Merge pull request #3 from wavetermdev/sawka/new-blockstore

new blockstore

pkg/blockstore/blockstore_dbops.go

@@ -3,7 +3,7 @@ package blockstore
 import (
 	"context"
 	"fmt"
-	"sync/atomic"
+	"os"
 
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil"
 )
@@ -54,14 +54,15 @@ func dbGetAllBlockIds(ctx context.Context) ([]string, error) {
 }
 
 func dbGetFileParts(ctx context.Context, blockId string, name string, parts []int) (map[int]*DataCacheEntry, error) {
+	if len(parts) == 0 {
+		return nil, nil
+	}
 	return WithTxRtn(ctx, func(tx *TxWrap) (map[int]*DataCacheEntry, error) {
 		var data []*DataCacheEntry
 		query := "SELECT partidx, data FROM db_block_data WHERE blockid = ? AND name = ? AND partidx IN (SELECT value FROM json_each(?))"
 		tx.Select(&data, query, blockId, name, dbutil.QuickJsonArr(parts))
 		rtn := make(map[int]*DataCacheEntry)
 		for _, d := range data {
-			d.Dirty = &atomic.Bool{}
-			d.Flushing = &atomic.Bool{}
 			if cap(d.Data) != int(partDataSize) {
 				newData := make([]byte, len(d.Data), partDataSize)
 				copy(newData, d.Data)
@@ -81,26 +82,26 @@ func dbGetBlockFiles(ctx context.Context, blockId string) ([]*BlockFile, error)
 	})
 }
 
-func dbWriteCacheEntry(ctx context.Context, fileEntry *FileCacheEntry, dataEntries []*DataCacheEntry) error {
-	if fileEntry == nil {
-		return fmt.Errorf("fileEntry or fileEntry.File is nil")
-	}
+func dbWriteCacheEntry(ctx context.Context, file *BlockFile, dataEntries map[int]*DataCacheEntry, replace bool) error {
 	return WithTx(ctx, func(tx *TxWrap) error {
 		query := `SELECT blockid FROM db_block_file WHERE blockid = ? AND name = ?`
-		if !tx.Exists(query, fileEntry.File.BlockId, fileEntry.File.Name) {
+		if !tx.Exists(query, file.BlockId, file.Name) {
 			// since deletion is synchronous this stops us from writing to a deleted file
-			return fmt.Errorf("file not found in db")
+			return os.ErrNotExist
 		}
-		if fileEntry.Dirty.Load() {
-			query := `UPDATE db_block_file SET size = ?, createdts = ?, modts = ?, opts = ?, meta = ? WHERE blockid = ? AND name = ?`
-			tx.Exec(query, fileEntry.File.Size, fileEntry.File.CreatedTs, fileEntry.File.ModTs, dbutil.QuickJson(fileEntry.File.Opts), dbutil.QuickJson(fileEntry.File.Meta), fileEntry.File.BlockId, fileEntry.File.Name)
+		// we don't update CreatedTs or Opts
+		query = `UPDATE db_block_file SET size = ?, modts = ?, meta = ? WHERE blockid = ? AND name = ?`
+		tx.Exec(query, file.Size, file.ModTs, dbutil.QuickJson(file.Meta), file.BlockId, file.Name)
+		if replace {
+			query = `DELETE FROM db_block_data WHERE blockid = ? AND name = ?`
+			tx.Exec(query, file.BlockId, file.Name)
 		}
 		dataPartQuery := `REPLACE INTO db_block_data (blockid, name, partidx, data) VALUES (?, ?, ?, ?)`
-		for _, dataEntry := range dataEntries {
-			if dataEntry == nil || !dataEntry.Dirty.Load() {
-				continue
+		for partIdx, dataEntry := range dataEntries {
+			if partIdx != dataEntry.PartIdx {
+				panic(fmt.Sprintf("partIdx:%d and dataEntry.PartIdx:%d do not match", partIdx, dataEntry.PartIdx))
 			}
-			tx.Exec(dataPartQuery, fileEntry.File.BlockId, fileEntry.File.Name, dataEntry.PartIdx, dataEntry.Data)
+			tx.Exec(dataPartQuery, file.BlockId, file.Name, dataEntry.PartIdx, dataEntry.Data)
 		}
 		return nil
 	})

pkg/blockstore/blockstore_test.go

@@ -6,7 +6,9 @@ package blockstore
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
+	"io/fs"
 	"log"
 	"sync"
 	"sync/atomic"
@@ -45,6 +47,32 @@ func cleanupDb(t *testing.T) {
 	}
 }
 
+func (s *BlockStore) getCacheSize() int {
+	s.Lock.Lock()
+	defer s.Lock.Unlock()
+	return len(s.Cache)
+}
+
+func (s *BlockStore) clearCache() {
+	s.Lock.Lock()
+	defer s.Lock.Unlock()
+	s.Cache = make(map[cacheKey]*CacheEntry)
+}
+
+//lint:ignore U1000 used for testing
+func (s *BlockStore) dump() string {
+	s.Lock.Lock()
+	defer s.Lock.Unlock()
+	var buf bytes.Buffer
+	buf.WriteString(fmt.Sprintf("BlockStore %d entries\n", len(s.Cache)))
+	for _, v := range s.Cache {
+		entryStr := v.dump()
+		buf.WriteString(entryStr)
+		buf.WriteString("\n")
+	}
+	return buf.String()
+}
+
 func TestCreate(t *testing.T) {
 	initDb(t)
 	defer cleanupDb(t)
@@ -134,12 +162,9 @@ func TestDelete(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error deleting file: %v", err)
 	}
-	file, err := GBS.Stat(ctx, blockId, "testfile")
-	if err != nil {
-		t.Fatalf("error stating file: %v", err)
-	}
-	if file != nil {
-		t.Fatalf("file should not be found")
+	_, err = GBS.Stat(ctx, blockId, "testfile")
+	if err == nil || errors.Is(err, fs.ErrNotExist) {
+		t.Errorf("expected file not found error")
 	}
 
 	// create two files in same block, use DeleteBlock to delete
@@ -376,7 +401,6 @@ func TestCircularWrites(t *testing.T) {
 		t.Fatalf("error writing data: %v", err)
 	}
 	checkFileData(t, ctx, blockId, "c1", "123456789 123456789 123456789 123456789 123456789 ")
-
 	err = GBS.AppendData(ctx, blockId, "c1", []byte("apple"))
 	if err != nil {
 		t.Fatalf("error appending data: %v", err)
@@ -426,14 +450,14 @@ func TestCircularWrites(t *testing.T) {
 	}
 	checkFileSize(t, ctx, blockId, "c1", 128)
 	checkFileData(t, ctx, blockId, "c1", " 123456789 123456789 123456789 bar456789 123456789")
-	GBS.withLock(blockId, "c1", false, func(entry *CacheEntry) {
+	err = withLock(GBS, blockId, "c1", func(entry *CacheEntry) error {
 		if entry == nil {
-			err = fmt.Errorf("entry not found")
-			return
+			return fmt.Errorf("entry not found")
 		}
 		if len(entry.DataEntries) != 1 {
-			err = fmt.Errorf("data entries mismatch: expected 1, got %d", len(entry.DataEntries))
+			return fmt.Errorf("data entries mismatch: expected 1, got %d", len(entry.DataEntries))
 		}
+		return nil
 	})
 	if err != nil {
 		t.Fatalf("error checking data entries: %v", err)