Merge pull request #3043 from kmlebedev/ydb_refactor

ydb test&refactor

docker/Makefile

@@ -7,7 +7,7 @@ gen: dev
 binary:
 	export SWCOMMIT=$(shell git rev-parse --short HEAD)
 	export SWLDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
-	cd ../weed; CGO_ENABLED=0 GOOS=linux go build -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/
+	cd ../weed; CGO_ENABLED=0 GOOS=linux go build --tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/
 
 build: binary
 	docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
@@ -49,9 +49,6 @@ dev_replicate: build
 dev_auditlog: build
 	docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up
 
-dev_ydb: build
-	docker-compose -f compose/local-ydb-compose.yml -p seaweedfs up
-
 cluster: build
 	docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up
 
@@ -73,6 +70,11 @@ filer_etcd: build
 test_etcd: build
 	docker-compose -f compose/test-etcd-filer.yml -p seaweedfs up
 
+test_ydb: tags = ydb
+test_ydb: build
+	export
+	docker-compose -f compose/test-ydb-filer.yml -p seaweedfs up
+
 clean:
 	rm ./weed
 

docker/compose/local-ydb-compose.yml → docker/compose/test-ydb-filer.yml

@@ -17,8 +17,6 @@ services:
     ports:
       - 9333:9333
       - 19333:19333
-      - 8084:8080
-      - 18084:18080
       - 8888:8888
       - 8000:8000
       - 18888:18888
@@ -33,3 +31,5 @@ services:
       YDB_ANONYMOUS_CREDENTIALS: 1
       WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
       WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+    depends_on:
+      - ydb

weed/filer/ydb/ydb_store.go

@@ -15,7 +15,6 @@ import (
 	"github.com/ydb-platform/ydb-go-sdk/v3"
 	"github.com/ydb-platform/ydb-go-sdk/v3/sugar"
 	"github.com/ydb-platform/ydb-go-sdk/v3/table"
-	"github.com/ydb-platform/ydb-go-sdk/v3/table/options"
 	"github.com/ydb-platform/ydb-go-sdk/v3/table/result"
 	"github.com/ydb-platform/ydb-go-sdk/v3/table/result/named"
 	"github.com/ydb-platform/ydb-go-sdk/v3/table/types"
@@ -28,7 +27,6 @@ import (
 
 const (
 	defaultDialTimeOut = 10
-	maxRowsInQuery     = 1000 // Limit number of rows in query results https://cloud.yandex.com/en-ru/docs/ydb/concepts/limits-ydb
 )
 
 var (
@@ -112,14 +110,13 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix
 func (store *YdbStore) doTxOrDB(ctx context.Context, query *string, params *table.QueryParameters, tc *table.TransactionControl, processResultFunc func(res result.Result) error) (err error) {
 	var res result.Result
 	if tx, ok := ctx.Value("tx").(table.Transaction); ok {
-		res, err = tx.Execute(ctx, *query, params, options.WithQueryCachePolicy(options.WithQueryCachePolicyKeepInCache()))
+		res, err = tx.Execute(ctx, *query, params)
 		if err != nil {
 			return fmt.Errorf("execute transaction: %v", err)
 		}
 	} else {
 		err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) {
-			_, res, err = s.Execute(ctx, tc, *query,
-				params, options.WithQueryCachePolicy(options.WithQueryCachePolicyKeepInCache()))
+			_, res, err = s.Execute(ctx, tc, *query, params)
 			if err != nil {
 				return fmt.Errorf("execute statement: %v", err)
 			}
@@ -240,10 +237,6 @@ func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath
 	}
 	truncated := true
 	eachEntryFuncIsNotBreake := true
-	shortLimit := limit
-	if limit > maxRowsInQuery {
-		shortLimit = maxRowsInQuery * 2
-	}
 	entryCount := int64(0)
 	for truncated && eachEntryFuncIsNotBreake {
 		if lastFileName != "" {
@@ -253,15 +246,12 @@ func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath
 			}
 		}
 		restLimit := limit - entryCount
-		if maxRowsInQuery > restLimit {
-			shortLimit = restLimit
-		}
 		queryParams := table.NewQueryParameters(
 			table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
 			table.ValueParam("$directory", types.UTF8Value(*shortDir)),
 			table.ValueParam("$start_name", types.UTF8Value(startFileName)),
 			table.ValueParam("$prefix", types.UTF8Value(prefix+"%")),
-			table.ValueParam("$limit", types.Uint64Value(uint64(shortLimit))),
+			table.ValueParam("$limit", types.Uint64Value(uint64(restLimit))),
 		)
 		err = store.doTxOrDB(ctx, query, queryParams, roTX, func(res result.Result) error {
 			var name string

weed/filer/ydb/ydb_store_test.go

@@ -0,0 +1,19 @@
+//go:build ydb
+// +build ydb
+
+package ydb
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filer/store_test"
+	"testing"
+)
+
+func TestStore(t *testing.T) {
+	// run "make test_ydb" under docker folder.
+	// to set up local env
+	if false {
+		store := &YdbStore{}
+		store.initialize("/buckets", "grpc://localhost:2136/?database=local", "seaweedfs", true, 10, 50)
+		store_test.TestFilerStore(t, store)
+	}
+}