Konstantin Lebedev
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 410 additions and 74 deletions
-
6README.md
-
6go.mod
-
41go.sum
-
4k8s/seaweedfs/Chart.yaml
-
2k8s/seaweedfs/values.yaml
-
2other/java/client/pom.xml
-
2other/java/client/pom.xml.deploy
-
2other/java/client/pom_debug.xml
-
2other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
-
8other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java
-
4other/java/examples/pom.xml
-
2other/java/hdfs2/dependency-reduced-pom.xml
-
2other/java/hdfs2/pom.xml
-
2other/java/hdfs3/dependency-reduced-pom.xml
-
2other/java/hdfs3/pom.xml
-
2unmaintained/load_test/load_test_leveldb/load_test_leveldb.go
-
108unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go
-
2weed/command/mount.go
-
20weed/command/mount_std.go
-
5weed/command/scaffold.go
-
1weed/filer/meta_aggregator.go
-
74weed/filer/sqlite/sqlite_store.go
-
14weed/filer/stream.go
-
11weed/filesys/dirty_pages_temp_file.go
-
37weed/filesys/wfs.go
-
39weed/filesys/wfs_filer_client.go
-
2weed/filesys/wfs_write.go
-
59weed/pb/grpc_client_server.go
-
6weed/s3api/s3api_object_handlers.go
-
6weed/s3api/s3err/s3api_errors.go
-
3weed/server/filer_server.go
-
2weed/shell/command_fs_meta_save.go
-
3weed/shell/command_s3_bucket_list.go
-
2weed/util/constants.go
-
1weed/wdclient/masterclient.go
@ -1,5 +1,5 @@ |
|||
apiVersion: v1 |
|||
description: SeaweedFS |
|||
name: seaweedfs |
|||
appVersion: "2.48" |
|||
version: 2.48 |
|||
appVersion: "2.49" |
|||
version: 2.49 |
|||
@ -0,0 +1,108 @@ |
|||
package main |
|||
|
|||
import ( |
|||
"context" |
|||
"flag" |
|||
"fmt" |
|||
"github.com/chrislusf/seaweedfs/weed/pb" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
"google.golang.org/grpc" |
|||
"io" |
|||
"strconv" |
|||
) |
|||
|
|||
var ( |
|||
dir = flag.String("dir", "/tmp", "directory to create files") |
|||
n = flag.Int("n", 100, "the number of metadata") |
|||
tailFiler = flag.String("filer", "localhost:8888", "the filer address") |
|||
isWrite = flag.Bool("write", false, "only write") |
|||
) |
|||
|
|||
func main() { |
|||
|
|||
flag.Parse() |
|||
|
|||
if *isWrite { |
|||
startGenerateMetadata() |
|||
return |
|||
} |
|||
|
|||
expected := 0 |
|||
startSubscribeMetadata(func(event *filer_pb.SubscribeMetadataResponse) error { |
|||
if event.Directory != *dir { |
|||
return nil |
|||
} |
|||
name := event.EventNotification.NewEntry.Name |
|||
fmt.Printf("=> %s\n", name) |
|||
id := name[4:] |
|||
if x, err := strconv.Atoi(id); err == nil { |
|||
if x != expected { |
|||
return fmt.Errorf("Expected file%d Actual %s\n", expected, name) |
|||
} |
|||
expected++ |
|||
} else { |
|||
return err |
|||
} |
|||
return nil |
|||
}) |
|||
|
|||
} |
|||
|
|||
func startGenerateMetadata() { |
|||
pb.WithFilerClient(*tailFiler, grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error { |
|||
|
|||
for i := 0; i < *n; i++ { |
|||
name := fmt.Sprintf("file%d", i) |
|||
if err := filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{ |
|||
Directory: *dir, |
|||
Entry: &filer_pb.Entry{ |
|||
Name: name, |
|||
}, |
|||
}); err != nil { |
|||
fmt.Printf("create entry %s: %v\n", name, err) |
|||
return err |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
|
|||
}) |
|||
} |
|||
|
|||
func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) { |
|||
|
|||
lastTsNs := int64(0) |
|||
|
|||
tailErr := pb.WithFilerClient(*tailFiler, grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error { |
|||
|
|||
ctx, cancel := context.WithCancel(context.Background()) |
|||
defer cancel() |
|||
|
|||
stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{ |
|||
ClientName: "tail", |
|||
PathPrefix: *dir, |
|||
SinceNs: lastTsNs, |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("listen: %v", err) |
|||
} |
|||
|
|||
for { |
|||
resp, listenErr := stream.Recv() |
|||
if listenErr == io.EOF { |
|||
return nil |
|||
} |
|||
if listenErr != nil { |
|||
return listenErr |
|||
} |
|||
if err = eachEntryFunc(resp); err != nil { |
|||
return err |
|||
} |
|||
lastTsNs = resp.TsNs |
|||
} |
|||
|
|||
}) |
|||
if tailErr != nil { |
|||
fmt.Printf("tail %s: %v\n", *tailFiler, tailErr) |
|||
} |
|||
} |
|||
@ -0,0 +1,74 @@ |
|||
package sqlite |
|||
|
|||
import ( |
|||
"context" |
|||
"database/sql" |
|||
"fmt" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/filer" |
|||
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" |
|||
"github.com/chrislusf/seaweedfs/weed/filer/mysql" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
_ "modernc.org/sqlite" |
|||
) |
|||
|
|||
func init() { |
|||
filer.Stores = append(filer.Stores, &SqliteStore{}) |
|||
} |
|||
|
|||
type SqliteStore struct { |
|||
abstract_sql.AbstractSqlStore |
|||
} |
|||
|
|||
func (store *SqliteStore) GetName() string { |
|||
return "sqlite" |
|||
} |
|||
|
|||
func (store *SqliteStore) Initialize(configuration util.Configuration, prefix string) (err error) { |
|||
dbFile := configuration.GetString(prefix + "dbFile") |
|||
createTable := `CREATE TABLE IF NOT EXISTS "%s" ( |
|||
dirhash BIGINT, |
|||
name VARCHAR(1000), |
|||
directory TEXT, |
|||
meta BLOB, |
|||
PRIMARY KEY (dirhash, name) |
|||
) WITHOUT ROWID;` |
|||
upsertQuery := `INSERT INTO "%s"(dirhash,name,directory,meta)VALUES(?,?,?,?) |
|||
ON CONFLICT(dirhash,name) DO UPDATE SET |
|||
directory=excluded.directory, |
|||
meta=excluded.meta; |
|||
` |
|||
return store.initialize( |
|||
dbFile, |
|||
createTable, |
|||
upsertQuery, |
|||
) |
|||
} |
|||
|
|||
func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (err error) { |
|||
|
|||
store.SupportBucketTable = true |
|||
store.SqlGenerator = &mysql.SqlGenMysql{ |
|||
CreateTableSqlTemplate: createTable, |
|||
DropTableSqlTemplate: "drop table `%s`", |
|||
UpsertQueryTemplate: upsertQuery, |
|||
} |
|||
|
|||
var dbErr error |
|||
store.DB, dbErr = sql.Open("sqlite", dbFile) |
|||
if dbErr != nil { |
|||
store.DB.Close() |
|||
store.DB = nil |
|||
return fmt.Errorf("can not connect to %s error:%v", dbFile, err) |
|||
} |
|||
|
|||
if err = store.DB.Ping(); err != nil { |
|||
return fmt.Errorf("connect to %s error:%v", dbFile, err) |
|||
} |
|||
|
|||
if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil { |
|||
return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue