hilimd
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
54 changed files with 682 additions and 128 deletions
-
11README.md
-
2docker/compose/local-s3tests-compose.yml
-
9go.mod
-
43go.sum
-
4k8s/seaweedfs/Chart.yaml
-
2k8s/seaweedfs/values.yaml
-
2other/java/client/pom.xml
-
2other/java/client/pom.xml.deploy
-
2other/java/client/pom_debug.xml
-
2other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
-
8other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java
-
4other/java/examples/pom.xml
-
2other/java/hdfs2/dependency-reduced-pom.xml
-
2other/java/hdfs2/pom.xml
-
2other/java/hdfs3/dependency-reduced-pom.xml
-
2other/java/hdfs3/pom.xml
-
2unmaintained/load_test/load_test_leveldb/load_test_leveldb.go
-
108unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go
-
1weed/command/command.go
-
165weed/command/fuse.go
-
2weed/command/mount.go
-
20weed/command/mount_std.go
-
11weed/command/scaffold.go
-
1weed/filer/meta_aggregator.go
-
78weed/filer/sqlite/sqlite_store.go
-
9weed/filer/sqlite/sqlite_store_unsupported.go
-
10weed/filer/stream.go
-
11weed/filesys/dirty_pages_temp_file.go
-
37weed/filesys/wfs.go
-
39weed/filesys/wfs_filer_client.go
-
2weed/filesys/wfs_write.go
-
59weed/pb/grpc_client_server.go
-
3weed/replication/sink/azuresink/azure_sink.go
-
5weed/replication/sink/b2sink/b2_sink.go
-
3weed/replication/sink/gcssink/gcs_sink.go
-
30weed/replication/sink/localsink/local_sink.go
-
3weed/replication/sink/s3sink/s3_sink.go
-
1weed/replication/source/filer_source.go
-
33weed/s3api/s3api_object_handlers.go
-
12weed/s3api/s3err/s3api_errors.go
-
8weed/server/common.go
-
3weed/server/filer_server.go
-
21weed/server/filer_server_handlers_read.go
-
2weed/server/filer_server_handlers_write_autochunk.go
-
7weed/server/volume_server_handlers_read.go
-
2weed/server/volume_server_ui/templates.go
-
2weed/shell/command_fs_meta_save.go
-
3weed/shell/command_s3_bucket_list.go
-
4weed/static/javascript/jquery-2.1.3.min.js
-
2weed/static/javascript/jquery-3.6.0.min.js
-
6weed/topology/data_node.go
-
3weed/util/config.go
-
2weed/util/constants.go
-
1weed/wdclient/masterclient.go
@ -1,5 +1,5 @@ |
|||
apiVersion: v1 |
|||
description: SeaweedFS |
|||
name: seaweedfs |
|||
appVersion: "2.48" |
|||
version: 2.48 |
|||
appVersion: "2.49" |
|||
version: 2.49 |
|||
@ -0,0 +1,108 @@ |
|||
package main |
|||
|
|||
import ( |
|||
"context" |
|||
"flag" |
|||
"fmt" |
|||
"github.com/chrislusf/seaweedfs/weed/pb" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
"google.golang.org/grpc" |
|||
"io" |
|||
"strconv" |
|||
) |
|||
|
|||
var ( |
|||
dir = flag.String("dir", "/tmp", "directory to create files") |
|||
n = flag.Int("n", 100, "the number of metadata") |
|||
tailFiler = flag.String("filer", "localhost:8888", "the filer address") |
|||
isWrite = flag.Bool("write", false, "only write") |
|||
) |
|||
|
|||
func main() { |
|||
|
|||
flag.Parse() |
|||
|
|||
if *isWrite { |
|||
startGenerateMetadata() |
|||
return |
|||
} |
|||
|
|||
expected := 0 |
|||
startSubscribeMetadata(func(event *filer_pb.SubscribeMetadataResponse) error { |
|||
if event.Directory != *dir { |
|||
return nil |
|||
} |
|||
name := event.EventNotification.NewEntry.Name |
|||
fmt.Printf("=> %s\n", name) |
|||
id := name[4:] |
|||
if x, err := strconv.Atoi(id); err == nil { |
|||
if x != expected { |
|||
return fmt.Errorf("Expected file%d Actual %s\n", expected, name) |
|||
} |
|||
expected++ |
|||
} else { |
|||
return err |
|||
} |
|||
return nil |
|||
}) |
|||
|
|||
} |
|||
|
|||
func startGenerateMetadata() { |
|||
pb.WithFilerClient(*tailFiler, grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error { |
|||
|
|||
for i := 0; i < *n; i++ { |
|||
name := fmt.Sprintf("file%d", i) |
|||
if err := filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{ |
|||
Directory: *dir, |
|||
Entry: &filer_pb.Entry{ |
|||
Name: name, |
|||
}, |
|||
}); err != nil { |
|||
fmt.Printf("create entry %s: %v\n", name, err) |
|||
return err |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
|
|||
}) |
|||
} |
|||
|
|||
func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) { |
|||
|
|||
lastTsNs := int64(0) |
|||
|
|||
tailErr := pb.WithFilerClient(*tailFiler, grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error { |
|||
|
|||
ctx, cancel := context.WithCancel(context.Background()) |
|||
defer cancel() |
|||
|
|||
stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{ |
|||
ClientName: "tail", |
|||
PathPrefix: *dir, |
|||
SinceNs: lastTsNs, |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("listen: %v", err) |
|||
} |
|||
|
|||
for { |
|||
resp, listenErr := stream.Recv() |
|||
if listenErr == io.EOF { |
|||
return nil |
|||
} |
|||
if listenErr != nil { |
|||
return listenErr |
|||
} |
|||
if err = eachEntryFunc(resp); err != nil { |
|||
return err |
|||
} |
|||
lastTsNs = resp.TsNs |
|||
} |
|||
|
|||
}) |
|||
if tailErr != nil { |
|||
fmt.Printf("tail %s: %v\n", *tailFiler, tailErr) |
|||
} |
|||
} |
|||
@ -0,0 +1,165 @@ |
|||
package command |
|||
|
|||
import ( |
|||
"fmt" |
|||
"strings" |
|||
"strconv" |
|||
"time" |
|||
"os" |
|||
) |
|||
|
|||
func init() { |
|||
cmdFuse.Run = runFuse // break init cycle
|
|||
} |
|||
|
|||
func runFuse(cmd *Command, args []string) bool { |
|||
argsLen := len(args) |
|||
options := []string{} |
|||
|
|||
// at least target mount path should be passed
|
|||
if argsLen < 1 { |
|||
return false |
|||
} |
|||
|
|||
// first option is always target mount path
|
|||
mountOptions.dir = &args[0] |
|||
|
|||
// scan parameters looking for one or more -o options
|
|||
// -o options receive parameters on format key=value[,key=value]...
|
|||
for i := 0; i < argsLen; i++ { |
|||
if args[i] == "-o" && i+1 <= argsLen { |
|||
options = strings.Split(args[i+1], ",") |
|||
i++ |
|||
} |
|||
} |
|||
|
|||
// for each option passed with -o
|
|||
for _, option := range options { |
|||
// split just first = character
|
|||
parts := strings.SplitN(option, "=", 2) |
|||
|
|||
// if doesn't key and value skip
|
|||
if len(parts) != 2 { |
|||
continue |
|||
} |
|||
|
|||
key, value := parts[0], parts[1] |
|||
|
|||
// switch key keeping "weed mount" parameters
|
|||
switch key { |
|||
case "filer": |
|||
mountOptions.filer = &value |
|||
case "filer.path": |
|||
mountOptions.filerMountRootPath = &value |
|||
case "dirAutoCreate": |
|||
if parsed, err := strconv.ParseBool(value); err != nil { |
|||
mountOptions.dirAutoCreate = &parsed |
|||
} else { |
|||
panic(fmt.Errorf("dirAutoCreate: %s", err)) |
|||
} |
|||
case "collection": |
|||
mountOptions.collection = &value |
|||
case "replication": |
|||
mountOptions.replication = &value |
|||
case "disk": |
|||
mountOptions.diskType = &value |
|||
case "ttl": |
|||
if parsed, err := strconv.ParseInt(value, 0, 32); err != nil { |
|||
intValue := int(parsed) |
|||
mountOptions.ttlSec = &intValue |
|||
} else { |
|||
panic(fmt.Errorf("ttl: %s", err)) |
|||
} |
|||
case "chunkSizeLimitMB": |
|||
if parsed, err := strconv.ParseInt(value, 0, 32); err != nil { |
|||
intValue := int(parsed) |
|||
mountOptions.chunkSizeLimitMB = &intValue |
|||
} else { |
|||
panic(fmt.Errorf("chunkSizeLimitMB: %s", err)) |
|||
} |
|||
case "concurrentWriters": |
|||
if parsed, err := strconv.ParseInt(value, 0, 32); err != nil { |
|||
intValue := int(parsed) |
|||
mountOptions.concurrentWriters = &intValue |
|||
} else { |
|||
panic(fmt.Errorf("concurrentWriters: %s", err)) |
|||
} |
|||
case "cacheDir": |
|||
mountOptions.cacheDir = &value |
|||
case "cacheCapacityMB": |
|||
if parsed, err := strconv.ParseInt(value, 0, 64); err != nil { |
|||
mountOptions.cacheSizeMB = &parsed |
|||
} else { |
|||
panic(fmt.Errorf("cacheCapacityMB: %s", err)) |
|||
} |
|||
case "dataCenter": |
|||
mountOptions.dataCenter = &value |
|||
case "allowOthers": |
|||
if parsed, err := strconv.ParseBool(value); err != nil { |
|||
mountOptions.allowOthers = &parsed |
|||
} else { |
|||
panic(fmt.Errorf("allowOthers: %s", err)) |
|||
} |
|||
case "umask": |
|||
mountOptions.umaskString = &value |
|||
case "nonempty": |
|||
if parsed, err := strconv.ParseBool(value); err != nil { |
|||
mountOptions.nonempty = &parsed |
|||
} else { |
|||
panic(fmt.Errorf("nonempty: %s", err)) |
|||
} |
|||
case "volumeServerAccess": |
|||
mountOptions.volumeServerAccess = &value |
|||
case "map.uid": |
|||
mountOptions.uidMap = &value |
|||
case "map.gid": |
|||
mountOptions.gidMap = &value |
|||
case "readOnly": |
|||
if parsed, err := strconv.ParseBool(value); err != nil { |
|||
mountOptions.readOnly = &parsed |
|||
} else { |
|||
panic(fmt.Errorf("readOnly: %s", err)) |
|||
} |
|||
case "cpuprofile": |
|||
mountCpuProfile = &value |
|||
case "memprofile": |
|||
mountMemProfile = &value |
|||
case "readRetryTime": |
|||
if parsed, err := time.ParseDuration(value); err != nil { |
|||
mountReadRetryTime = &parsed |
|||
} else { |
|||
panic(fmt.Errorf("readRetryTime: %s", err)) |
|||
} |
|||
} |
|||
} |
|||
|
|||
// I don't know why PATH environment variable is lost
|
|||
if err := os.Setenv("PATH", "/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"); err != nil { |
|||
panic(fmt.Errorf("setenv: %s", err)) |
|||
} |
|||
|
|||
// just call "weed mount" command
|
|||
return runMount(cmdMount, []string{}) |
|||
} |
|||
|
|||
var cmdFuse = &Command{ |
|||
UsageLine: "fuse /mnt/mount/point -o \"filer=localhost:8888,filer.path=/\"", |
|||
Short: "Allow use weed with linux's mount command", |
|||
Long: `Allow use weed with linux's mount command |
|||
|
|||
You can use -t weed on mount command: |
|||
mv weed /sbin/mount.weed |
|||
mount -t weed fuse /mnt -o "filer=localhost:8888,filer.path=/" |
|||
|
|||
Or you can use -t fuse on mount command: |
|||
mv weed /sbin/weed |
|||
mount -t fuse.weed fuse /mnt -o "filer=localhost:8888,filer.path=/" |
|||
mount -t fuse "weed#fuse" /mnt -o "filer=localhost:8888,filer.path=/" |
|||
|
|||
To use without mess with your /sbin: |
|||
mount -t fuse./home/user/bin/weed fuse /mnt -o "filer=localhost:8888,filer.path=/" |
|||
mount -t fuse "/home/user/bin/weed#fuse" /mnt -o "filer=localhost:8888,filer.path=/" |
|||
|
|||
To check valid options look "weed mount --help" |
|||
`, |
|||
} |
|||
@ -0,0 +1,78 @@ |
|||
// +build linux darwin windows
|
|||
|
|||
// limited GOOS due to modernc.org/libc/unistd
|
|||
|
|||
package sqlite |
|||
|
|||
import ( |
|||
"context" |
|||
"database/sql" |
|||
"fmt" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/filer" |
|||
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" |
|||
"github.com/chrislusf/seaweedfs/weed/filer/mysql" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
_ "modernc.org/sqlite" |
|||
) |
|||
|
|||
func init() { |
|||
filer.Stores = append(filer.Stores, &SqliteStore{}) |
|||
} |
|||
|
|||
type SqliteStore struct { |
|||
abstract_sql.AbstractSqlStore |
|||
} |
|||
|
|||
func (store *SqliteStore) GetName() string { |
|||
return "sqlite" |
|||
} |
|||
|
|||
func (store *SqliteStore) Initialize(configuration util.Configuration, prefix string) (err error) { |
|||
dbFile := configuration.GetString(prefix + "dbFile") |
|||
createTable := `CREATE TABLE IF NOT EXISTS "%s" ( |
|||
dirhash BIGINT, |
|||
name VARCHAR(1000), |
|||
directory TEXT, |
|||
meta BLOB, |
|||
PRIMARY KEY (dirhash, name) |
|||
) WITHOUT ROWID;` |
|||
upsertQuery := `INSERT INTO "%s"(dirhash,name,directory,meta)VALUES(?,?,?,?) |
|||
ON CONFLICT(dirhash,name) DO UPDATE SET |
|||
directory=excluded.directory, |
|||
meta=excluded.meta; |
|||
` |
|||
return store.initialize( |
|||
dbFile, |
|||
createTable, |
|||
upsertQuery, |
|||
) |
|||
} |
|||
|
|||
func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (err error) { |
|||
|
|||
store.SupportBucketTable = true |
|||
store.SqlGenerator = &mysql.SqlGenMysql{ |
|||
CreateTableSqlTemplate: createTable, |
|||
DropTableSqlTemplate: "drop table `%s`", |
|||
UpsertQueryTemplate: upsertQuery, |
|||
} |
|||
|
|||
var dbErr error |
|||
store.DB, dbErr = sql.Open("sqlite", dbFile) |
|||
if dbErr != nil { |
|||
store.DB.Close() |
|||
store.DB = nil |
|||
return fmt.Errorf("can not connect to %s error:%v", dbFile, err) |
|||
} |
|||
|
|||
if err = store.DB.Ping(); err != nil { |
|||
return fmt.Errorf("connect to %s error:%v", dbFile, err) |
|||
} |
|||
|
|||
if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil { |
|||
return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
@ -0,0 +1,9 @@ |
|||
// +build !linux,!darwin,!windows
|
|||
|
|||
// limited GOOS due to modernc.org/libc/unistd
|
|||
|
|||
package sqlite |
|||
|
|||
func init() { |
|||
// filer.Stores = append(filer.Stores, &SqliteStore{})
|
|||
} |
|||
4
weed/static/javascript/jquery-2.1.3.min.js
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
2
weed/static/javascript/jquery-3.6.0.min.js
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
Write
Preview
Loading…
Cancel
Save
Reference in new issue