hilimd
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
121 changed files with 3359 additions and 442 deletions
-
2.github/workflows/release.yml
-
41README.md
-
2backers.md
-
2docker/Dockerfile.go_build
-
2docker/Dockerfile.go_build_large
-
17docker/Makefile
-
0docker/compose/dev.env
-
7docker/compose/local-clusters-compose.yml
-
28docker/compose/local-dev-compose.yml
-
30docker/compose/master-cloud.toml
-
14docker/compose/tls.env
-
4docker/entrypoint.sh
-
3go.mod
-
2go.sum
-
8k8s/README.md
-
4k8s/seaweedfs/Chart.yaml
-
23k8s/seaweedfs/templates/_helpers.tpl
-
2k8s/seaweedfs/templates/cronjob.yaml
-
1k8s/seaweedfs/templates/filer-service-client.yaml
-
78k8s/seaweedfs/templates/volume-statefulset.yaml
-
20k8s/seaweedfs/values.yaml
-
BINnote/SeaweedFS_Architecture.png
-
BINnote/SeaweedFS_Cluster_Backup.png
-
BINnote/SeaweedFS_XDR.png
-
2other/java/client/pom.xml
-
2other/java/client/pom.xml.deploy
-
2other/java/client/pom_debug.xml
-
3other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
-
4other/java/examples/pom.xml
-
2other/java/hdfs2/dependency-reduced-pom.xml
-
2other/java/hdfs2/pom.xml
-
2other/java/hdfs3/dependency-reduced-pom.xml
-
2other/java/hdfs3/pom.xml
-
4weed/Makefile
-
25weed/command/benchmark.go
-
2weed/command/command.go
-
157weed/command/filer_backup.go
-
268weed/command/filer_meta_backup.go
-
8weed/command/filer_meta_tail.go
-
29weed/command/filer_replication.go
-
203weed/command/filer_sync.go
-
1weed/command/master.go
-
23weed/command/mount_std.go
-
2weed/command/msg_broker.go
-
2weed/command/s3.go
-
15weed/command/scaffold.go
-
13weed/command/server.go
-
26weed/command/volume.go
-
2weed/command/webdav.go
-
6weed/filer/filer_delete_entry.go
-
5weed/filer/filer_on_meta_event.go
-
1weed/filer/meta_aggregator.go
-
8weed/filer/stream.go
-
14weed/filesys/dir.go
-
29weed/filesys/dir_link.go
-
9weed/filesys/dirty_page.go
-
51weed/filesys/file.go
-
75weed/filesys/filehandle.go
-
5weed/filesys/fscache.go
-
2weed/operation/upload_content.go
-
14weed/pb/filer_pb/filer_client.go
-
65weed/pb/grpc_client_server.go
-
29weed/pb/volume_info.go
-
2weed/replication/repl_util/replication_util.go
-
2weed/replication/replicator.go
-
14weed/replication/sink/azuresink/azure_sink.go
-
14weed/replication/sink/b2sink/b2_sink.go
-
6weed/replication/sink/filersink/filer_sink.go
-
14weed/replication/sink/gcssink/gcs_sink.go
-
8weed/replication/sink/localsink/local_sink.go
-
1weed/replication/sink/replication_sink.go
-
29weed/replication/sink/s3sink/s3_sink.go
-
13weed/replication/sink/s3sink/s3_write.go
-
8weed/replication/source/filer_source.go
-
4weed/s3api/filer_util.go
-
2weed/s3api/s3api_bucket_handlers.go
-
20weed/s3api/s3api_object_handlers.go
-
2weed/s3api/s3api_object_handlers_postpolicy.go
-
66weed/security/tls.go
-
2weed/server/filer_server.go
-
11weed/server/filer_server_handlers_read.go
-
11weed/server/filer_server_handlers_write_autochunk.go
-
7weed/server/master_grpc_server_volume.go
-
137weed/server/volume_server_tcp_handlers_write.go
-
2weed/shell/command_ec_encode.go
-
92weed/shell/command_s3_clean_uploads.go
-
2weed/shell/command_volume_tier_move.go
-
6weed/storage/backend/backend.go
-
2weed/storage/backend/disk_file.go
-
5weed/storage/backend/s3_backend/s3_sessions.go
-
2weed/storage/erasure_coding/ec_volume.go
-
23weed/storage/needle/crc.go
-
2weed/storage/needle/needle_read_write.go
-
6weed/storage/needle_map_leveldb.go
-
8weed/storage/needle_map_sorted_file.go
-
18weed/storage/store.go
-
3weed/storage/super_block/replica_placement.go
-
10weed/storage/volume_loading.go
-
60weed/storage/volume_read_write.go
-
104weed/storage/volume_stream_write.go
@ -0,0 +1,30 @@ |
|||||
|
|
||||
|
# Put this file to one of the location, with descending priority |
||||
|
# ./master.toml |
||||
|
# $HOME/.seaweedfs/master.toml |
||||
|
# /etc/seaweedfs/master.toml |
||||
|
# this file is read by master |
||||
|
|
||||
|
[master.maintenance] |
||||
|
# periodically run these scripts are the same as running them from 'weed shell' |
||||
|
scripts = """ |
||||
|
lock |
||||
|
ec.encode -fullPercent=95 -quietFor=1h |
||||
|
ec.rebuild -force |
||||
|
ec.balance -force |
||||
|
volume.balance -force |
||||
|
volume.fix.replication |
||||
|
unlock |
||||
|
""" |
||||
|
sleep_minutes = 17 # sleep minutes between each script execution |
||||
|
|
||||
|
# configurations for tiered cloud storage |
||||
|
# old volumes are transparently moved to cloud for cost efficiency |
||||
|
[storage.backend] |
||||
|
[storage.backend.s3.default] |
||||
|
enabled = true |
||||
|
aws_access_key_id = "any" # if empty, loads from the shared credentials file (~/.aws/credentials). |
||||
|
aws_secret_access_key = "any" # if empty, loads from the shared credentials file (~/.aws/credentials). |
||||
|
region = "us-east-2" |
||||
|
bucket = "volume_bucket" # an existing bucket |
||||
|
endpoint = "http://server2:8333" |
||||
@ -0,0 +1,14 @@ |
|||||
|
WEED_GRPC_CA=/etc/seaweedfs/tls/SeaweedFS_CA.crt |
||||
|
WEED_GRPC_ALLOWED_WILDCARD_DOMAIN=".dev" |
||||
|
WEED_GRPC_MASTER_CERT=/etc/seaweedfs/tls/master01.dev.crt |
||||
|
WEED_GRPC_MASTER_KEY=/etc/seaweedfs/tls/master01.dev.key |
||||
|
WEED_GRPC_VOLUME_CERT=/etc/seaweedfs/tls/volume01.dev.crt |
||||
|
WEED_GRPC_VOLUME_KEY=/etc/seaweedfs/tls/volume01.dev.key |
||||
|
WEED_GRPC_FILER_CERT=/etc/seaweedfs/tls/filer01.dev.crt |
||||
|
WEED_GRPC_FILER_KEY=/etc/seaweedfs/tls/filer01.dev.key |
||||
|
WEED_GRPC_CLIENT_CERT=/etc/seaweedfs/tls/client01.dev.crt |
||||
|
WEED_GRPC_CLIENT_KEY=/etc/seaweedfs/tls/client01.dev.key |
||||
|
WEED_GRPC_MASTER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" |
||||
|
WEED_GRPC_VOLUME_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" |
||||
|
WEED_GRPC_FILER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" |
||||
|
WEED_GRPC_CLIENT_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" |
||||
@ -1,5 +1,5 @@ |
|||||
apiVersion: v1 |
apiVersion: v1 |
||||
description: SeaweedFS |
description: SeaweedFS |
||||
name: seaweedfs |
name: seaweedfs |
||||
appVersion: "2.27" |
|
||||
version: 2.27 |
|
||||
|
appVersion: "2.31" |
||||
|
version: 2.31 |
||||
|
After Width: 1121 | Height: 701 | Size: 90 KiB |
|
After Width: 1211 | Height: 586 | Size: 86 KiB |
|
After Width: 1271 | Height: 566 | Size: 62 KiB |
@ -0,0 +1,157 @@ |
|||||
|
package command |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"fmt" |
||||
|
"github.com/chrislusf/seaweedfs/weed/glog" |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/replication/source" |
||||
|
"github.com/chrislusf/seaweedfs/weed/security" |
||||
|
"github.com/chrislusf/seaweedfs/weed/util" |
||||
|
"google.golang.org/grpc" |
||||
|
"io" |
||||
|
"time" |
||||
|
) |
||||
|
|
||||
|
type FilerBackupOptions struct { |
||||
|
isActivePassive *bool |
||||
|
filer *string |
||||
|
path *string |
||||
|
debug *bool |
||||
|
proxyByFiler *bool |
||||
|
timeAgo *time.Duration |
||||
|
} |
||||
|
|
||||
|
var ( |
||||
|
filerBackupOptions FilerBackupOptions |
||||
|
) |
||||
|
|
||||
|
func init() { |
||||
|
cmdFilerBackup.Run = runFilerBackup // break init cycle
|
||||
|
filerBackupOptions.filer = cmdFilerBackup.Flag.String("filer", "localhost:8888", "filer of one SeaweedFS cluster") |
||||
|
filerBackupOptions.path = cmdFilerBackup.Flag.String("filerPath", "/", "directory to sync on filer") |
||||
|
filerBackupOptions.proxyByFiler = cmdFilerBackup.Flag.Bool("filerProxy", false, "read and write file chunks by filer instead of volume servers") |
||||
|
filerBackupOptions.debug = cmdFilerBackup.Flag.Bool("debug", false, "debug mode to print out received files") |
||||
|
filerBackupOptions.timeAgo = cmdFilerBackup.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"") |
||||
|
} |
||||
|
|
||||
|
var cmdFilerBackup = &Command{ |
||||
|
UsageLine: "filer.backup -filer=<filerHost>:<filerPort> ", |
||||
|
Short: "resume-able continuously replicate files from a SeaweedFS cluster to another location defined in replication.toml", |
||||
|
Long: `resume-able continuously replicate files from a SeaweedFS cluster to another location defined in replication.toml |
||||
|
|
||||
|
filer.backup listens on filer notifications. If any file is updated, it will fetch the updated content, |
||||
|
and write to the destination. This is to replace filer.replicate command since additional message queue is not needed. |
||||
|
|
||||
|
If restarted and "-timeAgo" is not set, the synchronization will resume from the previous checkpoints, persisted every minute. |
||||
|
A fresh sync will start from the earliest metadata logs. To reset the checkpoints, just set "-timeAgo" to a high value. |
||||
|
|
||||
|
`, |
||||
|
} |
||||
|
|
||||
|
func runFilerBackup(cmd *Command, args []string) bool { |
||||
|
|
||||
|
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") |
||||
|
|
||||
|
util.LoadConfiguration("security", false) |
||||
|
util.LoadConfiguration("replication", true) |
||||
|
|
||||
|
for { |
||||
|
err := doFilerBackup(grpcDialOption, &filerBackupOptions) |
||||
|
if err != nil { |
||||
|
glog.Errorf("backup from %s: %v", *filerBackupOptions.filer, err) |
||||
|
time.Sleep(1747 * time.Millisecond) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return true |
||||
|
} |
||||
|
|
||||
|
const ( |
||||
|
BackupKeyPrefix = "backup." |
||||
|
) |
||||
|
|
||||
|
func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions) error { |
||||
|
|
||||
|
// find data sink
|
||||
|
config := util.GetViper() |
||||
|
dataSink := findSink(config) |
||||
|
if dataSink == nil { |
||||
|
return fmt.Errorf("no data sink configured in replication.toml") |
||||
|
} |
||||
|
|
||||
|
sourceFiler := *backupOption.filer |
||||
|
sourcePath := *backupOption.path |
||||
|
timeAgo := *backupOption.timeAgo |
||||
|
targetPath := dataSink.GetSinkToDirectory() |
||||
|
debug := *backupOption.debug |
||||
|
|
||||
|
// get start time for the data sink
|
||||
|
startFrom := time.Unix(0, 0) |
||||
|
sinkId := util.HashStringToLong(dataSink.GetName() + dataSink.GetSinkToDirectory()) |
||||
|
if timeAgo.Milliseconds() == 0 { |
||||
|
lastOffsetTsNs, err := getOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId)) |
||||
|
if err != nil { |
||||
|
glog.V(0).Infof("starting from %v", startFrom) |
||||
|
} else { |
||||
|
startFrom = time.Unix(0, lastOffsetTsNs) |
||||
|
glog.V(0).Infof("resuming from %v", startFrom) |
||||
|
} |
||||
|
} else { |
||||
|
startFrom = time.Now().Add(-timeAgo) |
||||
|
glog.V(0).Infof("start time is set to %v", startFrom) |
||||
|
} |
||||
|
|
||||
|
// create filer sink
|
||||
|
filerSource := &source.FilerSource{} |
||||
|
filerSource.DoInitialize(sourceFiler, pb.ServerToGrpcAddress(sourceFiler), sourcePath, *backupOption.proxyByFiler) |
||||
|
dataSink.SetSourceFiler(filerSource) |
||||
|
|
||||
|
processEventFn := genProcessFunction(sourcePath, targetPath, dataSink, debug) |
||||
|
|
||||
|
return pb.WithFilerClient(sourceFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { |
||||
|
|
||||
|
ctx, cancel := context.WithCancel(context.Background()) |
||||
|
defer cancel() |
||||
|
|
||||
|
stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{ |
||||
|
ClientName: "backup_" + dataSink.GetName(), |
||||
|
PathPrefix: sourcePath, |
||||
|
SinceNs: startFrom.UnixNano(), |
||||
|
}) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("listen: %v", err) |
||||
|
} |
||||
|
|
||||
|
var counter int64 |
||||
|
var lastWriteTime time.Time |
||||
|
for { |
||||
|
resp, listenErr := stream.Recv() |
||||
|
|
||||
|
if listenErr == io.EOF { |
||||
|
return nil |
||||
|
} |
||||
|
if listenErr != nil { |
||||
|
return listenErr |
||||
|
} |
||||
|
|
||||
|
if err := processEventFn(resp); err != nil { |
||||
|
return fmt.Errorf("processEventFn: %v", err) |
||||
|
} |
||||
|
|
||||
|
counter++ |
||||
|
if lastWriteTime.Add(3 * time.Second).Before(time.Now()) { |
||||
|
glog.V(0).Infof("backup %s progressed to %v %0.2f/sec", sourceFiler, time.Unix(0, resp.TsNs), float64(counter)/float64(3)) |
||||
|
counter = 0 |
||||
|
lastWriteTime = time.Now() |
||||
|
if err := setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId), resp.TsNs); err != nil { |
||||
|
return fmt.Errorf("setOffset: %v", err) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
}) |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,268 @@ |
|||||
|
package command |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"fmt" |
||||
|
"github.com/chrislusf/seaweedfs/weed/filer" |
||||
|
"github.com/chrislusf/seaweedfs/weed/glog" |
||||
|
"github.com/spf13/viper" |
||||
|
"google.golang.org/grpc" |
||||
|
"io" |
||||
|
"reflect" |
||||
|
"time" |
||||
|
|
||||
|
"github.com/chrislusf/seaweedfs/weed/pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/security" |
||||
|
"github.com/chrislusf/seaweedfs/weed/util" |
||||
|
) |
||||
|
|
||||
|
var ( |
||||
|
metaBackup FilerMetaBackupOptions |
||||
|
) |
||||
|
|
||||
|
type FilerMetaBackupOptions struct { |
||||
|
grpcDialOption grpc.DialOption |
||||
|
filerAddress *string |
||||
|
filerDirectory *string |
||||
|
restart *bool |
||||
|
backupFilerConfig *string |
||||
|
|
||||
|
store filer.FilerStore |
||||
|
} |
||||
|
|
||||
|
func init() { |
||||
|
cmdFilerMetaBackup.Run = runFilerMetaBackup // break init cycle
|
||||
|
metaBackup.filerAddress = cmdFilerMetaBackup.Flag.String("filer", "localhost:8888", "filer hostname:port") |
||||
|
metaBackup.filerDirectory = cmdFilerMetaBackup.Flag.String("filerDir", "/", "a folder on the filer") |
||||
|
metaBackup.restart = cmdFilerMetaBackup.Flag.Bool("restart", false, "copy the full metadata before async incremental backup") |
||||
|
metaBackup.backupFilerConfig = cmdFilerMetaBackup.Flag.String("config", "", "path to filer.toml specifying backup filer store") |
||||
|
} |
||||
|
|
||||
|
var cmdFilerMetaBackup = &Command{ |
||||
|
UsageLine: "filer.meta.backup [-filer=localhost:8888] [-filerDir=/] [-restart] -config=/path/to/backup_filer.toml", |
||||
|
Short: "continuously backup filer meta data changes to anther filer store specified in a backup_filer.toml", |
||||
|
Long: `continuously backup filer meta data changes. |
||||
|
The backup writes to another filer store specified in a backup_filer.toml. |
||||
|
|
||||
|
weed filer.meta.backup -config=/path/to/backup_filer.toml -filer="localhost:8888" |
||||
|
weed filer.meta.backup -config=/path/to/backup_filer.toml -filer="localhost:8888" -restart |
||||
|
|
||||
|
`, |
||||
|
} |
||||
|
|
||||
|
func runFilerMetaBackup(cmd *Command, args []string) bool { |
||||
|
|
||||
|
metaBackup.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") |
||||
|
|
||||
|
// load backup_filer.toml
|
||||
|
v := viper.New() |
||||
|
v.SetConfigFile(*metaBackup.backupFilerConfig) |
||||
|
|
||||
|
if err := v.ReadInConfig(); err != nil { // Handle errors reading the config file
|
||||
|
glog.Fatalf("Failed to load %s file.\nPlease use this command to generate the a %s.toml file\n"+ |
||||
|
" weed scaffold -config=%s -output=.\n\n\n", |
||||
|
*metaBackup.backupFilerConfig, "backup_filer", "filer") |
||||
|
} |
||||
|
|
||||
|
if err := metaBackup.initStore(v); err != nil { |
||||
|
glog.V(0).Infof("init backup filer store: %v", err) |
||||
|
return true |
||||
|
} |
||||
|
|
||||
|
missingPreviousBackup := false |
||||
|
_, err := metaBackup.getOffset() |
||||
|
if err != nil { |
||||
|
missingPreviousBackup = true |
||||
|
} |
||||
|
|
||||
|
if *metaBackup.restart || missingPreviousBackup { |
||||
|
glog.V(0).Infof("traversing metadata tree...") |
||||
|
startTime := time.Now() |
||||
|
if err := metaBackup.traverseMetadata(); err != nil { |
||||
|
glog.Errorf("traverse meta data: %v", err) |
||||
|
return true |
||||
|
} |
||||
|
glog.V(0).Infof("metadata copied up to %v", startTime) |
||||
|
if err := metaBackup.setOffset(startTime); err != nil { |
||||
|
startTime = time.Now() |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
for { |
||||
|
err := metaBackup.streamMetadataBackup() |
||||
|
if err != nil { |
||||
|
glog.Errorf("filer meta backup from %s: %v", *metaBackup.filerAddress, err) |
||||
|
time.Sleep(1747 * time.Millisecond) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return true |
||||
|
} |
||||
|
|
||||
|
func (metaBackup *FilerMetaBackupOptions) initStore(v *viper.Viper) error { |
||||
|
// load configuration for default filer store
|
||||
|
hasDefaultStoreConfigured := false |
||||
|
for _, store := range filer.Stores { |
||||
|
if v.GetBool(store.GetName() + ".enabled") { |
||||
|
store = reflect.New(reflect.ValueOf(store).Elem().Type()).Interface().(filer.FilerStore) |
||||
|
if err := store.Initialize(v, store.GetName()+"."); err != nil { |
||||
|
glog.Fatalf("failed to initialize store for %s: %+v", store.GetName(), err) |
||||
|
} |
||||
|
glog.V(0).Infof("configured filer store to %s", store.GetName()) |
||||
|
hasDefaultStoreConfigured = true |
||||
|
metaBackup.store = filer.NewFilerStoreWrapper(store) |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
if !hasDefaultStoreConfigured { |
||||
|
return fmt.Errorf("no filer store enabled in %s", v.ConfigFileUsed()) |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (metaBackup *FilerMetaBackupOptions) traverseMetadata() (err error) { |
||||
|
var saveErr error |
||||
|
|
||||
|
traverseErr := filer_pb.TraverseBfs(metaBackup, util.FullPath(*metaBackup.filerDirectory), func(parentPath util.FullPath, entry *filer_pb.Entry) { |
||||
|
|
||||
|
println("+", parentPath.Child(entry.Name)) |
||||
|
if err := metaBackup.store.InsertEntry(context.Background(), filer.FromPbEntry(string(parentPath), entry)); err != nil { |
||||
|
saveErr = fmt.Errorf("insert entry error: %v\n", err) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
}) |
||||
|
|
||||
|
if traverseErr != nil { |
||||
|
return fmt.Errorf("traverse: %v", traverseErr) |
||||
|
} |
||||
|
return saveErr |
||||
|
} |
||||
|
|
||||
|
var ( |
||||
|
MetaBackupKey = []byte("metaBackup") |
||||
|
) |
||||
|
|
||||
|
func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error { |
||||
|
|
||||
|
startTime, err := metaBackup.getOffset() |
||||
|
if err != nil { |
||||
|
startTime = time.Now() |
||||
|
} |
||||
|
glog.V(0).Infof("streaming from %v", startTime) |
||||
|
|
||||
|
store := metaBackup.store |
||||
|
|
||||
|
eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error { |
||||
|
|
||||
|
ctx := context.Background() |
||||
|
message := resp.EventNotification |
||||
|
|
||||
|
if message.OldEntry == nil && message.NewEntry == nil { |
||||
|
return nil |
||||
|
} |
||||
|
if message.OldEntry == nil && message.NewEntry != nil { |
||||
|
println("+", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name)) |
||||
|
entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry) |
||||
|
return store.InsertEntry(ctx, entry) |
||||
|
} |
||||
|
if message.OldEntry != nil && message.NewEntry == nil { |
||||
|
println("-", util.FullPath(resp.Directory).Child(message.OldEntry.Name)) |
||||
|
return store.DeleteEntry(ctx, util.FullPath(resp.Directory).Child(message.OldEntry.Name)) |
||||
|
} |
||||
|
if message.OldEntry != nil && message.NewEntry != nil { |
||||
|
if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name { |
||||
|
println("~", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name)) |
||||
|
entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry) |
||||
|
return store.UpdateEntry(ctx, entry) |
||||
|
} |
||||
|
println("-", util.FullPath(resp.Directory).Child(message.OldEntry.Name)) |
||||
|
if err := store.DeleteEntry(ctx, util.FullPath(resp.Directory).Child(message.OldEntry.Name)); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
println("+", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name)) |
||||
|
return store.InsertEntry(ctx, filer.FromPbEntry(message.NewParentPath, message.NewEntry)) |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
tailErr := pb.WithFilerClient(*metaBackup.filerAddress, metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { |
||||
|
|
||||
|
ctx, cancel := context.WithCancel(context.Background()) |
||||
|
defer cancel() |
||||
|
|
||||
|
stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{ |
||||
|
ClientName: "meta_backup", |
||||
|
PathPrefix: *metaBackup.filerDirectory, |
||||
|
SinceNs: startTime.UnixNano(), |
||||
|
}) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("listen: %v", err) |
||||
|
} |
||||
|
|
||||
|
var counter int64 |
||||
|
var lastWriteTime time.Time |
||||
|
for { |
||||
|
resp, listenErr := stream.Recv() |
||||
|
if listenErr == io.EOF { |
||||
|
return nil |
||||
|
} |
||||
|
if listenErr != nil { |
||||
|
return listenErr |
||||
|
} |
||||
|
if err = eachEntryFunc(resp); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
counter++ |
||||
|
if lastWriteTime.Add(3 * time.Second).Before(time.Now()) { |
||||
|
glog.V(0).Infof("meta backup %s progressed to %v %0.2f/sec", *metaBackup.filerAddress, time.Unix(0, resp.TsNs), float64(counter)/float64(3)) |
||||
|
counter = 0 |
||||
|
lastWriteTime = time.Now() |
||||
|
if err2 := metaBackup.setOffset(lastWriteTime); err2 != nil { |
||||
|
return err2 |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
}) |
||||
|
return tailErr |
||||
|
} |
||||
|
|
||||
|
func (metaBackup *FilerMetaBackupOptions) getOffset() (lastWriteTime time.Time, err error) { |
||||
|
value, err := metaBackup.store.KvGet(context.Background(), MetaBackupKey) |
||||
|
if err != nil { |
||||
|
return |
||||
|
} |
||||
|
tsNs := util.BytesToUint64(value) |
||||
|
|
||||
|
return time.Unix(0, int64(tsNs)), nil |
||||
|
} |
||||
|
|
||||
|
func (metaBackup *FilerMetaBackupOptions) setOffset(lastWriteTime time.Time) error { |
||||
|
valueBuf := make([]byte, 8) |
||||
|
util.Uint64toBytes(valueBuf, uint64(lastWriteTime.UnixNano())) |
||||
|
|
||||
|
if err := metaBackup.store.KvPut(context.Background(), MetaBackupKey, valueBuf); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
var _ = filer_pb.FilerClient(&FilerMetaBackupOptions{}) |
||||
|
|
||||
|
func (metaBackup *FilerMetaBackupOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error { |
||||
|
|
||||
|
return pb.WithFilerClient(*metaBackup.filerAddress, metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { |
||||
|
return fn(client) |
||||
|
}) |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func (metaBackup *FilerMetaBackupOptions) AdjustedUrl(location *filer_pb.Location) string { |
||||
|
return location.Url |
||||
|
} |
||||
@ -0,0 +1,137 @@ |
|||||
|
package weed_server |
||||
|
|
||||
|
import ( |
||||
|
"bufio" |
||||
|
"fmt" |
||||
|
"github.com/chrislusf/seaweedfs/weed/glog" |
||||
|
"github.com/chrislusf/seaweedfs/weed/storage/needle" |
||||
|
"github.com/chrislusf/seaweedfs/weed/util" |
||||
|
"io" |
||||
|
"net" |
||||
|
"strings" |
||||
|
) |
||||
|
|
||||
|
func (vs *VolumeServer) HandleTcpConnection(c net.Conn) { |
||||
|
defer c.Close() |
||||
|
|
||||
|
glog.V(0).Infof("Serving writes from %s", c.RemoteAddr().String()) |
||||
|
|
||||
|
bufReader := bufio.NewReaderSize(c, 1024*1024) |
||||
|
bufWriter := bufio.NewWriterSize(c, 1024*1024) |
||||
|
|
||||
|
for { |
||||
|
cmd, err := bufReader.ReadString('\n') |
||||
|
if err != nil { |
||||
|
if err != io.EOF { |
||||
|
glog.Errorf("read command from %s: %v", c.RemoteAddr().String(), err) |
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
cmd = cmd[:len(cmd)-1] |
||||
|
switch cmd[0] { |
||||
|
case '+': |
||||
|
fileId := cmd[1:] |
||||
|
err = vs.handleTcpPut(fileId, bufReader) |
||||
|
if err == nil { |
||||
|
bufWriter.Write([]byte("+OK\n")) |
||||
|
} else { |
||||
|
bufWriter.Write([]byte("-ERR " + string(err.Error()) + "\n")) |
||||
|
} |
||||
|
case '-': |
||||
|
fileId := cmd[1:] |
||||
|
err = vs.handleTcpDelete(fileId) |
||||
|
if err == nil { |
||||
|
bufWriter.Write([]byte("+OK\n")) |
||||
|
} else { |
||||
|
bufWriter.Write([]byte("-ERR " + string(err.Error()) + "\n")) |
||||
|
} |
||||
|
case '?': |
||||
|
fileId := cmd[1:] |
||||
|
err = vs.handleTcpGet(fileId, bufWriter) |
||||
|
case '!': |
||||
|
bufWriter.Flush() |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func (vs *VolumeServer) handleTcpGet(fileId string, writer *bufio.Writer) (err error) { |
||||
|
|
||||
|
volumeId, n, err2 := vs.parseFileId(fileId) |
||||
|
if err2 != nil { |
||||
|
return err2 |
||||
|
} |
||||
|
|
||||
|
volume := vs.store.GetVolume(volumeId) |
||||
|
if volume == nil { |
||||
|
return fmt.Errorf("volume %d not found", volumeId) |
||||
|
} |
||||
|
|
||||
|
err = volume.StreamRead(n, writer) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (vs *VolumeServer) handleTcpPut(fileId string, bufReader *bufio.Reader) (err error) { |
||||
|
|
||||
|
volumeId, n, err2 := vs.parseFileId(fileId) |
||||
|
if err2 != nil { |
||||
|
return err2 |
||||
|
} |
||||
|
|
||||
|
volume := vs.store.GetVolume(volumeId) |
||||
|
if volume == nil { |
||||
|
return fmt.Errorf("volume %d not found", volumeId) |
||||
|
} |
||||
|
|
||||
|
sizeBuf := make([]byte, 4) |
||||
|
if _, err = bufReader.Read(sizeBuf); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
dataSize := util.BytesToUint32(sizeBuf) |
||||
|
|
||||
|
err = volume.StreamWrite(n, bufReader, dataSize) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (vs *VolumeServer) handleTcpDelete(fileId string) (err error) { |
||||
|
|
||||
|
volumeId, n, err2 := vs.parseFileId(fileId) |
||||
|
if err2 != nil { |
||||
|
return err2 |
||||
|
} |
||||
|
|
||||
|
_, err = vs.store.DeleteVolumeNeedle(volumeId, n) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (vs *VolumeServer) parseFileId(fileId string) (needle.VolumeId, *needle.Needle, error) { |
||||
|
|
||||
|
commaIndex := strings.LastIndex(fileId, ",") |
||||
|
if commaIndex <= 0 { |
||||
|
return 0, nil, fmt.Errorf("unknown fileId %s", fileId) |
||||
|
} |
||||
|
|
||||
|
vid, fid := fileId[0:commaIndex], fileId[commaIndex+1:] |
||||
|
|
||||
|
volumeId, ve := needle.NewVolumeId(vid) |
||||
|
if ve != nil { |
||||
|
return 0, nil, fmt.Errorf("unknown volume id in fileId %s", fileId) |
||||
|
} |
||||
|
|
||||
|
n := new(needle.Needle) |
||||
|
n.ParsePath(fid) |
||||
|
return volumeId, n, nil |
||||
|
} |
||||
@ -0,0 +1,92 @@ |
|||||
|
package shell |
||||
|
|
||||
|
import ( |
||||
|
"flag" |
||||
|
"fmt" |
||||
|
"github.com/chrislusf/seaweedfs/weed/util" |
||||
|
"io" |
||||
|
"math" |
||||
|
"time" |
||||
|
|
||||
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
||||
|
) |
||||
|
|
||||
|
func init() { |
||||
|
Commands = append(Commands, &commandS3CleanUploads{}) |
||||
|
} |
||||
|
|
||||
|
type commandS3CleanUploads struct { |
||||
|
} |
||||
|
|
||||
|
func (c *commandS3CleanUploads) Name() string { |
||||
|
return "s3.clean.uploads" |
||||
|
} |
||||
|
|
||||
|
func (c *commandS3CleanUploads) Help() string { |
||||
|
return `clean up stale multipart uploads |
||||
|
|
||||
|
Example: |
||||
|
s3.clean.uploads -replication 001 |
||||
|
|
||||
|
` |
||||
|
} |
||||
|
|
||||
|
func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { |
||||
|
|
||||
|
bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) |
||||
|
uploadedTimeAgo := bucketCommand.Duration("timeAgo", 24*time.Hour, "created time before now. \"1.5h\" or \"2h45m\". Valid time units are \"m\", \"h\"") |
||||
|
if err = bucketCommand.Parse(args); err != nil { |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
var filerBucketsPath string |
||||
|
filerBucketsPath, err = readFilerBucketsPath(commandEnv) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("read buckets: %v", err) |
||||
|
} |
||||
|
|
||||
|
var buckets []string |
||||
|
err = filer_pb.List(commandEnv, filerBucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error { |
||||
|
buckets = append(buckets, entry.Name) |
||||
|
return nil |
||||
|
}, "", false, math.MaxUint32) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("list buckets under %v: %v", filerBucketsPath, err) |
||||
|
} |
||||
|
|
||||
|
for _, bucket:= range buckets { |
||||
|
c.cleanupUploads(commandEnv, writer, filerBucketsPath, bucket, *uploadedTimeAgo) |
||||
|
} |
||||
|
|
||||
|
return err |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io.Writer, filerBucketsPath string, bucket string, timeAgo time.Duration) error { |
||||
|
uploadsDir := filerBucketsPath+"/"+bucket+"/.uploads" |
||||
|
var staleUploads []string |
||||
|
now := time.Now() |
||||
|
err := filer_pb.List(commandEnv, uploadsDir, "", func(entry *filer_pb.Entry, isLast bool) error { |
||||
|
ctime := time.Unix(entry.Attributes.Crtime, 0) |
||||
|
if ctime.Add(timeAgo).Before(now) { |
||||
|
staleUploads = append(staleUploads, entry.Name) |
||||
|
} |
||||
|
return nil |
||||
|
}, "", false, math.MaxUint32) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("list uploads under %v: %v", uploadsDir, err) |
||||
|
} |
||||
|
|
||||
|
for _, staleUpload:= range staleUploads { |
||||
|
deleteUrl := fmt.Sprintf("http://%s:%d%s/%s?recursive=true&ignoreRecursiveError=true",commandEnv.option.FilerHost, commandEnv.option.FilerPort,uploadsDir, staleUpload) |
||||
|
fmt.Fprintf(writer, "purge %s\n", deleteUrl) |
||||
|
|
||||
|
err = util.Delete(deleteUrl, "") |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("purge %s/%s: %v", uploadsDir, staleUpload, err) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,104 @@ |
|||||
|
package storage |
||||
|
|
||||
|
import ( |
||||
|
"bufio" |
||||
|
"fmt" |
||||
|
"github.com/chrislusf/seaweedfs/weed/util" |
||||
|
"io" |
||||
|
"time" |
||||
|
|
||||
|
"github.com/chrislusf/seaweedfs/weed/glog" |
||||
|
"github.com/chrislusf/seaweedfs/weed/storage/backend" |
||||
|
"github.com/chrislusf/seaweedfs/weed/storage/needle" |
||||
|
. "github.com/chrislusf/seaweedfs/weed/storage/types" |
||||
|
) |
||||
|
|
||||
|
func (v *Volume) StreamWrite(n *needle.Needle, data io.Reader, dataSize uint32) (err error) { |
||||
|
|
||||
|
v.dataFileAccessLock.Lock() |
||||
|
defer v.dataFileAccessLock.Unlock() |
||||
|
|
||||
|
df, ok := v.DataBackend.(*backend.DiskFile) |
||||
|
if !ok { |
||||
|
return fmt.Errorf("unexpected volume backend") |
||||
|
} |
||||
|
offset, _, _ := v.DataBackend.GetStat() |
||||
|
|
||||
|
header := make([]byte, NeedleHeaderSize+TimestampSize) // adding timestamp to reuse it and avoid extra allocation
|
||||
|
CookieToBytes(header[0:CookieSize], n.Cookie) |
||||
|
NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id) |
||||
|
n.Size = 4 + Size(dataSize) + 1 |
||||
|
SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size) |
||||
|
|
||||
|
n.DataSize = dataSize |
||||
|
|
||||
|
// needle header
|
||||
|
df.Write(header[0:NeedleHeaderSize]) |
||||
|
|
||||
|
// data size and data
|
||||
|
util.Uint32toBytes(header[0:4], n.DataSize) |
||||
|
df.Write(header[0:4]) |
||||
|
// write and calculate CRC
|
||||
|
crcWriter := needle.NewCRCwriter(df) |
||||
|
io.Copy(crcWriter, io.LimitReader(data, int64(dataSize))) |
||||
|
|
||||
|
// flags
|
||||
|
util.Uint8toBytes(header[0:1], n.Flags) |
||||
|
df.Write(header[0:1]) |
||||
|
|
||||
|
// data checksum
|
||||
|
util.Uint32toBytes(header[0:needle.NeedleChecksumSize], crcWriter.Sum()) |
||||
|
// write timestamp, padding
|
||||
|
n.AppendAtNs = uint64(time.Now().UnixNano()) |
||||
|
util.Uint64toBytes(header[needle.NeedleChecksumSize:needle.NeedleChecksumSize+TimestampSize], n.AppendAtNs) |
||||
|
padding := needle.PaddingLength(n.Size, needle.Version3) |
||||
|
df.Write(header[0 : needle.NeedleChecksumSize+TimestampSize+padding]) |
||||
|
|
||||
|
// add to needle map
|
||||
|
if err = v.nm.Put(n.Id, ToOffset(int64(offset)), n.Size); err != nil { |
||||
|
glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err) |
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func (v *Volume) StreamRead(n *needle.Needle, writer io.Writer) (err error) { |
||||
|
|
||||
|
v.dataFileAccessLock.Lock() |
||||
|
defer v.dataFileAccessLock.Unlock() |
||||
|
|
||||
|
nv, ok := v.nm.Get(n.Id) |
||||
|
if !ok || nv.Offset.IsZero() { |
||||
|
return ErrorNotFound |
||||
|
} |
||||
|
|
||||
|
sr := &StreamReader{ |
||||
|
readerAt: v.DataBackend, |
||||
|
offset: nv.Offset.ToActualOffset(), |
||||
|
} |
||||
|
bufReader := bufio.NewReader(sr) |
||||
|
bufReader.Discard(NeedleHeaderSize) |
||||
|
sizeBuf := make([]byte, 4) |
||||
|
bufReader.Read(sizeBuf) |
||||
|
if _, err = writer.Write(sizeBuf); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
dataSize := util.BytesToUint32(sizeBuf) |
||||
|
|
||||
|
_, err = io.Copy(writer, io.LimitReader(bufReader, int64(dataSize))) |
||||
|
|
||||
|
return |
||||
|
} |
||||
|
|
||||
|
type StreamReader struct { |
||||
|
offset int64 |
||||
|
readerAt io.ReaderAt |
||||
|
} |
||||
|
|
||||
|
func (sr *StreamReader) Read(p []byte) (n int, err error) { |
||||
|
n, err = sr.readerAt.ReadAt(p, sr.offset) |
||||
|
if err != nil { |
||||
|
return |
||||
|
} |
||||
|
sr.offset += int64(n) |
||||
|
return |
||||
|
} |
||||
Some files were not shown because too many files changed in this diff
Write
Preview
Loading…
Cancel
Save
Reference in new issue