bingoohuang
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
69 changed files with 735 additions and 329 deletions
-
3docker/compose/local-clusters-compose.yml
-
8k8s/README.md
-
4k8s/seaweedfs/Chart.yaml
-
23k8s/seaweedfs/templates/_helpers.tpl
-
2k8s/seaweedfs/templates/cronjob.yaml
-
1k8s/seaweedfs/templates/filer-service-client.yaml
-
78k8s/seaweedfs/templates/volume-statefulset.yaml
-
20k8s/seaweedfs/values.yaml
-
2other/java/client/pom.xml
-
2other/java/client/pom.xml.deploy
-
2other/java/client/pom_debug.xml
-
33other/java/client/src/main/java/seaweedfs/client/FilerClient.java
-
3other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
-
4other/java/examples/pom.xml
-
2other/java/hdfs2/dependency-reduced-pom.xml
-
2other/java/hdfs2/pom.xml
-
2other/java/hdfs3/dependency-reduced-pom.xml
-
2other/java/hdfs3/pom.xml
-
4weed/Makefile
-
2weed/command/benchmark.go
-
1weed/command/command.go
-
2weed/command/filer.go
-
158weed/command/filer_backup.go
-
2weed/command/filer_copy.go
-
4weed/command/filer_meta_tail.go
-
29weed/command/filer_replication.go
-
207weed/command/filer_sync.go
-
2weed/command/mount.go
-
8weed/command/scaffold.go
-
4weed/command/server.go
-
2weed/command/upload.go
-
2weed/command/volume.go
-
2weed/command/webdav.go
-
2weed/filer/stream.go
-
33weed/operation/upload_content.go
-
2weed/replication/repl_util/replication_util.go
-
2weed/replication/replicator.go
-
14weed/replication/sink/azuresink/azure_sink.go
-
6weed/replication/sink/b2sink/b2_sink.go
-
6weed/replication/sink/filersink/filer_sink.go
-
6weed/replication/sink/gcssink/gcs_sink.go
-
8weed/replication/sink/localsink/local_sink.go
-
1weed/replication/sink/replication_sink.go
-
29weed/replication/sink/s3sink/s3_sink.go
-
7weed/replication/sink/s3sink/s3_write.go
-
8weed/replication/source/filer_source.go
-
9weed/server/filer_server_handlers_read.go
-
7weed/server/master_grpc_server_volume.go
-
2weed/server/volume_grpc_copy.go
-
11weed/shell/command_ec_common.go
-
6weed/shell/command_ec_decode.go
-
11weed/shell/command_ec_encode.go
-
2weed/shell/command_fs_configure.go
-
20weed/shell/command_volume_balance.go
-
9weed/shell/command_volume_configure_replication.go
-
21weed/shell/command_volume_fix_replication.go
-
13weed/shell/command_volume_fsck.go
-
10weed/shell/command_volume_list.go
-
6weed/shell/command_volume_move.go
-
22weed/shell/command_volume_server_evacuate.go
-
2weed/shell/command_volume_tier_download.go
-
121weed/shell/command_volume_tier_move.go
-
7weed/storage/types/volume_disk_type.go
-
25weed/topology/topology_vacuum.go
-
2weed/topology/volume_layout.go
-
2weed/topology/volume_location_list.go
-
2weed/util/constants.go
-
4weed/util/file_util.go
-
4weed/util/fla9/fla9.go
@ -1,5 +1,5 @@ |
|||||
apiVersion: v1 |
apiVersion: v1 |
||||
description: SeaweedFS |
description: SeaweedFS |
||||
name: seaweedfs |
name: seaweedfs |
||||
appVersion: "2.26" |
|
||||
version: 2.26 |
|
||||
|
appVersion: "2.28" |
||||
|
version: 2.28 |
||||
@ -0,0 +1,158 @@ |
|||||
|
package command |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"fmt" |
||||
|
"github.com/chrislusf/seaweedfs/weed/glog" |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/replication/source" |
||||
|
"github.com/chrislusf/seaweedfs/weed/security" |
||||
|
"github.com/chrislusf/seaweedfs/weed/util" |
||||
|
"google.golang.org/grpc" |
||||
|
"io" |
||||
|
"time" |
||||
|
) |
||||
|
|
||||
|
type FilerBackupOptions struct { |
||||
|
isActivePassive *bool |
||||
|
filer *string |
||||
|
path *string |
||||
|
debug *bool |
||||
|
proxyByFiler *bool |
||||
|
timeAgo *time.Duration |
||||
|
} |
||||
|
|
||||
|
var ( |
||||
|
filerBackupOptions FilerBackupOptions |
||||
|
) |
||||
|
|
||||
|
func init() { |
||||
|
cmdFilerBackup.Run = runFilerBackup // break init cycle
|
||||
|
filerBackupOptions.filer = cmdFilerBackup.Flag.String("filer", "localhost:8888", "filer of one SeaweedFS cluster") |
||||
|
filerBackupOptions.path = cmdFilerBackup.Flag.String("filerPath", "/", "directory to sync on filer") |
||||
|
filerBackupOptions.proxyByFiler = cmdFilerBackup.Flag.Bool("filerProxy", false, "read and write file chunks by filer instead of volume servers") |
||||
|
filerBackupOptions.debug = cmdFilerBackup.Flag.Bool("debug", false, "debug mode to print out received files") |
||||
|
filerBackupOptions.timeAgo = cmdFilerBackup.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"") |
||||
|
} |
||||
|
|
||||
|
var cmdFilerBackup = &Command{ |
||||
|
UsageLine: "filer.backup -filer=<filerHost>:<filerPort> ", |
||||
|
Short: "resume-able continuously replicate files from a SeaweedFS cluster to another location defined in replication.toml", |
||||
|
Long: `resume-able continuously replicate files from a SeaweedFS cluster to another location defined in replication.toml |
||||
|
|
||||
|
filer.backup listens on filer notifications. If any file is updated, it will fetch the updated content, |
||||
|
and write to the destination. This is to replace filer.replicate command since additional message queue is not needed. |
||||
|
|
||||
|
If restarted and "-timeAgo" is not set, the synchronization will resume from the previous checkpoints, persisted every minute. |
||||
|
A fresh sync will start from the earliest metadata logs. To reset the checkpoints, just set "-timeAgo" to a high value. |
||||
|
|
||||
|
`, |
||||
|
} |
||||
|
|
||||
|
func runFilerBackup(cmd *Command, args []string) bool { |
||||
|
|
||||
|
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") |
||||
|
|
||||
|
util.LoadConfiguration("security", false) |
||||
|
util.LoadConfiguration("replication", true) |
||||
|
|
||||
|
for { |
||||
|
err := doFilerBackup(grpcDialOption, &filerBackupOptions) |
||||
|
if err != nil { |
||||
|
glog.Errorf("backup from %s: %v", *filerBackupOptions.filer, err) |
||||
|
time.Sleep(1747 * time.Millisecond) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return true |
||||
|
} |
||||
|
|
||||
|
const ( |
||||
|
BackupKeyPrefix = "backup." |
||||
|
) |
||||
|
|
||||
|
func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions) error { |
||||
|
|
||||
|
// find data sink
|
||||
|
config := util.GetViper() |
||||
|
dataSink := findSink(config) |
||||
|
if dataSink == nil { |
||||
|
return fmt.Errorf("no data sink configured in replication.toml") |
||||
|
} |
||||
|
|
||||
|
sourceFiler := *backupOption.filer |
||||
|
sourcePath := *backupOption.path |
||||
|
timeAgo := *backupOption.timeAgo |
||||
|
targetPath := dataSink.GetSinkToDirectory() |
||||
|
debug := *backupOption.debug |
||||
|
|
||||
|
// get start time for the data sink
|
||||
|
startFrom := time.Unix(0, 0) |
||||
|
sinkId := util.HashStringToLong(dataSink.GetName() + dataSink.GetSinkToDirectory()) |
||||
|
if timeAgo.Milliseconds() == 0 { |
||||
|
lastOffsetTsNs, err := getOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId)) |
||||
|
if err != nil { |
||||
|
glog.V(0).Infof("starting from %v", startFrom) |
||||
|
} else { |
||||
|
startFrom = time.Unix(0, lastOffsetTsNs) |
||||
|
glog.V(0).Infof("resuming from %v", startFrom) |
||||
|
} |
||||
|
} else { |
||||
|
startFrom = time.Now().Add(-timeAgo) |
||||
|
glog.V(0).Infof("start time is set to %v", startFrom) |
||||
|
} |
||||
|
|
||||
|
// create filer sink
|
||||
|
filerSource := &source.FilerSource{} |
||||
|
filerSource.DoInitialize(sourceFiler, pb.ServerToGrpcAddress(sourceFiler), sourcePath, *backupOption.proxyByFiler) |
||||
|
dataSink.SetSourceFiler(filerSource) |
||||
|
|
||||
|
processEventFn := genProcessFunction(sourcePath, targetPath, dataSink, debug) |
||||
|
|
||||
|
return pb.WithFilerClient(sourceFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { |
||||
|
|
||||
|
ctx, cancel := context.WithCancel(context.Background()) |
||||
|
defer cancel() |
||||
|
|
||||
|
stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{ |
||||
|
ClientName: "backup_" + dataSink.GetName(), |
||||
|
PathPrefix: sourcePath, |
||||
|
SinceNs: startFrom.UnixNano(), |
||||
|
}) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("listen: %v", err) |
||||
|
} |
||||
|
|
||||
|
var counter int64 |
||||
|
var lastWriteTime time.Time |
||||
|
for { |
||||
|
resp, listenErr := stream.Recv() |
||||
|
|
||||
|
if listenErr == io.EOF { |
||||
|
return nil |
||||
|
} |
||||
|
if listenErr != nil { |
||||
|
return listenErr |
||||
|
} |
||||
|
|
||||
|
if err := processEventFn(resp); err != nil { |
||||
|
return fmt.Errorf("processEventFn: %v", err) |
||||
|
} |
||||
|
|
||||
|
counter++ |
||||
|
if lastWriteTime.Add(3 * time.Second).Before(time.Now()) { |
||||
|
glog.V(0).Infof("backup %s progressed to %v %0.2f/sec", sourceFiler, time.Unix(0, resp.TsNs), float64(counter)/float64(3)) |
||||
|
counter = 0 |
||||
|
lastWriteTime = time.Now() |
||||
|
if err := setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId), resp.TsNs); err != nil { |
||||
|
return fmt.Errorf("setOffset: %v", err) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
}) |
||||
|
|
||||
|
} |
||||
|
|
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue