bingoohuang
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
69 changed files with 735 additions and 329 deletions
-
3docker/compose/local-clusters-compose.yml
-
8k8s/README.md
-
4k8s/seaweedfs/Chart.yaml
-
23k8s/seaweedfs/templates/_helpers.tpl
-
2k8s/seaweedfs/templates/cronjob.yaml
-
1k8s/seaweedfs/templates/filer-service-client.yaml
-
78k8s/seaweedfs/templates/volume-statefulset.yaml
-
20k8s/seaweedfs/values.yaml
-
2other/java/client/pom.xml
-
2other/java/client/pom.xml.deploy
-
2other/java/client/pom_debug.xml
-
33other/java/client/src/main/java/seaweedfs/client/FilerClient.java
-
3other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
-
4other/java/examples/pom.xml
-
2other/java/hdfs2/dependency-reduced-pom.xml
-
2other/java/hdfs2/pom.xml
-
2other/java/hdfs3/dependency-reduced-pom.xml
-
2other/java/hdfs3/pom.xml
-
4weed/Makefile
-
2weed/command/benchmark.go
-
1weed/command/command.go
-
2weed/command/filer.go
-
158weed/command/filer_backup.go
-
2weed/command/filer_copy.go
-
4weed/command/filer_meta_tail.go
-
29weed/command/filer_replication.go
-
207weed/command/filer_sync.go
-
2weed/command/mount.go
-
8weed/command/scaffold.go
-
4weed/command/server.go
-
2weed/command/upload.go
-
2weed/command/volume.go
-
2weed/command/webdav.go
-
2weed/filer/stream.go
-
17weed/operation/upload_content.go
-
2weed/replication/repl_util/replication_util.go
-
2weed/replication/replicator.go
-
6weed/replication/sink/azuresink/azure_sink.go
-
6weed/replication/sink/b2sink/b2_sink.go
-
6weed/replication/sink/filersink/filer_sink.go
-
6weed/replication/sink/gcssink/gcs_sink.go
-
8weed/replication/sink/localsink/local_sink.go
-
1weed/replication/sink/replication_sink.go
-
13weed/replication/sink/s3sink/s3_sink.go
-
7weed/replication/sink/s3sink/s3_write.go
-
6weed/replication/source/filer_source.go
-
9weed/server/filer_server_handlers_read.go
-
5weed/server/master_grpc_server_volume.go
-
2weed/server/volume_grpc_copy.go
-
11weed/shell/command_ec_common.go
-
6weed/shell/command_ec_decode.go
-
11weed/shell/command_ec_encode.go
-
2weed/shell/command_fs_configure.go
-
20weed/shell/command_volume_balance.go
-
9weed/shell/command_volume_configure_replication.go
-
21weed/shell/command_volume_fix_replication.go
-
13weed/shell/command_volume_fsck.go
-
10weed/shell/command_volume_list.go
-
6weed/shell/command_volume_move.go
-
22weed/shell/command_volume_server_evacuate.go
-
2weed/shell/command_volume_tier_download.go
-
121weed/shell/command_volume_tier_move.go
-
7weed/storage/types/volume_disk_type.go
-
25weed/topology/topology_vacuum.go
-
2weed/topology/volume_layout.go
-
2weed/topology/volume_location_list.go
-
2weed/util/constants.go
-
4weed/util/file_util.go
-
4weed/util/fla9/fla9.go
@ -1,5 +1,5 @@ |
|||
apiVersion: v1 |
|||
description: SeaweedFS |
|||
name: seaweedfs |
|||
appVersion: "2.26" |
|||
version: 2.26 |
|||
appVersion: "2.28" |
|||
version: 2.28 |
|||
@ -0,0 +1,158 @@ |
|||
package command |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/pb" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/replication/source" |
|||
"github.com/chrislusf/seaweedfs/weed/security" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
"google.golang.org/grpc" |
|||
"io" |
|||
"time" |
|||
) |
|||
|
|||
type FilerBackupOptions struct { |
|||
isActivePassive *bool |
|||
filer *string |
|||
path *string |
|||
debug *bool |
|||
proxyByFiler *bool |
|||
timeAgo *time.Duration |
|||
} |
|||
|
|||
var ( |
|||
filerBackupOptions FilerBackupOptions |
|||
) |
|||
|
|||
func init() { |
|||
cmdFilerBackup.Run = runFilerBackup // break init cycle
|
|||
filerBackupOptions.filer = cmdFilerBackup.Flag.String("filer", "localhost:8888", "filer of one SeaweedFS cluster") |
|||
filerBackupOptions.path = cmdFilerBackup.Flag.String("filerPath", "/", "directory to sync on filer") |
|||
filerBackupOptions.proxyByFiler = cmdFilerBackup.Flag.Bool("filerProxy", false, "read and write file chunks by filer instead of volume servers") |
|||
filerBackupOptions.debug = cmdFilerBackup.Flag.Bool("debug", false, "debug mode to print out received files") |
|||
filerBackupOptions.timeAgo = cmdFilerBackup.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"") |
|||
} |
|||
|
|||
var cmdFilerBackup = &Command{ |
|||
UsageLine: "filer.backup -filer=<filerHost>:<filerPort> ", |
|||
Short: "resume-able continuously replicate files from a SeaweedFS cluster to another location defined in replication.toml", |
|||
Long: `resume-able continuously replicate files from a SeaweedFS cluster to another location defined in replication.toml |
|||
|
|||
filer.backup listens on filer notifications. If any file is updated, it will fetch the updated content, |
|||
and write to the destination. This is to replace filer.replicate command since additional message queue is not needed. |
|||
|
|||
If restarted and "-timeAgo" is not set, the synchronization will resume from the previous checkpoints, persisted every minute. |
|||
A fresh sync will start from the earliest metadata logs. To reset the checkpoints, just set "-timeAgo" to a high value. |
|||
|
|||
`, |
|||
} |
|||
|
|||
func runFilerBackup(cmd *Command, args []string) bool { |
|||
|
|||
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") |
|||
|
|||
util.LoadConfiguration("security", false) |
|||
util.LoadConfiguration("replication", true) |
|||
|
|||
for { |
|||
err := doFilerBackup(grpcDialOption, &filerBackupOptions) |
|||
if err != nil { |
|||
glog.Errorf("backup from %s: %v", *filerBackupOptions.filer, err) |
|||
time.Sleep(1747 * time.Millisecond) |
|||
} |
|||
} |
|||
|
|||
return true |
|||
} |
|||
|
|||
const ( |
|||
BackupKeyPrefix = "backup." |
|||
) |
|||
|
|||
func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions) error { |
|||
|
|||
// find data sink
|
|||
config := util.GetViper() |
|||
dataSink := findSink(config) |
|||
if dataSink == nil { |
|||
return fmt.Errorf("no data sink configured in replication.toml") |
|||
} |
|||
|
|||
sourceFiler := *backupOption.filer |
|||
sourcePath := *backupOption.path |
|||
timeAgo := *backupOption.timeAgo |
|||
targetPath := dataSink.GetSinkToDirectory() |
|||
debug := *backupOption.debug |
|||
|
|||
// get start time for the data sink
|
|||
startFrom := time.Unix(0, 0) |
|||
sinkId := util.HashStringToLong(dataSink.GetName() + dataSink.GetSinkToDirectory()) |
|||
if timeAgo.Milliseconds() == 0 { |
|||
lastOffsetTsNs, err := getOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId)) |
|||
if err != nil { |
|||
glog.V(0).Infof("starting from %v", startFrom) |
|||
} else { |
|||
startFrom = time.Unix(0, lastOffsetTsNs) |
|||
glog.V(0).Infof("resuming from %v", startFrom) |
|||
} |
|||
} else { |
|||
startFrom = time.Now().Add(-timeAgo) |
|||
glog.V(0).Infof("start time is set to %v", startFrom) |
|||
} |
|||
|
|||
// create filer sink
|
|||
filerSource := &source.FilerSource{} |
|||
filerSource.DoInitialize(sourceFiler, pb.ServerToGrpcAddress(sourceFiler), sourcePath, *backupOption.proxyByFiler) |
|||
dataSink.SetSourceFiler(filerSource) |
|||
|
|||
processEventFn := genProcessFunction(sourcePath, targetPath, dataSink, debug) |
|||
|
|||
return pb.WithFilerClient(sourceFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { |
|||
|
|||
ctx, cancel := context.WithCancel(context.Background()) |
|||
defer cancel() |
|||
|
|||
stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{ |
|||
ClientName: "backup_" + dataSink.GetName(), |
|||
PathPrefix: sourcePath, |
|||
SinceNs: startFrom.UnixNano(), |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("listen: %v", err) |
|||
} |
|||
|
|||
var counter int64 |
|||
var lastWriteTime time.Time |
|||
for { |
|||
resp, listenErr := stream.Recv() |
|||
|
|||
if listenErr == io.EOF { |
|||
return nil |
|||
} |
|||
if listenErr != nil { |
|||
return listenErr |
|||
} |
|||
|
|||
if err := processEventFn(resp); err != nil { |
|||
return fmt.Errorf("processEventFn: %v", err) |
|||
} |
|||
|
|||
counter++ |
|||
if lastWriteTime.Add(3 * time.Second).Before(time.Now()) { |
|||
glog.V(0).Infof("backup %s progressed to %v %0.2f/sec", sourceFiler, time.Unix(0, resp.TsNs), float64(counter)/float64(3)) |
|||
counter = 0 |
|||
lastWriteTime = time.Now() |
|||
if err := setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId), resp.TsNs); err != nil { |
|||
return fmt.Errorf("setOffset: %v", err) |
|||
} |
|||
} |
|||
|
|||
} |
|||
|
|||
}) |
|||
|
|||
} |
|||
|
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue