hilimd
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
121 changed files with 3359 additions and 442 deletions
-
2.github/workflows/release.yml
-
41README.md
-
2backers.md
-
2docker/Dockerfile.go_build
-
2docker/Dockerfile.go_build_large
-
17docker/Makefile
-
0docker/compose/dev.env
-
7docker/compose/local-clusters-compose.yml
-
28docker/compose/local-dev-compose.yml
-
30docker/compose/master-cloud.toml
-
14docker/compose/tls.env
-
4docker/entrypoint.sh
-
3go.mod
-
2go.sum
-
8k8s/README.md
-
4k8s/seaweedfs/Chart.yaml
-
23k8s/seaweedfs/templates/_helpers.tpl
-
2k8s/seaweedfs/templates/cronjob.yaml
-
1k8s/seaweedfs/templates/filer-service-client.yaml
-
78k8s/seaweedfs/templates/volume-statefulset.yaml
-
20k8s/seaweedfs/values.yaml
-
BINnote/SeaweedFS_Architecture.png
-
BINnote/SeaweedFS_Cluster_Backup.png
-
BINnote/SeaweedFS_XDR.png
-
2other/java/client/pom.xml
-
2other/java/client/pom.xml.deploy
-
2other/java/client/pom_debug.xml
-
3other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
-
4other/java/examples/pom.xml
-
2other/java/hdfs2/dependency-reduced-pom.xml
-
2other/java/hdfs2/pom.xml
-
2other/java/hdfs3/dependency-reduced-pom.xml
-
2other/java/hdfs3/pom.xml
-
4weed/Makefile
-
25weed/command/benchmark.go
-
2weed/command/command.go
-
157weed/command/filer_backup.go
-
268weed/command/filer_meta_backup.go
-
8weed/command/filer_meta_tail.go
-
29weed/command/filer_replication.go
-
203weed/command/filer_sync.go
-
1weed/command/master.go
-
23weed/command/mount_std.go
-
2weed/command/msg_broker.go
-
2weed/command/s3.go
-
15weed/command/scaffold.go
-
13weed/command/server.go
-
26weed/command/volume.go
-
2weed/command/webdav.go
-
6weed/filer/filer_delete_entry.go
-
5weed/filer/filer_on_meta_event.go
-
1weed/filer/meta_aggregator.go
-
8weed/filer/stream.go
-
14weed/filesys/dir.go
-
29weed/filesys/dir_link.go
-
9weed/filesys/dirty_page.go
-
51weed/filesys/file.go
-
75weed/filesys/filehandle.go
-
5weed/filesys/fscache.go
-
2weed/operation/upload_content.go
-
14weed/pb/filer_pb/filer_client.go
-
65weed/pb/grpc_client_server.go
-
29weed/pb/volume_info.go
-
2weed/replication/repl_util/replication_util.go
-
2weed/replication/replicator.go
-
14weed/replication/sink/azuresink/azure_sink.go
-
14weed/replication/sink/b2sink/b2_sink.go
-
6weed/replication/sink/filersink/filer_sink.go
-
14weed/replication/sink/gcssink/gcs_sink.go
-
8weed/replication/sink/localsink/local_sink.go
-
1weed/replication/sink/replication_sink.go
-
29weed/replication/sink/s3sink/s3_sink.go
-
13weed/replication/sink/s3sink/s3_write.go
-
8weed/replication/source/filer_source.go
-
4weed/s3api/filer_util.go
-
2weed/s3api/s3api_bucket_handlers.go
-
20weed/s3api/s3api_object_handlers.go
-
2weed/s3api/s3api_object_handlers_postpolicy.go
-
66weed/security/tls.go
-
2weed/server/filer_server.go
-
11weed/server/filer_server_handlers_read.go
-
11weed/server/filer_server_handlers_write_autochunk.go
-
7weed/server/master_grpc_server_volume.go
-
137weed/server/volume_server_tcp_handlers_write.go
-
2weed/shell/command_ec_encode.go
-
92weed/shell/command_s3_clean_uploads.go
-
2weed/shell/command_volume_tier_move.go
-
6weed/storage/backend/backend.go
-
2weed/storage/backend/disk_file.go
-
5weed/storage/backend/s3_backend/s3_sessions.go
-
2weed/storage/erasure_coding/ec_volume.go
-
23weed/storage/needle/crc.go
-
2weed/storage/needle/needle_read_write.go
-
6weed/storage/needle_map_leveldb.go
-
8weed/storage/needle_map_sorted_file.go
-
18weed/storage/store.go
-
3weed/storage/super_block/replica_placement.go
-
10weed/storage/volume_loading.go
-
60weed/storage/volume_read_write.go
-
104weed/storage/volume_stream_write.go
@ -0,0 +1,30 @@ |
|||
|
|||
# Put this file to one of the location, with descending priority |
|||
# ./master.toml |
|||
# $HOME/.seaweedfs/master.toml |
|||
# /etc/seaweedfs/master.toml |
|||
# this file is read by master |
|||
|
|||
[master.maintenance] |
|||
# periodically run these scripts are the same as running them from 'weed shell' |
|||
scripts = """ |
|||
lock |
|||
ec.encode -fullPercent=95 -quietFor=1h |
|||
ec.rebuild -force |
|||
ec.balance -force |
|||
volume.balance -force |
|||
volume.fix.replication |
|||
unlock |
|||
""" |
|||
sleep_minutes = 17 # sleep minutes between each script execution |
|||
|
|||
# configurations for tiered cloud storage |
|||
# old volumes are transparently moved to cloud for cost efficiency |
|||
[storage.backend] |
|||
[storage.backend.s3.default] |
|||
enabled = true |
|||
aws_access_key_id = "any" # if empty, loads from the shared credentials file (~/.aws/credentials). |
|||
aws_secret_access_key = "any" # if empty, loads from the shared credentials file (~/.aws/credentials). |
|||
region = "us-east-2" |
|||
bucket = "volume_bucket" # an existing bucket |
|||
endpoint = "http://server2:8333" |
|||
@ -0,0 +1,14 @@ |
|||
WEED_GRPC_CA=/etc/seaweedfs/tls/SeaweedFS_CA.crt |
|||
WEED_GRPC_ALLOWED_WILDCARD_DOMAIN=".dev" |
|||
WEED_GRPC_MASTER_CERT=/etc/seaweedfs/tls/master01.dev.crt |
|||
WEED_GRPC_MASTER_KEY=/etc/seaweedfs/tls/master01.dev.key |
|||
WEED_GRPC_VOLUME_CERT=/etc/seaweedfs/tls/volume01.dev.crt |
|||
WEED_GRPC_VOLUME_KEY=/etc/seaweedfs/tls/volume01.dev.key |
|||
WEED_GRPC_FILER_CERT=/etc/seaweedfs/tls/filer01.dev.crt |
|||
WEED_GRPC_FILER_KEY=/etc/seaweedfs/tls/filer01.dev.key |
|||
WEED_GRPC_CLIENT_CERT=/etc/seaweedfs/tls/client01.dev.crt |
|||
WEED_GRPC_CLIENT_KEY=/etc/seaweedfs/tls/client01.dev.key |
|||
WEED_GRPC_MASTER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" |
|||
WEED_GRPC_VOLUME_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" |
|||
WEED_GRPC_FILER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" |
|||
WEED_GRPC_CLIENT_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev" |
|||
@ -1,5 +1,5 @@ |
|||
apiVersion: v1 |
|||
description: SeaweedFS |
|||
name: seaweedfs |
|||
appVersion: "2.27" |
|||
version: 2.27 |
|||
appVersion: "2.31" |
|||
version: 2.31 |
|||
|
After Width: 1121 | Height: 701 | Size: 90 KiB |
|
After Width: 1211 | Height: 586 | Size: 86 KiB |
|
After Width: 1271 | Height: 566 | Size: 62 KiB |
@ -0,0 +1,157 @@ |
|||
package command |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/pb" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/replication/source" |
|||
"github.com/chrislusf/seaweedfs/weed/security" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
"google.golang.org/grpc" |
|||
"io" |
|||
"time" |
|||
) |
|||
|
|||
type FilerBackupOptions struct { |
|||
isActivePassive *bool |
|||
filer *string |
|||
path *string |
|||
debug *bool |
|||
proxyByFiler *bool |
|||
timeAgo *time.Duration |
|||
} |
|||
|
|||
var ( |
|||
filerBackupOptions FilerBackupOptions |
|||
) |
|||
|
|||
func init() { |
|||
cmdFilerBackup.Run = runFilerBackup // break init cycle
|
|||
filerBackupOptions.filer = cmdFilerBackup.Flag.String("filer", "localhost:8888", "filer of one SeaweedFS cluster") |
|||
filerBackupOptions.path = cmdFilerBackup.Flag.String("filerPath", "/", "directory to sync on filer") |
|||
filerBackupOptions.proxyByFiler = cmdFilerBackup.Flag.Bool("filerProxy", false, "read and write file chunks by filer instead of volume servers") |
|||
filerBackupOptions.debug = cmdFilerBackup.Flag.Bool("debug", false, "debug mode to print out received files") |
|||
filerBackupOptions.timeAgo = cmdFilerBackup.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"") |
|||
} |
|||
|
|||
var cmdFilerBackup = &Command{ |
|||
UsageLine: "filer.backup -filer=<filerHost>:<filerPort> ", |
|||
Short: "resume-able continuously replicate files from a SeaweedFS cluster to another location defined in replication.toml", |
|||
Long: `resume-able continuously replicate files from a SeaweedFS cluster to another location defined in replication.toml |
|||
|
|||
filer.backup listens on filer notifications. If any file is updated, it will fetch the updated content, |
|||
and write to the destination. This is to replace filer.replicate command since additional message queue is not needed. |
|||
|
|||
If restarted and "-timeAgo" is not set, the synchronization will resume from the previous checkpoints, persisted every minute. |
|||
A fresh sync will start from the earliest metadata logs. To reset the checkpoints, just set "-timeAgo" to a high value. |
|||
|
|||
`, |
|||
} |
|||
|
|||
func runFilerBackup(cmd *Command, args []string) bool { |
|||
|
|||
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") |
|||
|
|||
util.LoadConfiguration("security", false) |
|||
util.LoadConfiguration("replication", true) |
|||
|
|||
for { |
|||
err := doFilerBackup(grpcDialOption, &filerBackupOptions) |
|||
if err != nil { |
|||
glog.Errorf("backup from %s: %v", *filerBackupOptions.filer, err) |
|||
time.Sleep(1747 * time.Millisecond) |
|||
} |
|||
} |
|||
|
|||
return true |
|||
} |
|||
|
|||
const ( |
|||
BackupKeyPrefix = "backup." |
|||
) |
|||
|
|||
func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions) error { |
|||
|
|||
// find data sink
|
|||
config := util.GetViper() |
|||
dataSink := findSink(config) |
|||
if dataSink == nil { |
|||
return fmt.Errorf("no data sink configured in replication.toml") |
|||
} |
|||
|
|||
sourceFiler := *backupOption.filer |
|||
sourcePath := *backupOption.path |
|||
timeAgo := *backupOption.timeAgo |
|||
targetPath := dataSink.GetSinkToDirectory() |
|||
debug := *backupOption.debug |
|||
|
|||
// get start time for the data sink
|
|||
startFrom := time.Unix(0, 0) |
|||
sinkId := util.HashStringToLong(dataSink.GetName() + dataSink.GetSinkToDirectory()) |
|||
if timeAgo.Milliseconds() == 0 { |
|||
lastOffsetTsNs, err := getOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId)) |
|||
if err != nil { |
|||
glog.V(0).Infof("starting from %v", startFrom) |
|||
} else { |
|||
startFrom = time.Unix(0, lastOffsetTsNs) |
|||
glog.V(0).Infof("resuming from %v", startFrom) |
|||
} |
|||
} else { |
|||
startFrom = time.Now().Add(-timeAgo) |
|||
glog.V(0).Infof("start time is set to %v", startFrom) |
|||
} |
|||
|
|||
// create filer sink
|
|||
filerSource := &source.FilerSource{} |
|||
filerSource.DoInitialize(sourceFiler, pb.ServerToGrpcAddress(sourceFiler), sourcePath, *backupOption.proxyByFiler) |
|||
dataSink.SetSourceFiler(filerSource) |
|||
|
|||
processEventFn := genProcessFunction(sourcePath, targetPath, dataSink, debug) |
|||
|
|||
return pb.WithFilerClient(sourceFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { |
|||
|
|||
ctx, cancel := context.WithCancel(context.Background()) |
|||
defer cancel() |
|||
|
|||
stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{ |
|||
ClientName: "backup_" + dataSink.GetName(), |
|||
PathPrefix: sourcePath, |
|||
SinceNs: startFrom.UnixNano(), |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("listen: %v", err) |
|||
} |
|||
|
|||
var counter int64 |
|||
var lastWriteTime time.Time |
|||
for { |
|||
resp, listenErr := stream.Recv() |
|||
|
|||
if listenErr == io.EOF { |
|||
return nil |
|||
} |
|||
if listenErr != nil { |
|||
return listenErr |
|||
} |
|||
|
|||
if err := processEventFn(resp); err != nil { |
|||
return fmt.Errorf("processEventFn: %v", err) |
|||
} |
|||
|
|||
counter++ |
|||
if lastWriteTime.Add(3 * time.Second).Before(time.Now()) { |
|||
glog.V(0).Infof("backup %s progressed to %v %0.2f/sec", sourceFiler, time.Unix(0, resp.TsNs), float64(counter)/float64(3)) |
|||
counter = 0 |
|||
lastWriteTime = time.Now() |
|||
if err := setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId), resp.TsNs); err != nil { |
|||
return fmt.Errorf("setOffset: %v", err) |
|||
} |
|||
} |
|||
|
|||
} |
|||
|
|||
}) |
|||
|
|||
} |
|||
@ -0,0 +1,268 @@ |
|||
package command |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"github.com/chrislusf/seaweedfs/weed/filer" |
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/spf13/viper" |
|||
"google.golang.org/grpc" |
|||
"io" |
|||
"reflect" |
|||
"time" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/pb" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/security" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
) |
|||
|
|||
var ( |
|||
metaBackup FilerMetaBackupOptions |
|||
) |
|||
|
|||
type FilerMetaBackupOptions struct { |
|||
grpcDialOption grpc.DialOption |
|||
filerAddress *string |
|||
filerDirectory *string |
|||
restart *bool |
|||
backupFilerConfig *string |
|||
|
|||
store filer.FilerStore |
|||
} |
|||
|
|||
func init() { |
|||
cmdFilerMetaBackup.Run = runFilerMetaBackup // break init cycle
|
|||
metaBackup.filerAddress = cmdFilerMetaBackup.Flag.String("filer", "localhost:8888", "filer hostname:port") |
|||
metaBackup.filerDirectory = cmdFilerMetaBackup.Flag.String("filerDir", "/", "a folder on the filer") |
|||
metaBackup.restart = cmdFilerMetaBackup.Flag.Bool("restart", false, "copy the full metadata before async incremental backup") |
|||
metaBackup.backupFilerConfig = cmdFilerMetaBackup.Flag.String("config", "", "path to filer.toml specifying backup filer store") |
|||
} |
|||
|
|||
var cmdFilerMetaBackup = &Command{ |
|||
UsageLine: "filer.meta.backup [-filer=localhost:8888] [-filerDir=/] [-restart] -config=/path/to/backup_filer.toml", |
|||
Short: "continuously backup filer meta data changes to anther filer store specified in a backup_filer.toml", |
|||
Long: `continuously backup filer meta data changes. |
|||
The backup writes to another filer store specified in a backup_filer.toml. |
|||
|
|||
weed filer.meta.backup -config=/path/to/backup_filer.toml -filer="localhost:8888" |
|||
weed filer.meta.backup -config=/path/to/backup_filer.toml -filer="localhost:8888" -restart |
|||
|
|||
`, |
|||
} |
|||
|
|||
func runFilerMetaBackup(cmd *Command, args []string) bool { |
|||
|
|||
metaBackup.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") |
|||
|
|||
// load backup_filer.toml
|
|||
v := viper.New() |
|||
v.SetConfigFile(*metaBackup.backupFilerConfig) |
|||
|
|||
if err := v.ReadInConfig(); err != nil { // Handle errors reading the config file
|
|||
glog.Fatalf("Failed to load %s file.\nPlease use this command to generate the a %s.toml file\n"+ |
|||
" weed scaffold -config=%s -output=.\n\n\n", |
|||
*metaBackup.backupFilerConfig, "backup_filer", "filer") |
|||
} |
|||
|
|||
if err := metaBackup.initStore(v); err != nil { |
|||
glog.V(0).Infof("init backup filer store: %v", err) |
|||
return true |
|||
} |
|||
|
|||
missingPreviousBackup := false |
|||
_, err := metaBackup.getOffset() |
|||
if err != nil { |
|||
missingPreviousBackup = true |
|||
} |
|||
|
|||
if *metaBackup.restart || missingPreviousBackup { |
|||
glog.V(0).Infof("traversing metadata tree...") |
|||
startTime := time.Now() |
|||
if err := metaBackup.traverseMetadata(); err != nil { |
|||
glog.Errorf("traverse meta data: %v", err) |
|||
return true |
|||
} |
|||
glog.V(0).Infof("metadata copied up to %v", startTime) |
|||
if err := metaBackup.setOffset(startTime); err != nil { |
|||
startTime = time.Now() |
|||
} |
|||
} |
|||
|
|||
for { |
|||
err := metaBackup.streamMetadataBackup() |
|||
if err != nil { |
|||
glog.Errorf("filer meta backup from %s: %v", *metaBackup.filerAddress, err) |
|||
time.Sleep(1747 * time.Millisecond) |
|||
} |
|||
} |
|||
|
|||
return true |
|||
} |
|||
|
|||
func (metaBackup *FilerMetaBackupOptions) initStore(v *viper.Viper) error { |
|||
// load configuration for default filer store
|
|||
hasDefaultStoreConfigured := false |
|||
for _, store := range filer.Stores { |
|||
if v.GetBool(store.GetName() + ".enabled") { |
|||
store = reflect.New(reflect.ValueOf(store).Elem().Type()).Interface().(filer.FilerStore) |
|||
if err := store.Initialize(v, store.GetName()+"."); err != nil { |
|||
glog.Fatalf("failed to initialize store for %s: %+v", store.GetName(), err) |
|||
} |
|||
glog.V(0).Infof("configured filer store to %s", store.GetName()) |
|||
hasDefaultStoreConfigured = true |
|||
metaBackup.store = filer.NewFilerStoreWrapper(store) |
|||
break |
|||
} |
|||
} |
|||
if !hasDefaultStoreConfigured { |
|||
return fmt.Errorf("no filer store enabled in %s", v.ConfigFileUsed()) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (metaBackup *FilerMetaBackupOptions) traverseMetadata() (err error) { |
|||
var saveErr error |
|||
|
|||
traverseErr := filer_pb.TraverseBfs(metaBackup, util.FullPath(*metaBackup.filerDirectory), func(parentPath util.FullPath, entry *filer_pb.Entry) { |
|||
|
|||
println("+", parentPath.Child(entry.Name)) |
|||
if err := metaBackup.store.InsertEntry(context.Background(), filer.FromPbEntry(string(parentPath), entry)); err != nil { |
|||
saveErr = fmt.Errorf("insert entry error: %v\n", err) |
|||
return |
|||
} |
|||
|
|||
}) |
|||
|
|||
if traverseErr != nil { |
|||
return fmt.Errorf("traverse: %v", traverseErr) |
|||
} |
|||
return saveErr |
|||
} |
|||
|
|||
var ( |
|||
MetaBackupKey = []byte("metaBackup") |
|||
) |
|||
|
|||
func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error { |
|||
|
|||
startTime, err := metaBackup.getOffset() |
|||
if err != nil { |
|||
startTime = time.Now() |
|||
} |
|||
glog.V(0).Infof("streaming from %v", startTime) |
|||
|
|||
store := metaBackup.store |
|||
|
|||
eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error { |
|||
|
|||
ctx := context.Background() |
|||
message := resp.EventNotification |
|||
|
|||
if message.OldEntry == nil && message.NewEntry == nil { |
|||
return nil |
|||
} |
|||
if message.OldEntry == nil && message.NewEntry != nil { |
|||
println("+", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name)) |
|||
entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry) |
|||
return store.InsertEntry(ctx, entry) |
|||
} |
|||
if message.OldEntry != nil && message.NewEntry == nil { |
|||
println("-", util.FullPath(resp.Directory).Child(message.OldEntry.Name)) |
|||
return store.DeleteEntry(ctx, util.FullPath(resp.Directory).Child(message.OldEntry.Name)) |
|||
} |
|||
if message.OldEntry != nil && message.NewEntry != nil { |
|||
if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name { |
|||
println("~", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name)) |
|||
entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry) |
|||
return store.UpdateEntry(ctx, entry) |
|||
} |
|||
println("-", util.FullPath(resp.Directory).Child(message.OldEntry.Name)) |
|||
if err := store.DeleteEntry(ctx, util.FullPath(resp.Directory).Child(message.OldEntry.Name)); err != nil { |
|||
return err |
|||
} |
|||
println("+", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name)) |
|||
return store.InsertEntry(ctx, filer.FromPbEntry(message.NewParentPath, message.NewEntry)) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
tailErr := pb.WithFilerClient(*metaBackup.filerAddress, metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { |
|||
|
|||
ctx, cancel := context.WithCancel(context.Background()) |
|||
defer cancel() |
|||
|
|||
stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{ |
|||
ClientName: "meta_backup", |
|||
PathPrefix: *metaBackup.filerDirectory, |
|||
SinceNs: startTime.UnixNano(), |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("listen: %v", err) |
|||
} |
|||
|
|||
var counter int64 |
|||
var lastWriteTime time.Time |
|||
for { |
|||
resp, listenErr := stream.Recv() |
|||
if listenErr == io.EOF { |
|||
return nil |
|||
} |
|||
if listenErr != nil { |
|||
return listenErr |
|||
} |
|||
if err = eachEntryFunc(resp); err != nil { |
|||
return err |
|||
} |
|||
|
|||
counter++ |
|||
if lastWriteTime.Add(3 * time.Second).Before(time.Now()) { |
|||
glog.V(0).Infof("meta backup %s progressed to %v %0.2f/sec", *metaBackup.filerAddress, time.Unix(0, resp.TsNs), float64(counter)/float64(3)) |
|||
counter = 0 |
|||
lastWriteTime = time.Now() |
|||
if err2 := metaBackup.setOffset(lastWriteTime); err2 != nil { |
|||
return err2 |
|||
} |
|||
} |
|||
|
|||
} |
|||
|
|||
}) |
|||
return tailErr |
|||
} |
|||
|
|||
func (metaBackup *FilerMetaBackupOptions) getOffset() (lastWriteTime time.Time, err error) { |
|||
value, err := metaBackup.store.KvGet(context.Background(), MetaBackupKey) |
|||
if err != nil { |
|||
return |
|||
} |
|||
tsNs := util.BytesToUint64(value) |
|||
|
|||
return time.Unix(0, int64(tsNs)), nil |
|||
} |
|||
|
|||
func (metaBackup *FilerMetaBackupOptions) setOffset(lastWriteTime time.Time) error { |
|||
valueBuf := make([]byte, 8) |
|||
util.Uint64toBytes(valueBuf, uint64(lastWriteTime.UnixNano())) |
|||
|
|||
if err := metaBackup.store.KvPut(context.Background(), MetaBackupKey, valueBuf); err != nil { |
|||
return err |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
var _ = filer_pb.FilerClient(&FilerMetaBackupOptions{}) |
|||
|
|||
func (metaBackup *FilerMetaBackupOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error { |
|||
|
|||
return pb.WithFilerClient(*metaBackup.filerAddress, metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { |
|||
return fn(client) |
|||
}) |
|||
|
|||
} |
|||
|
|||
func (metaBackup *FilerMetaBackupOptions) AdjustedUrl(location *filer_pb.Location) string { |
|||
return location.Url |
|||
} |
|||
@ -0,0 +1,137 @@ |
|||
package weed_server |
|||
|
|||
import ( |
|||
"bufio" |
|||
"fmt" |
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/storage/needle" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
"io" |
|||
"net" |
|||
"strings" |
|||
) |
|||
|
|||
func (vs *VolumeServer) HandleTcpConnection(c net.Conn) { |
|||
defer c.Close() |
|||
|
|||
glog.V(0).Infof("Serving writes from %s", c.RemoteAddr().String()) |
|||
|
|||
bufReader := bufio.NewReaderSize(c, 1024*1024) |
|||
bufWriter := bufio.NewWriterSize(c, 1024*1024) |
|||
|
|||
for { |
|||
cmd, err := bufReader.ReadString('\n') |
|||
if err != nil { |
|||
if err != io.EOF { |
|||
glog.Errorf("read command from %s: %v", c.RemoteAddr().String(), err) |
|||
} |
|||
return |
|||
} |
|||
cmd = cmd[:len(cmd)-1] |
|||
switch cmd[0] { |
|||
case '+': |
|||
fileId := cmd[1:] |
|||
err = vs.handleTcpPut(fileId, bufReader) |
|||
if err == nil { |
|||
bufWriter.Write([]byte("+OK\n")) |
|||
} else { |
|||
bufWriter.Write([]byte("-ERR " + string(err.Error()) + "\n")) |
|||
} |
|||
case '-': |
|||
fileId := cmd[1:] |
|||
err = vs.handleTcpDelete(fileId) |
|||
if err == nil { |
|||
bufWriter.Write([]byte("+OK\n")) |
|||
} else { |
|||
bufWriter.Write([]byte("-ERR " + string(err.Error()) + "\n")) |
|||
} |
|||
case '?': |
|||
fileId := cmd[1:] |
|||
err = vs.handleTcpGet(fileId, bufWriter) |
|||
case '!': |
|||
bufWriter.Flush() |
|||
} |
|||
|
|||
} |
|||
|
|||
} |
|||
|
|||
func (vs *VolumeServer) handleTcpGet(fileId string, writer *bufio.Writer) (err error) { |
|||
|
|||
volumeId, n, err2 := vs.parseFileId(fileId) |
|||
if err2 != nil { |
|||
return err2 |
|||
} |
|||
|
|||
volume := vs.store.GetVolume(volumeId) |
|||
if volume == nil { |
|||
return fmt.Errorf("volume %d not found", volumeId) |
|||
} |
|||
|
|||
err = volume.StreamRead(n, writer) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (vs *VolumeServer) handleTcpPut(fileId string, bufReader *bufio.Reader) (err error) { |
|||
|
|||
volumeId, n, err2 := vs.parseFileId(fileId) |
|||
if err2 != nil { |
|||
return err2 |
|||
} |
|||
|
|||
volume := vs.store.GetVolume(volumeId) |
|||
if volume == nil { |
|||
return fmt.Errorf("volume %d not found", volumeId) |
|||
} |
|||
|
|||
sizeBuf := make([]byte, 4) |
|||
if _, err = bufReader.Read(sizeBuf); err != nil { |
|||
return err |
|||
} |
|||
dataSize := util.BytesToUint32(sizeBuf) |
|||
|
|||
err = volume.StreamWrite(n, bufReader, dataSize) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (vs *VolumeServer) handleTcpDelete(fileId string) (err error) { |
|||
|
|||
volumeId, n, err2 := vs.parseFileId(fileId) |
|||
if err2 != nil { |
|||
return err2 |
|||
} |
|||
|
|||
_, err = vs.store.DeleteVolumeNeedle(volumeId, n) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (vs *VolumeServer) parseFileId(fileId string) (needle.VolumeId, *needle.Needle, error) { |
|||
|
|||
commaIndex := strings.LastIndex(fileId, ",") |
|||
if commaIndex <= 0 { |
|||
return 0, nil, fmt.Errorf("unknown fileId %s", fileId) |
|||
} |
|||
|
|||
vid, fid := fileId[0:commaIndex], fileId[commaIndex+1:] |
|||
|
|||
volumeId, ve := needle.NewVolumeId(vid) |
|||
if ve != nil { |
|||
return 0, nil, fmt.Errorf("unknown volume id in fileId %s", fileId) |
|||
} |
|||
|
|||
n := new(needle.Needle) |
|||
n.ParsePath(fid) |
|||
return volumeId, n, nil |
|||
} |
|||
@ -0,0 +1,92 @@ |
|||
package shell |
|||
|
|||
import ( |
|||
"flag" |
|||
"fmt" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
"io" |
|||
"math" |
|||
"time" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
) |
|||
|
|||
func init() { |
|||
Commands = append(Commands, &commandS3CleanUploads{}) |
|||
} |
|||
|
|||
type commandS3CleanUploads struct { |
|||
} |
|||
|
|||
func (c *commandS3CleanUploads) Name() string { |
|||
return "s3.clean.uploads" |
|||
} |
|||
|
|||
func (c *commandS3CleanUploads) Help() string { |
|||
return `clean up stale multipart uploads |
|||
|
|||
Example: |
|||
s3.clean.uploads -replication 001 |
|||
|
|||
` |
|||
} |
|||
|
|||
func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { |
|||
|
|||
bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) |
|||
uploadedTimeAgo := bucketCommand.Duration("timeAgo", 24*time.Hour, "created time before now. \"1.5h\" or \"2h45m\". Valid time units are \"m\", \"h\"") |
|||
if err = bucketCommand.Parse(args); err != nil { |
|||
return nil |
|||
} |
|||
|
|||
var filerBucketsPath string |
|||
filerBucketsPath, err = readFilerBucketsPath(commandEnv) |
|||
if err != nil { |
|||
return fmt.Errorf("read buckets: %v", err) |
|||
} |
|||
|
|||
var buckets []string |
|||
err = filer_pb.List(commandEnv, filerBucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error { |
|||
buckets = append(buckets, entry.Name) |
|||
return nil |
|||
}, "", false, math.MaxUint32) |
|||
if err != nil { |
|||
return fmt.Errorf("list buckets under %v: %v", filerBucketsPath, err) |
|||
} |
|||
|
|||
for _, bucket:= range buckets { |
|||
c.cleanupUploads(commandEnv, writer, filerBucketsPath, bucket, *uploadedTimeAgo) |
|||
} |
|||
|
|||
return err |
|||
|
|||
} |
|||
|
|||
func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io.Writer, filerBucketsPath string, bucket string, timeAgo time.Duration) error { |
|||
uploadsDir := filerBucketsPath+"/"+bucket+"/.uploads" |
|||
var staleUploads []string |
|||
now := time.Now() |
|||
err := filer_pb.List(commandEnv, uploadsDir, "", func(entry *filer_pb.Entry, isLast bool) error { |
|||
ctime := time.Unix(entry.Attributes.Crtime, 0) |
|||
if ctime.Add(timeAgo).Before(now) { |
|||
staleUploads = append(staleUploads, entry.Name) |
|||
} |
|||
return nil |
|||
}, "", false, math.MaxUint32) |
|||
if err != nil { |
|||
return fmt.Errorf("list uploads under %v: %v", uploadsDir, err) |
|||
} |
|||
|
|||
for _, staleUpload:= range staleUploads { |
|||
deleteUrl := fmt.Sprintf("http://%s:%d%s/%s?recursive=true&ignoreRecursiveError=true",commandEnv.option.FilerHost, commandEnv.option.FilerPort,uploadsDir, staleUpload) |
|||
fmt.Fprintf(writer, "purge %s\n", deleteUrl) |
|||
|
|||
err = util.Delete(deleteUrl, "") |
|||
if err != nil { |
|||
return fmt.Errorf("purge %s/%s: %v", uploadsDir, staleUpload, err) |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
|
|||
} |
|||
@ -0,0 +1,104 @@ |
|||
package storage |
|||
|
|||
import ( |
|||
"bufio" |
|||
"fmt" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
"io" |
|||
"time" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/storage/backend" |
|||
"github.com/chrislusf/seaweedfs/weed/storage/needle" |
|||
. "github.com/chrislusf/seaweedfs/weed/storage/types" |
|||
) |
|||
|
|||
func (v *Volume) StreamWrite(n *needle.Needle, data io.Reader, dataSize uint32) (err error) { |
|||
|
|||
v.dataFileAccessLock.Lock() |
|||
defer v.dataFileAccessLock.Unlock() |
|||
|
|||
df, ok := v.DataBackend.(*backend.DiskFile) |
|||
if !ok { |
|||
return fmt.Errorf("unexpected volume backend") |
|||
} |
|||
offset, _, _ := v.DataBackend.GetStat() |
|||
|
|||
header := make([]byte, NeedleHeaderSize+TimestampSize) // adding timestamp to reuse it and avoid extra allocation
|
|||
CookieToBytes(header[0:CookieSize], n.Cookie) |
|||
NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id) |
|||
n.Size = 4 + Size(dataSize) + 1 |
|||
SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size) |
|||
|
|||
n.DataSize = dataSize |
|||
|
|||
// needle header
|
|||
df.Write(header[0:NeedleHeaderSize]) |
|||
|
|||
// data size and data
|
|||
util.Uint32toBytes(header[0:4], n.DataSize) |
|||
df.Write(header[0:4]) |
|||
// write and calculate CRC
|
|||
crcWriter := needle.NewCRCwriter(df) |
|||
io.Copy(crcWriter, io.LimitReader(data, int64(dataSize))) |
|||
|
|||
// flags
|
|||
util.Uint8toBytes(header[0:1], n.Flags) |
|||
df.Write(header[0:1]) |
|||
|
|||
// data checksum
|
|||
util.Uint32toBytes(header[0:needle.NeedleChecksumSize], crcWriter.Sum()) |
|||
// write timestamp, padding
|
|||
n.AppendAtNs = uint64(time.Now().UnixNano()) |
|||
util.Uint64toBytes(header[needle.NeedleChecksumSize:needle.NeedleChecksumSize+TimestampSize], n.AppendAtNs) |
|||
padding := needle.PaddingLength(n.Size, needle.Version3) |
|||
df.Write(header[0 : needle.NeedleChecksumSize+TimestampSize+padding]) |
|||
|
|||
// add to needle map
|
|||
if err = v.nm.Put(n.Id, ToOffset(int64(offset)), n.Size); err != nil { |
|||
glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err) |
|||
} |
|||
return |
|||
} |
|||
|
|||
func (v *Volume) StreamRead(n *needle.Needle, writer io.Writer) (err error) { |
|||
|
|||
v.dataFileAccessLock.Lock() |
|||
defer v.dataFileAccessLock.Unlock() |
|||
|
|||
nv, ok := v.nm.Get(n.Id) |
|||
if !ok || nv.Offset.IsZero() { |
|||
return ErrorNotFound |
|||
} |
|||
|
|||
sr := &StreamReader{ |
|||
readerAt: v.DataBackend, |
|||
offset: nv.Offset.ToActualOffset(), |
|||
} |
|||
bufReader := bufio.NewReader(sr) |
|||
bufReader.Discard(NeedleHeaderSize) |
|||
sizeBuf := make([]byte, 4) |
|||
bufReader.Read(sizeBuf) |
|||
if _, err = writer.Write(sizeBuf); err != nil { |
|||
return err |
|||
} |
|||
dataSize := util.BytesToUint32(sizeBuf) |
|||
|
|||
_, err = io.Copy(writer, io.LimitReader(bufReader, int64(dataSize))) |
|||
|
|||
return |
|||
} |
|||
|
|||
type StreamReader struct { |
|||
offset int64 |
|||
readerAt io.ReaderAt |
|||
} |
|||
|
|||
func (sr *StreamReader) Read(p []byte) (n int, err error) { |
|||
n, err = sr.readerAt.ReadAt(p, sr.offset) |
|||
if err != nil { |
|||
return |
|||
} |
|||
sr.offset += int64(n) |
|||
return |
|||
} |
|||
Some files were not shown because too many files changed in this diff
Write
Preview
Loading…
Cancel
Save
Reference in new issue