|
@ -1,12 +1,14 @@ |
|
|
package rclone_backend |
|
|
package rclone_backend |
|
|
|
|
|
|
|
|
import ( |
|
|
import ( |
|
|
|
|
|
"bytes" |
|
|
"context" |
|
|
"context" |
|
|
"fmt" |
|
|
"fmt" |
|
|
"github.com/rclone/rclone/fs/config/configfile" |
|
|
"github.com/rclone/rclone/fs/config/configfile" |
|
|
"github.com/seaweedfs/seaweedfs/weed/util" |
|
|
"github.com/seaweedfs/seaweedfs/weed/util" |
|
|
"io" |
|
|
"io" |
|
|
"os" |
|
|
"os" |
|
|
|
|
|
"text/template" |
|
|
"time" |
|
|
"time" |
|
|
|
|
|
|
|
|
"github.com/google/uuid" |
|
|
"github.com/google/uuid" |
|
@ -40,6 +42,8 @@ func (factory *RcloneBackendFactory) BuildStorage(configuration backend.StringPr |
|
|
type RcloneBackendStorage struct { |
|
|
type RcloneBackendStorage struct { |
|
|
id string |
|
|
id string |
|
|
remoteName string |
|
|
remoteName string |
|
|
|
|
|
keyTemplate *template.Template |
|
|
|
|
|
keyTemplateText string |
|
|
fs fs.Fs |
|
|
fs fs.Fs |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
@ -47,6 +51,11 @@ func newRcloneBackendStorage(configuration backend.StringProperties, configPrefi |
|
|
s = &RcloneBackendStorage{} |
|
|
s = &RcloneBackendStorage{} |
|
|
s.id = id |
|
|
s.id = id |
|
|
s.remoteName = configuration.GetString(configPrefix + "remote_name") |
|
|
s.remoteName = configuration.GetString(configPrefix + "remote_name") |
|
|
|
|
|
s.keyTemplateText = configuration.GetString(configPrefix + "key_template") |
|
|
|
|
|
s.keyTemplate, err = template.New("keyTemplate").Parse(s.keyTemplateText) |
|
|
|
|
|
if err != nil { |
|
|
|
|
|
return |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
ctx := context.TODO() |
|
|
ctx := context.TODO() |
|
|
accounting.Start(ctx) |
|
|
accounting.Start(ctx) |
|
@ -65,9 +74,25 @@ func newRcloneBackendStorage(configuration backend.StringProperties, configPrefi |
|
|
func (s *RcloneBackendStorage) ToProperties() map[string]string { |
|
|
func (s *RcloneBackendStorage) ToProperties() map[string]string { |
|
|
m := make(map[string]string) |
|
|
m := make(map[string]string) |
|
|
m["remote_name"] = s.remoteName |
|
|
m["remote_name"] = s.remoteName |
|
|
|
|
|
if len(s.keyTemplateText) > 0 { |
|
|
|
|
|
m["key_template"] = s.keyTemplateText |
|
|
|
|
|
} |
|
|
return m |
|
|
return m |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
func formatKey(key string, storage RcloneBackendStorage) (fKey string, err error) { |
|
|
|
|
|
var b bytes.Buffer |
|
|
|
|
|
if len(storage.keyTemplateText) == 0 { |
|
|
|
|
|
fKey = key |
|
|
|
|
|
} else { |
|
|
|
|
|
err = storage.keyTemplate.Execute(&b, key) |
|
|
|
|
|
if err == nil { |
|
|
|
|
|
fKey = b.String() |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
return |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
func (s *RcloneBackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) backend.BackendStorageFile { |
|
|
func (s *RcloneBackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) backend.BackendStorageFile { |
|
|
f := &RcloneBackendStorageFile{ |
|
|
f := &RcloneBackendStorageFile{ |
|
|
backendStorage: s, |
|
|
backendStorage: s, |
|
@ -85,6 +110,11 @@ func (s *RcloneBackendStorage) CopyFile(f *os.File, fn func(progressed int64, pe |
|
|
} |
|
|
} |
|
|
key = randomUuid.String() |
|
|
key = randomUuid.String() |
|
|
|
|
|
|
|
|
|
|
|
key, err = formatKey(key, *s) |
|
|
|
|
|
if err != nil { |
|
|
|
|
|
return key, 0, err |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
glog.V(1).Infof("copy dat file of %s to remote rclone.%s as %s", f.Name(), s.id, key) |
|
|
glog.V(1).Infof("copy dat file of %s to remote rclone.%s as %s", f.Name(), s.id, key) |
|
|
|
|
|
|
|
|
util.Retry("upload via Rclone", func() error { |
|
|
util.Retry("upload via Rclone", func() error { |
|
|