Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

filer: option to encrypt data on volume server

tikv
Chris Lu 5 years ago
parent
31c481e3fc
commit
13e215ee5c
34 changed files with 419 additions and 247 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      other/java/client/src/main/proto/filer.proto
  2. 2
      unmaintained/repeated_vacuum/repeated_vacuum.go
  3. 3
      weed/command/filer.go
  4. 16
      weed/command/filer_copy.go
  5. 5
      weed/command/mount_std.go
  6. 4
      weed/command/scaffold.go
  7. 1
      weed/command/server.go
  8. 40
      weed/command/webdav.go
  9. 30
      weed/filer2/filechunks.go
  10. 1
      weed/filer2/filer.go
  11. 7
      weed/filer2/filer_client_util.go
  12. 2
      weed/filer2/stream.go
  13. 3
      weed/filesys/dirty_page.go
  14. 5
      weed/filesys/wfs.go
  15. 7
      weed/operation/submit.go
  16. 39
      weed/operation/upload_content.go
  17. 2
      weed/pb/filer.proto
  18. 235
      weed/pb/filer_pb/filer.pb.go
  19. 8
      weed/pb/master_pb/master.pb.go
  20. 12
      weed/pb/volume_server_pb/volume_server.pb.go
  21. 2
      weed/replication/sink/azuresink/azure_sink.go
  22. 2
      weed/replication/sink/b2sink/b2_sink.go
  23. 5
      weed/replication/sink/filersink/fetch_write.go
  24. 5
      weed/replication/sink/gcssink/gcs_sink.go
  25. 2
      weed/replication/sink/s3sink/s3_write.go
  26. 2
      weed/server/common.go
  27. 1
      weed/server/filer_grpc_server.go
  28. 2
      weed/server/filer_server.go
  29. 23
      weed/server/filer_server_handlers_read.go
  30. 2
      weed/server/filer_server_handlers_write_autochunk.go
  31. 4
      weed/server/webdav_server.go
  32. 5
      weed/topology/store_replicate.go
  33. 60
      weed/util/cipher.go
  34. 49
      weed/util/http_util.go

2
other/java/client/src/main/proto/filer.proto
View File

@ -99,6 +99,7 @@ message FileChunk {
string source_file_id = 6; // to be deprecated string source_file_id = 6; // to be deprecated
FileId fid = 7; FileId fid = 7;
FileId source_fid = 8; FileId source_fid = 8;
bytes cipher_key = 9;
} }
message FileId { message FileId {
@ -229,4 +230,5 @@ message GetFilerConfigurationResponse {
uint32 max_mb = 4; uint32 max_mb = 4;
string dir_buckets = 5; string dir_buckets = 5;
string dir_queues = 6; string dir_queues = 6;
bool cipher = 7;
} }

2
unmaintained/repeated_vacuum/repeated_vacuum.go
View File

@ -35,7 +35,7 @@ func main() {
targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid) targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
_, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), reader, false, "", nil, assignResult.Auth)
_, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), false, reader, false, "", nil, assignResult.Auth)
if err != nil { if err != nil {
log.Fatalf("upload: %v", err) log.Fatalf("upload: %v", err)
} }

3
weed/command/filer.go
View File

@ -34,6 +34,7 @@ type FilerOptions struct {
dataCenter *string dataCenter *string
enableNotification *bool enableNotification *bool
disableHttp *bool disableHttp *bool
cipher *bool
// default leveldb directory, used in "weed server" mode // default leveldb directory, used in "weed server" mode
defaultLevelDbDirectory *string defaultLevelDbDirectory *string
@ -53,6 +54,7 @@ func init() {
f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size") f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center") f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center")
f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed") f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers")
} }
var cmdFiler = &Command{ var cmdFiler = &Command{
@ -111,6 +113,7 @@ func (fo *FilerOptions) startFiler() {
DefaultLevelDbDir: defaultLevelDbDirectory, DefaultLevelDbDir: defaultLevelDbDirectory,
DisableHttp: *fo.disableHttp, DisableHttp: *fo.disableHttp,
Port: uint32(*fo.port), Port: uint32(*fo.port),
Cipher: *fo.cipher,
}) })
if nfs_err != nil { if nfs_err != nil {
glog.Fatalf("Filer startup error: %v", nfs_err) glog.Fatalf("Filer startup error: %v", nfs_err)

16
weed/command/filer_copy.go
View File

@ -41,6 +41,7 @@ type CopyOptions struct {
compressionLevel *int compressionLevel *int
grpcDialOption grpc.DialOption grpcDialOption grpc.DialOption
masters []string masters []string
cipher bool
} }
func init() { func init() {
@ -108,7 +109,7 @@ func runCopy(cmd *Command, args []string) bool {
filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort) filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort)
copy.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") copy.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
masters, collection, replication, maxMB, err := readFilerConfiguration(copy.grpcDialOption, filerGrpcAddress)
masters, collection, replication, maxMB, cipher, err := readFilerConfiguration(copy.grpcDialOption, filerGrpcAddress)
if err != nil { if err != nil {
fmt.Printf("read from filer %s: %v\n", filerGrpcAddress, err) fmt.Printf("read from filer %s: %v\n", filerGrpcAddress, err)
return false return false
@ -123,6 +124,7 @@ func runCopy(cmd *Command, args []string) bool {
*copy.maxMB = int(maxMB) *copy.maxMB = int(maxMB)
} }
copy.masters = masters copy.masters = masters
copy.cipher = cipher
if *cmdCopy.IsDebug { if *cmdCopy.IsDebug {
util.SetupProfiling("filer.copy.cpu.pprof", "filer.copy.mem.pprof") util.SetupProfiling("filer.copy.cpu.pprof", "filer.copy.mem.pprof")
@ -159,13 +161,14 @@ func runCopy(cmd *Command, args []string) bool {
return true return true
} }
func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress string) (masters []string, collection, replication string, maxMB uint32, err error) {
func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress string) (masters []string, collection, replication string, maxMB uint32, cipher bool, err error) {
err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
if err != nil { if err != nil {
return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err) return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
} }
masters, collection, replication, maxMB = resp.Masters, resp.Collection, resp.Replication, resp.MaxMb masters, collection, replication, maxMB = resp.Masters, resp.Collection, resp.Replication, resp.MaxMb
cipher = resp.Cipher
return nil return nil
}) })
return return
@ -300,7 +303,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId
uploadResult, err := operation.UploadWithLocalCompressionLevel(targetUrl, fileName, f, false, mimeType, nil, security.EncodedJwt(assignResult.Auth), *worker.options.compressionLevel)
uploadResult, err := operation.UploadWithLocalCompressionLevel(targetUrl, fileName, worker.options.cipher, f, false, mimeType, nil, security.EncodedJwt(assignResult.Auth), *worker.options.compressionLevel)
if err != nil { if err != nil {
return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err) return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
} }
@ -315,6 +318,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
Size: uint64(uploadResult.Size), Size: uint64(uploadResult.Size),
Mtime: time.Now().UnixNano(), Mtime: time.Now().UnixNano(),
ETag: uploadResult.ETag, ETag: uploadResult.ETag,
CipherKey: uploadResult.CipherKey,
}) })
fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName) fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName)
@ -409,10 +413,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
replication = assignResult.Replication replication = assignResult.Replication
} }
uploadResult, err := operation.Upload(targetUrl,
fileName+"-"+strconv.FormatInt(i+1, 10),
io.NewSectionReader(f, i*chunkSize, chunkSize),
false, "", nil, security.EncodedJwt(assignResult.Auth))
uploadResult, err := operation.Upload(targetUrl, fileName+"-"+strconv.FormatInt(i+1, 10), false, io.NewSectionReader(f, i*chunkSize, chunkSize), false, "", nil, security.EncodedJwt(assignResult.Auth))
if err != nil { if err != nil {
uploadError = fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err) uploadError = fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
return return
@ -427,6 +428,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
Size: uint64(uploadResult.Size), Size: uint64(uploadResult.Size),
Mtime: time.Now().UnixNano(), Mtime: time.Now().UnixNano(),
ETag: uploadResult.ETag, ETag: uploadResult.ETag,
CipherKey: uploadResult.CipherKey,
} }
fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size)) fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
}(i) }(i)

5
weed/command/mount_std.go
View File

@ -145,11 +145,13 @@ func RunMount(filer, filerMountRootPath, dir, collection, replication, dataCente
// try to connect to filer, filerBucketsPath may be useful later // try to connect to filer, filerBucketsPath may be useful later
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
var cipher bool
err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
_, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
if err != nil { if err != nil {
return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err) return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
} }
cipher = resp.Cipher
return nil return nil
}) })
if err != nil { if err != nil {
@ -183,6 +185,7 @@ func RunMount(filer, filerMountRootPath, dir, collection, replication, dataCente
MountMtime: time.Now(), MountMtime: time.Now(),
Umask: umask, Umask: umask,
OutsideContainerClusterMode: outsideContainerClusterMode, OutsideContainerClusterMode: outsideContainerClusterMode,
Cipher: cipher,
})) }))
if err != nil { if err != nil {
fuse.Unmount(dir) fuse.Unmount(dir)

4
weed/command/scaffold.go
View File

@ -75,9 +75,9 @@ const (
# recursive_delete will delete all sub folders and files, similar to "rm -Rf" # recursive_delete will delete all sub folders and files, similar to "rm -Rf"
recursive_delete = false recursive_delete = false
# directories under this folder will be automatically creating a separate bucket # directories under this folder will be automatically creating a separate bucket
buckets_folder = /buckets
buckets_folder = "/buckets"
# directories under this folder will be store message queue data # directories under this folder will be store message queue data
queues_folder = /queues
queues_folder = "/queues"
#################################################### ####################################################
# The following are filer store options # The following are filer store options

1
weed/command/server.go
View File

@ -82,6 +82,7 @@ func init() {
filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit") filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size") filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")
serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port") serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")

40
weed/command/webdav.go
View File

@ -1,6 +1,7 @@
package command package command
import ( import (
"context"
"fmt" "fmt"
"net/http" "net/http"
"os/user" "os/user"
@ -9,6 +10,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/server" "github.com/chrislusf/seaweedfs/weed/server"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
@ -55,12 +57,6 @@ func runWebDav(cmd *Command, args []string) bool {
func (wo *WebDavOption) startWebDav() bool { func (wo *WebDavOption) startWebDav() bool {
filerGrpcAddress, err := pb.ParseFilerGrpcAddress(*wo.filer)
if err != nil {
glog.Fatal(err)
return false
}
// detect current user // detect current user
uid, gid := uint32(0), uint32(0) uid, gid := uint32(0), uint32(0)
if u, err := user.Current(); err == nil { if u, err := user.Current(); err == nil {
@ -72,13 +68,43 @@ func (wo *WebDavOption) startWebDav() bool {
} }
} }
// parse filer grpc address
filerGrpcAddress, err := pb.ParseFilerGrpcAddress(*wo.filer)
if err != nil {
glog.Fatal(err)
return false
}
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
var cipher bool
// connect to filer
for {
err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
if err != nil {
return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
}
cipher = resp.Cipher
return nil
})
if err != nil {
glog.V(0).Infof("wait to connect to filer %s grpc address %s", *wo.filer, filerGrpcAddress)
time.Sleep(time.Second)
} else {
glog.V(0).Infof("connected to filer %s grpc address %s", *wo.filer, filerGrpcAddress)
break
}
}
ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{ ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{
Filer: *wo.filer, Filer: *wo.filer,
FilerGrpcAddress: filerGrpcAddress, FilerGrpcAddress: filerGrpcAddress,
GrpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.client"),
GrpcDialOption: grpcDialOption,
Collection: *wo.collection, Collection: *wo.collection,
Uid: uid, Uid: uid,
Gid: gid, Gid: gid,
Cipher: cipher,
}) })
if webdavServer_err != nil { if webdavServer_err != nil {
glog.Fatalf("WebDav Server startup error: %v", webdavServer_err) glog.Fatalf("WebDav Server startup error: %v", webdavServer_err)

30
weed/filer2/filechunks.go
View File

@ -71,6 +71,7 @@ type ChunkView struct {
Size uint64 Size uint64
LogicOffset int64 LogicOffset int64
IsFullChunk bool IsFullChunk bool
CipherKey []byte
} }
func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*ChunkView) { func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*ChunkView) {
@ -94,6 +95,7 @@ func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int
Size: uint64(min(chunk.stop, stop) - offset), Size: uint64(min(chunk.stop, stop) - offset),
LogicOffset: offset, LogicOffset: offset,
IsFullChunk: isFullChunk, IsFullChunk: isFullChunk,
CipherKey: chunk.cipherKey,
}) })
offset = min(chunk.stop, stop) offset = min(chunk.stop, stop)
} }
@ -120,13 +122,7 @@ var bufPool = sync.Pool{
func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.FileChunk) []VisibleInterval { func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.FileChunk) []VisibleInterval {
newV := newVisibleInterval(
chunk.Offset,
chunk.Offset+int64(chunk.Size),
chunk.GetFileIdString(),
chunk.Mtime,
true,
)
newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Mtime, true, chunk.CipherKey)
length := len(visibles) length := len(visibles)
if length == 0 { if length == 0 {
@ -140,23 +136,11 @@ func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.
logPrintf(" before", visibles) logPrintf(" before", visibles)
for _, v := range visibles { for _, v := range visibles {
if v.start < chunk.Offset && chunk.Offset < v.stop { if v.start < chunk.Offset && chunk.Offset < v.stop {
newVisibles = append(newVisibles, newVisibleInterval(
v.start,
chunk.Offset,
v.fileId,
v.modifiedTime,
false,
))
newVisibles = append(newVisibles, newVisibleInterval(v.start, chunk.Offset, v.fileId, v.modifiedTime, false, v.cipherKey))
} }
chunkStop := chunk.Offset + int64(chunk.Size) chunkStop := chunk.Offset + int64(chunk.Size)
if v.start < chunkStop && chunkStop < v.stop { if v.start < chunkStop && chunkStop < v.stop {
newVisibles = append(newVisibles, newVisibleInterval(
chunkStop,
v.stop,
v.fileId,
v.modifiedTime,
false,
))
newVisibles = append(newVisibles, newVisibleInterval(chunkStop, v.stop, v.fileId, v.modifiedTime, false, v.cipherKey))
} }
if chunkStop <= v.start || v.stop <= chunk.Offset { if chunkStop <= v.start || v.stop <= chunk.Offset {
newVisibles = append(newVisibles, v) newVisibles = append(newVisibles, v)
@ -208,15 +192,17 @@ type VisibleInterval struct {
modifiedTime int64 modifiedTime int64
fileId string fileId string
isFullChunk bool isFullChunk bool
cipherKey []byte
} }
func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool) VisibleInterval {
func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool, cipherKey []byte) VisibleInterval {
return VisibleInterval{ return VisibleInterval{
start: start, start: start,
stop: stop, stop: stop,
fileId: fileId, fileId: fileId,
modifiedTime: modifiedTime, modifiedTime: modifiedTime,
isFullChunk: isFullChunk, isFullChunk: isFullChunk,
cipherKey: cipherKey,
} }
} }

1
weed/filer2/filer.go
View File

@ -33,6 +33,7 @@ type Filer struct {
DirBucketsPath string DirBucketsPath string
DirQueuesPath string DirQueuesPath string
buckets *FilerBuckets buckets *FilerBuckets
Cipher bool
} }
func NewFiler(masters []string, grpcDialOption grpc.DialOption, filerGrpcPort uint32) *Filer { func NewFiler(masters []string, grpcDialOption grpc.DialOption, filerGrpcPort uint32) *Filer {

7
weed/filer2/filer_client_util.go
View File

@ -70,12 +70,7 @@ func ReadIntoBuffer(filerClient FilerClient, fullFilePath FullPath, buff []byte,
volumeServerAddress := filerClient.AdjustedUrl(locations.Locations[0].Url) volumeServerAddress := filerClient.AdjustedUrl(locations.Locations[0].Url)
var n int64 var n int64
n, err = util.ReadUrl(
fmt.Sprintf("http://%s/%s", volumeServerAddress, chunkView.FileId),
chunkView.Offset,
int(chunkView.Size),
buff[chunkView.LogicOffset-baseOffset:chunkView.LogicOffset-baseOffset+int64(chunkView.Size)],
!chunkView.IsFullChunk)
n, err = util.ReadUrl(fmt.Sprintf("http://%s/%s", volumeServerAddress, chunkView.FileId), chunkView.CipherKey, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), buff[chunkView.LogicOffset-baseOffset:chunkView.LogicOffset-baseOffset+int64(chunkView.Size)])
if err != nil { if err != nil {

2
weed/filer2/stream.go
View File

@ -27,7 +27,7 @@ func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*f
for _, chunkView := range chunkViews { for _, chunkView := range chunkViews {
urlString := fileId2Url[chunkView.FileId] urlString := fileId2Url[chunkView.FileId]
_, err := util.ReadUrlAsStream(urlString, chunkView.Offset, int(chunkView.Size), func(data []byte) {
err := util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), func(data []byte) {
w.Write(data) w.Write(data)
}) })
if err != nil { if err != nil {

3
weed/filesys/dirty_page.go
View File

@ -174,7 +174,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64,
} }
fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
uploadResult, err := operation.Upload(fileUrl, pages.f.Name, reader, false, "", nil, auth)
uploadResult, err := operation.Upload(fileUrl, pages.f.Name, pages.f.wfs.option.Cipher, reader, false, "", nil, auth)
if err != nil { if err != nil {
glog.V(0).Infof("upload data %v to %s: %v", pages.f.Name, fileUrl, err) glog.V(0).Infof("upload data %v to %s: %v", pages.f.Name, fileUrl, err)
return nil, fmt.Errorf("upload data: %v", err) return nil, fmt.Errorf("upload data: %v", err)
@ -190,6 +190,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64,
Size: uint64(size), Size: uint64(size),
Mtime: time.Now().UnixNano(), Mtime: time.Now().UnixNano(),
ETag: uploadResult.ETag, ETag: uploadResult.ETag,
CipherKey: uploadResult.CipherKey,
}, nil }, nil
} }

5
weed/filesys/wfs.go
View File

@ -39,8 +39,9 @@ type Option struct {
MountCtime time.Time MountCtime time.Time
MountMtime time.Time MountMtime time.Time
// whether the mount runs outside SeaweedFS containers
OutsideContainerClusterMode bool
OutsideContainerClusterMode bool // whether the mount runs outside SeaweedFS containers
Cipher bool // whether encrypt data on volume server
} }
var _ = fs.FS(&WFS{}) var _ = fs.FS(&WFS{})

7
weed/operation/submit.go
View File

@ -189,7 +189,7 @@ func (fi FilePart) Upload(maxMB int, master string, jwt security.EncodedJwt, grp
cm.DeleteChunks(master, grpcDialOption) cm.DeleteChunks(master, grpcDialOption)
} }
} else { } else {
ret, e := Upload(fileUrl, baseName, fi.Reader, false, fi.MimeType, nil, jwt)
ret, e := Upload(fileUrl, baseName, false, fi.Reader, false, fi.MimeType, nil, jwt)
if e != nil { if e != nil {
return 0, e return 0, e
} }
@ -202,8 +202,7 @@ func upload_one_chunk(filename string, reader io.Reader, master,
fileUrl string, jwt security.EncodedJwt, fileUrl string, jwt security.EncodedJwt,
) (size uint32, e error) { ) (size uint32, e error) {
glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...") glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...")
uploadResult, uploadError := Upload(fileUrl, filename, reader, false,
"", nil, jwt)
uploadResult, uploadError := Upload(fileUrl, filename, false, reader, false, "", nil, jwt)
if uploadError != nil { if uploadError != nil {
return 0, uploadError return 0, uploadError
} }
@ -221,6 +220,6 @@ func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt s
q := u.Query() q := u.Query()
q.Set("cm", "true") q.Set("cm", "true")
u.RawQuery = q.Encode() u.RawQuery = q.Encode()
_, e = Upload(u.String(), manifest.Name, bufReader, false, "application/json", nil, jwt)
_, e = Upload(u.String(), manifest.Name, false, bufReader, false, "application/json", nil, jwt)
return e return e
} }

39
weed/operation/upload_content.go
View File

@ -26,6 +26,7 @@ type UploadResult struct {
Size uint32 `json:"size,omitempty"` Size uint32 `json:"size,omitempty"`
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
ETag string `json:"eTag,omitempty"` ETag string `json:"eTag,omitempty"`
CipherKey []byte `json:"cipherKey,omitempty"`
} }
var ( var (
@ -41,22 +42,22 @@ func init() {
var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
// Upload sends a POST request to a volume server to upload the content with adjustable compression level // Upload sends a POST request to a volume server to upload the content with adjustable compression level
func UploadWithLocalCompressionLevel(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt, compressionLevel int) (*UploadResult, error) {
func UploadWithLocalCompressionLevel(uploadUrl string, filename string, cipher bool, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt, compressionLevel int) (*UploadResult, error) {
if compressionLevel < 1 { if compressionLevel < 1 {
compressionLevel = 1 compressionLevel = 1
} }
if compressionLevel > 9 { if compressionLevel > 9 {
compressionLevel = 9 compressionLevel = 9
} }
return doUpload(uploadUrl, filename, reader, isGzipped, mtype, pairMap, compressionLevel, jwt)
return doUpload(uploadUrl, filename, cipher, reader, isGzipped, mtype, pairMap, compressionLevel, jwt)
} }
// Upload sends a POST request to a volume server to upload the content with fast compression // Upload sends a POST request to a volume server to upload the content with fast compression
func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
return doUpload(uploadUrl, filename, reader, isGzipped, mtype, pairMap, flate.BestSpeed, jwt)
func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
return doUpload(uploadUrl, filename, cipher, reader, isGzipped, mtype, pairMap, flate.BestSpeed, jwt)
} }
func doUpload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, compression int, jwt security.EncodedJwt) (*UploadResult, error) {
func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, compression int, jwt security.EncodedJwt) (*UploadResult, error) {
contentIsGzipped := isGzipped contentIsGzipped := isGzipped
shouldGzipNow := false shouldGzipNow := false
if !isGzipped { if !isGzipped {
@ -65,7 +66,25 @@ func doUpload(uploadUrl string, filename string, reader io.Reader, isGzipped boo
contentIsGzipped = true contentIsGzipped = true
} }
} }
return upload_content(uploadUrl, func(w io.Writer) (err error) {
// encrypt data
var cipherKey util.CipherKey
var clearDataLen int
if cipher {
clearData, err := ioutil.ReadAll(reader)
if err != nil {
return nil, fmt.Errorf("read raw input: %v", err)
}
clearDataLen = len(clearData)
cipherKey = util.GenCipherKey()
encryptedData, err := util.Encrypt(clearData, cipherKey)
if err != nil {
return nil, fmt.Errorf("encrypt input: %v", err)
}
reader = bytes.NewReader(encryptedData)
}
// upload data
uploadResult, err := upload_content(uploadUrl, func(w io.Writer) (err error) {
if shouldGzipNow { if shouldGzipNow {
gzWriter, _ := gzip.NewWriterLevel(w, compression) gzWriter, _ := gzip.NewWriterLevel(w, compression)
_, err = io.Copy(gzWriter, reader) _, err = io.Copy(gzWriter, reader)
@ -75,6 +94,14 @@ func doUpload(uploadUrl string, filename string, reader io.Reader, isGzipped boo
} }
return return
}, filename, contentIsGzipped, mtype, pairMap, jwt) }, filename, contentIsGzipped, mtype, pairMap, jwt)
// remember cipher key
if uploadResult != nil && cipherKey != nil {
uploadResult.CipherKey = cipherKey
uploadResult.Size = uint32(clearDataLen)
}
return uploadResult, err
} }
func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {

2
weed/pb/filer.proto
View File

@ -99,6 +99,7 @@ message FileChunk {
string source_file_id = 6; // to be deprecated string source_file_id = 6; // to be deprecated
FileId fid = 7; FileId fid = 7;
FileId source_fid = 8; FileId source_fid = 8;
bytes cipher_key = 9;
} }
message FileId { message FileId {
@ -229,4 +230,5 @@ message GetFilerConfigurationResponse {
uint32 max_mb = 4; uint32 max_mb = 4;
string dir_buckets = 5; string dir_buckets = 5;
string dir_queues = 6; string dir_queues = 6;
bool cipher = 7;
} }

235
weed/pb/filer_pb/filer.pb.go
View File

@ -287,6 +287,7 @@ type FileChunk struct {
SourceFileId string `protobuf:"bytes,6,opt,name=source_file_id,json=sourceFileId" json:"source_file_id,omitempty"` SourceFileId string `protobuf:"bytes,6,opt,name=source_file_id,json=sourceFileId" json:"source_file_id,omitempty"`
Fid *FileId `protobuf:"bytes,7,opt,name=fid" json:"fid,omitempty"` Fid *FileId `protobuf:"bytes,7,opt,name=fid" json:"fid,omitempty"`
SourceFid *FileId `protobuf:"bytes,8,opt,name=source_fid,json=sourceFid" json:"source_fid,omitempty"` SourceFid *FileId `protobuf:"bytes,8,opt,name=source_fid,json=sourceFid" json:"source_fid,omitempty"`
CipherKey []byte `protobuf:"bytes,9,opt,name=cipher_key,json=cipherKey,proto3" json:"cipher_key,omitempty"`
} }
func (m *FileChunk) Reset() { *m = FileChunk{} } func (m *FileChunk) Reset() { *m = FileChunk{} }
@ -350,6 +351,13 @@ func (m *FileChunk) GetSourceFid() *FileId {
return nil return nil
} }
func (m *FileChunk) GetCipherKey() []byte {
if m != nil {
return m.CipherKey
}
return nil
}
type FileId struct { type FileId struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
FileKey uint64 `protobuf:"varint,2,opt,name=file_key,json=fileKey" json:"file_key,omitempty"` FileKey uint64 `protobuf:"varint,2,opt,name=file_key,json=fileKey" json:"file_key,omitempty"`
@ -1014,6 +1022,7 @@ type GetFilerConfigurationResponse struct {
MaxMb uint32 `protobuf:"varint,4,opt,name=max_mb,json=maxMb" json:"max_mb,omitempty"` MaxMb uint32 `protobuf:"varint,4,opt,name=max_mb,json=maxMb" json:"max_mb,omitempty"`
DirBuckets string `protobuf:"bytes,5,opt,name=dir_buckets,json=dirBuckets" json:"dir_buckets,omitempty"` DirBuckets string `protobuf:"bytes,5,opt,name=dir_buckets,json=dirBuckets" json:"dir_buckets,omitempty"`
DirQueues string `protobuf:"bytes,6,opt,name=dir_queues,json=dirQueues" json:"dir_queues,omitempty"` DirQueues string `protobuf:"bytes,6,opt,name=dir_queues,json=dirQueues" json:"dir_queues,omitempty"`
Cipher bool `protobuf:"varint,7,opt,name=cipher" json:"cipher,omitempty"`
} }
func (m *GetFilerConfigurationResponse) Reset() { *m = GetFilerConfigurationResponse{} } func (m *GetFilerConfigurationResponse) Reset() { *m = GetFilerConfigurationResponse{} }
@ -1063,6 +1072,13 @@ func (m *GetFilerConfigurationResponse) GetDirQueues() string {
return "" return ""
} }
func (m *GetFilerConfigurationResponse) GetCipher() bool {
if m != nil {
return m.Cipher
}
return false
}
func init() { func init() {
proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest") proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest")
proto.RegisterType((*LookupDirectoryEntryResponse)(nil), "filer_pb.LookupDirectoryEntryResponse") proto.RegisterType((*LookupDirectoryEntryResponse)(nil), "filer_pb.LookupDirectoryEntryResponse")
@ -1594,113 +1610,114 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{ var fileDescriptor0 = []byte{
// 1713 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xcb, 0x6e, 0xdb, 0xca,
0x19, 0x36, 0x75, 0xe7, 0x2f, 0x29, 0xb1, 0x47, 0x76, 0xa2, 0xc8, 0x97, 0x3a, 0x74, 0x93, 0xba,
0x48, 0xe0, 0x1a, 0x6e, 0x16, 0x49, 0xd3, 0x2e, 0x12, 0x5f, 0x0a, 0xa3, 0xce, 0xa5, 0x74, 0x52,
0xa4, 0x28, 0x50, 0x82, 0x26, 0x47, 0xf2, 0xd4, 0x24, 0x47, 0x19, 0x0e, 0x6d, 0xa7, 0x8f, 0x52,
0xa0, 0x8b, 0xbe, 0x47, 0xd1, 0x4d, 0x51, 0xa0, 0xeb, 0xf3, 0x08, 0xe7, 0x01, 0xce, 0xf2, 0xac,
0x0f, 0x66, 0x86, 0xa4, 0x86, 0xa2, 0x6c, 0x27, 0xe7, 0x20, 0x3b, 0xce, 0x7f, 0x9b, 0x7f, 0xbe,
0xff, 0x2a, 0x41, 0x7b, 0x48, 0x02, 0xcc, 0xb6, 0xc6, 0x8c, 0x72, 0x8a, 0x5a, 0xf2, 0xe0, 0x8c,
0x4f, 0xac, 0x37, 0xb0, 0x7c, 0x44, 0xe9, 0x59, 0x32, 0xde, 0x23, 0x0c, 0x7b, 0x9c, 0xb2, 0x4f,
0xfb, 0x11, 0x67, 0x9f, 0x6c, 0xfc, 0x31, 0xc1, 0x31, 0x47, 0x2b, 0x60, 0xfa, 0x19, 0xa3, 0x6f,
0xac, 0x1b, 0x9b, 0xa6, 0x3d, 0x21, 0x20, 0x04, 0xb5, 0xc8, 0x0d, 0x71, 0xbf, 0x22, 0x19, 0xf2,
0xdb, 0xda, 0x87, 0x95, 0xd9, 0x06, 0xe3, 0x31, 0x8d, 0x62, 0x8c, 0x1e, 0x40, 0x1d, 0x0b, 0x82,
0xb4, 0xd6, 0xde, 0xb9, 0xbd, 0x95, 0xb9, 0xb2, 0xa5, 0xe4, 0x14, 0xd7, 0xfa, 0x8f, 0x01, 0xe8,
0x88, 0xc4, 0x5c, 0x10, 0x09, 0x8e, 0x3f, 0xcf, 0x9f, 0x3b, 0xd0, 0x18, 0x33, 0x3c, 0x24, 0x97,
0xa9, 0x47, 0xe9, 0x09, 0x3d, 0x86, 0x85, 0x98, 0xbb, 0x8c, 0x1f, 0x30, 0x1a, 0x1e, 0x90, 0x00,
0xbf, 0x16, 0x4e, 0x57, 0xa5, 0x48, 0x99, 0x81, 0xb6, 0x00, 0x91, 0xc8, 0x0b, 0x92, 0x98, 0x9c,
0xe3, 0xe3, 0x8c, 0xdb, 0xaf, 0xad, 0x1b, 0x9b, 0x2d, 0x7b, 0x06, 0x07, 0x2d, 0x42, 0x3d, 0x20,
0x21, 0xe1, 0xfd, 0xfa, 0xba, 0xb1, 0xd9, 0xb5, 0xd5, 0xc1, 0xfa, 0x2d, 0xf4, 0x0a, 0xfe, 0x7f,
0xd9, 0xf3, 0xff, 0x59, 0x81, 0xba, 0x24, 0xe4, 0x18, 0x1b, 0x13, 0x8c, 0xd1, 0x7d, 0xe8, 0x90,
0xd8, 0x99, 0x00, 0x51, 0x91, 0xbe, 0xb5, 0x49, 0x9c, 0x63, 0x8e, 0x1e, 0x41, 0xc3, 0x3b, 0x4d,
0xa2, 0xb3, 0xb8, 0x5f, 0x5d, 0xaf, 0x6e, 0xb6, 0x77, 0x7a, 0x93, 0x8b, 0xc4, 0x43, 0x77, 0x05,
0xcf, 0x4e, 0x45, 0xd0, 0x53, 0x00, 0x97, 0x73, 0x46, 0x4e, 0x12, 0x8e, 0x63, 0xf9, 0xd2, 0xf6,
0x4e, 0x5f, 0x53, 0x48, 0x62, 0xfc, 0x22, 0xe7, 0xdb, 0x9a, 0x2c, 0x7a, 0x06, 0x2d, 0x7c, 0xc9,
0x71, 0xe4, 0x63, 0xbf, 0x5f, 0x97, 0x17, 0xad, 0x4e, 0xbd, 0x68, 0x6b, 0x3f, 0xe5, 0xab, 0xf7,
0xe5, 0xe2, 0x83, 0xe7, 0xd0, 0x2d, 0xb0, 0xd0, 0x3c, 0x54, 0xcf, 0x70, 0x16, 0x55, 0xf1, 0x29,
0x90, 0x3d, 0x77, 0x83, 0x44, 0x25, 0x58, 0xc7, 0x56, 0x87, 0xdf, 0x54, 0x9e, 0x1a, 0xd6, 0x1e,
0x98, 0x07, 0x49, 0x10, 0xe4, 0x8a, 0x3e, 0x61, 0x99, 0xa2, 0x4f, 0xd8, 0x04, 0xe5, 0xca, 0xb5,
0x28, 0xff, 0xdb, 0x80, 0x85, 0xfd, 0x73, 0x1c, 0xf1, 0xd7, 0x94, 0x93, 0x21, 0xf1, 0x5c, 0x4e,
0x68, 0x84, 0x1e, 0x83, 0x49, 0x03, 0xdf, 0xb9, 0x36, 0x4c, 0x2d, 0x1a, 0xa4, 0x5e, 0x3f, 0x06,
0x33, 0xc2, 0x17, 0xce, 0xb5, 0xd7, 0xb5, 0x22, 0x7c, 0xa1, 0xa4, 0x37, 0xa0, 0xeb, 0xe3, 0x00,
0x73, 0xec, 0xe4, 0xd1, 0x11, 0xa1, 0xeb, 0x28, 0xe2, 0xae, 0x0a, 0xc7, 0x43, 0xb8, 0x2d, 0x4c,
0x8e, 0x5d, 0x86, 0x23, 0xee, 0x8c, 0x5d, 0x7e, 0x2a, 0x63, 0x62, 0xda, 0xdd, 0x08, 0x5f, 0xbc,
0x95, 0xd4, 0xb7, 0x2e, 0x3f, 0xb5, 0xbe, 0x37, 0xc0, 0xcc, 0x83, 0x89, 0xee, 0x42, 0x53, 0x5c,
0xeb, 0x10, 0x3f, 0x45, 0xa2, 0x21, 0x8e, 0x87, 0xbe, 0xa8, 0x0a, 0x3a, 0x1c, 0xc6, 0x98, 0x4b,
0xf7, 0xaa, 0x76, 0x7a, 0x12, 0x99, 0x15, 0x93, 0xbf, 0xab, 0x42, 0xa8, 0xd9, 0xf2, 0x5b, 0x20,
0x1e, 0x72, 0x12, 0x62, 0x79, 0x61, 0xd5, 0x56, 0x07, 0xd4, 0x83, 0x3a, 0x76, 0xb8, 0x3b, 0x92,
0x19, 0x6e, 0xda, 0x35, 0xfc, 0xce, 0x1d, 0xa1, 0x9f, 0xc3, 0xad, 0x98, 0x26, 0xcc, 0xc3, 0x4e,
0x76, 0x6d, 0x43, 0x72, 0x3b, 0x8a, 0x7a, 0xa0, 0x2e, 0xb7, 0xa0, 0x3a, 0x24, 0x7e, 0xbf, 0x29,
0x81, 0x99, 0x2f, 0x26, 0xe1, 0xa1, 0x6f, 0x0b, 0x26, 0xfa, 0x15, 0x40, 0x6e, 0xc9, 0xef, 0xb7,
0xae, 0x10, 0x35, 0x33, 0xbb, 0xbe, 0xf5, 0x01, 0x1a, 0xa9, 0xf9, 0x65, 0x30, 0xcf, 0x69, 0x90,
0x84, 0xf9, 0xb3, 0xbb, 0x76, 0x4b, 0x11, 0x0e, 0x7d, 0x74, 0x0f, 0x64, 0x9f, 0x73, 0x44, 0x56,
0x55, 0xe4, 0x23, 0x25, 0x42, 0x7f, 0xc0, 0xb2, 0x53, 0x78, 0x94, 0x9e, 0x11, 0xf5, 0xfa, 0xa6,
0x9d, 0x9e, 0xac, 0xef, 0x2a, 0x70, 0xab, 0x98, 0xee, 0xe2, 0x0a, 0x69, 0x45, 0x62, 0x65, 0x48,
0x33, 0xd2, 0xec, 0x71, 0x01, 0xaf, 0x8a, 0x8e, 0x57, 0xa6, 0x12, 0x52, 0x5f, 0x5d, 0xd0, 0x55,
0x2a, 0xaf, 0xa8, 0x8f, 0x45, 0xb6, 0x26, 0xc4, 0x97, 0x00, 0x77, 0x6d, 0xf1, 0x29, 0x28, 0x23,
0xe2, 0xa7, 0xed, 0x43, 0x7c, 0x4a, 0xf7, 0x98, 0xb4, 0xdb, 0x50, 0x21, 0x53, 0x27, 0x11, 0xb2,
0x50, 0x50, 0x9b, 0x2a, 0x0e, 0xe2, 0x1b, 0xad, 0x43, 0x9b, 0xe1, 0x71, 0x90, 0x66, 0xaf, 0x84,
0xcf, 0xb4, 0x75, 0x12, 0x5a, 0x03, 0xf0, 0x68, 0x10, 0x60, 0x4f, 0x0a, 0x98, 0x52, 0x40, 0xa3,
0x88, 0xcc, 0xe1, 0x3c, 0x70, 0x62, 0xec, 0xf5, 0x61, 0xdd, 0xd8, 0xac, 0xdb, 0x0d, 0xce, 0x83,
0x63, 0xec, 0x89, 0x77, 0x24, 0x31, 0x66, 0x8e, 0x6c, 0x40, 0x6d, 0xa9, 0xd7, 0x12, 0x04, 0xd9,
0x26, 0x57, 0x01, 0x46, 0x8c, 0x26, 0x63, 0xc5, 0xed, 0xac, 0x57, 0x45, 0x2f, 0x96, 0x14, 0xc9,
0x7e, 0x00, 0xb7, 0xe2, 0x4f, 0x61, 0x40, 0xa2, 0x33, 0x87, 0xbb, 0x6c, 0x84, 0x79, 0xbf, 0xab,
0x72, 0x38, 0xa5, 0xbe, 0x93, 0x44, 0x6b, 0x0c, 0x68, 0x97, 0x61, 0x97, 0xe3, 0x2f, 0x18, 0x3b,
0x9f, 0x57, 0xdd, 0x68, 0x09, 0x1a, 0xd4, 0xc1, 0x97, 0x5e, 0x90, 0x16, 0x59, 0x9d, 0xee, 0x5f,
0x7a, 0x81, 0xf5, 0x08, 0x7a, 0x85, 0x1b, 0xd3, 0xc6, 0xbc, 0x08, 0x75, 0xcc, 0x18, 0xcd, 0xda,
0x88, 0x3a, 0x58, 0x7f, 0x06, 0xf4, 0x7e, 0xec, 0x7f, 0x0d, 0xf7, 0xac, 0x25, 0xe8, 0x15, 0x4c,
0x2b, 0x3f, 0xac, 0xff, 0x19, 0x80, 0xf6, 0x64, 0x37, 0xf8, 0x69, 0x83, 0x58, 0xd4, 0xa7, 0x18,
0x12, 0xaa, 0xdb, 0xf8, 0x2e, 0x77, 0xd3, 0x11, 0xd6, 0x21, 0xb1, 0xb2, 0xbf, 0xe7, 0x72, 0x37,
0x1d, 0x25, 0x0c, 0x7b, 0x09, 0x13, 0x53, 0x4d, 0x26, 0xa1, 0x1c, 0x25, 0x76, 0x46, 0x42, 0x4f,
0xe0, 0x0e, 0x19, 0x45, 0x94, 0xe1, 0x89, 0x98, 0xa3, 0xa0, 0x6a, 0x48, 0xe1, 0x45, 0xc5, 0xcd,
0x15, 0xf6, 0x25, 0x72, 0x8f, 0xa0, 0x57, 0x78, 0xc6, 0xb5, 0x30, 0xff, 0xc3, 0x80, 0xfe, 0x0b,
0x4e, 0x43, 0xe2, 0xd9, 0x58, 0x38, 0x5f, 0x78, 0xfa, 0x06, 0x74, 0x45, 0x3f, 0x9e, 0x7e, 0x7e,
0x87, 0x06, 0xfe, 0x64, 0xde, 0xdd, 0x03, 0xd1, 0x92, 0x1d, 0x0d, 0x85, 0x26, 0x0d, 0x7c, 0x99,
0x89, 0x1b, 0x20, 0xfa, 0xa6, 0xa6, 0xaf, 0x26, 0x7f, 0x27, 0xc2, 0x17, 0x05, 0x7d, 0x21, 0x24,
0xf5, 0x55, 0xb3, 0x6d, 0x46, 0xf8, 0x42, 0xe8, 0x5b, 0xcb, 0x70, 0x6f, 0x86, 0x6f, 0x69, 0xb8,
0xfe, 0x6f, 0x40, 0xef, 0x45, 0x1c, 0x93, 0x51, 0xf4, 0x27, 0xd9, 0x76, 0x32, 0xa7, 0x17, 0xa1,
0xee, 0xd1, 0x24, 0xe2, 0xd2, 0xd9, 0xba, 0xad, 0x0e, 0x53, 0x95, 0x58, 0x29, 0x55, 0xe2, 0x54,
0x2d, 0x57, 0xcb, 0xb5, 0xac, 0xd5, 0x6a, 0xad, 0x50, 0xab, 0x3f, 0x83, 0xb6, 0x08, 0xb2, 0xe3,
0xe1, 0x88, 0x63, 0x96, 0x76, 0x6a, 0x10, 0xa4, 0x5d, 0x49, 0x11, 0x02, 0xfa, 0x44, 0x51, 0xcd,
0x1a, 0xc6, 0x93, 0x71, 0xf2, 0xad, 0x01, 0x8b, 0xc5, 0xa7, 0xa4, 0x31, 0xbb, 0x72, 0xb2, 0x88,
0x56, 0xc6, 0x82, 0xf4, 0x1d, 0xe2, 0x53, 0x34, 0x85, 0x71, 0x72, 0x12, 0x10, 0xcf, 0x11, 0x0c,
0xe5, 0xbf, 0xa9, 0x28, 0xef, 0x59, 0x30, 0x41, 0xa5, 0xa6, 0xa3, 0x82, 0xa0, 0xe6, 0x26, 0xfc,
0x34, 0x9b, 0x2e, 0xe2, 0x7b, 0x0a, 0xa9, 0xc6, 0x4d, 0x48, 0x35, 0xcb, 0x48, 0xe5, 0x99, 0xd6,
0xd2, 0x33, 0xed, 0x09, 0xf4, 0xd4, 0x7a, 0x5a, 0x0c, 0xd7, 0x2a, 0x40, 0x3e, 0x47, 0xe2, 0xbe,
0xa1, 0x9a, 0x59, 0x36, 0x48, 0x62, 0xeb, 0x77, 0x60, 0x1e, 0x51, 0x65, 0x37, 0x46, 0xdb, 0x60,
0x06, 0xd9, 0x41, 0x8a, 0xb6, 0x77, 0xd0, 0xa4, 0xc6, 0x33, 0x39, 0x7b, 0x22, 0x64, 0x3d, 0x87,
0x56, 0x46, 0xce, 0x30, 0x33, 0xae, 0xc2, 0xac, 0x32, 0x85, 0x99, 0xf5, 0x5f, 0x03, 0x16, 0x8b,
0x2e, 0xa7, 0x61, 0x79, 0x0f, 0xdd, 0xfc, 0x0a, 0x27, 0x74, 0xc7, 0xa9, 0x2f, 0xdb, 0xba, 0x2f,
0x65, 0xb5, 0xdc, 0xc1, 0xf8, 0x95, 0x3b, 0x56, 0xb9, 0xdc, 0x09, 0x34, 0xd2, 0xe0, 0x1d, 0x2c,
0x94, 0x44, 0x66, 0xec, 0x66, 0xbf, 0xd4, 0x77, 0xb3, 0xc2, 0x7e, 0x99, 0x6b, 0xeb, 0x0b, 0xdb,
0x33, 0xb8, 0xab, 0xda, 0xc1, 0x6e, 0x1e, 0xc3, 0x0c, 0xfb, 0x62, 0xa8, 0x8d, 0xe9, 0x50, 0x5b,
0x03, 0xe8, 0x97, 0x55, 0xd3, 0xf2, 0x1b, 0xc1, 0xc2, 0x31, 0x77, 0x39, 0x89, 0x39, 0xf1, 0xf2,
0x1f, 0x09, 0x53, 0xb9, 0x61, 0xdc, 0x34, 0x11, 0xcb, 0x75, 0x38, 0x0f, 0x55, 0xce, 0xb3, 0xfc,
0x15, 0x9f, 0x22, 0x0a, 0x48, 0xbf, 0x29, 0x8d, 0xc1, 0x57, 0xb8, 0x4a, 0xe4, 0x03, 0xa7, 0xdc,
0x0d, 0xd4, 0xc6, 0x51, 0x93, 0x1b, 0x87, 0x29, 0x29, 0x72, 0xe5, 0x50, 0x43, 0xd9, 0x57, 0xdc,
0xba, 0xda, 0x47, 0x04, 0x41, 0x32, 0x57, 0x01, 0x64, 0xa9, 0xaa, 0x2a, 0x6b, 0x28, 0x5d, 0x41,
0xd9, 0x15, 0x04, 0x6b, 0x0d, 0x56, 0x7e, 0x8f, 0xb9, 0xd8, 0x9d, 0xd8, 0x2e, 0x8d, 0x86, 0x64,
0x94, 0x30, 0x57, 0x0b, 0x85, 0xf5, 0x8d, 0x01, 0xab, 0x57, 0x08, 0xa4, 0x0f, 0xee, 0x43, 0x33,
0x74, 0x63, 0x8e, 0x59, 0x56, 0x25, 0xd9, 0x71, 0x1a, 0x8a, 0xca, 0x4d, 0x50, 0x54, 0x4b, 0x50,
0x2c, 0x41, 0x23, 0x74, 0x2f, 0x9d, 0xf0, 0x24, 0x5d, 0x8e, 0xea, 0xa1, 0x7b, 0xf9, 0xea, 0x44,
0x76, 0x36, 0xc2, 0x9c, 0x93, 0xc4, 0x3b, 0xc3, 0x3c, 0xce, 0x3b, 0x1b, 0x61, 0x2f, 0x15, 0x45,
0x3c, 0x5a, 0x08, 0x7c, 0x4c, 0x70, 0x82, 0xe3, 0xb4, 0x57, 0x88, 0xe1, 0xf8, 0x47, 0x49, 0xd8,
0xf9, 0x57, 0x0b, 0x3a, 0xc7, 0xd8, 0xbd, 0xc0, 0xd8, 0x97, 0x0f, 0x43, 0xa3, 0xac, 0xa0, 0x8a,
0x3f, 0x51, 0xd1, 0x83, 0xe9, 0xca, 0x99, 0xf9, 0x9b, 0x78, 0xf0, 0xf0, 0x26, 0xb1, 0x34, 0x37,
0xe7, 0xd0, 0x6b, 0x68, 0x6b, 0xbf, 0x01, 0xd1, 0x8a, 0xa6, 0x58, 0xfa, 0x69, 0x3b, 0x58, 0xbd,
0x82, 0x9b, 0x59, 0xdb, 0x36, 0xd0, 0x11, 0xb4, 0xb5, 0xd5, 0x45, 0xb7, 0x57, 0xde, 0xa1, 0x74,
0x7b, 0x33, 0xf6, 0x1d, 0x6b, 0x4e, 0x58, 0xd3, 0x16, 0x10, 0xdd, 0x5a, 0x79, 0xe5, 0xd1, 0xad,
0xcd, 0xda, 0x5a, 0xa4, 0x35, 0x6d, 0xde, 0xeb, 0xd6, 0xca, 0xdb, 0x8c, 0x6e, 0x6d, 0xc6, 0x92,
0x60, 0xcd, 0xa1, 0x0f, 0xd0, 0x3b, 0xe6, 0x0c, 0xbb, 0xe1, 0x84, 0x3d, 0x85, 0xe0, 0x8f, 0xb0,
0xba, 0x69, 0x6c, 0x1b, 0xe8, 0xaf, 0xb0, 0x50, 0x9a, 0xe6, 0xc8, 0x9a, 0x68, 0x5e, 0xb5, 0x86,
0x0c, 0x36, 0xae, 0x95, 0xc9, 0x3d, 0x7f, 0x03, 0x1d, 0x7d, 0x88, 0x22, 0xcd, 0xa9, 0x19, 0x7b,
0xc2, 0x60, 0xed, 0x2a, 0xb6, 0x6e, 0x50, 0xef, 0xe3, 0xba, 0xc1, 0x19, 0x93, 0x4c, 0x37, 0x38,
0xab, 0xfd, 0x5b, 0x73, 0xe8, 0x2f, 0x30, 0x3f, 0xdd, 0x4f, 0xd1, 0xfd, 0x69, 0xe8, 0x4a, 0x6d,
0x7a, 0x60, 0x5d, 0x27, 0x92, 0x1b, 0x3f, 0x04, 0x98, 0xb4, 0x49, 0xb4, 0x3c, 0xd1, 0x29, 0xb5,
0xe9, 0xc1, 0xca, 0x6c, 0x66, 0x6e, 0xea, 0x6f, 0xb0, 0x34, 0xb3, 0x17, 0x21, 0xad, 0x00, 0xaf,
0xeb, 0x66, 0x83, 0x5f, 0xdc, 0x28, 0x97, 0xdd, 0xf5, 0x72, 0x0d, 0xe6, 0x63, 0xd5, 0x22, 0x86,
0xf1, 0x96, 0x17, 0x10, 0x1c, 0xf1, 0x97, 0x20, 0x35, 0xde, 0x32, 0xca, 0xe9, 0x49, 0x43, 0xfe,
0x6f, 0xf6, 0xeb, 0x1f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xdb, 0x25, 0xa7, 0x6f, 0x46, 0x13, 0x00,
0x00,
// 1742 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x4f, 0x6f, 0xdc, 0xc6,
0x15, 0x37, 0xf7, 0x3f, 0xdf, 0xee, 0x3a, 0xd2, 0xac, 0x9c, 0xac, 0xd7, 0x92, 0xab, 0xd0, 0x75,
0xaa, 0xc2, 0x86, 0x6a, 0xa8, 0x39, 0x24, 0x4d, 0x7b, 0xb0, 0x65, 0xb9, 0x30, 0x62, 0x3b, 0x2e,
0x65, 0x17, 0x29, 0x0a, 0x94, 0xa0, 0xc8, 0xd9, 0xd5, 0x54, 0x24, 0x87, 0x19, 0x0e, 0x2d, 0xb9,
0xdf, 0xa3, 0x97, 0x02, 0x3d, 0xf4, 0x7b, 0x14, 0xbd, 0x14, 0x05, 0xfa, 0x39, 0x7a, 0xec, 0xa1,
0x9f, 0xa1, 0x98, 0x37, 0x24, 0x77, 0xb8, 0x5c, 0x49, 0x49, 0x8b, 0xdc, 0x38, 0xef, 0xdf, 0xbc,
0xf9, 0xbd, 0xbf, 0xbb, 0x30, 0x9c, 0xb3, 0x88, 0x8a, 0xfd, 0x54, 0x70, 0xc9, 0xc9, 0x00, 0x0f,
0x5e, 0x7a, 0xe2, 0x7c, 0x05, 0x77, 0x5e, 0x70, 0x7e, 0x96, 0xa7, 0x4f, 0x99, 0xa0, 0x81, 0xe4,
0xe2, 0xfd, 0x51, 0x22, 0xc5, 0x7b, 0x97, 0x7e, 0x93, 0xd3, 0x4c, 0x92, 0x6d, 0xb0, 0xc3, 0x92,
0x31, 0xb5, 0x76, 0xad, 0x3d, 0xdb, 0x5d, 0x12, 0x08, 0x81, 0x4e, 0xe2, 0xc7, 0x74, 0xda, 0x42,
0x06, 0x7e, 0x3b, 0x47, 0xb0, 0xbd, 0xde, 0x60, 0x96, 0xf2, 0x24, 0xa3, 0xe4, 0x3e, 0x74, 0xa9,
0x22, 0xa0, 0xb5, 0xe1, 0xc1, 0x07, 0xfb, 0xa5, 0x2b, 0xfb, 0x5a, 0x4e, 0x73, 0x9d, 0xbf, 0x59,
0x40, 0x5e, 0xb0, 0x4c, 0x2a, 0x22, 0xa3, 0xd9, 0xb7, 0xf3, 0xe7, 0x43, 0xe8, 0xa5, 0x82, 0xce,
0xd9, 0x45, 0xe1, 0x51, 0x71, 0x22, 0x0f, 0x61, 0x33, 0x93, 0xbe, 0x90, 0xcf, 0x04, 0x8f, 0x9f,
0xb1, 0x88, 0xbe, 0x52, 0x4e, 0xb7, 0x51, 0xa4, 0xc9, 0x20, 0xfb, 0x40, 0x58, 0x12, 0x44, 0x79,
0xc6, 0xde, 0xd1, 0xe3, 0x92, 0x3b, 0xed, 0xec, 0x5a, 0x7b, 0x03, 0x77, 0x0d, 0x87, 0x6c, 0x41,
0x37, 0x62, 0x31, 0x93, 0xd3, 0xee, 0xae, 0xb5, 0x37, 0x76, 0xf5, 0xc1, 0xf9, 0x39, 0x4c, 0x6a,
0xfe, 0x7f, 0xb7, 0xe7, 0xff, 0xb9, 0x05, 0x5d, 0x24, 0x54, 0x18, 0x5b, 0x4b, 0x8c, 0xc9, 0xc7,
0x30, 0x62, 0x99, 0xb7, 0x04, 0xa2, 0x85, 0xbe, 0x0d, 0x59, 0x56, 0x61, 0x4e, 0x1e, 0x40, 0x2f,
0x38, 0xcd, 0x93, 0xb3, 0x6c, 0xda, 0xde, 0x6d, 0xef, 0x0d, 0x0f, 0x26, 0xcb, 0x8b, 0xd4, 0x43,
0x0f, 0x15, 0xcf, 0x2d, 0x44, 0xc8, 0x67, 0x00, 0xbe, 0x94, 0x82, 0x9d, 0xe4, 0x92, 0x66, 0xf8,
0xd2, 0xe1, 0xc1, 0xd4, 0x50, 0xc8, 0x33, 0xfa, 0xb8, 0xe2, 0xbb, 0x86, 0x2c, 0xf9, 0x1c, 0x06,
0xf4, 0x42, 0xd2, 0x24, 0xa4, 0xe1, 0xb4, 0x8b, 0x17, 0xed, 0xac, 0xbc, 0x68, 0xff, 0xa8, 0xe0,
0xeb, 0xf7, 0x55, 0xe2, 0xb3, 0x2f, 0x60, 0x5c, 0x63, 0x91, 0x0d, 0x68, 0x9f, 0xd1, 0x32, 0xaa,
0xea, 0x53, 0x21, 0xfb, 0xce, 0x8f, 0x72, 0x9d, 0x60, 0x23, 0x57, 0x1f, 0x7e, 0xd6, 0xfa, 0xcc,
0x72, 0x9e, 0x82, 0xfd, 0x2c, 0x8f, 0xa2, 0x4a, 0x31, 0x64, 0xa2, 0x54, 0x0c, 0x99, 0x58, 0xa2,
0xdc, 0xba, 0x12, 0xe5, 0xbf, 0x5a, 0xb0, 0x79, 0xf4, 0x8e, 0x26, 0xf2, 0x15, 0x97, 0x6c, 0xce,
0x02, 0x5f, 0x32, 0x9e, 0x90, 0x87, 0x60, 0xf3, 0x28, 0xf4, 0xae, 0x0c, 0xd3, 0x80, 0x47, 0x85,
0xd7, 0x0f, 0xc1, 0x4e, 0xe8, 0xb9, 0x77, 0xe5, 0x75, 0x83, 0x84, 0x9e, 0x6b, 0xe9, 0x7b, 0x30,
0x0e, 0x69, 0x44, 0x25, 0xf5, 0xaa, 0xe8, 0xa8, 0xd0, 0x8d, 0x34, 0xf1, 0x50, 0x87, 0xe3, 0x13,
0xf8, 0x40, 0x99, 0x4c, 0x7d, 0x41, 0x13, 0xe9, 0xa5, 0xbe, 0x3c, 0xc5, 0x98, 0xd8, 0xee, 0x38,
0xa1, 0xe7, 0xaf, 0x91, 0xfa, 0xda, 0x97, 0xa7, 0xce, 0x1f, 0x5b, 0x60, 0x57, 0xc1, 0x24, 0x1f,
0x41, 0x5f, 0x5d, 0xeb, 0xb1, 0xb0, 0x40, 0xa2, 0xa7, 0x8e, 0xcf, 0x43, 0x55, 0x15, 0x7c, 0x3e,
0xcf, 0xa8, 0x44, 0xf7, 0xda, 0x6e, 0x71, 0x52, 0x99, 0x95, 0xb1, 0x3f, 0xe8, 0x42, 0xe8, 0xb8,
0xf8, 0xad, 0x10, 0x8f, 0x25, 0x8b, 0x29, 0x5e, 0xd8, 0x76, 0xf5, 0x81, 0x4c, 0xa0, 0x4b, 0x3d,
0xe9, 0x2f, 0x30, 0xc3, 0x6d, 0xb7, 0x43, 0xdf, 0xf8, 0x0b, 0xf2, 0x43, 0xb8, 0x99, 0xf1, 0x5c,
0x04, 0xd4, 0x2b, 0xaf, 0xed, 0x21, 0x77, 0xa4, 0xa9, 0xcf, 0xf4, 0xe5, 0x0e, 0xb4, 0xe7, 0x2c,
0x9c, 0xf6, 0x11, 0x98, 0x8d, 0x7a, 0x12, 0x3e, 0x0f, 0x5d, 0xc5, 0x24, 0x3f, 0x01, 0xa8, 0x2c,
0x85, 0xd3, 0xc1, 0x25, 0xa2, 0x76, 0x69, 0x37, 0x24, 0x3b, 0x00, 0x01, 0x4b, 0x4f, 0xa9, 0xf0,
0x54, 0xc2, 0xd8, 0x98, 0x1c, 0xb6, 0xa6, 0x7c, 0x49, 0xdf, 0x3b, 0x5f, 0x43, 0xaf, 0xb8, 0xfd,
0x0e, 0xd8, 0xef, 0x78, 0x94, 0xc7, 0x15, 0x2a, 0x63, 0x77, 0xa0, 0x09, 0xcf, 0x43, 0x72, 0x1b,
0xb0, 0x0d, 0xa2, 0x8d, 0x16, 0x62, 0x80, 0x00, 0x7e, 0x49, 0xb1, 0x91, 0x04, 0x9c, 0x9f, 0x31,
0x0d, 0x4e, 0xdf, 0x2d, 0x4e, 0xce, 0x7f, 0x5a, 0x70, 0xb3, 0x5e, 0x0d, 0xea, 0x0a, 0xb4, 0x82,
0x50, 0x5a, 0x68, 0x06, 0xcd, 0x1e, 0xd7, 0xe0, 0x6c, 0x99, 0x70, 0x96, 0x2a, 0x31, 0x0f, 0xf5,
0x05, 0x63, 0xad, 0xf2, 0x92, 0x87, 0x54, 0x25, 0x73, 0xce, 0x42, 0xc4, 0x7f, 0xec, 0xaa, 0x4f,
0x45, 0x59, 0xb0, 0xb0, 0xe8, 0x2e, 0xea, 0x13, 0xdd, 0x13, 0x68, 0xb7, 0xa7, 0x23, 0xaa, 0x4f,
0x2a, 0xa2, 0xb1, 0xa2, 0xf6, 0x75, 0x98, 0xd4, 0x37, 0xd9, 0x85, 0xa1, 0xa0, 0x69, 0x54, 0x24,
0x37, 0xa2, 0x6b, 0xbb, 0x26, 0x89, 0xdc, 0x05, 0x08, 0x78, 0x14, 0xd1, 0x00, 0x05, 0x6c, 0x14,
0x30, 0x28, 0x2a, 0xb1, 0xa4, 0x8c, 0xbc, 0x8c, 0x06, 0x53, 0xd8, 0xb5, 0xf6, 0xba, 0x6e, 0x4f,
0xca, 0xe8, 0x98, 0x06, 0xea, 0x1d, 0x79, 0x46, 0x85, 0x87, 0xfd, 0x69, 0x88, 0x7a, 0x03, 0x45,
0xc0, 0x2e, 0xba, 0x03, 0xb0, 0x10, 0x3c, 0x4f, 0x35, 0x77, 0xb4, 0xdb, 0x56, 0xad, 0x1a, 0x29,
0xc8, 0xbe, 0x0f, 0x37, 0xb3, 0xf7, 0x71, 0xc4, 0x92, 0x33, 0x4f, 0xfa, 0x62, 0x41, 0xe5, 0x74,
0xac, 0x53, 0xbc, 0xa0, 0xbe, 0x41, 0xa2, 0x93, 0x02, 0x39, 0x14, 0xd4, 0x97, 0xf4, 0x3b, 0x4c,
0xa5, 0x6f, 0x57, 0xfc, 0xe4, 0x16, 0xf4, 0xb8, 0x47, 0x2f, 0x82, 0xa8, 0xa8, 0xc1, 0x2e, 0x3f,
0xba, 0x08, 0x22, 0xe7, 0x01, 0x4c, 0x6a, 0x37, 0x16, 0x7d, 0x7b, 0x0b, 0xba, 0x54, 0x08, 0x5e,
0x76, 0x19, 0x7d, 0x70, 0x7e, 0x03, 0xe4, 0x6d, 0x1a, 0x7e, 0x1f, 0xee, 0x39, 0xb7, 0x60, 0x52,
0x33, 0xad, 0xfd, 0x70, 0xfe, 0x61, 0x01, 0x79, 0x8a, 0xcd, 0xe2, 0xff, 0x9b, 0xd3, 0xaa, 0x7c,
0xd5, 0x0c, 0xd1, 0xcd, 0x28, 0xf4, 0xa5, 0x5f, 0x4c, 0xb8, 0x11, 0xcb, 0xb4, 0xfd, 0xa7, 0xbe,
0xf4, 0x8b, 0x49, 0x23, 0x68, 0x90, 0x0b, 0x35, 0xf4, 0x30, 0x09, 0x71, 0xd2, 0xb8, 0x25, 0x89,
0x7c, 0x0a, 0x1f, 0xb2, 0x45, 0xc2, 0x05, 0x5d, 0x8a, 0x79, 0x1a, 0xaa, 0x1e, 0x0a, 0x6f, 0x69,
0x6e, 0xa5, 0x70, 0x84, 0xc8, 0x3d, 0x80, 0x49, 0xed, 0x19, 0x57, 0xc2, 0xfc, 0x27, 0x0b, 0xa6,
0x8f, 0x25, 0x8f, 0x59, 0xe0, 0x52, 0xe5, 0x7c, 0xed, 0xe9, 0xf7, 0x60, 0xac, 0xda, 0xf5, 0xea,
0xf3, 0x47, 0x3c, 0x0a, 0x97, 0xe3, 0xf0, 0x36, 0xa8, 0x8e, 0xed, 0x19, 0x28, 0xf4, 0x79, 0x14,
0x62, 0x26, 0xde, 0x03, 0xd5, 0x56, 0x0d, 0x7d, 0xbd, 0x18, 0x8c, 0x12, 0x7a, 0x5e, 0xd3, 0x57,
0x42, 0xa8, 0xaf, 0x7b, 0x71, 0x3f, 0xa1, 0xe7, 0x4a, 0xdf, 0xb9, 0x03, 0xb7, 0xd7, 0xf8, 0x56,
0x84, 0xeb, 0x9f, 0x16, 0x4c, 0x1e, 0x67, 0x19, 0x5b, 0x24, 0xbf, 0xc6, 0xb6, 0x53, 0x3a, 0xbd,
0x05, 0xdd, 0x80, 0xe7, 0x89, 0x44, 0x67, 0xbb, 0xae, 0x3e, 0xac, 0x54, 0x62, 0xab, 0x51, 0x89,
0x2b, 0xb5, 0xdc, 0x6e, 0xd6, 0xb2, 0x51, 0xab, 0x9d, 0x5a, 0xad, 0xfe, 0x00, 0x86, 0x2a, 0xc8,
0x5e, 0x40, 0x13, 0x49, 0x45, 0xd1, 0xc8, 0x41, 0x91, 0x0e, 0x91, 0xa2, 0x04, 0xcc, 0x81, 0xa3,
0x7b, 0x39, 0xa4, 0xcb, 0x69, 0xf3, 0x2f, 0x0b, 0xb6, 0xea, 0x4f, 0x29, 0x62, 0x76, 0xe9, 0xe0,
0x51, 0xad, 0x4c, 0x44, 0xc5, 0x3b, 0xd4, 0xa7, 0x6a, 0x0a, 0x69, 0x7e, 0x12, 0xb1, 0xc0, 0x53,
0x0c, 0xed, 0xbf, 0xad, 0x29, 0x6f, 0x45, 0xb4, 0x44, 0xa5, 0x63, 0xa2, 0x42, 0xa0, 0xe3, 0xe7,
0xf2, 0xb4, 0x1c, 0x3e, 0xea, 0x7b, 0x05, 0xa9, 0xde, 0x75, 0x48, 0xf5, 0x9b, 0x48, 0x55, 0x99,
0x36, 0x30, 0x33, 0xed, 0x53, 0x98, 0xe8, 0xed, 0xb5, 0x1e, 0xae, 0x1d, 0x80, 0x6a, 0x8e, 0x64,
0x53, 0x4b, 0x37, 0xb3, 0x72, 0x90, 0x64, 0xce, 0x2f, 0xc0, 0x7e, 0xc1, 0xb5, 0xdd, 0x8c, 0x3c,
0x02, 0x3b, 0x2a, 0x0f, 0x28, 0x3a, 0x3c, 0x20, 0xcb, 0x1a, 0x2f, 0xe5, 0xdc, 0xa5, 0x90, 0xf3,
0x05, 0x0c, 0x4a, 0x72, 0x89, 0x99, 0x75, 0x19, 0x66, 0xad, 0x15, 0xcc, 0x9c, 0xbf, 0x5b, 0xb0,
0x55, 0x77, 0xb9, 0x08, 0xcb, 0x5b, 0x18, 0x57, 0x57, 0x78, 0xb1, 0x9f, 0x16, 0xbe, 0x3c, 0x32,
0x7d, 0x69, 0xaa, 0x55, 0x0e, 0x66, 0x2f, 0xfd, 0x54, 0xe7, 0xf2, 0x28, 0x32, 0x48, 0xb3, 0x37,
0xb0, 0xd9, 0x10, 0x59, 0xb3, 0xba, 0xfd, 0xd8, 0x5c, 0xdd, 0x6a, 0xeb, 0x67, 0xa5, 0x6d, 0xee,
0x73, 0x9f, 0xc3, 0x47, 0xba, 0x1d, 0x1c, 0x56, 0x31, 0x2c, 0xb1, 0xaf, 0x87, 0xda, 0x5a, 0x0d,
0xb5, 0x33, 0x83, 0x69, 0x53, 0xb5, 0x28, 0xbf, 0x05, 0x6c, 0x1e, 0x4b, 0x5f, 0xb2, 0x4c, 0xb2,
0xa0, 0xfa, 0x0d, 0xb1, 0x92, 0x1b, 0xd6, 0x75, 0x13, 0xb1, 0x59, 0x87, 0x1b, 0xd0, 0x96, 0xb2,
0xcc, 0x5f, 0xf5, 0xa9, 0xa2, 0x40, 0xcc, 0x9b, 0x8a, 0x18, 0x7c, 0x0f, 0x57, 0xa9, 0x7c, 0x90,
0x5c, 0xfa, 0x91, 0xde, 0x38, 0x3a, 0xb8, 0x71, 0xd8, 0x48, 0xc1, 0x95, 0x43, 0x0f, 0xe5, 0x50,
0x73, 0xbb, 0x7a, 0x1f, 0x51, 0x04, 0x64, 0xee, 0x00, 0x60, 0xa9, 0xea, 0x2a, 0xeb, 0x69, 0x5d,
0x45, 0x39, 0x54, 0x04, 0xe7, 0x2e, 0x6c, 0xff, 0x92, 0x4a, 0xb5, 0x3b, 0x89, 0x43, 0x9e, 0xcc,
0xd9, 0x22, 0x17, 0xbe, 0x11, 0x0a, 0xe7, 0xdf, 0x16, 0xec, 0x5c, 0x22, 0x50, 0x3c, 0x78, 0x0a,
0xfd, 0xd8, 0xcf, 0x24, 0x15, 0x65, 0x95, 0x94, 0xc7, 0x55, 0x28, 0x5a, 0xd7, 0x41, 0xd1, 0x6e,
0x40, 0x71, 0x0b, 0x7a, 0xb1, 0x7f, 0xe1, 0xc5, 0x27, 0xc5, 0x72, 0xd4, 0x8d, 0xfd, 0x8b, 0x97,
0x27, 0xd8, 0xd9, 0x98, 0xf0, 0x4e, 0xf2, 0xe0, 0x8c, 0xca, 0xac, 0xea, 0x6c, 0x4c, 0x3c, 0xd1,
0x14, 0xf5, 0x68, 0x25, 0xf0, 0x4d, 0x4e, 0x73, 0x9a, 0x15, 0xbd, 0x42, 0x0d, 0xc7, 0x5f, 0x21,
0x01, 0x97, 0x29, 0x5c, 0x1d, 0xb1, 0x4b, 0x0c, 0xdc, 0xe2, 0x74, 0xf0, 0x97, 0x01, 0x8c, 0x8e,
0xa9, 0x7f, 0x4e, 0x69, 0x88, 0x0f, 0x26, 0x8b, 0xb2, 0xd0, 0xea, 0xbf, 0x6c, 0xc9, 0xfd, 0xd5,
0x8a, 0x5a, 0xfb, 0x53, 0x7a, 0xf6, 0xc9, 0x75, 0x62, 0x45, 0xce, 0xde, 0x20, 0xaf, 0x60, 0x68,
0xfc, 0x74, 0x24, 0xdb, 0x86, 0x62, 0xe3, 0x17, 0xf1, 0x6c, 0xe7, 0x12, 0x6e, 0x69, 0xed, 0x91,
0x45, 0x5e, 0xc0, 0xd0, 0x58, 0x69, 0x4c, 0x7b, 0xcd, 0xdd, 0xca, 0xb4, 0xb7, 0x66, 0x0f, 0x72,
0x6e, 0x28, 0x6b, 0xc6, 0x62, 0x62, 0x5a, 0x6b, 0xae, 0x42, 0xa6, 0xb5, 0x75, 0xdb, 0x0c, 0x5a,
0x33, 0xf6, 0x00, 0xd3, 0x5a, 0x73, 0xcb, 0x31, 0xad, 0xad, 0x59, 0x1e, 0x9c, 0x1b, 0xe4, 0x6b,
0x98, 0x1c, 0x4b, 0x41, 0xfd, 0x78, 0xc9, 0x5e, 0x41, 0xf0, 0x7f, 0xb0, 0xba, 0x67, 0x3d, 0xb2,
0xc8, 0xef, 0x60, 0xb3, 0x31, 0xe5, 0x89, 0xb3, 0xd4, 0xbc, 0x6c, 0x3d, 0x99, 0xdd, 0xbb, 0x52,
0xa6, 0xf2, 0xfc, 0x2b, 0x18, 0x99, 0xc3, 0x95, 0x18, 0x4e, 0xad, 0xd9, 0x1f, 0x66, 0x77, 0x2f,
0x63, 0x9b, 0x06, 0xcd, 0xfe, 0x6e, 0x1a, 0x5c, 0x33, 0xe1, 0x4c, 0x83, 0xeb, 0xc6, 0x82, 0x73,
0x83, 0xfc, 0x16, 0x36, 0x56, 0xfb, 0x2c, 0xf9, 0x78, 0x15, 0xba, 0x46, 0xfb, 0x9e, 0x39, 0x57,
0x89, 0x54, 0xc6, 0x9f, 0x03, 0x2c, 0xdb, 0x27, 0xb9, 0xb3, 0xd4, 0x69, 0xb4, 0xef, 0xd9, 0xf6,
0x7a, 0x66, 0x65, 0xea, 0xf7, 0x70, 0x6b, 0x6d, 0x8f, 0x22, 0x46, 0x01, 0x5e, 0xd5, 0xe5, 0x66,
0x3f, 0xba, 0x56, 0xae, 0xbc, 0xeb, 0xc9, 0x5d, 0xd8, 0xc8, 0x74, 0x8b, 0x98, 0x67, 0xfb, 0x41,
0xc4, 0x68, 0x22, 0x9f, 0x00, 0x6a, 0xbc, 0x16, 0x5c, 0xf2, 0x93, 0x1e, 0xfe, 0xdd, 0xf6, 0xd3,
0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xf6, 0xb5, 0x1d, 0x19, 0x7d, 0x13, 0x00, 0x00,
} }

8
weed/pb/master_pb/master.pb.go
View File

@ -431,9 +431,7 @@ type VolumeEcShardInformationMessage struct {
func (m *VolumeEcShardInformationMessage) Reset() { *m = VolumeEcShardInformationMessage{} } func (m *VolumeEcShardInformationMessage) Reset() { *m = VolumeEcShardInformationMessage{} }
func (m *VolumeEcShardInformationMessage) String() string { return proto.CompactTextString(m) } func (m *VolumeEcShardInformationMessage) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardInformationMessage) ProtoMessage() {} func (*VolumeEcShardInformationMessage) ProtoMessage() {}
func (*VolumeEcShardInformationMessage) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{4}
}
func (*VolumeEcShardInformationMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
func (m *VolumeEcShardInformationMessage) GetId() uint32 { func (m *VolumeEcShardInformationMessage) GetId() uint32 {
if m != nil { if m != nil {
@ -1427,9 +1425,7 @@ type GetMasterConfigurationResponse struct {
func (m *GetMasterConfigurationResponse) Reset() { *m = GetMasterConfigurationResponse{} } func (m *GetMasterConfigurationResponse) Reset() { *m = GetMasterConfigurationResponse{} }
func (m *GetMasterConfigurationResponse) String() string { return proto.CompactTextString(m) } func (m *GetMasterConfigurationResponse) String() string { return proto.CompactTextString(m) }
func (*GetMasterConfigurationResponse) ProtoMessage() {} func (*GetMasterConfigurationResponse) ProtoMessage() {}
func (*GetMasterConfigurationResponse) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{32}
}
func (*GetMasterConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{32} }
func (m *GetMasterConfigurationResponse) GetMetricsAddress() string { func (m *GetMasterConfigurationResponse) GetMetricsAddress() string {
if m != nil { if m != nil {

12
weed/pb/volume_server_pb/volume_server.pb.go
View File

@ -1038,9 +1038,7 @@ type VolumeEcShardsGenerateResponse struct {
func (m *VolumeEcShardsGenerateResponse) Reset() { *m = VolumeEcShardsGenerateResponse{} } func (m *VolumeEcShardsGenerateResponse) Reset() { *m = VolumeEcShardsGenerateResponse{} }
func (m *VolumeEcShardsGenerateResponse) String() string { return proto.CompactTextString(m) } func (m *VolumeEcShardsGenerateResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardsGenerateResponse) ProtoMessage() {} func (*VolumeEcShardsGenerateResponse) ProtoMessage() {}
func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{41}
}
func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{41} }
type VolumeEcShardsRebuildRequest struct { type VolumeEcShardsRebuildRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
@ -1416,9 +1414,7 @@ type VolumeEcShardsToVolumeResponse struct {
func (m *VolumeEcShardsToVolumeResponse) Reset() { *m = VolumeEcShardsToVolumeResponse{} } func (m *VolumeEcShardsToVolumeResponse) Reset() { *m = VolumeEcShardsToVolumeResponse{} }
func (m *VolumeEcShardsToVolumeResponse) String() string { return proto.CompactTextString(m) } func (m *VolumeEcShardsToVolumeResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardsToVolumeResponse) ProtoMessage() {} func (*VolumeEcShardsToVolumeResponse) ProtoMessage() {}
func (*VolumeEcShardsToVolumeResponse) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{57}
}
func (*VolumeEcShardsToVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{57} }
type ReadVolumeFileStatusRequest struct { type ReadVolumeFileStatusRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
@ -2085,9 +2081,7 @@ type QueryRequest_InputSerialization_JSONInput struct {
func (m *QueryRequest_InputSerialization_JSONInput) Reset() { func (m *QueryRequest_InputSerialization_JSONInput) Reset() {
*m = QueryRequest_InputSerialization_JSONInput{} *m = QueryRequest_InputSerialization_JSONInput{}
} }
func (m *QueryRequest_InputSerialization_JSONInput) String() string {
return proto.CompactTextString(m)
}
func (m *QueryRequest_InputSerialization_JSONInput) String() string { return proto.CompactTextString(m) }
func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {} func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {}
func (*QueryRequest_InputSerialization_JSONInput) Descriptor() ([]byte, []int) { func (*QueryRequest_InputSerialization_JSONInput) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{70, 1, 1} return fileDescriptor0, []int{70, 1, 1}

2
weed/replication/sink/azuresink/azure_sink.go
View File

@ -115,7 +115,7 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry) error {
} }
var writeErr error var writeErr error
_, readErr := util.ReadUrlAsStream(fileUrl, chunk.Offset, int(chunk.Size), func(data []byte) {
readErr := util.ReadUrlAsStream(fileUrl, nil, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) {
_, writeErr = appendBlobURL.AppendBlock(context.Background(), bytes.NewReader(data), azblob.AppendBlobAccessConditions{}, nil) _, writeErr = appendBlobURL.AppendBlock(context.Background(), bytes.NewReader(data), azblob.AppendBlobAccessConditions{}, nil)
}) })

2
weed/replication/sink/b2sink/b2_sink.go
View File

@ -103,7 +103,7 @@ func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
} }
var writeErr error var writeErr error
_, readErr := util.ReadUrlAsStream(fileUrl, chunk.Offset, int(chunk.Size), func(data []byte) {
readErr := util.ReadUrlAsStream(fileUrl, nil, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) {
_, err := writer.Write(data) _, err := writer.Write(data)
if err != nil { if err != nil {
writeErr = err writeErr = err

5
weed/replication/sink/filersink/fetch_write.go
View File

@ -50,6 +50,7 @@ func (fs *FilerSink) replicateOneChunk(sourceChunk *filer_pb.FileChunk, dir stri
Mtime: sourceChunk.Mtime, Mtime: sourceChunk.Mtime,
ETag: sourceChunk.ETag, ETag: sourceChunk.ETag,
SourceFileId: sourceChunk.GetFileIdString(), SourceFileId: sourceChunk.GetFileIdString(),
CipherKey: sourceChunk.CipherKey,
}, nil }, nil
} }
@ -95,8 +96,8 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, dir string)
glog.V(4).Infof("replicating %s to %s header:%+v", filename, fileUrl, header) glog.V(4).Infof("replicating %s to %s header:%+v", filename, fileUrl, header)
uploadResult, err := operation.Upload(fileUrl, filename, readCloser,
"gzip" == header.Get("Content-Encoding"), header.Get("Content-Type"), nil, auth)
// fetch data as is, regardless whether it is encrypted or not
uploadResult, err := operation.Upload(fileUrl, filename, false, readCloser, "gzip" == header.Get("Content-Encoding"), header.Get("Content-Type"), nil, auth)
if err != nil { if err != nil {
glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err) glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err)
return "", fmt.Errorf("upload data: %v", err) return "", fmt.Errorf("upload data: %v", err)

5
weed/replication/sink/gcssink/gcs_sink.go
View File

@ -6,13 +6,14 @@ import (
"os" "os"
"cloud.google.com/go/storage" "cloud.google.com/go/storage"
"google.golang.org/api/option"
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/replication/sink" "github.com/chrislusf/seaweedfs/weed/replication/sink"
"github.com/chrislusf/seaweedfs/weed/replication/source" "github.com/chrislusf/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
"google.golang.org/api/option"
) )
type GcsSink struct { type GcsSink struct {
@ -100,7 +101,7 @@ func (g *GcsSink) CreateEntry(key string, entry *filer_pb.Entry) error {
return err return err
} }
_, err = util.ReadUrlAsStream(fileUrl, chunk.Offset, int(chunk.Size), func(data []byte) {
err = util.ReadUrlAsStream(fileUrl, nil, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) {
wc.Write(data) wc.Write(data)
}) })

2
weed/replication/sink/s3sink/s3_write.go
View File

@ -162,6 +162,6 @@ func (s3sink *S3Sink) buildReadSeeker(chunk *filer2.ChunkView) (io.ReadSeeker, e
return nil, err return nil, err
} }
buf := make([]byte, chunk.Size) buf := make([]byte, chunk.Size)
util.ReadUrl(fileUrl, chunk.Offset, int(chunk.Size), buf, true)
util.ReadUrl(fileUrl, nil, false, chunk.Offset, int(chunk.Size), buf)
return bytes.NewReader(buf), nil return bytes.NewReader(buf), nil
} }

2
weed/server/common.go
View File

@ -134,7 +134,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st
} }
debug("upload file to store", url) debug("upload file to store", url)
uploadResult, err := operation.Upload(url, fname, bytes.NewReader(data), isGzipped, mimeType, pairMap, assignResult.Auth)
uploadResult, err := operation.Upload(url, fname, false, bytes.NewReader(data), isGzipped, mimeType, pairMap, assignResult.Auth)
if err != nil { if err != nil {
writeJsonError(w, r, http.StatusInternalServerError, err) writeJsonError(w, r, http.StatusInternalServerError, err)
return return

1
weed/server/filer_grpc_server.go
View File

@ -338,5 +338,6 @@ func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb.
MaxMb: uint32(fs.option.MaxMB), MaxMb: uint32(fs.option.MaxMB),
DirBuckets: fs.filer.DirBucketsPath, DirBuckets: fs.filer.DirBucketsPath,
DirQueues: fs.filer.DirQueuesPath, DirQueues: fs.filer.DirQueuesPath,
Cipher: fs.filer.Cipher,
}, nil }, nil
} }

2
weed/server/filer_server.go
View File

@ -46,6 +46,7 @@ type FilerOption struct {
DisableHttp bool DisableHttp bool
Port uint32 Port uint32
recursiveDelete bool recursiveDelete bool
Cipher bool
} }
type FilerServer struct { type FilerServer struct {
@ -67,6 +68,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
} }
fs.filer = filer2.NewFiler(option.Masters, fs.grpcDialOption, option.Port+10000) fs.filer = filer2.NewFiler(option.Masters, fs.grpcDialOption, option.Port+10000)
fs.filer.Cipher = option.Cipher
go fs.filer.KeepConnectedToMaster() go fs.filer.KeepConnectedToMaster()

23
weed/server/filer_server_handlers_read.go
View File

@ -14,6 +14,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
) )
@ -93,7 +94,7 @@ func (fs *FilerServer) handleSingleChunk(w http.ResponseWriter, r *http.Request,
return return
} }
if fs.option.RedirectOnRead {
if fs.option.RedirectOnRead && entry.Chunks[0].CipherKey == nil {
stats.FilerRequestCounter.WithLabelValues("redirect").Inc() stats.FilerRequestCounter.WithLabelValues("redirect").Inc()
http.Redirect(w, r, urlString, http.StatusFound) http.Redirect(w, r, urlString, http.StatusFound)
return return
@ -136,7 +137,27 @@ func (fs *FilerServer) handleSingleChunk(w http.ResponseWriter, r *http.Request,
w.Header().Set("Content-Type", entry.Attr.Mime) w.Header().Set("Content-Type", entry.Attr.Mime)
} }
w.WriteHeader(resp.StatusCode) w.WriteHeader(resp.StatusCode)
if entry.Chunks[0].CipherKey == nil {
io.Copy(w, resp.Body) io.Copy(w, resp.Body)
} else {
fs.writeEncryptedChunk(w, resp, entry.Chunks[0])
}
}
func (fs *FilerServer) writeEncryptedChunk(w http.ResponseWriter, resp *http.Response, chunk *filer_pb.FileChunk) {
encryptedData, err := ioutil.ReadAll(resp.Body)
if err != nil {
glog.V(1).Infof("read encrypted %s failed, err: %v", chunk.FileId, err)
w.WriteHeader(http.StatusNotFound)
return
}
decryptedData, err := util.Decrypt(encryptedData, util.CipherKey(chunk.CipherKey))
if err != nil {
glog.V(1).Infof("decrypt %s failed, err: %v", chunk.FileId, err)
w.WriteHeader(http.StatusNotFound)
return
}
w.Write(decryptedData)
} }
func (fs *FilerServer) handleMultipleChunks(w http.ResponseWriter, r *http.Request, entry *filer2.Entry) { func (fs *FilerServer) handleMultipleChunks(w http.ResponseWriter, r *http.Request, entry *filer2.Entry) {

2
weed/server/filer_server_handlers_write_autochunk.go
View File

@ -182,7 +182,7 @@ func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *ht
stats.FilerRequestHistogram.WithLabelValues("postAutoChunkUpload").Observe(time.Since(start).Seconds()) stats.FilerRequestHistogram.WithLabelValues("postAutoChunkUpload").Observe(time.Since(start).Seconds())
}() }()
uploadResult, uploadError := operation.Upload(urlLocation, fileName, limitedReader, false, contentType, nil, auth)
uploadResult, uploadError := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, nil, auth)
if uploadError != nil { if uploadError != nil {
return 0, uploadError return 0, uploadError
} }

4
weed/server/webdav_server.go
View File

@ -32,6 +32,7 @@ type WebDavOption struct {
Collection string Collection string
Uid uint32 Uid uint32
Gid uint32 Gid uint32
Cipher bool
} }
type WebDavServer struct { type WebDavServer struct {
@ -418,7 +419,7 @@ func (f *WebDavFile) Write(buf []byte) (int, error) {
fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
bufReader := bytes.NewReader(buf) bufReader := bytes.NewReader(buf)
uploadResult, err := operation.Upload(fileUrl, f.name, bufReader, false, "", nil, auth)
uploadResult, err := operation.Upload(fileUrl, f.name, f.fs.option.Cipher, bufReader, false, "", nil, auth)
if err != nil { if err != nil {
glog.V(0).Infof("upload data %v to %s: %v", f.name, fileUrl, err) glog.V(0).Infof("upload data %v to %s: %v", f.name, fileUrl, err)
return 0, fmt.Errorf("upload data: %v", err) return 0, fmt.Errorf("upload data: %v", err)
@ -434,6 +435,7 @@ func (f *WebDavFile) Write(buf []byte) (int, error) {
Size: uint64(len(buf)), Size: uint64(len(buf)),
Mtime: time.Now().UnixNano(), Mtime: time.Now().UnixNano(),
ETag: uploadResult.ETag, ETag: uploadResult.ETag,
CipherKey: uploadResult.CipherKey,
} }
f.entry.Chunks = append(f.entry.Chunks, chunk) f.entry.Chunks = append(f.entry.Chunks, chunk)

5
weed/topology/store_replicate.go
View File

@ -72,9 +72,8 @@ func ReplicatedWrite(masterNode string, s *storage.Store,
} }
} }
_, err := operation.Upload(u.String(),
string(n.Name), bytes.NewReader(n.Data), n.IsGzipped(), string(n.Mime),
pairMap, jwt)
// volume server do not know about encryption
_, err := operation.Upload(u.String(), string(n.Name), false, bytes.NewReader(n.Data), n.IsGzipped(), string(n.Mime), pairMap, jwt)
return err return err
}); err != nil { }); err != nil {
size = 0 size = 0

60
weed/util/cipher.go
View File

@ -0,0 +1,60 @@
package util
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"errors"
"io"
"github.com/chrislusf/seaweedfs/weed/glog"
)
type CipherKey []byte
func GenCipherKey() CipherKey {
key := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, key); err != nil {
glog.Fatalf("random key gen: %v", err)
}
return CipherKey(key)
}
func Encrypt(plaintext []byte, key CipherKey) ([]byte, error) {
c, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
gcm, err := cipher.NewGCM(c)
if err != nil {
return nil, err
}
nonce := make([]byte, gcm.NonceSize())
if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
return nil, err
}
return gcm.Seal(nonce, nonce, plaintext, nil), nil
}
func Decrypt(ciphertext []byte, key CipherKey) ([]byte, error) {
c, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
gcm, err := cipher.NewGCM(c)
if err != nil {
return nil, err
}
nonceSize := gcm.NonceSize()
if len(ciphertext) < nonceSize {
return nil, errors.New("ciphertext too short")
}
nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:]
return gcm.Open(nil, nonce, ciphertext, nil)
}

49
weed/util/http_util.go
View File

@ -189,13 +189,21 @@ func NormalizeUrl(url string) string {
return "http://" + url return "http://" + url
} }
func ReadUrl(fileUrl string, offset int64, size int, buf []byte, isReadRange bool) (int64, error) {
func ReadUrl(fileUrl string, cipherKey []byte, isFullChunk bool, offset int64, size int, buf []byte) (int64, error) {
if cipherKey != nil {
var n int
err := readEncryptedUrl(fileUrl, cipherKey, offset, size, func(data []byte) {
n = copy(buf, data)
})
return int64(n), err
}
req, err := http.NewRequest("GET", fileUrl, nil) req, err := http.NewRequest("GET", fileUrl, nil)
if err != nil { if err != nil {
return 0, err return 0, err
} }
if isReadRange {
if !isFullChunk {
req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1)) req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1))
} else { } else {
req.Header.Set("Accept-Encoding", "gzip") req.Header.Set("Accept-Encoding", "gzip")
@ -250,41 +258,62 @@ func ReadUrl(fileUrl string, offset int64, size int, buf []byte, isReadRange boo
return n, err return n, err
} }
func ReadUrlAsStream(fileUrl string, offset int64, size int, fn func(data []byte)) (int64, error) {
func ReadUrlAsStream(fileUrl string, cipherKey []byte, isFullChunk bool, offset int64, size int, fn func(data []byte)) error {
if cipherKey != nil {
return readEncryptedUrl(fileUrl, cipherKey, offset, size, fn)
}
req, err := http.NewRequest("GET", fileUrl, nil) req, err := http.NewRequest("GET", fileUrl, nil)
if err != nil { if err != nil {
return 0, err
return err
} }
if !isFullChunk {
req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1)) req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1))
}
r, err := client.Do(req) r, err := client.Do(req)
if err != nil { if err != nil {
return 0, err
return err
} }
defer CloseResponse(r) defer CloseResponse(r)
if r.StatusCode >= 400 { if r.StatusCode >= 400 {
return 0, fmt.Errorf("%s: %s", fileUrl, r.Status)
return fmt.Errorf("%s: %s", fileUrl, r.Status)
} }
var ( var (
m int m int
n int64
) )
buf := make([]byte, 64*1024) buf := make([]byte, 64*1024)
for { for {
m, err = r.Body.Read(buf) m, err = r.Body.Read(buf)
fn(buf[:m]) fn(buf[:m])
n += int64(m)
if err == io.EOF { if err == io.EOF {
return n, nil
return nil
} }
if err != nil { if err != nil {
return n, err
return err
}
} }
} }
func readEncryptedUrl(fileUrl string, cipherKey []byte, offset int64, size int, fn func(data []byte)) error {
encryptedData, err := Get(fileUrl)
if err != nil {
return fmt.Errorf("fetch %s: %v", fileUrl, err)
}
decryptedData, err := Decrypt(encryptedData, CipherKey(cipherKey))
if err != nil {
return fmt.Errorf("decrypt %s: %v", fileUrl, err)
}
if len(decryptedData) < int(offset)+size {
return fmt.Errorf("read decrypted %s size %d [%d, %d)", fileUrl, len(decryptedData), offset, int(offset)+size)
}
fn(decryptedData[int(offset) : int(offset)+size])
return nil
} }
func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, error) { func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, error) {

Write Preview
Loading…
Cancel
Save