From b832ddd1ef949a99f60f4bf1fd321b70ee592626 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Wed, 3 Jan 2024 21:50:32 +0500 Subject: [PATCH 1/4] fix: return etag with md5 in webdav responses (#5158) --- weed/server/webdav_server.go | 6 ++++++ weed/server/wrapped_webdav_fs.go | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index 35c554839..217caa59b 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -96,6 +96,7 @@ type FileInfo struct { size int64 mode os.FileMode modifiedTime time.Time + etag string isDirectory bool } @@ -106,6 +107,10 @@ func (fi *FileInfo) ModTime() time.Time { return fi.modifiedTime } func (fi *FileInfo) IsDir() bool { return fi.isDirectory } func (fi *FileInfo) Sys() interface{} { return nil } +func (fi *FileInfo) ETag(ctx context.Context) (string, error) { + return fi.etag, nil +} + type WebDavFile struct { fs *WebDavFileSystem name string @@ -369,6 +374,7 @@ func (fs *WebDavFileSystem) stat(ctx context.Context, fullFilePath string) (os.F fi.name = string(fullpath) fi.mode = os.FileMode(entry.Attributes.FileMode) fi.modifiedTime = time.Unix(entry.Attributes.Mtime, 0) + fi.etag = filer.ETag(entry) fi.isDirectory = entry.IsDirectory if fi.name == "/" { diff --git a/weed/server/wrapped_webdav_fs.go b/weed/server/wrapped_webdav_fs.go index 93e2b3122..f110d41a4 100644 --- a/weed/server/wrapped_webdav_fs.go +++ b/weed/server/wrapped_webdav_fs.go @@ -95,3 +95,11 @@ func (w wrappedFileInfo) Name() string { name := w.FileInfo.Name() return strings.TrimPrefix(name, *w.subFolder) } + +func (w wrappedFileInfo) ETag(ctx context.Context) (string, error) { + etag, _ := w.FileInfo.(webdav.ETager).ETag(ctx) + if len(etag) == 0 { + return etag, webdav.ErrNotImplemented + } + return etag, nil +} From 3e43778869ccbb1ddab207e48f58e5be5d37fd53 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Wed, 3 Jan 2024 22:52:51 +0500 Subject: [PATCH 2/4] fix: webdav avoid create empty files (#5160) --- weed/server/webdav_server.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index 217caa59b..c4b1a1c03 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -241,7 +241,7 @@ func (fs *WebDavFileSystem) OpenFile(ctx context.Context, fullFilePath string, f Name: name, IsDirectory: perm&os.ModeDir > 0, Attributes: &filer_pb.FuseAttributes{ - Mtime: time.Now().Unix(), + Mtime: 0, Crtime: time.Now().Unix(), FileMode: uint32(perm), Uid: fs.option.Uid, @@ -429,12 +429,13 @@ func (f *WebDavFile) Write(buf []byte) (int, error) { glog.V(2).Infof("WebDavFileSystem.Write %v", f.name) - dir, _ := util.FullPath(f.name).DirAndName() + fullPath := util.FullPath(f.name) + dir, _ := fullPath.DirAndName() var getErr error ctx := context.Background() if f.entry == nil { - f.entry, getErr = filer_pb.GetEntry(f.fs, util.FullPath(f.name)) + f.entry, getErr = filer_pb.GetEntry(f.fs, fullPath) } if f.entry == nil { @@ -451,6 +452,11 @@ func (f *WebDavFile) Write(buf []byte) (int, error) { chunk, flushErr = f.saveDataAsChunk(util.NewBytesReader(data), f.name, offset, time.Now().UnixNano()) if flushErr != nil { + if f.entry.Attributes.Mtime == 0 { + if err := f.fs.removeAll(ctx, f.name); err != nil { + glog.Errorf("bufWriter.Flush remove file error: %+v", f.name) + } + } return fmt.Errorf("%s upload result: %v", f.name, flushErr) } From 4f6172f3699cc47107dea3fe4f3664f283296cdb Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Thu, 4 Jan 2024 01:27:13 +0500 Subject: [PATCH 3/4] Fix missing VersionConfiguration node in get-bucket-versioning response (#5162) * fix: s3 response for get bucket version https://github.com/seaweedfs/seaweedfs/issues/5155 * fix: s3 response for get bucket version https://github.com/seaweedfs/seaweedfs/issues/5155 --- weed/s3api/s3api_bucket_handlers.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index c1bb815a9..6ae6011f2 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -545,8 +545,9 @@ func (s3a *S3ApiServer) GetBucketVersioningHandler(w http.ResponseWriter, r *htt return } - result := &s3.VersioningConfiguration{} - result.SetStatus(s3.BucketVersioningStatusSuspended) - - s3err.WriteAwsXMLResponse(w, r, http.StatusOK, result) + s3err.WriteAwsXMLResponse(w, r, http.StatusOK, &s3.PutBucketVersioningInput{ + VersioningConfiguration: &s3.VersioningConfiguration{ + Status: aws.String(s3.BucketVersioningStatusSuspended), + }, + }) } From 3b62b338a5532639fa34cd2a56101ea5507f6eb3 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Sat, 6 Jan 2024 00:10:20 +0500 Subject: [PATCH 4/4] chore: add maxMB option for webdav (#5165) --- weed/command/server.go | 1 + weed/command/webdav.go | 3 +++ weed/server/webdav_server.go | 5 +++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/weed/command/server.go b/weed/command/server.go index 9631f6bfd..b8ef25fad 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -164,6 +164,7 @@ func init() { webdavOptions.tlsCertificate = cmdServer.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB") + webdavOptions.maxMB = cmdServer.Flag.Int("webdav.maxMB", 4, "split files larger than the limit") webdavOptions.filerRootPath = cmdServer.Flag.String("webdav.filer.path", "/", "use this remote path from filer server") mqBrokerOptions.port = cmdServer.Flag.Int("mq.broker.port", 17777, "message queue broker gRPC listen port") diff --git a/weed/command/webdav.go b/weed/command/webdav.go index 67e6ce69c..f0e738f4a 100644 --- a/weed/command/webdav.go +++ b/weed/command/webdav.go @@ -32,6 +32,7 @@ type WebDavOption struct { tlsCertificate *string cacheDir *string cacheSizeMB *int64 + maxMB *int } func init() { @@ -45,6 +46,7 @@ func init() { webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file") webDavStandaloneOptions.cacheDir = cmdWebDav.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks") webDavStandaloneOptions.cacheSizeMB = cmdWebDav.Flag.Int64("cacheCapacityMB", 0, "local cache capacity in MB") + webDavStandaloneOptions.maxMB = cmdWebDav.Flag.Int("maxMB", 4, "split files larger than the limit") webDavStandaloneOptions.filerRootPath = cmdWebDav.Flag.String("filer.path", "/", "use this remote path from filer server") } @@ -116,6 +118,7 @@ func (wo *WebDavOption) startWebDav() bool { Cipher: cipher, CacheDir: util.ResolvePath(*wo.cacheDir), CacheSizeMB: *wo.cacheSizeMB, + MaxMB: *wo.maxMB, }) if webdavServer_err != nil { glog.Fatalf("WebDav Server startup error: %v", webdavServer_err) diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index c4b1a1c03..97d51dad7 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -38,6 +38,7 @@ type WebDavOption struct { Cipher bool CacheDir string CacheSizeMB int64 + MaxMB int } type WebDavServer struct { @@ -262,7 +263,7 @@ func (fs *WebDavFileSystem) OpenFile(ctx context.Context, fullFilePath string, f fs: fs, name: fullFilePath, isDirectory: false, - bufWriter: buffered_writer.NewBufferedWriteCloser(4 * 1024 * 1024), + bufWriter: buffered_writer.NewBufferedWriteCloser(fs.option.MaxMB * 1024 * 1024), }, nil } @@ -278,7 +279,7 @@ func (fs *WebDavFileSystem) OpenFile(ctx context.Context, fullFilePath string, f fs: fs, name: fullFilePath, isDirectory: false, - bufWriter: buffered_writer.NewBufferedWriteCloser(4 * 1024 * 1024), + bufWriter: buffered_writer.NewBufferedWriteCloser(fs.option.MaxMB * 1024 * 1024), }, nil }