Browse Source

prevents any potential data loss or corruption in the small-file inline storage path

pull/7481/head
chrislu 1 month ago
parent
commit
d4da42ed7c
  1. 30
      weed/operation/upload_chunked.go

30
weed/operation/upload_chunked.go

@ -93,21 +93,25 @@ func UploadReaderInChunks(ctx context.Context, reader io.Reader, opt *ChunkedUpl
break break
} }
// For small files at offset 0, store inline instead of uploading
if chunkOffset == 0 && opt.SaveSmallInline && dataSize < opt.SmallFileLimit {
smallContent := make([]byte, dataSize)
bytesBuffer.Read(smallContent)
chunkBufferPool.Put(bytesBuffer)
<-bytesBufferLimitChan
return &ChunkedUploadResult{
FileChunks: nil,
Md5Hash: md5Hash,
TotalSize: dataSize,
SmallContent: smallContent,
}, nil
// For small files at offset 0, store inline instead of uploading
if chunkOffset == 0 && opt.SaveSmallInline && dataSize < opt.SmallFileLimit {
smallContent := make([]byte, dataSize)
n, readErr := io.ReadFull(bytesBuffer, smallContent)
chunkBufferPool.Put(bytesBuffer)
<-bytesBufferLimitChan
if readErr != nil {
return nil, fmt.Errorf("failed to read small content: read %d of %d bytes: %w", n, dataSize, readErr)
} }
return &ChunkedUploadResult{
FileChunks: nil,
Md5Hash: md5Hash,
TotalSize: dataSize,
SmallContent: smallContent,
}, nil
}
// Upload chunk in parallel goroutine // Upload chunk in parallel goroutine
wg.Add(1) wg.Add(1)
go func(offset int64, buf *bytes.Buffer) { go func(offset int64, buf *bytes.Buffer) {

Loading…
Cancel
Save