From 308688c8d0b29caa06739338748cb3882c3a8559 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 20 Mar 2020 20:31:11 -0700 Subject: [PATCH] filer: add back image resizing capability --- weed/filer2/stream.go | 58 ++++++++++++++++++++++ weed/operation/chunked_file.go | 1 + weed/server/common.go | 2 +- weed/server/filer_server_handlers_read.go | 16 +++++- weed/server/volume_server_handlers_read.go | 17 +++++-- 5 files changed, 87 insertions(+), 7 deletions(-) diff --git a/weed/filer2/stream.go b/weed/filer2/stream.go index 381d99144..bf985f8bd 100644 --- a/weed/filer2/stream.go +++ b/weed/filer2/stream.go @@ -1,7 +1,10 @@ package filer2 import ( + "bytes" + "fmt" "io" + "math" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -40,3 +43,58 @@ func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*f return nil } + +type ChunkStreamReader struct { + masterClient *wdclient.MasterClient + chunkViews []*ChunkView + logicOffset int64 + buffer bytes.Buffer + bufferOffset int64 + chunkIndex int +} + +var _ = io.ReadSeeker(&ChunkStreamReader{}) + +func NewChunkStreamReader(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader { + + chunkViews := ViewFromChunks(chunks, 0, math.MaxInt32) + + return &ChunkStreamReader{ + masterClient: masterClient, + chunkViews: chunkViews, + bufferOffset: -1, + } +} + +func (c *ChunkStreamReader) Read(p []byte) (n int, err error) { + if c.buffer.Len() == 0 { + if c.chunkIndex >= len(c.chunkViews) { + return 0, io.EOF + } + chunkView := c.chunkViews[c.chunkIndex] + c.fetchChunkToBuffer(chunkView) + c.chunkIndex++ + } + return c.buffer.Read(p) +} + +func (c *ChunkStreamReader) Seek(offset int64, whence int) (int64, error) { + return 0, fmt.Errorf("ChunkStreamReader: seek not supported") +} + +func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error { + urlString, err := c.masterClient.LookupFileId(chunkView.FileId) + if err != nil { + glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err) + return err + } + c.buffer.Reset() + err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.isGzipped, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), func(data []byte) { + c.buffer.Write(data) + }) + if err != nil { + glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err) + return err + } + return nil +} diff --git a/weed/operation/chunked_file.go b/weed/operation/chunked_file.go index d874de4ee..1ab3c59ed 100644 --- a/weed/operation/chunked_file.go +++ b/weed/operation/chunked_file.go @@ -131,6 +131,7 @@ func NewChunkedFileReader(chunkList []*ChunkInfo, master string) *ChunkedFileRea for _, chunk := range chunkList { totalSize += chunk.Size } + sort.Sort(ChunkList(chunkList)) return &ChunkedFileReader{ totalSize: totalSize, chunkList: chunkList, diff --git a/weed/server/common.go b/weed/server/common.go index e06142d7f..f064ca707 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -224,7 +224,7 @@ func adjustHeadersAfterHEAD(w http.ResponseWriter, r *http.Request, filename str } } -func processRangeRequst(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) { +func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) { rangeReq := r.Header.Get("Range") if rangeReq == "" { diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 878b52ffa..5322492dc 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -11,6 +11,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/images" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/stats" ) @@ -89,8 +90,19 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, totalSize := int64(filer2.TotalSize(entry.Chunks)) - processRangeRequst(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error { - return filer2.StreamContent(fs.filer.MasterClient, w, entry.Chunks, offset, int(size)) + if rangeReq := r.Header.Get("Range"); rangeReq == "" { + ext := filepath.Ext(filename) + width, height, mode, shouldResize := shouldResizeImages(ext, r) + if shouldResize { + chunkedFileReader := filer2.NewChunkStreamReader(fs.filer.MasterClient, entry.Chunks) + rs, _, _ := images.Resized(ext, chunkedFileReader, width, height, mode) + io.Copy(w, rs) + return + } + } + + processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error { + return filer2.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, int(size)) }) } diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index 996eb3a2f..a222f198d 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -200,20 +200,29 @@ func (vs *VolumeServer) tryHandleChunkedFile(n *needle.Needle, fileName string, func conditionallyResizeImages(originalDataReaderSeeker io.ReadSeeker, ext string, r *http.Request) io.ReadSeeker { rs := originalDataReaderSeeker + + width, height, mode, shouldResize := shouldResizeImages(ext, r) + if shouldResize { + rs, _, _ = images.Resized(ext, originalDataReaderSeeker, width, height, mode) + } + return rs +} + +func shouldResizeImages(ext string, r *http.Request) (width, height int, mode string, shouldResize bool) { if len(ext) > 0 { ext = strings.ToLower(ext) } if ext == ".png" || ext == ".jpg" || ext == ".jpeg" || ext == ".gif" { - width, height := 0, 0 if r.FormValue("width") != "" { width, _ = strconv.Atoi(r.FormValue("width")) } if r.FormValue("height") != "" { height, _ = strconv.Atoi(r.FormValue("height")) } - rs, _, _ = images.Resized(ext, originalDataReaderSeeker, width, height, r.FormValue("mode")) } - return rs + mode = r.FormValue("mode") + shouldResize = width > 0 || height > 0 + return } func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.ResponseWriter, r *http.Request) error { @@ -235,7 +244,7 @@ func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.Re adjustHeadersAfterHEAD(w, r, filename) - processRangeRequst(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error { + processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error { if _, e = rs.Seek(offset, 0); e != nil { return e }