Browse Source

revert PR #1903 avoid http error: superfluous response.WriteHeader

pull/2050/head
Chris Lu 4 years ago
parent
commit
ac71117ee6
  1. 4
      go.mod
  2. 2
      go.sum
  3. 2
      weed/command/filer_cat.go
  4. 2
      weed/filer/filer_on_meta_event.go
  5. 2
      weed/filer/read_write.go
  6. 25
      weed/filer/stream.go
  7. 6
      weed/s3api/s3api_object_handlers.go
  8. 8
      weed/server/common.go
  9. 10
      weed/server/filer_server_handlers_read.go
  10. 7
      weed/server/volume_server_handlers_read.go
  11. 2
      weed/shell/command_fs_cat.go

4
go.mod

@ -39,7 +39,7 @@ require (
github.com/google/uuid v1.1.1 github.com/google/uuid v1.1.1
github.com/gorilla/mux v1.7.4 github.com/gorilla/mux v1.7.4
github.com/gorilla/websocket v1.4.1 // indirect github.com/gorilla/websocket v1.4.1 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
github.com/jcmturner/gofork v1.0.0 // indirect github.com/jcmturner/gofork v1.0.0 // indirect
github.com/jinzhu/copier v0.2.8 github.com/jinzhu/copier v0.2.8
@ -90,7 +90,7 @@ require (
gocloud.dev/pubsub/rabbitpubsub v0.20.0 gocloud.dev/pubsub/rabbitpubsub v0.20.0
golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sync v0.0.0-20200930132711-30421366ff76 // indirect
golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd
golang.org/x/tools v0.0.0-20200608174601-1b747fd94509 golang.org/x/tools v0.0.0-20200608174601-1b747fd94509
google.golang.org/api v0.26.0 google.golang.org/api v0.26.0

2
go.sum

@ -969,8 +969,6 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200930132711-30421366ff76 h1:JnxiSYT3Nm0BT2a8CyvYyM6cnrWpidecD1UuSYbhKm0= golang.org/x/sync v0.0.0-20200930132711-30421366ff76 h1:JnxiSYT3Nm0BT2a8CyvYyM6cnrWpidecD1UuSYbhKm0=
golang.org/x/sync v0.0.0-20200930132711-30421366ff76/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200930132711-30421366ff76/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

2
weed/command/filer_cat.go

@ -110,7 +110,7 @@ func runFilerCat(cmd *Command, args []string) bool {
filerCat.filerClient = client filerCat.filerClient = client
return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
}) })

2
weed/filer/filer_on_meta_event.go

@ -52,7 +52,7 @@ func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataR
func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) { func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {
var buf bytes.Buffer var buf bytes.Buffer
err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64, false)
err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64)
if err != nil { if err != nil {
return nil, err return nil, err
} }

2
weed/filer/read_write.go

@ -27,7 +27,7 @@ func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.Seaweed
return err return err
} }
return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
} }

25
weed/filer/stream.go

@ -3,7 +3,6 @@ package filer
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"golang.org/x/sync/errgroup"
"io" "io"
"math" "math"
"strings" "strings"
@ -14,7 +13,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/wdclient" "github.com/chrislusf/seaweedfs/weed/wdclient"
) )
func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64, isCheck bool) error {
func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
glog.V(9).Infof("start to stream content for chunks: %+v\n", chunks) glog.V(9).Infof("start to stream content for chunks: %+v\n", chunks)
chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size) chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size)
@ -34,16 +33,6 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c
fileId2Url[chunkView.FileId] = urlStrings fileId2Url[chunkView.FileId] = urlStrings
} }
if isCheck {
// Pre-check all chunkViews urls
gErr := new(errgroup.Group)
CheckAllChunkViews(chunkViews, &fileId2Url, gErr)
if err := gErr.Wait(); err != nil {
glog.Errorf("check all chunks: %v", err)
return fmt.Errorf("check all chunks: %v", err)
}
return nil
}
for _, chunkView := range chunkViews { for _, chunkView := range chunkViews {
@ -53,7 +42,6 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c
glog.Errorf("read chunk: %v", err) glog.Errorf("read chunk: %v", err)
return fmt.Errorf("read chunk: %v", err) return fmt.Errorf("read chunk: %v", err)
} }
_, err = w.Write(data) _, err = w.Write(data)
if err != nil { if err != nil {
glog.Errorf("write chunk: %v", err) glog.Errorf("write chunk: %v", err)
@ -65,17 +53,6 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c
} }
func CheckAllChunkViews(chunkViews []*ChunkView, fileId2Url *map[string][]string, gErr *errgroup.Group) {
for _, chunkView := range chunkViews {
urlStrings := (*fileId2Url)[chunkView.FileId]
glog.V(9).Infof("Check chunk: %+v\n url: %v", chunkView, urlStrings)
gErr.Go(func() error {
_, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
return err
})
}
}
// ---------------- ReadAllReader ---------------------------------- // ---------------- ReadAllReader ----------------------------------
func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) ([]byte, error) { func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) ([]byte, error) {

6
weed/s3api/s3api_object_handlers.go

@ -326,11 +326,7 @@ func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) {
for k, v := range proxyResponse.Header { for k, v := range proxyResponse.Header {
w.Header()[k] = v w.Header()[k] = v
} }
if proxyResponse.Header.Get("Content-Range") != "" && proxyResponse.StatusCode == 200 {
w.WriteHeader(http.StatusPartialContent)
} else {
w.WriteHeader(proxyResponse.StatusCode)
}
w.WriteHeader(proxyResponse.StatusCode)
io.Copy(w, proxyResponse.Body) io.Copy(w, proxyResponse.Body)
} }

8
weed/server/common.go

@ -234,12 +234,12 @@ func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, file
} }
} }
func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) {
func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64, httpStatusCode int) error) {
rangeReq := r.Header.Get("Range") rangeReq := r.Header.Get("Range")
if rangeReq == "" { if rangeReq == "" {
w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
if err := writeFn(w, 0, totalSize); err != nil {
if err := writeFn(w, 0, totalSize, 0); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError) http.Error(w, err.Error(), http.StatusInternalServerError)
return return
} }
@ -279,7 +279,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10)) w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
w.Header().Set("Content-Range", ra.contentRange(totalSize)) w.Header().Set("Content-Range", ra.contentRange(totalSize))
err = writeFn(w, ra.start, ra.length)
err = writeFn(w, ra.start, ra.length, http.StatusPartialContent)
if err != nil { if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError) http.Error(w, err.Error(), http.StatusInternalServerError)
return return
@ -307,7 +307,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
pw.CloseWithError(e) pw.CloseWithError(e)
return return
} }
if e = writeFn(part, ra.start, ra.length); e != nil {
if e = writeFn(part, ra.start, ra.length, 0); e != nil {
pw.CloseWithError(e) pw.CloseWithError(e)
return return
} }

10
weed/server/filer_server_handlers_read.go

@ -131,9 +131,6 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
if r.Method == "HEAD" { if r.Method == "HEAD" {
w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size, true)
})
return return
} }
@ -153,7 +150,10 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
} }
} }
processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64, httpStatusCode int) error {
if httpStatusCode != 0 {
w.WriteHeader(httpStatusCode)
}
if offset+size <= int64(len(entry.Content)) { if offset+size <= int64(len(entry.Content)) {
_, err := writer.Write(entry.Content[offset : offset+size]) _, err := writer.Write(entry.Content[offset : offset+size])
if err != nil { if err != nil {
@ -161,7 +161,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
} }
return err return err
} }
return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size, false)
return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size)
}) })
} }

7
weed/server/volume_server_handlers_read.go

@ -27,7 +27,7 @@ var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)
func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
glog.V(9).Info(r.Method + " " + r.URL.Path + " " + r.Header.Get("Range"))
// println(r.Method + " " + r.URL.Path)
stats.VolumeServerRequestCounter.WithLabelValues("get").Inc() stats.VolumeServerRequestCounter.WithLabelValues("get").Inc()
start := time.Now() start := time.Now()
@ -261,10 +261,13 @@ func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.Re
return nil return nil
} }
processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64, httpStatusCode int) error {
if _, e = rs.Seek(offset, 0); e != nil { if _, e = rs.Seek(offset, 0); e != nil {
return e return e
} }
if httpStatusCode != 0 {
w.WriteHeader(httpStatusCode)
}
_, e = io.CopyN(writer, rs, size) _, e = io.CopyN(writer, rs, size)
return e return e
}) })

2
weed/shell/command_fs_cat.go

@ -52,7 +52,7 @@ func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Write
return err return err
} }
return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
}) })

Loading…
Cancel
Save