AWS S3 Select feature now available

go.mod

@@ -10,10 +10,9 @@ require (
 	contrib.go.opencensus.io/resource v0.1.2 // indirect
 	github.com/Azure/azure-amqp-common-go v1.1.4 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.2 // indirect
-	github.com/Azure/azure-sdk-for-go v33.0.0+incompatible // indirect
 	github.com/Azure/azure-storage-blob-go v0.8.0
 	github.com/Azure/go-autorest v13.0.0+incompatible // indirect
-	github.com/Azure/go-autorest/tracing v0.5.0 // indirect
+	github.com/Azure/go-autorest/autorest v0.9.2 // indirect
 	github.com/DataDog/zstd v1.4.1 // indirect
 	github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190828224159-d93c53a4824c // indirect
 	github.com/Shopify/sarama v1.23.1
@@ -22,12 +21,10 @@ require (
 	github.com/aws/aws-sdk-go v1.23.13
 	github.com/chrislusf/raft v0.0.0-20190225081310-10d6e2182d92
 	github.com/coreos/etcd v3.3.15+incompatible // indirect
-	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/disintegration/imaging v1.6.1
 	github.com/dustin/go-humanize v1.0.0
-	github.com/eapache/go-resiliency v1.2.0 // indirect
 	github.com/gabriel-vasile/mimetype v0.3.17
 	github.com/go-kit/kit v0.9.0 // indirect
 	github.com/go-redis/redis v6.15.2+incompatible
@@ -39,28 +36,23 @@ require (
 	github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70 // indirect
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.3
-	github.com/gorilla/websocket v1.4.1 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
 	github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f
 	github.com/jcmturner/gofork v1.0.0 // indirect
 	github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9 // indirect
 	github.com/karlseguin/ccache v2.0.3+incompatible
 	github.com/karlseguin/expect v1.0.1 // indirect
-	github.com/klauspost/cpuid v1.2.1 // indirect
 	github.com/klauspost/crc32 v1.2.0
-	github.com/klauspost/reedsolomon v1.9.2
+	github.com/klauspost/reedsolomon v1.9.3
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/kr/pty v1.1.8 // indirect
-	github.com/kurin/blazer v0.5.3
+	github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3
 	github.com/lib/pq v1.2.0
 	github.com/magiconair/properties v1.8.1 // indirect
 	github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect
 	github.com/mattn/go-isatty v0.0.9 // indirect
-	github.com/mattn/go-runewidth v0.0.4 // indirect
+	github.com/minio/minio v0.0.0-20191122214605-0a56e33ce1b5
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
-	github.com/nats-io/gnatsd v1.4.1 // indirect
-	github.com/nats-io/go-nats v1.7.2 // indirect
-	github.com/nats-io/nats-server/v2 v2.0.4 // indirect
 	github.com/onsi/ginkgo v1.10.1 // indirect
 	github.com/onsi/gomega v1.7.0 // indirect
 	github.com/opentracing/opentracing-go v1.1.0 // indirect
@@ -81,7 +73,6 @@ require (
 	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
 	github.com/satori/go.uuid v1.2.0
 	github.com/seaweedfs/fuse v0.0.0-20190510212405-310228904eff
-	github.com/sirupsen/logrus v1.4.2 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.2.2 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
@@ -107,13 +98,12 @@ require (
 	golang.org/x/exp v0.0.0-20190829153037-c13cbed26979 // indirect
 	golang.org/x/image v0.0.0-20190829233526-b3c06291d021 // indirect
 	golang.org/x/mobile v0.0.0-20190830201351-c6da95954960 // indirect
-	golang.org/x/net v0.0.0-20190909003024-a7b16738d86b
-	golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b
+	golang.org/x/net v0.0.0-20190923162816-aa69164e4478
+	golang.org/x/sys v0.0.0-20190922100055-0a153f010e69
 	golang.org/x/tools v0.0.0-20190911022129-16c5e0f7d110
 	google.golang.org/api v0.9.0
 	google.golang.org/appengine v1.6.2 // indirect
 	google.golang.org/grpc v1.23.0
-	gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
 	gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
 	gopkg.in/karlseguin/expect.v1 v1.0.1 // indirect
 	honnef.co/go/tools v0.0.1-2019.2.2 // indirect

weed/s3api/s3api_object_handlers.go

@@ -4,6 +4,8 @@ import (
 	"crypto/md5"
 	"encoding/json"
 	"fmt"
+	"github.com/minio/minio/cmd"
+	"github.com/minio/minio/pkg/s3select"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -12,6 +14,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
 var (
@@ -224,3 +227,98 @@ func getObject(vars map[string]string) string {
 	}
 	return object
 }
+
+func (s3a *S3ApiServer) SelectObjectContent(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := getObject(vars)
+
+	s3Select, err := s3select.NewS3Select(r.Body)
+	if err != nil {
+		if serr, ok := err.(s3select.SelectError); ok {
+			encodedErrorResponse := encodeResponse(cmd.APIErrorResponse{
+				Code: serr.ErrorCode(),
+				Message: serr.ErrorMessage(),
+				BucketName: bucket,
+				Key: object,
+				Resource: r.URL.Path,
+				RequestID: w.Header().Get(xhttp.AmzRequestID),
+				HostID: "",
+			})
+			writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, "application/xml")
+		} else {
+			writeErrorResponse(w, ErrInternalError, r.URL)
+		}
+		return
+	}
+
+	if strings.HasSuffix(r.URL.Path, "/") {
+		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		return
+	}
+
+	destUrl := fmt.Sprintf("http://%s%s/%s%s",
+		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)
+	passThroughResponseSelectObjectContent := func(proxyResonse *http.Response,
+		w http.ResponseWriter) {
+		getObject := func(offset, length int64) (io.ReadCloser, error) {
+			return proxyResonse.Body, nil
+		}
+		if err = s3Select.Open(getObject); err != nil {
+			if serr, ok := err.(s3select.SelectError); ok {
+				encodedErrorResponse := encodeResponse(cmd.APIErrorResponse{
+					Code:       serr.ErrorCode(),
+					Message:    serr.ErrorMessage(),
+					BucketName: bucket,
+					Key:        object,
+					Resource:   r.URL.Path,
+					RequestID:  w.Header().Get(xhttp.AmzRequestID),
+					HostID:     "",
+				})
+				writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML)
+			} else {
+				writeResponse(w, http.StatusInternalServerError, encodeResponse("not s3select.SelectError"), mimeXML)
+			}
+			return
+		}
+
+		s3Select.Evaluate(w)
+		s3Select.Close()
+	}
+	s3a.proxyToFilerS3Select(w, r, destUrl, passThroughResponseSelectObjectContent)
+}
+
+func (s3a *S3ApiServer) proxyToFilerS3Select(w http.ResponseWriter, r *http.Request, destUrl string,
+	responseFn func(proxyResonse *http.Response, w http.ResponseWriter)) {
+
+	glog.V(2).Infof("s3 proxying %s to %s", "GET", destUrl)
+
+	proxyReq, err := http.NewRequest("GET", destUrl, nil)
+
+	if err != nil {
+		glog.Errorf("NewRequest %s: %v", destUrl, err)
+		writeErrorResponse(w, ErrInternalError, r.URL)
+		return
+	}
+
+	proxyReq.Header.Set("Host", s3a.option.Filer)
+	proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr)
+	proxyReq.Header.Set("Etag-MD5", "True")
+
+	for header, values := range r.Header {
+		for _, value := range values {
+			proxyReq.Header.Add(header, value)
+		}
+	}
+
+	resp, postErr := client.Do(proxyReq)
+
+	if postErr != nil {
+		glog.Errorf("post to filer: %v", postErr)
+		writeErrorResponse(w, ErrInternalError, r.URL)
+		return
+	}
+	defer resp.Body.Close()
+
+	responseFn(resp, w)
+}

weed/s3api/s3api_server.go

@@ -79,6 +79,8 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.GetObjectHandler)
 		// ListObjectsV1 (Legacy)
 		bucket.Methods("GET").HandlerFunc(s3a.ListObjectsV1Handler)
+		// SelectObjectContent
+		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(s3a.SelectObjectContent).Queries("select", "", "select-type", "2")
 
 		// DeleteMultipleObjects
 		bucket.Methods("POST").HandlerFunc(s3a.DeleteMultipleObjectsHandler).Queries("delete", "")