Merge 9a5ed08fab into a337b844ec

README.md

@@ -1,4 +1,4 @@
-# SeaweedFS
+# SeaweedFS CCCDDD
 
 [![Build Status](https://travis-ci.org/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.org/chrislusf/seaweedfs)
 [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)

weed/.gitignore

@@ -0,0 +1 @@
+/weed

weed/command/server.go

@@ -39,15 +39,11 @@ var cmdServer = &Command{
 	Short:     "start a server, including volume server, and automatically elect a master server",
 	Long: `start both a volume server to provide storage spaces
   and a master server to provide volume=>location mapping service and sequence number of file ids
-
   This is provided as a convenient way to start both volume server and master server.
   The servers are exactly the same as starting them separately.
-
   So other volume servers can use this embedded master server also.
-
   Optionally, one filer server can be started. Logically, filer servers should not be in a cluster.
   They run with meta data on disk, not shared. So each filer server is different.
-
   `,
 }
 

weed/command/volume.go

@@ -62,7 +62,6 @@ var cmdVolume = &Command{
 	UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
 	Short:     "start a volume server",
 	Long: `start a volume server to provide storage spaces
-
   `,
 }
 

weed/filer/cassandra_store/cassandra_store.go

@@ -11,20 +11,16 @@ import (
 )
 
 /*
-
 Basically you need a table just like this:
-
 CREATE TABLE seaweed_files (
    path varchar,
    fids list<varchar>,
    PRIMARY KEY (path)
 );
-
 Need to match flat_namespace.FlatNamespaceStore interface
 	Put(fullFileName string, fid string) (err error)
 	Get(fullFileName string) (fid string, err error)
 	Delete(fullFileName string) (fid string, err error)
-
 */
 type CassandraStore struct {
 	cluster *gocql.ClusterConfig

weed/filer/postgres_store/postgres_store.go

@@ -278,7 +278,6 @@ func (s *PostgresStore) FindFiles(dirPath string, lastFileName string, limit int
 }
 
 var createDirectoryTable = `
-
 CREATE TABLE IF NOT EXISTS %s (
   id BIGSERIAL NOT NULL,
   directoryRoot VARCHAR(1024) NOT NULL DEFAULT '',
@@ -288,7 +287,6 @@ CREATE TABLE IF NOT EXISTS %s (
 `
 
 var createFileTable = `
-
 CREATE TABLE IF NOT EXISTS %s (
   id BIGSERIAL NOT NULL,
   directoryPart VARCHAR(1024) NOT NULL DEFAULT '',

weed/images/resizing.go

@@ -16,6 +16,22 @@ func Resized(ext string, data []byte, width, height int) (resized []byte, w int,
 		return data, 0, 0
 	}
 	srcImage, _, err := image.Decode(bytes.NewReader(data))
+
+	dx := srcImage.Bounds().Dx()
+	dy := srcImage.Bounds().Dy()
+
+	if height == 0 {
+		height = width * dy / dx
+	} else if width == 0 {
+		width = height * dx / dy
+	} else {
+		if width/height > dx/dy { //定高
+			width = height * dx / dy
+		} else if width/height < dx/dy { //定宽
+			height = width * dy / dx
+		}
+	}
+
 	if err == nil {
 		bounds := srcImage.Bounds()
 		var dstImage *image.NRGBA

weed/images/rotate.go

@@ -0,0 +1,42 @@
+package images
+
+import (
+	"bytes"
+	"image"
+	"image/color"
+	"image/gif"
+	"image/jpeg"
+	"image/png"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/disintegration/gift"
+)
+
+func Rotate(ext string, data []byte, rotate int) (resized []byte) {
+	if rotate < 1 {
+		return data
+	}
+	srcImage, _, err := image.Decode(bytes.NewReader(data))
+	if err == nil {
+		var dstImage *image.NRGBA
+		g := gift.New(
+			gift.Rotate(float32(rotate), color.Opaque, gift.CubicInterpolation),
+		)
+		dstImage = image.NewNRGBA(g.Bounds(srcImage.Bounds()))
+		g.Draw(dstImage, srcImage)
+
+		var buf bytes.Buffer
+		switch ext {
+		case ".png":
+			png.Encode(&buf, dstImage)
+		case ".jpg", ".jpeg":
+			jpeg.Encode(&buf, dstImage, nil)
+		case ".gif":
+			gif.Encode(&buf, dstImage, nil)
+		}
+		return buf.Bytes()
+	} else {
+		glog.Error(err)
+	}
+	return data
+}

weed/server/filer_server_handlers_read.go

@@ -1,20 +1,28 @@
 package weed_server
 
 import (
+	"bytes"
+	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
+	"path"
 	"strconv"
 	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/security"
 	ui "github.com/chrislusf/seaweedfs/weed/server/filer_ui"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
+//七牛资源域名
+var resourceUrl string = "http://resource.k12cloud.cn"
+
 // listDirectoryHandler lists directories and folers under a directory
 // files are sorted by name and paginated via "lastFileName" and "limit".
 // sub directories are listed on the first page, when "lastFileName"
@@ -83,6 +91,10 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
 }
 
 func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) {
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("Access-Control-Allow-Methods", "GET")
+	w.Header().Set("Access-Control-Max-Age", "1000")
+
 	if strings.HasSuffix(r.URL.Path, "/") {
 		if fs.disableDirListing {
 			w.WriteHeader(http.StatusMethodNotAllowed)
@@ -92,11 +104,42 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 		return
 	}
 
+	//先在文件存储中查找该文件是否存在，不存在则去七牛查找，下载并上传到文件存储中
 	fileId, err := fs.filer.FindFile(r.URL.Path)
 	if err == filer.ErrNotFound {
-		glog.V(3).Infoln("Not found in db", r.URL.Path)
-		w.WriteHeader(http.StatusNotFound)
-		return
+		data, fileName, contentType, err := download(resourceUrl + r.URL.Path)
+		if err != nil {
+			glog.V(0).Infoln(err)
+			return
+		}
+
+		jwt := security.GetJwt(r)
+		_, err = operation.Upload("http://"+r.Host+r.URL.Path, fileName, bytes.NewReader(data), false, contentType, nil, jwt)
+		if err != nil {
+			glog.V(0).Infoln(err)
+			return
+		}
+		glog.V(0).Infoln("path", resourceUrl+r.URL.Path)
+	}
+
+	reqUrl := r.URL.RequestURI()
+	if r.FormValue("w") != "" || r.FormValue("h") != "" || r.FormValue("r") != "" {
+		reqUrl = r.URL.Path + "?w=" + r.FormValue("w") + "&h=" + r.FormValue("h") + "&r=" + r.FormValue("r")
+	}
+	fileId, err = fs.filer.FindFile(reqUrl)
+	if err == filer.ErrNotFound {
+		glog.V(0).Infoln(reqUrl, "not exist")
+		r.Header.Add("exist", "0")
+		r.Header.Add("path", r.URL.Path)
+		fileId, err = fs.filer.FindFile(r.URL.Path)
+		if err == filer.ErrNotFound {
+			glog.V(3).Infoln("Not found in db", r.URL.Path)
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+	} else {
+		glog.V(0).Infoln(reqUrl, "exist")
+		r.Header.Add("exist", "1")
 	}
 
 	urlLocation, err := operation.LookupFileId(fs.getMasterNode(), fileId)
@@ -144,3 +187,35 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 	io.Copy(w, resp.Body)
 
 }
+
+func download(u string) (data []byte, fileName, contentType string, err error) {
+	client := &http.Client{}
+	request, err := http.NewRequest("GET", u, nil)
+	if err != nil {
+		return
+	}
+
+	resp, err := client.Do(request)
+	if err != nil {
+		return
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		err = fmt.Errorf("%s", "下载失败")
+	}
+	data, err = ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+
+	contentType = resp.Header.Get("Content-type")
+
+	ur, err := url.Parse(u)
+	if err != nil {
+		return
+	}
+	_, fileName = path.Split(ur.Path)
+
+	return
+}

weed/server/filer_server_handlers_write.go

@@ -298,7 +298,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 		writeJsonError(w, r, http.StatusInternalServerError, errors.New(ret.Error))
 		return
 	}
-	path := r.URL.Path
+	path := r.URL.RequestURI()
 	if strings.HasSuffix(path, "/") {
 		if ret.Name != "" {
 			path += ret.Name

weed/server/volume_server_handlers_read.go

@@ -2,6 +2,10 @@ package weed_server
 
 import (
 	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
 	"io"
 	"mime"
 	"mime/multipart"
@@ -12,18 +16,27 @@ import (
 	"strings"
 	"time"
 
-	"encoding/json"
-
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/images"
 	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
 
+type JsonEncode struct {
+	Data   interface{} `json:"data"`
+	Msg    string      `json:"msg"`
+	Status int         `json:"status"`
+}
+
 func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("Access-Control-Allow-Methods", "GET")
+	w.Header().Set("Access-Control-Max-Age", "1000")
+
 	n := new(storage.Needle)
 	vid, fid, filename, ext, _ := parseURLPath(r.URL.Path)
 	volumeId, err := storage.NewVolumeId(vid)
@@ -124,7 +137,7 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 			mtype = mt
 		}
 	}
-
+	ext = strings.ToLower(ext) // 后缀先转小写，防止匹配不上大写的后缀
 	if ext != ".gz" {
 		if n.IsGzipped() {
 			if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
@@ -136,15 +149,51 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 			}
 		}
 	}
-	if ext == ".png" || ext == ".jpg" || ext == ".gif" {
-		width, height := 0, 0
-		if r.FormValue("width") != "" {
-			width, _ = strconv.Atoi(r.FormValue("width"))
+
+	if ext == ".png" || ext == ".jpg" || ext == ".jpeg" || ext == ".gif" {
+		width, height, rotate := 0, 0, 0
+		if r.FormValue("w") != "" {
+			width, _ = strconv.Atoi(r.FormValue("w"))
+		}
+		if r.FormValue("h") != "" {
+			height, _ = strconv.Atoi(r.FormValue("h"))
+		}
+		if r.FormValue("r") != "" {
+			rotate, _ = strconv.Atoi(r.FormValue("r"))
+		}
+
+		//缓存
+		if r.Header.Get("exist") != "1" && (r.FormValue("w") != "" || r.FormValue("h") != "" || r.FormValue("r") != "") {
+			glog.V(0).Infoln("生成")
+			n.Data, _, _ = images.Resized(ext, n.Data, width, height)
+			n.Data = images.Rotate(ext, n.Data, rotate)
+
+			reqUrl := r.Header.Get("path") + "?w=" + r.FormValue("w") + "&h=" + r.FormValue("h") + "&r=" + r.FormValue("r")
+			jwt := security.GetJwt(r)
+			_, err = operation.Upload("http://"+r.Host+reqUrl, filename, bytes.NewReader(n.Data), false, "image/jpeg", nil, jwt)
+			if err != nil {
+				glog.V(0).Infoln(err)
+			}
 		}
-		if r.FormValue("height") != "" {
-			height, _ = strconv.Atoi(r.FormValue("height"))
+	}
+
+	if r.FormValue("md5") != "" {
+		md5Ctx := md5.New()
+		md5Ctx.Write(n.Data)
+		cipherStr := md5Ctx.Sum(nil)
+		glog.V(0).Infoln("md5", hex.EncodeToString(cipherStr))
+
+		fileInfo := map[string]interface{}{
+			"url": r.Header.Get("path"),
+			"md5": hex.EncodeToString(cipherStr),
 		}
-		n.Data, _, _ = images.Resized(ext, n.Data, width, height)
+		fmt.Fprintf(w, "%v", (&JsonEncode{fileInfo, "success", 200}).ReturnJson())
+		return
+	}
+
+	//重命名
+	if r.FormValue("rename") != "" {
+		filename = r.FormValue("rename") + ext
 	}
 
 	if e := writeResponseContent(filename, mtype, bytes.NewReader(n.Data), w, r); e != nil {
@@ -311,3 +360,9 @@ func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.Re
 	_, e = io.CopyN(w, sendContent, sendSize)
 	return e
 }
+
+//返回json
+func (j *JsonEncode) ReturnJson() string {
+	b, _ := json.MarshalIndent(j, "", "	")
+	return string(b)
+}

weed/storage/needle_byte_cache.go

@@ -18,9 +18,7 @@ var (
 
 /*
 There are one level of caching, and one level of pooling.
-
 In pooling, all []byte are fetched and returned to the pool bytesPool.
-
 In caching, the string~[]byte mapping is cached
 */
 func init() {