diff --git a/README.md b/README.md index cbedafe1e..7e046812b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# SeaweedFS +# SeaweedFS CCCDDD [![Build Status](https://travis-ci.org/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.org/chrislusf/seaweedfs) [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed) diff --git a/weed/.gitignore b/weed/.gitignore new file mode 100644 index 000000000..aa3146581 --- /dev/null +++ b/weed/.gitignore @@ -0,0 +1 @@ +/weed diff --git a/weed/command/server.go b/weed/command/server.go index e1152f23f..409feddb9 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -39,15 +39,11 @@ var cmdServer = &Command{ Short: "start a server, including volume server, and automatically elect a master server", Long: `start both a volume server to provide storage spaces and a master server to provide volume=>location mapping service and sequence number of file ids - This is provided as a convenient way to start both volume server and master server. The servers are exactly the same as starting them separately. - So other volume servers can use this embedded master server also. - Optionally, one filer server can be started. Logically, filer servers should not be in a cluster. They run with meta data on disk, not shared. So each filer server is different. - `, } diff --git a/weed/command/volume.go b/weed/command/volume.go index 0e69325b6..0ab59ee0d 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -62,7 +62,6 @@ var cmdVolume = &Command{ UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333", Short: "start a volume server", Long: `start a volume server to provide storage spaces - `, } diff --git a/weed/filer/cassandra_store/cassandra_store.go b/weed/filer/cassandra_store/cassandra_store.go index 75af48bcd..b58c7779b 100644 --- a/weed/filer/cassandra_store/cassandra_store.go +++ b/weed/filer/cassandra_store/cassandra_store.go @@ -11,20 +11,16 @@ import ( ) /* - Basically you need a table just like this: - CREATE TABLE seaweed_files ( path varchar, fids list, PRIMARY KEY (path) ); - Need to match flat_namespace.FlatNamespaceStore interface Put(fullFileName string, fid string) (err error) Get(fullFileName string) (fid string, err error) Delete(fullFileName string) (fid string, err error) - */ type CassandraStore struct { cluster *gocql.ClusterConfig diff --git a/weed/filer/postgres_store/postgres_store.go b/weed/filer/postgres_store/postgres_store.go index 68a6298ee..270f267dd 100644 --- a/weed/filer/postgres_store/postgres_store.go +++ b/weed/filer/postgres_store/postgres_store.go @@ -278,7 +278,6 @@ func (s *PostgresStore) FindFiles(dirPath string, lastFileName string, limit int } var createDirectoryTable = ` - CREATE TABLE IF NOT EXISTS %s ( id BIGSERIAL NOT NULL, directoryRoot VARCHAR(1024) NOT NULL DEFAULT '', @@ -288,7 +287,6 @@ CREATE TABLE IF NOT EXISTS %s ( ` var createFileTable = ` - CREATE TABLE IF NOT EXISTS %s ( id BIGSERIAL NOT NULL, directoryPart VARCHAR(1024) NOT NULL DEFAULT '', @@ -620,4 +618,4 @@ func (s *PostgresStore) findFiles(dirPath string, lastFileName string, limit int dirPath, len(files), limit, lastFileName) return files, err -} +} \ No newline at end of file diff --git a/weed/images/resizing.go b/weed/images/resizing.go index 7e4a88c42..0e4a7b9bc 100644 --- a/weed/images/resizing.go +++ b/weed/images/resizing.go @@ -16,6 +16,22 @@ func Resized(ext string, data []byte, width, height int) (resized []byte, w int, return data, 0, 0 } srcImage, _, err := image.Decode(bytes.NewReader(data)) + + dx := srcImage.Bounds().Dx() + dy := srcImage.Bounds().Dy() + + if height == 0 { + height = width * dy / dx + } else if width == 0 { + width = height * dx / dy + } else { + if width/height > dx/dy { //定高 + width = height * dx / dy + } else if width/height < dx/dy { //定宽 + height = width * dy / dx + } + } + if err == nil { bounds := srcImage.Bounds() var dstImage *image.NRGBA diff --git a/weed/images/rotate.go b/weed/images/rotate.go new file mode 100644 index 000000000..9e26d602c --- /dev/null +++ b/weed/images/rotate.go @@ -0,0 +1,42 @@ +package images + +import ( + "bytes" + "image" + "image/color" + "image/gif" + "image/jpeg" + "image/png" + + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/disintegration/gift" +) + +func Rotate(ext string, data []byte, rotate int) (resized []byte) { + if rotate < 1 { + return data + } + srcImage, _, err := image.Decode(bytes.NewReader(data)) + if err == nil { + var dstImage *image.NRGBA + g := gift.New( + gift.Rotate(float32(rotate), color.Opaque, gift.CubicInterpolation), + ) + dstImage = image.NewNRGBA(g.Bounds(srcImage.Bounds())) + g.Draw(dstImage, srcImage) + + var buf bytes.Buffer + switch ext { + case ".png": + png.Encode(&buf, dstImage) + case ".jpg", ".jpeg": + jpeg.Encode(&buf, dstImage, nil) + case ".gif": + gif.Encode(&buf, dstImage, nil) + } + return buf.Bytes() + } else { + glog.Error(err) + } + return data +} diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index e95c7fcd0..0af1cf341 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -1,20 +1,28 @@ package weed_server import ( + "bytes" + "fmt" "io" + "io/ioutil" "net/http" "net/url" + "path" "strconv" "strings" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/security" ui "github.com/chrislusf/seaweedfs/weed/server/filer_ui" "github.com/chrislusf/seaweedfs/weed/util" "github.com/syndtr/goleveldb/leveldb" ) +//七牛资源域名 +var resourceUrl string = "http://resource.k12cloud.cn" + // listDirectoryHandler lists directories and folers under a directory // files are sorted by name and paginated via "lastFileName" and "limit". // sub directories are listed on the first page, when "lastFileName" @@ -83,6 +91,10 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque } func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET") + w.Header().Set("Access-Control-Max-Age", "1000") + if strings.HasSuffix(r.URL.Path, "/") { if fs.disableDirListing { w.WriteHeader(http.StatusMethodNotAllowed) @@ -92,11 +104,42 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, return } + //先在文件存储中查找该文件是否存在,不存在则去七牛查找,下载并上传到文件存储中 fileId, err := fs.filer.FindFile(r.URL.Path) if err == filer.ErrNotFound { - glog.V(3).Infoln("Not found in db", r.URL.Path) - w.WriteHeader(http.StatusNotFound) - return + data, fileName, contentType, err := download(resourceUrl + r.URL.Path) + if err != nil { + glog.V(0).Infoln(err) + return + } + + jwt := security.GetJwt(r) + _, err = operation.Upload("http://"+r.Host+r.URL.Path, fileName, bytes.NewReader(data), false, contentType, nil, jwt) + if err != nil { + glog.V(0).Infoln(err) + return + } + glog.V(0).Infoln("path", resourceUrl+r.URL.Path) + } + + reqUrl := r.URL.RequestURI() + if r.FormValue("w") != "" || r.FormValue("h") != "" || r.FormValue("r") != "" { + reqUrl = r.URL.Path + "?w=" + r.FormValue("w") + "&h=" + r.FormValue("h") + "&r=" + r.FormValue("r") + } + fileId, err = fs.filer.FindFile(reqUrl) + if err == filer.ErrNotFound { + glog.V(0).Infoln(reqUrl, "not exist") + r.Header.Add("exist", "0") + r.Header.Add("path", r.URL.Path) + fileId, err = fs.filer.FindFile(r.URL.Path) + if err == filer.ErrNotFound { + glog.V(3).Infoln("Not found in db", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + } else { + glog.V(0).Infoln(reqUrl, "exist") + r.Header.Add("exist", "1") } urlLocation, err := operation.LookupFileId(fs.getMasterNode(), fileId) @@ -144,3 +187,35 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, io.Copy(w, resp.Body) } + +func download(u string) (data []byte, fileName, contentType string, err error) { + client := &http.Client{} + request, err := http.NewRequest("GET", u, nil) + if err != nil { + return + } + + resp, err := client.Do(request) + if err != nil { + return + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + err = fmt.Errorf("%s", "下载失败") + } + data, err = ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + contentType = resp.Header.Get("Content-type") + + ur, err := url.Parse(u) + if err != nil { + return + } + _, fileName = path.Split(ur.Path) + + return +} diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 07452cd77..145d0936a 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -298,7 +298,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { writeJsonError(w, r, http.StatusInternalServerError, errors.New(ret.Error)) return } - path := r.URL.Path + path := r.URL.RequestURI() if strings.HasSuffix(path, "/") { if ret.Name != "" { path += ret.Name diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index 2e33b415c..9b6cd9f71 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -2,6 +2,10 @@ package weed_server import ( "bytes" + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" "io" "mime" "mime/multipart" @@ -12,18 +16,27 @@ import ( "strings" "time" - "encoding/json" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/images" "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" ) var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") +type JsonEncode struct { + Data interface{} `json:"data"` + Msg string `json:"msg"` + Status int `json:"status"` +} + func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET") + w.Header().Set("Access-Control-Max-Age", "1000") + n := new(storage.Needle) vid, fid, filename, ext, _ := parseURLPath(r.URL.Path) volumeId, err := storage.NewVolumeId(vid) @@ -124,7 +137,7 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) mtype = mt } } - + ext = strings.ToLower(ext) // 后缀先转小写,防止匹配不上大写的后缀 if ext != ".gz" { if n.IsGzipped() { if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { @@ -136,15 +149,51 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) } } } - if ext == ".png" || ext == ".jpg" || ext == ".gif" { - width, height := 0, 0 - if r.FormValue("width") != "" { - width, _ = strconv.Atoi(r.FormValue("width")) + + if ext == ".png" || ext == ".jpg" || ext == ".jpeg" || ext == ".gif" { + width, height, rotate := 0, 0, 0 + if r.FormValue("w") != "" { + width, _ = strconv.Atoi(r.FormValue("w")) + } + if r.FormValue("h") != "" { + height, _ = strconv.Atoi(r.FormValue("h")) + } + if r.FormValue("r") != "" { + rotate, _ = strconv.Atoi(r.FormValue("r")) + } + + //缓存 + if r.Header.Get("exist") != "1" && (r.FormValue("w") != "" || r.FormValue("h") != "" || r.FormValue("r") != "") { + glog.V(0).Infoln("生成") + n.Data, _, _ = images.Resized(ext, n.Data, width, height) + n.Data = images.Rotate(ext, n.Data, rotate) + + reqUrl := r.Header.Get("path") + "?w=" + r.FormValue("w") + "&h=" + r.FormValue("h") + "&r=" + r.FormValue("r") + jwt := security.GetJwt(r) + _, err = operation.Upload("http://"+r.Host+reqUrl, filename, bytes.NewReader(n.Data), false, "image/jpeg", nil, jwt) + if err != nil { + glog.V(0).Infoln(err) + } } - if r.FormValue("height") != "" { - height, _ = strconv.Atoi(r.FormValue("height")) + } + + if r.FormValue("md5") != "" { + md5Ctx := md5.New() + md5Ctx.Write(n.Data) + cipherStr := md5Ctx.Sum(nil) + glog.V(0).Infoln("md5", hex.EncodeToString(cipherStr)) + + fileInfo := map[string]interface{}{ + "url": r.Header.Get("path"), + "md5": hex.EncodeToString(cipherStr), } - n.Data, _, _ = images.Resized(ext, n.Data, width, height) + fmt.Fprintf(w, "%v", (&JsonEncode{fileInfo, "success", 200}).ReturnJson()) + return + } + + //重命名 + if r.FormValue("rename") != "" { + filename = r.FormValue("rename") + ext } if e := writeResponseContent(filename, mtype, bytes.NewReader(n.Data), w, r); e != nil { @@ -311,3 +360,9 @@ func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.Re _, e = io.CopyN(w, sendContent, sendSize) return e } + +//返回json +func (j *JsonEncode) ReturnJson() string { + b, _ := json.MarshalIndent(j, "", " ") + return string(b) +} diff --git a/weed/storage/needle_byte_cache.go b/weed/storage/needle_byte_cache.go index dfc32bcbf..8b6c54a55 100644 --- a/weed/storage/needle_byte_cache.go +++ b/weed/storage/needle_byte_cache.go @@ -18,9 +18,7 @@ var ( /* There are one level of caching, and one level of pooling. - In pooling, all []byte are fetched and returned to the pool bytesPool. - In caching, the string~[]byte mapping is cached */ func init() {