From 87fee21ef597a8b1bac5352d1327c13f87eeb000 Mon Sep 17 00:00:00 2001 From: Mike Tolman Date: Fri, 5 Aug 2016 15:14:24 -0600 Subject: [PATCH 01/31] Changing needle_byte_cache so that it doesn't grow so big when larger files are added. --- weed/storage/needle_byte_cache.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/weed/storage/needle_byte_cache.go b/weed/storage/needle_byte_cache.go index ae35a48ba..930ead81d 100644 --- a/weed/storage/needle_byte_cache.go +++ b/weed/storage/needle_byte_cache.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/golang-lru" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/glog" ) var ( @@ -24,7 +25,7 @@ In caching, the string~[]byte mapping is cached */ func init() { bytesPool = util.NewBytesPool() - bytesCache, _ = lru.NewWithEvict(512, func(key interface{}, value interface{}) { + bytesCache, _ = lru.NewWithEvict(50, func(key interface{}, value interface{}) { value.(*Block).decreaseReference() }) } @@ -46,22 +47,37 @@ func (block *Block) increaseReference() { // get bytes from the LRU cache of []byte first, then from the bytes pool // when []byte in LRU cache is evicted, it will be put back to the bytes pool func getBytesForFileBlock(r *os.File, offset int64, readSize int) (dataSlice []byte, block *Block, err error) { + //Skip the cache if we are looking for a block that is too big to fit in the cache (defaulting to 10MB) + cacheable := readSize <= (1024*1024*10) + if !cacheable { + glog.V(4).Infoln("Block too big to keep in cache. Size:", readSize) + } + cacheKey := string("") + if cacheable { // check cache, return if found - cacheKey := fmt.Sprintf("%d:%d:%d", r.Fd(), offset>>3, readSize) + cacheKey = fmt.Sprintf("%d:%d:%d", r.Fd(), offset >> 3, readSize) if obj, found := bytesCache.Get(cacheKey); found { + glog.V(4).Infoln("Found block in cache. Size:", readSize) block = obj.(*Block) block.increaseReference() dataSlice = block.Bytes[0:readSize] return dataSlice, block, nil + } } // get the []byte from pool b := bytesPool.Get(readSize) // refCount = 2, one by the bytesCache, one by the actual needle object - block = &Block{Bytes: b, refCount: 2} + refCount := int32(1) + if cacheable { + refCount = 2 + } + block = &Block{Bytes: b, refCount: refCount} dataSlice = block.Bytes[0:readSize] _, err = r.ReadAt(dataSlice, offset) + if cacheable { bytesCache.Add(cacheKey, block) + } return dataSlice, block, err } From b6ce40e87f11c1f85f6285345edd701a4508ca4b Mon Sep 17 00:00:00 2001 From: Mike Tolman Date: Fri, 5 Aug 2016 15:16:17 -0600 Subject: [PATCH 02/31] Add AutoChunking to the Filer API, so that you can upload really large files through the filer API. --- weed/server/filer_server_handlers_write.go | 208 +++++++++++++++++++++ 1 file changed, 208 insertions(+) diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index e2d40f532..872d8c4b9 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -20,6 +20,8 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" "github.com/syndtr/goleveldb/leveldb" + "path" + "strconv" ) type FilerPostResult struct { @@ -217,6 +219,7 @@ func (fs *FilerServer) monolithicUploadAnalyzer(w http.ResponseWriter, r *http.R } func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() replication := query.Get("replication") if replication == "" { @@ -227,6 +230,10 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { collection = fs.collection } + if autoChunked := fs.autoChunk(w, r, replication, collection); autoChunked { + return + } + var fileId, urlLocation string var err error @@ -243,7 +250,17 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { } u, _ := url.Parse(urlLocation) + + // This allows a client to generate a chunk manifest and submit it to the filer -- it is a little off + // because they need to provide FIDs instead of file paths... + cm, _ := strconv.ParseBool(query.Get("cm")) + if cm { + q := u.Query() + q.Set("cm", "true") + u.RawQuery = q.Encode() + } glog.V(4).Infoln("post to", u) + request := &http.Request{ Method: r.Method, URL: u, @@ -319,6 +336,197 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { writeJsonQuiet(w, r, http.StatusCreated, reply) } +func (fs *FilerServer) autoChunk(w http.ResponseWriter, r *http.Request, replication string, collection string) bool { + if r.Method != "POST" { + glog.V(4).Infoln("AutoChunking not supported for method", r.Method) + return false + } + + // autoChunking can be set at the command-line level or as a query param. Query param overrides command-line + query := r.URL.Query() + + parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32) + maxMB := int32(parsedMaxMB) + if maxMB <= 0 && fs.maxMB > 0 { + maxMB = int32(fs.maxMB) + } + if maxMB <= 0 { + glog.V(4).Infoln("AutoChunking not enabled") + return false + } + glog.V(4).Infoln("AutoChunking level set to", maxMB, "(MB)") + + chunkSize := 1024 * 1024 * maxMB + + contentLength := int64(0) + if contentLengthHeader := r.Header["Content-Length"]; len(contentLengthHeader) == 1 { + contentLength, _ = strconv.ParseInt(contentLengthHeader[0], 10, 64) + if contentLength <= int64(chunkSize) { + glog.V(4).Infoln("Content-Length of", contentLength, "is less than the chunk size of", chunkSize, "so autoChunking will be skipped.") + return false + } + } + + if contentLength <= 0 { + glog.V(4).Infoln("Content-Length value is missing or unexpected so autoChunking will be skipped.") + return false + } + + reply, err := fs.doAutoChunk(w, r, contentLength, chunkSize, replication, collection) + if err != nil { + writeJsonError(w, r, http.StatusInternalServerError, err) + } else if reply != nil { + writeJsonQuiet(w, r, http.StatusCreated, reply) + } + return true +} + +func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, contentLength int64, chunkSize int32, replication string, collection string) (filerResult *FilerPostResult, replyerr error) { + + multipartReader, multipartReaderErr := r.MultipartReader() + if multipartReaderErr != nil { + return nil, multipartReaderErr + } + + part1, part1Err := multipartReader.NextPart() + if part1Err != nil { + return nil, part1Err + } + + fileName := part1.FileName() + if fileName != "" { + fileName = path.Base(fileName) + } + + chunks := (int64(contentLength) / int64(chunkSize)) + 1 + cm := operation.ChunkManifest{ + Name: fileName, + Size: 0, // don't know yet + Mime: "application/octet-stream", + Chunks: make([]*operation.ChunkInfo, 0, chunks), + } + + totalBytesRead := int64(0) + tmpBufferSize := int32(1024 * 1024) + tmpBuffer := bytes.NewBuffer(make([]byte, 0, tmpBufferSize)) + chunkBuf := make([]byte, chunkSize+tmpBufferSize, chunkSize+tmpBufferSize) // chunk size plus a little overflow + chunkBufOffset := int32(0) + chunkOffset := int64(0) + writtenChunks := 0 + + filerResult = &FilerPostResult{ + Name: fileName, + } + + for totalBytesRead < contentLength { + tmpBuffer.Reset() + bytesRead, readErr := io.CopyN(tmpBuffer, part1, int64(tmpBufferSize)) + readFully := readErr != nil && readErr == io.EOF + tmpBuf := tmpBuffer.Bytes() + bytesToCopy := tmpBuf[0:int(bytesRead)] + + copy(chunkBuf[chunkBufOffset:chunkBufOffset+int32(bytesRead)], bytesToCopy) + chunkBufOffset = chunkBufOffset + int32(bytesRead) + + if chunkBufOffset >= chunkSize || readFully || (chunkBufOffset > 0 && bytesRead == 0) { + writtenChunks = writtenChunks + 1 + fileId, urlLocation, assignErr := fs.assignNewFileInfo(w, r, replication, collection) + if assignErr != nil { + return nil, assignErr + } + + // upload the chunk to the volume server + chunkName := fileName + "_chunk_" + strconv.FormatInt(int64(cm.Chunks.Len()+1), 10) + uploadErr := fs.doUpload(urlLocation, w, r, chunkBuf[0:chunkBufOffset], chunkName, "application/octet-stream", fileId) + if uploadErr != nil { + return nil, uploadErr + } + + // Save to chunk manifest structure + cm.Chunks = append(cm.Chunks, + &operation.ChunkInfo{ + Offset: chunkOffset, + Size: int64(chunkBufOffset), + Fid: fileId, + }, + ) + + // reset variables for the next chunk + chunkBufOffset = 0 + chunkOffset = totalBytesRead + int64(bytesRead) + } + + totalBytesRead = totalBytesRead + int64(bytesRead) + + if bytesRead == 0 || readFully { + break + } + + if readErr != nil { + return nil, readErr + } + } + + cm.Size = totalBytesRead + manifestBuf, marshalErr := cm.Marshal() + if marshalErr != nil { + return nil, marshalErr + } + + manifestStr := string(manifestBuf) + glog.V(4).Infoln("Generated chunk manifest: ", manifestStr) + + manifestFileId, manifestUrlLocation, manifestAssignmentErr := fs.assignNewFileInfo(w, r, replication, collection) + if manifestAssignmentErr != nil { + return nil, manifestAssignmentErr + } + glog.V(4).Infoln("Manifest uploaded to:", manifestUrlLocation, "Fid:", manifestFileId) + filerResult.Fid = manifestFileId + + u, _ := url.Parse(manifestUrlLocation) + q := u.Query() + q.Set("cm", "true") + u.RawQuery = q.Encode() + + manifestUploadErr := fs.doUpload(u.String(), w, r, manifestBuf, fileName+"_manifest", "application/json", manifestFileId) + if manifestUploadErr != nil { + return nil, manifestUploadErr + } + + path := r.URL.Path + // also delete the old fid unless PUT operation + if r.Method != "PUT" { + if oldFid, err := fs.filer.FindFile(path); err == nil { + operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) + } + } + + glog.V(4).Infoln("saving", path, "=>", manifestFileId) + if db_err := fs.filer.CreateFile(path, manifestFileId); db_err != nil { + replyerr = db_err + filerResult.Error = db_err.Error() + operation.DeleteFile(fs.getMasterNode(), manifestFileId, fs.jwt(manifestFileId)) //clean up + glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err) + return + } + + return +} + +func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, chunkBuf []byte, fileName string, contentType string, fileId string) (err error) { + err = nil + + ioReader := ioutil.NopCloser(bytes.NewBuffer(chunkBuf)) + uploadResult, uploadError := operation.Upload(urlLocation, fileName, ioReader, false, contentType, fs.jwt(fileId)) + if uploadResult != nil { + glog.V(0).Infoln("Chunk upload result. Name:", uploadResult.Name, "Fid:", fileId, "Size:", uploadResult.Size) + } + if uploadError != nil { + err = uploadError + } + return +} + // curl -X DELETE http://localhost:8888/path/to // curl -X DELETE http://localhost:8888/path/to?recursive=true func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { From 09059bfdccdeff1a588ee1326318075adb068b0f Mon Sep 17 00:00:00 2001 From: Mike Tolman Date: Fri, 5 Aug 2016 15:19:10 -0600 Subject: [PATCH 03/31] Add AutoChunking to the Filer API, so that you can upload really large files through the filer API. --- weed/command/filer.go | 3 + weed/command/server.go | 2 + weed/server/filer_server.go | 2 + weed/server/filer_server_handlers_write.go | 208 +++++++++++++++++++++ 4 files changed, 215 insertions(+) diff --git a/weed/command/filer.go b/weed/command/filer.go index 582d4e9c8..0bd508e0b 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -24,6 +24,7 @@ type FilerOptions struct { dir *string redirectOnRead *bool disableDirListing *bool + maxMB *int secretKey *string cassandra_server *string cassandra_keyspace *string @@ -42,6 +43,7 @@ func init() { f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified") f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing") + f.maxMB = cmdFiler.Flag.Int("maxMB", 0, "split files larger than the limit") f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server") f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server") f.redis_server = cmdFiler.Flag.String("redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379") @@ -82,6 +84,7 @@ func runFiler(cmd *Command, args []string) bool { r := http.NewServeMux() _, nfs_err := weed_server.NewFilerServer(r, *f.ip, *f.port, *f.master, *f.dir, *f.collection, *f.defaultReplicaPlacement, *f.redirectOnRead, *f.disableDirListing, + *f.maxMB, *f.secretKey, *f.cassandra_server, *f.cassandra_keyspace, *f.redis_server, *f.redis_password, *f.redis_database, diff --git a/weed/command/server.go b/weed/command/server.go index 1211c7137..7a6677a65 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -86,6 +86,7 @@ func init() { filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.") filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") + filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 0, "split files larger than the limit") filerOptions.cassandra_server = cmdServer.Flag.String("filer.cassandra.server", "", "host[:port] of the cassandra server") filerOptions.cassandra_keyspace = cmdServer.Flag.String("filer.cassandra.keyspace", "seaweed", "keyspace of the cassandra server") filerOptions.redis_server = cmdServer.Flag.String("filer.redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379") @@ -169,6 +170,7 @@ func runServer(cmd *Command, args []string) bool { _, nfs_err := weed_server.NewFilerServer(r, *serverBindIp, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection, *filerOptions.defaultReplicaPlacement, *filerOptions.redirectOnRead, *filerOptions.disableDirListing, + *filerOptions.maxMB, *filerOptions.secretKey, *filerOptions.cassandra_server, *filerOptions.cassandra_keyspace, *filerOptions.redis_server, *filerOptions.redis_password, *filerOptions.redis_database, diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index b99bbd7c9..c9bc0e021 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -28,6 +28,7 @@ type FilerServer struct { disableDirListing bool secret security.Secret filer filer.Filer + maxMB int masterNodes *storage.MasterNodes } @@ -43,6 +44,7 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st defaultReplication: replication, redirectOnRead: redirectOnRead, disableDirListing: disableDirListing, + maxMB: maxMB, port: ip + ":" + strconv.Itoa(port), } diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index e2d40f532..872d8c4b9 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -20,6 +20,8 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" "github.com/syndtr/goleveldb/leveldb" + "path" + "strconv" ) type FilerPostResult struct { @@ -217,6 +219,7 @@ func (fs *FilerServer) monolithicUploadAnalyzer(w http.ResponseWriter, r *http.R } func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() replication := query.Get("replication") if replication == "" { @@ -227,6 +230,10 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { collection = fs.collection } + if autoChunked := fs.autoChunk(w, r, replication, collection); autoChunked { + return + } + var fileId, urlLocation string var err error @@ -243,7 +250,17 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { } u, _ := url.Parse(urlLocation) + + // This allows a client to generate a chunk manifest and submit it to the filer -- it is a little off + // because they need to provide FIDs instead of file paths... + cm, _ := strconv.ParseBool(query.Get("cm")) + if cm { + q := u.Query() + q.Set("cm", "true") + u.RawQuery = q.Encode() + } glog.V(4).Infoln("post to", u) + request := &http.Request{ Method: r.Method, URL: u, @@ -319,6 +336,197 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { writeJsonQuiet(w, r, http.StatusCreated, reply) } +func (fs *FilerServer) autoChunk(w http.ResponseWriter, r *http.Request, replication string, collection string) bool { + if r.Method != "POST" { + glog.V(4).Infoln("AutoChunking not supported for method", r.Method) + return false + } + + // autoChunking can be set at the command-line level or as a query param. Query param overrides command-line + query := r.URL.Query() + + parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32) + maxMB := int32(parsedMaxMB) + if maxMB <= 0 && fs.maxMB > 0 { + maxMB = int32(fs.maxMB) + } + if maxMB <= 0 { + glog.V(4).Infoln("AutoChunking not enabled") + return false + } + glog.V(4).Infoln("AutoChunking level set to", maxMB, "(MB)") + + chunkSize := 1024 * 1024 * maxMB + + contentLength := int64(0) + if contentLengthHeader := r.Header["Content-Length"]; len(contentLengthHeader) == 1 { + contentLength, _ = strconv.ParseInt(contentLengthHeader[0], 10, 64) + if contentLength <= int64(chunkSize) { + glog.V(4).Infoln("Content-Length of", contentLength, "is less than the chunk size of", chunkSize, "so autoChunking will be skipped.") + return false + } + } + + if contentLength <= 0 { + glog.V(4).Infoln("Content-Length value is missing or unexpected so autoChunking will be skipped.") + return false + } + + reply, err := fs.doAutoChunk(w, r, contentLength, chunkSize, replication, collection) + if err != nil { + writeJsonError(w, r, http.StatusInternalServerError, err) + } else if reply != nil { + writeJsonQuiet(w, r, http.StatusCreated, reply) + } + return true +} + +func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, contentLength int64, chunkSize int32, replication string, collection string) (filerResult *FilerPostResult, replyerr error) { + + multipartReader, multipartReaderErr := r.MultipartReader() + if multipartReaderErr != nil { + return nil, multipartReaderErr + } + + part1, part1Err := multipartReader.NextPart() + if part1Err != nil { + return nil, part1Err + } + + fileName := part1.FileName() + if fileName != "" { + fileName = path.Base(fileName) + } + + chunks := (int64(contentLength) / int64(chunkSize)) + 1 + cm := operation.ChunkManifest{ + Name: fileName, + Size: 0, // don't know yet + Mime: "application/octet-stream", + Chunks: make([]*operation.ChunkInfo, 0, chunks), + } + + totalBytesRead := int64(0) + tmpBufferSize := int32(1024 * 1024) + tmpBuffer := bytes.NewBuffer(make([]byte, 0, tmpBufferSize)) + chunkBuf := make([]byte, chunkSize+tmpBufferSize, chunkSize+tmpBufferSize) // chunk size plus a little overflow + chunkBufOffset := int32(0) + chunkOffset := int64(0) + writtenChunks := 0 + + filerResult = &FilerPostResult{ + Name: fileName, + } + + for totalBytesRead < contentLength { + tmpBuffer.Reset() + bytesRead, readErr := io.CopyN(tmpBuffer, part1, int64(tmpBufferSize)) + readFully := readErr != nil && readErr == io.EOF + tmpBuf := tmpBuffer.Bytes() + bytesToCopy := tmpBuf[0:int(bytesRead)] + + copy(chunkBuf[chunkBufOffset:chunkBufOffset+int32(bytesRead)], bytesToCopy) + chunkBufOffset = chunkBufOffset + int32(bytesRead) + + if chunkBufOffset >= chunkSize || readFully || (chunkBufOffset > 0 && bytesRead == 0) { + writtenChunks = writtenChunks + 1 + fileId, urlLocation, assignErr := fs.assignNewFileInfo(w, r, replication, collection) + if assignErr != nil { + return nil, assignErr + } + + // upload the chunk to the volume server + chunkName := fileName + "_chunk_" + strconv.FormatInt(int64(cm.Chunks.Len()+1), 10) + uploadErr := fs.doUpload(urlLocation, w, r, chunkBuf[0:chunkBufOffset], chunkName, "application/octet-stream", fileId) + if uploadErr != nil { + return nil, uploadErr + } + + // Save to chunk manifest structure + cm.Chunks = append(cm.Chunks, + &operation.ChunkInfo{ + Offset: chunkOffset, + Size: int64(chunkBufOffset), + Fid: fileId, + }, + ) + + // reset variables for the next chunk + chunkBufOffset = 0 + chunkOffset = totalBytesRead + int64(bytesRead) + } + + totalBytesRead = totalBytesRead + int64(bytesRead) + + if bytesRead == 0 || readFully { + break + } + + if readErr != nil { + return nil, readErr + } + } + + cm.Size = totalBytesRead + manifestBuf, marshalErr := cm.Marshal() + if marshalErr != nil { + return nil, marshalErr + } + + manifestStr := string(manifestBuf) + glog.V(4).Infoln("Generated chunk manifest: ", manifestStr) + + manifestFileId, manifestUrlLocation, manifestAssignmentErr := fs.assignNewFileInfo(w, r, replication, collection) + if manifestAssignmentErr != nil { + return nil, manifestAssignmentErr + } + glog.V(4).Infoln("Manifest uploaded to:", manifestUrlLocation, "Fid:", manifestFileId) + filerResult.Fid = manifestFileId + + u, _ := url.Parse(manifestUrlLocation) + q := u.Query() + q.Set("cm", "true") + u.RawQuery = q.Encode() + + manifestUploadErr := fs.doUpload(u.String(), w, r, manifestBuf, fileName+"_manifest", "application/json", manifestFileId) + if manifestUploadErr != nil { + return nil, manifestUploadErr + } + + path := r.URL.Path + // also delete the old fid unless PUT operation + if r.Method != "PUT" { + if oldFid, err := fs.filer.FindFile(path); err == nil { + operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) + } + } + + glog.V(4).Infoln("saving", path, "=>", manifestFileId) + if db_err := fs.filer.CreateFile(path, manifestFileId); db_err != nil { + replyerr = db_err + filerResult.Error = db_err.Error() + operation.DeleteFile(fs.getMasterNode(), manifestFileId, fs.jwt(manifestFileId)) //clean up + glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err) + return + } + + return +} + +func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, chunkBuf []byte, fileName string, contentType string, fileId string) (err error) { + err = nil + + ioReader := ioutil.NopCloser(bytes.NewBuffer(chunkBuf)) + uploadResult, uploadError := operation.Upload(urlLocation, fileName, ioReader, false, contentType, fs.jwt(fileId)) + if uploadResult != nil { + glog.V(0).Infoln("Chunk upload result. Name:", uploadResult.Name, "Fid:", fileId, "Size:", uploadResult.Size) + } + if uploadError != nil { + err = uploadError + } + return +} + // curl -X DELETE http://localhost:8888/path/to // curl -X DELETE http://localhost:8888/path/to?recursive=true func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { From 14d4252904ed0fad8a7d6d6156a70fcbc3eda12c Mon Sep 17 00:00:00 2001 From: Mike Tolman Date: Fri, 5 Aug 2016 15:22:46 -0600 Subject: [PATCH 04/31] Ooops. Missed a line. --- weed/server/filer_server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index c9bc0e021..3c7c1fd9e 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -34,6 +34,7 @@ type FilerServer struct { func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir string, collection string, replication string, redirectOnRead bool, disableDirListing bool, + maxMB int, secret string, cassandra_server string, cassandra_keyspace string, redis_server string, redis_password string, redis_database int, From 34837afc7adb8ea6955d5cf962af10f8f30fb476 Mon Sep 17 00:00:00 2001 From: Mike Tolman Date: Fri, 5 Aug 2016 15:23:43 -0600 Subject: [PATCH 05/31] Adding HTTP verb whitelisting options. --- weed/command/filer.go | 74 ++++++++++++++++++++ weed/command/master.go | 15 ++-- weed/command/server.go | 77 +++++++++++++++++++-- weed/command/volume.go | 16 +++-- weed/security/guard.go | 48 +++++++++++-- weed/server/filer_server.go | 13 ++++ weed/server/filer_server_handlers.go | 10 +-- weed/server/master_server.go | 40 ++++++----- weed/server/master_server_handlers_admin.go | 2 +- weed/server/volume_server.go | 57 +++++++++------ weed/server/volume_server_handlers.go | 10 +-- 11 files changed, 290 insertions(+), 72 deletions(-) diff --git a/weed/command/filer.go b/weed/command/filer.go index 0bd508e0b..f58e38403 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -9,6 +9,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/server" "github.com/chrislusf/seaweedfs/weed/util" + "strings" ) var ( @@ -31,6 +32,31 @@ type FilerOptions struct { redis_server *string redis_password *string redis_database *int + get_ip_whitelist_option *string + get_root_whitelist_option *string + head_ip_whitelist_option *string + head_root_whitelist_option *string + delete_ip_whitelist_option *string + delete_root_whitelist_option *string + put_ip_whitelist_option *string + put_root_whitelist_option *string + post_ip_whitelist_option *string + post_root_whitelist_option *string + get_secure_key *string + head_secure_key *string + delete_secure_key *string + put_secure_key *string + post_secure_key *string + get_ip_whitelist []string + get_root_whitelist []string + head_ip_whitelist []string + head_root_whitelist []string + delete_ip_whitelist []string + delete_root_whitelist []string + put_ip_whitelist []string + put_root_whitelist []string + post_ip_whitelist []string + post_root_whitelist []string } func init() { @@ -50,6 +76,21 @@ func init() { f.redis_password = cmdFiler.Flag.String("redis.password", "", "password in clear text") f.redis_database = cmdFiler.Flag.Int("redis.database", 0, "the database on the redis server") f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)") + f.get_ip_whitelist_option = cmdFiler.Flag.String("whitelist.ip.get", "", "comma separated Ip addresses having get permission. No limit if empty.") + f.get_root_whitelist_option = cmdFiler.Flag.String("whitelist.root.get", "", "comma separated root paths having get permission. No limit if empty.") + f.head_ip_whitelist_option = cmdFiler.Flag.String("whitelist.ip.head", "", "comma separated Ip addresses having head permission. No limit if empty.") + f.head_root_whitelist_option = cmdFiler.Flag.String("whitelist.root.head", "", "comma separated root paths having head permission. No limit if empty.") + f.delete_ip_whitelist_option = cmdFiler.Flag.String("whitelist.ip.delete", "", "comma separated Ip addresses having delete permission. No limit if empty.") + f.delete_root_whitelist_option = cmdFiler.Flag.String("whitelist.root.delete", "", "comma separated root paths having delete permission. No limit if empty.") + f.put_ip_whitelist_option = cmdFiler.Flag.String("whitelist.ip.put", "", "comma separated Ip addresses having put permission. No limit if empty.") + f.put_root_whitelist_option = cmdFiler.Flag.String("whitelist.root.put", "", "comma separated root paths having put permission. No limit if empty.") + f.post_ip_whitelist_option = cmdFiler.Flag.String("whitelist.ip.post", "", "comma separated Ip addresses having post permission. No limit if empty.") + f.post_root_whitelist_option = cmdFiler.Flag.String("whitelist.root.post", "", "comma separated root paths having post permission. No limit if empty.") + f.get_secure_key = cmdFiler.Flag.String("secure.secret.get", "", "secret to encrypt Json Web Token(JWT)") + f.head_secure_key = cmdFiler.Flag.String("secure.secret.head", "", "secret to encrypt Json Web Token(JWT)") + f.delete_secure_key = cmdFiler.Flag.String("secure.secret.delete", "", "secret to encrypt Json Web Token(JWT)") + f.put_secure_key = cmdFiler.Flag.String("secure.secret.put", "", "secret to encrypt Json Web Token(JWT)") + f.post_secure_key = cmdFiler.Flag.String("secure.secret.post", "", "secret to encrypt Json Web Token(JWT)") } @@ -81,6 +122,36 @@ func runFiler(cmd *Command, args []string) bool { glog.Fatalf("Check Meta Folder (-dir) Writable %s : %s", *f.dir, err) } + if *f.get_ip_whitelist_option != "" { + f.get_ip_whitelist = strings.Split(*f.get_ip_whitelist_option, ",") + } + if *f.get_root_whitelist_option != "" { + f.get_root_whitelist = strings.Split(*f.get_root_whitelist_option, ",") + } + if *f.head_ip_whitelist_option != "" { + f.head_ip_whitelist = strings.Split(*f.head_ip_whitelist_option, ",") + } + if *f.head_root_whitelist_option != "" { + f.head_root_whitelist = strings.Split(*f.head_root_whitelist_option, ",") + } + if *f.delete_ip_whitelist_option != "" { + f.delete_ip_whitelist = strings.Split(*f.delete_ip_whitelist_option, ",") + } + if *f.delete_root_whitelist_option != "" { + f.delete_root_whitelist = strings.Split(*f.delete_root_whitelist_option, ",") + } + if *f.put_ip_whitelist_option != "" { + f.put_ip_whitelist = strings.Split(*f.put_ip_whitelist_option, ",") + } + if *f.put_root_whitelist_option != "" { + f.put_root_whitelist = strings.Split(*f.put_root_whitelist_option, ",") + } + if *f.post_ip_whitelist_option != "" { + f.post_ip_whitelist = strings.Split(*f.post_ip_whitelist_option, ",") + } + if *f.post_root_whitelist_option != "" { + f.post_root_whitelist = strings.Split(*f.post_root_whitelist_option, ",") + } r := http.NewServeMux() _, nfs_err := weed_server.NewFilerServer(r, *f.ip, *f.port, *f.master, *f.dir, *f.collection, *f.defaultReplicaPlacement, *f.redirectOnRead, *f.disableDirListing, @@ -88,6 +159,9 @@ func runFiler(cmd *Command, args []string) bool { *f.secretKey, *f.cassandra_server, *f.cassandra_keyspace, *f.redis_server, *f.redis_password, *f.redis_database, + f.get_ip_whitelist, f.head_ip_whitelist, f.delete_ip_whitelist, f.put_ip_whitelist, f.post_ip_whitelist, + f.get_root_whitelist, f.head_root_whitelist, f.delete_root_whitelist, f.put_root_whitelist, f.post_root_whitelist, + *f.get_secure_key, *f.head_secure_key, *f.delete_secure_key, *f.put_secure_key, *f.post_secure_key, ) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) diff --git a/weed/command/master.go b/weed/command/master.go index cd15defce..f140750ea 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -41,11 +41,13 @@ var ( mTimeout = cmdMaster.Flag.Int("idleTimeout", 10, "connection idle seconds") mMaxCpu = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") garbageThreshold = cmdMaster.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces") - masterWhiteListOption = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") + masterReadWhiteListOption = cmdMaster.Flag.String("readWhiteList", "", "comma separated Ip addresses having read permission. No limit if empty.") + masterWriteWhiteListOption = cmdMaster.Flag.String("writeWhiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") masterSecureKey = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)") masterCpuProfile = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file") - masterWhiteList []string + masterReadWhiteList []string + masterWriteWhiteList []string ) func runMaster(cmd *Command, args []string) bool { @@ -67,14 +69,17 @@ func runMaster(cmd *Command, args []string) bool { if err := util.TestFolderWritable(*metaFolder); err != nil { glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *metaFolder, err) } - if *masterWhiteListOption != "" { - masterWhiteList = strings.Split(*masterWhiteListOption, ",") + if *masterReadWhiteListOption != "" { + masterReadWhiteList = strings.Split(*masterReadWhiteListOption, ",") + } + if *masterWriteWhiteListOption != "" { + masterWriteWhiteList = strings.Split(*masterWriteWhiteListOption, ",") } r := mux.NewRouter() ms := weed_server.NewMasterServer(r, *mport, *metaFolder, *volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold, - masterWhiteList, *masterSecureKey, + masterReadWhiteList, masterWriteWhiteList, nil, *masterSecureKey, ) listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport) diff --git a/weed/command/server.go b/weed/command/server.go index 7a6677a65..9a19ef2af 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -54,7 +54,8 @@ var ( serverTimeout = cmdServer.Flag.Int("idleTimeout", 10, "connection idle seconds") serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name") serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name") - serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") + serverReadWhiteListOption = cmdServer.Flag.String("read.whitelist", "", "comma separated Ip addresses having read permission. No limit if empty.") + serverWriteWhiteListOption = cmdServer.Flag.String("write.whitelist", "", "comma separated Ip addresses having write permission. No limit if empty.") serverPeers = cmdServer.Flag.String("master.peers", "", "other master nodes in comma separated ip:masterPort list") serverSecureKey = cmdServer.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)") serverGarbageThreshold = cmdServer.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces") @@ -74,7 +75,8 @@ var ( volumeServerPublicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address") isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer") - serverWhiteList []string + serverReadWhiteList []string + serverWriteWhiteList []string ) func init() { @@ -82,7 +84,7 @@ func init() { filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server") filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection") filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") - filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -dir is specified") + filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified") filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.") filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") @@ -92,6 +94,21 @@ func init() { filerOptions.redis_server = cmdServer.Flag.String("filer.redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379") filerOptions.redis_password = cmdServer.Flag.String("filer.redis.password", "", "redis password in clear text") filerOptions.redis_database = cmdServer.Flag.Int("filer.redis.database", 0, "the database on the redis server") + filerOptions.get_ip_whitelist_option = cmdServer.Flag.String("filer.whitelist.ip.get", "", "comma separated Ip addresses having filer GET permission. No limit if empty.") + filerOptions.get_root_whitelist_option = cmdServer.Flag.String("filer.whitelist.root.get", "", "comma separated root paths having filer GET permission. No limit if empty.") + filerOptions.head_ip_whitelist_option = cmdServer.Flag.String("filer.whitelist.ip.head", "", "comma separated Ip addresses having filer HEAD permission. No limit if empty.") + filerOptions.head_root_whitelist_option = cmdServer.Flag.String("filer.whitelist.root.head", "", "comma separated root paths having filer HEAD permission. No limit if empty.") + filerOptions.delete_ip_whitelist_option = cmdServer.Flag.String("filer.whitelist.ip.delete", "", "comma separated Ip addresses having filer DELETE permission. No limit if empty.") + filerOptions.delete_root_whitelist_option = cmdServer.Flag.String("filer.whitelist.root.delete", "", "comma separated root paths having filer DELETE permission. No limit if empty.") + filerOptions.put_ip_whitelist_option = cmdServer.Flag.String("filer.whitelist.ip.put", "", "comma separated Ip addresses having filer PUT permission. No limit if empty.") + filerOptions.put_root_whitelist_option = cmdServer.Flag.String("filer.whitelist.root.put", "", "comma separated root paths having filer PUT permission. No limit if empty.") + filerOptions.post_ip_whitelist_option = cmdServer.Flag.String("filer.whitelist.ip.post", "", "comma separated Ip addresses having filer POST permission. No limit if empty.") + filerOptions.post_root_whitelist_option = cmdServer.Flag.String("filer.whitelist.root.post", "", "comma separated root paths having filer POST permission. No limit if empty.") + filerOptions.get_secure_key = cmdServer.Flag.String("filer.secure.secret.get", "", "secret to encrypt Json Web Token(JWT)") + filerOptions.head_secure_key = cmdServer.Flag.String("filer.secure.secret.head", "", "secret to encrypt Json Web Token(JWT)") + filerOptions.delete_secure_key = cmdServer.Flag.String("filer.secure.secret.delete", "", "secret to encrypt Json Web Token(JWT)") + filerOptions.put_secure_key = cmdServer.Flag.String("filer.secure.secret.put", "", "secret to encrypt Json Web Token(JWT)") + filerOptions.post_secure_key = cmdServer.Flag.String("filer.secure.secret.post", "", "secret to encrypt Json Web Token(JWT)") } func runServer(cmd *Command, args []string) bool { @@ -154,13 +171,56 @@ func runServer(cmd *Command, args []string) bool { if err := util.TestFolderWritable(*filerOptions.dir); err != nil { glog.Fatalf("Check Mapping Meta Folder (-filer.dir=\"%s\") Writable: %s", *filerOptions.dir, err) } + if *filerOptions.get_ip_whitelist_option != "" { + glog.V(0).Infof("Filer GET IP whitelist: %s", *filerOptions.get_ip_whitelist_option) + filerOptions.get_ip_whitelist = strings.Split(*filerOptions.get_ip_whitelist_option, ",") + } + if *filerOptions.get_root_whitelist_option != "" { + glog.V(0).Infof("Filer GET root whitelist: %s", *filerOptions.get_root_whitelist_option) + filerOptions.get_root_whitelist = strings.Split(*filerOptions.get_root_whitelist_option, ",") + } + if *filerOptions.head_ip_whitelist_option != "" { + glog.V(0).Infof("Filer HEAD IP whitelist: %s", *filerOptions.head_ip_whitelist_option) + filerOptions.head_ip_whitelist = strings.Split(*filerOptions.head_ip_whitelist_option, ",") + } + if *filerOptions.head_root_whitelist_option != "" { + glog.V(0).Infof("Filer HEAD root whitelist: %s", *filerOptions.head_root_whitelist_option) + filerOptions.head_root_whitelist = strings.Split(*filerOptions.head_root_whitelist_option, ",") + } + if *filerOptions.delete_ip_whitelist_option != "" { + glog.V(0).Infof("Filer DELETE IP whitelist: %s", *filerOptions.delete_ip_whitelist_option) + filerOptions.delete_ip_whitelist = strings.Split(*filerOptions.delete_ip_whitelist_option, ",") + } + if *filerOptions.delete_root_whitelist_option != "" { + glog.V(0).Infof("Filer DELETE root whitelist: %s", *filerOptions.delete_root_whitelist_option) + filerOptions.delete_root_whitelist = strings.Split(*filerOptions.delete_root_whitelist_option, ",") + } + if *filerOptions.put_ip_whitelist_option != "" { + glog.V(0).Infof("Filer PUT IP whitelist: %s", *filerOptions.put_ip_whitelist_option) + filerOptions.put_ip_whitelist = strings.Split(*filerOptions.put_ip_whitelist_option, ",") + } + if *filerOptions.put_root_whitelist_option != "" { + glog.V(0).Infof("Filer PUT root whitelist: %s", *filerOptions.put_root_whitelist_option) + filerOptions.put_root_whitelist = strings.Split(*filerOptions.put_root_whitelist_option, ",") + } + if *filerOptions.post_ip_whitelist_option != "" { + glog.V(0).Infof("Filer POST IP whitelist: %s", *filerOptions.post_ip_whitelist_option) + filerOptions.post_ip_whitelist = strings.Split(*filerOptions.post_ip_whitelist_option, ",") + } + if *filerOptions.post_root_whitelist_option != "" { + glog.V(0).Infof("Filer POST root whitelist: %s", *filerOptions.post_root_whitelist_option) + filerOptions.post_root_whitelist = strings.Split(*filerOptions.post_root_whitelist_option, ",") + } } if err := util.TestFolderWritable(*masterMetaFolder); err != nil { glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err) } - if *serverWhiteListOption != "" { - serverWhiteList = strings.Split(*serverWhiteListOption, ",") + if *serverReadWhiteListOption != "" { + serverReadWhiteList = strings.Split(*serverReadWhiteListOption, ",") + } + if *serverWriteWhiteListOption != "" { + serverWriteWhiteList = strings.Split(*serverWriteWhiteListOption, ",") } if *isStartingFiler { @@ -174,6 +234,9 @@ func runServer(cmd *Command, args []string) bool { *filerOptions.secretKey, *filerOptions.cassandra_server, *filerOptions.cassandra_keyspace, *filerOptions.redis_server, *filerOptions.redis_password, *filerOptions.redis_database, + filerOptions.get_ip_whitelist, filerOptions.head_ip_whitelist, filerOptions.delete_ip_whitelist, filerOptions.put_ip_whitelist, filerOptions.post_ip_whitelist, + filerOptions.get_root_whitelist, filerOptions.head_root_whitelist, filerOptions.delete_root_whitelist, filerOptions.put_root_whitelist, filerOptions.post_root_whitelist, + *f.get_secure_key, *f.head_secure_key, *f.delete_secure_key, *f.put_secure_key, *f.post_secure_key, ) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) @@ -202,7 +265,7 @@ func runServer(cmd *Command, args []string) bool { r := mux.NewRouter() ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder, *masterVolumeSizeLimitMB, *volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *serverGarbageThreshold, - serverWhiteList, *serverSecureKey, + serverReadWhiteList, serverWriteWhiteList, nil, *serverSecureKey, ) glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort)) @@ -256,7 +319,7 @@ func runServer(cmd *Command, args []string) bool { folders, maxCounts, volumeNeedleMapKind, *serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack, - serverWhiteList, *volumeFixJpgOrientation, *volumeReadRedirect, + serverReadWhiteList, serverWriteWhiteList, nil, *volumeFixJpgOrientation, *volumeReadRedirect, ) glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort)) diff --git a/weed/command/volume.go b/weed/command/volume.go index 21369cbe9..68f5edd9e 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -2,6 +2,7 @@ package command import ( "net/http" + _ "net/http/pprof" "os" "runtime" "strconv" @@ -32,7 +33,8 @@ type VolumeServerOptions struct { maxCpu *int dataCenter *string rack *string - whiteList []string + readWhitelist []string + writeWhitelist []string indexType *string fixJpgOrientation *bool readRedirect *bool @@ -67,7 +69,8 @@ var cmdVolume = &Command{ var ( volumeFolders = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...") maxVolumeCounts = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...") - volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") + volumeReadWhiteListOption = cmdVolume.Flag.String("read.whitelist", "", "comma separated Ip addresses having read permission. No limit if empty.") + volumeWriteWhiteListOption = cmdVolume.Flag.String("write.whitelist", "", "comma separated Ip addresses having write permission. No limit if empty.") ) func runVolume(cmd *Command, args []string) bool { @@ -96,8 +99,11 @@ func runVolume(cmd *Command, args []string) bool { } //security related white list configuration - if *volumeWhiteListOption != "" { - v.whiteList = strings.Split(*volumeWhiteListOption, ",") + if *volumeReadWhiteListOption != "" { + v.readWhitelist = strings.Split(*volumeReadWhiteListOption, ",") + } + if *volumeWriteWhiteListOption != "" { + v.writeWhitelist = strings.Split(*volumeWriteWhiteListOption, ",") } if *v.ip == "" { @@ -130,7 +136,7 @@ func runVolume(cmd *Command, args []string) bool { v.folders, v.folderMaxLimits, volumeNeedleMapKind, *v.master, *v.pulseSeconds, *v.dataCenter, *v.rack, - v.whiteList, + v.readWhitelist, v.writeWhitelist, nil, *v.fixJpgOrientation, *v.readRedirect, ) diff --git a/weed/security/guard.go b/weed/security/guard.go index dea3b12f2..6292c67c9 100644 --- a/weed/security/guard.go +++ b/weed/security/guard.go @@ -41,17 +41,31 @@ https://github.com/pkieltyka/jwtauth/blob/master/jwtauth.go */ type Guard struct { - whiteList []string + ipWhiteList []string + rootWhiteList []string SecretKey Secret isActive bool } -func NewGuard(whiteList []string, secretKey string) *Guard { - g := &Guard{whiteList: whiteList, SecretKey: Secret(secretKey)} - g.isActive = len(g.whiteList) != 0 || len(g.SecretKey) != 0 +func NewGuard(ipWhiteList []string, rootWhiteList []string, secretKey string) *Guard { + g := &Guard{ipWhiteList: ipWhiteList, rootWhiteList: rootWhiteList, SecretKey: Secret(secretKey)} + g.isActive = len(g.ipWhiteList) != 0 || len(g.SecretKey) != 0 return g } +func (g *Guard) WhiteList2(f func(w http.ResponseWriter, r *http.Request, b bool)) func(w http.ResponseWriter, r *http.Request, b bool) { + if !g.isActive { + //if no security needed, just skip all checkings + return f + } + return func(w http.ResponseWriter, r *http.Request, b bool) { + if err := g.checkWhiteList(w, r); err != nil { + w.WriteHeader(http.StatusUnauthorized) + return + } + f(w, r, b) + } +} func (g *Guard) WhiteList(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { if !g.isActive { @@ -96,13 +110,14 @@ func GetActualRemoteHost(r *http.Request) (host string, err error) { } func (g *Guard) checkWhiteList(w http.ResponseWriter, r *http.Request) error { - if len(g.whiteList) == 0 { + if len(g.ipWhiteList) == 0 { + glog.V(0).Info("No whitelist specified for operation") return nil } host, err := GetActualRemoteHost(r) if err == nil { - for _, ip := range g.whiteList { + for _, ip := range g.ipWhiteList { // If the whitelist entry contains a "/" it // is a CIDR range, and we should check the @@ -114,6 +129,7 @@ func (g *Guard) checkWhiteList(w http.ResponseWriter, r *http.Request) error { } remote := net.ParseIP(host) if cidrnet.Contains(remote) { + glog.V(0).Infof("Found %s in CIDR whitelist.", r.RemoteAddr) return nil } } @@ -122,10 +138,30 @@ func (g *Guard) checkWhiteList(w http.ResponseWriter, r *http.Request) error { // Otherwise we're looking for a literal match. // if ip == host { + glog.V(0).Infof("Found %s in whitelist.", r.RemoteAddr) + return nil + } + // ::1 is the same as 127.0.0.1 and localhost + if host == "::1" && (ip == "127.0.0.1" || ip == "localhost") { + glog.V(0).Infof("Found %s (localhost) in whitelist.", r.RemoteAddr) return nil } } } + // The root whitelist allows exceptions to the IP whitelist, but only by certain root paths in the request. + if len(g.rootWhiteList) > 0 { + pathParts := strings.Split(r.RequestURI, "/") + if len(pathParts) > 0 { + requestedRoot := pathParts[1] + for _, root := range g.rootWhiteList { + if root == requestedRoot { + glog.V(0).Infof("Found %s in root whitelist.", requestedRoot) + return nil + } + } + glog.V(0).Infof("Not in root whitelist: %s", requestedRoot) + } + } glog.V(0).Infof("Not in whitelist: %s", r.RemoteAddr) return fmt.Errorf("Not in whitelis: %s", r.RemoteAddr) diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 3c7c1fd9e..2a432af65 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -30,6 +30,11 @@ type FilerServer struct { filer filer.Filer maxMB int masterNodes *storage.MasterNodes + get_guard *security.Guard + head_guard *security.Guard + delete_guard *security.Guard + put_guard *security.Guard + post_guard *security.Guard } func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir string, collection string, @@ -38,6 +43,9 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st secret string, cassandra_server string, cassandra_keyspace string, redis_server string, redis_password string, redis_database int, + get_ip_whitelist []string, head_ip_whitelist []string, delete_ip_whitelist []string, put_ip_whitelist []string, post_ip_whitelist []string, + get_root_whitelist []string, head_root_whitelist []string, delete_root_whitelist []string, put_root_whitelist []string, post_root_whitelist []string, + get_secure_key string, head_secure_key string, delete_secure_key string, put_secure_key string, post_secure_key string, ) (fs *FilerServer, err error) { fs = &FilerServer{ master: master, @@ -46,6 +54,11 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st redirectOnRead: redirectOnRead, disableDirListing: disableDirListing, maxMB: maxMB, + get_guard: security.NewGuard(get_ip_whitelist, get_root_whitelist, get_secure_key), + head_guard: security.NewGuard(head_ip_whitelist, head_root_whitelist, head_secure_key), + delete_guard: security.NewGuard(delete_ip_whitelist, delete_root_whitelist, delete_secure_key), + put_guard: security.NewGuard(put_ip_whitelist, put_root_whitelist, put_secure_key), + post_guard: security.NewGuard(post_ip_whitelist, post_root_whitelist, post_secure_key), port: ip + ":" + strconv.Itoa(port), } diff --git a/weed/server/filer_server_handlers.go b/weed/server/filer_server_handlers.go index 4e39258af..4db170590 100644 --- a/weed/server/filer_server_handlers.go +++ b/weed/server/filer_server_handlers.go @@ -7,14 +7,14 @@ import ( func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { switch r.Method { case "GET": - fs.GetOrHeadHandler(w, r, true) + fs.get_guard.WhiteList2(fs.GetOrHeadHandler)(w, r, true) case "HEAD": - fs.GetOrHeadHandler(w, r, false) + fs.head_guard.WhiteList2(fs.GetOrHeadHandler)(w, r, false) case "DELETE": - fs.DeleteHandler(w, r) + fs.delete_guard.WhiteList(fs.DeleteHandler)(w, r) case "PUT": - fs.PostHandler(w, r) + fs.put_guard.WhiteList(fs.PostHandler)(w, r) case "POST": - fs.PostHandler(w, r) + fs.post_guard.WhiteList(fs.PostHandler)(w, r) } } diff --git a/weed/server/master_server.go b/weed/server/master_server.go index 61bda6988..3bd77c819 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -23,7 +23,8 @@ type MasterServer struct { pulseSeconds int defaultReplicaPlacement string garbageThreshold string - guard *security.Guard + read_guard *security.Guard + write_guard *security.Guard Topo *topology.Topology vg *topology.VolumeGrowth @@ -38,7 +39,9 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string, confFile string, defaultReplicaPlacement string, garbageThreshold string, - whiteList []string, + ipReadWhiteList []string, + ipWriteWhiteList []string, + rootWhiteList []string, secureKey string, ) *MasterServer { ms := &MasterServer{ @@ -58,24 +61,25 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string, ms.vg = topology.NewDefaultVolumeGrowth() glog.V(0).Infoln("Volume Size Limit is", volumeSizeLimitMB, "MB") - ms.guard = security.NewGuard(whiteList, secureKey) + ms.read_guard = security.NewGuard(ipReadWhiteList, rootWhiteList, secureKey) + ms.write_guard = security.NewGuard(ipWriteWhiteList, rootWhiteList, secureKey) r.HandleFunc("/", ms.uiStatusHandler) - r.HandleFunc("/ui/index.html", ms.uiStatusHandler) - r.HandleFunc("/dir/assign", ms.proxyToLeader(ms.guard.WhiteList(ms.dirAssignHandler))) - r.HandleFunc("/dir/lookup", ms.proxyToLeader(ms.guard.WhiteList(ms.dirLookupHandler))) - r.HandleFunc("/dir/join", ms.proxyToLeader(ms.guard.WhiteList(ms.dirJoinHandler))) - r.HandleFunc("/dir/status", ms.proxyToLeader(ms.guard.WhiteList(ms.dirStatusHandler))) - r.HandleFunc("/col/delete", ms.proxyToLeader(ms.guard.WhiteList(ms.collectionDeleteHandler))) - r.HandleFunc("/vol/lookup", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeLookupHandler))) - r.HandleFunc("/vol/grow", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeGrowHandler))) - r.HandleFunc("/vol/status", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeStatusHandler))) - r.HandleFunc("/vol/vacuum", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeVacuumHandler))) - r.HandleFunc("/submit", ms.guard.WhiteList(ms.submitFromMasterServerHandler)) - r.HandleFunc("/delete", ms.guard.WhiteList(ms.deleteFromMasterServerHandler)) - r.HandleFunc("/{fileId}", ms.proxyToLeader(ms.redirectHandler)) - r.HandleFunc("/stats/counter", ms.guard.WhiteList(statsCounterHandler)) - r.HandleFunc("/stats/memory", ms.guard.WhiteList(statsMemoryHandler)) + r.HandleFunc("/ui/index.html", ms.read_guard.WhiteList(ms.uiStatusHandler)) + r.HandleFunc("/dir/assign", ms.proxyToLeader(ms.write_guard.WhiteList(ms.dirAssignHandler))) + r.HandleFunc("/dir/lookup", ms.proxyToLeader(ms.read_guard.WhiteList(ms.dirLookupHandler))) + r.HandleFunc("/dir/join", ms.proxyToLeader(ms.write_guard.WhiteList(ms.dirJoinHandler))) + r.HandleFunc("/dir/status", ms.proxyToLeader(ms.read_guard.WhiteList(ms.dirStatusHandler))) + r.HandleFunc("/col/delete", ms.proxyToLeader(ms.write_guard.WhiteList(ms.collectionDeleteHandler))) + r.HandleFunc("/vol/lookup", ms.proxyToLeader(ms.read_guard.WhiteList(ms.volumeLookupHandler))) + r.HandleFunc("/vol/grow", ms.proxyToLeader(ms.write_guard.WhiteList(ms.volumeGrowHandler))) + r.HandleFunc("/vol/status", ms.proxyToLeader(ms.read_guard.WhiteList(ms.volumeStatusHandler))) + r.HandleFunc("/vol/vacuum", ms.proxyToLeader(ms.write_guard.WhiteList(ms.volumeVacuumHandler))) + r.HandleFunc("/submit", ms.write_guard.WhiteList(ms.submitFromMasterServerHandler)) + r.HandleFunc("/delete", ms.write_guard.WhiteList(ms.deleteFromMasterServerHandler)) + r.HandleFunc("/{fileId}", ms.proxyToLeader(ms.read_guard.WhiteList(ms.redirectHandler))) + r.HandleFunc("/stats/counter", ms.read_guard.WhiteList(statsCounterHandler)) + r.HandleFunc("/stats/memory", ms.read_guard.WhiteList(statsMemoryHandler)) ms.Topo.StartRefreshWritableVolumes(garbageThreshold) diff --git a/weed/server/master_server_handlers_admin.go b/weed/server/master_server_handlers_admin.go index a762bf416..385290079 100644 --- a/weed/server/master_server_handlers_admin.go +++ b/weed/server/master_server_handlers_admin.go @@ -61,7 +61,7 @@ func (ms *MasterServer) dirJoinHandler(w http.ResponseWriter, r *http.Request) { ms.Topo.ProcessJoinMessage(joinMessage) writeJsonQuiet(w, r, http.StatusOK, operation.JoinResult{ VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024, - SecretKey: string(ms.guard.SecretKey), + SecretKey: string(ms.write_guard.SecretKey), }) } diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index 79a4276b1..a40eeb9c0 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -9,6 +9,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/storage" + //"net/http/pprof" ) type VolumeServer struct { @@ -18,7 +19,8 @@ type VolumeServer struct { dataCenter string rack string store *storage.Store - guard *security.Guard + read_guard *security.Guard + write_guard *security.Guard needleMapKind storage.NeedleMapType FixJpgOrientation bool @@ -31,7 +33,9 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, needleMapKind storage.NeedleMapType, masterNode string, pulseSeconds int, dataCenter string, rack string, - whiteList []string, + ipReadWhiteList []string, + ipWriteWhiteList []string, + rootWhiteList []string, fixJpgOrientation bool, readRedirect bool) *VolumeServer { vs := &VolumeServer{ @@ -45,28 +49,40 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, vs.SetMasterNode(masterNode) vs.store = storage.NewStore(port, ip, publicUrl, folders, maxCounts, vs.needleMapKind) - vs.guard = security.NewGuard(whiteList, "") + vs.read_guard = security.NewGuard(ipReadWhiteList, rootWhiteList, "") + vs.write_guard = security.NewGuard(ipWriteWhiteList, rootWhiteList, "") - adminMux.HandleFunc("/ui/index.html", vs.uiStatusHandler) - adminMux.HandleFunc("/status", vs.guard.WhiteList(vs.statusHandler)) - adminMux.HandleFunc("/admin/assign_volume", vs.guard.WhiteList(vs.assignVolumeHandler)) - adminMux.HandleFunc("/admin/vacuum/check", vs.guard.WhiteList(vs.vacuumVolumeCheckHandler)) - adminMux.HandleFunc("/admin/vacuum/compact", vs.guard.WhiteList(vs.vacuumVolumeCompactHandler)) - adminMux.HandleFunc("/admin/vacuum/commit", vs.guard.WhiteList(vs.vacuumVolumeCommitHandler)) - adminMux.HandleFunc("/admin/delete_collection", vs.guard.WhiteList(vs.deleteCollectionHandler)) - adminMux.HandleFunc("/admin/sync/status", vs.guard.WhiteList(vs.getVolumeSyncStatusHandler)) - adminMux.HandleFunc("/admin/sync/index", vs.guard.WhiteList(vs.getVolumeIndexContentHandler)) - adminMux.HandleFunc("/admin/sync/data", vs.guard.WhiteList(vs.getVolumeDataContentHandler)) - adminMux.HandleFunc("/stats/counter", vs.guard.WhiteList(statsCounterHandler)) - adminMux.HandleFunc("/stats/memory", vs.guard.WhiteList(statsMemoryHandler)) - adminMux.HandleFunc("/stats/disk", vs.guard.WhiteList(vs.statsDiskHandler)) - adminMux.HandleFunc("/delete", vs.guard.WhiteList(vs.batchDeleteHandler)) - adminMux.HandleFunc("/", vs.privateStoreHandler) + adminMux.HandleFunc("/ui/index.html", vs.read_guard.WhiteList(vs.uiStatusHandler)) + adminMux.HandleFunc("/status", vs.read_guard.WhiteList(vs.statusHandler)) + adminMux.HandleFunc("/admin/assign_volume", vs.write_guard.WhiteList(vs.assignVolumeHandler)) + adminMux.HandleFunc("/admin/vacuum/check", vs.write_guard.WhiteList(vs.vacuumVolumeCheckHandler)) + adminMux.HandleFunc("/admin/vacuum/compact", vs.write_guard.WhiteList(vs.vacuumVolumeCompactHandler)) + adminMux.HandleFunc("/admin/vacuum/commit", vs.write_guard.WhiteList(vs.vacuumVolumeCommitHandler)) + adminMux.HandleFunc("/admin/delete_collection", vs.write_guard.WhiteList(vs.deleteCollectionHandler)) + adminMux.HandleFunc("/admin/sync/status", vs.read_guard.WhiteList(vs.getVolumeSyncStatusHandler)) + adminMux.HandleFunc("/admin/sync/index", vs.write_guard.WhiteList(vs.getVolumeIndexContentHandler)) + adminMux.HandleFunc("/admin/sync/data", vs.write_guard.WhiteList(vs.getVolumeDataContentHandler)) + adminMux.HandleFunc("/stats/counter", vs.read_guard.WhiteList(statsCounterHandler)) + adminMux.HandleFunc("/stats/memory", vs.read_guard.WhiteList(statsMemoryHandler)) + adminMux.HandleFunc("/stats/disk", vs.read_guard.WhiteList(vs.statsDiskHandler)) + adminMux.HandleFunc("/delete", vs.write_guard.WhiteList(vs.batchDeleteHandler)) + adminMux.HandleFunc("/", vs.read_guard.WhiteList(vs.privateStoreHandler)) if publicMux != adminMux { // separated admin and public port publicMux.HandleFunc("/favicon.ico", vs.faviconHandler) publicMux.HandleFunc("/", vs.publicReadOnlyHandler) } + /* + // add in profiling support + adminMux.HandleFunc("/debug/pprof/", pprof.Index) + adminMux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + adminMux.HandleFunc("/debug/pprof/profile", pprof.Profile) + adminMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + adminMux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) + adminMux.Handle("/debug/pprof/heap", pprof.Handler("heap")) + adminMux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) + adminMux.Handle("/debug/pprof/block", pprof.Handler("block")) + */ go func() { connected := true @@ -82,7 +98,8 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, if !connected { connected = true vs.SetMasterNode(master) - vs.guard.SecretKey = secretKey + vs.read_guard.SecretKey = secretKey + vs.write_guard.SecretKey = secretKey glog.V(0).Infoln("Volume Server Connected with master at", master) } } else { @@ -121,5 +138,5 @@ func (vs *VolumeServer) Shutdown() { } func (vs *VolumeServer) jwt(fileId string) security.EncodedJwt { - return security.GenJwt(vs.guard.SecretKey, fileId) + return security.GenJwt(vs.read_guard.SecretKey, fileId) } diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index 2d6fe7849..f223af6c0 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -25,19 +25,19 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque switch r.Method { case "GET": stats.ReadRequest() - vs.GetOrHeadHandler(w, r) + vs.write_guard.WhiteList(vs.GetOrHeadHandler)(w, r) case "HEAD": stats.ReadRequest() - vs.GetOrHeadHandler(w, r) + vs.write_guard.WhiteList(vs.GetOrHeadHandler)(w, r) case "DELETE": stats.DeleteRequest() - vs.guard.WhiteList(vs.DeleteHandler)(w, r) + vs.write_guard.WhiteList(vs.DeleteHandler)(w, r) case "PUT": stats.WriteRequest() - vs.guard.WhiteList(vs.PostHandler)(w, r) + vs.write_guard.WhiteList(vs.PostHandler)(w, r) case "POST": stats.WriteRequest() - vs.guard.WhiteList(vs.PostHandler)(w, r) + vs.write_guard.WhiteList(vs.PostHandler)(w, r) } } From ce99bb927d163707e83de6a265ce9b77dd4f9d44 Mon Sep 17 00:00:00 2001 From: Mike Tolman Date: Fri, 5 Aug 2016 15:45:48 -0600 Subject: [PATCH 06/31] Revert "Adding HTTP verb whitelisting options." This reverts commit 34837afc7adb8ea6955d5cf962af10f8f30fb476. --- weed/command/filer.go | 74 -------------------- weed/command/master.go | 15 ++-- weed/command/server.go | 77 ++------------------- weed/command/volume.go | 16 ++--- weed/security/guard.go | 48 ++----------- weed/server/filer_server.go | 13 ---- weed/server/filer_server_handlers.go | 10 +-- weed/server/master_server.go | 40 +++++------ weed/server/master_server_handlers_admin.go | 2 +- weed/server/volume_server.go | 57 ++++++--------- weed/server/volume_server_handlers.go | 10 +-- 11 files changed, 72 insertions(+), 290 deletions(-) diff --git a/weed/command/filer.go b/weed/command/filer.go index f58e38403..0bd508e0b 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -9,7 +9,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/server" "github.com/chrislusf/seaweedfs/weed/util" - "strings" ) var ( @@ -32,31 +31,6 @@ type FilerOptions struct { redis_server *string redis_password *string redis_database *int - get_ip_whitelist_option *string - get_root_whitelist_option *string - head_ip_whitelist_option *string - head_root_whitelist_option *string - delete_ip_whitelist_option *string - delete_root_whitelist_option *string - put_ip_whitelist_option *string - put_root_whitelist_option *string - post_ip_whitelist_option *string - post_root_whitelist_option *string - get_secure_key *string - head_secure_key *string - delete_secure_key *string - put_secure_key *string - post_secure_key *string - get_ip_whitelist []string - get_root_whitelist []string - head_ip_whitelist []string - head_root_whitelist []string - delete_ip_whitelist []string - delete_root_whitelist []string - put_ip_whitelist []string - put_root_whitelist []string - post_ip_whitelist []string - post_root_whitelist []string } func init() { @@ -76,21 +50,6 @@ func init() { f.redis_password = cmdFiler.Flag.String("redis.password", "", "password in clear text") f.redis_database = cmdFiler.Flag.Int("redis.database", 0, "the database on the redis server") f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)") - f.get_ip_whitelist_option = cmdFiler.Flag.String("whitelist.ip.get", "", "comma separated Ip addresses having get permission. No limit if empty.") - f.get_root_whitelist_option = cmdFiler.Flag.String("whitelist.root.get", "", "comma separated root paths having get permission. No limit if empty.") - f.head_ip_whitelist_option = cmdFiler.Flag.String("whitelist.ip.head", "", "comma separated Ip addresses having head permission. No limit if empty.") - f.head_root_whitelist_option = cmdFiler.Flag.String("whitelist.root.head", "", "comma separated root paths having head permission. No limit if empty.") - f.delete_ip_whitelist_option = cmdFiler.Flag.String("whitelist.ip.delete", "", "comma separated Ip addresses having delete permission. No limit if empty.") - f.delete_root_whitelist_option = cmdFiler.Flag.String("whitelist.root.delete", "", "comma separated root paths having delete permission. No limit if empty.") - f.put_ip_whitelist_option = cmdFiler.Flag.String("whitelist.ip.put", "", "comma separated Ip addresses having put permission. No limit if empty.") - f.put_root_whitelist_option = cmdFiler.Flag.String("whitelist.root.put", "", "comma separated root paths having put permission. No limit if empty.") - f.post_ip_whitelist_option = cmdFiler.Flag.String("whitelist.ip.post", "", "comma separated Ip addresses having post permission. No limit if empty.") - f.post_root_whitelist_option = cmdFiler.Flag.String("whitelist.root.post", "", "comma separated root paths having post permission. No limit if empty.") - f.get_secure_key = cmdFiler.Flag.String("secure.secret.get", "", "secret to encrypt Json Web Token(JWT)") - f.head_secure_key = cmdFiler.Flag.String("secure.secret.head", "", "secret to encrypt Json Web Token(JWT)") - f.delete_secure_key = cmdFiler.Flag.String("secure.secret.delete", "", "secret to encrypt Json Web Token(JWT)") - f.put_secure_key = cmdFiler.Flag.String("secure.secret.put", "", "secret to encrypt Json Web Token(JWT)") - f.post_secure_key = cmdFiler.Flag.String("secure.secret.post", "", "secret to encrypt Json Web Token(JWT)") } @@ -122,36 +81,6 @@ func runFiler(cmd *Command, args []string) bool { glog.Fatalf("Check Meta Folder (-dir) Writable %s : %s", *f.dir, err) } - if *f.get_ip_whitelist_option != "" { - f.get_ip_whitelist = strings.Split(*f.get_ip_whitelist_option, ",") - } - if *f.get_root_whitelist_option != "" { - f.get_root_whitelist = strings.Split(*f.get_root_whitelist_option, ",") - } - if *f.head_ip_whitelist_option != "" { - f.head_ip_whitelist = strings.Split(*f.head_ip_whitelist_option, ",") - } - if *f.head_root_whitelist_option != "" { - f.head_root_whitelist = strings.Split(*f.head_root_whitelist_option, ",") - } - if *f.delete_ip_whitelist_option != "" { - f.delete_ip_whitelist = strings.Split(*f.delete_ip_whitelist_option, ",") - } - if *f.delete_root_whitelist_option != "" { - f.delete_root_whitelist = strings.Split(*f.delete_root_whitelist_option, ",") - } - if *f.put_ip_whitelist_option != "" { - f.put_ip_whitelist = strings.Split(*f.put_ip_whitelist_option, ",") - } - if *f.put_root_whitelist_option != "" { - f.put_root_whitelist = strings.Split(*f.put_root_whitelist_option, ",") - } - if *f.post_ip_whitelist_option != "" { - f.post_ip_whitelist = strings.Split(*f.post_ip_whitelist_option, ",") - } - if *f.post_root_whitelist_option != "" { - f.post_root_whitelist = strings.Split(*f.post_root_whitelist_option, ",") - } r := http.NewServeMux() _, nfs_err := weed_server.NewFilerServer(r, *f.ip, *f.port, *f.master, *f.dir, *f.collection, *f.defaultReplicaPlacement, *f.redirectOnRead, *f.disableDirListing, @@ -159,9 +88,6 @@ func runFiler(cmd *Command, args []string) bool { *f.secretKey, *f.cassandra_server, *f.cassandra_keyspace, *f.redis_server, *f.redis_password, *f.redis_database, - f.get_ip_whitelist, f.head_ip_whitelist, f.delete_ip_whitelist, f.put_ip_whitelist, f.post_ip_whitelist, - f.get_root_whitelist, f.head_root_whitelist, f.delete_root_whitelist, f.put_root_whitelist, f.post_root_whitelist, - *f.get_secure_key, *f.head_secure_key, *f.delete_secure_key, *f.put_secure_key, *f.post_secure_key, ) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) diff --git a/weed/command/master.go b/weed/command/master.go index f140750ea..cd15defce 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -41,13 +41,11 @@ var ( mTimeout = cmdMaster.Flag.Int("idleTimeout", 10, "connection idle seconds") mMaxCpu = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") garbageThreshold = cmdMaster.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces") - masterReadWhiteListOption = cmdMaster.Flag.String("readWhiteList", "", "comma separated Ip addresses having read permission. No limit if empty.") - masterWriteWhiteListOption = cmdMaster.Flag.String("writeWhiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") + masterWhiteListOption = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") masterSecureKey = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)") masterCpuProfile = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file") - masterReadWhiteList []string - masterWriteWhiteList []string + masterWhiteList []string ) func runMaster(cmd *Command, args []string) bool { @@ -69,17 +67,14 @@ func runMaster(cmd *Command, args []string) bool { if err := util.TestFolderWritable(*metaFolder); err != nil { glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *metaFolder, err) } - if *masterReadWhiteListOption != "" { - masterReadWhiteList = strings.Split(*masterReadWhiteListOption, ",") - } - if *masterWriteWhiteListOption != "" { - masterWriteWhiteList = strings.Split(*masterWriteWhiteListOption, ",") + if *masterWhiteListOption != "" { + masterWhiteList = strings.Split(*masterWhiteListOption, ",") } r := mux.NewRouter() ms := weed_server.NewMasterServer(r, *mport, *metaFolder, *volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold, - masterReadWhiteList, masterWriteWhiteList, nil, *masterSecureKey, + masterWhiteList, *masterSecureKey, ) listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport) diff --git a/weed/command/server.go b/weed/command/server.go index 9a19ef2af..7a6677a65 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -54,8 +54,7 @@ var ( serverTimeout = cmdServer.Flag.Int("idleTimeout", 10, "connection idle seconds") serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name") serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name") - serverReadWhiteListOption = cmdServer.Flag.String("read.whitelist", "", "comma separated Ip addresses having read permission. No limit if empty.") - serverWriteWhiteListOption = cmdServer.Flag.String("write.whitelist", "", "comma separated Ip addresses having write permission. No limit if empty.") + serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") serverPeers = cmdServer.Flag.String("master.peers", "", "other master nodes in comma separated ip:masterPort list") serverSecureKey = cmdServer.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)") serverGarbageThreshold = cmdServer.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces") @@ -75,8 +74,7 @@ var ( volumeServerPublicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address") isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer") - serverReadWhiteList []string - serverWriteWhiteList []string + serverWhiteList []string ) func init() { @@ -84,7 +82,7 @@ func init() { filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server") filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection") filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") - filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified") + filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -dir is specified") filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.") filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") @@ -94,21 +92,6 @@ func init() { filerOptions.redis_server = cmdServer.Flag.String("filer.redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379") filerOptions.redis_password = cmdServer.Flag.String("filer.redis.password", "", "redis password in clear text") filerOptions.redis_database = cmdServer.Flag.Int("filer.redis.database", 0, "the database on the redis server") - filerOptions.get_ip_whitelist_option = cmdServer.Flag.String("filer.whitelist.ip.get", "", "comma separated Ip addresses having filer GET permission. No limit if empty.") - filerOptions.get_root_whitelist_option = cmdServer.Flag.String("filer.whitelist.root.get", "", "comma separated root paths having filer GET permission. No limit if empty.") - filerOptions.head_ip_whitelist_option = cmdServer.Flag.String("filer.whitelist.ip.head", "", "comma separated Ip addresses having filer HEAD permission. No limit if empty.") - filerOptions.head_root_whitelist_option = cmdServer.Flag.String("filer.whitelist.root.head", "", "comma separated root paths having filer HEAD permission. No limit if empty.") - filerOptions.delete_ip_whitelist_option = cmdServer.Flag.String("filer.whitelist.ip.delete", "", "comma separated Ip addresses having filer DELETE permission. No limit if empty.") - filerOptions.delete_root_whitelist_option = cmdServer.Flag.String("filer.whitelist.root.delete", "", "comma separated root paths having filer DELETE permission. No limit if empty.") - filerOptions.put_ip_whitelist_option = cmdServer.Flag.String("filer.whitelist.ip.put", "", "comma separated Ip addresses having filer PUT permission. No limit if empty.") - filerOptions.put_root_whitelist_option = cmdServer.Flag.String("filer.whitelist.root.put", "", "comma separated root paths having filer PUT permission. No limit if empty.") - filerOptions.post_ip_whitelist_option = cmdServer.Flag.String("filer.whitelist.ip.post", "", "comma separated Ip addresses having filer POST permission. No limit if empty.") - filerOptions.post_root_whitelist_option = cmdServer.Flag.String("filer.whitelist.root.post", "", "comma separated root paths having filer POST permission. No limit if empty.") - filerOptions.get_secure_key = cmdServer.Flag.String("filer.secure.secret.get", "", "secret to encrypt Json Web Token(JWT)") - filerOptions.head_secure_key = cmdServer.Flag.String("filer.secure.secret.head", "", "secret to encrypt Json Web Token(JWT)") - filerOptions.delete_secure_key = cmdServer.Flag.String("filer.secure.secret.delete", "", "secret to encrypt Json Web Token(JWT)") - filerOptions.put_secure_key = cmdServer.Flag.String("filer.secure.secret.put", "", "secret to encrypt Json Web Token(JWT)") - filerOptions.post_secure_key = cmdServer.Flag.String("filer.secure.secret.post", "", "secret to encrypt Json Web Token(JWT)") } func runServer(cmd *Command, args []string) bool { @@ -171,56 +154,13 @@ func runServer(cmd *Command, args []string) bool { if err := util.TestFolderWritable(*filerOptions.dir); err != nil { glog.Fatalf("Check Mapping Meta Folder (-filer.dir=\"%s\") Writable: %s", *filerOptions.dir, err) } - if *filerOptions.get_ip_whitelist_option != "" { - glog.V(0).Infof("Filer GET IP whitelist: %s", *filerOptions.get_ip_whitelist_option) - filerOptions.get_ip_whitelist = strings.Split(*filerOptions.get_ip_whitelist_option, ",") - } - if *filerOptions.get_root_whitelist_option != "" { - glog.V(0).Infof("Filer GET root whitelist: %s", *filerOptions.get_root_whitelist_option) - filerOptions.get_root_whitelist = strings.Split(*filerOptions.get_root_whitelist_option, ",") - } - if *filerOptions.head_ip_whitelist_option != "" { - glog.V(0).Infof("Filer HEAD IP whitelist: %s", *filerOptions.head_ip_whitelist_option) - filerOptions.head_ip_whitelist = strings.Split(*filerOptions.head_ip_whitelist_option, ",") - } - if *filerOptions.head_root_whitelist_option != "" { - glog.V(0).Infof("Filer HEAD root whitelist: %s", *filerOptions.head_root_whitelist_option) - filerOptions.head_root_whitelist = strings.Split(*filerOptions.head_root_whitelist_option, ",") - } - if *filerOptions.delete_ip_whitelist_option != "" { - glog.V(0).Infof("Filer DELETE IP whitelist: %s", *filerOptions.delete_ip_whitelist_option) - filerOptions.delete_ip_whitelist = strings.Split(*filerOptions.delete_ip_whitelist_option, ",") - } - if *filerOptions.delete_root_whitelist_option != "" { - glog.V(0).Infof("Filer DELETE root whitelist: %s", *filerOptions.delete_root_whitelist_option) - filerOptions.delete_root_whitelist = strings.Split(*filerOptions.delete_root_whitelist_option, ",") - } - if *filerOptions.put_ip_whitelist_option != "" { - glog.V(0).Infof("Filer PUT IP whitelist: %s", *filerOptions.put_ip_whitelist_option) - filerOptions.put_ip_whitelist = strings.Split(*filerOptions.put_ip_whitelist_option, ",") - } - if *filerOptions.put_root_whitelist_option != "" { - glog.V(0).Infof("Filer PUT root whitelist: %s", *filerOptions.put_root_whitelist_option) - filerOptions.put_root_whitelist = strings.Split(*filerOptions.put_root_whitelist_option, ",") - } - if *filerOptions.post_ip_whitelist_option != "" { - glog.V(0).Infof("Filer POST IP whitelist: %s", *filerOptions.post_ip_whitelist_option) - filerOptions.post_ip_whitelist = strings.Split(*filerOptions.post_ip_whitelist_option, ",") - } - if *filerOptions.post_root_whitelist_option != "" { - glog.V(0).Infof("Filer POST root whitelist: %s", *filerOptions.post_root_whitelist_option) - filerOptions.post_root_whitelist = strings.Split(*filerOptions.post_root_whitelist_option, ",") - } } if err := util.TestFolderWritable(*masterMetaFolder); err != nil { glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err) } - if *serverReadWhiteListOption != "" { - serverReadWhiteList = strings.Split(*serverReadWhiteListOption, ",") - } - if *serverWriteWhiteListOption != "" { - serverWriteWhiteList = strings.Split(*serverWriteWhiteListOption, ",") + if *serverWhiteListOption != "" { + serverWhiteList = strings.Split(*serverWhiteListOption, ",") } if *isStartingFiler { @@ -234,9 +174,6 @@ func runServer(cmd *Command, args []string) bool { *filerOptions.secretKey, *filerOptions.cassandra_server, *filerOptions.cassandra_keyspace, *filerOptions.redis_server, *filerOptions.redis_password, *filerOptions.redis_database, - filerOptions.get_ip_whitelist, filerOptions.head_ip_whitelist, filerOptions.delete_ip_whitelist, filerOptions.put_ip_whitelist, filerOptions.post_ip_whitelist, - filerOptions.get_root_whitelist, filerOptions.head_root_whitelist, filerOptions.delete_root_whitelist, filerOptions.put_root_whitelist, filerOptions.post_root_whitelist, - *f.get_secure_key, *f.head_secure_key, *f.delete_secure_key, *f.put_secure_key, *f.post_secure_key, ) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) @@ -265,7 +202,7 @@ func runServer(cmd *Command, args []string) bool { r := mux.NewRouter() ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder, *masterVolumeSizeLimitMB, *volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *serverGarbageThreshold, - serverReadWhiteList, serverWriteWhiteList, nil, *serverSecureKey, + serverWhiteList, *serverSecureKey, ) glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort)) @@ -319,7 +256,7 @@ func runServer(cmd *Command, args []string) bool { folders, maxCounts, volumeNeedleMapKind, *serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack, - serverReadWhiteList, serverWriteWhiteList, nil, *volumeFixJpgOrientation, *volumeReadRedirect, + serverWhiteList, *volumeFixJpgOrientation, *volumeReadRedirect, ) glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort)) diff --git a/weed/command/volume.go b/weed/command/volume.go index 68f5edd9e..21369cbe9 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -2,7 +2,6 @@ package command import ( "net/http" - _ "net/http/pprof" "os" "runtime" "strconv" @@ -33,8 +32,7 @@ type VolumeServerOptions struct { maxCpu *int dataCenter *string rack *string - readWhitelist []string - writeWhitelist []string + whiteList []string indexType *string fixJpgOrientation *bool readRedirect *bool @@ -69,8 +67,7 @@ var cmdVolume = &Command{ var ( volumeFolders = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...") maxVolumeCounts = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...") - volumeReadWhiteListOption = cmdVolume.Flag.String("read.whitelist", "", "comma separated Ip addresses having read permission. No limit if empty.") - volumeWriteWhiteListOption = cmdVolume.Flag.String("write.whitelist", "", "comma separated Ip addresses having write permission. No limit if empty.") + volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") ) func runVolume(cmd *Command, args []string) bool { @@ -99,11 +96,8 @@ func runVolume(cmd *Command, args []string) bool { } //security related white list configuration - if *volumeReadWhiteListOption != "" { - v.readWhitelist = strings.Split(*volumeReadWhiteListOption, ",") - } - if *volumeWriteWhiteListOption != "" { - v.writeWhitelist = strings.Split(*volumeWriteWhiteListOption, ",") + if *volumeWhiteListOption != "" { + v.whiteList = strings.Split(*volumeWhiteListOption, ",") } if *v.ip == "" { @@ -136,7 +130,7 @@ func runVolume(cmd *Command, args []string) bool { v.folders, v.folderMaxLimits, volumeNeedleMapKind, *v.master, *v.pulseSeconds, *v.dataCenter, *v.rack, - v.readWhitelist, v.writeWhitelist, nil, + v.whiteList, *v.fixJpgOrientation, *v.readRedirect, ) diff --git a/weed/security/guard.go b/weed/security/guard.go index 6292c67c9..dea3b12f2 100644 --- a/weed/security/guard.go +++ b/weed/security/guard.go @@ -41,31 +41,17 @@ https://github.com/pkieltyka/jwtauth/blob/master/jwtauth.go */ type Guard struct { - ipWhiteList []string - rootWhiteList []string + whiteList []string SecretKey Secret isActive bool } -func NewGuard(ipWhiteList []string, rootWhiteList []string, secretKey string) *Guard { - g := &Guard{ipWhiteList: ipWhiteList, rootWhiteList: rootWhiteList, SecretKey: Secret(secretKey)} - g.isActive = len(g.ipWhiteList) != 0 || len(g.SecretKey) != 0 +func NewGuard(whiteList []string, secretKey string) *Guard { + g := &Guard{whiteList: whiteList, SecretKey: Secret(secretKey)} + g.isActive = len(g.whiteList) != 0 || len(g.SecretKey) != 0 return g } -func (g *Guard) WhiteList2(f func(w http.ResponseWriter, r *http.Request, b bool)) func(w http.ResponseWriter, r *http.Request, b bool) { - if !g.isActive { - //if no security needed, just skip all checkings - return f - } - return func(w http.ResponseWriter, r *http.Request, b bool) { - if err := g.checkWhiteList(w, r); err != nil { - w.WriteHeader(http.StatusUnauthorized) - return - } - f(w, r, b) - } -} func (g *Guard) WhiteList(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { if !g.isActive { @@ -110,14 +96,13 @@ func GetActualRemoteHost(r *http.Request) (host string, err error) { } func (g *Guard) checkWhiteList(w http.ResponseWriter, r *http.Request) error { - if len(g.ipWhiteList) == 0 { - glog.V(0).Info("No whitelist specified for operation") + if len(g.whiteList) == 0 { return nil } host, err := GetActualRemoteHost(r) if err == nil { - for _, ip := range g.ipWhiteList { + for _, ip := range g.whiteList { // If the whitelist entry contains a "/" it // is a CIDR range, and we should check the @@ -129,7 +114,6 @@ func (g *Guard) checkWhiteList(w http.ResponseWriter, r *http.Request) error { } remote := net.ParseIP(host) if cidrnet.Contains(remote) { - glog.V(0).Infof("Found %s in CIDR whitelist.", r.RemoteAddr) return nil } } @@ -138,30 +122,10 @@ func (g *Guard) checkWhiteList(w http.ResponseWriter, r *http.Request) error { // Otherwise we're looking for a literal match. // if ip == host { - glog.V(0).Infof("Found %s in whitelist.", r.RemoteAddr) - return nil - } - // ::1 is the same as 127.0.0.1 and localhost - if host == "::1" && (ip == "127.0.0.1" || ip == "localhost") { - glog.V(0).Infof("Found %s (localhost) in whitelist.", r.RemoteAddr) return nil } } } - // The root whitelist allows exceptions to the IP whitelist, but only by certain root paths in the request. - if len(g.rootWhiteList) > 0 { - pathParts := strings.Split(r.RequestURI, "/") - if len(pathParts) > 0 { - requestedRoot := pathParts[1] - for _, root := range g.rootWhiteList { - if root == requestedRoot { - glog.V(0).Infof("Found %s in root whitelist.", requestedRoot) - return nil - } - } - glog.V(0).Infof("Not in root whitelist: %s", requestedRoot) - } - } glog.V(0).Infof("Not in whitelist: %s", r.RemoteAddr) return fmt.Errorf("Not in whitelis: %s", r.RemoteAddr) diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 2a432af65..3c7c1fd9e 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -30,11 +30,6 @@ type FilerServer struct { filer filer.Filer maxMB int masterNodes *storage.MasterNodes - get_guard *security.Guard - head_guard *security.Guard - delete_guard *security.Guard - put_guard *security.Guard - post_guard *security.Guard } func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir string, collection string, @@ -43,9 +38,6 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st secret string, cassandra_server string, cassandra_keyspace string, redis_server string, redis_password string, redis_database int, - get_ip_whitelist []string, head_ip_whitelist []string, delete_ip_whitelist []string, put_ip_whitelist []string, post_ip_whitelist []string, - get_root_whitelist []string, head_root_whitelist []string, delete_root_whitelist []string, put_root_whitelist []string, post_root_whitelist []string, - get_secure_key string, head_secure_key string, delete_secure_key string, put_secure_key string, post_secure_key string, ) (fs *FilerServer, err error) { fs = &FilerServer{ master: master, @@ -54,11 +46,6 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st redirectOnRead: redirectOnRead, disableDirListing: disableDirListing, maxMB: maxMB, - get_guard: security.NewGuard(get_ip_whitelist, get_root_whitelist, get_secure_key), - head_guard: security.NewGuard(head_ip_whitelist, head_root_whitelist, head_secure_key), - delete_guard: security.NewGuard(delete_ip_whitelist, delete_root_whitelist, delete_secure_key), - put_guard: security.NewGuard(put_ip_whitelist, put_root_whitelist, put_secure_key), - post_guard: security.NewGuard(post_ip_whitelist, post_root_whitelist, post_secure_key), port: ip + ":" + strconv.Itoa(port), } diff --git a/weed/server/filer_server_handlers.go b/weed/server/filer_server_handlers.go index 4db170590..4e39258af 100644 --- a/weed/server/filer_server_handlers.go +++ b/weed/server/filer_server_handlers.go @@ -7,14 +7,14 @@ import ( func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { switch r.Method { case "GET": - fs.get_guard.WhiteList2(fs.GetOrHeadHandler)(w, r, true) + fs.GetOrHeadHandler(w, r, true) case "HEAD": - fs.head_guard.WhiteList2(fs.GetOrHeadHandler)(w, r, false) + fs.GetOrHeadHandler(w, r, false) case "DELETE": - fs.delete_guard.WhiteList(fs.DeleteHandler)(w, r) + fs.DeleteHandler(w, r) case "PUT": - fs.put_guard.WhiteList(fs.PostHandler)(w, r) + fs.PostHandler(w, r) case "POST": - fs.post_guard.WhiteList(fs.PostHandler)(w, r) + fs.PostHandler(w, r) } } diff --git a/weed/server/master_server.go b/weed/server/master_server.go index 3bd77c819..61bda6988 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -23,8 +23,7 @@ type MasterServer struct { pulseSeconds int defaultReplicaPlacement string garbageThreshold string - read_guard *security.Guard - write_guard *security.Guard + guard *security.Guard Topo *topology.Topology vg *topology.VolumeGrowth @@ -39,9 +38,7 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string, confFile string, defaultReplicaPlacement string, garbageThreshold string, - ipReadWhiteList []string, - ipWriteWhiteList []string, - rootWhiteList []string, + whiteList []string, secureKey string, ) *MasterServer { ms := &MasterServer{ @@ -61,25 +58,24 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string, ms.vg = topology.NewDefaultVolumeGrowth() glog.V(0).Infoln("Volume Size Limit is", volumeSizeLimitMB, "MB") - ms.read_guard = security.NewGuard(ipReadWhiteList, rootWhiteList, secureKey) - ms.write_guard = security.NewGuard(ipWriteWhiteList, rootWhiteList, secureKey) + ms.guard = security.NewGuard(whiteList, secureKey) r.HandleFunc("/", ms.uiStatusHandler) - r.HandleFunc("/ui/index.html", ms.read_guard.WhiteList(ms.uiStatusHandler)) - r.HandleFunc("/dir/assign", ms.proxyToLeader(ms.write_guard.WhiteList(ms.dirAssignHandler))) - r.HandleFunc("/dir/lookup", ms.proxyToLeader(ms.read_guard.WhiteList(ms.dirLookupHandler))) - r.HandleFunc("/dir/join", ms.proxyToLeader(ms.write_guard.WhiteList(ms.dirJoinHandler))) - r.HandleFunc("/dir/status", ms.proxyToLeader(ms.read_guard.WhiteList(ms.dirStatusHandler))) - r.HandleFunc("/col/delete", ms.proxyToLeader(ms.write_guard.WhiteList(ms.collectionDeleteHandler))) - r.HandleFunc("/vol/lookup", ms.proxyToLeader(ms.read_guard.WhiteList(ms.volumeLookupHandler))) - r.HandleFunc("/vol/grow", ms.proxyToLeader(ms.write_guard.WhiteList(ms.volumeGrowHandler))) - r.HandleFunc("/vol/status", ms.proxyToLeader(ms.read_guard.WhiteList(ms.volumeStatusHandler))) - r.HandleFunc("/vol/vacuum", ms.proxyToLeader(ms.write_guard.WhiteList(ms.volumeVacuumHandler))) - r.HandleFunc("/submit", ms.write_guard.WhiteList(ms.submitFromMasterServerHandler)) - r.HandleFunc("/delete", ms.write_guard.WhiteList(ms.deleteFromMasterServerHandler)) - r.HandleFunc("/{fileId}", ms.proxyToLeader(ms.read_guard.WhiteList(ms.redirectHandler))) - r.HandleFunc("/stats/counter", ms.read_guard.WhiteList(statsCounterHandler)) - r.HandleFunc("/stats/memory", ms.read_guard.WhiteList(statsMemoryHandler)) + r.HandleFunc("/ui/index.html", ms.uiStatusHandler) + r.HandleFunc("/dir/assign", ms.proxyToLeader(ms.guard.WhiteList(ms.dirAssignHandler))) + r.HandleFunc("/dir/lookup", ms.proxyToLeader(ms.guard.WhiteList(ms.dirLookupHandler))) + r.HandleFunc("/dir/join", ms.proxyToLeader(ms.guard.WhiteList(ms.dirJoinHandler))) + r.HandleFunc("/dir/status", ms.proxyToLeader(ms.guard.WhiteList(ms.dirStatusHandler))) + r.HandleFunc("/col/delete", ms.proxyToLeader(ms.guard.WhiteList(ms.collectionDeleteHandler))) + r.HandleFunc("/vol/lookup", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeLookupHandler))) + r.HandleFunc("/vol/grow", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeGrowHandler))) + r.HandleFunc("/vol/status", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeStatusHandler))) + r.HandleFunc("/vol/vacuum", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeVacuumHandler))) + r.HandleFunc("/submit", ms.guard.WhiteList(ms.submitFromMasterServerHandler)) + r.HandleFunc("/delete", ms.guard.WhiteList(ms.deleteFromMasterServerHandler)) + r.HandleFunc("/{fileId}", ms.proxyToLeader(ms.redirectHandler)) + r.HandleFunc("/stats/counter", ms.guard.WhiteList(statsCounterHandler)) + r.HandleFunc("/stats/memory", ms.guard.WhiteList(statsMemoryHandler)) ms.Topo.StartRefreshWritableVolumes(garbageThreshold) diff --git a/weed/server/master_server_handlers_admin.go b/weed/server/master_server_handlers_admin.go index 385290079..a762bf416 100644 --- a/weed/server/master_server_handlers_admin.go +++ b/weed/server/master_server_handlers_admin.go @@ -61,7 +61,7 @@ func (ms *MasterServer) dirJoinHandler(w http.ResponseWriter, r *http.Request) { ms.Topo.ProcessJoinMessage(joinMessage) writeJsonQuiet(w, r, http.StatusOK, operation.JoinResult{ VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024, - SecretKey: string(ms.write_guard.SecretKey), + SecretKey: string(ms.guard.SecretKey), }) } diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index a40eeb9c0..79a4276b1 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -9,7 +9,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/storage" - //"net/http/pprof" ) type VolumeServer struct { @@ -19,8 +18,7 @@ type VolumeServer struct { dataCenter string rack string store *storage.Store - read_guard *security.Guard - write_guard *security.Guard + guard *security.Guard needleMapKind storage.NeedleMapType FixJpgOrientation bool @@ -33,9 +31,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, needleMapKind storage.NeedleMapType, masterNode string, pulseSeconds int, dataCenter string, rack string, - ipReadWhiteList []string, - ipWriteWhiteList []string, - rootWhiteList []string, + whiteList []string, fixJpgOrientation bool, readRedirect bool) *VolumeServer { vs := &VolumeServer{ @@ -49,40 +45,28 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, vs.SetMasterNode(masterNode) vs.store = storage.NewStore(port, ip, publicUrl, folders, maxCounts, vs.needleMapKind) - vs.read_guard = security.NewGuard(ipReadWhiteList, rootWhiteList, "") - vs.write_guard = security.NewGuard(ipWriteWhiteList, rootWhiteList, "") + vs.guard = security.NewGuard(whiteList, "") - adminMux.HandleFunc("/ui/index.html", vs.read_guard.WhiteList(vs.uiStatusHandler)) - adminMux.HandleFunc("/status", vs.read_guard.WhiteList(vs.statusHandler)) - adminMux.HandleFunc("/admin/assign_volume", vs.write_guard.WhiteList(vs.assignVolumeHandler)) - adminMux.HandleFunc("/admin/vacuum/check", vs.write_guard.WhiteList(vs.vacuumVolumeCheckHandler)) - adminMux.HandleFunc("/admin/vacuum/compact", vs.write_guard.WhiteList(vs.vacuumVolumeCompactHandler)) - adminMux.HandleFunc("/admin/vacuum/commit", vs.write_guard.WhiteList(vs.vacuumVolumeCommitHandler)) - adminMux.HandleFunc("/admin/delete_collection", vs.write_guard.WhiteList(vs.deleteCollectionHandler)) - adminMux.HandleFunc("/admin/sync/status", vs.read_guard.WhiteList(vs.getVolumeSyncStatusHandler)) - adminMux.HandleFunc("/admin/sync/index", vs.write_guard.WhiteList(vs.getVolumeIndexContentHandler)) - adminMux.HandleFunc("/admin/sync/data", vs.write_guard.WhiteList(vs.getVolumeDataContentHandler)) - adminMux.HandleFunc("/stats/counter", vs.read_guard.WhiteList(statsCounterHandler)) - adminMux.HandleFunc("/stats/memory", vs.read_guard.WhiteList(statsMemoryHandler)) - adminMux.HandleFunc("/stats/disk", vs.read_guard.WhiteList(vs.statsDiskHandler)) - adminMux.HandleFunc("/delete", vs.write_guard.WhiteList(vs.batchDeleteHandler)) - adminMux.HandleFunc("/", vs.read_guard.WhiteList(vs.privateStoreHandler)) + adminMux.HandleFunc("/ui/index.html", vs.uiStatusHandler) + adminMux.HandleFunc("/status", vs.guard.WhiteList(vs.statusHandler)) + adminMux.HandleFunc("/admin/assign_volume", vs.guard.WhiteList(vs.assignVolumeHandler)) + adminMux.HandleFunc("/admin/vacuum/check", vs.guard.WhiteList(vs.vacuumVolumeCheckHandler)) + adminMux.HandleFunc("/admin/vacuum/compact", vs.guard.WhiteList(vs.vacuumVolumeCompactHandler)) + adminMux.HandleFunc("/admin/vacuum/commit", vs.guard.WhiteList(vs.vacuumVolumeCommitHandler)) + adminMux.HandleFunc("/admin/delete_collection", vs.guard.WhiteList(vs.deleteCollectionHandler)) + adminMux.HandleFunc("/admin/sync/status", vs.guard.WhiteList(vs.getVolumeSyncStatusHandler)) + adminMux.HandleFunc("/admin/sync/index", vs.guard.WhiteList(vs.getVolumeIndexContentHandler)) + adminMux.HandleFunc("/admin/sync/data", vs.guard.WhiteList(vs.getVolumeDataContentHandler)) + adminMux.HandleFunc("/stats/counter", vs.guard.WhiteList(statsCounterHandler)) + adminMux.HandleFunc("/stats/memory", vs.guard.WhiteList(statsMemoryHandler)) + adminMux.HandleFunc("/stats/disk", vs.guard.WhiteList(vs.statsDiskHandler)) + adminMux.HandleFunc("/delete", vs.guard.WhiteList(vs.batchDeleteHandler)) + adminMux.HandleFunc("/", vs.privateStoreHandler) if publicMux != adminMux { // separated admin and public port publicMux.HandleFunc("/favicon.ico", vs.faviconHandler) publicMux.HandleFunc("/", vs.publicReadOnlyHandler) } - /* - // add in profiling support - adminMux.HandleFunc("/debug/pprof/", pprof.Index) - adminMux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - adminMux.HandleFunc("/debug/pprof/profile", pprof.Profile) - adminMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - adminMux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) - adminMux.Handle("/debug/pprof/heap", pprof.Handler("heap")) - adminMux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) - adminMux.Handle("/debug/pprof/block", pprof.Handler("block")) - */ go func() { connected := true @@ -98,8 +82,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, if !connected { connected = true vs.SetMasterNode(master) - vs.read_guard.SecretKey = secretKey - vs.write_guard.SecretKey = secretKey + vs.guard.SecretKey = secretKey glog.V(0).Infoln("Volume Server Connected with master at", master) } } else { @@ -138,5 +121,5 @@ func (vs *VolumeServer) Shutdown() { } func (vs *VolumeServer) jwt(fileId string) security.EncodedJwt { - return security.GenJwt(vs.read_guard.SecretKey, fileId) + return security.GenJwt(vs.guard.SecretKey, fileId) } diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index f223af6c0..2d6fe7849 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -25,19 +25,19 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque switch r.Method { case "GET": stats.ReadRequest() - vs.write_guard.WhiteList(vs.GetOrHeadHandler)(w, r) + vs.GetOrHeadHandler(w, r) case "HEAD": stats.ReadRequest() - vs.write_guard.WhiteList(vs.GetOrHeadHandler)(w, r) + vs.GetOrHeadHandler(w, r) case "DELETE": stats.DeleteRequest() - vs.write_guard.WhiteList(vs.DeleteHandler)(w, r) + vs.guard.WhiteList(vs.DeleteHandler)(w, r) case "PUT": stats.WriteRequest() - vs.write_guard.WhiteList(vs.PostHandler)(w, r) + vs.guard.WhiteList(vs.PostHandler)(w, r) case "POST": stats.WriteRequest() - vs.write_guard.WhiteList(vs.PostHandler)(w, r) + vs.guard.WhiteList(vs.PostHandler)(w, r) } } From 0f4c7dd8fdb2cc9278b8f1342a74b0ac1afa61e1 Mon Sep 17 00:00:00 2001 From: Mike Tolman Date: Fri, 5 Aug 2016 15:46:08 -0600 Subject: [PATCH 07/31] Revert "Ooops. Missed a line." This reverts commit 14d4252904ed0fad8a7d6d6156a70fcbc3eda12c. --- weed/server/filer_server.go | 1 - 1 file changed, 1 deletion(-) diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 3c7c1fd9e..c9bc0e021 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -34,7 +34,6 @@ type FilerServer struct { func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir string, collection string, replication string, redirectOnRead bool, disableDirListing bool, - maxMB int, secret string, cassandra_server string, cassandra_keyspace string, redis_server string, redis_password string, redis_database int, From 0d331c1e3ae0d038ae972279a63d2ff9a70e25f4 Mon Sep 17 00:00:00 2001 From: Mike Tolman Date: Fri, 5 Aug 2016 15:46:45 -0600 Subject: [PATCH 08/31] Revert "Changing needle_byte_cache so that it doesn't grow so big when larger files are added." This reverts commit 87fee21ef597a8b1bac5352d1327c13f87eeb000. --- weed/storage/needle_byte_cache.go | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/weed/storage/needle_byte_cache.go b/weed/storage/needle_byte_cache.go index 930ead81d..ae35a48ba 100644 --- a/weed/storage/needle_byte_cache.go +++ b/weed/storage/needle_byte_cache.go @@ -8,7 +8,6 @@ import ( "github.com/hashicorp/golang-lru" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/chrislusf/seaweedfs/weed/glog" ) var ( @@ -25,7 +24,7 @@ In caching, the string~[]byte mapping is cached */ func init() { bytesPool = util.NewBytesPool() - bytesCache, _ = lru.NewWithEvict(50, func(key interface{}, value interface{}) { + bytesCache, _ = lru.NewWithEvict(512, func(key interface{}, value interface{}) { value.(*Block).decreaseReference() }) } @@ -47,37 +46,22 @@ func (block *Block) increaseReference() { // get bytes from the LRU cache of []byte first, then from the bytes pool // when []byte in LRU cache is evicted, it will be put back to the bytes pool func getBytesForFileBlock(r *os.File, offset int64, readSize int) (dataSlice []byte, block *Block, err error) { - //Skip the cache if we are looking for a block that is too big to fit in the cache (defaulting to 10MB) - cacheable := readSize <= (1024*1024*10) - if !cacheable { - glog.V(4).Infoln("Block too big to keep in cache. Size:", readSize) - } - cacheKey := string("") - if cacheable { // check cache, return if found - cacheKey = fmt.Sprintf("%d:%d:%d", r.Fd(), offset >> 3, readSize) + cacheKey := fmt.Sprintf("%d:%d:%d", r.Fd(), offset>>3, readSize) if obj, found := bytesCache.Get(cacheKey); found { - glog.V(4).Infoln("Found block in cache. Size:", readSize) block = obj.(*Block) block.increaseReference() dataSlice = block.Bytes[0:readSize] return dataSlice, block, nil - } } // get the []byte from pool b := bytesPool.Get(readSize) // refCount = 2, one by the bytesCache, one by the actual needle object - refCount := int32(1) - if cacheable { - refCount = 2 - } - block = &Block{Bytes: b, refCount: refCount} + block = &Block{Bytes: b, refCount: 2} dataSlice = block.Bytes[0:readSize] _, err = r.ReadAt(dataSlice, offset) - if cacheable { bytesCache.Add(cacheKey, block) - } return dataSlice, block, err } From a89a3c86d0bfa20ead98fec1d286cdc6018c3bde Mon Sep 17 00:00:00 2001 From: Mike Tolman Date: Fri, 5 Aug 2016 15:47:46 -0600 Subject: [PATCH 09/31] Revert "Add AutoChunking to the Filer API, so that you can upload really large files through the filer API." This reverts commit 09059bfdccdeff1a588ee1326318075adb068b0f. --- weed/command/filer.go | 3 - weed/command/server.go | 2 - weed/server/filer_server.go | 2 - weed/server/filer_server_handlers_write.go | 208 --------------------- 4 files changed, 215 deletions(-) diff --git a/weed/command/filer.go b/weed/command/filer.go index 0bd508e0b..582d4e9c8 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -24,7 +24,6 @@ type FilerOptions struct { dir *string redirectOnRead *bool disableDirListing *bool - maxMB *int secretKey *string cassandra_server *string cassandra_keyspace *string @@ -43,7 +42,6 @@ func init() { f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified") f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing") - f.maxMB = cmdFiler.Flag.Int("maxMB", 0, "split files larger than the limit") f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server") f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server") f.redis_server = cmdFiler.Flag.String("redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379") @@ -84,7 +82,6 @@ func runFiler(cmd *Command, args []string) bool { r := http.NewServeMux() _, nfs_err := weed_server.NewFilerServer(r, *f.ip, *f.port, *f.master, *f.dir, *f.collection, *f.defaultReplicaPlacement, *f.redirectOnRead, *f.disableDirListing, - *f.maxMB, *f.secretKey, *f.cassandra_server, *f.cassandra_keyspace, *f.redis_server, *f.redis_password, *f.redis_database, diff --git a/weed/command/server.go b/weed/command/server.go index 7a6677a65..1211c7137 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -86,7 +86,6 @@ func init() { filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.") filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") - filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 0, "split files larger than the limit") filerOptions.cassandra_server = cmdServer.Flag.String("filer.cassandra.server", "", "host[:port] of the cassandra server") filerOptions.cassandra_keyspace = cmdServer.Flag.String("filer.cassandra.keyspace", "seaweed", "keyspace of the cassandra server") filerOptions.redis_server = cmdServer.Flag.String("filer.redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379") @@ -170,7 +169,6 @@ func runServer(cmd *Command, args []string) bool { _, nfs_err := weed_server.NewFilerServer(r, *serverBindIp, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection, *filerOptions.defaultReplicaPlacement, *filerOptions.redirectOnRead, *filerOptions.disableDirListing, - *filerOptions.maxMB, *filerOptions.secretKey, *filerOptions.cassandra_server, *filerOptions.cassandra_keyspace, *filerOptions.redis_server, *filerOptions.redis_password, *filerOptions.redis_database, diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index c9bc0e021..b99bbd7c9 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -28,7 +28,6 @@ type FilerServer struct { disableDirListing bool secret security.Secret filer filer.Filer - maxMB int masterNodes *storage.MasterNodes } @@ -44,7 +43,6 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st defaultReplication: replication, redirectOnRead: redirectOnRead, disableDirListing: disableDirListing, - maxMB: maxMB, port: ip + ":" + strconv.Itoa(port), } diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 872d8c4b9..e2d40f532 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -20,8 +20,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" "github.com/syndtr/goleveldb/leveldb" - "path" - "strconv" ) type FilerPostResult struct { @@ -219,7 +217,6 @@ func (fs *FilerServer) monolithicUploadAnalyzer(w http.ResponseWriter, r *http.R } func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { - query := r.URL.Query() replication := query.Get("replication") if replication == "" { @@ -230,10 +227,6 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { collection = fs.collection } - if autoChunked := fs.autoChunk(w, r, replication, collection); autoChunked { - return - } - var fileId, urlLocation string var err error @@ -250,17 +243,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { } u, _ := url.Parse(urlLocation) - - // This allows a client to generate a chunk manifest and submit it to the filer -- it is a little off - // because they need to provide FIDs instead of file paths... - cm, _ := strconv.ParseBool(query.Get("cm")) - if cm { - q := u.Query() - q.Set("cm", "true") - u.RawQuery = q.Encode() - } glog.V(4).Infoln("post to", u) - request := &http.Request{ Method: r.Method, URL: u, @@ -336,197 +319,6 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { writeJsonQuiet(w, r, http.StatusCreated, reply) } -func (fs *FilerServer) autoChunk(w http.ResponseWriter, r *http.Request, replication string, collection string) bool { - if r.Method != "POST" { - glog.V(4).Infoln("AutoChunking not supported for method", r.Method) - return false - } - - // autoChunking can be set at the command-line level or as a query param. Query param overrides command-line - query := r.URL.Query() - - parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32) - maxMB := int32(parsedMaxMB) - if maxMB <= 0 && fs.maxMB > 0 { - maxMB = int32(fs.maxMB) - } - if maxMB <= 0 { - glog.V(4).Infoln("AutoChunking not enabled") - return false - } - glog.V(4).Infoln("AutoChunking level set to", maxMB, "(MB)") - - chunkSize := 1024 * 1024 * maxMB - - contentLength := int64(0) - if contentLengthHeader := r.Header["Content-Length"]; len(contentLengthHeader) == 1 { - contentLength, _ = strconv.ParseInt(contentLengthHeader[0], 10, 64) - if contentLength <= int64(chunkSize) { - glog.V(4).Infoln("Content-Length of", contentLength, "is less than the chunk size of", chunkSize, "so autoChunking will be skipped.") - return false - } - } - - if contentLength <= 0 { - glog.V(4).Infoln("Content-Length value is missing or unexpected so autoChunking will be skipped.") - return false - } - - reply, err := fs.doAutoChunk(w, r, contentLength, chunkSize, replication, collection) - if err != nil { - writeJsonError(w, r, http.StatusInternalServerError, err) - } else if reply != nil { - writeJsonQuiet(w, r, http.StatusCreated, reply) - } - return true -} - -func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, contentLength int64, chunkSize int32, replication string, collection string) (filerResult *FilerPostResult, replyerr error) { - - multipartReader, multipartReaderErr := r.MultipartReader() - if multipartReaderErr != nil { - return nil, multipartReaderErr - } - - part1, part1Err := multipartReader.NextPart() - if part1Err != nil { - return nil, part1Err - } - - fileName := part1.FileName() - if fileName != "" { - fileName = path.Base(fileName) - } - - chunks := (int64(contentLength) / int64(chunkSize)) + 1 - cm := operation.ChunkManifest{ - Name: fileName, - Size: 0, // don't know yet - Mime: "application/octet-stream", - Chunks: make([]*operation.ChunkInfo, 0, chunks), - } - - totalBytesRead := int64(0) - tmpBufferSize := int32(1024 * 1024) - tmpBuffer := bytes.NewBuffer(make([]byte, 0, tmpBufferSize)) - chunkBuf := make([]byte, chunkSize+tmpBufferSize, chunkSize+tmpBufferSize) // chunk size plus a little overflow - chunkBufOffset := int32(0) - chunkOffset := int64(0) - writtenChunks := 0 - - filerResult = &FilerPostResult{ - Name: fileName, - } - - for totalBytesRead < contentLength { - tmpBuffer.Reset() - bytesRead, readErr := io.CopyN(tmpBuffer, part1, int64(tmpBufferSize)) - readFully := readErr != nil && readErr == io.EOF - tmpBuf := tmpBuffer.Bytes() - bytesToCopy := tmpBuf[0:int(bytesRead)] - - copy(chunkBuf[chunkBufOffset:chunkBufOffset+int32(bytesRead)], bytesToCopy) - chunkBufOffset = chunkBufOffset + int32(bytesRead) - - if chunkBufOffset >= chunkSize || readFully || (chunkBufOffset > 0 && bytesRead == 0) { - writtenChunks = writtenChunks + 1 - fileId, urlLocation, assignErr := fs.assignNewFileInfo(w, r, replication, collection) - if assignErr != nil { - return nil, assignErr - } - - // upload the chunk to the volume server - chunkName := fileName + "_chunk_" + strconv.FormatInt(int64(cm.Chunks.Len()+1), 10) - uploadErr := fs.doUpload(urlLocation, w, r, chunkBuf[0:chunkBufOffset], chunkName, "application/octet-stream", fileId) - if uploadErr != nil { - return nil, uploadErr - } - - // Save to chunk manifest structure - cm.Chunks = append(cm.Chunks, - &operation.ChunkInfo{ - Offset: chunkOffset, - Size: int64(chunkBufOffset), - Fid: fileId, - }, - ) - - // reset variables for the next chunk - chunkBufOffset = 0 - chunkOffset = totalBytesRead + int64(bytesRead) - } - - totalBytesRead = totalBytesRead + int64(bytesRead) - - if bytesRead == 0 || readFully { - break - } - - if readErr != nil { - return nil, readErr - } - } - - cm.Size = totalBytesRead - manifestBuf, marshalErr := cm.Marshal() - if marshalErr != nil { - return nil, marshalErr - } - - manifestStr := string(manifestBuf) - glog.V(4).Infoln("Generated chunk manifest: ", manifestStr) - - manifestFileId, manifestUrlLocation, manifestAssignmentErr := fs.assignNewFileInfo(w, r, replication, collection) - if manifestAssignmentErr != nil { - return nil, manifestAssignmentErr - } - glog.V(4).Infoln("Manifest uploaded to:", manifestUrlLocation, "Fid:", manifestFileId) - filerResult.Fid = manifestFileId - - u, _ := url.Parse(manifestUrlLocation) - q := u.Query() - q.Set("cm", "true") - u.RawQuery = q.Encode() - - manifestUploadErr := fs.doUpload(u.String(), w, r, manifestBuf, fileName+"_manifest", "application/json", manifestFileId) - if manifestUploadErr != nil { - return nil, manifestUploadErr - } - - path := r.URL.Path - // also delete the old fid unless PUT operation - if r.Method != "PUT" { - if oldFid, err := fs.filer.FindFile(path); err == nil { - operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) - } - } - - glog.V(4).Infoln("saving", path, "=>", manifestFileId) - if db_err := fs.filer.CreateFile(path, manifestFileId); db_err != nil { - replyerr = db_err - filerResult.Error = db_err.Error() - operation.DeleteFile(fs.getMasterNode(), manifestFileId, fs.jwt(manifestFileId)) //clean up - glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err) - return - } - - return -} - -func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, chunkBuf []byte, fileName string, contentType string, fileId string) (err error) { - err = nil - - ioReader := ioutil.NopCloser(bytes.NewBuffer(chunkBuf)) - uploadResult, uploadError := operation.Upload(urlLocation, fileName, ioReader, false, contentType, fs.jwt(fileId)) - if uploadResult != nil { - glog.V(0).Infoln("Chunk upload result. Name:", uploadResult.Name, "Fid:", fileId, "Size:", uploadResult.Size) - } - if uploadError != nil { - err = uploadError - } - return -} - // curl -X DELETE http://localhost:8888/path/to // curl -X DELETE http://localhost:8888/path/to?recursive=true func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { From 01d3f69c5230c31dd37c5b6f2476f155bab9c9a6 Mon Sep 17 00:00:00 2001 From: Mike Tolman Date: Fri, 5 Aug 2016 16:01:30 -0600 Subject: [PATCH 10/31] Adding AutoChunk/MaxMB Support to Filer API This is related to the following issue I added to chrislusf/seaweedfs: https://github.com/chrislusf/seaweedfs/issues/342 --- weed/command/filer.go | 3 + weed/command/server.go | 2 + weed/server/filer_server.go | 3 + weed/server/filer_server_handlers_write.go | 208 +++++++++++++++++++++ 4 files changed, 216 insertions(+) diff --git a/weed/command/filer.go b/weed/command/filer.go index 582d4e9c8..0bd508e0b 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -24,6 +24,7 @@ type FilerOptions struct { dir *string redirectOnRead *bool disableDirListing *bool + maxMB *int secretKey *string cassandra_server *string cassandra_keyspace *string @@ -42,6 +43,7 @@ func init() { f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified") f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing") + f.maxMB = cmdFiler.Flag.Int("maxMB", 0, "split files larger than the limit") f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server") f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server") f.redis_server = cmdFiler.Flag.String("redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379") @@ -82,6 +84,7 @@ func runFiler(cmd *Command, args []string) bool { r := http.NewServeMux() _, nfs_err := weed_server.NewFilerServer(r, *f.ip, *f.port, *f.master, *f.dir, *f.collection, *f.defaultReplicaPlacement, *f.redirectOnRead, *f.disableDirListing, + *f.maxMB, *f.secretKey, *f.cassandra_server, *f.cassandra_keyspace, *f.redis_server, *f.redis_password, *f.redis_database, diff --git a/weed/command/server.go b/weed/command/server.go index 1211c7137..7a6677a65 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -86,6 +86,7 @@ func init() { filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.") filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") + filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 0, "split files larger than the limit") filerOptions.cassandra_server = cmdServer.Flag.String("filer.cassandra.server", "", "host[:port] of the cassandra server") filerOptions.cassandra_keyspace = cmdServer.Flag.String("filer.cassandra.keyspace", "seaweed", "keyspace of the cassandra server") filerOptions.redis_server = cmdServer.Flag.String("filer.redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379") @@ -169,6 +170,7 @@ func runServer(cmd *Command, args []string) bool { _, nfs_err := weed_server.NewFilerServer(r, *serverBindIp, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection, *filerOptions.defaultReplicaPlacement, *filerOptions.redirectOnRead, *filerOptions.disableDirListing, + *filerOptions.maxMB, *filerOptions.secretKey, *filerOptions.cassandra_server, *filerOptions.cassandra_keyspace, *filerOptions.redis_server, *filerOptions.redis_password, *filerOptions.redis_database, diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index b99bbd7c9..3c7c1fd9e 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -28,11 +28,13 @@ type FilerServer struct { disableDirListing bool secret security.Secret filer filer.Filer + maxMB int masterNodes *storage.MasterNodes } func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir string, collection string, replication string, redirectOnRead bool, disableDirListing bool, + maxMB int, secret string, cassandra_server string, cassandra_keyspace string, redis_server string, redis_password string, redis_database int, @@ -43,6 +45,7 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st defaultReplication: replication, redirectOnRead: redirectOnRead, disableDirListing: disableDirListing, + maxMB: maxMB, port: ip + ":" + strconv.Itoa(port), } diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index e2d40f532..872d8c4b9 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -20,6 +20,8 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" "github.com/syndtr/goleveldb/leveldb" + "path" + "strconv" ) type FilerPostResult struct { @@ -217,6 +219,7 @@ func (fs *FilerServer) monolithicUploadAnalyzer(w http.ResponseWriter, r *http.R } func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() replication := query.Get("replication") if replication == "" { @@ -227,6 +230,10 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { collection = fs.collection } + if autoChunked := fs.autoChunk(w, r, replication, collection); autoChunked { + return + } + var fileId, urlLocation string var err error @@ -243,7 +250,17 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { } u, _ := url.Parse(urlLocation) + + // This allows a client to generate a chunk manifest and submit it to the filer -- it is a little off + // because they need to provide FIDs instead of file paths... + cm, _ := strconv.ParseBool(query.Get("cm")) + if cm { + q := u.Query() + q.Set("cm", "true") + u.RawQuery = q.Encode() + } glog.V(4).Infoln("post to", u) + request := &http.Request{ Method: r.Method, URL: u, @@ -319,6 +336,197 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { writeJsonQuiet(w, r, http.StatusCreated, reply) } +func (fs *FilerServer) autoChunk(w http.ResponseWriter, r *http.Request, replication string, collection string) bool { + if r.Method != "POST" { + glog.V(4).Infoln("AutoChunking not supported for method", r.Method) + return false + } + + // autoChunking can be set at the command-line level or as a query param. Query param overrides command-line + query := r.URL.Query() + + parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32) + maxMB := int32(parsedMaxMB) + if maxMB <= 0 && fs.maxMB > 0 { + maxMB = int32(fs.maxMB) + } + if maxMB <= 0 { + glog.V(4).Infoln("AutoChunking not enabled") + return false + } + glog.V(4).Infoln("AutoChunking level set to", maxMB, "(MB)") + + chunkSize := 1024 * 1024 * maxMB + + contentLength := int64(0) + if contentLengthHeader := r.Header["Content-Length"]; len(contentLengthHeader) == 1 { + contentLength, _ = strconv.ParseInt(contentLengthHeader[0], 10, 64) + if contentLength <= int64(chunkSize) { + glog.V(4).Infoln("Content-Length of", contentLength, "is less than the chunk size of", chunkSize, "so autoChunking will be skipped.") + return false + } + } + + if contentLength <= 0 { + glog.V(4).Infoln("Content-Length value is missing or unexpected so autoChunking will be skipped.") + return false + } + + reply, err := fs.doAutoChunk(w, r, contentLength, chunkSize, replication, collection) + if err != nil { + writeJsonError(w, r, http.StatusInternalServerError, err) + } else if reply != nil { + writeJsonQuiet(w, r, http.StatusCreated, reply) + } + return true +} + +func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, contentLength int64, chunkSize int32, replication string, collection string) (filerResult *FilerPostResult, replyerr error) { + + multipartReader, multipartReaderErr := r.MultipartReader() + if multipartReaderErr != nil { + return nil, multipartReaderErr + } + + part1, part1Err := multipartReader.NextPart() + if part1Err != nil { + return nil, part1Err + } + + fileName := part1.FileName() + if fileName != "" { + fileName = path.Base(fileName) + } + + chunks := (int64(contentLength) / int64(chunkSize)) + 1 + cm := operation.ChunkManifest{ + Name: fileName, + Size: 0, // don't know yet + Mime: "application/octet-stream", + Chunks: make([]*operation.ChunkInfo, 0, chunks), + } + + totalBytesRead := int64(0) + tmpBufferSize := int32(1024 * 1024) + tmpBuffer := bytes.NewBuffer(make([]byte, 0, tmpBufferSize)) + chunkBuf := make([]byte, chunkSize+tmpBufferSize, chunkSize+tmpBufferSize) // chunk size plus a little overflow + chunkBufOffset := int32(0) + chunkOffset := int64(0) + writtenChunks := 0 + + filerResult = &FilerPostResult{ + Name: fileName, + } + + for totalBytesRead < contentLength { + tmpBuffer.Reset() + bytesRead, readErr := io.CopyN(tmpBuffer, part1, int64(tmpBufferSize)) + readFully := readErr != nil && readErr == io.EOF + tmpBuf := tmpBuffer.Bytes() + bytesToCopy := tmpBuf[0:int(bytesRead)] + + copy(chunkBuf[chunkBufOffset:chunkBufOffset+int32(bytesRead)], bytesToCopy) + chunkBufOffset = chunkBufOffset + int32(bytesRead) + + if chunkBufOffset >= chunkSize || readFully || (chunkBufOffset > 0 && bytesRead == 0) { + writtenChunks = writtenChunks + 1 + fileId, urlLocation, assignErr := fs.assignNewFileInfo(w, r, replication, collection) + if assignErr != nil { + return nil, assignErr + } + + // upload the chunk to the volume server + chunkName := fileName + "_chunk_" + strconv.FormatInt(int64(cm.Chunks.Len()+1), 10) + uploadErr := fs.doUpload(urlLocation, w, r, chunkBuf[0:chunkBufOffset], chunkName, "application/octet-stream", fileId) + if uploadErr != nil { + return nil, uploadErr + } + + // Save to chunk manifest structure + cm.Chunks = append(cm.Chunks, + &operation.ChunkInfo{ + Offset: chunkOffset, + Size: int64(chunkBufOffset), + Fid: fileId, + }, + ) + + // reset variables for the next chunk + chunkBufOffset = 0 + chunkOffset = totalBytesRead + int64(bytesRead) + } + + totalBytesRead = totalBytesRead + int64(bytesRead) + + if bytesRead == 0 || readFully { + break + } + + if readErr != nil { + return nil, readErr + } + } + + cm.Size = totalBytesRead + manifestBuf, marshalErr := cm.Marshal() + if marshalErr != nil { + return nil, marshalErr + } + + manifestStr := string(manifestBuf) + glog.V(4).Infoln("Generated chunk manifest: ", manifestStr) + + manifestFileId, manifestUrlLocation, manifestAssignmentErr := fs.assignNewFileInfo(w, r, replication, collection) + if manifestAssignmentErr != nil { + return nil, manifestAssignmentErr + } + glog.V(4).Infoln("Manifest uploaded to:", manifestUrlLocation, "Fid:", manifestFileId) + filerResult.Fid = manifestFileId + + u, _ := url.Parse(manifestUrlLocation) + q := u.Query() + q.Set("cm", "true") + u.RawQuery = q.Encode() + + manifestUploadErr := fs.doUpload(u.String(), w, r, manifestBuf, fileName+"_manifest", "application/json", manifestFileId) + if manifestUploadErr != nil { + return nil, manifestUploadErr + } + + path := r.URL.Path + // also delete the old fid unless PUT operation + if r.Method != "PUT" { + if oldFid, err := fs.filer.FindFile(path); err == nil { + operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) + } + } + + glog.V(4).Infoln("saving", path, "=>", manifestFileId) + if db_err := fs.filer.CreateFile(path, manifestFileId); db_err != nil { + replyerr = db_err + filerResult.Error = db_err.Error() + operation.DeleteFile(fs.getMasterNode(), manifestFileId, fs.jwt(manifestFileId)) //clean up + glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err) + return + } + + return +} + +func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, chunkBuf []byte, fileName string, contentType string, fileId string) (err error) { + err = nil + + ioReader := ioutil.NopCloser(bytes.NewBuffer(chunkBuf)) + uploadResult, uploadError := operation.Upload(urlLocation, fileName, ioReader, false, contentType, fs.jwt(fileId)) + if uploadResult != nil { + glog.V(0).Infoln("Chunk upload result. Name:", uploadResult.Name, "Fid:", fileId, "Size:", uploadResult.Size) + } + if uploadError != nil { + err = uploadError + } + return +} + // curl -X DELETE http://localhost:8888/path/to // curl -X DELETE http://localhost:8888/path/to?recursive=true func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { From f04d8fcbcc7baea562e2afb526328a467cd37ab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=8D=E6=99=93=E6=A0=8B?= Date: Tue, 9 Aug 2016 20:12:39 +0800 Subject: [PATCH 11/31] if replicated volume has one copy in readonly mode at one node,it should be removed from writable list --- weed/topology/data_node.go | 11 +++++++++++ weed/topology/volume_layout.go | 13 +++++++++++++ 2 files changed, 24 insertions(+) diff --git a/weed/topology/data_node.go b/weed/topology/data_node.go index 1404d4aa8..da6ed2895 100644 --- a/weed/topology/data_node.go +++ b/weed/topology/data_node.go @@ -79,6 +79,17 @@ func (dn *DataNode) GetVolumes() (ret []storage.VolumeInfo) { return ret } +func (dn *DataNode) GetVolumesById(id storage.VolumeId) (storage.VolumeInfo, error) { + dn.RLock() + defer dn.RUnlock() + v_info, ok := dn.volumes[id] + if ok { + return v_info, nil + } else { + return storage.VolumeInfo{}, fmt.Errorf("volumeInfo not found") + } +} + func (dn *DataNode) GetDataCenter() *DataCenter { return dn.Parent().Parent().(*NodeImpl).value.(*DataCenter) } diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index 066f5f69a..af8503b29 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -45,6 +45,19 @@ func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) { } vl.vid2location[v.Id].Set(dn) glog.V(4).Infoln("volume", v.Id, "added to dn", dn.Id(), "len", vl.vid2location[v.Id].Length(), "copy", v.ReplicaPlacement.GetCopyCount()) + for _, dn := range vl.vid2location[v.Id].list { + if v_info, err := dn.GetVolumesById(v.Id); err == nil { + if v_info.ReadOnly { + glog.V(3).Infof("vid %d removed from writable", v.Id) + vl.removeFromWritable(v.Id) + return + } + } else { + glog.V(3).Infof("vid %d removed from writable", v.Id) + vl.removeFromWritable(v.Id) + return + } + } if vl.vid2location[v.Id].Length() == vl.rp.GetCopyCount() && vl.isWritable(v) { if _, ok := vl.oversizedVolumes[v.Id]; !ok { vl.addToWritable(v.Id) From a3c8235f867bb79214083e168912935c90b4fe4a Mon Sep 17 00:00:00 2001 From: ChengLei Shao Date: Wed, 10 Aug 2016 16:11:00 +0800 Subject: [PATCH 12/31] Make Docker build works correctly --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b2291c15f..e87538fdb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN echo tlsv1 >> ~/.curlrc RUN \ curl -Lks https://bintray.com$(curl -Lk http://bintray.com/chrislusf/seaweedfs/seaweedfs/_latestVersion | grep linux_amd64.tar.gz | sed -n "/href/ s/.*href=['\"]\([^'\"]*\)['\"].*/\1/gp") | gunzip | tar -xf - -C /opt/weed/ && \ - mkdir ./bin && mv weed_*/* ./bin && \ + mkdir ./bin && mv ./*/* ./bin && \ chmod +x ./bin/weed EXPOSE 8080 From b0035747e33913f4a6f998c410f3fee67d76f7d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=8D=E6=99=93=E6=A0=8B?= Date: Wed, 31 Aug 2016 11:32:30 +0800 Subject: [PATCH 13/31] add filer support --- weed/command/filer.go | 5 +- weed/command/server.go | 2 + weed/filer/mysql_store/filer_mapping.sql | 11 + weed/filer/mysql_store/mysql_store.go | 224 +++++++++++++++++++++ weed/filer/mysql_store/mysql_store_test.go | 60 ++++++ weed/server/filer_server.go | 42 +++- 6 files changed, 340 insertions(+), 4 deletions(-) create mode 100644 weed/filer/mysql_store/filer_mapping.sql create mode 100644 weed/filer/mysql_store/mysql_store.go create mode 100644 weed/filer/mysql_store/mysql_store_test.go diff --git a/weed/command/filer.go b/weed/command/filer.go index 0bd508e0b..7d90707a6 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -24,7 +24,8 @@ type FilerOptions struct { dir *string redirectOnRead *bool disableDirListing *bool - maxMB *int + confFile *string + maxMB *int secretKey *string cassandra_server *string cassandra_keyspace *string @@ -43,6 +44,7 @@ func init() { f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified") f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing") + f.confFile = cmdFiler.Flag.String("confFile", "", "json encoded filer conf file") f.maxMB = cmdFiler.Flag.Int("maxMB", 0, "split files larger than the limit") f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server") f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server") @@ -84,6 +86,7 @@ func runFiler(cmd *Command, args []string) bool { r := http.NewServeMux() _, nfs_err := weed_server.NewFilerServer(r, *f.ip, *f.port, *f.master, *f.dir, *f.collection, *f.defaultReplicaPlacement, *f.redirectOnRead, *f.disableDirListing, + *f.confFile, *f.maxMB, *f.secretKey, *f.cassandra_server, *f.cassandra_keyspace, diff --git a/weed/command/server.go b/weed/command/server.go index 7a6677a65..eed7dcae4 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -86,6 +86,7 @@ func init() { filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.") filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") + filerOptions.confFile = cmdServer.Flag.String("filer.confFile", "", "json encoded filer conf file") filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 0, "split files larger than the limit") filerOptions.cassandra_server = cmdServer.Flag.String("filer.cassandra.server", "", "host[:port] of the cassandra server") filerOptions.cassandra_keyspace = cmdServer.Flag.String("filer.cassandra.keyspace", "seaweed", "keyspace of the cassandra server") @@ -170,6 +171,7 @@ func runServer(cmd *Command, args []string) bool { _, nfs_err := weed_server.NewFilerServer(r, *serverBindIp, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection, *filerOptions.defaultReplicaPlacement, *filerOptions.redirectOnRead, *filerOptions.disableDirListing, + *filerOptions.confFile, *filerOptions.maxMB, *filerOptions.secretKey, *filerOptions.cassandra_server, *filerOptions.cassandra_keyspace, diff --git a/weed/filer/mysql_store/filer_mapping.sql b/weed/filer/mysql_store/filer_mapping.sql new file mode 100644 index 000000000..6bbe4e880 --- /dev/null +++ b/weed/filer/mysql_store/filer_mapping.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS `filer_mapping` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `uriPath` char(256) NOT NULL DEFAULT "" COMMENT 'http uriPath', + `fid` char(36) NOT NULL DEFAULT "" COMMENT 'seaweedfs fid', + `createTime` int(10) NOT NULL DEFAULT 0 COMMENT 'createdTime in unix timestamp', + `updateTime` int(10) NOT NULL DEFAULT 0 COMMENT 'updatedTime in unix timestamp', + `remark` varchar(20) NOT NULL DEFAULT "" COMMENT 'reserverd field', + `status` tinyint(2) DEFAULT '1' COMMENT 'resource status', + PRIMARY KEY (`id`), + UNIQUE KEY `index_uriPath` (`uriPath`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; \ No newline at end of file diff --git a/weed/filer/mysql_store/mysql_store.go b/weed/filer/mysql_store/mysql_store.go new file mode 100644 index 000000000..44d0d88a7 --- /dev/null +++ b/weed/filer/mysql_store/mysql_store.go @@ -0,0 +1,224 @@ +package mysql_store + +import ( + "database/sql" + "fmt" + "hash/crc32" + "sync" + "time" + + _ "github.com/go-sql-driver/mysql" +) + +const ( + sqlUrl = "%s:%s@tcp(%s:%d)/%s?charset=utf8" + maxIdleConnections = 100 + maxOpenConnections = 50 + maxTableNums = 1024 + tableName = "filer_mapping" +) + +var ( + _init_db sync.Once + _db_connections []*sql.DB +) + +type MySqlConf struct { + User string + Password string + HostName string + Port int + DataBase string +} + +type MySqlStore struct { + dbs []*sql.DB +} + +func getDbConnection(confs []MySqlConf) []*sql.DB { + _init_db.Do(func() { + for _, conf := range confs { + + sqlUrl := fmt.Sprintf(sqlUrl, conf.User, conf.Password, conf.HostName, conf.Port, conf.DataBase) + var dbErr error + _db_connection, dbErr := sql.Open("mysql", sqlUrl) + if dbErr != nil { + _db_connection.Close() + _db_connection = nil + panic(dbErr) + } + _db_connection.SetMaxIdleConns(maxIdleConnections) + _db_connection.SetMaxOpenConns(maxOpenConnections) + _db_connections = append(_db_connections, _db_connection) + } + }) + return _db_connections +} + +func NewMysqlStore(confs []MySqlConf) *MySqlStore { + ms := &MySqlStore{ + dbs: getDbConnection(confs), + } + + for _, db := range ms.dbs { + for i := 0; i < maxTableNums; i++ { + if err := ms.createTables(db, tableName, i); err != nil { + fmt.Printf("create table failed %s", err.Error()) + } + } + } + + return ms +} + +func (s *MySqlStore) hash(fullFileName string) (instance_offset, table_postfix int) { + hash_value := crc32.ChecksumIEEE([]byte(fullFileName)) + instance_offset = int(hash_value) % len(s.dbs) + table_postfix = int(hash_value) % maxTableNums + return +} + +func (s *MySqlStore) parseFilerMappingInfo(path string) (instanceId int, tableFullName string, err error) { + instance_offset, table_postfix := s.hash(path) + instanceId = instance_offset + tableFullName = fmt.Sprintf("%s_%04d", tableName, table_postfix) + return +} + +func (s *MySqlStore) Get(fullFilePath string) (fid string, err error) { + instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath) + if err != nil { + return "", err + } + fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName) + if err == sql.ErrNoRows { + //Could not found + err = nil + } + return fid, err +} + +func (s *MySqlStore) Put(fullFilePath string, fid string) (err error) { + var tableFullName string + + instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath) + if err != nil { + return err + } + if old_fid, localErr := s.query(fullFilePath, s.dbs[instance_offset], tableFullName); localErr != nil && localErr != sql.ErrNoRows { + err = localErr + return + } else { + if len(old_fid) == 0 { + err = s.insert(fullFilePath, fid, s.dbs[instance_offset], tableFullName) + } else { + err = s.update(fullFilePath, fid, s.dbs[instance_offset], tableFullName) + } + } + return +} + +func (s *MySqlStore) Delete(fullFilePath string) (err error) { + var fid string + instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath) + if err != nil { + return err + } + if fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil { + return err + } else if fid == "" { + return nil + } + if err := s.delete(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil { + return err + } else { + return nil + } +} + +func (s *MySqlStore) Close() { + for _, db := range s.dbs { + db.Close() + } +} + +var createTable = ` +CREATE TABLE IF NOT EXISTS %s_%04d ( + id bigint(20) NOT NULL AUTO_INCREMENT, + uriPath char(256) NOT NULL DEFAULT "" COMMENT 'http uriPath', + fid char(36) NOT NULL DEFAULT "" COMMENT 'seaweedfs fid', + createTime int(10) NOT NULL DEFAULT 0 COMMENT 'createdTime in unix timestamp', + updateTime int(10) NOT NULL DEFAULT 0 COMMENT 'updatedTime in unix timestamp', + remark varchar(20) NOT NULL DEFAULT "" COMMENT 'reserverd field', + status tinyint(2) DEFAULT '1' COMMENT 'resource status', + PRIMARY KEY (id), + UNIQUE KEY index_uriPath (uriPath) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +` + +func (s *MySqlStore) createTables(db *sql.DB, tableName string, postfix int) error { + stmt, err := db.Prepare(fmt.Sprintf(createTable, tableName, postfix)) + if err != nil { + return err + } + defer stmt.Close() + + _, err = stmt.Exec() + if err != nil { + return err + } + return nil +} + +func (s *MySqlStore) query(uriPath string, db *sql.DB, tableName string) (string, error) { + sqlStatement := "SELECT fid FROM %s WHERE uriPath=?" + row := db.QueryRow(fmt.Sprintf(sqlStatement, tableName), uriPath) + var fid string + err := row.Scan(&fid) + if err != nil { + return "", err + } + return fid, nil +} + +func (s *MySqlStore) update(uriPath string, fid string, db *sql.DB, tableName string) error { + sqlStatement := "UPDATE %s SET fid=?, updateTime=? WHERE uriPath=?" + res, err := db.Exec(fmt.Sprintf(sqlStatement, tableName), fid, time.Now().Unix(), uriPath) + if err != nil { + return err + } + + _, err = res.RowsAffected() + if err != nil { + return err + } + return nil +} + +func (s *MySqlStore) insert(uriPath string, fid string, db *sql.DB, tableName string) error { + sqlStatement := "INSERT INTO %s (uriPath,fid,createTime) VALUES(?,?,?)" + res, err := db.Exec(fmt.Sprintf(sqlStatement, tableName), uriPath, fid, time.Now().Unix()) + if err != nil { + return err + } + + _, err = res.RowsAffected() + if err != nil { + return err + } + return nil +} + +func (s *MySqlStore) delete(uriPath string, db *sql.DB, tableName string) error { + sqlStatement := "DELETE FROM %s WHERE uriPath=?" + res, err := db.Exec(fmt.Sprintf(sqlStatement, tableName), uriPath) + if err != nil { + return err + } + + _, err = res.RowsAffected() + if err != nil { + return err + } + return nil +} diff --git a/weed/filer/mysql_store/mysql_store_test.go b/weed/filer/mysql_store/mysql_store_test.go new file mode 100644 index 000000000..e89ca9020 --- /dev/null +++ b/weed/filer/mysql_store/mysql_store_test.go @@ -0,0 +1,60 @@ +package mysql_store + +import ( + "encoding/json" + "hash/crc32" + "testing" +) + +/* +To improve performance when storing billion of files, you could shar +At each mysql instance, we will try to create 1024 tables if not exist, table name will be something like: +filer_mapping_0000 +filer_mapping_0001 +..... +filer_mapping_1023 +sample conf should be + +>$cat filer_conf.json +{ + "mysql": [ + { + "User": "root", + "Password": "root", + "HostName": "127.0.0.1", + "Port": 3306, + "DataBase": "seaweedfs" + }, + { + "User": "root", + "Password": "root", + "HostName": "127.0.0.2", + "Port": 3306, + "DataBase": "seaweedfs" + } + ] +} +*/ + +func TestGenerateMysqlConf(t *testing.T) { + var conf MySqlConf + conf = append(conf, MySqlInstConf{ + User: "root", + Password: "root", + HostName: "localhost", + Port: 3306, + DataBase: "seaweedfs", + }) + body, err := json.Marshal(conf) + if err != nil { + t.Errorf("json encoding err %s", err.Error()) + } + t.Logf("json output is %s", string(body)) +} + +func TestCRC32FullPathName(t *testing.T) { + fullPathName := "/prod-bucket/law632191483895612493300-signed.pdf" + hash_value := crc32.ChecksumIEEE([]byte(fullPathName)) + table_postfix := int(hash_value) % 1024 + t.Logf("table postfix %d", table_postfix) +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 3c7c1fd9e..1da0d065d 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -1,8 +1,10 @@ package weed_server import ( + "encoding/json" "math/rand" "net/http" + "os" "strconv" "sync" "time" @@ -11,6 +13,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/filer/cassandra_store" "github.com/chrislusf/seaweedfs/weed/filer/embedded_filer" "github.com/chrislusf/seaweedfs/weed/filer/flat_namespace" + "github.com/chrislusf/seaweedfs/weed/filer/mysql_store" "github.com/chrislusf/seaweedfs/weed/filer/redis_store" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/security" @@ -18,6 +21,25 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) +type filerConf struct { + MysqlConf []mysql_store.MySqlConf `json:"mysql"` +} + +func parseConfFile(confPath string) (*filerConf, error) { + var setting filerConf + configFile, err := os.Open(confPath) + defer configFile.Close() + if err != nil { + return nil, err + } + + jsonParser := json.NewDecoder(configFile) + if err = jsonParser.Decode(&setting); err != nil { + return nil, err + } + return &setting, nil +} + type FilerServer struct { port string master string @@ -28,12 +50,13 @@ type FilerServer struct { disableDirListing bool secret security.Secret filer filer.Filer - maxMB int + maxMB int masterNodes *storage.MasterNodes } func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir string, collection string, replication string, redirectOnRead bool, disableDirListing bool, + confFile string, maxMB int, secret string, cassandra_server string, cassandra_keyspace string, @@ -45,11 +68,24 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st defaultReplication: replication, redirectOnRead: redirectOnRead, disableDirListing: disableDirListing, - maxMB: maxMB, + maxMB: maxMB, port: ip + ":" + strconv.Itoa(port), } - if cassandra_server != "" { + var setting *filerConf + if confFile != "" { + setting, err = parseConfFile(confFile) + if err != nil { + return nil, err + } + } else { + setting = new(filerConf) + } + + if setting.MysqlConf != nil && len(setting.MysqlConf) != 0 { + mysql_store := mysql_store.NewMysqlStore(setting.MysqlConf) + fs.filer = flat_namespace.NewFlatNamespaceFiler(master, mysql_store) + } else if cassandra_server != "" { cassandra_store, err := cassandra_store.NewCassandraStore(cassandra_keyspace, cassandra_server) if err != nil { glog.Fatalf("Can not connect to cassandra server %s with keyspace %s: %v", cassandra_server, cassandra_keyspace, err) From e7b237c8dadc8f5d65fed8db2115c1d946d8c519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=8D=E6=99=93=E6=A0=8B?= Date: Wed, 31 Aug 2016 11:55:02 +0800 Subject: [PATCH 14/31] UT case fix --- weed/filer/mysql_store/mysql_store_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/filer/mysql_store/mysql_store_test.go b/weed/filer/mysql_store/mysql_store_test.go index e89ca9020..2bfe26dc8 100644 --- a/weed/filer/mysql_store/mysql_store_test.go +++ b/weed/filer/mysql_store/mysql_store_test.go @@ -37,8 +37,8 @@ sample conf should be */ func TestGenerateMysqlConf(t *testing.T) { - var conf MySqlConf - conf = append(conf, MySqlInstConf{ + var conf []MySqlConf + conf = append(conf, MySqlConf{ User: "root", Password: "root", HostName: "localhost", From 3aa021a812317fa640ebe2c7ec3b2a78492fa319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=8D=E6=99=93=E6=A0=8B?= Date: Mon, 5 Sep 2016 14:10:22 +0800 Subject: [PATCH 15/31] refactoring mysql store code --- weed/filer/mysql_store/README.md | 66 +++++++++++++ weed/filer/mysql_store/filer_mapping.sql | 11 --- weed/filer/mysql_store/mysql_store.go | 102 +++++++++++++++------ weed/filer/mysql_store/mysql_store_test.go | 30 ------ weed/server/filer_server.go | 3 +- 5 files changed, 141 insertions(+), 71 deletions(-) create mode 100644 weed/filer/mysql_store/README.md delete mode 100644 weed/filer/mysql_store/filer_mapping.sql diff --git a/weed/filer/mysql_store/README.md b/weed/filer/mysql_store/README.md new file mode 100644 index 000000000..4ce6438da --- /dev/null +++ b/weed/filer/mysql_store/README.md @@ -0,0 +1,66 @@ +#MySQL filer mapping store + +## Schema format + + +Basically, uriPath and fid are the key elements stored in MySQL. In view of the optimization and user's usage, +adding primary key with integer type and involving createTime, updateTime, status fields should be somewhat meaningful. +Of course, you could customize the schema per your concretely circumstance freely. + +

+CREATE TABLE IF NOT EXISTS `filer_mapping` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `uriPath` char(256) NOT NULL DEFAULT "" COMMENT 'http uriPath',
+  `fid` char(36) NOT NULL DEFAULT "" COMMENT 'seaweedfs fid',
+  `createTime` int(10) NOT NULL DEFAULT 0 COMMENT 'createdTime in unix timestamp',
+  `updateTime` int(10) NOT NULL DEFAULT 0 COMMENT 'updatedTime in unix timestamp',
+  `remark` varchar(20) NOT NULL DEFAULT "" COMMENT 'reserverd field',
+  `status` tinyint(2) DEFAULT '1' COMMENT 'resource status',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `index_uriPath` (`uriPath`)
+) DEFAULT CHARSET=utf8;
+
+ + +The MySQL 's config params is not added into the weed command option as other stores(redis,cassandra). Instead, +We created a config file(json format) for them. TOML,YAML or XML also should be OK. But TOML and YAML need import thirdparty package +while XML is a little bit complex. + +The sample config file's content is below: + +

+{
+    "mysql": [
+        {
+            "User": "root",
+            "Password": "root",
+            "HostName": "127.0.0.1",
+            "Port": 3306,
+            "DataBase": "seaweedfs"
+        },
+        {
+            "User": "root",
+            "Password": "root",
+            "HostName": "127.0.0.2",
+            "Port": 3306,
+            "DataBase": "seaweedfs"
+        }
+    ],
+    "IsSharding":true,
+    "ShardingNum":1024
+}
+
+ + +The "mysql" field in above conf file is an array which include all mysql instances you prepared to store sharding data. +1. If one mysql instance is enough, just keep one instance in "mysql" field. +2. If table sharding at a specific mysql instance is needed , mark "IsSharding" field with true and specify total table +sharding numbers using "ShardingNum" field. +3. If the mysql service could be auto scaled transparently in your environment, just config one mysql instance(usually it's a frondend proxy or VIP), +and mark "IsSharding" with false value +4. If your prepare more than one mysql instances and have no plan to use table sharding for any instance(mark isSharding with false), instance sharding +will still be done implicitly + + + + diff --git a/weed/filer/mysql_store/filer_mapping.sql b/weed/filer/mysql_store/filer_mapping.sql deleted file mode 100644 index 6bbe4e880..000000000 --- a/weed/filer/mysql_store/filer_mapping.sql +++ /dev/null @@ -1,11 +0,0 @@ -CREATE TABLE IF NOT EXISTS `filer_mapping` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `uriPath` char(256) NOT NULL DEFAULT "" COMMENT 'http uriPath', - `fid` char(36) NOT NULL DEFAULT "" COMMENT 'seaweedfs fid', - `createTime` int(10) NOT NULL DEFAULT 0 COMMENT 'createdTime in unix timestamp', - `updateTime` int(10) NOT NULL DEFAULT 0 COMMENT 'updatedTime in unix timestamp', - `remark` varchar(20) NOT NULL DEFAULT "" COMMENT 'reserverd field', - `status` tinyint(2) DEFAULT '1' COMMENT 'resource status', - PRIMARY KEY (`id`), - UNIQUE KEY `index_uriPath` (`uriPath`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; \ No newline at end of file diff --git a/weed/filer/mysql_store/mysql_store.go b/weed/filer/mysql_store/mysql_store.go index 44d0d88a7..439ed22f7 100644 --- a/weed/filer/mysql_store/mysql_store.go +++ b/weed/filer/mysql_store/mysql_store.go @@ -11,11 +11,11 @@ import ( ) const ( - sqlUrl = "%s:%s@tcp(%s:%d)/%s?charset=utf8" - maxIdleConnections = 100 - maxOpenConnections = 50 - maxTableNums = 1024 - tableName = "filer_mapping" + sqlUrl = "%s:%s@tcp(%s:%d)/%s?charset=utf8" + default_maxIdleConnections = 100 + default_maxOpenConnections = 50 + default_maxTableNums = 1024 + tableName = "filer_mapping" ) var ( @@ -24,15 +24,24 @@ var ( ) type MySqlConf struct { - User string - Password string - HostName string - Port int - DataBase string + User string + Password string + HostName string + Port int + DataBase string + MaxIdleConnections int + MaxOpenConnections int +} + +type ShardingConf struct { + IsSharding bool `json:"isSharding"` + ShardingNum int `json:"shardingNum"` } type MySqlStore struct { - dbs []*sql.DB + dbs []*sql.DB + isSharding bool + shardingNum int } func getDbConnection(confs []MySqlConf) []*sql.DB { @@ -47,6 +56,19 @@ func getDbConnection(confs []MySqlConf) []*sql.DB { _db_connection = nil panic(dbErr) } + var maxIdleConnections, maxOpenConnections int + + if conf.MaxIdleConnections != 0 { + maxIdleConnections = conf.MaxIdleConnections + } else { + maxIdleConnections = default_maxIdleConnections + } + if conf.MaxOpenConnections != 0 { + maxOpenConnections = conf.MaxOpenConnections + } else { + maxOpenConnections = default_maxOpenConnections + } + _db_connection.SetMaxIdleConns(maxIdleConnections) _db_connection.SetMaxOpenConns(maxOpenConnections) _db_connections = append(_db_connections, _db_connection) @@ -55,15 +77,24 @@ func getDbConnection(confs []MySqlConf) []*sql.DB { return _db_connections } -func NewMysqlStore(confs []MySqlConf) *MySqlStore { +func NewMysqlStore(confs []MySqlConf, isSharding bool, shardingNum int) *MySqlStore { ms := &MySqlStore{ - dbs: getDbConnection(confs), + dbs: getDbConnection(confs), + isSharding: isSharding, + shardingNum: shardingNum, } for _, db := range ms.dbs { - for i := 0; i < maxTableNums; i++ { + if !isSharding { + ms.shardingNum = 1 + } else { + if ms.shardingNum == 0 { + ms.shardingNum = default_maxTableNums + } + } + for i := 0; i < ms.shardingNum; i++ { if err := ms.createTables(db, tableName, i); err != nil { - fmt.Printf("create table failed %s", err.Error()) + fmt.Printf("create table failed %v", err) } } } @@ -74,21 +105,25 @@ func NewMysqlStore(confs []MySqlConf) *MySqlStore { func (s *MySqlStore) hash(fullFileName string) (instance_offset, table_postfix int) { hash_value := crc32.ChecksumIEEE([]byte(fullFileName)) instance_offset = int(hash_value) % len(s.dbs) - table_postfix = int(hash_value) % maxTableNums + table_postfix = int(hash_value) % s.shardingNum return } func (s *MySqlStore) parseFilerMappingInfo(path string) (instanceId int, tableFullName string, err error) { instance_offset, table_postfix := s.hash(path) instanceId = instance_offset - tableFullName = fmt.Sprintf("%s_%04d", tableName, table_postfix) + if s.isSharding { + tableFullName = fmt.Sprintf("%s_%04d", tableName, table_postfix) + } else { + tableFullName = tableName + } return } func (s *MySqlStore) Get(fullFilePath string) (fid string, err error) { instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath) if err != nil { - return "", err + return "", fmt.Errorf("MySqlStore Get operation can not parse file path %s: err is %v", fullFilePath, err) } fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName) if err == sql.ErrNoRows { @@ -103,16 +138,18 @@ func (s *MySqlStore) Put(fullFilePath string, fid string) (err error) { instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath) if err != nil { - return err + return fmt.Errorf("MySqlStore Put operation can not parse file path %s: err is %v", fullFilePath, err) } - if old_fid, localErr := s.query(fullFilePath, s.dbs[instance_offset], tableFullName); localErr != nil && localErr != sql.ErrNoRows { - err = localErr - return + var old_fid string + if old_fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil && err != sql.ErrNoRows { + return fmt.Errorf("MySqlStore Put operation failed when querying path %s: err is %v", fullFilePath, err) } else { if len(old_fid) == 0 { err = s.insert(fullFilePath, fid, s.dbs[instance_offset], tableFullName) + err = fmt.Errorf("MySqlStore Put operation failed when inserting path %s with fid %s : err is %v", fullFilePath, fid, err) } else { err = s.update(fullFilePath, fid, s.dbs[instance_offset], tableFullName) + err = fmt.Errorf("MySqlStore Put operation failed when updating path %s with fid %s : err is %v", fullFilePath, fid, err) } } return @@ -122,15 +159,15 @@ func (s *MySqlStore) Delete(fullFilePath string) (err error) { var fid string instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath) if err != nil { - return err + return fmt.Errorf("MySqlStore Delete operation can not parse file path %s: err is %v", fullFilePath, err) } if fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil { - return err + return fmt.Errorf("MySqlStore Delete operation failed when querying path %s: err is %v", fullFilePath, err) } else if fid == "" { return nil } - if err := s.delete(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil { - return err + if err = s.delete(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil { + return fmt.Errorf("MySqlStore Delete operation failed when deleting path %s: err is %v", fullFilePath, err) } else { return nil } @@ -143,7 +180,7 @@ func (s *MySqlStore) Close() { } var createTable = ` -CREATE TABLE IF NOT EXISTS %s_%04d ( +CREATE TABLE IF NOT EXISTS %s ( id bigint(20) NOT NULL AUTO_INCREMENT, uriPath char(256) NOT NULL DEFAULT "" COMMENT 'http uriPath', fid char(36) NOT NULL DEFAULT "" COMMENT 'seaweedfs fid', @@ -153,11 +190,18 @@ CREATE TABLE IF NOT EXISTS %s_%04d ( status tinyint(2) DEFAULT '1' COMMENT 'resource status', PRIMARY KEY (id), UNIQUE KEY index_uriPath (uriPath) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) DEFAULT CHARSET=utf8; ` func (s *MySqlStore) createTables(db *sql.DB, tableName string, postfix int) error { - stmt, err := db.Prepare(fmt.Sprintf(createTable, tableName, postfix)) + var realTableName string + if s.isSharding { + realTableName = fmt.Sprintf("%s_%4d", tableName, postfix) + } else { + realTableName = tableName + } + + stmt, err := db.Prepare(fmt.Sprintf(createTable, realTableName)) if err != nil { return err } diff --git a/weed/filer/mysql_store/mysql_store_test.go b/weed/filer/mysql_store/mysql_store_test.go index 2bfe26dc8..1c9765c59 100644 --- a/weed/filer/mysql_store/mysql_store_test.go +++ b/weed/filer/mysql_store/mysql_store_test.go @@ -6,36 +6,6 @@ import ( "testing" ) -/* -To improve performance when storing billion of files, you could shar -At each mysql instance, we will try to create 1024 tables if not exist, table name will be something like: -filer_mapping_0000 -filer_mapping_0001 -..... -filer_mapping_1023 -sample conf should be - ->$cat filer_conf.json -{ - "mysql": [ - { - "User": "root", - "Password": "root", - "HostName": "127.0.0.1", - "Port": 3306, - "DataBase": "seaweedfs" - }, - { - "User": "root", - "Password": "root", - "HostName": "127.0.0.2", - "Port": 3306, - "DataBase": "seaweedfs" - } - ] -} -*/ - func TestGenerateMysqlConf(t *testing.T) { var conf []MySqlConf conf = append(conf, MySqlConf{ diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 1da0d065d..1bcbd046f 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -23,6 +23,7 @@ import ( type filerConf struct { MysqlConf []mysql_store.MySqlConf `json:"mysql"` + mysql_store.ShardingConf } func parseConfFile(confPath string) (*filerConf, error) { @@ -83,7 +84,7 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st } if setting.MysqlConf != nil && len(setting.MysqlConf) != 0 { - mysql_store := mysql_store.NewMysqlStore(setting.MysqlConf) + mysql_store := mysql_store.NewMysqlStore(setting.MysqlConf, setting.IsSharding, setting.ShardingNum) fs.filer = flat_namespace.NewFlatNamespaceFiler(master, mysql_store) } else if cassandra_server != "" { cassandra_store, err := cassandra_store.NewCassandraStore(cassandra_keyspace, cassandra_server) From c4b7966dbee50970ebaf70269d74e96592ff72c9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 7 Sep 2016 18:05:57 -0700 Subject: [PATCH 16/31] minor help message change --- weed/command/export.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/command/export.go b/weed/command/export.go index 481aa111b..5a7dc71d9 100644 --- a/weed/command/export.go +++ b/weed/command/export.go @@ -51,7 +51,7 @@ func init() { var ( output = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout") format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename formatted with {{.Mime}} {{.Id}} {{.Name}} {{.Ext}}") - newer = cmdExport.Flag.String("newer", "", "export only files newer than this time, default is all files. Must be specified in RFC3339 without timezone") + newer = cmdExport.Flag.String("newer", "", "export only files newer than this time, default is all files. Must be specified in RFC3339 without timezone, e.g. 2006-01-02T15:04:05") tarOutputFile *tar.Writer tarHeader tar.Header From 0559aa96738d727e055190301bbb0624c74ced7f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 7 Sep 2016 18:13:49 -0700 Subject: [PATCH 17/31] use Lock instead of RLock fix https://github.com/chrislusf/seaweedfs/issues/364 --- weed/topology/data_node.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/weed/topology/data_node.go b/weed/topology/data_node.go index da6ed2895..b7f039559 100644 --- a/weed/topology/data_node.go +++ b/weed/topology/data_node.go @@ -53,7 +53,7 @@ func (dn *DataNode) UpdateVolumes(actualVolumes []storage.VolumeInfo) (deletedVo for _, v := range actualVolumes { actualVolumeMap[v.Id] = v } - dn.RLock() + dn.Lock() for vid, v := range dn.volumes { if _, ok := actualVolumeMap[vid]; !ok { glog.V(0).Infoln("Deleting volume id:", vid) @@ -62,8 +62,8 @@ func (dn *DataNode) UpdateVolumes(actualVolumes []storage.VolumeInfo) (deletedVo dn.UpAdjustVolumeCountDelta(-1) dn.UpAdjustActiveVolumeCountDelta(-1) } - } //TODO: adjust max volume id, if need to reclaim volume ids - dn.RUnlock() + } + dn.Unlock() for _, v := range actualVolumes { dn.AddOrUpdateVolume(v) } From 78474409a596aabd8e579716ba1f4939e7d62579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=8D=E6=99=93=E6=A0=8B?= Date: Thu, 8 Sep 2016 11:35:54 +0800 Subject: [PATCH 18/31] filer mysqlstore bug fix --- weed/filer/cassandra_store/cassandra_store.go | 2 ++ weed/filer/embedded_filer/files_in_leveldb.go | 4 ++- weed/filer/filer.go | 6 ++++ weed/filer/mysql_store/README.md | 15 +++++---- weed/filer/mysql_store/mysql_store.go | 32 ++++++++++--------- weed/filer/redis_store/redis_store.go | 4 ++- weed/server/filer_server.go | 2 +- weed/server/filer_server_handlers_read.go | 3 +- weed/server/filer_server_handlers_write.go | 12 ++++--- 9 files changed, 50 insertions(+), 30 deletions(-) diff --git a/weed/filer/cassandra_store/cassandra_store.go b/weed/filer/cassandra_store/cassandra_store.go index cdb9d3e3c..50a792a65 100644 --- a/weed/filer/cassandra_store/cassandra_store.go +++ b/weed/filer/cassandra_store/cassandra_store.go @@ -3,6 +3,7 @@ package cassandra_store import ( "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/gocql/gocql" @@ -59,6 +60,7 @@ func (c *CassandraStore) Get(fullFileName string) (fid string, err error) { fullFileName).Consistency(gocql.One).Scan(&output); err != nil { if err != gocql.ErrNotFound { glog.V(0).Infof("Failed to find file %s: %v", fullFileName, fid, err) + return "", filer.ErrNotFound } } if len(output) == 0 { diff --git a/weed/filer/embedded_filer/files_in_leveldb.go b/weed/filer/embedded_filer/files_in_leveldb.go index 19f6dd7e8..c40d7adaf 100644 --- a/weed/filer/embedded_filer/files_in_leveldb.go +++ b/weed/filer/embedded_filer/files_in_leveldb.go @@ -53,7 +53,9 @@ func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string } func (fl *FileListInLevelDb) FindFile(dirId filer.DirectoryId, fileName string) (fid string, err error) { data, e := fl.db.Get(genKey(dirId, fileName), nil) - if e != nil { + if e == leveldb.ErrNotFound { + return "", filer.ErrNotFound + } else if e != nil { return "", e } return string(data), nil diff --git a/weed/filer/filer.go b/weed/filer/filer.go index fd23e119c..5d5acb68d 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -1,5 +1,9 @@ package filer +import ( + "errors" +) + type FileId string //file id in SeaweedFS type FileEntry struct { @@ -26,3 +30,5 @@ type Filer interface { DeleteDirectory(dirPath string, recursive bool) (err error) Move(fromPath string, toPath string) (err error) } + +var ErrNotFound = errors.New("filer: no entry is found in filer store") diff --git a/weed/filer/mysql_store/README.md b/weed/filer/mysql_store/README.md index 4ce6438da..6efeb1c54 100644 --- a/weed/filer/mysql_store/README.md +++ b/weed/filer/mysql_store/README.md @@ -47,19 +47,20 @@ The sample config file's content is below: } ], "IsSharding":true, - "ShardingNum":1024 + "ShardCount":1024 } The "mysql" field in above conf file is an array which include all mysql instances you prepared to store sharding data. + 1. If one mysql instance is enough, just keep one instance in "mysql" field. -2. If table sharding at a specific mysql instance is needed , mark "IsSharding" field with true and specify total table -sharding numbers using "ShardingNum" field. -3. If the mysql service could be auto scaled transparently in your environment, just config one mysql instance(usually it's a frondend proxy or VIP), -and mark "IsSharding" with false value -4. If your prepare more than one mysql instances and have no plan to use table sharding for any instance(mark isSharding with false), instance sharding -will still be done implicitly + +2. If table sharding at a specific mysql instance is needed , mark "IsSharding" field with true and specify total table sharding numbers using "ShardCount" field. + +3. If the mysql service could be auto scaled transparently in your environment, just config one mysql instance(usually it's a frondend proxy or VIP),and mark "IsSharding" with false value + +4. If you prepare more than one mysql instance and have no plan to use table sharding for any instance(mark isSharding with false), instance sharding will still be done implicitly diff --git a/weed/filer/mysql_store/mysql_store.go b/weed/filer/mysql_store/mysql_store.go index 439ed22f7..6910206ce 100644 --- a/weed/filer/mysql_store/mysql_store.go +++ b/weed/filer/mysql_store/mysql_store.go @@ -7,6 +7,8 @@ import ( "sync" "time" + "github.com/chrislusf/seaweedfs/weed/filer" + _ "github.com/go-sql-driver/mysql" ) @@ -34,14 +36,14 @@ type MySqlConf struct { } type ShardingConf struct { - IsSharding bool `json:"isSharding"` - ShardingNum int `json:"shardingNum"` + IsSharding bool `json:"isSharding"` + ShardCount int `json:"shardCount"` } type MySqlStore struct { - dbs []*sql.DB - isSharding bool - shardingNum int + dbs []*sql.DB + isSharding bool + shardCount int } func getDbConnection(confs []MySqlConf) []*sql.DB { @@ -77,22 +79,22 @@ func getDbConnection(confs []MySqlConf) []*sql.DB { return _db_connections } -func NewMysqlStore(confs []MySqlConf, isSharding bool, shardingNum int) *MySqlStore { +func NewMysqlStore(confs []MySqlConf, isSharding bool, shardCount int) *MySqlStore { ms := &MySqlStore{ - dbs: getDbConnection(confs), - isSharding: isSharding, - shardingNum: shardingNum, + dbs: getDbConnection(confs), + isSharding: isSharding, + shardCount: shardCount, } for _, db := range ms.dbs { if !isSharding { - ms.shardingNum = 1 + ms.shardCount = 1 } else { - if ms.shardingNum == 0 { - ms.shardingNum = default_maxTableNums + if ms.shardCount == 0 { + ms.shardCount = default_maxTableNums } } - for i := 0; i < ms.shardingNum; i++ { + for i := 0; i < ms.shardCount; i++ { if err := ms.createTables(db, tableName, i); err != nil { fmt.Printf("create table failed %v", err) } @@ -105,7 +107,7 @@ func NewMysqlStore(confs []MySqlConf, isSharding bool, shardingNum int) *MySqlSt func (s *MySqlStore) hash(fullFileName string) (instance_offset, table_postfix int) { hash_value := crc32.ChecksumIEEE([]byte(fullFileName)) instance_offset = int(hash_value) % len(s.dbs) - table_postfix = int(hash_value) % s.shardingNum + table_postfix = int(hash_value) % s.shardCount return } @@ -128,7 +130,7 @@ func (s *MySqlStore) Get(fullFilePath string) (fid string, err error) { fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName) if err == sql.ErrNoRows { //Could not found - err = nil + err = filer.ErrNotFound } return fid, err } diff --git a/weed/filer/redis_store/redis_store.go b/weed/filer/redis_store/redis_store.go index 5e51b5455..2ad49a805 100644 --- a/weed/filer/redis_store/redis_store.go +++ b/weed/filer/redis_store/redis_store.go @@ -1,6 +1,8 @@ package redis_store import ( + "github.com/chrislusf/seaweedfs/weed/filer" + redis "gopkg.in/redis.v2" ) @@ -20,7 +22,7 @@ func NewRedisStore(hostPort string, password string, database int) *RedisStore { func (s *RedisStore) Get(fullFileName string) (fid string, err error) { fid, err = s.Client.Get(fullFileName).Result() if err == redis.Nil { - err = nil + err = filer.ErrNotFound } return fid, err } diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 1bcbd046f..959bb92cb 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -84,7 +84,7 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st } if setting.MysqlConf != nil && len(setting.MysqlConf) != 0 { - mysql_store := mysql_store.NewMysqlStore(setting.MysqlConf, setting.IsSharding, setting.ShardingNum) + mysql_store := mysql_store.NewMysqlStore(setting.MysqlConf, setting.IsSharding, setting.ShardCount) fs.filer = flat_namespace.NewFlatNamespaceFiler(master, mysql_store) } else if cassandra_server != "" { cassandra_store, err := cassandra_store.NewCassandraStore(cassandra_keyspace, cassandra_server) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 6b9505377..bf95e37b9 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" ui "github.com/chrislusf/seaweedfs/weed/server/filer_ui" @@ -87,7 +88,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, } fileId, err := fs.filer.FindFile(r.URL.Path) - if err == leveldb.ErrNotFound { + if err == filer.ErrNotFound { glog.V(3).Infoln("Not found in db", r.URL.Path) w.WriteHeader(http.StatusNotFound) return diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 872d8c4b9..464cb81ef 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -15,11 +15,11 @@ import ( "net/url" "strings" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/syndtr/goleveldb/leveldb" "path" "strconv" ) @@ -73,17 +73,17 @@ func makeFormData(filename, mimeType string, content io.Reader) (formData io.Rea } func (fs *FilerServer) queryFileInfoByPath(w http.ResponseWriter, r *http.Request, path string) (fileId, urlLocation string, err error) { - if fileId, err = fs.filer.FindFile(path); err != nil && err != leveldb.ErrNotFound { + if fileId, err = fs.filer.FindFile(path); err != nil && err != filer.ErrNotFound { glog.V(0).Infoln("failing to find path in filer store", path, err.Error()) writeJsonError(w, r, http.StatusInternalServerError, err) - return } else if fileId != "" && err == nil { urlLocation, err = operation.LookupFileId(fs.getMasterNode(), fileId) if err != nil { glog.V(1).Infoln("operation LookupFileId %s failed, err is %s", fileId, err.Error()) w.WriteHeader(http.StatusNotFound) - return } + } else if fileId == "" && err == filer.ErrNotFound { + w.WriteHeader(http.StatusNotFound) } return } @@ -315,6 +315,8 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { if r.Method != "PUT" { if oldFid, err := fs.filer.FindFile(path); err == nil { operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) + } else if err != nil && err != filer.ErrNotFound { + glog.V(0).Infof("error %v occur when finding %s in filer store", err, path) } } @@ -498,6 +500,8 @@ func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, conte if r.Method != "PUT" { if oldFid, err := fs.filer.FindFile(path); err == nil { operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) + } else if err != nil && err != filer.ErrNotFound { + glog.V(0).Infof("error %v occur when finding %s in filer store", err, path) } } From 1c3670630682a330be460cc367509390c818712d Mon Sep 17 00:00:00 2001 From: Jesper Zedlitz Date: Thu, 8 Sep 2016 09:19:06 +0200 Subject: [PATCH 19/31] Changed Dockerfile so it uses a special entrypoint script. All parameters are passed through to weed. Depending on the command the entrypoint.sh script adds parameters to link containers. --- Dockerfile | 21 --------------------- docker/Dockerfile | 18 ++++++++++++++++++ docker/entrypoint.sh | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 21 deletions(-) delete mode 100644 Dockerfile create mode 100644 docker/Dockerfile create mode 100755 docker/entrypoint.sh diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index e87538fdb..000000000 --- a/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM progrium/busybox - -WORKDIR /opt/weed - -RUN opkg-install curl -RUN echo tlsv1 >> ~/.curlrc - -RUN \ - curl -Lks https://bintray.com$(curl -Lk http://bintray.com/chrislusf/seaweedfs/seaweedfs/_latestVersion | grep linux_amd64.tar.gz | sed -n "/href/ s/.*href=['\"]\([^'\"]*\)['\"].*/\1/gp") | gunzip | tar -xf - -C /opt/weed/ && \ - mkdir ./bin && mv ./*/* ./bin && \ - chmod +x ./bin/weed - -EXPOSE 8080 -EXPOSE 9333 - -VOLUME /data - -ENV WEED_HOME /opt/weed -ENV PATH ${PATH}:${WEED_HOME}/bin - -ENTRYPOINT ["weed"] diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..ed1c4dfe3 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,18 @@ +FROM progrium/busybox + +COPY entrypoint.sh /entrypoint.sh +COPY Dockerfile /etc/Dockerfile + +RUN opkg-install curl +RUN echo tlsv1 >> ~/.curlrc + +RUN curl -Lks https://bintray.com$(curl -Lk http://bintray.com/chrislusf/seaweedfs/seaweedfs/_latestVersion | grep linux_amd64.tar.gz | sed -n "/href/ s/.*href=['\"]\([^'\"]*\)['\"].*/\1/gp") | gunzip | tar -xf - && \ + mv go_*amd64/weed /usr/bin/ && \ + rm -r go_*amd64 + +EXPOSE 8080 +EXPOSE 9333 + +VOLUME /data + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 000000000..bdde2caa4 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +case "$1" in + + 'master') + ARGS="-ip `hostname -i` -mdir /data" + # Is this instance linked with an other master? (Docker commandline "--link master1:master") + if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then + ARGS="$ARGS -peers=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT" + fi + /usr/bin/weed $@ $ARGS + ;; + + 'volume') + ARGS="-ip `hostname -i` -dir /data" + # Is this instance linked with a master? (Docker commandline "--link master1:master") + if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then + ARGS="$ARGS -mserver=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT" + fi + /usr/bin/weed $@ $ARGS + ;; + + 'server') + ARGS="-ip `hostname -i` -dir /data" + if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then + ARGS="$ARGS -master.peers=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT" + fi + /usr/bin/weed $@ $ARGS + ;; + + *) + /usr/bin/weed $@ + ;; +esac From 01cbd5cb589f27ba4ffc5db3ca868c91f0d2c8a9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 8 Sep 2016 09:50:27 -0700 Subject: [PATCH 20/31] lock fix https://github.com/chrislusf/seaweedfs/issues/367 --- weed/storage/compact_map.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/storage/compact_map.go b/weed/storage/compact_map.go index 6afaf7df8..721be2ec7 100644 --- a/weed/storage/compact_map.go +++ b/weed/storage/compact_map.go @@ -41,10 +41,10 @@ func NewCompactSection(start Key) *CompactSection { //return old entry size func (cs *CompactSection) Set(key Key, offset uint32, size uint32) uint32 { ret := uint32(0) + cs.Lock() if key > cs.end { cs.end = key } - cs.Lock() if i := cs.binarySearchValues(key); i >= 0 { ret = cs.values[i].Size //println("key", key, "old size", ret) From 1bc041b46de0db2e1baa9e56aa0fca2bb61a48b8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 22 Sep 2016 20:31:17 -0700 Subject: [PATCH 21/31] add a new way to manually compact corrupted volume fix https://github.com/chrislusf/seaweedfs/issues/371 --- weed/command/compact.go | 11 +++++- weed/storage/volume_vacuum.go | 69 +++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/weed/command/compact.go b/weed/command/compact.go index ba2fbf867..db11880ec 100644 --- a/weed/command/compact.go +++ b/weed/command/compact.go @@ -23,6 +23,7 @@ var ( compactVolumePath = cmdCompact.Flag.String("dir", ".", "data directory to store files") compactVolumeCollection = cmdCompact.Flag.String("collection", "", "volume collection name") compactVolumeId = cmdCompact.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir.") + compactMethod = cmdCompact.Flag.Int("method", 0, "option to choose which compact method. use 0 or 1.") ) func runCompact(cmd *Command, args []string) bool { @@ -37,8 +38,14 @@ func runCompact(cmd *Command, args []string) bool { if err != nil { glog.Fatalf("Load Volume [ERROR] %s\n", err) } - if err = v.Compact(); err != nil { - glog.Fatalf("Compact Volume [ERROR] %s\n", err) + if *compactMethod == 0 { + if err = v.Compact(); err != nil { + glog.Fatalf("Compact Volume [ERROR] %s\n", err) + } + } else { + if err = v.Compact2(); err != nil { + glog.Fatalf("Compact Volume [ERROR] %s\n", err) + } } return true diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index 9b9a27816..785e2ff88 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -23,6 +23,14 @@ func (v *Volume) Compact() error { glog.V(3).Infof("creating copies for volume %d ...", v.Id) return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx") } + +func (v *Volume) Compact2() error { + glog.V(3).Infof("Compact2 ...") + filePath := v.FileName() + glog.V(3).Infof("creating copies for volume %d ...", v.Id) + return v.copyDataBasedOnIndexFile(filePath+".cpd", filePath+".cpx") +} + func (v *Volume) commitCompact() error { glog.V(3).Infof("Committing vacuuming...") v.dataFileAccessLock.Lock() @@ -91,3 +99,64 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err erro return } + +func (v *Volume) copyDataBasedOnIndexFile(dstName, idxName string) (err error) { + var ( + dst, idx, oldIndexFile *os.File + ) + if dst, err = os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil { + return + } + defer dst.Close() + + if idx, err = os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil { + return + } + defer idx.Close() + + if oldIndexFile, err = os.OpenFile(v.FileName()+".idx", os.O_RDONLY, 0644); err != nil { + return + } + defer oldIndexFile.Close() + + nm := NewNeedleMap(idx) + now := uint64(time.Now().Unix()) + + v.SuperBlock.CompactRevision++ + dst.Write(v.SuperBlock.Bytes()) + new_offset := int64(SuperBlockSize) + + WalkIndexFile(oldIndexFile, func(key uint64, offset, size uint32) error { + if size <= 0 { + return nil + } + + nv, ok := v.nm.Get(key) + if !ok { + return nil + } + + n := new(Needle) + n.ReadData(v.dataFile, int64(offset)*NeedlePaddingSize, size, v.Version()) + defer n.ReleaseMemory() + + if n.HasTtl() && now >= n.LastModified+uint64(v.Ttl.Minutes()*60) { + return nil + } + + glog.V(4).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv) + if nv.Offset == offset && nv.Size > 0 { + if err = nm.Put(n.Id, uint32(new_offset/NeedlePaddingSize), n.Size); err != nil { + return fmt.Errorf("cannot put needle: %s", err) + } + if _, err = n.Append(dst, v.Version()); err != nil { + return fmt.Errorf("cannot append needle: %s", err) + } + new_offset += n.DiskSize() + glog.V(3).Infoln("saving key", n.Id, "volume offset", offset, "=>", new_offset, "data_size", n.Size) + } + return nil + }) + + return +} From 0fbec288584fac5a8dcc46177878cd4ed5741167 Mon Sep 17 00:00:00 2001 From: Felipe Roberto Date: Fri, 23 Sep 2016 13:49:38 -0300 Subject: [PATCH 22/31] how to use with docker --- README.md | 23 ++++++++++++++++++++++- docker/docker-compose.yml | 23 +++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 docker/docker-compose.yml diff --git a/README.md b/README.md index 3582bdb32..6c5bd35b1 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,6 @@ SeaweedFS uses HTTP REST operations to write, read, delete. The responses are in ``` > ./weed master - ``` ### Start Volume Servers ### @@ -61,8 +60,30 @@ SeaweedFS uses HTTP REST operations to write, read, delete. The responses are in ``` > weed volume -dir="/tmp/data1" -max=5 -mserver="localhost:9333" -port=8080 & > weed volume -dir="/tmp/data2" -max=10 -mserver="localhost:9333" -port=8081 & +``` + +### Running with Docker ### +Use with docker is easy as run locally, you can pass all args like above. But you don't need to worry about "-ip". It'll be treated by the entrypoint script. + +``` +docker run -p 9333:9333 --name master chrislusf/seaweedfs master +``` ``` +docker run -p 8080:8080 --name volume --link master chrislusf/seaweedfs volume -max=5 -mserver="master:9333" -port=8080 +``` +#### With Compose #### +But with Compose it's easiest. +To startup just run: +``` +docker-compose -f docker/docker-compose.yml up +``` +And to scale: +``` +docker-compose -f docker/docker-compose.yml scale volume=2 +``` +Remember that if multiple containers for volume are created on a single host, the port will clash. + ### Write File ### diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 000000000..3ccf596d5 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,23 @@ +version: '2' + +services: + master: + image: chrislusf/seaweedfs + ports: + - 9333:9333 + command: "master" + networks: + default: + aliases: + - seaweed_master + volume: + image: chrislusf/seaweedfs + ports: + - 8080:8080 + command: 'volume -max=5 -mserver="master:9333" -port=8080' + depends_on: + - master + networks: + default: + aliases: + - seaweed_volume \ No newline at end of file From 8fa61c28e4745d5ec7e89508740ce9cb2900fffb Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Sep 2016 10:07:06 -0700 Subject: [PATCH 23/31] Update README.md --- README.md | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/README.md b/README.md index 6c5bd35b1..41b8f6be3 100644 --- a/README.md +++ b/README.md @@ -62,28 +62,6 @@ SeaweedFS uses HTTP REST operations to write, read, delete. The responses are in > weed volume -dir="/tmp/data2" -max=10 -mserver="localhost:9333" -port=8081 & ``` -### Running with Docker ### - -Use with docker is easy as run locally, you can pass all args like above. But you don't need to worry about "-ip". It'll be treated by the entrypoint script. - -``` -docker run -p 9333:9333 --name master chrislusf/seaweedfs master -``` -``` -docker run -p 8080:8080 --name volume --link master chrislusf/seaweedfs volume -max=5 -mserver="master:9333" -port=8080 -``` -#### With Compose #### -But with Compose it's easiest. -To startup just run: -``` -docker-compose -f docker/docker-compose.yml up -``` -And to scale: -``` -docker-compose -f docker/docker-compose.yml scale volume=2 -``` -Remember that if multiple containers for volume are created on a single host, the port will clash. - ### Write File ### From 7e2921832752b2787a4392b9000d13ac00a4fdfe Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 26 Sep 2016 22:26:39 -0700 Subject: [PATCH 24/31] add a template for makeupDiff --- weed/storage/volume_vacuum.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index 785e2ff88..81db68492 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -38,6 +38,7 @@ func (v *Volume) commitCompact() error { glog.V(3).Infof("Got Committing lock...") v.nm.Close() _ = v.dataFile.Close() + makeupDiff(v.FileName()+".cpd", v.FileName()+".cpx", v.FileName()+".dat", v.FileName()+".idx") var e error if e = os.Rename(v.FileName()+".cpd", v.FileName()+".dat"); e != nil { return e @@ -54,6 +55,9 @@ func (v *Volume) commitCompact() error { return nil } +func makeupDiff(newDatFile, newIdxFile, oldDatFile, oldIdxFile string) (err error) { +} + func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err error) { var ( dst, idx *os.File From dffad65f2f3b1e87c7ac5274065730cabceeee99 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 26 Sep 2016 22:30:44 -0700 Subject: [PATCH 25/31] fix compilation --- weed/storage/volume_vacuum.go | 1 + 1 file changed, 1 insertion(+) diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index 81db68492..51d74e311 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -56,6 +56,7 @@ func (v *Volume) commitCompact() error { } func makeupDiff(newDatFile, newIdxFile, oldDatFile, oldIdxFile string) (err error) { + return nil } func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err error) { From ed848425c77ccca0a9d2f30c7f631bc50f28cd32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=8D=E6=99=93=E6=A0=8B?= Date: Thu, 29 Sep 2016 13:57:23 +0800 Subject: [PATCH 26/31] supplemental data between compacting and commit compacting --- weed/storage/volume.go | 8 +++ weed/storage/volume_vacuum.go | 110 +++++++++++++++++++++++++++++++--- 2 files changed, 110 insertions(+), 8 deletions(-) diff --git a/weed/storage/volume.go b/weed/storage/volume.go index 801dfe267..258787701 100644 --- a/weed/storage/volume.go +++ b/weed/storage/volume.go @@ -10,6 +10,11 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" ) +type keyField struct { + offset uint32 + size uint32 +} + type Volume struct { Id VolumeId dir string @@ -23,6 +28,9 @@ type Volume struct { dataFileAccessLock sync.Mutex lastModifiedTime uint64 //unix time in seconds + + lastCompactingIndexOffset uint64 + incrementedHasUpdatedIndexEntry map[uint64]keyField } func NewVolume(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL) (v *Volume, e error) { diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index 51d74e311..55c248894 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -6,6 +6,7 @@ import ( "time" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util" ) func (v *Volume) garbageLevel() float64 { @@ -20,7 +21,8 @@ func (v *Volume) Compact() error { //glog.V(3).Infof("Got Compaction lock...") filePath := v.FileName() - glog.V(3).Infof("creating copies for volume %d ...", v.Id) + v.lastCompactingIndexOffset = v.nm.IndexFileSize() + glog.V(3).Infof("creating copies for volume %d ,last offset %d...", v.Id, v.lastCompactingIndexOffset) return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx") } @@ -38,14 +40,28 @@ func (v *Volume) commitCompact() error { glog.V(3).Infof("Got Committing lock...") v.nm.Close() _ = v.dataFile.Close() - makeupDiff(v.FileName()+".cpd", v.FileName()+".cpx", v.FileName()+".dat", v.FileName()+".idx") + var e error - if e = os.Rename(v.FileName()+".cpd", v.FileName()+".dat"); e != nil { - return e - } - if e = os.Rename(v.FileName()+".cpx", v.FileName()+".idx"); e != nil { - return e + if e = v.makeupDiff(v.FileName()+".cpd", v.FileName()+".cpx", v.FileName()+".dat", v.FileName()+".idx"); e != nil { + glog.V(0).Infof("makeupDiff in commitCompact failed %v", e) + e = os.Remove(v.FileName() + ".cpd") + if e != nil { + return e + } + e = os.Remove(v.FileName() + ".cpx") + if e != nil { + return e + } + } else { + var e error + if e = os.Rename(v.FileName()+".cpd", v.FileName()+".dat"); e != nil { + return e + } + if e = os.Rename(v.FileName()+".cpx", v.FileName()+".idx"); e != nil { + return e + } } + //glog.V(3).Infof("Pretending to be vacuuming...") //time.Sleep(20 * time.Second) glog.V(3).Infof("Loading Commit file...") @@ -55,7 +71,85 @@ func (v *Volume) commitCompact() error { return nil } -func makeupDiff(newDatFile, newIdxFile, oldDatFile, oldIdxFile string) (err error) { +func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldIdxFileName string) (err error) { + var indexSize int64 + + oldIdxFile, err := os.Open(oldIdxFileName) + defer oldIdxFile.Close() + + oldDatFile, err := os.Open(oldDatFileName) + defer oldDatFile.Close() + + if indexSize, err = verifyIndexFileIntegrity(oldIdxFile); err != nil { + return fmt.Errorf("verifyIndexFileIntegrity %s failed: %v", oldIdxFileName, err) + } + if indexSize == 0 || uint64(indexSize) <= v.lastCompactingIndexOffset { + return nil + } + + v.incrementedHasUpdatedIndexEntry = make(map[uint64]keyField) + for idx_offset := indexSize; uint64(idx_offset) >= v.lastCompactingIndexOffset; idx_offset -= NeedleIndexSize { + var IdxEntry []byte + if IdxEntry, err = readIndexEntryAtOffset(oldIdxFile, idx_offset); err != nil { + return fmt.Errorf("readIndexEntry %s at offset %d failed: %v", oldIdxFileName, idx_offset, err) + } + key, offset, size := idxFileEntry(IdxEntry) + if _, found := v.incrementedHasUpdatedIndexEntry[key]; !found { + v.incrementedHasUpdatedIndexEntry[key] = keyField{ + offset: offset, + size: size, + } + } else { + continue + } + } + + if len(v.incrementedHasUpdatedIndexEntry) > 0 { + var ( + dst, idx *os.File + ) + if dst, err = os.OpenFile(newDatFileName, os.O_WRONLY, 0644); err != nil { + return + } + defer dst.Close() + + if idx, err = os.OpenFile(newIdxFileName, os.O_WRONLY, 0644); err != nil { + return + } + defer idx.Close() + + idx_entry_bytes := make([]byte, 16) + for key, incre_idx_entry := range v.incrementedHasUpdatedIndexEntry { + util.Uint64toBytes(idx_entry_bytes[0:8], key) + util.Uint32toBytes(idx_entry_bytes[8:12], incre_idx_entry.offset) + util.Uint32toBytes(idx_entry_bytes[12:16], incre_idx_entry.size) + + if _, err := idx.Seek(0, 2); err != nil { + return fmt.Errorf("cannot seek end of indexfile %s: %v", + newIdxFileName, err) + } + _, err = idx.Write(idx_entry_bytes) + + //even the needle cache in memory is hit, the need_bytes is correct + needle_bytes, _, _ := ReadNeedleBlob(dst, int64(incre_idx_entry.offset)*NeedlePaddingSize, incre_idx_entry.size) + + var offset int64 + if offset, err = dst.Seek(0, 2); err != nil { + glog.V(0).Infof("failed to seek the end of file: %v", err) + return + } + //ensure file writing starting from aligned positions + if offset%NeedlePaddingSize != 0 { + offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize) + if offset, err = v.dataFile.Seek(offset, 0); err != nil { + glog.V(0).Infof("failed to align in datafile %s: %v", v.dataFile.Name(), err) + return + } + } + dst.Write(needle_bytes) + } + } + return nil } From ce1f7ab66250cf3ec82e85c458ec273170e7531a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=8D=E6=99=93=E6=A0=8B?= Date: Fri, 7 Oct 2016 16:22:24 +0800 Subject: [PATCH 27/31] makediff func with UT case --- weed/storage/volume.go | 9 +-- weed/storage/volume_checking.go | 1 - weed/storage/volume_vacuum.go | 100 ++++++++++++++++++++++------- weed/storage/volume_vacuum_test.go | 53 +++++++++++++++ 4 files changed, 132 insertions(+), 31 deletions(-) create mode 100644 weed/storage/volume_vacuum_test.go diff --git a/weed/storage/volume.go b/weed/storage/volume.go index 258787701..c1d531376 100644 --- a/weed/storage/volume.go +++ b/weed/storage/volume.go @@ -10,11 +10,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" ) -type keyField struct { - offset uint32 - size uint32 -} - type Volume struct { Id VolumeId dir string @@ -29,8 +24,8 @@ type Volume struct { dataFileAccessLock sync.Mutex lastModifiedTime uint64 //unix time in seconds - lastCompactingIndexOffset uint64 - incrementedHasUpdatedIndexEntry map[uint64]keyField + lastCompactIndexOffset uint64 + lastCompactRevision uint16 } func NewVolume(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL) (v *Volume, e error) { diff --git a/weed/storage/volume_checking.go b/weed/storage/volume_checking.go index d424010f1..48f707594 100644 --- a/weed/storage/volume_checking.go +++ b/weed/storage/volume_checking.go @@ -21,7 +21,6 @@ func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) error { return fmt.Errorf("readLastIndexEntry %s failed: %v", indexFile.Name(), e) } key, offset, size := idxFileEntry(lastIdxEntry) - //deleted index entry could not point to deleted needle if offset == 0 { return nil } diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index 55c248894..723300557 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -21,8 +21,9 @@ func (v *Volume) Compact() error { //glog.V(3).Infof("Got Compaction lock...") filePath := v.FileName() - v.lastCompactingIndexOffset = v.nm.IndexFileSize() - glog.V(3).Infof("creating copies for volume %d ,last offset %d...", v.Id, v.lastCompactingIndexOffset) + v.lastCompactIndexOffset = v.nm.IndexFileSize() + v.lastCompactRevision = v.SuperBlock.CompactRevision + glog.V(3).Infof("creating copies for volume %d ,last offset %d...", v.Id, v.lastCompactIndexOffset) return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx") } @@ -71,6 +72,21 @@ func (v *Volume) commitCompact() error { return nil } +func fetchCompactRevisionFromDatFile(file *os.File) (compactRevision uint16, err error) { + if _, err = file.Seek(0, 0); err != nil { + return 0, fmt.Errorf("cannot seek to the beginning of %s: %v", file.Name(), err) + } + header := make([]byte, SuperBlockSize) + if _, e := file.Read(header); e != nil { + return 0, fmt.Errorf("cannot read file %s 's super block: %v", file.Name(), e) + } + superBlock, err := ParseSuperBlock(header) + if err != nil { + return 0, err + } + return superBlock.CompactRevision, nil +} + func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldIdxFileName string) (err error) { var indexSize int64 @@ -83,56 +99,67 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI if indexSize, err = verifyIndexFileIntegrity(oldIdxFile); err != nil { return fmt.Errorf("verifyIndexFileIntegrity %s failed: %v", oldIdxFileName, err) } - if indexSize == 0 || uint64(indexSize) <= v.lastCompactingIndexOffset { + if indexSize == 0 || uint64(indexSize) <= v.lastCompactIndexOffset { return nil } - v.incrementedHasUpdatedIndexEntry = make(map[uint64]keyField) - for idx_offset := indexSize; uint64(idx_offset) >= v.lastCompactingIndexOffset; idx_offset -= NeedleIndexSize { + oldDatCompactRevision, err := fetchCompactRevisionFromDatFile(oldDatFile) + if err != nil { + return + } + if oldDatCompactRevision != v.lastCompactRevision { + return fmt.Errorf("current old dat file's compact revision %d is not the expected one %d", oldDatCompactRevision, v.lastCompactRevision) + } + + type keyField struct { + offset uint32 + size uint32 + } + incrementedHasUpdatedIndexEntry := make(map[uint64]keyField) + + for idx_offset := indexSize - NeedleIndexSize; uint64(idx_offset) >= v.lastCompactIndexOffset; idx_offset -= NeedleIndexSize { var IdxEntry []byte if IdxEntry, err = readIndexEntryAtOffset(oldIdxFile, idx_offset); err != nil { return fmt.Errorf("readIndexEntry %s at offset %d failed: %v", oldIdxFileName, idx_offset, err) } key, offset, size := idxFileEntry(IdxEntry) - if _, found := v.incrementedHasUpdatedIndexEntry[key]; !found { - v.incrementedHasUpdatedIndexEntry[key] = keyField{ + if _, found := incrementedHasUpdatedIndexEntry[key]; !found { + incrementedHasUpdatedIndexEntry[key] = keyField{ offset: offset, size: size, } - } else { - continue } } - if len(v.incrementedHasUpdatedIndexEntry) > 0 { + if len(incrementedHasUpdatedIndexEntry) > 0 { var ( dst, idx *os.File ) - if dst, err = os.OpenFile(newDatFileName, os.O_WRONLY, 0644); err != nil { + if dst, err = os.OpenFile(newDatFileName, os.O_RDWR, 0644); err != nil { return } defer dst.Close() - if idx, err = os.OpenFile(newIdxFileName, os.O_WRONLY, 0644); err != nil { + if idx, err = os.OpenFile(newIdxFileName, os.O_RDWR, 0644); err != nil { return } defer idx.Close() + var newDatCompactRevision uint16 + newDatCompactRevision, err = fetchCompactRevisionFromDatFile(dst) + if err != nil { + return + } + if oldDatCompactRevision+1 != newDatCompactRevision { + return fmt.Errorf("oldDatFile %s 's compact revision is %d while newDatFile %s 's compact revision is %d", oldDatFileName, oldDatCompactRevision, newDatFileName, newDatCompactRevision) + } + idx_entry_bytes := make([]byte, 16) - for key, incre_idx_entry := range v.incrementedHasUpdatedIndexEntry { + for key, incre_idx_entry := range incrementedHasUpdatedIndexEntry { util.Uint64toBytes(idx_entry_bytes[0:8], key) util.Uint32toBytes(idx_entry_bytes[8:12], incre_idx_entry.offset) util.Uint32toBytes(idx_entry_bytes[12:16], incre_idx_entry.size) - if _, err := idx.Seek(0, 2); err != nil { - return fmt.Errorf("cannot seek end of indexfile %s: %v", - newIdxFileName, err) - } - _, err = idx.Write(idx_entry_bytes) - - //even the needle cache in memory is hit, the need_bytes is correct - needle_bytes, _, _ := ReadNeedleBlob(dst, int64(incre_idx_entry.offset)*NeedlePaddingSize, incre_idx_entry.size) - var offset int64 if offset, err = dst.Seek(0, 2); err != nil { glog.V(0).Infof("failed to seek the end of file: %v", err) @@ -146,7 +173,34 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI return } } - dst.Write(needle_bytes) + + //updated needle + if incre_idx_entry.offset != 0 && incre_idx_entry.size != 0 { + //even the needle cache in memory is hit, the need_bytes is correct + var needle_bytes []byte + needle_bytes, _, err = ReadNeedleBlob(oldDatFile, int64(incre_idx_entry.offset)*NeedlePaddingSize, incre_idx_entry.size) + if err != nil { + return + } + dst.Write(needle_bytes) + util.Uint32toBytes(idx_entry_bytes[8:12], uint32(offset/NeedlePaddingSize)) + } else { //deleted needle + //fakeDelNeedle 's default Data field is nil + fakeDelNeedle := new(Needle) + fakeDelNeedle.Id = key + fakeDelNeedle.Cookie = 0x12345678 + _, err = fakeDelNeedle.Append(dst, v.Version()) + if err != nil { + return + } + util.Uint32toBytes(idx_entry_bytes[8:12], uint32(0)) + } + + if _, err := idx.Seek(0, 2); err != nil { + return fmt.Errorf("cannot seek end of indexfile %s: %v", + newIdxFileName, err) + } + _, err = idx.Write(idx_entry_bytes) } } diff --git a/weed/storage/volume_vacuum_test.go b/weed/storage/volume_vacuum_test.go new file mode 100644 index 000000000..02d1a2b56 --- /dev/null +++ b/weed/storage/volume_vacuum_test.go @@ -0,0 +1,53 @@ +package storage + +import ( + "testing" +) + +/* +makediff test steps +1. launch weed server at your local/dev environment, (option +"garbageThreshold" for master and option "max" for volume should be set with specific value which would let +preparing test prerequisite easier ) + a) ./weed master -garbageThreshold=0.99 -mdir=./m + b) ./weed volume -dir=./data -max=1 -mserver=localhost:9333 -port=8080 +2. upload 4 different files, you could call dir/assign to get 4 different fids + a) upload file A with fid a + b) upload file B with fid b + c) upload file C with fid c + d) upload file D with fid d +3. update file A and C + a) modify file A and upload file A with fid a + b) modify file C and upload file C with fid c + c) record the current 1.idx's file size(lastCompactIndexOffset value) +4. Compacting the data file + a) run curl http://localhost:8080/admin/vacuum/compact?volumeId=1 + b) verify the 1.cpd and 1.cpx is created under volume directory +5. update file B and delete file D + a) modify file B and upload file B with fid b + d) delete file B with fid b +6. Now you could run the following UT case, the case should be run successfully +7. Compact commit manually + a) mv 1.cpd 1.dat + b) mv 1.cpx 1.idx +8. Restart Volume Server +9. Now you should get updated file A,B,C +*/ + +func TestMakeDiff(t *testing.T) { + + v := new(Volume) + //lastCompactIndexOffset value is the index file size before step 4 + v.lastCompactIndexOffset = 96 + v.SuperBlock.version = 0x2 + err := v.makeupDiff( + "/yourpath/1.cpd", + "/yourpath/1.cpx", + "/yourpath/1.dat", + "/yourpath/1.idx") + if err != nil { + t.Errorf("makeupDiff err is %v", err) + } else { + t.Log("makeupDiff Succeeded") + } +} From 7d382ba5fec2539821047e81b4f9a8ce20af0331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=8D=E6=99=93=E6=A0=8B?= Date: Fri, 7 Oct 2016 16:34:22 +0800 Subject: [PATCH 28/31] comment UT case --- weed/storage/volume_vacuum_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/weed/storage/volume_vacuum_test.go b/weed/storage/volume_vacuum_test.go index 02d1a2b56..8ab59404d 100644 --- a/weed/storage/volume_vacuum_test.go +++ b/weed/storage/volume_vacuum_test.go @@ -34,6 +34,7 @@ preparing test prerequisite easier ) 9. Now you should get updated file A,B,C */ +/* func TestMakeDiff(t *testing.T) { v := new(Volume) @@ -51,3 +52,4 @@ func TestMakeDiff(t *testing.T) { t.Log("makeupDiff Succeeded") } } +*/ From 7d73bbb07399cc504ae2ebdcfdc164dc01295916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=8D=E6=99=93=E6=A0=8B?= Date: Fri, 7 Oct 2016 16:40:51 +0800 Subject: [PATCH 29/31] comment UT case --- weed/storage/volume_vacuum_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/weed/storage/volume_vacuum_test.go b/weed/storage/volume_vacuum_test.go index 8ab59404d..c2fac6ce8 100644 --- a/weed/storage/volume_vacuum_test.go +++ b/weed/storage/volume_vacuum_test.go @@ -34,22 +34,22 @@ preparing test prerequisite easier ) 9. Now you should get updated file A,B,C */ -/* func TestMakeDiff(t *testing.T) { v := new(Volume) //lastCompactIndexOffset value is the index file size before step 4 v.lastCompactIndexOffset = 96 v.SuperBlock.version = 0x2 - err := v.makeupDiff( - "/yourpath/1.cpd", - "/yourpath/1.cpx", - "/yourpath/1.dat", - "/yourpath/1.idx") - if err != nil { - t.Errorf("makeupDiff err is %v", err) - } else { - t.Log("makeupDiff Succeeded") - } + /* + err := v.makeupDiff( + "/yourpath/1.cpd", + "/yourpath/1.cpx", + "/yourpath/1.dat", + "/yourpath/1.idx") + if err != nil { + t.Errorf("makeupDiff err is %v", err) + } else { + t.Log("makeupDiff Succeeded") + } + */ } -*/ From 48a24559a3c46efaa99ca6f3d43715e545381441 Mon Sep 17 00:00:00 2001 From: Amin Cheloh Date: Wed, 12 Oct 2016 16:47:56 +0700 Subject: [PATCH 30/31] Update entrypoint.sh Running application becomes container's PID 1 allow to receives Unix signals --- docker/entrypoint.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index bdde2caa4..34ab61148 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -8,7 +8,7 @@ case "$1" in if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then ARGS="$ARGS -peers=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT" fi - /usr/bin/weed $@ $ARGS + exec /usr/bin/weed $@ $ARGS ;; 'volume') @@ -17,7 +17,7 @@ case "$1" in if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then ARGS="$ARGS -mserver=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT" fi - /usr/bin/weed $@ $ARGS + exec /usr/bin/weed $@ $ARGS ;; 'server') @@ -25,10 +25,10 @@ case "$1" in if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then ARGS="$ARGS -master.peers=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT" fi - /usr/bin/weed $@ $ARGS + exec /usr/bin/weed $@ $ARGS ;; *) - /usr/bin/weed $@ + exec /usr/bin/weed $@ ;; -esac +esac From 54bd1c406a70fab3f71332511897460fed32e568 Mon Sep 17 00:00:00 2001 From: Amin Cheloh Date: Wed, 12 Oct 2016 17:10:38 +0700 Subject: [PATCH 31/31] Update Dockerfile Move COPY /entrypoint.sh to bottom and make sure entrypoint.sh have execute permission --- docker/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ed1c4dfe3..21e5a7b47 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,8 +1,5 @@ FROM progrium/busybox -COPY entrypoint.sh /entrypoint.sh -COPY Dockerfile /etc/Dockerfile - RUN opkg-install curl RUN echo tlsv1 >> ~/.curlrc @@ -15,4 +12,7 @@ EXPOSE 9333 VOLUME /data +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + ENTRYPOINT ["/entrypoint.sh"]