Merge branch 'master' into add_s3

weed/filesys/dir.go

@@ -4,17 +4,20 @@ import (
 	"context"
 	"os"
 	"path"
+	"path/filepath"
+	"time"
+
 	"bazil.org/fuse"
 	"bazil.org/fuse/fs"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"path/filepath"
-	"time"
+	"github.com/chrislusf/seaweedfs/weed/filer2"
 )
 
 type Dir struct {
 	Path       string
 	wfs        *WFS
+	attributes *filer_pb.FuseAttributes
 }
 
 var _ = fs.Node(&Dir{})
@@ -24,6 +27,7 @@ var _ = fs.NodeRequestLookuper(&Dir{})
 var _ = fs.HandleReadDirAller(&Dir{})
 var _ = fs.NodeRemover(&Dir{})
 var _ = fs.NodeRenamer(&Dir{})
+var _ = fs.NodeSetattrer(&Dir{})
 
 func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 
@@ -48,8 +52,6 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 
 	parent, name := filepath.Split(dir.Path)
 
-	var attributes *filer_pb.FuseAttributes
-
 	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.GetEntryAttributesRequest{
@@ -64,7 +66,7 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 			return err
 		}
 
-		attributes = resp.Attributes
+		dir.attributes = resp.Attributes
 
 		return nil
 	})
@@ -76,25 +78,25 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 	// glog.V(1).Infof("dir %s: %v", dir.Path, attributes)
 	// glog.V(1).Infof("dir %s permission: %v", dir.Path, os.FileMode(attributes.FileMode))
 
-	attr.Mode = os.FileMode(attributes.FileMode) | os.ModeDir
-	if dir.Path == "/" && attributes.FileMode == 0 {
+	attr.Mode = os.FileMode(dir.attributes.FileMode) | os.ModeDir
+	if dir.Path == "/" && dir.attributes.FileMode == 0 {
 		attr.Valid = time.Second
 	}
 
-	attr.Mtime = time.Unix(attributes.Mtime, 0)
-	attr.Ctime = time.Unix(attributes.Crtime, 0)
-	attr.Gid = attributes.Gid
-	attr.Uid = attributes.Uid
+	attr.Mtime = time.Unix(dir.attributes.Mtime, 0)
+	attr.Ctime = time.Unix(dir.attributes.Crtime, 0)
+	attr.Gid = dir.attributes.Gid
+	attr.Uid = dir.attributes.Uid
 
 	return nil
 }
 
-func (dir *Dir) newFile(name string, chunks []*filer_pb.FileChunk) *File {
+func (dir *Dir) newFile(name string, chunks []*filer_pb.FileChunk, attr *filer_pb.FuseAttributes) *File {
 	return &File{
 		Name:       name,
 		dir:        dir,
 		wfs:        dir.wfs,
-		// attributes: &filer_pb.FuseAttributes{},
+		attributes: attr,
 		Chunks:     chunks,
 	}
 }
@@ -132,7 +134,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
 	})
 
 	if err == nil {
-		file := dir.newFile(req.Name, nil)
+		file := dir.newFile(req.Name, nil, &filer_pb.FuseAttributes{})
 		file.isOpen = true
 		return file, dir.wfs.AcquireHandle(file, req.Uid, req.Gid), nil
 	}
@@ -200,9 +202,9 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
 
 	if entry != nil {
 		if entry.IsDirectory {
-			node = &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs}
+			node = &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs, attributes: entry.Attributes}
 		} else {
-			node = dir.newFile(req.Name, entry.Chunks)
+			node = dir.newFile(req.Name, entry.Chunks, entry.Attributes)
 		}
 
 		resp.EntryValid = time.Duration(0)
@@ -272,3 +274,45 @@ func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
 	})
 
 }
+
+func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
+
+	glog.V(3).Infof("%v dir setattr %+v, fh=%d", dir.Path, req, req.Handle)
+	if req.Valid.Mode() {
+		dir.attributes.FileMode = uint32(req.Mode)
+	}
+
+	if req.Valid.Uid() {
+		dir.attributes.Uid = req.Uid
+	}
+
+	if req.Valid.Gid() {
+		dir.attributes.Gid = req.Gid
+	}
+
+	if req.Valid.Mtime() {
+		dir.attributes.Mtime = req.Mtime.Unix()
+	}
+
+	parentDir, name := filer2.FullPath(dir.Path).DirAndName()
+	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.UpdateEntryRequest{
+			Directory: parentDir,
+			Entry: &filer_pb.Entry{
+				Name:       name,
+				Attributes: dir.attributes,
+			},
+		}
+
+		glog.V(1).Infof("set attr directory entry: %v", request)
+		_, err := client.UpdateEntry(ctx, request)
+		if err != nil {
+			glog.V(0).Infof("UpdateEntry %s: %v", dir.Path, err)
+			return fuse.EIO
+		}
+
+		return nil
+	})
+
+}

weed/filesys/file.go

@@ -34,7 +34,7 @@ func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error {
 
 	if file.attributes == nil || !file.isOpen {
 		item := file.wfs.listDirectoryEntriesCache.Get(file.fullpath())
-		if item != nil && !item.Expired(){
+		if item != nil && !item.Expired() {
 			entry := item.Value().(*filer_pb.Entry)
 			file.Chunks = entry.Chunks
 			file.attributes = entry.Attributes
@@ -121,7 +121,26 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 		file.attributes.Mtime = req.Mtime.Unix()
 	}
 
+	return file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.UpdateEntryRequest{
+			Directory: file.dir.Path,
+			Entry: &filer_pb.Entry{
+				Name:       file.Name,
+				Attributes: file.attributes,
+				Chunks:     file.Chunks,
+			},
+		}
+
+		glog.V(1).Infof("set attr file entry: %v", request)
+		_, err := client.UpdateEntry(ctx, request)
+		if err != nil {
+			glog.V(0).Infof("UpdateEntry file %s/%s: %v", file.dir.Path, file.Name, err)
+			return fuse.EIO
+		}
+
 		return nil
+	})
 
 }
 

weed/server/master_ui/templates.go

@@ -39,7 +39,7 @@ var StatusTpl = template.Must(template.New("status").Parse(`<!DOCTYPE html>
                 <td><a href="http://{{ .Leader }}">{{ .Leader }}</a></td>
               </tr>
               <tr>
-                <td class="col-sm-2 field-label"><label>Peers:</label></td>
+                <td class="col-sm-2 field-label"><label>Other Masters:</label></td>
                 <td class="col-sm-10"><ul class="list-unstyled">
                 {{ range $k, $p := .Peers }}
                   <li><a href="{{ $p.ConnectionString }}">{{ $p.Name }}</a></li>

weed/storage/needle.go

@@ -3,18 +3,14 @@ package storage
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"mime"
 	"net/http"
-	"path"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/images"
-	"github.com/chrislusf/seaweedfs/weed/operation"
 	. "github.com/chrislusf/seaweedfs/weed/storage/types"
+	"io/ioutil"
 )
 
 const (
@@ -62,91 +58,20 @@ func ParseUpload(r *http.Request) (
 		}
 	}
 
-	form, fe := r.MultipartReader()
-	if fe != nil {
-		glog.V(0).Infoln("MultipartReader [ERROR]", fe)
-		e = fe
-		return
-	}
-
-	//first multi-part item
-	part, fe := form.NextPart()
-	if fe != nil {
-		glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
-		e = fe
-		return
-	}
+	isChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
 
-	fileName = part.FileName()
-	if fileName != "" {
-		fileName = path.Base(fileName)
+	if r.Method == "POST" {
+		fileName, data, mimeType, isGzipped, e = parseMultipart(r, isChunkedFile)
+	} else {
+		isGzipped = false
+		mimeType = r.Header.Get("Content-Type")
+		fileName = ""
+		data, e = ioutil.ReadAll(r.Body)
 	}
-
-	data, e = ioutil.ReadAll(part)
 	if e != nil {
-		glog.V(0).Infoln("Reading Content [ERROR]", e)
-		return
-	}
-
-	//if the filename is empty string, do a search on the other multi-part items
-	for fileName == "" {
-		part2, fe := form.NextPart()
-		if fe != nil {
-			break // no more or on error, just safely break
-		}
-
-		fName := part2.FileName()
-
-		//found the first <file type> multi-part has filename
-		if fName != "" {
-			data2, fe2 := ioutil.ReadAll(part2)
-			if fe2 != nil {
-				glog.V(0).Infoln("Reading Content [ERROR]", fe2)
-				e = fe2
 		return
 	}
 
-			//update
-			data = data2
-			fileName = path.Base(fName)
-			break
-		}
-	}
-
-	isChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
-
-	if !isChunkedFile {
-
-		dotIndex := strings.LastIndex(fileName, ".")
-		ext, mtype := "", ""
-		if dotIndex > 0 {
-			ext = strings.ToLower(fileName[dotIndex:])
-			mtype = mime.TypeByExtension(ext)
-		}
-		contentType := part.Header.Get("Content-Type")
-		if contentType != "" && mtype != contentType {
-			mimeType = contentType //only return mime type if not deductable
-			mtype = contentType
-		}
-
-		if part.Header.Get("Content-Encoding") == "gzip" {
-			isGzipped = true
-		} else if operation.IsGzippable(ext, mtype) {
-			if data, e = operation.GzipData(data); e != nil {
-				return
-			}
-			isGzipped = true
-		}
-		if ext == ".gz" {
-			if strings.HasSuffix(fileName, ".css.gz") ||
-				strings.HasSuffix(fileName, ".html.gz") ||
-				strings.HasSuffix(fileName, ".txt.gz") ||
-				strings.HasSuffix(fileName, ".js.gz") {
-				fileName = fileName[:len(fileName)-3]
-				isGzipped = true
-			}
-		}
-	}
 	modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
 	ttl, _ = ReadTTL(r.FormValue("ttl"))
 

weed/storage/needle_parse_multipart.go

@@ -0,0 +1,100 @@
+package storage
+
+import (
+	"mime"
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"net/http"
+	"path"
+	"io/ioutil"
+	"strings"
+)
+
+func parseMultipart(r *http.Request, isChunkedFile bool) (
+	fileName string, data []byte, mimeType string, isGzipped bool, e error) {
+	form, fe := r.MultipartReader()
+	if fe != nil {
+		glog.V(0).Infoln("MultipartReader [ERROR]", fe)
+		e = fe
+		return
+	}
+
+	//first multi-part item
+	part, fe := form.NextPart()
+	if fe != nil {
+		glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
+		e = fe
+		return
+	}
+
+	fileName = part.FileName()
+	if fileName != "" {
+		fileName = path.Base(fileName)
+	}
+
+	data, e = ioutil.ReadAll(part)
+	if e != nil {
+		glog.V(0).Infoln("Reading Content [ERROR]", e)
+		return
+	}
+
+	//if the filename is empty string, do a search on the other multi-part items
+	for fileName == "" {
+		part2, fe := form.NextPart()
+		if fe != nil {
+			break // no more or on error, just safely break
+		}
+
+		fName := part2.FileName()
+
+		//found the first <file type> multi-part has filename
+		if fName != "" {
+			data2, fe2 := ioutil.ReadAll(part2)
+			if fe2 != nil {
+				glog.V(0).Infoln("Reading Content [ERROR]", fe2)
+				e = fe2
+				return
+			}
+
+			//update
+			data = data2
+			fileName = path.Base(fName)
+			break
+		}
+	}
+
+	if !isChunkedFile {
+
+		dotIndex := strings.LastIndex(fileName, ".")
+		ext, mtype := "", ""
+		if dotIndex > 0 {
+			ext = strings.ToLower(fileName[dotIndex:])
+			mtype = mime.TypeByExtension(ext)
+		}
+		contentType := part.Header.Get("Content-Type")
+		if contentType != "" && mtype != contentType {
+			mimeType = contentType //only return mime type if not deductable
+			mtype = contentType
+		}
+
+		if part.Header.Get("Content-Encoding") == "gzip" {
+			isGzipped = true
+		} else if operation.IsGzippable(ext, mtype) {
+			if data, e = operation.GzipData(data); e != nil {
+				return
+			}
+			isGzipped = true
+		}
+		if ext == ".gz" {
+			if strings.HasSuffix(fileName, ".css.gz") ||
+				strings.HasSuffix(fileName, ".html.gz") ||
+				strings.HasSuffix(fileName, ".txt.gz") ||
+				strings.HasSuffix(fileName, ".js.gz") {
+				fileName = fileName[:len(fileName)-3]
+				isGzipped = true
+			}
+		}
+	}
+
+	return
+}