diff --git a/weed/command/command.go b/weed/command/command.go index c6665a7be..dbc18a053 100644 --- a/weed/command/command.go +++ b/weed/command/command.go @@ -30,7 +30,6 @@ var Commands = []*Command{ cmdMaster, cmdMasterFollower, cmdMount, - cmdMount2, cmdS3, cmdIam, cmdMsgBroker, diff --git a/weed/command/mount.go b/weed/command/mount.go deleted file mode 100644 index 545ba8a43..000000000 --- a/weed/command/mount.go +++ /dev/null @@ -1,86 +0,0 @@ -package command - -import ( - "os" - "time" -) - -type MountOptions struct { - filer *string - filerMountRootPath *string - dir *string - dirAutoCreate *bool - collection *string - replication *string - diskType *string - ttlSec *int - chunkSizeLimitMB *int - concurrentWriters *int - cacheDir *string - cacheSizeMB *int64 - dataCenter *string - allowOthers *bool - umaskString *string - nonempty *bool - volumeServerAccess *string - uidMap *string - gidMap *string - readOnly *bool - debug *bool - debugPort *int -} - -var ( - mountOptions MountOptions - mountCpuProfile *string - mountMemProfile *string - mountReadRetryTime *time.Duration -) - -func init() { - cmdMount.Run = runMount // break init cycle - mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "comma-separated weed filer location") - mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server") - mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory") - mountOptions.dirAutoCreate = cmdMount.Flag.Bool("dirAutoCreate", false, "auto create the directory to mount to") - mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files") - mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.") - mountOptions.diskType = cmdMount.Flag.String("disk", "", "[hdd|ssd|] hard drive or solid state drive or any tag") - mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds") - mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files") - mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0") - mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data") - mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 0, "local file chunk cache capacity in MB") - mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center") - mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system") - mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111") - mountOptions.nonempty = cmdMount.Flag.Bool("nonempty", false, "allows the mounting over a non-empty directory") - mountOptions.volumeServerAccess = cmdMount.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]") - mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated :") - mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated :") - mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only") - mountOptions.debug = cmdMount.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:/debug/pprof/goroutine?debug=2") - mountOptions.debugPort = cmdMount.Flag.Int("debug.port", 6061, "http port for debugging") - - mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file") - mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file") - mountReadRetryTime = cmdMount.Flag.Duration("readRetryTime", 6*time.Second, "maximum read retry wait time") -} - -var cmdMount = &Command{ - UsageLine: "mount -filer=localhost:8888 -dir=/some/dir", - Short: "mount weed filer to a directory as file system in userspace(FUSE)", - Long: `mount weed filer to userspace. - - Pre-requisites: - 1) have SeaweedFS master and volume servers running - 2) have a "weed filer" running - These 2 requirements can be achieved with one command "weed server -filer=true" - - This uses github.com/seaweedfs/fuse, which enables writing FUSE file systems on - Linux, and OS X. - - On OS X, it requires OSXFUSE (http://osxfuse.github.com/). - - `, -} diff --git a/weed/command/mount2.go b/weed/command/mount2.go index b285f5d3f..545ba8a43 100644 --- a/weed/command/mount2.go +++ b/weed/command/mount2.go @@ -5,7 +5,7 @@ import ( "time" ) -type Mount2Options struct { +type MountOptions struct { filer *string filerMountRootPath *string dir *string @@ -31,42 +31,45 @@ type Mount2Options struct { } var ( - mount2Options Mount2Options + mountOptions MountOptions + mountCpuProfile *string + mountMemProfile *string + mountReadRetryTime *time.Duration ) func init() { - cmdMount2.Run = runMount2 // break init cycle - mount2Options.filer = cmdMount2.Flag.String("filer", "localhost:8888", "comma-separated weed filer location") - mount2Options.filerMountRootPath = cmdMount2.Flag.String("filer.path", "/", "mount this remote path from filer server") - mount2Options.dir = cmdMount2.Flag.String("dir", ".", "mount weed filer to this directory") - mount2Options.dirAutoCreate = cmdMount2.Flag.Bool("dirAutoCreate", false, "auto create the directory to mount to") - mount2Options.collection = cmdMount2.Flag.String("collection", "", "collection to create the files") - mount2Options.replication = cmdMount2.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.") - mount2Options.diskType = cmdMount2.Flag.String("disk", "", "[hdd|ssd|] hard drive or solid state drive or any tag") - mount2Options.ttlSec = cmdMount2.Flag.Int("ttl", 0, "file ttl in seconds") - mount2Options.chunkSizeLimitMB = cmdMount2.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files") - mount2Options.concurrentWriters = cmdMount2.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0") - mount2Options.cacheDir = cmdMount2.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data") - mount2Options.cacheSizeMB = cmdMount2.Flag.Int64("cacheCapacityMB", 0, "local file chunk cache capacity in MB") - mount2Options.dataCenter = cmdMount2.Flag.String("dataCenter", "", "prefer to write to the data center") - mount2Options.allowOthers = cmdMount2.Flag.Bool("allowOthers", true, "allows other users to access the file system") - mount2Options.umaskString = cmdMount2.Flag.String("umask", "022", "octal umask, e.g., 022, 0111") - mount2Options.nonempty = cmdMount2.Flag.Bool("nonempty", false, "allows the mounting over a non-empty directory") - mount2Options.volumeServerAccess = cmdMount2.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]") - mount2Options.uidMap = cmdMount2.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated :") - mount2Options.gidMap = cmdMount2.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated :") - mount2Options.readOnly = cmdMount2.Flag.Bool("readOnly", false, "read only") - mount2Options.debug = cmdMount2.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:/debug/pprof/goroutine?debug=2") - mount2Options.debugPort = cmdMount2.Flag.Int("debug.port", 6061, "http port for debugging") + cmdMount.Run = runMount // break init cycle + mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "comma-separated weed filer location") + mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server") + mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory") + mountOptions.dirAutoCreate = cmdMount.Flag.Bool("dirAutoCreate", false, "auto create the directory to mount to") + mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files") + mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.") + mountOptions.diskType = cmdMount.Flag.String("disk", "", "[hdd|ssd|] hard drive or solid state drive or any tag") + mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds") + mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files") + mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0") + mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data") + mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 0, "local file chunk cache capacity in MB") + mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center") + mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system") + mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111") + mountOptions.nonempty = cmdMount.Flag.Bool("nonempty", false, "allows the mounting over a non-empty directory") + mountOptions.volumeServerAccess = cmdMount.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]") + mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated :") + mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated :") + mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only") + mountOptions.debug = cmdMount.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:/debug/pprof/goroutine?debug=2") + mountOptions.debugPort = cmdMount.Flag.Int("debug.port", 6061, "http port for debugging") - mountCpuProfile = cmdMount2.Flag.String("cpuprofile", "", "cpu profile output file") - mountMemProfile = cmdMount2.Flag.String("memprofile", "", "memory profile output file") - mountReadRetryTime = cmdMount2.Flag.Duration("readRetryTime", 6*time.Second, "maximum read retry wait time") + mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file") + mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file") + mountReadRetryTime = cmdMount.Flag.Duration("readRetryTime", 6*time.Second, "maximum read retry wait time") } -var cmdMount2 = &Command{ - UsageLine: "mount2 -filer=localhost:8888 -dir=/some/dir", - Short: " mount weed filer to a directory as file system in userspace(FUSE)", +var cmdMount = &Command{ + UsageLine: "mount -filer=localhost:8888 -dir=/some/dir", + Short: "mount weed filer to a directory as file system in userspace(FUSE)", Long: `mount weed filer to userspace. Pre-requisites: diff --git a/weed/command/mount2_std.go b/weed/command/mount2_std.go index 61555d8f2..e53f9831e 100644 --- a/weed/command/mount2_std.go +++ b/weed/command/mount2_std.go @@ -27,10 +27,10 @@ import ( "github.com/chrislusf/seaweedfs/weed/util/grace" ) -func runMount2(cmd *Command, args []string) bool { +func runMount(cmd *Command, args []string) bool { - if *mount2Options.debug { - go http.ListenAndServe(fmt.Sprintf(":%d", *mount2Options.debugPort), nil) + if *mountOptions.debug { + go http.ListenAndServe(fmt.Sprintf(":%d", *mountOptions.debugPort), nil) } grace.SetupProfiling(*mountCpuProfile, *mountMemProfile) @@ -39,9 +39,9 @@ func runMount2(cmd *Command, args []string) bool { } util.RetryWaitTime = *mountReadRetryTime - umask, umaskErr := strconv.ParseUint(*mount2Options.umaskString, 8, 64) + umask, umaskErr := strconv.ParseUint(*mountOptions.umaskString, 8, 64) if umaskErr != nil { - fmt.Printf("can not parse umask %s", *mount2Options.umaskString) + fmt.Printf("can not parse umask %s", *mountOptions.umaskString) return false } @@ -49,13 +49,13 @@ func runMount2(cmd *Command, args []string) bool { return false } - return RunMount2(&mount2Options, os.FileMode(umask)) + return RunMount2(&mountOptions, os.FileMode(umask)) } -func RunMount2(option *Mount2Options, umask os.FileMode) bool { +func RunMount2(option *MountOptions, umask os.FileMode) bool { // basic checks - chunkSizeLimitMB := *mount2Options.chunkSizeLimitMB + chunkSizeLimitMB := *mountOptions.chunkSizeLimitMB if chunkSizeLimitMB <= 0 { fmt.Printf("Please specify a reasonable buffer size.") return false @@ -213,7 +213,7 @@ func RunMount2(option *Mount2Options, umask os.FileMode) bool { MountCtime: fileInfo.ModTime(), MountMtime: time.Now(), Umask: umask, - VolumeServerAccess: *mount2Options.volumeServerAccess, + VolumeServerAccess: *mountOptions.volumeServerAccess, Cipher: cipher, UidGidMapper: uidGidMapper, }) diff --git a/weed/command/mount_freebsd.go b/weed/command/mount_freebsd.go deleted file mode 100644 index f0a5581e7..000000000 --- a/weed/command/mount_freebsd.go +++ /dev/null @@ -1,13 +0,0 @@ -package command - -import ( - "github.com/seaweedfs/fuse" -) - -func osSpecificMountOptions() []fuse.MountOption { - return []fuse.MountOption{} -} - -func checkMountPointAvailable(dir string) bool { - return true -} diff --git a/weed/command/mount_notsupported.go b/weed/command/mount_notsupported.go deleted file mode 100644 index 1e5c9f53d..000000000 --- a/weed/command/mount_notsupported.go +++ /dev/null @@ -1,15 +0,0 @@ -//go:build !linux && !darwin && !freebsd -// +build !linux,!darwin,!freebsd - -package command - -import ( - "fmt" - "runtime" -) - -func runMount(cmd *Command, args []string) bool { - fmt.Printf("Mount is not supported on %s %s\n", runtime.GOOS, runtime.GOARCH) - - return true -} diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go deleted file mode 100644 index 8f62b4ec9..000000000 --- a/weed/command/mount_std.go +++ /dev/null @@ -1,261 +0,0 @@ -//go:build linux || darwin || freebsd -// +build linux darwin freebsd - -package command - -import ( - "context" - "fmt" - "net/http" - "os" - "os/user" - "path" - "path/filepath" - "runtime" - "strconv" - "strings" - "syscall" - "time" - - "github.com/chrislusf/seaweedfs/weed/storage/types" - - "github.com/chrislusf/seaweedfs/weed/filesys/meta_cache" - - "github.com/seaweedfs/fuse" - "github.com/seaweedfs/fuse/fs" - - "github.com/chrislusf/seaweedfs/weed/filesys" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/security" - "github.com/chrislusf/seaweedfs/weed/util" - "github.com/chrislusf/seaweedfs/weed/util/grace" -) - -func runMount(cmd *Command, args []string) bool { - - if *mountOptions.debug { - go http.ListenAndServe(fmt.Sprintf(":%d", *mountOptions.debugPort), nil) - } - - grace.SetupProfiling(*mountCpuProfile, *mountMemProfile) - if *mountReadRetryTime < time.Second { - *mountReadRetryTime = time.Second - } - util.RetryWaitTime = *mountReadRetryTime - - umask, umaskErr := strconv.ParseUint(*mountOptions.umaskString, 8, 64) - if umaskErr != nil { - fmt.Printf("can not parse umask %s", *mountOptions.umaskString) - return false - } - - if len(args) > 0 { - return false - } - - return RunMount(&mountOptions, os.FileMode(umask)) -} - -func getParentInode(mountDir string) (uint64, error) { - parentDir := filepath.Clean(filepath.Join(mountDir, "..")) - fi, err := os.Stat(parentDir) - if err != nil { - return 0, err - } - - stat, ok := fi.Sys().(*syscall.Stat_t) - if !ok { - return 0, nil - } - - return stat.Ino, nil -} - -func RunMount(option *MountOptions, umask os.FileMode) bool { - - filerAddresses := pb.ServerAddresses(*option.filer).ToAddresses() - - util.LoadConfiguration("security", false) - // try to connect to filer, filerBucketsPath may be useful later - grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") - var cipher bool - var err error - for i := 0; i < 10; i++ { - err = pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { - resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) - if err != nil { - return fmt.Errorf("get filer grpc address %v configuration: %v", filerAddresses, err) - } - cipher = resp.Cipher - return nil - }) - if err != nil { - glog.V(0).Infof("failed to talk to filer %v: %v", filerAddresses, err) - glog.V(0).Infof("wait for %d seconds ...", i+1) - time.Sleep(time.Duration(i+1) * time.Second) - } - } - if err != nil { - glog.Errorf("failed to talk to filer %v: %v", filerAddresses, err) - return true - } - - filerMountRootPath := *option.filerMountRootPath - dir := util.ResolvePath(*option.dir) - parentInode, err := getParentInode(dir) - if err != nil { - glog.Errorf("failed to retrieve inode for parent directory of %s: %v", dir, err) - return true - } - - fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH) - if dir == "" { - fmt.Printf("Please specify the mount directory via \"-dir\"") - return false - } - - chunkSizeLimitMB := *mountOptions.chunkSizeLimitMB - if chunkSizeLimitMB <= 0 { - fmt.Printf("Please specify a reasonable buffer size.") - return false - } - - fuse.Unmount(dir) - - // detect mount folder mode - if *option.dirAutoCreate { - os.MkdirAll(dir, os.FileMode(0777)&^umask) - } - fileInfo, err := os.Stat(dir) - - uid, gid := uint32(0), uint32(0) - mountMode := os.ModeDir | 0777 - if err == nil { - mountMode = os.ModeDir | os.FileMode(0777)&^umask - uid, gid = util.GetFileUidGid(fileInfo) - fmt.Printf("mount point owner uid=%d gid=%d mode=%s\n", uid, gid, mountMode) - } else { - fmt.Printf("can not stat %s\n", dir) - return false - } - - if uid == 0 { - if u, err := user.Current(); err == nil { - if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil { - uid = uint32(parsedId) - } - if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil { - gid = uint32(parsedId) - } - fmt.Printf("current uid=%d gid=%d\n", uid, gid) - } - } - - // mapping uid, gid - uidGidMapper, err := meta_cache.NewUidGidMapper(*option.uidMap, *option.gidMap) - if err != nil { - fmt.Printf("failed to parse %s %s: %v\n", *option.uidMap, *option.gidMap, err) - return false - } - - // Ensure target mount point availability - if isValid := checkMountPointAvailable(dir); !isValid { - glog.Fatalf("Expected mount to still be active, target mount point: %s, please check!", dir) - return true - } - - mountName := path.Base(dir) - - options := []fuse.MountOption{ - fuse.VolumeName(mountName), - fuse.FSName(*option.filer + ":" + filerMountRootPath), - fuse.Subtype("seaweedfs"), - // fuse.NoAppleDouble(), // include .DS_Store, otherwise can not delete non-empty folders - fuse.NoAppleXattr(), - fuse.ExclCreate(), - fuse.DaemonTimeout("3600"), - fuse.AllowDev(), - fuse.AllowSUID(), - fuse.DefaultPermissions(), - fuse.MaxReadahead(1024 * 512), - fuse.AsyncRead(), - // fuse.WritebackCache(), - // fuse.MaxBackground(1024), - // fuse.CongestionThreshold(1024), - } - - options = append(options, osSpecificMountOptions()...) - if *option.allowOthers { - options = append(options, fuse.AllowOther()) - } - if *option.nonempty { - options = append(options, fuse.AllowNonEmptyMount()) - } - if *option.readOnly { - options = append(options, fuse.ReadOnly()) - } - - // find mount point - mountRoot := filerMountRootPath - if mountRoot != "/" && strings.HasSuffix(mountRoot, "/") { - mountRoot = mountRoot[0 : len(mountRoot)-1] - } - - diskType := types.ToDiskType(*option.diskType) - - seaweedFileSystem := filesys.NewSeaweedFileSystem(&filesys.Option{ - MountDirectory: dir, - FilerAddresses: filerAddresses, - GrpcDialOption: grpcDialOption, - FilerMountRootPath: mountRoot, - Collection: *option.collection, - Replication: *option.replication, - TtlSec: int32(*option.ttlSec), - DiskType: diskType, - ChunkSizeLimit: int64(chunkSizeLimitMB) * 1024 * 1024, - ConcurrentWriters: *option.concurrentWriters, - CacheDir: *option.cacheDir, - CacheSizeMB: *option.cacheSizeMB, - DataCenter: *option.dataCenter, - MountUid: uid, - MountGid: gid, - MountMode: mountMode, - MountCtime: fileInfo.ModTime(), - MountMtime: time.Now(), - MountParentInode: parentInode, - Umask: umask, - VolumeServerAccess: *mountOptions.volumeServerAccess, - Cipher: cipher, - UidGidMapper: uidGidMapper, - }) - - // mount - c, err := fuse.Mount(dir, options...) - if err != nil { - glog.V(0).Infof("mount: %v", err) - return true - } - defer fuse.Unmount(dir) - - grace.OnInterrupt(func() { - fuse.Unmount(dir) - c.Close() - }) - - glog.V(0).Infof("mounted %s%s to %v", *option.filer, mountRoot, dir) - server := fs.New(c, nil) - seaweedFileSystem.Server = server - seaweedFileSystem.StartBackgroundTasks() - err = server.Serve(seaweedFileSystem) - - // check if the mount process has an error to report - <-c.Ready - if err := c.MountError; err != nil { - glog.V(0).Infof("mount process: %v", err) - return true - } - - return true -}