add dir list limit option

weed/command/filer.go

@@ -30,6 +30,7 @@ type FilerOptions struct {
 	disableDirListing       *bool
 	maxMB                   *int
 	secretKey               *string
+	dirListingLimit         *int
 }
 
 func init() {
@@ -45,6 +46,7 @@ func init() {
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
 	f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
 	f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
+	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 1000, "limit sub dir listing size")
 }
 
 var cmdFiler = &Command{
@@ -86,12 +88,16 @@ func (fo *FilerOptions) start() {
 
 	masters := *f.masters
 
-	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux,
-		strings.Split(masters, ","), *fo.collection,
-		*fo.defaultReplicaPlacement, *fo.redirectOnRead, *fo.disableDirListing,
-		*fo.maxMB,
-		*fo.secretKey,
-	)
+	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
+		Masters:            strings.Split(masters, ","),
+		Collection:         *fo.collection,
+		DefaultReplication: *fo.defaultReplicaPlacement,
+		RedirectOnRead:     *fo.redirectOnRead,
+		DisableDirListing:  *fo.disableDirListing,
+		MaxMB:              *fo.maxMB,
+		SecretKey:          *fo.secretKey,
+		DirListingLimit:    *fo.dirListingLimit,
+	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)
 	}

weed/server/filer_grpc_server.go

@@ -33,7 +33,7 @@ func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.L
 
 func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntriesRequest) (*filer_pb.ListEntriesResponse, error) {
 
-	entries, err := fs.filer.ListDirectoryEntries(filer2.FullPath(req.Directory), "", false, 1000)
+	entries, err := fs.filer.ListDirectoryEntries(filer2.FullPath(req.Directory), "", false, fs.option.DirListingLimit)
 	if err != nil {
 		return nil, err
 	}

weed/server/filer_server.go

@@ -13,36 +13,33 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 )
 
+type FilerOption struct {
+	Masters            []string
+	Collection         string
+	DefaultReplication string
+	RedirectOnRead     bool
+	DisableDirListing  bool
+	MaxMB              int
+	SecretKey          string
+	DirListingLimit    int
+}
+
 type FilerServer struct {
-	masters            []string
-	collection         string
-	defaultReplication string
-	redirectOnRead     bool
-	disableDirListing  bool
-	secret             security.Secret
-	filer              *filer2.Filer
-	maxMB              int
+	option *FilerOption
+	secret security.Secret
+	filer  *filer2.Filer
 }
 
-func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, masters []string, collection string,
-	replication string, redirectOnRead bool, disableDirListing bool,
-	maxMB int,
-	secret string,
-) (fs *FilerServer, err error) {
+func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) (fs *FilerServer, err error) {
 	fs = &FilerServer{
-		masters:            masters,
-		collection:         collection,
-		defaultReplication: replication,
-		redirectOnRead:     redirectOnRead,
-		disableDirListing:  disableDirListing,
-		maxMB:              maxMB,
+		option: option,
 	}
 
-	if len(masters) == 0 {
+	if len(option.Masters) == 0 {
 		glog.Fatal("master list is required!")
 	}
 
-	fs.filer = filer2.NewFiler(masters)
+	fs.filer = filer2.NewFiler(option.Masters)
 
 	go fs.filer.KeepConnectedToMaster()
 

weed/server/filer_server_handlers_read.go

@@ -30,7 +30,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 	}
 
 	if entry.IsDirectory() {
-		if fs.disableDirListing {
+		if fs.option.DisableDirListing {
 			w.WriteHeader(http.StatusMethodNotAllowed)
 			return
 		}
@@ -70,7 +70,7 @@ func (fs *FilerServer) handleSingleChunk(w http.ResponseWriter, r *http.Request,
 		return
 	}
 
-	if fs.redirectOnRead {
+	if fs.option.RedirectOnRead {
 		http.Redirect(w, r, urlString, http.StatusFound)
 		return
 	}

weed/server/filer_server_handlers_write.go

@@ -65,11 +65,11 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 	query := r.URL.Query()
 	replication := query.Get("replication")
 	if replication == "" {
-		replication = fs.defaultReplication
+		replication = fs.option.DefaultReplication
 	}
 	collection := query.Get("collection")
 	if collection == "" {
-		collection = fs.collection
+		collection = fs.option.Collection
 	}
 
 	if autoChunked := fs.autoChunk(w, r, replication, collection); autoChunked {

weed/server/filer_server_handlers_write_autochunk.go

@@ -27,8 +27,8 @@ func (fs *FilerServer) autoChunk(w http.ResponseWriter, r *http.Request, replica
 
 	parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32)
 	maxMB := int32(parsedMaxMB)
-	if maxMB <= 0 && fs.maxMB > 0 {
-		maxMB = int32(fs.maxMB)
+	if maxMB <= 0 && fs.option.MaxMB > 0 {
+		maxMB = int32(fs.option.MaxMB)
 	}
 	if maxMB <= 0 {
 		glog.V(4).Infoln("AutoChunking not enabled")