Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

change limit to int64 in case of overflow

pull/1748/head
Chris Lu 4 years ago
parent
19e52fd414
commit
f002e668de
29 changed files with 90 additions and 90 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      weed/filer/abstract_sql/abstract_sql_store.go
  2. 6
      weed/filer/cassandra/cassandra_store.go
  3. 8
      weed/filer/elastic/v7/elastic_store.go
  4. 4
      weed/filer/etcd/etcd_store.go
  5. 7
      weed/filer/filer.go
  6. 4
      weed/filer/filer_buckets.go
  7. 2
      weed/filer/filer_delete_entry.go
  8. 4
      weed/filer/filer_notify.go
  9. 24
      weed/filer/filer_search.go
  10. 4
      weed/filer/filerstore.go
  11. 4
      weed/filer/filerstore_translate_path.go
  12. 8
      weed/filer/filerstore_wrapper.go
  13. 4
      weed/filer/hbase/hbase_store.go
  14. 12
      weed/filer/leveldb/leveldb_store.go
  15. 6
      weed/filer/leveldb/leveldb_store_test.go
  16. 12
      weed/filer/leveldb2/leveldb2_store.go
  17. 6
      weed/filer/leveldb2/leveldb2_store_test.go
  18. 4
      weed/filer/leveldb3/leveldb3_store.go
  19. 6
      weed/filer/leveldb3/leveldb3_store_test.go
  20. 6
      weed/filer/mongodb/mongodb_store.go
  21. 12
      weed/filer/redis/universal_redis_store.go
  22. 6
      weed/filer/redis2/universal_redis_store.go
  23. 4
      weed/filer/rocksdb/rocksdb_store.go
  24. 6
      weed/filer/rocksdb/rocksdb_store_test.go
  25. 2
      weed/filesys/dir.go
  26. 2
      weed/filesys/meta_cache/meta_cache.go
  27. 4
      weed/server/filer_grpc_server.go
  28. 4
      weed/server/filer_grpc_server_rename.go
  29. 3
      weed/server/filer_server_handlers_read_dir.go

6
weed/filer/abstract_sql/abstract_sql_store.go
View File

@ -172,7 +172,7 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat
return nil return nil
} }
func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
sqlText := store.SqlListExclusive sqlText := store.SqlListExclusive
if includeStartFile { if includeStartFile {
sqlText = store.SqlListInclusive sqlText = store.SqlListInclusive
@ -203,7 +203,7 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context,
entries = append(entries, entry) entries = append(entries, entry)
} }
hasMore = len(entries) == limit + 1
hasMore = int64(len(entries)) == limit + 1
if hasMore { if hasMore {
entries = entries[:limit] entries = entries[:limit]
} }
@ -211,7 +211,7 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context,
return entries, hasMore, nil return entries, hasMore, nil
} }
func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) {
func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) {
return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "")
} }

6
weed/filer/cassandra/cassandra_store.go
View File

@ -168,11 +168,11 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath
return nil return nil
} }
func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
return nil, false, filer.ErrUnsupportedListDirectoryPrefixed return nil, false, filer.ErrUnsupportedListDirectoryPrefixed
} }
func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) {
func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) {
if _, ok := store.isSuperLargeDirectory(string(dirPath)); ok { if _, ok := store.isSuperLargeDirectory(string(dirPath)); ok {
return // nil, filer.ErrUnsupportedSuperLargeDirectoryListing return // nil, filer.ErrUnsupportedSuperLargeDirectoryListing
@ -201,7 +201,7 @@ func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, dirPath u
glog.V(0).Infof("list iterator close: %v", err) glog.V(0).Infof("list iterator close: %v", err)
} }
hasMore = len(entries) == limit + 1
hasMore = int64(len(entries)) == limit + 1
if hasMore { if hasMore {
entries = entries[:limit] entries = entries[:limit]
} }

8
weed/filer/elastic/v7/elastic_store.go
View File

@ -96,7 +96,7 @@ func (store *ElasticStore) RollbackTransaction(ctx context.Context) error {
return nil return nil
} }
func (store *ElasticStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
func (store *ElasticStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
return nil, false, filer.ErrUnsupportedListDirectoryPrefixed return nil, false, filer.ErrUnsupportedListDirectoryPrefixed
} }
@ -195,14 +195,14 @@ func (store *ElasticStore) DeleteFolderChildren(ctx context.Context, fullpath we
return nil return nil
} }
func (store *ElasticStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) {
func (store *ElasticStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) {
if string(dirPath) == "/" { if string(dirPath) == "/" {
return store.listRootDirectoryEntries(ctx, startFileName, includeStartFile, limit) return store.listRootDirectoryEntries(ctx, startFileName, includeStartFile, limit)
} }
return store.listDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) return store.listDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit)
} }
func (store *ElasticStore) listRootDirectoryEntries(ctx context.Context, startFileName string, inclusive bool, limit int) (entries []*filer.Entry, hasMore bool, err error) {
func (store *ElasticStore) listRootDirectoryEntries(ctx context.Context, startFileName string, inclusive bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) {
indexResult, err := store.client.CatIndices().Do(ctx) indexResult, err := store.client.CatIndices().Do(ctx)
if err != nil { if err != nil {
glog.Errorf("list indices %v.", err) glog.Errorf("list indices %v.", err)
@ -232,7 +232,7 @@ func (store *ElasticStore) listRootDirectoryEntries(ctx context.Context, startFi
} }
func (store *ElasticStore) listDirectoryEntries( func (store *ElasticStore) listDirectoryEntries(
ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int,
ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int64,
) (entries []*filer.Entry, hasMore bool, err error) { ) (entries []*filer.Entry, hasMore bool, err error) {
first := true first := true
index := getIndex(fullpath) index := getIndex(fullpath)

4
weed/filer/etcd/etcd_store.go
View File

@ -139,11 +139,11 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_
return nil return nil
} }
func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
return nil, false, filer.ErrUnsupportedListDirectoryPrefixed return nil, false, filer.ErrUnsupportedListDirectoryPrefixed
} }
func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) {
func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) {
directoryPrefix := genDirectoryKeyPrefix(dirPath, "") directoryPrefix := genDirectoryKeyPrefix(dirPath, "")
resp, err := store.client.Get(ctx, string(directoryPrefix), resp, err := store.client.Get(ctx, string(directoryPrefix),

7
weed/filer/filer.go
View File

@ -281,10 +281,11 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e
} }
func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*Entry, expiredCount int, lastFileName string, err error) {
listedEntries, _, listErr := f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix)
func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string) (entries []*Entry, hasMore bool, expiredCount int64, lastFileName string, err error) {
listedEntries, listHasMore, listErr := f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix)
hasMore = listHasMore
if listErr != nil { if listErr != nil {
return listedEntries, expiredCount, "", listErr
return listedEntries, hasMore, expiredCount, "", listErr
} }
for _, entry := range listedEntries { for _, entry := range listedEntries {
lastFileName = entry.Name() lastFileName = entry.Name()

4
weed/filer/filer_buckets.go
View File

@ -27,9 +27,9 @@ func (f *Filer) LoadBuckets() {
buckets: make(map[BucketName]*BucketOption), buckets: make(map[BucketName]*BucketOption),
} }
limit := math.MaxInt32
limit := int64(math.MaxInt32)
entries, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit, "", "")
entries, _, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit, "", "")
if err != nil { if err != nil {
glog.V(1).Infof("no buckets found: %v", err) glog.V(1).Infof("no buckets found: %v", err)

2
weed/filer/filer_delete_entry.go
View File

@ -69,7 +69,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
includeLastFile := false includeLastFile := false
if !isDeletingBucket { if !isDeletingBucket {
for { for {
entries, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "")
entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "")
if err != nil { if err != nil {
glog.Errorf("list folder %s: %v", entry.FullPath, err) glog.Errorf("list folder %s: %v", entry.FullPath, err)
return nil, nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err) return nil, nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err)

4
weed/filer/filer_notify.go
View File

@ -113,13 +113,13 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func(
sizeBuf := make([]byte, 4) sizeBuf := make([]byte, 4)
startTsNs := startTime.UnixNano() startTsNs := startTime.UnixNano()
dayEntries, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "", "")
dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "", "")
if listDayErr != nil { if listDayErr != nil {
return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr) return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr)
} }
for _, dayEntry := range dayEntries { for _, dayEntry := range dayEntries {
// println("checking day", dayEntry.FullPath) // println("checking day", dayEntry.FullPath)
hourMinuteEntries, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "", "")
hourMinuteEntries, _, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "", "")
if listHourMinuteErr != nil { if listHourMinuteErr != nil {
return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr) return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr)
} }

24
weed/filer/filer_search.go
View File

@ -20,7 +20,7 @@ func splitPattern(pattern string) (prefix string, restPattern string) {
} }
// For now, prefix and namePattern are mutually exclusive // For now, prefix and namePattern are mutually exclusive
func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string, namePattern string) (entries []*Entry, err error) {
func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string) (entries []*Entry, hasMore bool, err error) {
if strings.HasSuffix(string(p), "/") && len(p) > 1 { if strings.HasSuffix(string(p), "/") && len(p) > 1 {
p = p[0 : len(p)-1] p = p[0 : len(p)-1]
} }
@ -29,31 +29,31 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start
if prefixInNamePattern != "" { if prefixInNamePattern != "" {
prefix = prefixInNamePattern prefix = prefixInNamePattern
} }
var missedCount int
var missedCount int64
var lastFileName string var lastFileName string
entries, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern)
entries, hasMore, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern)
for missedCount > 0 && err == nil { for missedCount > 0 && err == nil {
var makeupEntries []*Entry var makeupEntries []*Entry
makeupEntries, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern)
makeupEntries, hasMore, missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern)
for _, entry := range makeupEntries { for _, entry := range makeupEntries {
entries = append(entries, entry) entries = append(entries, entry)
} }
} }
return entries, err
return entries, hasMore, err
} }
func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix, restNamePattern string) (matchedEntries []*Entry, missedCount int, lastFileName string, err error) {
func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix, restNamePattern string) (matchedEntries []*Entry, hasMore bool, missedCount int64, lastFileName string, err error) {
var foundEntries []*Entry var foundEntries []*Entry
foundEntries, lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix)
foundEntries, hasMore, lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix)
if err != nil { if err != nil {
return return
} }
if len(restNamePattern) == 0 { if len(restNamePattern) == 0 {
return foundEntries, 0, lastFileName, nil
return foundEntries, false,0, lastFileName, nil
} }
for _, entry := range foundEntries { for _, entry := range foundEntries {
nameToTest := strings.ToLower(entry.Name()) nameToTest := strings.ToLower(entry.Name())
@ -66,12 +66,12 @@ func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath
return return
} }
func (f *Filer) doListValidEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*Entry, lastFileName string, err error) {
func (f *Filer) doListValidEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string) (entries []*Entry, hasMore bool, lastFileName string, err error) {
var makeupEntries []*Entry var makeupEntries []*Entry
var expiredCount int
entries, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix)
var expiredCount int64
entries, hasMore, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix)
for expiredCount > 0 && err == nil { for expiredCount > 0 && err == nil {
makeupEntries, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount, prefix)
makeupEntries, hasMore, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount, prefix)
if err == nil { if err == nil {
entries = append(entries, makeupEntries...) entries = append(entries, makeupEntries...)
} }

4
weed/filer/filerstore.go
View File

@ -24,8 +24,8 @@ type FilerStore interface {
FindEntry(context.Context, util.FullPath) (entry *Entry, err error) FindEntry(context.Context, util.FullPath) (entry *Entry, err error)
DeleteEntry(context.Context, util.FullPath) (err error) DeleteEntry(context.Context, util.FullPath) (err error)
DeleteFolderChildren(context.Context, util.FullPath) (err error) DeleteFolderChildren(context.Context, util.FullPath) (err error)
ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, bool, error)
ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, bool, error)
ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) ([]*Entry, bool, error)
ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) ([]*Entry, bool, error)
BeginTransaction(ctx context.Context) (context.Context, error) BeginTransaction(ctx context.Context) (context.Context, error)
CommitTransaction(ctx context.Context) error CommitTransaction(ctx context.Context) error

4
weed/filer/filerstore_translate_path.go
View File

@ -106,7 +106,7 @@ func (t *FilerStorePathTranlator) DeleteFolderChildren(ctx context.Context, fp u
return t.actualStore.DeleteFolderChildren(ctx, newFullPath) return t.actualStore.DeleteFolderChildren(ctx, newFullPath)
} }
func (t *FilerStorePathTranlator) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, bool, error) {
func (t *FilerStorePathTranlator) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) ([]*Entry, bool, error) {
newFullPath := t.translatePath(dirPath) newFullPath := t.translatePath(dirPath)
@ -120,7 +120,7 @@ func (t *FilerStorePathTranlator) ListDirectoryEntries(ctx context.Context, dirP
return entries, hasMore, err return entries, hasMore, err
} }
func (t *FilerStorePathTranlator) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, bool, error) {
func (t *FilerStorePathTranlator) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) ([]*Entry, bool, error) {
newFullPath := t.translatePath(dirPath) newFullPath := t.translatePath(dirPath)

8
weed/filer/filerstore_wrapper.go
View File

@ -194,7 +194,7 @@ func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.
return actualStore.DeleteFolderChildren(ctx, fp) return actualStore.DeleteFolderChildren(ctx, fp)
} }
func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, bool, error) {
func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) ([]*Entry, bool, error) {
actualStore := fsw.getActualStore(dirPath + "/") actualStore := fsw.getActualStore(dirPath + "/")
stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "list").Inc() stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "list").Inc()
start := time.Now() start := time.Now()
@ -214,7 +214,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath
return entries, hasMore, err return entries, hasMore, err
} }
func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, bool, error) {
func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) ([]*Entry, bool, error) {
actualStore := fsw.getActualStore(dirPath + "/") actualStore := fsw.getActualStore(dirPath + "/")
stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "prefixList").Inc() stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "prefixList").Inc()
start := time.Now() start := time.Now()
@ -236,7 +236,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context,
return entries, hasMore, nil return entries, hasMore, nil
} }
func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*Entry, hasMore bool, err error) {
func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*Entry, hasMore bool, err error) {
actualStore := fsw.getActualStore(dirPath + "/") actualStore := fsw.getActualStore(dirPath + "/")
entries, hasMore, err = actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) entries, hasMore, err = actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit)
if err != nil { if err != nil {
@ -247,7 +247,7 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u
return return
} }
count := 0
count := int64(0)
var lastFileName string var lastFileName string
notPrefixed := entries notPrefixed := entries
entries = nil entries = nil

4
weed/filer/hbase/hbase_store.go
View File

@ -148,11 +148,11 @@ func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.Ful
return return
} }
func (store *HbaseStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*filer.Entry, bool, error) {
func (store *HbaseStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) ([]*filer.Entry, bool, error) {
return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "")
} }
func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*filer.Entry, bool, error) {
func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) ([]*filer.Entry, bool, error) {
family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}} family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}}
expectedPrefix := []byte(dirPath.Child(prefix)) expectedPrefix := []byte(dirPath.Child(prefix))
scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family)) scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family))

12
weed/filer/leveldb/leveldb_store.go
View File

@ -162,16 +162,16 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we
return nil return nil
} }
func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) {
func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) {
return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "")
} }
func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
directoryPrefix := genDirectoryKeyPrefix(fullpath, prefix)
directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix)
lastFileStart := directoryPrefix lastFileStart := directoryPrefix
if startFileName != "" { if startFileName != "" {
lastFileStart = genDirectoryKeyPrefix(fullpath, startFileName)
lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName)
} }
iter := store.db.NewIterator(&leveldb_util.Range{Start: lastFileStart}, nil) iter := store.db.NewIterator(&leveldb_util.Range{Start: lastFileStart}, nil)
@ -184,7 +184,7 @@ func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful
if fileName == "" { if fileName == "" {
continue continue
} }
if fileName == startFileName && !inclusive {
if fileName == startFileName && !includeStartFile {
continue continue
} }
limit-- limit--
@ -193,7 +193,7 @@ func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful
break break
} }
entry := &filer.Entry{ entry := &filer.Entry{
FullPath: weed_util.NewFullPath(string(fullpath), fileName),
FullPath: weed_util.NewFullPath(string(dirPath), fileName),
} }
if decodeErr := entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(iter.Value())); decodeErr != nil { if decodeErr := entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(iter.Value())); decodeErr != nil {
err = decodeErr err = decodeErr

6
weed/filer/leveldb/leveldb_store_test.go
View File

@ -51,14 +51,14 @@ func TestCreateAndFind(t *testing.T) {
} }
// checking one upper directory // checking one upper directory
entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "")
entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "")
if len(entries) != 1 { if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries)) t.Errorf("list entries count: %v", len(entries))
return return
} }
// checking one upper directory // checking one upper directory
entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
if len(entries) != 1 { if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries)) t.Errorf("list entries count: %v", len(entries))
return return
@ -77,7 +77,7 @@ func TestEmptyRoot(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// checking one upper directory // checking one upper directory
entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
if err != nil { if err != nil {
t.Errorf("list entries: %v", err) t.Errorf("list entries: %v", err)
return return

12
weed/filer/leveldb2/leveldb2_store.go
View File

@ -171,16 +171,16 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w
return nil return nil
} }
func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) {
func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) {
return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "")
} }
func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
directoryPrefix, partitionId := genDirectoryKeyPrefix(fullpath, prefix, store.dbCount)
directoryPrefix, partitionId := genDirectoryKeyPrefix(dirPath, prefix, store.dbCount)
lastFileStart := directoryPrefix lastFileStart := directoryPrefix
if startFileName != "" { if startFileName != "" {
lastFileStart, _ = genDirectoryKeyPrefix(fullpath, startFileName, store.dbCount)
lastFileStart, _ = genDirectoryKeyPrefix(dirPath, startFileName, store.dbCount)
} }
iter := store.dbs[partitionId].NewIterator(&leveldb_util.Range{Start: lastFileStart}, nil) iter := store.dbs[partitionId].NewIterator(&leveldb_util.Range{Start: lastFileStart}, nil)
@ -193,7 +193,7 @@ func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fu
if fileName == "" { if fileName == "" {
continue continue
} }
if fileName == startFileName && !inclusive {
if fileName == startFileName && !includeStartFile {
continue continue
} }
limit-- limit--
@ -202,7 +202,7 @@ func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fu
break break
} }
entry := &filer.Entry{ entry := &filer.Entry{
FullPath: weed_util.NewFullPath(string(fullpath), fileName),
FullPath: weed_util.NewFullPath(string(dirPath), fileName),
} }
// println("list", entry.FullPath, "chunks", len(entry.Chunks)) // println("list", entry.FullPath, "chunks", len(entry.Chunks))

6
weed/filer/leveldb2/leveldb2_store_test.go
View File

@ -49,14 +49,14 @@ func TestCreateAndFind(t *testing.T) {
} }
// checking one upper directory // checking one upper directory
entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "")
entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "")
if len(entries) != 1 { if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries)) t.Errorf("list entries count: %v", len(entries))
return return
} }
// checking one upper directory // checking one upper directory
entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
if len(entries) != 1 { if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries)) t.Errorf("list entries count: %v", len(entries))
return return
@ -75,7 +75,7 @@ func TestEmptyRoot(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// checking one upper directory // checking one upper directory
entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
if err != nil { if err != nil {
t.Errorf("list entries: %v", err) t.Errorf("list entries: %v", err)
return return

4
weed/filer/leveldb3/leveldb3_store.go
View File

@ -286,11 +286,11 @@ func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath w
return nil return nil
} }
func (store *LevelDB3Store) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) {
func (store *LevelDB3Store) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) {
return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "")
} }
func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
db, _, shortPath, err := store.findDB(dirPath, true) db, _, shortPath, err := store.findDB(dirPath, true)
if err != nil { if err != nil {

6
weed/filer/leveldb3/leveldb3_store_test.go
View File

@ -49,14 +49,14 @@ func TestCreateAndFind(t *testing.T) {
} }
// checking one upper directory // checking one upper directory
entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "")
entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "")
if len(entries) != 1 { if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries)) t.Errorf("list entries count: %v", len(entries))
return return
} }
// checking one upper directory // checking one upper directory
entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
if len(entries) != 1 { if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries)) t.Errorf("list entries count: %v", len(entries))
return return
@ -75,7 +75,7 @@ func TestEmptyRoot(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// checking one upper directory // checking one upper directory
entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
if err != nil { if err != nil {
t.Errorf("list entries: %v", err) t.Errorf("list entries: %v", err)
return return

6
weed/filer/mongodb/mongodb_store.go
View File

@ -178,11 +178,11 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut
return nil return nil
} }
func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
return nil, false, filer.ErrUnsupportedListDirectoryPrefixed return nil, false, filer.ErrUnsupportedListDirectoryPrefixed
} }
func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) {
func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) {
var where = bson.M{"directory": string(dirPath), "name": bson.M{"$gt": startFileName}} var where = bson.M{"directory": string(dirPath), "name": bson.M{"$gt": startFileName}}
if includeStartFile { if includeStartFile {
@ -212,7 +212,7 @@ func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath uti
entries = append(entries, entry) entries = append(entries, entry)
} }
hasMore = len(entries) == limit + 1
hasMore = int64(len(entries)) == limit + 1
if hasMore { if hasMore {
entries = entries[:limit] entries = entries[:limit]
} }

12
weed/filer/redis/universal_redis_store.go
View File

@ -125,16 +125,16 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full
return nil return nil
} }
func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
return nil, false, filer.ErrUnsupportedListDirectoryPrefixed return nil, false, filer.ErrUnsupportedListDirectoryPrefixed
} }
func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) {
func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) {
dirListKey := genDirectoryListKey(string(fullpath))
dirListKey := genDirectoryListKey(string(dirPath))
members, err := store.Client.SMembers(ctx, dirListKey).Result() members, err := store.Client.SMembers(ctx, dirListKey).Result()
if err != nil { if err != nil {
return nil, false, fmt.Errorf("list %s : %v", fullpath, err)
return nil, false, fmt.Errorf("list %s : %v", dirPath, err)
} }
// skip // skip
@ -160,14 +160,14 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, full
}) })
// limit // limit
if limit < len(members) {
if limit < int64(len(entries)) {
members = members[:limit] members = members[:limit]
hasMore = true hasMore = true
} }
// fetch entry meta // fetch entry meta
for _, fileName := range members { for _, fileName := range members {
path := util.NewFullPath(string(fullpath), fileName)
path := util.NewFullPath(string(dirPath), fileName)
entry, err := store.FindEntry(ctx, path) entry, err := store.FindEntry(ctx, path)
if err != nil { if err != nil {
glog.V(0).Infof("list %s : %v", path, err) glog.V(0).Infof("list %s : %v", path, err)

6
weed/filer/redis2/universal_redis_store.go
View File

@ -149,11 +149,11 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful
return nil return nil
} }
func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
return nil, false, filer.ErrUnsupportedListDirectoryPrefixed return nil, false, filer.ErrUnsupportedListDirectoryPrefixed
} }
func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) {
func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) {
dirListKey := genDirectoryListKey(string(dirPath)) dirListKey := genDirectoryListKey(string(dirPath))
start := int64(0) start := int64(0)
@ -168,7 +168,7 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dir
return nil, false, fmt.Errorf("list %s : %v", dirPath, err) return nil, false, fmt.Errorf("list %s : %v", dirPath, err)
} }
hasMore = len(members) == limit+1
hasMore = int64(len(entries)) == limit+1
if hasMore { if hasMore {
members = members[:len(members)-1] members = members[:len(members)-1]
} }

4
weed/filer/rocksdb/rocksdb_store.go
View File

@ -221,11 +221,11 @@ func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey
return hasMore, nil return hasMore, nil
} }
func (store *RocksDBStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int) (entries []*filer.Entry, hasMore bool, err error) {
func (store *RocksDBStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64) (entries []*filer.Entry, hasMore bool, err error) {
return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "") return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "")
} }
func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string) (entries []*filer.Entry, hasMore bool, err error) {
directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix) directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix)
lastFileStart := directoryPrefix lastFileStart := directoryPrefix

6
weed/filer/rocksdb/rocksdb_store_test.go
View File

@ -53,14 +53,14 @@ func TestCreateAndFind(t *testing.T) {
} }
// checking one upper directory // checking one upper directory
entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "")
entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "")
if len(entries) != 1 { if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries)) t.Errorf("list entries count: %v", len(entries))
return return
} }
// checking one upper directory // checking one upper directory
entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
if len(entries) != 1 { if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries)) t.Errorf("list entries count: %v", len(entries))
return return
@ -79,7 +79,7 @@ func TestEmptyRoot(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// checking one upper directory // checking one upper directory
entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
if err != nil { if err != nil {
t.Errorf("list entries: %v", err) t.Errorf("list entries: %v", err)
return return

2
weed/filesys/dir.go
View File

@ -319,7 +319,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
glog.Errorf("dir ReadDirAll %s: %v", dirPath, err) glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
return nil, fuse.EIO return nil, fuse.EIO
} }
listedEntries, listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), util.FullPath(dir.FullPath()), "", false, int(math.MaxInt32))
listedEntries, listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), util.FullPath(dir.FullPath()), "", false, int64(math.MaxInt32))
if listErr != nil { if listErr != nil {
glog.Errorf("list meta cache: %v", listErr) glog.Errorf("list meta cache: %v", listErr)
return nil, fuse.EIO return nil, fuse.EIO

2
weed/filesys/meta_cache/meta_cache.go
View File

@ -117,7 +117,7 @@ func (mc *MetaCache) DeleteEntry(ctx context.Context, fp util.FullPath) (err err
return mc.localStore.DeleteEntry(ctx, fp) return mc.localStore.DeleteEntry(ctx, fp)
} }
func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*filer.Entry, error) {
func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64) ([]*filer.Entry, error) {
mc.RLock() mc.RLock()
defer mc.RUnlock() defer mc.RUnlock()

4
weed/server/filer_grpc_server.go
View File

@ -61,7 +61,7 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file
lastFileName := req.StartFromFileName lastFileName := req.StartFromFileName
includeLastFile := req.InclusiveStartFrom includeLastFile := req.InclusiveStartFrom
for limit > 0 { for limit > 0 {
entries, err := fs.filer.ListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, paginationLimit, req.Prefix, "")
entries, hasMore, err := fs.filer.ListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, int64(paginationLimit), req.Prefix, "")
if err != nil { if err != nil {
return err return err
@ -97,7 +97,7 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file
} }
} }
if len(entries) < paginationLimit {
if !hasMore {
break break
} }

4
weed/server/filer_grpc_server_rename.go
View File

@ -75,7 +75,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.
includeLastFile := false includeLastFile := false
for { for {
entries, err := fs.filer.ListDirectoryEntries(ctx, currentDirPath, lastFileName, includeLastFile, 1024, "", "")
entries, hasMore, err := fs.filer.ListDirectoryEntries(ctx, currentDirPath, lastFileName, includeLastFile, 1024, "", "")
if err != nil { if err != nil {
return err return err
} }
@ -90,7 +90,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.
return err return err
} }
} }
if len(entries) < 1024 {
if !hasMore {
break break
} }
} }

3
weed/server/filer_server_handlers_read_dir.go
View File

@ -36,7 +36,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
lastFileName := r.FormValue("lastFileName") lastFileName := r.FormValue("lastFileName")
namePattern := r.FormValue("namePattern") namePattern := r.FormValue("namePattern")
entries, err := fs.filer.ListDirectoryEntries(context.Background(), util.FullPath(path), lastFileName, false, limit, "", namePattern)
entries, shouldDisplayLoadMore, err := fs.filer.ListDirectoryEntries(context.Background(), util.FullPath(path), lastFileName, false, int64(limit), "", namePattern)
if err != nil { if err != nil {
glog.V(0).Infof("listDirectory %s %s %d: %s", path, lastFileName, limit, err) glog.V(0).Infof("listDirectory %s %s %d: %s", path, lastFileName, limit, err)
@ -44,7 +44,6 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
return return
} }
shouldDisplayLoadMore := len(entries) == limit
if path == "/" { if path == "/" {
path = "" path = ""
} }

Write Preview
Loading…
Cancel
Save