From 7198c3b776837e47bef68c2f72f2da68372732c8 Mon Sep 17 00:00:00 2001 From: Fan Date: Thu, 26 Oct 2023 17:43:47 +0000 Subject: [PATCH] Improve the performance of prefix list after fix #4668 --- weed/filer/filerstore_wrapper.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/weed/filer/filerstore_wrapper.go b/weed/filer/filerstore_wrapper.go index cc3e58363..600992a32 100644 --- a/weed/filer/filerstore_wrapper.go +++ b/weed/filer/filerstore_wrapper.go @@ -281,8 +281,16 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u return actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, eachEntryFunc) } + // after remove compare in pr #4924 , the loop will fetch some objects based on the limit parameter + // When the limit parameter is set too small and the number of objects is too large, it can cause performance issues. + // so it will at least fetch 500 objects a time if limit < 500 + fetchLimit := int64(500) + if limit > fetchLimit { + fetchLimit = limit + } + var notPrefixed []*Entry - lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *Entry) bool { + lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, fetchLimit, func(entry *Entry) bool { notPrefixed = append(notPrefixed, entry) return true }) @@ -305,7 +313,7 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u } if count < limit && lastFileName <= prefix { notPrefixed = notPrefixed[:0] - lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool { + lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, fetchLimit, func(entry *Entry) bool { notPrefixed = append(notPrefixed, entry) return true })