filer: optimize for less number of directory lookup

bottom up directory lookup

weed/filer/filer.go

@@ -134,14 +134,57 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFr
 		return nil
 	}
 
+	oldEntry, _ := f.FindEntry(ctx, entry.FullPath)
+
+	/*
+		if !hasWritePermission(lastDirectoryEntry, entry) {
+			glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d",
+				lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid)
+			return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath)
+		}
+	*/
+
+	if oldEntry == nil {
+
 		dirParts := strings.Split(string(entry.FullPath), "/")
+		if err := f.ensureParentDirecotryEntry(ctx, entry, dirParts, len(dirParts)-1, isFromOtherCluster); err != nil {
+			return err
+		}
 
-	// fmt.Printf("directory parts: %+v\n", dirParts)
+		glog.V(4).Infof("InsertEntry %s: new entry: %v", entry.FullPath, entry.Name())
+		if err := f.Store.InsertEntry(ctx, entry); err != nil {
+			glog.Errorf("insert entry %s: %v", entry.FullPath, err)
+			return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
+		}
+	} else {
+		if o_excl {
+			glog.V(3).Infof("EEXIST: entry %s already exists", entry.FullPath)
+			return fmt.Errorf("EEXIST: entry %s already exists", entry.FullPath)
+		}
+		glog.V(4).Infof("UpdateEntry %s: old entry: %v", entry.FullPath, oldEntry.Name())
+		if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
+			glog.Errorf("update entry %s: %v", entry.FullPath, err)
+			return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
+		}
+	}
 
-	var lastDirectoryEntry *Entry
+	f.maybeAddBucket(entry)
+	f.NotifyUpdateEvent(ctx, oldEntry, entry, true, isFromOtherCluster, signatures)
+
+	f.deleteChunksIfNotNew(oldEntry, entry)
 
-	for i := 1; i < len(dirParts); i++ {
-		dirPath := "/" + util.Join(dirParts[:i]...)
+	glog.V(4).Infof("CreateEntry %s: created", entry.FullPath)
+
+	return nil
+}
+
+func (f *Filer) ensureParentDirecotryEntry(ctx context.Context, entry *Entry, dirParts []string, level int, isFromOtherCluster bool) (err error) {
+
+	if level == 0 {
+		return nil
+	}
+
+	dirPath := "/" + util.Join(dirParts[:level]...)
 	// fmt.Printf("%d directory: %+v\n", i, dirPath)
 
 	// check the store directly
@@ -151,6 +194,11 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFr
 	// no such existing directory
 	if dirEntry == nil {
 
+		// ensure parent directory
+		if err = f.ensureParentDirecotryEntry(ctx, entry, dirParts, level-1, isFromOtherCluster); err != nil {
+			return err
+		}
+
 		// create the directory
 		now := time.Now()
 
@@ -186,53 +234,6 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFr
 		return fmt.Errorf("%s is a file", dirPath)
 	}
 
-		// remember the direct parent directory entry
-		if i == len(dirParts)-1 {
-			lastDirectoryEntry = dirEntry
-		}
-
-	}
-
-	if lastDirectoryEntry == nil {
-		glog.Errorf("CreateEntry %s: lastDirectoryEntry is nil", entry.FullPath)
-		return fmt.Errorf("parent folder not found: %v", entry.FullPath)
-	}
-
-	/*
-		if !hasWritePermission(lastDirectoryEntry, entry) {
-			glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d",
-				lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid)
-			return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath)
-		}
-	*/
-
-	oldEntry, _ := f.FindEntry(ctx, entry.FullPath)
-
-	if oldEntry == nil {
-		glog.V(4).Infof("InsertEntry %s: new entry: %v", entry.FullPath, entry.Name())
-		if err := f.Store.InsertEntry(ctx, entry); err != nil {
-			glog.Errorf("insert entry %s: %v", entry.FullPath, err)
-			return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
-		}
-	} else {
-		if o_excl {
-			glog.V(3).Infof("EEXIST: entry %s already exists", entry.FullPath)
-			return fmt.Errorf("EEXIST: entry %s already exists", entry.FullPath)
-		}
-		glog.V(4).Infof("UpdateEntry %s: old entry: %v", entry.FullPath, oldEntry.Name())
-		if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
-			glog.Errorf("update entry %s: %v", entry.FullPath, err)
-			return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
-		}
-	}
-
-	f.maybeAddBucket(entry)
-	f.NotifyUpdateEvent(ctx, oldEntry, entry, true, isFromOtherCluster, signatures)
-
-	f.deleteChunksIfNotNew(oldEntry, entry)
-
-	glog.V(4).Infof("CreateEntry %s: created", entry.FullPath)
-
 	return nil
 }