add RetryForever

weed/filesys/meta_cache/meta_cache_subscribe.go

@@ -59,8 +59,11 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
 
 	}
 
-	return util.Retry("followMetaUpdates", func() error {
+	util.RetryForever("followMetaUpdates", func() error {
 		return pb.WithFilerClientFollowMetadata(client, "mount", dir, lastTsNs, selfSignature, processEventFn, true)
+	}, func(err error) bool {
+		glog.Errorf("follow metadata updates: %v", err)
+		return true
 	})
 
 }

weed/util/retry.go

@@ -32,6 +32,26 @@ func Retry(name string, job func() error) (err error) {
 	return err
 }
 
+func RetryForever(name string, job func() error, onErrFn func(err error) bool) {
+	waitTime := time.Second
+	for {
+		err = job()
+		if err == nil {
+			break
+		}
+		if onErrFn(err) {
+			if strings.Contains(err.Error(), "transport") {
+				glog.V(0).Infof("retry %s: err: %v", name, err)
+				time.Sleep(waitTime)
+				if waitTime < RetryWaitTime {
+					waitTime += waitTime / 2
+				}
+			}
+			continue
+		}
+	}
+}
+
 // return the first non empty string
 func Nvl(values ...string) string {
 	for _, s := range values {