store task cli: update

go/storage/store_task_cli.go

@@ -6,6 +6,7 @@ import (
 	"net/url"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/go/glog"
 	"github.com/chrislusf/seaweedfs/go/util"
 )
 
@@ -44,12 +45,29 @@ func (c *TaskCli) WaitAndQueryResult(timeout time.Duration) error {
 	startTime := time.Now()
 	args := url.Values{}
 	args.Set("task", c.TID)
+	args.Set("timeout", time.Minute.String())
+	tryTimes := 0
 	for time.Since(startTime) < timeout {
 		_, e := util.RemoteApiCall(c.DataNode, "/admin/task/query", args)
-		if e.Error() == ErrTaskNotFinish.Error() {
-			continue
+		if e == nil {
+			//task finished and have no error
+			return nil
 		}
-		return e
+		if util.IsRemoteApiError(e) {
+			if e.Error() == ErrTaskNotFinish.Error() {
+				tryTimes = 0
+				continue
+			}
+			return e
+		} else {
+			tryTimes++
+			if tryTimes >= 10 {
+				return e
+			}
+			glog.V(0).Infof("query task (%s) error %v, wait 1 minute then retry %d times", c.TID, e, tryTimes)
+			time.Sleep(time.Minute)
+		}
+
 	}
 	return ErrTaskTimeout
 }

go/util/http_util.go

@@ -72,6 +72,22 @@ func Post(host, path string, values url.Values) (content []byte, e error) {
 	return
 }
 
+type RApiError struct {
+	E string
+}
+
+func (e *RApiError) Error() string {
+	return e.E
+}
+
+func IsRemoteApiError(e error) bool {
+	switch e.(type) {
+	case *RApiError:
+		return true
+	}
+	return false
+}
+
 func RemoteApiCall(host, path string, values url.Values) (result map[string]interface{}, e error) {
 	jsonBlob, code, e := PostEx(host, path, values)
 	if e != nil {
@@ -82,7 +98,7 @@ func RemoteApiCall(host, path string, values url.Values) (result map[string]inte
 		return nil, e
 	}
 	if err, ok := result["error"]; ok && err.(string) != "" {
-		return nil, errors.New(err.(string))
+		return nil, &RApiError{E: err.(string)}
 	}
 	if code != http.StatusOK {
 		return nil, fmt.Errorf("RemoteApiCall %s/%s return %d", host, path, code)