diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 43b58a94a..56aee18be 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -37,11 +37,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) } if err == filer_pb.ErrNotFound { glog.V(1).Infof("Not found %s: %v", path, err) - stats.FilerRequestCounter.WithLabelValues("read.notfound").Inc() + stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadNotFound).Inc() w.WriteHeader(http.StatusNotFound) } else { glog.Errorf("Internal %s: %v", path, err) - stats.FilerRequestCounter.WithLabelValues("read.internalerror").Inc() + stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadInternal).Inc() w.WriteHeader(http.StatusInternalServerError) } return @@ -162,7 +162,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) if offset+size <= int64(len(entry.Content)) { _, err := writer.Write(entry.Content[offset : offset+size]) if err != nil { - stats.FilerRequestCounter.WithLabelValues("write.entry.failed").Inc() + stats.FilerRequestCounter.WithLabelValues(stats.ErrorWriteEntry).Inc() glog.Errorf("failed to write entry content: %v", err) } return err @@ -174,7 +174,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) Directory: dir, Name: name, }); err != nil { - stats.FilerRequestCounter.WithLabelValues("read.cache.failed").Inc() + stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadCache).Inc() glog.Errorf("CacheRemoteObjectToLocalCluster %s: %v", entry.FullPath, err) return fmt.Errorf("cache %s: %v", entry.FullPath, err) } else { @@ -184,7 +184,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) err = filer.StreamContent(fs.filer.MasterClient, writer, chunks, offset, size) if err != nil { - stats.FilerRequestCounter.WithLabelValues("read.stream.failed").Inc() + stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadStream).Inc() glog.Errorf("failed to stream content %s: %v", r.URL, err) } return err diff --git a/weed/stats/metrics_names.go b/weed/stats/metrics_names.go new file mode 100644 index 000000000..59ae022aa --- /dev/null +++ b/weed/stats/metrics_names.go @@ -0,0 +1,31 @@ +package stats + +// This file contains metric names for all errors +// The naming convention is ErrorSomeThing = "error.some.thing" +const ( + // volume server + ErrorSizeMismatchOffsetSize = "errorSizeMismatchOffsetSize" + ErrorSizeMismatch = "errorSizeMismatch" + ErrorCRC = "errorCRC" + ErrorIndexOutOfRange = "errorIndexOutOfRange" + + // master topology + ErrorWriteToLocalDisk = "errorWriteToLocalDisk" + ErrorUnmarshalPairs = "errorUnmarshalPairs" + ErrorWriteToReplicas = "errorWriteToReplicas" + + // master client + FailedToKeepConnected = "failedToKeepConnected" + FailedToSend = "failedToSend" + FailedToReceive = "failedToReceive" + RedirectedToleader = "redirectedToleader" + OnPeerUpdate = "onPeerUpdate" + Failed = "failed" + + // filer handler + ErrorReadNotFound = "read.notfound" + ErrorReadInternal = "read.internalerror" + ErrorWriteEntry = "write.entry.failed" + ErrorReadCache = "read.cache.failed" + ErrorReadStream = "read.stream.failed" +) diff --git a/weed/storage/needle/needle_read_write.go b/weed/storage/needle/needle_read_write.go index 7557702e1..9ab2f0b49 100644 --- a/weed/storage/needle/needle_read_write.go +++ b/weed/storage/needle/needle_read_write.go @@ -218,11 +218,11 @@ func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Versio if n.Size != size { // cookie is not always passed in for this API. Use size to do preliminary checking. if OffsetSize == 4 && offset < int64(MaxPossibleVolumeSize) { - stats.VolumeServerRequestCounter.WithLabelValues("errorSizeMismatchOffsetSize").Inc() + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorSizeMismatchOffsetSize).Inc() glog.Errorf("entry not found1: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size) return ErrorSizeMismatch } - stats.VolumeServerRequestCounter.WithLabelValues("errorSizeMismatch").Inc() + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorSizeMismatch).Inc() return fmt.Errorf("entry not found: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size) } switch version { @@ -238,7 +238,7 @@ func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Versio checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize]) newChecksum := NewCRC(n.Data) if checksum != newChecksum.Value() { - stats.VolumeServerRequestCounter.WithLabelValues("errorCRC").Inc() + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorCRC).Inc() return errors.New("CRC error! Data On Disk Corrupted") } n.Checksum = newChecksum @@ -271,7 +271,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) { n.DataSize = util.BytesToUint32(bytes[index : index+4]) index = index + 4 if int(n.DataSize)+index > lenBytes { - stats.VolumeServerRequestCounter.WithLabelValues("errorIndexOutOfRange").Inc() + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() return fmt.Errorf("index out of range %d", 1) } n.Data = bytes[index : index+int(n.DataSize)] @@ -283,7 +283,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) { n.NameSize = uint8(bytes[index]) index = index + 1 if int(n.NameSize)+index > lenBytes { - stats.VolumeServerRequestCounter.WithLabelValues("errorIndexOutOfRange").Inc() + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() return fmt.Errorf("index out of range %d", 2) } n.Name = bytes[index : index+int(n.NameSize)] @@ -293,7 +293,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) { n.MimeSize = uint8(bytes[index]) index = index + 1 if int(n.MimeSize)+index > lenBytes { - stats.VolumeServerRequestCounter.WithLabelValues("errorIndexOutOfRange").Inc() + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() return fmt.Errorf("index out of range %d", 3) } n.Mime = bytes[index : index+int(n.MimeSize)] @@ -301,7 +301,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) { } if index < lenBytes && n.HasLastModifiedDate() { if LastModifiedBytesLength+index > lenBytes { - stats.VolumeServerRequestCounter.WithLabelValues("errorIndexOutOfRange").Inc() + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() return fmt.Errorf("index out of range %d", 4) } n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength]) @@ -309,7 +309,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) { } if index < lenBytes && n.HasTtl() { if TtlBytesLength+index > lenBytes { - stats.VolumeServerRequestCounter.WithLabelValues("errorIndexOutOfRange").Inc() + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() return fmt.Errorf("index out of range %d", 5) } n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength]) @@ -317,13 +317,13 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) { } if index < lenBytes && n.HasPairs() { if 2+index > lenBytes { - stats.VolumeServerRequestCounter.WithLabelValues("errorIndexOutOfRange").Inc() + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() return fmt.Errorf("index out of range %d", 6) } n.PairsSize = util.BytesToUint16(bytes[index : index+2]) index += 2 if int(n.PairsSize)+index > lenBytes { - stats.VolumeServerRequestCounter.WithLabelValues("errorIndexOutOfRange").Inc() + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() return fmt.Errorf("index out of range %d", 7) } end := index + int(n.PairsSize) diff --git a/weed/topology/store_replicate.go b/weed/topology/store_replicate.go index 0efbf9ab5..b0d063ac9 100644 --- a/weed/topology/store_replicate.go +++ b/weed/topology/store_replicate.go @@ -45,7 +45,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt if s.GetVolume(volumeId) != nil { isUnchanged, err = s.WriteVolumeNeedle(volumeId, n, true, fsync) if err != nil { - stats.VolumeServerRequestCounter.WithLabelValues("errorWriteToLocalDisk").Inc() + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToLocalDisk).Inc() err = fmt.Errorf("failed to write to local disk: %v", err) glog.V(0).Infoln(err) return @@ -76,7 +76,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt tmpMap := make(map[string]string) err := json.Unmarshal(n.Pairs, &tmpMap) if err != nil { - stats.VolumeServerRequestCounter.WithLabelValues("errorUnmarshalPairs").Inc() + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorUnmarshalPairs).Inc() glog.V(0).Infoln("Unmarshal pairs error:", err) } for k, v := range tmpMap { @@ -98,7 +98,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt _, err := operation.UploadData(n.Data, uploadOption) return err }); err != nil { - stats.VolumeServerRequestCounter.WithLabelValues("errorWriteToReplicas").Inc() + stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToReplicas).Inc() err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err) glog.V(0).Infoln(err) } diff --git a/weed/wdclient/masterclient.go b/weed/wdclient/masterclient.go index 0d62422dd..5280305e2 100644 --- a/weed/wdclient/masterclient.go +++ b/weed/wdclient/masterclient.go @@ -105,7 +105,7 @@ func (mc *MasterClient) tryConnectToMaster(master pb.ServerAddress) (nextHintedL stream, err := client.KeepConnected(ctx) if err != nil { glog.V(1).Infof("%s masterClient failed to keep connected to %s: %v", mc.clientType, master, err) - stats.MasterClientConnectCounter.WithLabelValues("failedToKeepConnected").Inc() + stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToKeepConnected).Inc() return err } @@ -115,7 +115,7 @@ func (mc *MasterClient) tryConnectToMaster(master pb.ServerAddress) (nextHintedL Version: util.Version(), }); err != nil { glog.V(0).Infof("%s masterClient failed to send to %s: %v", mc.clientType, master, err) - stats.MasterClientConnectCounter.WithLabelValues("failedToSend").Inc() + stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToSend).Inc() return err } @@ -126,7 +126,7 @@ func (mc *MasterClient) tryConnectToMaster(master pb.ServerAddress) (nextHintedL resp, err := stream.Recv() if err != nil { glog.V(0).Infof("%s masterClient failed to receive from %s: %v", mc.clientType, master, err) - stats.MasterClientConnectCounter.WithLabelValues("failedToReceive").Inc() + stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc() return err } @@ -135,7 +135,7 @@ func (mc *MasterClient) tryConnectToMaster(master pb.ServerAddress) (nextHintedL if resp.VolumeLocation.Leader != "" { glog.V(0).Infof("redirected to leader %v", resp.VolumeLocation.Leader) nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader) - stats.MasterClientConnectCounter.WithLabelValues("redirectedToleader").Inc() + stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToleader).Inc() return nil } @@ -164,7 +164,7 @@ func (mc *MasterClient) tryConnectToMaster(master pb.ServerAddress) (nextHintedL } else { glog.V(0).Infof("- %s %s leader:%v\n", update.NodeType, update.Address, update.IsLeader) } - stats.MasterClientConnectCounter.WithLabelValues("onPeerUpdate").Inc() + stats.MasterClientConnectCounter.WithLabelValues(stats.OnPeerUpdate).Inc() mc.OnPeerUpdate(update) } } @@ -173,7 +173,7 @@ func (mc *MasterClient) tryConnectToMaster(master pb.ServerAddress) (nextHintedL }) if gprcErr != nil { - stats.MasterClientConnectCounter.WithLabelValues("failed").Inc() + stats.MasterClientConnectCounter.WithLabelValues(stats.Failed).Inc() glog.V(1).Infof("%s masterClient failed to connect with master %v: %v", mc.clientType, master, gprcErr) } return