Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

error metrics for filer and store

pull/2634/head
Konstantin Lebedev 3 years ago
parent
6cf2e7d493
commit
3f4e17aa24
3 changed files with 18 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      weed/server/filer_server_handlers_read.go
  2. 11
      weed/storage/needle/needle_read_write.go
  3. 4
      weed/topology/store_replicate.go

3
weed/server/filer_server_handlers_read.go
View File

@ -162,6 +162,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
if offset+size <= int64(len(entry.Content)) { if offset+size <= int64(len(entry.Content)) {
_, err := writer.Write(entry.Content[offset : offset+size]) _, err := writer.Write(entry.Content[offset : offset+size])
if err != nil { if err != nil {
stats.FilerRequestCounter.WithLabelValues("write.entryfailed").Inc()
glog.Errorf("failed to write entry content: %v", err) glog.Errorf("failed to write entry content: %v", err)
} }
return err return err
@ -173,6 +174,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
Directory: dir, Directory: dir,
Name: name, Name: name,
}); err != nil { }); err != nil {
stats.FilerRequestCounter.WithLabelValues("read.cachefailed").Inc()
glog.Errorf("CacheRemoteObjectToLocalCluster %s: %v", entry.FullPath, err) glog.Errorf("CacheRemoteObjectToLocalCluster %s: %v", entry.FullPath, err)
return fmt.Errorf("cache %s: %v", entry.FullPath, err) return fmt.Errorf("cache %s: %v", entry.FullPath, err)
} else { } else {
@ -182,6 +184,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
err = filer.StreamContent(fs.filer.MasterClient, writer, chunks, offset, size) err = filer.StreamContent(fs.filer.MasterClient, writer, chunks, offset, size)
if err != nil { if err != nil {
stats.FilerRequestCounter.WithLabelValues("stream.contentFailed").Inc()
glog.Errorf("failed to stream content %s: %v", r.URL, err) glog.Errorf("failed to stream content %s: %v", r.URL, err)
} }
return err return err

11
weed/storage/needle/needle_read_write.go
View File

@ -5,6 +5,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/storage/backend" "github.com/chrislusf/seaweedfs/weed/storage/backend"
. "github.com/chrislusf/seaweedfs/weed/storage/types" . "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
@ -217,9 +218,11 @@ func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Versio
if n.Size != size { if n.Size != size {
// cookie is not always passed in for this API. Use size to do preliminary checking. // cookie is not always passed in for this API. Use size to do preliminary checking.
if OffsetSize == 4 && offset < int64(MaxPossibleVolumeSize) { if OffsetSize == 4 && offset < int64(MaxPossibleVolumeSize) {
stats.VolumeServerRequestCounter.WithLabelValues("errorSizeMismatchOffsetSize").Inc()
glog.Errorf("entry not found1: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size) glog.Errorf("entry not found1: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size)
return ErrorSizeMismatch return ErrorSizeMismatch
} }
stats.VolumeServerRequestCounter.WithLabelValues("errorSizeMismatch").Inc()
return fmt.Errorf("entry not found: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size) return fmt.Errorf("entry not found: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size)
} }
switch version { switch version {
@ -235,6 +238,7 @@ func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Versio
checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize]) checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize])
newChecksum := NewCRC(n.Data) newChecksum := NewCRC(n.Data)
if checksum != newChecksum.Value() { if checksum != newChecksum.Value() {
stats.VolumeServerRequestCounter.WithLabelValues("errorCRC").Inc()
return errors.New("CRC error! Data On Disk Corrupted") return errors.New("CRC error! Data On Disk Corrupted")
} }
n.Checksum = newChecksum n.Checksum = newChecksum
@ -267,6 +271,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
n.DataSize = util.BytesToUint32(bytes[index : index+4]) n.DataSize = util.BytesToUint32(bytes[index : index+4])
index = index + 4 index = index + 4
if int(n.DataSize)+index > lenBytes { if int(n.DataSize)+index > lenBytes {
stats.VolumeServerRequestCounter.WithLabelValues("errorIndexOutOfRange").Inc()
return fmt.Errorf("index out of range %d", 1) return fmt.Errorf("index out of range %d", 1)
} }
n.Data = bytes[index : index+int(n.DataSize)] n.Data = bytes[index : index+int(n.DataSize)]
@ -278,6 +283,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
n.NameSize = uint8(bytes[index]) n.NameSize = uint8(bytes[index])
index = index + 1 index = index + 1
if int(n.NameSize)+index > lenBytes { if int(n.NameSize)+index > lenBytes {
stats.VolumeServerRequestCounter.WithLabelValues("errorIndexOutOfRange").Inc()
return fmt.Errorf("index out of range %d", 2) return fmt.Errorf("index out of range %d", 2)
} }
n.Name = bytes[index : index+int(n.NameSize)] n.Name = bytes[index : index+int(n.NameSize)]
@ -287,6 +293,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
n.MimeSize = uint8(bytes[index]) n.MimeSize = uint8(bytes[index])
index = index + 1 index = index + 1
if int(n.MimeSize)+index > lenBytes { if int(n.MimeSize)+index > lenBytes {
stats.VolumeServerRequestCounter.WithLabelValues("errorIndexOutOfRange").Inc()
return fmt.Errorf("index out of range %d", 3) return fmt.Errorf("index out of range %d", 3)
} }
n.Mime = bytes[index : index+int(n.MimeSize)] n.Mime = bytes[index : index+int(n.MimeSize)]
@ -294,6 +301,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
} }
if index < lenBytes && n.HasLastModifiedDate() { if index < lenBytes && n.HasLastModifiedDate() {
if LastModifiedBytesLength+index > lenBytes { if LastModifiedBytesLength+index > lenBytes {
stats.VolumeServerRequestCounter.WithLabelValues("errorIndexOutOfRange").Inc()
return fmt.Errorf("index out of range %d", 4) return fmt.Errorf("index out of range %d", 4)
} }
n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength]) n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
@ -301,6 +309,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
} }
if index < lenBytes && n.HasTtl() { if index < lenBytes && n.HasTtl() {
if TtlBytesLength+index > lenBytes { if TtlBytesLength+index > lenBytes {
stats.VolumeServerRequestCounter.WithLabelValues("errorIndexOutOfRange").Inc()
return fmt.Errorf("index out of range %d", 5) return fmt.Errorf("index out of range %d", 5)
} }
n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength]) n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
@ -308,11 +317,13 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
} }
if index < lenBytes && n.HasPairs() { if index < lenBytes && n.HasPairs() {
if 2+index > lenBytes { if 2+index > lenBytes {
stats.VolumeServerRequestCounter.WithLabelValues("errorIndexOutOfRange").Inc()
return fmt.Errorf("index out of range %d", 6) return fmt.Errorf("index out of range %d", 6)
} }
n.PairsSize = util.BytesToUint16(bytes[index : index+2]) n.PairsSize = util.BytesToUint16(bytes[index : index+2])
index += 2 index += 2
if int(n.PairsSize)+index > lenBytes { if int(n.PairsSize)+index > lenBytes {
stats.VolumeServerRequestCounter.WithLabelValues("errorIndexOutOfRange").Inc()
return fmt.Errorf("index out of range %d", 7) return fmt.Errorf("index out of range %d", 7)
} }
end := index + int(n.PairsSize) end := index + int(n.PairsSize)

4
weed/topology/store_replicate.go
View File

@ -13,6 +13,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/storage/types"
@ -44,6 +45,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt
if s.GetVolume(volumeId) != nil { if s.GetVolume(volumeId) != nil {
isUnchanged, err = s.WriteVolumeNeedle(volumeId, n, true, fsync) isUnchanged, err = s.WriteVolumeNeedle(volumeId, n, true, fsync)
if err != nil { if err != nil {
stats.VolumeServerRequestCounter.WithLabelValues("errorWriteToLocalDisk").Inc()
err = fmt.Errorf("failed to write to local disk: %v", err) err = fmt.Errorf("failed to write to local disk: %v", err)
glog.V(0).Infoln(err) glog.V(0).Infoln(err)
return return
@ -74,6 +76,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt
tmpMap := make(map[string]string) tmpMap := make(map[string]string)
err := json.Unmarshal(n.Pairs, &tmpMap) err := json.Unmarshal(n.Pairs, &tmpMap)
if err != nil { if err != nil {
stats.VolumeServerRequestCounter.WithLabelValues("errorUnmarshalPairs").Inc()
glog.V(0).Infoln("Unmarshal pairs error:", err) glog.V(0).Infoln("Unmarshal pairs error:", err)
} }
for k, v := range tmpMap { for k, v := range tmpMap {
@ -95,6 +98,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt
_, err := operation.UploadData(n.Data, uploadOption) _, err := operation.UploadData(n.Data, uploadOption)
return err return err
}); err != nil { }); err != nil {
stats.VolumeServerRequestCounter.WithLabelValues("errorWriteToReplicas").Inc()
err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err) err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err)
glog.V(0).Infoln(err) glog.V(0).Infoln(err)
} }

Write Preview
Loading…
Cancel
Save