From 6b1e60582c20a36e5b8e76b47d3fc0510bc99c9c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 9 Oct 2012 20:53:31 -0700 Subject: [PATCH] adding file count and deletion count --- weed-fs/src/pkg/storage/needle_map.go | 10 ++++++++- weed-fs/src/pkg/storage/store.go | 4 ++-- weed-fs/src/pkg/storage/volume.go | 2 +- weed-fs/src/pkg/storage/volume_info.go | 2 ++ weed-fs/src/pkg/topology/node.go | 28 +++++++++++++------------- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/weed-fs/src/pkg/storage/needle_map.go b/weed-fs/src/pkg/storage/needle_map.go index 53a640052..44b74e8c1 100644 --- a/weed-fs/src/pkg/storage/needle_map.go +++ b/weed-fs/src/pkg/storage/needle_map.go @@ -9,7 +9,11 @@ import ( type NeedleMap struct { indexFile *os.File m CompactMap - bytes []byte + + //transient + bytes []byte + deletionCounter int + fileCounter int } func NewNeedleMap(file *os.File) *NeedleMap { @@ -40,8 +44,10 @@ func LoadNeedleMap(file *os.File) *NeedleMap { size := util.BytesToUint32(bytes[i+12 : i+16]) if offset > 0 { nm.m.Set(Key(key), offset, size) + nm.fileCounter++ } else { nm.m.Delete(Key(key)) + nm.deletionCounter++ } } @@ -55,6 +61,7 @@ func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) (int, error) { util.Uint64toBytes(nm.bytes[0:8], key) util.Uint32toBytes(nm.bytes[8:12], offset) util.Uint32toBytes(nm.bytes[12:16], size) + nm.fileCounter++ return nm.indexFile.Write(nm.bytes) } func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) { @@ -67,6 +74,7 @@ func (nm *NeedleMap) Delete(key uint64) { util.Uint32toBytes(nm.bytes[8:12], 0) util.Uint32toBytes(nm.bytes[12:16], 0) nm.indexFile.Write(nm.bytes) + nm.deletionCounter++ } func (nm *NeedleMap) Close() { nm.indexFile.Close() diff --git a/weed-fs/src/pkg/storage/store.go b/weed-fs/src/pkg/storage/store.go index 7ebc028ba..225fc9d92 100644 --- a/weed-fs/src/pkg/storage/store.go +++ b/weed-fs/src/pkg/storage/store.go @@ -89,7 +89,7 @@ func (s *Store) Status() []*VolumeInfo { var stats []*VolumeInfo for k, v := range s.volumes { s := new(VolumeInfo) - s.Id, s.Size, s.RepType = VolumeId(k), v.Size(), v.replicaType + s.Id, s.Size, s.RepType, s.FileCount, s.DeleteCount = VolumeId(k), v.Size(), v.replicaType, v.nm.fileCounter, v.nm.deletionCounter stats = append(stats, s) } return stats @@ -98,7 +98,7 @@ func (s *Store) Join(mserver string) error { stats := new([]*VolumeInfo) for k, v := range s.volumes { s := new(VolumeInfo) - s.Id, s.Size, s.RepType = VolumeId(k), v.Size(), v.replicaType + s.Id, s.Size, s.RepType, s.FileCount, s.DeleteCount = VolumeId(k), v.Size(), v.replicaType, v.nm.fileCounter, v.nm.deletionCounter *stats = append(*stats, s) } bytes, _ := json.Marshal(stats) diff --git a/weed-fs/src/pkg/storage/volume.go b/weed-fs/src/pkg/storage/volume.go index 0e6095089..a9713c36b 100644 --- a/weed-fs/src/pkg/storage/volume.go +++ b/weed-fs/src/pkg/storage/volume.go @@ -21,7 +21,7 @@ type Volume struct { replicaType ReplicationType accessLock sync.Mutex - + } func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume) { diff --git a/weed-fs/src/pkg/storage/volume_info.go b/weed-fs/src/pkg/storage/volume_info.go index 3f96db21b..b8eb62f0a 100644 --- a/weed-fs/src/pkg/storage/volume_info.go +++ b/weed-fs/src/pkg/storage/volume_info.go @@ -8,6 +8,8 @@ type VolumeInfo struct { Id VolumeId Size int64 RepType ReplicationType + FileCount int + DeleteCount int } type ReplicationType string diff --git a/weed-fs/src/pkg/topology/node.go b/weed-fs/src/pkg/topology/node.go index 1a33faa12..c3184b3da 100644 --- a/weed-fs/src/pkg/topology/node.go +++ b/weed-fs/src/pkg/topology/node.go @@ -14,6 +14,7 @@ type Node interface { UpAdjustMaxVolumeCountDelta(maxVolumeCountDelta int) UpAdjustActiveVolumeCountDelta(activeVolumeCountDelta int) UpAdjustMaxVolumeId(vid storage.VolumeId) + GetActiveVolumeCount() int GetMaxVolumeCount() int GetMaxVolumeId() storage.VolumeId @@ -25,8 +26,8 @@ type Node interface { IsDataNode() bool Children() map[NodeId]Node Parent() Node - - GetValue()interface{} //get reference to the topology,dc,rack,datanode + + GetValue() interface{} //get reference to the topology,dc,rack,datanode } type NodeImpl struct { id NodeId @@ -38,7 +39,7 @@ type NodeImpl struct { //for rack, data center, topology nodeType string - value interface{} + value interface{} } func (n *NodeImpl) IsDataNode() bool { @@ -71,8 +72,8 @@ func (n *NodeImpl) Children() map[NodeId]Node { func (n *NodeImpl) Parent() Node { return n.parent } -func (n *NodeImpl) GetValue()interface{}{ - return n.value +func (n *NodeImpl) GetValue() interface{} { + return n.value } func (n *NodeImpl) ReserveOneVolume(r int, vid storage.VolumeId) (bool, *DataNode) { ret := false @@ -119,7 +120,6 @@ func (n *NodeImpl) UpAdjustMaxVolumeId(vid storage.VolumeId) { //can be negative } } } - func (n *NodeImpl) GetMaxVolumeId() storage.VolumeId { return n.maxVolumeId } @@ -164,7 +164,7 @@ func (n *NodeImpl) CollectDeadNodeAndFullVolumes(freshThreshHold int64, volumeSi } for _, v := range dn.volumes { if uint64(v.Size) >= volumeSizeLimit { - n.GetTopology().chanFullVolumes <- &v + n.GetTopology().chanFullVolumes <- &v } } } @@ -175,11 +175,11 @@ func (n *NodeImpl) CollectDeadNodeAndFullVolumes(freshThreshHold int64, volumeSi } } -func (n *NodeImpl) GetTopology() *Topology{ - var p Node - p = n - for p.Parent() != nil { - p = p.Parent() - } - return p.GetValue().(*Topology) +func (n *NodeImpl) GetTopology() *Topology { + var p Node + p = n + for p.Parent() != nil { + p = p.Parent() + } + return p.GetValue().(*Topology) }