Browse Source

add more for volume placement

fix possible nil volume cases
pull/2/head
Chris Lu 13 years ago
parent
commit
2dceb44ae4
  1. 29
      weed-fs/src/pkg/replication/volume_growth.go
  2. 10
      weed-fs/src/pkg/storage/storage_limit.go
  3. 11
      weed-fs/src/pkg/storage/storage_limit_test.go
  4. 18
      weed-fs/src/pkg/storage/store.go
  5. 15
      weed-fs/src/pkg/topology/node.go
  6. 9
      weed-fs/src/pkg/topology/topology.go

29
weed-fs/src/pkg/replication/volume_growth.go

@ -0,0 +1,29 @@
package replication
import (
"pkg/topology"
)
/*
This package is created to resolve these replica placement issues:
1. growth factor for each replica level, e.g., add 10 volumes for 1 copy, 20 volumes for 2 copies, 30 volumes for 3 copies
2. in time of tight storage, how to reduce replica level
3. optimizing for hot data on faster disk, cold data on cheaper storage,
4. volume allocation for each bucket
*/
type VolumeGrowth struct {
copy1factor int
copy2factor int
copy3factor int
copyAll int
}
func (vg *VolumeGrowth) GrowVolumeCopy(copyLevel int, topo topology.Topology) {
if copyLevel == 1 {
for i := 0; i <vg.copy1factor; i++ {
topo.RandomlyCreateOneVolume()
}
}
}

10
weed-fs/src/pkg/storage/storage_limit.go

@ -1,21 +1,13 @@
package storage package storage
import ( import (
"syscall"
) )
type StorageLimit struct { type StorageLimit struct {
sizeLimit uint64 sizeLimit uint64
detectedLimit uint64
} }
func NewStorageLimit(desiredLimit uint64) *StorageLimit { func NewStorageLimit(desiredLimit uint64) *StorageLimit {
s := syscall.Statfs_t{}
errNo := syscall.Statfs(".", &s)
detected := uint64(0)
if errNo==nil {
detected = s.Bavail*uint64(s.Bsize)
}
sl := &StorageLimit{sizeLimit: desiredLimit, detectedLimit: detected}
sl := &StorageLimit{sizeLimit: desiredLimit}
return sl return sl
} }

11
weed-fs/src/pkg/storage/storage_limit_test.go

@ -1,11 +0,0 @@
package storage
import (
"testing"
)
func TestReadStorageLimit(t *testing.T) {
sl := NewStorageLimit(1000)
println("detected:",sl.detectedLimit)
}

18
weed-fs/src/pkg/storage/store.go

@ -89,11 +89,23 @@ func (s *Store) Close() {
} }
} }
func (s *Store) Write(i VolumeId, n *Needle) uint32 { func (s *Store) Write(i VolumeId, n *Needle) uint32 {
return s.volumes[i].write(n)
v := s.volumes[i]
if v!=nil{
return v.write(n)
}
return 0
} }
func (s *Store) Delete(i VolumeId, n *Needle) uint32 { func (s *Store) Delete(i VolumeId, n *Needle) uint32 {
return s.volumes[i].delete(n)
v := s.volumes[i]
if v!=nil{
return v.delete(n)
}
return 0
} }
func (s *Store) Read(i VolumeId, n *Needle) (int, error) { func (s *Store) Read(i VolumeId, n *Needle) (int, error) {
return s.volumes[i].read(n)
v := s.volumes[i]
if v!=nil{
return v.read(n)
}
return 0, errors.New("Not Found")
} }

15
weed-fs/src/pkg/topology/node.go

@ -11,4 +11,19 @@ type Node struct {
Ip string Ip string
Port int Port int
PublicUrl string PublicUrl string
//transient
allocation *Allocation
}
type Allocation struct {
count int
limit int
} }
func (n *Node) GetAllocation() *Allocation{
if n.allocation == nil {
n.allocation = &Allocation{count:len(n.volumes), limit : n.volumeLimit}
}
return n.allocation
}

9
weed-fs/src/pkg/topology/topology.go

@ -1,9 +1,16 @@
package topology package topology
import ( import (
"pkg/storage"
) )
type Topology struct { type Topology struct {
datacenters map[DataCenterId]*DataCenter datacenters map[DataCenterId]*DataCenter
} }
//FIXME
func (t *Topology) RandomlyCreateOneVolume() storage.VolumeId{
return t.findMaxVolumeId()
}
func (t *Topology) findMaxVolumeId() storage.VolumeId{
return storage.VolumeId(0);
}
Loading…
Cancel
Save