Browse Source
git-svn-id: https://weed-fs.googlecode.com/svn/trunk@4 282b0af5-e82d-9cf1-ede4-77906d7719d0
pull/2/head
git-svn-id: https://weed-fs.googlecode.com/svn/trunk@4 282b0af5-e82d-9cf1-ede4-77906d7719d0
pull/2/head
chris.lu@gmail.com
13 years ago
7 changed files with 199 additions and 67 deletions
-
46weed-fs/note/weedfs.txt
-
18weed-fs/src/cmd/gdir.go
-
57weed-fs/src/cmd/gstore.go
-
18weed-fs/src/pkg/directory/volume_mapping.go
-
90weed-fs/src/pkg/store/store.go
-
22weed-fs/src/pkg/store/util.go
-
15weed-fs/src/pkg/store/volume.go
@ -0,0 +1,46 @@ |
|||||
|
How to submit a content |
||||
|
1. Find physical volumes |
||||
|
1.c Create a hash value |
||||
|
1.d find a write logic volume id, and return [logic volume id, {physical volume ids}] |
||||
|
2. submit to physical volumes |
||||
|
2.c |
||||
|
generate the cookie |
||||
|
generate a unique id as key |
||||
|
choose the right altKey |
||||
|
send bytes to physical volumes |
||||
|
2.s each |
||||
|
save bytes |
||||
|
store map[key uint64, altKey uint32]<offset, size> |
||||
|
for updated entry, set old entry's offset to zero |
||||
|
3.c |
||||
|
wait for all physical volumes to finish |
||||
|
store the /<logic volume id>/<key>_<cookie>_<altKey>.<ext> |
||||
|
|
||||
|
How to retrieve a content |
||||
|
1.c |
||||
|
send logic volume id |
||||
|
1.d |
||||
|
find least busy volume's id |
||||
|
2.c |
||||
|
send URI /<physical volume id>/<key>_<cookie>_<altKey>.<ext> |
||||
|
|
||||
|
|
||||
|
How to submit a content |
||||
|
1. send bytes to weedfs, got <volume id, key uint64, cookie code> |
||||
|
store <key uint64, volume id uint32, cookie code uint32, ext>, and other information |
||||
|
|
||||
|
To read a content |
||||
|
2. use logic volume id to lookup a <machine id> |
||||
|
render url as /<machine id>/<volume id>/<key>/<cookie>.ext |
||||
|
|
||||
|
The directory server |
||||
|
0.init |
||||
|
load and collect <logic volume id, machine ids> mapping |
||||
|
1.on submit content |
||||
|
find a free logic volume id, start sending content to 3 machines |
||||
|
if all of them finishes, return <logic volume id, key, cookie code> |
||||
|
2.on read content |
||||
|
based on logic volume id, pick a machine with less load, |
||||
|
return <machine id> |
||||
|
|
||||
|
|
@ -1,18 +0,0 @@ |
|||||
package main |
|
||||
|
|
||||
import ( |
|
||||
"directory" |
|
||||
// "runtime"
|
|
||||
"log" |
|
||||
) |
|
||||
|
|
||||
func main() { |
|
||||
m := directory.NewMapper("/tmp", "directory") |
|
||||
log.Println("map size", len(m.Virtual2physical)) |
|
||||
m.Add(10, 11,12,13) |
|
||||
m.Add(20, 21,22,23) |
|
||||
log.Println("map(10)", m.Get(10)) |
|
||||
log.Println("map size", len(m.Virtual2physical)) |
|
||||
m.Save() |
|
||||
defer m.Save() |
|
||||
} |
|
@ -1,39 +1,69 @@ |
|||||
package store |
package store |
||||
|
|
||||
import ( |
import ( |
||||
"log" |
|
||||
"strconv" |
|
||||
|
"log" |
||||
|
"io/ioutil" |
||||
|
"json" |
||||
|
"strings" |
||||
|
"strconv" |
||||
|
"url" |
||||
) |
) |
||||
type Store struct{ |
|
||||
volumes []*Volume |
|
||||
dir string |
|
||||
|
|
||||
freeVolumeChannel chan int |
|
||||
|
|
||||
|
type Store struct { |
||||
|
volumes map[uint64]*Volume |
||||
|
dir string |
||||
|
Server string |
||||
|
PublicServer string |
||||
|
} |
||||
|
type VolumeStat struct { |
||||
|
Id uint64 "id" |
||||
|
Status int "status" //0:read, 1:write
|
||||
|
} |
||||
|
|
||||
|
func NewStore(server, publicServer, dirname string) (s *Store) { |
||||
|
s = new(Store) |
||||
|
s.Server, s.PublicServer, s.dir = server, publicServer, dirname |
||||
|
s.volumes = make(map[uint64]*Volume) |
||||
|
|
||||
|
counter := uint64(0) |
||||
|
files, _ := ioutil.ReadDir(dirname) |
||||
|
for _, f := range files { |
||||
|
if f.IsDirectory() || !strings.HasSuffix(f.Name, ".dat") { |
||||
|
continue |
||||
|
} |
||||
|
id, err := strconv.Atoui64(f.Name[:-4]) |
||||
|
if err == nil { |
||||
|
continue |
||||
|
} |
||||
|
s.volumes[counter] = NewVolume(s.dir, id) |
||||
|
counter++ |
||||
|
} |
||||
|
log.Println("Store started on dir:", dirname, "with", counter, "existing volumes") |
||||
|
return |
||||
} |
} |
||||
func NewStore(dirname string, count int) (s *Store){ |
|
||||
s = new(Store) |
|
||||
s.dir = dirname |
|
||||
s.volumes = make([]*Volume,count) |
|
||||
s.freeVolumeChannel = make(chan int, count) |
|
||||
for i:=0;i<count;i++{ |
|
||||
s.volumes[i] = NewVolume(s.dir, strconv.Itob(i,16)) |
|
||||
s.freeVolumeChannel <- i |
|
||||
} |
|
||||
log.Println("Store started on dir:", dirname, "with", count,"volumes"); |
|
||||
return |
|
||||
|
|
||||
|
func (s *Store) Join(mserver string) { |
||||
|
stats := make([]*VolumeStat, len(s.volumes)) |
||||
|
for k, _ := range s.volumes { |
||||
|
s := new(VolumeStat) |
||||
|
s.Id, s.Status = k, 1 |
||||
|
stats = append(stats, s) |
||||
|
} |
||||
|
bytes, _ := json.Marshal(stats) |
||||
|
values := new(url.Values) |
||||
|
values.Add("server", s.Server) |
||||
|
values.Add("publicServer", s.PublicServer) |
||||
|
values.Add("volumes", string(bytes)) |
||||
|
post("http://"+mserver+"/join", *values) |
||||
} |
} |
||||
func (s *Store)Close(){ |
|
||||
close(s.freeVolumeChannel) |
|
||||
for _, v := range s.volumes{ |
|
||||
v.Close() |
|
||||
} |
|
||||
|
func (s *Store) Close() { |
||||
|
for _, v := range s.volumes { |
||||
|
v.Close() |
||||
|
} |
||||
} |
} |
||||
func (s *Store)Write(n *Needle)(int){ |
|
||||
i := <- s.freeVolumeChannel |
|
||||
s.volumes[i].write(n) |
|
||||
s.freeVolumeChannel <- i |
|
||||
return i |
|
||||
|
func (s *Store) Write(i uint64, n *Needle) { |
||||
|
s.volumes[i].write(n) |
||||
} |
} |
||||
func (s *Store)Read(i int, n *Needle){ |
|
||||
s.volumes[i].read(n) |
|
||||
|
func (s *Store) Read(i uint64, n *Needle) { |
||||
|
s.volumes[i].read(n) |
||||
} |
} |
Write
Preview
Loading…
Cancel
Save
Reference in new issue