Browse Source

efficiently assign file id

git-svn-id: https://weed-fs.googlecode.com/svn/trunk@17 282b0af5-e82d-9cf1-ede4-77906d7719d0
pull/2/head
chris.lu@gmail.com 13 years ago
parent
commit
d018809830
  1. 10
      weed-fs/src/cmd/weeds.go
  2. 44
      weed-fs/src/pkg/directory/volume_mapping.go

10
weed-fs/src/cmd/weeds.go

@ -8,6 +8,7 @@ import (
"http" "http"
"json" "json"
"log" "log"
"rand"
"strconv" "strconv"
"strings" "strings"
) )
@ -17,6 +18,7 @@ var (
metaFolder = flag.String("mdir", "/tmp", "data directory to store mappings") metaFolder = flag.String("mdir", "/tmp", "data directory to store mappings")
capacity = flag.Int("capacity", 100, "maximum number of volumes to hold") capacity = flag.Int("capacity", 100, "maximum number of volumes to hold")
mapper *directory.Mapper mapper *directory.Mapper
) )
func dirReadHandler(w http.ResponseWriter, r *http.Request) { func dirReadHandler(w http.ResponseWriter, r *http.Request) {
@ -25,12 +27,14 @@ func dirReadHandler(w http.ResponseWriter, r *http.Request) {
writeJson(w, r, machine.Server) writeJson(w, r, machine.Server)
} }
func dirWriteHandler(w http.ResponseWriter, r *http.Request) { func dirWriteHandler(w http.ResponseWriter, r *http.Request) {
machine := mapper.PickForWrite()
_, machine := mapper.PickForWrite()
writeJson(w, r, machine) writeJson(w, r, machine)
} }
func dirPickHandler(w http.ResponseWriter, r *http.Request) { func dirPickHandler(w http.ResponseWriter, r *http.Request) {
machine := mapper.PickForWrite()
writeJson(w, r, machine)
vid, machine := mapper.PickForWrite()
hashcode := rand.Uint32()
fid := strconv.Uitoa64(vid) + "," + strconv.Uitoa64(mapper.NextFileId())+","+strconv.Uitoa64(uint64(hashcode))
writeJson(w, r, map[string]string{"fid":fid,"url":machine.Url})
} }
func dirJoinHandler(w http.ResponseWriter, r *http.Request) { func dirJoinHandler(w http.ResponseWriter, r *http.Request) {
s := r.RemoteAddr[0:strings.Index(r.RemoteAddr, ":")+1] + r.FormValue("port") s := r.RemoteAddr[0:strings.Index(r.RemoteAddr, ":")+1] + r.FormValue("port")

44
weed-fs/src/pkg/directory/volume_mapping.go

@ -11,7 +11,8 @@ import (
) )
const ( const (
ChunkSizeLimit = 1 * 1024 * 1024 * 1024 //1G, can not be more than max(uint32)*8
ChunkSizeLimit = 1 * 1000 * 1000 * 1000 //1G, can not be more than max(uint32)*8
FileIdSaveInterval = 10000
) )
type MachineInfo struct { type MachineInfo struct {
@ -35,6 +36,9 @@ type Mapper struct {
Writers []int // transient array of Writers volume id Writers []int // transient array of Writers volume id
GlobalVolumeSequence uint64 GlobalVolumeSequence uint64
FileIdSequence uint64
fileIdCounter uint64
} }
func NewMachine(server, publicServer string, volumes []storage.VolumeInfo, capacity int) *Machine { func NewMachine(server, publicServer string, volumes []storage.VolumeInfo, capacity int) *Machine {
@ -67,11 +71,33 @@ func NewMapper(dirname string, filename string, capacity int) (m *Mapper) {
} }
log.Println("Loaded mapping size", len(m.Machines)) log.Println("Loaded mapping size", len(m.Machines))
} }
seqFile, se := os.OpenFile(path.Join(m.dir, m.fileName+".seq"), os.O_RDONLY, 0644)
if se != nil {
m.FileIdSequence = FileIdSaveInterval
log.Println("Setting file id sequence", m.FileIdSequence)
} else {
decoder := gob.NewDecoder(seqFile)
defer seqFile.Close()
decoder.Decode(&m.FileIdSequence)
log.Println("Loading file id sequence", m.FileIdSequence, "=>", m.FileIdSequence + FileIdSaveInterval)
//in case the server stops between intervals
m.FileIdSequence += FileIdSaveInterval
}
return return
} }
func (m *Mapper) PickForWrite() MachineInfo {
vid := rand.Intn(len(m.Writers))
return m.Machines[m.Writers[vid]].Server
func (m *Mapper) PickForWrite() (vid uint64, server MachineInfo) {
machine := m.Machines[m.Writers[rand.Intn(len(m.Writers))]]
vid = machine.Volumes[rand.Intn(len(machine.Volumes))].Id
return vid, machine.Server
}
func (m *Mapper) NextFileId() uint64 {
if m.fileIdCounter <= 0 {
m.fileIdCounter = FileIdSaveInterval
m.saveSequence()
}
m.fileIdCounter--
return m.FileIdSequence - m.fileIdCounter
} }
func (m *Mapper) Get(vid uint64) *Machine { func (m *Mapper) Get(vid uint64) *Machine {
return m.Machines[m.vid2machineId[vid]] return m.Machines[m.vid2machineId[vid]]
@ -141,3 +167,13 @@ func (m *Mapper) Save() {
encoder.Encode(m.Machines) encoder.Encode(m.Machines)
encoder.Encode(m.GlobalVolumeSequence) encoder.Encode(m.GlobalVolumeSequence)
} }
func (m *Mapper) saveSequence() {
log.Println("Saving file id sequence", m.FileIdSequence, "to", path.Join(m.dir, m.fileName+".seq"))
seqFile, e := os.OpenFile(path.Join(m.dir, m.fileName+".seq"), os.O_CREATE|os.O_WRONLY, 0644)
if e != nil {
log.Fatalf("Sequence File Save [ERROR] %s\n", e)
}
defer seqFile.Close()
encoder := gob.NewEncoder(seqFile)
encoder.Encode(m.FileIdSequence)
}
Loading…
Cancel
Save