Browse Source

Add "weed backup" command.

This is a pre-cursor for asynchronous replication.
pull/174/head
chrislusf 10 years ago
parent
commit
86cd40fba8
  1. 54
      go/operation/sync_volume.go
  2. 77
      go/storage/needle_map.go
  3. 26
      go/storage/needle_map_boltdb.go
  4. 26
      go/storage/needle_map_leveldb.go
  5. 34
      go/storage/needle_map_memory.go
  6. 62
      go/storage/needle_read_write.go
  7. 4
      go/storage/store.go
  8. 39
      go/storage/volume.go
  9. 213
      go/storage/volume_sync.go
  10. 38
      go/util/http_util.go
  11. 90
      go/weed/backup.go
  12. 1
      go/weed/compact.go
  13. 1
      go/weed/weed.go
  14. 3
      go/weed/weed_server/volume_server.go
  15. 2
      go/weed/weed_server/volume_server_handlers_read.go
  16. 86
      go/weed/weed_server/volume_server_handlers_sync.go
  17. 4
      go/weed/weed_server/volume_server_handlers_write.go

54
go/operation/sync_volume.go

@ -0,0 +1,54 @@
package operation
import (
"encoding/json"
"fmt"
"net/url"
"github.com/chrislusf/seaweedfs/go/glog"
"github.com/chrislusf/seaweedfs/go/util"
)
type SyncVolumeResponse struct {
Replication string `json:"Replication,omitempty"`
Ttl string `json:"Ttl,omitempty"`
TailOffset uint64 `json:"TailOffset,omitempty"`
CompactRevision uint16 `json:"CompactRevision,omitempty"`
IdxFileSize uint64 `json:"IdxFileSize,omitempty"`
Error string `json:"error,omitempty"`
}
func GetVolumeSyncStatus(server string, vid string) (*SyncVolumeResponse, error) {
values := make(url.Values)
values.Add("volume", vid)
jsonBlob, err := util.Post("http://"+server+"/admin/sync/status", values)
glog.V(2).Info("sync volume result :", string(jsonBlob))
if err != nil {
return nil, err
}
var ret SyncVolumeResponse
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return nil, err
}
if ret.Error != "" {
return nil, fmt.Errorf("Volume %s get sync status error: %s", vid, ret.Error)
}
return &ret, nil
}
func GetVolumeIdxEntries(server string, vid string, eachEntryFn func(key uint64, offset, size uint32)) error {
values := make(url.Values)
values.Add("volume", vid)
line := make([]byte, 16)
err := util.GetBufferStream("http://"+server+"/admin/sync/index", values, line, func(bytes []byte) {
key := util.BytesToUint64(bytes[:8])
offset := util.BytesToUint32(bytes[8:12])
size := util.BytesToUint32(bytes[12:16])
eachEntryFn(key, offset, size)
})
if err != nil {
return err
}
return nil
}

77
go/storage/needle_map.go

@ -2,7 +2,9 @@ package storage
import ( import (
"fmt" "fmt"
"io/ioutil"
"os" "os"
"sync"
"github.com/chrislusf/seaweedfs/go/util" "github.com/chrislusf/seaweedfs/go/util"
) )
@ -26,29 +28,66 @@ type NeedleMapper interface {
FileCount() int FileCount() int
DeletedCount() int DeletedCount() int
MaxFileKey() uint64 MaxFileKey() uint64
IndexFileSize() uint64
IndexFileContent() ([]byte, error)
IndexFileName() string
} }
type mapMetric struct {
DeletionCounter int `json:"DeletionCounter"`
FileCounter int `json:"FileCounter"`
DeletionByteCounter uint64 `json:"DeletionByteCounter"`
FileByteCounter uint64 `json:"FileByteCounter"`
MaximumFileKey uint64 `json:"MaxFileKey"`
type baseNeedleMapper struct {
indexFile *os.File
indexFileAccessLock sync.Mutex
mapMetric
} }
func appendToIndexFile(indexFile *os.File,
key uint64, offset uint32, size uint32) error {
func (nm baseNeedleMapper) IndexFileSize() uint64 {
stat, err := nm.indexFile.Stat()
if err == nil {
return uint64(stat.Size())
}
return 0
}
func (nm baseNeedleMapper) IndexFileName() string {
return nm.indexFile.Name()
}
func idxFileEntry(bytes []byte) (key uint64, offset uint32, size uint32) {
key = util.BytesToUint64(bytes[:8])
offset = util.BytesToUint32(bytes[8:12])
size = util.BytesToUint32(bytes[12:16])
return
}
func (nm baseNeedleMapper) appendToIndexFile(key uint64, offset uint32, size uint32) error {
bytes := make([]byte, 16) bytes := make([]byte, 16)
util.Uint64toBytes(bytes[0:8], key) util.Uint64toBytes(bytes[0:8], key)
util.Uint32toBytes(bytes[8:12], offset) util.Uint32toBytes(bytes[8:12], offset)
util.Uint32toBytes(bytes[12:16], size) util.Uint32toBytes(bytes[12:16], size)
if _, err := indexFile.Seek(0, 2); err != nil {
nm.indexFileAccessLock.Lock()
defer nm.indexFileAccessLock.Unlock()
if _, err := nm.indexFile.Seek(0, 2); err != nil {
return fmt.Errorf("cannot seek end of indexfile %s: %v", return fmt.Errorf("cannot seek end of indexfile %s: %v",
indexFile.Name(), err)
nm.indexFile.Name(), err)
} }
_, err := indexFile.Write(bytes)
_, err := nm.indexFile.Write(bytes)
return err return err
} }
func (nm baseNeedleMapper) IndexFileContent() ([]byte, error) {
nm.indexFileAccessLock.Lock()
defer nm.indexFileAccessLock.Unlock()
return ioutil.ReadFile(nm.indexFile.Name())
}
type mapMetric struct {
indexFile *os.File
DeletionCounter int `json:"DeletionCounter"`
FileCounter int `json:"FileCounter"`
DeletionByteCounter uint64 `json:"DeletionByteCounter"`
FileByteCounter uint64 `json:"FileByteCounter"`
MaximumFileKey uint64 `json:"MaxFileKey"`
}
func (mm *mapMetric) logDelete(deletedByteCount uint32) { func (mm *mapMetric) logDelete(deletedByteCount uint32) {
mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(deletedByteCount) mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(deletedByteCount)
@ -66,3 +105,19 @@ func (mm *mapMetric) logPut(key uint64, oldSize uint32, newSize uint32) {
mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(oldSize) mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(oldSize)
} }
} }
func (mm mapMetric) ContentSize() uint64 {
return mm.FileByteCounter
}
func (mm mapMetric) DeletedSize() uint64 {
return mm.DeletionByteCounter
}
func (mm mapMetric) FileCount() int {
return mm.FileCounter
}
func (mm mapMetric) DeletedCount() int {
return mm.DeletionCounter
}
func (mm mapMetric) MaxFileKey() uint64 {
return mm.MaximumFileKey
}

26
go/storage/needle_map_boltdb.go

@ -12,15 +12,15 @@ import (
type BoltDbNeedleMap struct { type BoltDbNeedleMap struct {
dbFileName string dbFileName string
indexFile *os.File
db *bolt.DB db *bolt.DB
mapMetric
baseNeedleMapper
} }
var boltdbBucket = []byte("weed") var boltdbBucket = []byte("weed")
func NewBoltDbNeedleMap(dbFileName string, indexFile *os.File) (m *BoltDbNeedleMap, err error) { func NewBoltDbNeedleMap(dbFileName string, indexFile *os.File) (m *BoltDbNeedleMap, err error) {
m = &BoltDbNeedleMap{indexFile: indexFile, dbFileName: dbFileName}
m = &BoltDbNeedleMap{dbFileName: dbFileName}
m.indexFile = indexFile
if !isBoltDbFresh(dbFileName, indexFile) { if !isBoltDbFresh(dbFileName, indexFile) {
glog.V(1).Infof("Start to Generate %s from %s", dbFileName, indexFile.Name()) glog.V(1).Infof("Start to Generate %s from %s", dbFileName, indexFile.Name())
generateBoltDbFile(dbFileName, indexFile) generateBoltDbFile(dbFileName, indexFile)
@ -101,7 +101,7 @@ func (m *BoltDbNeedleMap) Put(key uint64, offset uint32, size uint32) error {
} }
m.logPut(key, oldSize, size) m.logPut(key, oldSize, size)
// write to index file first // write to index file first
if err := appendToIndexFile(m.indexFile, key, offset, size); err != nil {
if err := m.appendToIndexFile(key, offset, size); err != nil {
return fmt.Errorf("cannot write to indexfile %s: %v", m.indexFile.Name(), err) return fmt.Errorf("cannot write to indexfile %s: %v", m.indexFile.Name(), err)
} }
return boltDbWrite(m.db, key, offset, size) return boltDbWrite(m.db, key, offset, size)
@ -148,7 +148,7 @@ func (m *BoltDbNeedleMap) Delete(key uint64) error {
m.logDelete(oldNeedle.Size) m.logDelete(oldNeedle.Size)
} }
// write to index file first // write to index file first
if err := appendToIndexFile(m.indexFile, key, 0, 0); err != nil {
if err := m.appendToIndexFile(key, 0, 0); err != nil {
return err return err
} }
return boltDbDelete(m.db, key) return boltDbDelete(m.db, key)
@ -163,19 +163,3 @@ func (m *BoltDbNeedleMap) Destroy() error {
os.Remove(m.indexFile.Name()) os.Remove(m.indexFile.Name())
return os.Remove(m.dbFileName) return os.Remove(m.dbFileName)
} }
func (m *BoltDbNeedleMap) ContentSize() uint64 {
return m.FileByteCounter
}
func (m *BoltDbNeedleMap) DeletedSize() uint64 {
return m.DeletionByteCounter
}
func (m *BoltDbNeedleMap) FileCount() int {
return m.FileCounter
}
func (m *BoltDbNeedleMap) DeletedCount() int {
return m.DeletionCounter
}
func (m *BoltDbNeedleMap) MaxFileKey() uint64 {
return m.MaximumFileKey
}

26
go/storage/needle_map_leveldb.go

@ -12,13 +12,13 @@ import (
type LevelDbNeedleMap struct { type LevelDbNeedleMap struct {
dbFileName string dbFileName string
indexFile *os.File
db *leveldb.DB db *leveldb.DB
mapMetric
baseNeedleMapper
} }
func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File) (m *LevelDbNeedleMap, err error) { func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File) (m *LevelDbNeedleMap, err error) {
m = &LevelDbNeedleMap{indexFile: indexFile, dbFileName: dbFileName}
m = &LevelDbNeedleMap{dbFileName: dbFileName}
m.indexFile = indexFile
if !isLevelDbFresh(dbFileName, indexFile) { if !isLevelDbFresh(dbFileName, indexFile) {
glog.V(1).Infof("Start to Generate %s from %s", dbFileName, indexFile.Name()) glog.V(1).Infof("Start to Generate %s from %s", dbFileName, indexFile.Name())
generateLevelDbFile(dbFileName, indexFile) generateLevelDbFile(dbFileName, indexFile)
@ -89,7 +89,7 @@ func (m *LevelDbNeedleMap) Put(key uint64, offset uint32, size uint32) error {
} }
m.logPut(key, oldSize, size) m.logPut(key, oldSize, size)
// write to index file first // write to index file first
if err := appendToIndexFile(m.indexFile, key, offset, size); err != nil {
if err := m.appendToIndexFile(key, offset, size); err != nil {
return fmt.Errorf("cannot write to indexfile %s: %v", m.indexFile.Name(), err) return fmt.Errorf("cannot write to indexfile %s: %v", m.indexFile.Name(), err)
} }
return levelDbWrite(m.db, key, offset, size) return levelDbWrite(m.db, key, offset, size)
@ -117,7 +117,7 @@ func (m *LevelDbNeedleMap) Delete(key uint64) error {
m.logDelete(oldNeedle.Size) m.logDelete(oldNeedle.Size)
} }
// write to index file first // write to index file first
if err := appendToIndexFile(m.indexFile, key, 0, 0); err != nil {
if err := m.appendToIndexFile(key, 0, 0); err != nil {
return err return err
} }
return levelDbDelete(m.db, key) return levelDbDelete(m.db, key)
@ -132,19 +132,3 @@ func (m *LevelDbNeedleMap) Destroy() error {
os.Remove(m.indexFile.Name()) os.Remove(m.indexFile.Name())
return os.Remove(m.dbFileName) return os.Remove(m.dbFileName)
} }
func (m *LevelDbNeedleMap) ContentSize() uint64 {
return m.FileByteCounter
}
func (m *LevelDbNeedleMap) DeletedSize() uint64 {
return m.DeletionByteCounter
}
func (m *LevelDbNeedleMap) FileCount() int {
return m.FileCounter
}
func (m *LevelDbNeedleMap) DeletedCount() int {
return m.DeletionCounter
}
func (m *LevelDbNeedleMap) MaxFileKey() uint64 {
return m.MaximumFileKey
}

34
go/storage/needle_map_memory.go

@ -5,21 +5,19 @@ import (
"os" "os"
"github.com/chrislusf/seaweedfs/go/glog" "github.com/chrislusf/seaweedfs/go/glog"
"github.com/chrislusf/seaweedfs/go/util"
) )
type NeedleMap struct { type NeedleMap struct {
indexFile *os.File
m CompactMap
m CompactMap
mapMetric
baseNeedleMapper
} }
func NewNeedleMap(file *os.File) *NeedleMap { func NewNeedleMap(file *os.File) *NeedleMap {
nm := &NeedleMap{ nm := &NeedleMap{
m: NewCompactMap(),
indexFile: file,
m: NewCompactMap(),
} }
nm.indexFile = file
return nm return nm
} }
@ -70,9 +68,7 @@ func WalkIndexFile(r *os.File, fn func(key uint64, offset, size uint32) error) e
for count > 0 && e == nil || e == io.EOF { for count > 0 && e == nil || e == io.EOF {
for i = 0; i+16 <= count; i += 16 { for i = 0; i+16 <= count; i += 16 {
key = util.BytesToUint64(bytes[i : i+8])
offset = util.BytesToUint32(bytes[i+8 : i+12])
size = util.BytesToUint32(bytes[i+12 : i+16])
key, offset, size = idxFileEntry(bytes[i : i+16])
if e = fn(key, offset, size); e != nil { if e = fn(key, offset, size); e != nil {
return e return e
} }
@ -90,7 +86,7 @@ func WalkIndexFile(r *os.File, fn func(key uint64, offset, size uint32) error) e
func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) error { func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) error {
oldSize := nm.m.Set(Key(key), offset, size) oldSize := nm.m.Set(Key(key), offset, size)
nm.logPut(key, oldSize, size) nm.logPut(key, oldSize, size)
return appendToIndexFile(nm.indexFile, key, offset, size)
return nm.appendToIndexFile(key, offset, size)
} }
func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) { func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
element, ok = nm.m.Get(Key(key)) element, ok = nm.m.Get(Key(key))
@ -99,7 +95,7 @@ func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
func (nm *NeedleMap) Delete(key uint64) error { func (nm *NeedleMap) Delete(key uint64) error {
deletedBytes := nm.m.Delete(Key(key)) deletedBytes := nm.m.Delete(Key(key))
nm.logDelete(deletedBytes) nm.logDelete(deletedBytes)
return appendToIndexFile(nm.indexFile, key, 0, 0)
return nm.appendToIndexFile(key, 0, 0)
} }
func (nm *NeedleMap) Close() { func (nm *NeedleMap) Close() {
_ = nm.indexFile.Close() _ = nm.indexFile.Close()
@ -108,19 +104,3 @@ func (nm *NeedleMap) Destroy() error {
nm.Close() nm.Close()
return os.Remove(nm.indexFile.Name()) return os.Remove(nm.indexFile.Name())
} }
func (nm NeedleMap) ContentSize() uint64 {
return nm.FileByteCounter
}
func (nm NeedleMap) DeletedSize() uint64 {
return nm.DeletionByteCounter
}
func (nm NeedleMap) FileCount() int {
return nm.FileCounter
}
func (nm NeedleMap) DeletedCount() int {
return nm.DeletionCounter
}
func (nm NeedleMap) MaxFileKey() uint64 {
return nm.MaximumFileKey
}

62
go/storage/needle_read_write.go

@ -135,49 +135,37 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) {
return 0, fmt.Errorf("Unsupported Version! (%d)", version) return 0, fmt.Errorf("Unsupported Version! (%d)", version)
} }
func (n *Needle) Read(r *os.File, offset int64, size uint32, version Version) (ret int, err error) {
func ReadNeedleBlob(r *os.File, offset int64, size uint32) (bytes []byte, err error) {
padding := NeedlePaddingSize - ((NeedleHeaderSize + size + NeedleChecksumSize) % NeedlePaddingSize)
bytes = make([]byte, NeedleHeaderSize+size+NeedleChecksumSize+padding)
_, err = r.ReadAt(bytes, offset)
return
}
func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error) {
bytes, err := ReadNeedleBlob(r, offset, size)
if err != nil {
return err
}
n.ParseNeedleHeader(bytes)
if n.Size != size {
return fmt.Errorf("File Entry Not Found. Needle %d Memory %d", n.Size, size)
}
switch version { switch version {
case Version1: case Version1:
bytes := make([]byte, NeedleHeaderSize+size+NeedleChecksumSize)
if ret, err = r.ReadAt(bytes, offset); err != nil {
return
}
n.readNeedleHeader(bytes)
n.Data = bytes[NeedleHeaderSize : NeedleHeaderSize+size] n.Data = bytes[NeedleHeaderSize : NeedleHeaderSize+size]
checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize])
newChecksum := NewCRC(n.Data)
if checksum != newChecksum.Value() {
return 0, errors.New("CRC error! Data On Disk Corrupted")
}
n.Checksum = newChecksum
return
case Version2: case Version2:
if size == 0 {
return 0, nil
}
bytes := make([]byte, NeedleHeaderSize+size+NeedleChecksumSize)
if ret, err = r.ReadAt(bytes, offset); err != nil {
return
}
if ret != int(NeedleHeaderSize+size+NeedleChecksumSize) {
return 0, errors.New("File Entry Not Found")
}
n.readNeedleHeader(bytes)
if n.Size != size {
return 0, fmt.Errorf("File Entry Not Found. Needle %d Memory %d", n.Size, size)
}
n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)]) n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)])
checksum := util.BytesToUint32(bytes[NeedleHeaderSize+n.Size : NeedleHeaderSize+n.Size+NeedleChecksumSize])
newChecksum := NewCRC(n.Data)
if checksum != newChecksum.Value() {
return 0, errors.New("CRC Found Data On Disk Corrupted")
}
n.Checksum = newChecksum
return
} }
return 0, fmt.Errorf("Unsupported Version! (%d)", version)
checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize])
newChecksum := NewCRC(n.Data)
if checksum != newChecksum.Value() {
return errors.New("CRC error! Data On Disk Corrupted")
}
n.Checksum = newChecksum
return nil
} }
func (n *Needle) readNeedleHeader(bytes []byte) {
func (n *Needle) ParseNeedleHeader(bytes []byte) {
n.Cookie = util.BytesToUint32(bytes[0:4]) n.Cookie = util.BytesToUint32(bytes[0:4])
n.Id = util.BytesToUint64(bytes[4:12]) n.Id = util.BytesToUint64(bytes[4:12])
n.Size = util.BytesToUint32(bytes[12:NeedleHeaderSize]) n.Size = util.BytesToUint32(bytes[12:NeedleHeaderSize])
@ -228,7 +216,7 @@ func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, bod
if count <= 0 || err != nil { if count <= 0 || err != nil {
return nil, 0, err return nil, 0, err
} }
n.readNeedleHeader(bytes)
n.ParseNeedleHeader(bytes)
padding := NeedlePaddingSize - ((n.Size + NeedleHeaderSize + NeedleChecksumSize) % NeedlePaddingSize) padding := NeedlePaddingSize - ((n.Size + NeedleHeaderSize + NeedleChecksumSize) % NeedlePaddingSize)
bodyLength = n.Size + NeedleChecksumSize + padding bodyLength = n.Size + NeedleChecksumSize + padding
} }

4
go/storage/store.go

@ -371,9 +371,9 @@ func (s *Store) Delete(i VolumeId, n *Needle) (uint32, error) {
} }
return 0, nil return 0, nil
} }
func (s *Store) Read(i VolumeId, n *Needle) (int, error) {
func (s *Store) ReadVolumeNeedle(i VolumeId, n *Needle) (int, error) {
if v := s.findVolume(i); v != nil { if v := s.findVolume(i); v != nil {
return v.read(n)
return v.readNeedle(n)
} }
return 0, fmt.Errorf("Volume %v not found!", i) return 0, fmt.Errorf("Volume %v not found!", i)
} }

39
go/storage/volume.go

@ -54,6 +54,9 @@ func (v *Volume) FileName() (fileName string) {
} }
return return
} }
func (v *Volume) DataFile() *os.File {
return v.dataFile
}
func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType) error { func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType) error {
var e error var e error
fileName := v.FileName() fileName := v.FileName()
@ -152,7 +155,7 @@ func (v *Volume) isFileUnchanged(n *Needle) bool {
nv, ok := v.nm.Get(n.Id) nv, ok := v.nm.Get(n.Id)
if ok && nv.Offset > 0 { if ok && nv.Offset > 0 {
oldNeedle := new(Needle) oldNeedle := new(Needle)
_, err := oldNeedle.Read(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
err := oldNeedle.ReadData(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
if err != nil { if err != nil {
glog.V(0).Infof("Failed to check updated file %v", err) glog.V(0).Infof("Failed to check updated file %v", err)
return false return false
@ -180,6 +183,30 @@ func (v *Volume) Destroy() (err error) {
return return
} }
// AppendBlob append a blob to end of the data file, used in replication
func (v *Volume) AppendBlob(b []byte) (offset int64, err error) {
if v.readOnly {
err = fmt.Errorf("%s is read-only", v.dataFile.Name())
return
}
v.dataFileAccessLock.Lock()
defer v.dataFileAccessLock.Unlock()
if offset, err = v.dataFile.Seek(0, 2); err != nil {
glog.V(0).Infof("failed to seek the end of file: %v", err)
return
}
//ensure file writing starting from aligned positions
if offset%NeedlePaddingSize != 0 {
offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize)
if offset, err = v.dataFile.Seek(offset, 0); err != nil {
glog.V(0).Infof("failed to align in datafile %s: %v", v.dataFile.Name(), err)
return
}
}
v.dataFile.Write(b)
return
}
func (v *Volume) write(n *Needle) (size uint32, err error) { func (v *Volume) write(n *Needle) (size uint32, err error) {
glog.V(4).Infof("writing needle %s", NewFileIdFromNeedle(v.Id, n).String()) glog.V(4).Infof("writing needle %s", NewFileIdFromNeedle(v.Id, n).String())
if v.readOnly { if v.readOnly {
@ -250,17 +277,19 @@ func (v *Volume) delete(n *Needle) (uint32, error) {
return 0, nil return 0, nil
} }
func (v *Volume) read(n *Needle) (int, error) {
// read fills in Needle content by looking up n.Id from NeedleMapper
func (v *Volume) readNeedle(n *Needle) (int, error) {
nv, ok := v.nm.Get(n.Id) nv, ok := v.nm.Get(n.Id)
if !ok || nv.Offset == 0 { if !ok || nv.Offset == 0 {
return -1, errors.New("Not Found") return -1, errors.New("Not Found")
} }
bytesRead, err := n.Read(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
err := n.ReadData(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
if err != nil { if err != nil {
return bytesRead, err
return 0, err
} }
bytesRead := len(n.Data)
if !n.HasTtl() { if !n.HasTtl() {
return bytesRead, err
return bytesRead, nil
} }
ttlMinutes := n.Ttl.Minutes() ttlMinutes := n.Ttl.Minutes()
if ttlMinutes == 0 { if ttlMinutes == 0 {

213
go/storage/volume_sync.go

@ -0,0 +1,213 @@
package storage
import (
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"sort"
"strconv"
"github.com/chrislusf/seaweedfs/go/glog"
"github.com/chrislusf/seaweedfs/go/operation"
"github.com/chrislusf/seaweedfs/go/util"
)
// The volume sync with a master volume via 2 steps:
// 1. The slave checks master side to find subscription checkpoint
// to setup the replication.
// 2. The slave receives the updates from master
/*
Assume the slave volume needs to follow the master volume.
The master volume could be compacted, and could be many files ahead of
slave volume.
Step 1:
The slave volume will ask the master volume for a snapshot
of (existing file entries, last offset, number of compacted times).
For each entry x in master existing file entries:
if x does not exist locally:
add x locally
For each entry y in local slave existing file entries:
if y does not exist on master:
delete y locally
Step 2:
After this, use the last offset and number of compacted times to request
the master volume to send a new file, and keep looping. If the number of
compacted times is changed, go back to step 1 (very likely this can be
optimized more later).
*/
func (v *Volume) Synchronize(volumeServer string) (err error) {
var lastCompactRevision uint16 = 0
var compactRevision uint16 = 0
var masterMap CompactMap
for i := 0; i < 3; i++ {
if masterMap, _, compactRevision, err = fetchVolumeFileEntries(volumeServer, v.Id); err != nil {
return fmt.Errorf("Failed to sync volume %d entries with %s: %v", v.Id, volumeServer, err)
}
if lastCompactRevision != compactRevision && lastCompactRevision != 0 {
if err = v.Compact(); err != nil {
return fmt.Errorf("Compact Volume before synchronizing %v", err)
}
if err = v.commitCompact(); err != nil {
return fmt.Errorf("Commit Compact before synchronizing %v", err)
}
}
lastCompactRevision = compactRevision
if err = v.trySynchronizing(volumeServer, masterMap, compactRevision); err == nil {
return
}
}
return
}
type ByOffset []NeedleValue
func (a ByOffset) Len() int { return len(a) }
func (a ByOffset) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByOffset) Less(i, j int) bool { return a[i].Offset < a[j].Offset }
// trySynchronizing sync with remote volume server incrementally by
// make up the local and remote delta.
func (v *Volume) trySynchronizing(volumeServer string, masterMap CompactMap, compactRevision uint16) error {
slaveIdxFile, err := os.Open(v.nm.IndexFileName())
if err != nil {
return fmt.Errorf("Open volume %d index file: %v", v.Id, err)
}
defer slaveIdxFile.Close()
slaveMap, err := LoadNeedleMap(slaveIdxFile)
if err != nil {
return fmt.Errorf("Load volume %d index file: %v", v.Id, err)
}
var delta []NeedleValue
if err := masterMap.Visit(func(needleValue NeedleValue) error {
if needleValue.Key == 0 {
return nil
}
if _, ok := slaveMap.Get(uint64(needleValue.Key)); ok {
return nil // skip intersection
}
delta = append(delta, needleValue)
return nil
}); err != nil {
return fmt.Errorf("Add master entry: %v", err)
}
if err := slaveMap.m.Visit(func(needleValue NeedleValue) error {
if needleValue.Key == 0 {
return nil
}
if _, ok := masterMap.Get(needleValue.Key); ok {
return nil // skip intersection
}
needleValue.Size = 0
delta = append(delta, needleValue)
return nil
}); err != nil {
return fmt.Errorf("Remove local entry: %v", err)
}
// simulate to same ordering of remote .dat file needle entries
sort.Sort(ByOffset(delta))
// make up the delta
fetchCount := 0
volumeDataContentHandlerUrl := "http://" + volumeServer + "/admin/sync/data"
for _, needleValue := range delta {
if needleValue.Size == 0 {
// remove file entry from local
v.removeNeedle(needleValue.Key)
continue
}
// add master file entry to local data file
if err := v.fetchNeedle(volumeDataContentHandlerUrl, needleValue, compactRevision); err != nil {
glog.V(0).Infof("Fetch needle %v from %s: %v", needleValue, volumeServer, err)
return err
}
fetchCount++
}
glog.V(1).Infof("Fetched %d needles from %s", fetchCount, volumeServer)
return nil
}
func fetchVolumeFileEntries(volumeServer string, vid VolumeId) (m CompactMap, lastOffset uint64, compactRevision uint16, err error) {
m = NewCompactMap()
syncStatus, err := operation.GetVolumeSyncStatus(volumeServer, vid.String())
if err != nil {
return m, 0, 0, err
}
total := 0
err = operation.GetVolumeIdxEntries(volumeServer, vid.String(), func(key uint64, offset, size uint32) {
// println("remote key", key, "offset", offset*NeedlePaddingSize, "size", size)
if offset != 0 && size != 0 {
m.Set(Key(key), offset, size)
} else {
m.Delete(Key(key))
}
total++
})
glog.V(2).Infof("server %s volume %d, entries %d, last offset %d, revision %d", volumeServer, vid, total, syncStatus.TailOffset, syncStatus.CompactRevision)
return m, syncStatus.TailOffset, syncStatus.CompactRevision, err
}
func (v *Volume) GetVolumeSyncStatus() operation.SyncVolumeResponse {
var syncStatus = operation.SyncVolumeResponse{}
if stat, err := v.dataFile.Stat(); err == nil {
syncStatus.TailOffset = uint64(stat.Size())
}
syncStatus.IdxFileSize = v.nm.IndexFileSize()
syncStatus.CompactRevision = v.SuperBlock.CompactRevision
syncStatus.Ttl = v.SuperBlock.Ttl.String()
syncStatus.Replication = v.SuperBlock.ReplicaPlacement.String()
return syncStatus
}
func (v *Volume) IndexFileContent() ([]byte, error) {
return v.nm.IndexFileContent()
}
// removeNeedle removes one needle by needle key
func (v *Volume) removeNeedle(key Key) {
n := new(Needle)
n.Id = uint64(key)
v.delete(n)
}
// fetchNeedle fetches a remote volume needle by vid, id, offset
// The compact revision is checked first in case the remote volume
// is compacted and the offset is invalid any more.
func (v *Volume) fetchNeedle(volumeDataContentHandlerUrl string,
needleValue NeedleValue, compactRevision uint16) error {
// add master file entry to local data file
values := make(url.Values)
values.Add("revision", strconv.Itoa(int(compactRevision)))
values.Add("volume", v.Id.String())
values.Add("id", needleValue.Key.String())
values.Add("offset", strconv.FormatUint(uint64(needleValue.Offset), 10))
values.Add("size", strconv.FormatUint(uint64(needleValue.Size), 10))
glog.V(4).Infof("Fetch %+v", needleValue)
return util.GetUrlStream(volumeDataContentHandlerUrl, values, func(r io.Reader) error {
b, err := ioutil.ReadAll(r)
if err != nil {
return fmt.Errorf("Reading from %s error: %v", volumeDataContentHandlerUrl, err)
}
offset, err := v.AppendBlob(b)
if err != nil {
return fmt.Errorf("Appending volume %d error: %v", v.Id, err)
}
// println("add key", needleValue.Key, "offset", offset, "size", needleValue.Size)
v.nm.Put(uint64(needleValue.Key), uint32(offset/NeedlePaddingSize), needleValue.Size)
return nil
})
}

38
go/util/http_util.go

@ -3,6 +3,7 @@ package util
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"io"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
@ -84,6 +85,43 @@ func Delete(url string, jwt security.EncodedJwt) error {
return nil return nil
} }
func GetBufferStream(url string, values url.Values, allocatedBytes []byte, eachBuffer func([]byte)) error {
r, err := client.PostForm(url, values)
if err != nil {
return err
}
defer r.Body.Close()
if r.StatusCode != 200 {
return fmt.Errorf("%s: %s", url, r.Status)
}
bufferSize := len(allocatedBytes)
for {
n, err := r.Body.Read(allocatedBytes)
if n == bufferSize {
eachBuffer(allocatedBytes)
}
if err != nil {
if err == io.EOF {
return nil
}
return err
}
}
return nil
}
func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) error {
r, err := client.PostForm(url, values)
if err != nil {
return err
}
defer r.Body.Close()
if r.StatusCode != 200 {
return fmt.Errorf("%s: %s", url, r.Status)
}
return readFn(r.Body)
}
func DownloadUrl(fileUrl string) (filename string, content []byte, e error) { func DownloadUrl(fileUrl string) (filename string, content []byte, e error) {
response, err := client.Get(fileUrl) response, err := client.Get(fileUrl)
if err != nil { if err != nil {

90
go/weed/backup.go

@ -0,0 +1,90 @@
package main
import (
"fmt"
"github.com/chrislusf/seaweedfs/go/operation"
"github.com/chrislusf/seaweedfs/go/storage"
)
var (
s BackupOptions
)
type BackupOptions struct {
master *string
collection *string
dir *string
volumeId *int
}
func init() {
cmdBackup.Run = runBackup // break init cycle
s.master = cmdBackup.Flag.String("server", "localhost:9333", "SeaweedFS master location")
s.collection = cmdBackup.Flag.String("collection", "", "collection name")
s.dir = cmdBackup.Flag.String("dir", ".", "directory to store volume data files")
s.volumeId = cmdBackup.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
}
var cmdBackup = &Command{
UsageLine: "backup -dir=. -volumeId=234 -server=localhost:9333",
Short: "incrementally backup a volume to local folder",
Long: `Incrementally backup volume data.
It is expected that you use this inside a script, to loop through
all possible volume ids that needs to be backup to local folder.
The volume id does not need to exist locally or even remotely.
This will help to backup future new volumes.
Usually backing up is just copying the .dat (and .idx) files.
But it's tricky to incremententally copy the differences.
The complexity comes when there are multiple addition, deletion and compaction.
This tool will handle them correctly and efficiently, avoiding unnecessary data transporation.
`,
}
func runBackup(cmd *Command, args []string) bool {
if *s.volumeId == -1 {
return false
}
vid := storage.VolumeId(*s.volumeId)
// find volume location, replication, ttl info
lookup, err := operation.Lookup(*s.master, vid.String())
if err != nil {
fmt.Printf("Error looking up volume %d: %v\n", vid, err)
return true
}
volumeServer := lookup.Locations[0].Url
stats, err := operation.GetVolumeSyncStatus(volumeServer, vid.String())
if err != nil {
fmt.Printf("Error get volume %d status: %v\n", vid, err)
return true
}
ttl, err := storage.ReadTTL(stats.Ttl)
if err != nil {
fmt.Printf("Error get volume %d ttl %s: %v\n", vid, stats.Ttl, err)
return true
}
replication, err := storage.NewReplicaPlacementFromString(stats.Replication)
if err != nil {
fmt.Printf("Error get volume %d replication %s : %v\n", vid, stats.Replication, err)
return true
}
v, err := storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl)
if err != nil {
fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
return true
}
if err := v.Synchronize(volumeServer); err != nil {
fmt.Printf("Error synchronizing volume %d: %v\n", vid, err)
return true
}
return true
}

1
go/weed/compact.go

@ -7,7 +7,6 @@ import (
func init() { func init() {
cmdCompact.Run = runCompact // break init cycle cmdCompact.Run = runCompact // break init cycle
cmdCompact.IsDebug = cmdCompact.Flag.Bool("debug", false, "enable debug mode")
} }
var cmdCompact = &Command{ var cmdCompact = &Command{

1
go/weed/weed.go

@ -21,6 +21,7 @@ var server *string
var commands = []*Command{ var commands = []*Command{
cmdBenchmark, cmdBenchmark,
cmdBackup,
cmdCompact, cmdCompact,
cmdFix, cmdFix,
cmdServer, cmdServer,

3
go/weed/weed_server/volume_server.go

@ -51,6 +51,9 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
adminMux.HandleFunc("/admin/vacuum/compact", vs.guard.WhiteList(vs.vacuumVolumeCompactHandler)) adminMux.HandleFunc("/admin/vacuum/compact", vs.guard.WhiteList(vs.vacuumVolumeCompactHandler))
adminMux.HandleFunc("/admin/vacuum/commit", vs.guard.WhiteList(vs.vacuumVolumeCommitHandler)) adminMux.HandleFunc("/admin/vacuum/commit", vs.guard.WhiteList(vs.vacuumVolumeCommitHandler))
adminMux.HandleFunc("/admin/delete_collection", vs.guard.WhiteList(vs.deleteCollectionHandler)) adminMux.HandleFunc("/admin/delete_collection", vs.guard.WhiteList(vs.deleteCollectionHandler))
adminMux.HandleFunc("/admin/sync/status", vs.guard.WhiteList(vs.getVolumeSyncStatusHandler))
adminMux.HandleFunc("/admin/sync/index", vs.guard.WhiteList(vs.getVolumeIndexContentHandler))
adminMux.HandleFunc("/admin/sync/data", vs.guard.WhiteList(vs.getVolumeDataContentHandler))
adminMux.HandleFunc("/stats/counter", vs.guard.WhiteList(statsCounterHandler)) adminMux.HandleFunc("/stats/counter", vs.guard.WhiteList(statsCounterHandler))
adminMux.HandleFunc("/stats/memory", vs.guard.WhiteList(statsMemoryHandler)) adminMux.HandleFunc("/stats/memory", vs.guard.WhiteList(statsMemoryHandler))
adminMux.HandleFunc("/stats/disk", vs.guard.WhiteList(vs.statsDiskHandler)) adminMux.HandleFunc("/stats/disk", vs.guard.WhiteList(vs.statsDiskHandler))

2
go/weed/weed_server/volume_server_handlers_read.go

@ -47,7 +47,7 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
return return
} }
cookie := n.Cookie cookie := n.Cookie
count, e := vs.store.Read(volumeId, n)
count, e := vs.store.ReadVolumeNeedle(volumeId, n)
glog.V(4).Infoln("read bytes", count, "error", e) glog.V(4).Infoln("read bytes", count, "error", e)
if e != nil || count <= 0 { if e != nil || count <= 0 {
glog.V(0).Infoln("read error:", e, r.URL.Path) glog.V(0).Infoln("read error:", e, r.URL.Path)

86
go/weed/weed_server/volume_server_handlers_sync.go

@ -0,0 +1,86 @@
package weed_server
import (
"fmt"
"net/http"
"github.com/chrislusf/seaweedfs/go/glog"
"github.com/chrislusf/seaweedfs/go/storage"
"github.com/chrislusf/seaweedfs/go/util"
)
func (vs *VolumeServer) getVolumeSyncStatusHandler(w http.ResponseWriter, r *http.Request) {
v, err := vs.getVolume("volume", r)
if v == nil {
writeJsonError(w, r, http.StatusBadRequest, err)
return
}
syncStat := v.GetVolumeSyncStatus()
if syncStat.Error != "" {
writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Get Volume %d status error: %s", v.Id, syncStat.Error))
glog.V(2).Infoln("getVolumeSyncStatusHandler volume =", r.FormValue("volume"), ", error =", err)
} else {
writeJsonQuiet(w, r, http.StatusOK, syncStat)
}
}
func (vs *VolumeServer) getVolumeIndexContentHandler(w http.ResponseWriter, r *http.Request) {
v, err := vs.getVolume("volume", r)
if v == nil {
writeJsonError(w, r, http.StatusBadRequest, err)
return
}
content, err := v.IndexFileContent()
if err != nil {
writeJsonError(w, r, http.StatusInternalServerError, err)
return
}
w.Write(content)
}
func (vs *VolumeServer) getVolumeDataContentHandler(w http.ResponseWriter, r *http.Request) {
v, err := vs.getVolume("volume", r)
if v == nil {
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("Not Found volume: %v", err))
return
}
if int(v.SuperBlock.CompactRevision) != util.ParseInt(r.FormValue("revision"), 0) {
writeJsonError(w, r, http.StatusExpectationFailed, fmt.Errorf("Requested Volume Revision is %s, but current revision is %d", r.FormValue("revision"), v.SuperBlock.CompactRevision))
return
}
offset := uint32(util.ParseUint64(r.FormValue("offset"), 0))
size := uint32(util.ParseUint64(r.FormValue("size"), 0))
content, err := storage.ReadNeedleBlob(v.DataFile(), int64(offset)*storage.NeedlePaddingSize, size)
if err != nil {
writeJsonError(w, r, http.StatusInternalServerError, err)
return
}
id := util.ParseUint64(r.FormValue("id"), 0)
n := new(storage.Needle)
n.ParseNeedleHeader(content)
if id != n.Id {
writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("Expected file entry id %d, but found %d", id, n.Id))
return
}
w.Write(content)
}
func (vs *VolumeServer) getVolume(volumeParameterName string, r *http.Request) (*storage.Volume, error) {
volumeIdString := r.FormValue(volumeParameterName)
if volumeIdString == "" {
err := fmt.Errorf("Empty Volume Id: Need to pass in %s=the_volume_id.", volumeParameterName)
return nil, err
}
vid, err := storage.NewVolumeId(volumeIdString)
if err != nil {
err = fmt.Errorf("Volume Id %s is not a valid unsigned integer", volumeIdString)
return nil, err
}
v := vs.store.GetVolume(vid)
if v == nil {
return nil, fmt.Errorf("Not Found Volume Id %s: %d", volumeIdString, vid)
}
return v, nil
}

4
go/weed/weed_server/volume_server_handlers_write.go

@ -53,7 +53,7 @@ func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
glog.V(2).Infoln("deleting", n) glog.V(2).Infoln("deleting", n)
cookie := n.Cookie cookie := n.Cookie
count, ok := vs.store.Read(volumeId, n)
count, ok := vs.store.ReadVolumeNeedle(volumeId, n)
if ok != nil { if ok != nil {
m := make(map[string]uint32) m := make(map[string]uint32)
@ -94,7 +94,7 @@ func (vs *VolumeServer) batchDeleteHandler(w http.ResponseWriter, r *http.Reques
n.ParsePath(id_cookie) n.ParsePath(id_cookie)
glog.V(4).Infoln("batch deleting", n) glog.V(4).Infoln("batch deleting", n)
cookie := n.Cookie cookie := n.Cookie
if _, err := vs.store.Read(volumeId, n); err != nil {
if _, err := vs.store.ReadVolumeNeedle(volumeId, n); err != nil {
ret = append(ret, operation.DeleteResult{Fid: fid, Error: err.Error()}) ret = append(ret, operation.DeleteResult{Fid: fid, Error: err.Error()})
continue continue
} }

Loading…
Cancel
Save