Browse Source

re-factoring volume file scanning code

add export function
fix bugs on deleted byte counting
pull/2/head
Chris Lu 12 years ago
parent
commit
81b9175c7a
  1. 120
      weed-fs/src/cmd/weed/export.go
  2. 50
      weed-fs/src/cmd/weed/fix.go
  3. 25
      weed-fs/src/cmd/weed/master.go
  4. 3
      weed-fs/src/cmd/weed/upload.go
  5. 1
      weed-fs/src/cmd/weed/weed.go
  6. 1
      weed-fs/src/pkg/storage/needle.go
  7. 12
      weed-fs/src/pkg/storage/needle_map.go
  8. 5
      weed-fs/src/pkg/storage/needle_read_write.go
  9. 8
      weed-fs/src/pkg/storage/store.go
  10. 163
      weed-fs/src/pkg/storage/volume.go
  11. 2
      weed-fs/src/pkg/topology/topology_compact.go

120
weed-fs/src/cmd/weed/export.go

@ -0,0 +1,120 @@
package main
import (
"archive/tar"
"fmt"
"log"
"os"
"path"
"pkg/directory"
"pkg/storage"
"strconv"
"strings"
"time"
)
func init() {
cmdExport.Run = runExport // break init cycle
cmdExport.IsDebug = cmdExport.Flag.Bool("debug", false, "enable debug mode")
}
var cmdExport = &Command{
UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar",
Short: "export files out of one volume",
Long: `export all files in a volume
`,
}
var (
exportVolumePath = cmdExport.Flag.String("dir", "/tmp", "data directory to store files")
exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
dest = cmdExport.Flag.String("o", "", "output tar file name")
tarFh *tar.Writer
tarHeader tar.Header
counter int
)
func runExport(cmd *Command, args []string) bool {
if *exportVolumeId == -1 {
return false
}
var err error
if strings.HasSuffix(*dest, ".tar") {
var fh *os.File
if *dest == "" {
fh = os.Stdout
} else {
if fh, err = os.Create(*dest); err != nil {
log.Fatalf("cannot open output tar %s: %s", *dest, err)
}
}
defer fh.Close()
tarFh = tar.NewWriter(fh)
defer tarFh.Close()
t := time.Now()
tarHeader = tar.Header{Mode: 0644,
ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(),
Typeflag: tar.TypeReg,
AccessTime: t, ChangeTime: t}
}
fileName := strconv.Itoa(*exportVolumeId)
vid := storage.VolumeId(*exportVolumeId)
indexFile, err := os.OpenFile(path.Join(*exportVolumePath, fileName+".idx"), os.O_RDONLY, 0644)
if err != nil {
log.Fatalf("Create Volume Index [ERROR] %s\n", err)
}
defer indexFile.Close()
nm := storage.LoadNeedleMap(indexFile)
err = storage.ScanVolumeFile(*exportVolumePath, vid, func(superBlock storage.SuperBlock) error {
return nil
}, func(n *storage.Needle, offset uint32) error {
debug("key", n.Id, "offset", offset, "size", n.Size, "disk_size", n.DiskSize(), "gzip", n.IsGzipped())
nv, ok := nm.Get(n.Id)
if ok && nv.Size > 0 {
return walker(vid, n)
} else {
if !ok {
debug("This seems deleted", n.Id)
} else {
debug("Id", n.Id, "size", n.Size)
}
}
return nil
})
if err != nil {
log.Fatalf("Export Volume File [ERROR] %s\n", err)
}
return true
}
func walker(vid storage.VolumeId, n *storage.Needle) (err error) {
nm := fmt.Sprintf("%s/%d#%s", n.Mime, n.Id, n.Name)
if n.IsGzipped() && path.Ext(nm) != ".gz" {
nm = nm + ".gz"
}
if tarFh != nil {
tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data))
if err = tarFh.WriteHeader(&tarHeader); err != nil {
return err
}
_, err = tarFh.Write(n.Data)
} else {
fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n",
directory.NewFileId(vid, n.Id, n.Cookie).String(),
n.Name,
n.DataSize,
n.IsGzipped(),
n.Mime,
)
}
if err == nil {
counter++
}
return
}

50
weed-fs/src/cmd/weed/fix.go

@ -33,46 +33,32 @@ func runFix(cmd *Command, args []string) bool {
} }
fileName := strconv.Itoa(*fixVolumeId) fileName := strconv.Itoa(*fixVolumeId)
dataFile, e := os.OpenFile(path.Join(*fixVolumePath, fileName+".dat"), os.O_RDONLY, 0644)
if e != nil {
log.Fatalf("Read Volume [ERROR] %s\n", e)
}
defer dataFile.Close()
indexFile, ie := os.OpenFile(path.Join(*fixVolumePath, fileName+".idx"), os.O_WRONLY|os.O_CREATE, 0644)
if ie != nil {
log.Fatalf("Create Volume Index [ERROR] %s\n", ie)
indexFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".idx"), os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
log.Fatalf("Create Volume Index [ERROR] %s\n", err)
} }
defer indexFile.Close() defer indexFile.Close()
dataFile.Seek(0, 0)
header := make([]byte, storage.SuperBlockSize)
if _, e := dataFile.Read(header); e != nil {
log.Fatalf("cannot read superblock: %s", e)
}
ver, _, e := storage.ParseSuperBlock(header)
if e != nil {
log.Fatalf("error parsing superblock: %s", e)
}
n, rest, e := storage.ReadNeedleHeader(dataFile, ver)
if e != nil {
log.Fatalf("error reading needle header: %s", e)
}
dataFile.Seek(int64(rest), 1)
nm := storage.NewNeedleMap(indexFile) nm := storage.NewNeedleMap(indexFile)
offset := uint32(storage.SuperBlockSize)
for n != nil {
debug("key", n.Id, "volume offset", offset, "data_size", n.Size, "rest", rest)
defer nm.Close()
vid := storage.VolumeId(*fixVolumeId)
err = storage.ScanVolumeFile(*fixVolumePath, vid, func(superBlock storage.SuperBlock) error {
return nil
}, func(n *storage.Needle, offset uint32) error {
debug("key", n.Id, "offset", offset, "size", n.Size, "disk_size", n.DiskSize(), "gzip", n.IsGzipped())
if n.Size > 0 { if n.Size > 0 {
count, pe := nm.Put(n.Id, offset/storage.NeedlePaddingSize, n.Size) count, pe := nm.Put(n.Id, offset/storage.NeedlePaddingSize, n.Size)
debug("saved", count, "with error", pe) debug("saved", count, "with error", pe)
}else{
debug("skipping deleted file ...")
nm.Delete(n.Id)
} }
offset += rest + 16
if n, rest, e = storage.ReadNeedleHeader(dataFile, ver); e != nil {
log.Fatalf("error reading needle header: %s", e)
}
dataFile.Seek(int64(rest), 1)
return nil
})
if err != nil {
log.Fatalf("Export Volume File [ERROR] %s\n", err)
} }
return true return true
} }

25
weed-fs/src/cmd/weed/master.go

@ -128,24 +128,13 @@ func dirStatusHandler(w http.ResponseWriter, r *http.Request) {
} }
func volumeVacuumHandler(w http.ResponseWriter, r *http.Request) { func volumeVacuumHandler(w http.ResponseWriter, r *http.Request) {
count := 0
rt, err := storage.NewReplicationTypeFromString(r.FormValue("replication"))
if err == nil {
if count, err = strconv.Atoi(r.FormValue("count")); err == nil {
if topo.FreeSpace() < count*rt.GetCopyCount() {
err = errors.New("Only " + strconv.Itoa(topo.FreeSpace()) + " volumes left! Not enough for " + strconv.Itoa(count*rt.GetCopyCount()))
} else {
count, err = vg.GrowByCountAndType(count, rt, topo)
}
}
}
if err != nil {
w.WriteHeader(http.StatusNotAcceptable)
writeJson(w, r, map[string]string{"error": err.Error()})
} else {
w.WriteHeader(http.StatusNotAcceptable)
writeJson(w, r, map[string]interface{}{"count": count})
gcThreshold := r.FormValue("garbageThreshold")
if gcThreshold == "" {
gcThreshold = *garbageThreshold
} }
debug("garbageThreshold =", gcThreshold)
topo.Vacuum(gcThreshold)
dirStatusHandler(w, r)
} }
func volumeGrowHandler(w http.ResponseWriter, r *http.Request) { func volumeGrowHandler(w http.ResponseWriter, r *http.Request) {
@ -208,6 +197,8 @@ func runMaster(cmd *Command, args []string) bool {
http.HandleFunc("/dir/status", dirStatusHandler) http.HandleFunc("/dir/status", dirStatusHandler)
http.HandleFunc("/vol/grow", volumeGrowHandler) http.HandleFunc("/vol/grow", volumeGrowHandler)
http.HandleFunc("/vol/status", volumeStatusHandler) http.HandleFunc("/vol/status", volumeStatusHandler)
http.HandleFunc("/vol/vacuum", volumeVacuumHandler)
http.HandleFunc("/", redirectHandler) http.HandleFunc("/", redirectHandler)
topo.StartRefreshWritableVolumes(*garbageThreshold) topo.StartRefreshWritableVolumes(*garbageThreshold)

3
weed-fs/src/cmd/weed/upload.go

@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"net/url" "net/url"
"os" "os"
"path"
"pkg/operation" "pkg/operation"
"pkg/util" "pkg/util"
"strconv" "strconv"
@ -65,7 +66,7 @@ func upload(filename string, server string, fid string) (int, error) {
debug("Failed to open file:", filename) debug("Failed to open file:", filename)
return 0, err return 0, err
} }
ret, e := operation.Upload("http://"+server+"/"+fid, filename, fh)
ret, e := operation.Upload("http://"+server+"/"+fid, path.Base(filename), fh)
if e != nil { if e != nil {
return 0, e return 0, e
} }

1
weed-fs/src/cmd/weed/weed.go

@ -26,6 +26,7 @@ var commands = []*Command{
cmdShell, cmdShell,
cmdVersion, cmdVersion,
cmdVolume, cmdVolume,
cmdExport,
} }
var exitStatus = 0 var exitStatus = 0

1
weed-fs/src/pkg/storage/needle.go

@ -116,6 +116,7 @@ func (n *Needle) ParsePath(fid string) {
} }
} }
} }
func ParseKeyHash(key_hash_string string) (uint64, uint32) { func ParseKeyHash(key_hash_string string) (uint64, uint32) {
key_hash_bytes, khe := hex.DecodeString(key_hash_string) key_hash_bytes, khe := hex.DecodeString(key_hash_string)
key_hash_len := len(key_hash_bytes) key_hash_len := len(key_hash_bytes)

12
weed-fs/src/pkg/storage/needle_map.go

@ -1,7 +1,7 @@
package storage package storage
import ( import (
"log"
//"log"
"os" "os"
"pkg/util" "pkg/util"
) )
@ -36,10 +36,6 @@ func LoadNeedleMap(file *os.File) *NeedleMap {
nm := NewNeedleMap(file) nm := NewNeedleMap(file)
bytes := make([]byte, 16*RowsToRead) bytes := make([]byte, 16*RowsToRead)
count, e := nm.indexFile.Read(bytes) count, e := nm.indexFile.Read(bytes)
if count > 0 {
fstat, _ := file.Stat()
log.Println("Loading index file", fstat.Name(), "size", fstat.Size())
}
for count > 0 && e == nil { for count > 0 && e == nil {
for i := 0; i < count; i += 16 { for i := 0; i < count; i += 16 {
key := util.BytesToUint64(bytes[i : i+8]) key := util.BytesToUint64(bytes[i : i+8])
@ -55,10 +51,10 @@ func LoadNeedleMap(file *os.File) *NeedleMap {
nm.deletionByteCounter = nm.deletionByteCounter + uint64(oldSize) nm.deletionByteCounter = nm.deletionByteCounter + uint64(oldSize)
} }
} else { } else {
nm.m.Delete(Key(key))
//log.Println("removing key", key)
oldSize := nm.m.Delete(Key(key))
//log.Println("removing key", key, "offset", offset, "size", size, "oldSize", oldSize)
nm.deletionCounter++ nm.deletionCounter++
nm.deletionByteCounter = nm.deletionByteCounter + uint64(size)
nm.deletionByteCounter = nm.deletionByteCounter + uint64(oldSize)
} }
} }

5
weed-fs/src/pkg/storage/needle_read_write.go

@ -14,6 +14,10 @@ const (
FlagHasMime = 0x04 FlagHasMime = 0x04
) )
func (n *Needle) DiskSize() uint32 {
padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize)
return NeedleHeaderSize + n.Size + padding + NeedleChecksumSize
}
func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) {
switch version { switch version {
case Version1: case Version1:
@ -156,6 +160,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) {
n.Mime = bytes[index : index+int(n.MimeSize)] n.Mime = bytes[index : index+int(n.MimeSize)]
} }
} }
func ReadNeedleHeader(r *os.File, version Version) (n *Needle, bodyLength uint32, err error) { func ReadNeedleHeader(r *os.File, version Version) (n *Needle, bodyLength uint32, err error) {
n = new(Needle) n = new(Needle)
if version == Version1 || version == Version2 { if version == Version1 || version == Version2 {

8
weed-fs/src/pkg/storage/store.go

@ -109,7 +109,7 @@ func (s *Store) loadExistingVolumes() {
if s.volumes[vid] == nil { if s.volumes[vid] == nil {
if v, e := NewVolume(s.dir, vid, CopyNil); e == nil { if v, e := NewVolume(s.dir, vid, CopyNil); e == nil {
s.volumes[vid] = v s.volumes[vid] = v
log.Println("In dir", s.dir, "read volume =", vid, "replicationType =", v.replicaType, "version =", v.version, "size =", v.Size())
log.Println("In dir", s.dir, "read volume =", vid, "replicationType =", v.ReplicaType, "version =", v.Version(), "size =", v.Size())
} }
} }
} }
@ -121,7 +121,8 @@ func (s *Store) Status() []*VolumeInfo {
var stats []*VolumeInfo var stats []*VolumeInfo
for k, v := range s.volumes { for k, v := range s.volumes {
s := new(VolumeInfo) s := new(VolumeInfo)
s.Id, s.Size, s.RepType, s.Version, s.FileCount, s.DeleteCount, s.DeletedByteCount = VolumeId(k), v.ContentSize(), v.replicaType, v.Version(), v.nm.fileCounter, v.nm.deletionCounter, v.nm.deletionByteCounter
s.Id, s.Size, s.RepType, s.Version, s.FileCount, s.DeleteCount, s.DeletedByteCount =
VolumeId(k), v.ContentSize(), v.ReplicaType, v.Version(), v.nm.fileCounter, v.nm.deletionCounter, v.nm.deletionByteCounter
stats = append(stats, s) stats = append(stats, s)
} }
return stats return stats
@ -138,7 +139,8 @@ func (s *Store) Join() error {
stats := new([]*VolumeInfo) stats := new([]*VolumeInfo)
for k, v := range s.volumes { for k, v := range s.volumes {
s := new(VolumeInfo) s := new(VolumeInfo)
s.Id, s.Size, s.RepType, s.Version, s.FileCount, s.DeleteCount, s.DeletedByteCount = VolumeId(k), uint64(v.Size()), v.replicaType, v.Version(), v.nm.fileCounter, v.nm.deletionCounter, v.nm.deletionByteCounter
s.Id, s.Size, s.RepType, s.Version, s.FileCount, s.DeleteCount, s.DeletedByteCount =
VolumeId(k), uint64(v.Size()), v.ReplicaType, v.Version(), v.nm.fileCounter, v.nm.deletionCounter, v.nm.deletionByteCounter
*stats = append(*stats, s) *stats = append(*stats, s)
} }
bytes, _ := json.Marshal(stats) bytes, _ := json.Marshal(stats)

163
weed-fs/src/pkg/storage/volume.go

@ -3,6 +3,7 @@ package storage
import ( import (
"errors" "errors"
"fmt" "fmt"
"io"
"os" "os"
"path" "path"
"sync" "sync"
@ -12,46 +13,66 @@ const (
SuperBlockSize = 8 SuperBlockSize = 8
) )
type SuperBlock struct {
Version Version
ReplicaType ReplicationType
}
func (s *SuperBlock) Bytes() []byte {
header := make([]byte, SuperBlockSize)
header[0] = byte(s.Version)
header[1] = s.ReplicaType.Byte()
return header
}
type Volume struct { type Volume struct {
Id VolumeId Id VolumeId
dir string dir string
dataFile *os.File dataFile *os.File
nm *NeedleMap nm *NeedleMap
version Version
replicaType ReplicationType
SuperBlock
accessLock sync.Mutex accessLock sync.Mutex
} }
func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume, e error) { func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume, e error) {
v = &Volume{dir: dirname, Id: id, replicaType: replicationType}
e = v.load()
v = &Volume{dir: dirname, Id: id}
v.SuperBlock = SuperBlock{ReplicaType: replicationType}
e = v.load(true)
return
}
func LoadVolumeOnly(dirname string, id VolumeId) (v *Volume, e error) {
v = &Volume{dir: dirname, Id: id}
v.SuperBlock = SuperBlock{ReplicaType: CopyNil}
e = v.load(false)
return return
} }
func (v *Volume) load() error {
func (v *Volume) load(alsoLoadIndex bool) error {
var e error var e error
fileName := path.Join(v.dir, v.Id.String()) fileName := path.Join(v.dir, v.Id.String())
v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644) v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
if e != nil { if e != nil {
return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e) return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
} }
if v.replicaType == CopyNil {
if v.ReplicaType == CopyNil {
if e = v.readSuperBlock(); e != nil { if e = v.readSuperBlock(); e != nil {
return e return e
} }
} else { } else {
v.maybeWriteSuperBlock() v.maybeWriteSuperBlock()
} }
indexFile, ie := os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644)
if ie != nil {
return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
if alsoLoadIndex {
indexFile, ie := os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644)
if ie != nil {
return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
}
v.nm = LoadNeedleMap(indexFile)
} }
v.nm = LoadNeedleMap(indexFile)
return nil return nil
} }
func (v *Volume) Version() Version { func (v *Volume) Version() Version {
return v.version
return v.SuperBlock.Version
} }
func (v *Volume) Size() int64 { func (v *Volume) Size() int64 {
v.accessLock.Lock() v.accessLock.Lock()
@ -76,11 +97,8 @@ func (v *Volume) maybeWriteSuperBlock() {
return return
} }
if stat.Size() == 0 { if stat.Size() == 0 {
v.version = CurrentVersion
header := make([]byte, SuperBlockSize)
header[0] = byte(v.version)
header[1] = v.replicaType.Byte()
v.dataFile.Write(header)
v.SuperBlock.Version = CurrentVersion
v.dataFile.Write(v.SuperBlock.Bytes())
} }
} }
func (v *Volume) readSuperBlock() (err error) { func (v *Volume) readSuperBlock() (err error) {
@ -89,18 +107,18 @@ func (v *Volume) readSuperBlock() (err error) {
if _, e := v.dataFile.Read(header); e != nil { if _, e := v.dataFile.Read(header); e != nil {
return fmt.Errorf("cannot read superblock: %s", e) return fmt.Errorf("cannot read superblock: %s", e)
} }
v.version, v.replicaType, err = ParseSuperBlock(header)
v.SuperBlock, err = ParseSuperBlock(header)
return err return err
} }
func ParseSuperBlock(header []byte) (version Version, replicaType ReplicationType, err error) {
version = Version(header[0])
if replicaType, err = NewReplicationTypeFromByte(header[1]); err != nil {
func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error) {
superBlock.Version = Version(header[0])
if superBlock.ReplicaType, err = NewReplicationTypeFromByte(header[1]); err != nil {
err = fmt.Errorf("cannot read replica type: %s", err) err = fmt.Errorf("cannot read replica type: %s", err)
} }
return return
} }
func (v *Volume) NeedToReplicate() bool { func (v *Volume) NeedToReplicate() bool {
return v.replicaType.GetCopyCount() > 1
return v.ReplicaType.GetCopyCount() > 1
} }
func (v *Volume) write(n *Needle) (size uint32, err error) { func (v *Volume) write(n *Needle) (size uint32, err error) {
@ -110,7 +128,7 @@ func (v *Volume) write(n *Needle) (size uint32, err error) {
if offset, err = v.dataFile.Seek(0, 2); err != nil { if offset, err = v.dataFile.Seek(0, 2); err != nil {
return return
} }
if size, err = n.Append(v.dataFile, v.version); err != nil {
if size, err = n.Append(v.dataFile, v.Version()); err != nil {
return return
} }
nv, ok := v.nm.Get(n.Id) nv, ok := v.nm.Get(n.Id)
@ -127,7 +145,7 @@ func (v *Volume) delete(n *Needle) (uint32, error) {
if ok { if ok {
v.nm.Delete(n.Id) v.nm.Delete(n.Id)
v.dataFile.Seek(int64(nv.Offset*NeedlePaddingSize), 0) v.dataFile.Seek(int64(nv.Offset*NeedlePaddingSize), 0)
_, err := n.Append(v.dataFile, v.version)
_, err := n.Append(v.dataFile, v.Version())
return nv.Size, err return nv.Size, err
} }
return 0, nil return 0, nil
@ -139,7 +157,7 @@ func (v *Volume) read(n *Needle) (int, error) {
nv, ok := v.nm.Get(n.Id) nv, ok := v.nm.Get(n.Id)
if ok && nv.Offset > 0 { if ok && nv.Offset > 0 {
v.dataFile.Seek(int64(nv.Offset)*NeedlePaddingSize, 0) v.dataFile.Seek(int64(nv.Offset)*NeedlePaddingSize, 0)
return n.Read(v.dataFile, nv.Size, v.version)
return n.Read(v.dataFile, nv.Size, v.Version())
} }
return -1, errors.New("Not Found") return -1, errors.New("Not Found")
} }
@ -153,7 +171,7 @@ func (v *Volume) compact() error {
defer v.accessLock.Unlock() defer v.accessLock.Unlock()
filePath := path.Join(v.dir, v.Id.String()) filePath := path.Join(v.dir, v.Id.String())
return v.copyDataAndGenerateIndexFile(filePath+".dat", filePath+".cpd", filePath+".cpx")
return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx")
} }
func (v *Volume) commitCompact() error { func (v *Volume) commitCompact() error {
v.accessLock.Lock() v.accessLock.Lock()
@ -166,83 +184,90 @@ func (v *Volume) commitCompact() error {
if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpx"), path.Join(v.dir, v.Id.String()+".idx")); e != nil { if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpx"), path.Join(v.dir, v.Id.String()+".idx")); e != nil {
return e return e
} }
if e = v.load(); e != nil {
if e = v.load(true); e != nil {
return e return e
} }
return nil return nil
} }
func (v *Volume) copyDataAndGenerateIndexFile(srcName, dstName, idxName string) (err error) {
var (
src, dst, idx *os.File
)
if src, err = os.OpenFile(srcName, os.O_RDONLY, 0644); err != nil {
func ScanVolumeFile(dirname string, id VolumeId,
visitSuperBlock func(SuperBlock) error,
visitNeedle func(n *Needle, offset uint32) error) (err error) {
var v *Volume
if v, err = LoadVolumeOnly(dirname, id); err != nil {
return return
} }
defer src.Close()
if dst, err = os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644); err != nil {
if err = visitSuperBlock(v.SuperBlock); err != nil {
return return
} }
defer dst.Close()
if idx, err = os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE, 0644); err != nil {
return
}
defer idx.Close()
version := v.Version()
src.Seek(0, 0)
header := make([]byte, SuperBlockSize)
if _, err = src.Read(header); err == nil {
_, err = dst.Write(header)
offset := uint32(SuperBlockSize)
n, rest, e := ReadNeedleHeader(v.dataFile, version)
if e != nil {
err = fmt.Errorf("cannot read needle header: %s", e)
return
} }
if err != nil {
return err
for n != nil {
if err = n.ReadNeedleBody(v.dataFile, version, rest); err != nil {
err = fmt.Errorf("cannot read needle body: %s", err)
return
}
if err = visitNeedle(n, offset); err != nil {
return
}
offset += NeedleHeaderSize + rest
if n, rest, err = ReadNeedleHeader(v.dataFile, version); err != nil {
if err == io.EOF {
return nil
}
return fmt.Errorf("cannot read needle header: %s", err)
}
} }
version, _, e := ParseSuperBlock(header)
if e != nil {
err = fmt.Errorf("cannot parse superblock: %s", e)
return
}
func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err error) {
var (
dst, idx *os.File
)
if dst, err = os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644); err != nil {
return return
} }
defer dst.Close()
n, rest, e := ReadNeedleHeader(src, version)
if e != nil {
err = fmt.Errorf("cannot read needle header: %s", e)
if idx, err = os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE, 0644); err != nil {
return return
} }
defer idx.Close()
nm := NewNeedleMap(idx) nm := NewNeedleMap(idx)
old_offset := uint32(SuperBlockSize)
new_offset := uint32(SuperBlockSize) new_offset := uint32(SuperBlockSize)
for n != nil {
err = ScanVolumeFile(v.dir, v.Id, func(superBlock SuperBlock) error {
_, err = dst.Write(superBlock.Bytes())
return err
}, func(n *Needle, offset uint32) error {
nv, ok := v.nm.Get(n.Id) nv, ok := v.nm.Get(n.Id)
//log.Println("file size is", n.Size, "rest", rest) //log.Println("file size is", n.Size, "rest", rest)
if !ok || nv.Offset*NeedlePaddingSize != old_offset {
src.Seek(int64(rest), 1)
} else {
if ok && nv.Offset*NeedlePaddingSize == offset {
if nv.Size > 0 { if nv.Size > 0 {
if _, err = nm.Put(n.Id, new_offset/NeedlePaddingSize, n.Size); err != nil { if _, err = nm.Put(n.Id, new_offset/NeedlePaddingSize, n.Size); err != nil {
return fmt.Errorf("cannot put needle: %s", err) return fmt.Errorf("cannot put needle: %s", err)
} }
if err = n.ReadNeedleBody(src, version, rest); err != nil {
return fmt.Errorf("cannot read needle body: %s", err)
}
if _, err = n.Append(dst, v.version); err != nil {
if _, err = n.Append(dst, v.Version()); err != nil {
return fmt.Errorf("cannot append needle: %s", err) return fmt.Errorf("cannot append needle: %s", err)
} }
new_offset += rest + NeedleHeaderSize
new_offset += n.DiskSize()
//log.Println("saving key", n.Id, "volume offset", old_offset, "=>", new_offset, "data_size", n.Size, "rest", rest) //log.Println("saving key", n.Id, "volume offset", old_offset, "=>", new_offset, "data_size", n.Size, "rest", rest)
} else {
src.Seek(int64(rest), 1)
} }
} }
old_offset += rest + NeedleHeaderSize
if n, rest, err = ReadNeedleHeader(src, version); err != nil {
return fmt.Errorf("cannot read needle header: %s", err)
}
}
return nil
})
return nil
return
} }
func (v *Volume) ContentSize() uint64 { func (v *Volume) ContentSize() uint64 {
return v.nm.fileByteCounter return v.nm.fileByteCounter

2
weed-fs/src/pkg/topology/topology_compact.go

@ -19,7 +19,7 @@ func batchVacuumVolumeCheck(vl *VolumeLayout, vid storage.VolumeId, locationlist
//fmt.Println(index, "Error when checking vacuuming", vid, "on", url, e) //fmt.Println(index, "Error when checking vacuuming", vid, "on", url, e)
ch <- false ch <- false
} else { } else {
//fmt.Println(index, "Checked vacuuming", vid, "on", url)
//fmt.Println(index, "Checked vacuuming", vid, "on", url, "needVacuum", ret)
ch <- ret ch <- ret
} }
}(index, dn.Url(), vid) }(index, dn.Url(), vid)

Loading…
Cancel
Save