Browse Source
re-factoring volume file scanning code
re-factoring volume file scanning code
add export function fix bugs on deleted byte countingpull/2/head
Chris Lu
12 years ago
11 changed files with 259 additions and 131 deletions
-
120weed-fs/src/cmd/weed/export.go
-
50weed-fs/src/cmd/weed/fix.go
-
25weed-fs/src/cmd/weed/master.go
-
3weed-fs/src/cmd/weed/upload.go
-
1weed-fs/src/cmd/weed/weed.go
-
1weed-fs/src/pkg/storage/needle.go
-
12weed-fs/src/pkg/storage/needle_map.go
-
5weed-fs/src/pkg/storage/needle_read_write.go
-
8weed-fs/src/pkg/storage/store.go
-
153weed-fs/src/pkg/storage/volume.go
-
2weed-fs/src/pkg/topology/topology_compact.go
@ -0,0 +1,120 @@ |
|||||
|
package main |
||||
|
|
||||
|
import ( |
||||
|
"archive/tar" |
||||
|
"fmt" |
||||
|
"log" |
||||
|
"os" |
||||
|
"path" |
||||
|
"pkg/directory" |
||||
|
"pkg/storage" |
||||
|
"strconv" |
||||
|
"strings" |
||||
|
"time" |
||||
|
) |
||||
|
|
||||
|
func init() { |
||||
|
cmdExport.Run = runExport // break init cycle
|
||||
|
cmdExport.IsDebug = cmdExport.Flag.Bool("debug", false, "enable debug mode") |
||||
|
} |
||||
|
|
||||
|
var cmdExport = &Command{ |
||||
|
UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar", |
||||
|
Short: "export files out of one volume", |
||||
|
Long: `export all files in a volume |
||||
|
|
||||
|
`, |
||||
|
} |
||||
|
|
||||
|
var ( |
||||
|
exportVolumePath = cmdExport.Flag.String("dir", "/tmp", "data directory to store files") |
||||
|
exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.") |
||||
|
dest = cmdExport.Flag.String("o", "", "output tar file name") |
||||
|
tarFh *tar.Writer |
||||
|
tarHeader tar.Header |
||||
|
counter int |
||||
|
) |
||||
|
|
||||
|
func runExport(cmd *Command, args []string) bool { |
||||
|
|
||||
|
if *exportVolumeId == -1 { |
||||
|
return false |
||||
|
} |
||||
|
|
||||
|
var err error |
||||
|
if strings.HasSuffix(*dest, ".tar") { |
||||
|
var fh *os.File |
||||
|
if *dest == "" { |
||||
|
fh = os.Stdout |
||||
|
} else { |
||||
|
if fh, err = os.Create(*dest); err != nil { |
||||
|
log.Fatalf("cannot open output tar %s: %s", *dest, err) |
||||
|
} |
||||
|
} |
||||
|
defer fh.Close() |
||||
|
tarFh = tar.NewWriter(fh) |
||||
|
defer tarFh.Close() |
||||
|
t := time.Now() |
||||
|
tarHeader = tar.Header{Mode: 0644, |
||||
|
ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(), |
||||
|
Typeflag: tar.TypeReg, |
||||
|
AccessTime: t, ChangeTime: t} |
||||
|
} |
||||
|
|
||||
|
fileName := strconv.Itoa(*exportVolumeId) |
||||
|
vid := storage.VolumeId(*exportVolumeId) |
||||
|
indexFile, err := os.OpenFile(path.Join(*exportVolumePath, fileName+".idx"), os.O_RDONLY, 0644) |
||||
|
if err != nil { |
||||
|
log.Fatalf("Create Volume Index [ERROR] %s\n", err) |
||||
|
} |
||||
|
defer indexFile.Close() |
||||
|
|
||||
|
nm := storage.LoadNeedleMap(indexFile) |
||||
|
|
||||
|
err = storage.ScanVolumeFile(*exportVolumePath, vid, func(superBlock storage.SuperBlock) error { |
||||
|
return nil |
||||
|
}, func(n *storage.Needle, offset uint32) error { |
||||
|
debug("key", n.Id, "offset", offset, "size", n.Size, "disk_size", n.DiskSize(), "gzip", n.IsGzipped()) |
||||
|
nv, ok := nm.Get(n.Id) |
||||
|
if ok && nv.Size > 0 { |
||||
|
return walker(vid, n) |
||||
|
} else { |
||||
|
if !ok { |
||||
|
debug("This seems deleted", n.Id) |
||||
|
} else { |
||||
|
debug("Id", n.Id, "size", n.Size) |
||||
|
} |
||||
|
} |
||||
|
return nil |
||||
|
}) |
||||
|
if err != nil { |
||||
|
log.Fatalf("Export Volume File [ERROR] %s\n", err) |
||||
|
} |
||||
|
return true |
||||
|
} |
||||
|
|
||||
|
func walker(vid storage.VolumeId, n *storage.Needle) (err error) { |
||||
|
nm := fmt.Sprintf("%s/%d#%s", n.Mime, n.Id, n.Name) |
||||
|
if n.IsGzipped() && path.Ext(nm) != ".gz" { |
||||
|
nm = nm + ".gz" |
||||
|
} |
||||
|
if tarFh != nil { |
||||
|
tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data)) |
||||
|
if err = tarFh.WriteHeader(&tarHeader); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
_, err = tarFh.Write(n.Data) |
||||
|
} else { |
||||
|
fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n", |
||||
|
directory.NewFileId(vid, n.Id, n.Cookie).String(), |
||||
|
n.Name, |
||||
|
n.DataSize, |
||||
|
n.IsGzipped(), |
||||
|
n.Mime, |
||||
|
) |
||||
|
} |
||||
|
if err == nil { |
||||
|
counter++ |
||||
|
} |
||||
|
return |
||||
|
} |
Write
Preview
Loading…
Cancel
Save
Reference in new issue