Browse Source
re-factoring volume file scanning code
re-factoring volume file scanning code
add export function fix bugs on deleted byte countingpull/2/head
Chris Lu
12 years ago
11 changed files with 259 additions and 131 deletions
-
120weed-fs/src/cmd/weed/export.go
-
50weed-fs/src/cmd/weed/fix.go
-
25weed-fs/src/cmd/weed/master.go
-
3weed-fs/src/cmd/weed/upload.go
-
1weed-fs/src/cmd/weed/weed.go
-
1weed-fs/src/pkg/storage/needle.go
-
12weed-fs/src/pkg/storage/needle_map.go
-
5weed-fs/src/pkg/storage/needle_read_write.go
-
8weed-fs/src/pkg/storage/store.go
-
153weed-fs/src/pkg/storage/volume.go
-
2weed-fs/src/pkg/topology/topology_compact.go
@ -0,0 +1,120 @@ |
|||
package main |
|||
|
|||
import ( |
|||
"archive/tar" |
|||
"fmt" |
|||
"log" |
|||
"os" |
|||
"path" |
|||
"pkg/directory" |
|||
"pkg/storage" |
|||
"strconv" |
|||
"strings" |
|||
"time" |
|||
) |
|||
|
|||
func init() { |
|||
cmdExport.Run = runExport // break init cycle
|
|||
cmdExport.IsDebug = cmdExport.Flag.Bool("debug", false, "enable debug mode") |
|||
} |
|||
|
|||
var cmdExport = &Command{ |
|||
UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar", |
|||
Short: "export files out of one volume", |
|||
Long: `export all files in a volume |
|||
|
|||
`, |
|||
} |
|||
|
|||
var ( |
|||
exportVolumePath = cmdExport.Flag.String("dir", "/tmp", "data directory to store files") |
|||
exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.") |
|||
dest = cmdExport.Flag.String("o", "", "output tar file name") |
|||
tarFh *tar.Writer |
|||
tarHeader tar.Header |
|||
counter int |
|||
) |
|||
|
|||
func runExport(cmd *Command, args []string) bool { |
|||
|
|||
if *exportVolumeId == -1 { |
|||
return false |
|||
} |
|||
|
|||
var err error |
|||
if strings.HasSuffix(*dest, ".tar") { |
|||
var fh *os.File |
|||
if *dest == "" { |
|||
fh = os.Stdout |
|||
} else { |
|||
if fh, err = os.Create(*dest); err != nil { |
|||
log.Fatalf("cannot open output tar %s: %s", *dest, err) |
|||
} |
|||
} |
|||
defer fh.Close() |
|||
tarFh = tar.NewWriter(fh) |
|||
defer tarFh.Close() |
|||
t := time.Now() |
|||
tarHeader = tar.Header{Mode: 0644, |
|||
ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(), |
|||
Typeflag: tar.TypeReg, |
|||
AccessTime: t, ChangeTime: t} |
|||
} |
|||
|
|||
fileName := strconv.Itoa(*exportVolumeId) |
|||
vid := storage.VolumeId(*exportVolumeId) |
|||
indexFile, err := os.OpenFile(path.Join(*exportVolumePath, fileName+".idx"), os.O_RDONLY, 0644) |
|||
if err != nil { |
|||
log.Fatalf("Create Volume Index [ERROR] %s\n", err) |
|||
} |
|||
defer indexFile.Close() |
|||
|
|||
nm := storage.LoadNeedleMap(indexFile) |
|||
|
|||
err = storage.ScanVolumeFile(*exportVolumePath, vid, func(superBlock storage.SuperBlock) error { |
|||
return nil |
|||
}, func(n *storage.Needle, offset uint32) error { |
|||
debug("key", n.Id, "offset", offset, "size", n.Size, "disk_size", n.DiskSize(), "gzip", n.IsGzipped()) |
|||
nv, ok := nm.Get(n.Id) |
|||
if ok && nv.Size > 0 { |
|||
return walker(vid, n) |
|||
} else { |
|||
if !ok { |
|||
debug("This seems deleted", n.Id) |
|||
} else { |
|||
debug("Id", n.Id, "size", n.Size) |
|||
} |
|||
} |
|||
return nil |
|||
}) |
|||
if err != nil { |
|||
log.Fatalf("Export Volume File [ERROR] %s\n", err) |
|||
} |
|||
return true |
|||
} |
|||
|
|||
func walker(vid storage.VolumeId, n *storage.Needle) (err error) { |
|||
nm := fmt.Sprintf("%s/%d#%s", n.Mime, n.Id, n.Name) |
|||
if n.IsGzipped() && path.Ext(nm) != ".gz" { |
|||
nm = nm + ".gz" |
|||
} |
|||
if tarFh != nil { |
|||
tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data)) |
|||
if err = tarFh.WriteHeader(&tarHeader); err != nil { |
|||
return err |
|||
} |
|||
_, err = tarFh.Write(n.Data) |
|||
} else { |
|||
fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n", |
|||
directory.NewFileId(vid, n.Id, n.Cookie).String(), |
|||
n.Name, |
|||
n.DataSize, |
|||
n.IsGzipped(), |
|||
n.Mime, |
|||
) |
|||
} |
|||
if err == nil { |
|||
counter++ |
|||
} |
|||
return |
|||
} |
Write
Preview
Loading…
Cancel
Save
Reference in new issue