Chris Lu
12 years ago
45 changed files with 1086 additions and 555 deletions
-
96weed-fs/src/cmd/dump/main.go
-
1weed-fs/src/cmd/weed/command.go
-
27weed-fs/src/cmd/weed/fix.go
-
10weed-fs/src/cmd/weed/master.go
-
9weed-fs/src/cmd/weed/shell.go
-
4weed-fs/src/cmd/weed/volume.go
-
1weed-fs/src/cmd/weed/weed.go
-
2weed-fs/src/pkg/directory/file_id.go
-
2weed-fs/src/pkg/operation/delete_content.go
-
4weed-fs/src/pkg/operation/lookup_volume_id.go
-
2weed-fs/src/pkg/operation/upload_content.go
-
1weed-fs/src/pkg/replication/volume_growth_test.go
-
2weed-fs/src/pkg/sequence/sequence.go
-
112weed-fs/src/pkg/storage/cdb_map.go
-
24weed-fs/src/pkg/storage/compact_map.go
-
2weed-fs/src/pkg/storage/compact_map_perf_test.go
-
38weed-fs/src/pkg/storage/compress.go
-
4weed-fs/src/pkg/storage/needle.go
-
151weed-fs/src/pkg/storage/needle_map.go
-
9weed-fs/src/pkg/storage/needle_read_write.go
-
39weed-fs/src/pkg/storage/store.go
-
83weed-fs/src/pkg/storage/volume.go
-
1weed-fs/src/pkg/storage/volume_id.go
-
1weed-fs/src/pkg/storage/volume_info.go
-
3weed-fs/src/pkg/storage/volume_version.go
-
3weed-fs/src/pkg/topology/data_center.go
-
2weed-fs/src/pkg/topology/node_list_test.go
-
3weed-fs/src/pkg/topology/topology_map.go
-
41weed-fs/src/pkg/topology/volume_layout.go
-
1weed-fs/src/pkg/util/bytes.go
-
63weed-fs/src/pkg/util/file.go
@ -0,0 +1,96 @@ |
|||
// Copyright Tamás Gulácsi 2013 All rights reserved
|
|||
// Use of this source is governed by the same rules as the weed-fs library.
|
|||
// If this would be ambigous, than Apache License 2.0 has to be used.
|
|||
//
|
|||
// dump dumps the files of a volume to tar or unique files.
|
|||
// Each file will have id#mimetype#original_name file format
|
|||
|
|||
package main |
|||
|
|||
import ( |
|||
"archive/tar" |
|||
"bytes" |
|||
"flag" |
|||
"fmt" |
|||
// "io"
|
|||
"log" |
|||
"os" |
|||
"pkg/storage" |
|||
"strings" |
|||
"time" |
|||
) |
|||
|
|||
var ( |
|||
volumePath = flag.String("dir", "/tmp", "volume directory") |
|||
volumeId = flag.Int("id", 0, "volume Id") |
|||
dest = flag.String("out", "-", "output path. Produces tar if path ends with .tar; creates files otherwise.") |
|||
tarFh *tar.Writer |
|||
tarHeader tar.Header |
|||
counter int |
|||
) |
|||
|
|||
func main() { |
|||
var err error |
|||
|
|||
flag.Parse() |
|||
|
|||
if *dest == "-" { |
|||
*dest = "" |
|||
} |
|||
if *dest == "" || strings.HasSuffix(*dest, ".tar") { |
|||
var fh *os.File |
|||
if *dest == "" { |
|||
fh = os.Stdout |
|||
} else { |
|||
if fh, err = os.Create(*dest); err != nil { |
|||
log.Printf("cannot open output tar %s: %s", *dest, err) |
|||
return |
|||
} |
|||
} |
|||
defer fh.Close() |
|||
tarFh = tar.NewWriter(fh) |
|||
defer tarFh.Close() |
|||
t := time.Now() |
|||
tarHeader = tar.Header{Mode: 0644, |
|||
ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(), |
|||
Typeflag: tar.TypeReg, |
|||
AccessTime: t, ChangeTime: t} |
|||
} |
|||
|
|||
v, err := storage.NewVolume(*volumePath, storage.VolumeId(*volumeId), storage.CopyNil) |
|||
if v == nil || v.Version() == 0 || err != nil { |
|||
log.Printf("cannot load volume %d from %s (%s): %s", *volumeId, *volumePath, v, err) |
|||
return |
|||
} |
|||
log.Printf("volume: %s (ver. %d)", v, v.Version()) |
|||
if err := v.WalkValues(walker); err != nil { |
|||
log.Printf("error while walking: %s", err) |
|||
return |
|||
} |
|||
|
|||
log.Printf("%d files written.", counter) |
|||
} |
|||
|
|||
func walker(n *storage.Needle) (err error) { |
|||
// log.Printf("Id=%d Size=%d Name=%s mime=%s", n.Id, n.Size, n.Name, n.Mime)
|
|||
nm := fmt.Sprintf("%d#%s#%s", n.Id, bytes.Replace(n.Mime, []byte{'/'}, []byte{'_'}, -1), n.Name) |
|||
// log.Print(nm)
|
|||
if tarFh != nil { |
|||
tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data)) |
|||
if err = tarFh.WriteHeader(&tarHeader); err != nil { |
|||
return err |
|||
} |
|||
_, err = tarFh.Write(n.Data) |
|||
} else { |
|||
if fh, e := os.Create(*dest + "/" + nm); e != nil { |
|||
return e |
|||
} else { |
|||
defer fh.Close() |
|||
_, err = fh.Write(n.Data) |
|||
} |
|||
} |
|||
if err == nil { |
|||
counter++ |
|||
} |
|||
return |
|||
} |
|||
@ -0,0 +1,112 @@ |
|||
package storage |
|||
|
|||
import ( |
|||
"github.com/tgulacsi/go-cdb" |
|||
"io" |
|||
"log" |
|||
"os" |
|||
"pkg/util" |
|||
"strings" |
|||
) |
|||
|
|||
type CdbMap struct { |
|||
db *cdb.Cdb |
|||
transient []byte |
|||
Filename string |
|||
} |
|||
|
|||
// Opens the CDB file and servers as a needle map
|
|||
func NewCdbMap(filename string) (*CdbMap, error) { |
|||
m, err := cdb.Open(filename) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
return &CdbMap{db: m, transient: make([]byte, 8), |
|||
Filename: filename}, nil |
|||
} |
|||
|
|||
// writes the content of the index file to a CDB and returns that
|
|||
func NewCdbMapFromIndex(indexFile *os.File) (*CdbMap, error) { |
|||
nm := indexFile.Name() |
|||
nm = nm[:strings.LastIndex(nm, ".")+1] + "cdb" |
|||
|
|||
var ( |
|||
key uint64 |
|||
offset uint32 |
|||
ok bool |
|||
) |
|||
deleted := make(map[uint64]bool, 16) |
|||
gatherDeletes := func(buf []byte) error { |
|||
key = util.BytesToUint64(buf[:8]) |
|||
offset = util.BytesToUint32(buf[8:12]) |
|||
if offset > 0 { |
|||
if _, ok = deleted[key]; ok { //undelete
|
|||
delete(deleted, key) |
|||
} |
|||
} else { |
|||
deleted[key] = true |
|||
} |
|||
return nil |
|||
} |
|||
if err := readIndexFile(indexFile, gatherDeletes); err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
log.Printf("deleted: %s\nnm=%s", deleted, nm) |
|||
w, err := cdb.NewWriter(nm) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
iterFun := func(buf []byte) error { |
|||
key = util.BytesToUint64(buf[:8]) |
|||
log.Printf("iter key=%d", key) |
|||
if _, ok = deleted[key]; !ok { |
|||
w.PutPair(buf[:8], buf[8:16]) |
|||
} |
|||
return nil |
|||
} |
|||
indexFile.Seek(0, 0) |
|||
err = readIndexFile(indexFile, iterFun) |
|||
w.Close() |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
if err = util.SetFilePerm(nil, nm, 0444, -1); err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
return NewCdbMap(nm) |
|||
} |
|||
|
|||
func (m *CdbMap) Get(key Key) (element *NeedleValue, ok bool) { |
|||
util.Uint64toBytes(m.transient, uint64(key)) |
|||
data, err := m.db.Data(m.transient) |
|||
if err != nil { |
|||
if err == io.EOF { |
|||
return nil, false |
|||
} |
|||
log.Printf("error getting %s: %s", key, err) |
|||
return nil, false |
|||
} |
|||
return &NeedleValue{Key: key, |
|||
Offset: util.BytesToUint32(data[:4]), |
|||
Size: util.BytesToUint32(data[4:8]), |
|||
}, true |
|||
} |
|||
|
|||
func (m *CdbMap) Walk(pedestrian func(*NeedleValue) error) (err error) { |
|||
r, err := os.Open(m.Filename) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
defer r.Close() |
|||
|
|||
iterFunc := func(elt cdb.Element) error { |
|||
return pedestrian(&NeedleValue{ |
|||
Key: Key(util.BytesToUint64(elt.Key[:8])), |
|||
Offset: util.BytesToUint32(elt.Data[:4]), |
|||
Size: util.BytesToUint32(elt.Data[4:8]), |
|||
}) |
|||
} |
|||
return cdb.DumpMap(r, iterFunc) |
|||
} |
|||
@ -0,0 +1,63 @@ |
|||
package util |
|||
|
|||
import ( |
|||
"errors" |
|||
"log" |
|||
"os" |
|||
) |
|||
|
|||
// sets file (fh if not nil, otherwise fileName) permission to mask
|
|||
// it will
|
|||
// AND with the permission iff direction < 0
|
|||
// OR with the permission iff direction > 0
|
|||
// otherwise it will SET the permission to the mask
|
|||
func SetFilePerm(fh *os.File, fileName string, mask os.FileMode, direction int8) (err error) { |
|||
var stat os.FileInfo |
|||
if fh == nil { |
|||
stat, err = os.Stat(fileName) |
|||
} else { |
|||
stat, err = fh.Stat() |
|||
} |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
mode := stat.Mode() & ^os.ModePerm |
|||
// log.Printf("mode1=%d mask=%d", mode, mask)
|
|||
if direction == 0 { |
|||
mode |= mask |
|||
} else if direction > 0 { |
|||
mode |= stat.Mode().Perm() | mask |
|||
} else { |
|||
mode |= stat.Mode().Perm() & mask |
|||
} |
|||
log.Printf("pmode=%d operm=%d => nmode=%d nperm=%d", |
|||
stat.Mode(), stat.Mode()&os.ModePerm, |
|||
mode, mode&os.ModePerm) |
|||
if mode == 0 { |
|||
return errors.New("Zero FileMode") |
|||
} |
|||
if fh == nil { |
|||
err = os.Chmod(fileName, mode) |
|||
} else { |
|||
err = fh.Chmod(mode) |
|||
} |
|||
return err |
|||
} |
|||
|
|||
// returns whether the filename exists - errors doesn't mean not exists!
|
|||
func FileExists(fileName string) bool { |
|||
if _, e := os.Stat(fileName); e != nil && os.IsNotExist(e) { |
|||
return false |
|||
} |
|||
return true |
|||
} |
|||
|
|||
// returns whether the filename is POSSIBLY writable
|
|||
//- whether it has some kind of writable bit set
|
|||
func FileIsWritable(fileName string) bool { |
|||
if stat, e := os.Stat(fileName); e == nil { |
|||
return stat.Mode().Perm()&0222 > 0 |
|||
} |
|||
return false |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue