You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
107 lines
2.2 KiB
107 lines
2.2 KiB
package storage
|
|
|
|
import (
|
|
"github.com/tgulacsi/go-cdb"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"pkg/util"
|
|
"strings"
|
|
)
|
|
|
|
type CdbMap struct {
|
|
db *cdb.Cdb
|
|
transient []byte
|
|
Filename string
|
|
}
|
|
|
|
// Opens the CDB file and servers as a needle map
|
|
func NewCdbMap(filename string) (*CdbMap, error) {
|
|
m, err := cdb.Open(filename)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &CdbMap{db: m, transient: make([]byte, 8),
|
|
Filename: filename}, nil
|
|
}
|
|
|
|
// writes the content of the index file to a CDB and returns that
|
|
func NewCdbMapFromIndex(indexFile *os.File) (*CdbMap, error) {
|
|
nm := indexFile.Name()
|
|
nm = nm[strings.LastIndex(nm, ".")+1:] + "cdb"
|
|
|
|
var (
|
|
key uint64
|
|
offset uint32
|
|
ok bool
|
|
)
|
|
deleted := make(map[uint64]bool, 16)
|
|
gatherDeletes := func(buf []byte) error {
|
|
key = util.BytesToUint64(buf[:8])
|
|
offset = util.BytesToUint32(buf[8:12])
|
|
if offset > 0 {
|
|
if _, ok = deleted[key]; ok { //undelete
|
|
delete(deleted, key)
|
|
}
|
|
} else {
|
|
deleted[key] = true
|
|
}
|
|
return nil
|
|
}
|
|
if err := readIndexFile(indexFile, gatherDeletes); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
w, err := cdb.NewWriter(nm)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
iterFun := func(buf []byte) error {
|
|
key = util.BytesToUint64(buf[:8])
|
|
if _, ok = deleted[key]; !ok {
|
|
w.PutPair(buf[:8], buf[8:16])
|
|
}
|
|
return nil
|
|
}
|
|
indexFile.Seek(0, 0)
|
|
err = readIndexFile(indexFile, iterFun)
|
|
w.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return NewCdbMap(nm)
|
|
}
|
|
|
|
func (m *CdbMap) Get(key Key) (element *NeedleValue, ok bool) {
|
|
util.Uint64toBytes(m.transient, uint64(key))
|
|
data, err := m.db.Data(m.transient)
|
|
if err != nil {
|
|
if err == io.EOF {
|
|
return nil, false
|
|
}
|
|
log.Printf("error getting %s: %s", key, err)
|
|
return nil, false
|
|
}
|
|
return &NeedleValue{Key: key,
|
|
Offset: util.BytesToUint32(data[:4]),
|
|
Size: util.BytesToUint32(data[4:8]),
|
|
}, true
|
|
}
|
|
|
|
func (m *CdbMap) Walk(pedestrian func(*NeedleValue) error) (err error) {
|
|
r, err := os.Open(m.Filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer r.Close()
|
|
|
|
iterFunc := func(elt cdb.Element) error {
|
|
return pedestrian(&NeedleValue{
|
|
Key: Key(util.BytesToUint64(elt.Key[:8])),
|
|
Offset: util.BytesToUint32(elt.Data[:4]),
|
|
Size: util.BytesToUint32(elt.Data[4:8]),
|
|
})
|
|
}
|
|
return cdb.DumpMap(r, iterFunc)
|
|
}
|