Browse Source

can now delete a collection! Is this a dangerous feature? Only enabling

deleting "benchmark" collections for now.
pull/2/head
Chris Lu 11 years ago
parent
commit
cd10c277b2
  1. 4
      go/storage/cdb_map.go
  2. 5
      go/storage/needle_map.go
  3. 14
      go/storage/store.go
  4. 15
      go/storage/volume.go
  5. 11
      go/topology/collection.go
  6. 22
      go/topology/data_node.go
  7. 11
      go/topology/topology.go
  8. 2
      go/topology/topology_event_handling.go
  9. 16
      go/topology/volume_layout.go
  10. 1
      go/weed/weed_server/master_server.go
  11. 17
      go/weed/weed_server/master_server_handlers.go
  12. 1
      go/weed/weed_server/volume_server.go
  13. 13
      go/weed/weed_server/volume_server_handlers.go

4
go/storage/cdb_map.go

@ -64,6 +64,10 @@ func (m *cdbMap) Close() {
}
}
func (m *cdbMap) Destroy() error {
return errors.New("Can not delete readonly volumes")
}
func (m cdbMap) ContentSize() uint64 {
return m.FileByteCounter
}

5
go/storage/needle_map.go

@ -14,6 +14,7 @@ type NeedleMapper interface {
Get(key uint64) (element *NeedleValue, ok bool)
Delete(key uint64) error
Close()
Destroy() error
ContentSize() uint64
DeletedSize() uint64
FileCount() int
@ -155,6 +156,10 @@ func (nm *NeedleMap) Delete(key uint64) error {
func (nm *NeedleMap) Close() {
_ = nm.indexFile.Close()
}
func (nm *NeedleMap) Destroy() error {
nm.Close()
return os.Remove(nm.indexFile.Name())
}
func (nm NeedleMap) ContentSize() uint64 {
return nm.FileByteCounter
}

14
go/storage/store.go

@ -111,6 +111,20 @@ func (s *Store) AddVolume(volumeListString string, collection string, replicaPla
}
return e
}
func (s *Store) DeleteCollection(collection string) (e error) {
for _, location := range s.locations {
for k, v := range location.volumes {
if v.Collection == collection {
e = v.Destroy()
if e != nil {
return
}
delete(location.volumes, k)
}
}
}
return
}
func (s *Store) findVolume(vid VolumeId) *Volume {
for _, location := range s.locations {
if v, found := location.volumes[vid]; found {

15
go/storage/volume.go

@ -197,6 +197,21 @@ func (v *Volume) isFileUnchanged(n *Needle) bool {
}
return false
}
func (v *Volume) Destroy() (err error) {
if v.readOnly {
err = fmt.Errorf("%s is read-only", v.dataFile)
return
}
v.Close()
err = os.Remove(v.dataFile.Name())
if err != nil {
return
}
err = v.nm.Destroy()
return
}
func (v *Volume) write(n *Needle) (size uint32, err error) {
if v.readOnly {
err = fmt.Errorf("%s is read-only", v.dataFile)

11
go/topology/collection.go

@ -36,3 +36,14 @@ func (c *Collection) Lookup(vid storage.VolumeId) []*DataNode {
}
return nil
}
func (c *Collection) ListVolumeServers() (nodes []*DataNode) {
for _, vl := range c.replicaType2VolumeLayout {
if vl != nil {
if list := vl.ListVolumeServers(); list != nil {
nodes = append(nodes, list...)
}
}
}
return
}

22
go/topology/data_node.go

@ -1,8 +1,8 @@
package topology
import (
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/storage"
_ "fmt"
"strconv"
)
@ -28,12 +28,32 @@ func (dn *DataNode) AddOrUpdateVolume(v storage.VolumeInfo) {
if _, ok := dn.volumes[v.Id]; !ok {
dn.volumes[v.Id] = v
dn.UpAdjustVolumeCountDelta(1)
if !v.ReadOnly {
dn.UpAdjustActiveVolumeCountDelta(1)
}
dn.UpAdjustMaxVolumeId(v.Id)
} else {
dn.volumes[v.Id] = v
}
}
func (dn *DataNode) UpdateVolumes(actualVolumes []storage.VolumeInfo) {
actualVolumeMap := make(map[storage.VolumeId]storage.VolumeInfo)
for _, v := range actualVolumes {
actualVolumeMap[v.Id] = v
}
for vid, _ := range dn.volumes {
glog.V(2).Infoln("Checking volume id:", vid)
if _, ok := actualVolumeMap[vid]; !ok {
glog.V(0).Infoln("Deleting volume id:", vid)
delete(dn.volumes, vid)
dn.UpAdjustVolumeCountDelta(-1)
dn.UpAdjustActiveVolumeCountDelta(-1)
}
} //TODO: adjust max volume id, if need to reclaim volume ids
for _, v := range actualVolumes {
dn.AddOrUpdateVolume(v)
}
}
func (dn *DataNode) GetDataCenter() *DataCenter {
return dn.Parent().Parent().(*NodeImpl).value.(*DataCenter)
}

11
go/topology/topology.go

@ -99,6 +99,15 @@ func (t *Topology) GetVolumeLayout(collectionName string, rp *storage.ReplicaPla
return t.collectionMap[collectionName].GetOrCreateVolumeLayout(rp)
}
func (t *Topology) GetCollection(collectionName string) (collection *Collection, ok bool) {
collection, ok = t.collectionMap[collectionName]
return
}
func (t *Topology) DeleteCollection(collectionName string) {
delete(t.collectionMap, collectionName)
}
func (t *Topology) RegisterVolumeLayout(v *storage.VolumeInfo, dn *DataNode) {
t.GetVolumeLayout(v.Collection, v.ReplicaPlacement).RegisterVolume(v, dn)
}
@ -112,8 +121,8 @@ func (t *Topology) RegisterVolumes(init bool, volumeInfos []storage.VolumeInfo,
t.UnRegisterDataNode(dn)
}
dn = rack.GetOrCreateDataNode(ip, port, publicUrl, maxVolumeCount)
dn.UpdateVolumes(volumeInfos)
for _, v := range volumeInfos {
dn.AddOrUpdateVolume(v)
t.RegisterVolumeLayout(&v, dn)
}
}

2
go/topology/topology_event_handling.go

@ -46,8 +46,10 @@ func (t *Topology) SetVolumeCapacityFull(volumeInfo storage.VolumeInfo) bool {
return false
}
for _, dn := range vl.vid2location[volumeInfo.Id].list {
if !volumeInfo.ReadOnly {
dn.UpAdjustActiveVolumeCountDelta(-1)
}
}
return true
}
func (t *Topology) UnRegisterDataNode(dn *DataNode) {

16
go/topology/volume_layout.go

@ -8,6 +8,7 @@ import (
"sync"
)
// mapping from volume to its locations, inverted from server to volume
type VolumeLayout struct {
rp *storage.ReplicaPlacement
vid2location map[storage.VolumeId]*VolumeLocationList
@ -56,6 +57,13 @@ func (vl *VolumeLayout) Lookup(vid storage.VolumeId) []*DataNode {
return nil
}
func (vl *VolumeLayout) ListVolumeServers() (nodes []*DataNode) {
for _, location := range vl.vid2location {
nodes = append(nodes, location.list...)
}
return
}
func (vl *VolumeLayout) PickForWrite(count int, dataCenter string) (*storage.VolumeId, int, *VolumeLocationList, error) {
len_writers := len(vl.writables)
if len_writers <= 0 {
@ -134,12 +142,14 @@ func (vl *VolumeLayout) SetVolumeUnavailable(dn *DataNode, vid storage.VolumeId)
vl.accessLock.Lock()
defer vl.accessLock.Unlock()
if vl.vid2location[vid].Remove(dn) {
if vl.vid2location[vid].Length() < vl.rp.GetCopyCount() {
glog.V(0).Infoln("Volume", vid, "has", vl.vid2location[vid].Length(), "replica, less than required", vl.rp.GetCopyCount())
if location, ok := vl.vid2location[vid]; ok {
if location.Remove(dn) {
if location.Length() < vl.rp.GetCopyCount() {
glog.V(0).Infoln("Volume", vid, "has", location.Length(), "replica, less than required", vl.rp.GetCopyCount())
return vl.removeFromWritable(vid)
}
}
}
return false
}
func (vl *VolumeLayout) SetVolumeAvailable(dn *DataNode, vid storage.VolumeId) bool {

1
go/weed/weed_server/master_server.go

@ -60,6 +60,7 @@ func NewMasterServer(r *mux.Router, version string, port int, metaFolder string,
r.HandleFunc("/dir/lookup", ms.proxyToLeader(secure(ms.whiteList, ms.dirLookupHandler)))
r.HandleFunc("/dir/join", ms.proxyToLeader(secure(ms.whiteList, ms.dirJoinHandler)))
r.HandleFunc("/dir/status", ms.proxyToLeader(secure(ms.whiteList, ms.dirStatusHandler)))
r.HandleFunc("/col/delete", ms.proxyToLeader(secure(ms.whiteList, ms.collectionDeleteHandler)))
r.HandleFunc("/vol/grow", ms.proxyToLeader(secure(ms.whiteList, ms.volumeGrowHandler)))
r.HandleFunc("/vol/status", ms.proxyToLeader(secure(ms.whiteList, ms.volumeStatusHandler)))
r.HandleFunc("/vol/vacuum", ms.proxyToLeader(secure(ms.whiteList, ms.volumeVacuumHandler)))

17
go/weed/weed_server/master_server_handlers.go

@ -2,6 +2,7 @@ package weed_server
import (
"code.google.com/p/weed-fs/go/storage"
"code.google.com/p/weed-fs/go/util"
"encoding/json"
"errors"
"net/http"
@ -78,6 +79,22 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request)
}
}
func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) {
collection, ok := ms.topo.GetCollection(r.FormValue("collection"))
if !ok {
writeJsonQuiet(w, r, map[string]interface{}{"error": "collection " + r.FormValue("collection") + "does not exist!"})
return
}
for _, server := range collection.ListVolumeServers() {
_, err := util.Get("http://" + server.Ip + ":" + strconv.Itoa(server.Port) + "/admin/delete_collection?collection=" + r.FormValue("collection"))
if err != nil {
writeJsonQuiet(w, r, map[string]string{"error": err.Error()})
return
}
}
ms.topo.DeleteCollection(r.FormValue("collection"))
}
func (ms *MasterServer) dirJoinHandler(w http.ResponseWriter, r *http.Request) {
init := r.FormValue("init") == "true"
ip := r.FormValue("ip")

1
go/weed/weed_server/volume_server.go

@ -39,6 +39,7 @@ func NewVolumeServer(r *http.ServeMux, version string, ip string, port int, publ
r.HandleFunc("/admin/vacuum_volume_compact", secure(vs.whiteList, vs.vacuumVolumeCompactHandler))
r.HandleFunc("/admin/vacuum_volume_commit", secure(vs.whiteList, vs.vacuumVolumeCommitHandler))
r.HandleFunc("/admin/freeze_volume", secure(vs.whiteList, vs.freezeVolumeHandler))
r.HandleFunc("/admin/delete_collection", secure(vs.whiteList, vs.deleteCollectionHandler))
r.HandleFunc("/", vs.storeHandler)
go func() {

13
go/weed/weed_server/volume_server_handlers.go

@ -29,6 +29,19 @@ func (vs *VolumeServer) assignVolumeHandler(w http.ResponseWriter, r *http.Reque
}
glog.V(2).Infoln("assign volume =", r.FormValue("volume"), ", collection =", r.FormValue("collection"), ", replication =", r.FormValue("replication"), ", error =", err)
}
func (vs *VolumeServer) deleteCollectionHandler(w http.ResponseWriter, r *http.Request) {
if "benchmark" != r.FormValue("collection") {
glog.V(0).Infoln("deleting collection =", r.FormValue("collection"), "!!!")
return
}
err := vs.store.DeleteCollection(r.FormValue("collection"))
if err == nil {
writeJsonQuiet(w, r, map[string]string{"error": ""})
} else {
writeJsonQuiet(w, r, map[string]string{"error": err.Error()})
}
glog.V(2).Infoln("deleting collection =", r.FormValue("collection"), ", error =", err)
}
func (vs *VolumeServer) vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.Request) {
err, ret := vs.store.CheckCompactVolume(r.FormValue("volume"), r.FormValue("garbageThreshold"))
if err == nil {

Loading…
Cancel
Save