Chris Lu
13 years ago
8 changed files with 297 additions and 16 deletions
-
1weed-fs/bin/.gitignore
-
59weed-fs/note/replication.txt
-
54weed-fs/src/cmd/weed/shell.go
-
13weed-fs/src/cmd/weed/weed.go
-
6weed-fs/src/pkg/topology/data_center.go
-
16weed-fs/src/pkg/topology/node.go
-
10weed-fs/src/pkg/topology/rack.go
-
128weed-fs/src/pkg/util/config.go
@ -0,0 +1 @@ |
|||
/weed |
@ -0,0 +1,59 @@ |
|||
1. each file can choose the replication factor |
|||
2. replication granularity is in volume level |
|||
3. if not enough spaces, we can automatically decrease some volume's the replication factor, especially for cold data |
|||
4. support migrating data to cheaper storage |
|||
5. manual volume placement, access-based volume placement, auction based volume placement |
|||
|
|||
When a new volume server is started, it reports |
|||
1. how many volumes it can hold |
|||
2. current list of existing volumes |
|||
Each volume server remembers: |
|||
1. current volume ids, replica locations |
|||
|
|||
The master assign volume ids based on |
|||
1. replication factor |
|||
data center, rack |
|||
2. concurrent write support |
|||
On master, stores the replication configuration |
|||
{ |
|||
replication:{ |
|||
{factor:1, min_volume_count:3, weight:10}, |
|||
{factor:2, min_volume_count:2, weight:20}, |
|||
{factor:3, min_volume_count:3, weight:30} |
|||
}, |
|||
port:9333, |
|||
|
|||
} |
|||
Or manually via command line |
|||
1. add volume with specified replication factor |
|||
2. add volume with specified volume id |
|||
|
|||
|
|||
If duplicated volume ids are reported from different volume servers, |
|||
the master determines the replication factor of the volume, |
|||
if less than the replication factor, the volume is in readonly mode |
|||
if more than the replication factor, the volume will purge the smallest/oldest volume |
|||
if equal, the volume will function as usual |
|||
|
|||
maybe use gossip to send the volumeServer~volumes information |
|||
|
|||
|
|||
Use cases: |
|||
on volume server |
|||
1. weed volume -mserver="xx.xx.xx.xx:9333" -publicUrl="good.com:8080" -dir="/tmp" -volumes=50 |
|||
on weed master |
|||
1. weed master -port=9333 |
|||
generate a default json configuration file if doesn't exist |
|||
|
|||
Bootstrap |
|||
1. at the very beginning, the system has no volumes at all. |
|||
2. if maxReplicationFactor==1, always initialize volumes right away |
|||
3. if nServersHasFreeSpaces >= maxReplicationFactor, auto initialize |
|||
4. if maxReplicationFactor>1 |
|||
weed shell |
|||
> disable_auto_initialize |
|||
> enable_auto_initialize |
|||
> assign_free_volume vid "server1:port","server2:port","server3:port" |
|||
> status |
|||
5. |
|||
|
@ -0,0 +1,54 @@ |
|||
package main |
|||
|
|||
import ( |
|||
"bufio" |
|||
"os" |
|||
"fmt" |
|||
) |
|||
|
|||
func init() { |
|||
cmdShell.Run = runShell // break init cycle
|
|||
} |
|||
|
|||
var cmdShell = &Command{ |
|||
UsageLine: "shell", |
|||
Short: "run interactive commands, now just echo", |
|||
Long: `run interactive commands. |
|||
|
|||
`, |
|||
} |
|||
|
|||
var ( |
|||
) |
|||
|
|||
func runShell(command *Command, args []string) bool { |
|||
r := bufio.NewReader(os.Stdin) |
|||
o := bufio.NewWriter(os.Stdout) |
|||
e := bufio.NewWriter(os.Stderr) |
|||
prompt := func () { |
|||
o.WriteString("> ") |
|||
o.Flush() |
|||
}; |
|||
readLine := func () string { |
|||
ret, err := r.ReadString('\n') |
|||
if err != nil { |
|||
fmt.Fprint(e,err); |
|||
os.Exit(1) |
|||
} |
|||
return ret |
|||
} |
|||
execCmd := func (cmd string) int { |
|||
if cmd != "" { |
|||
o.WriteString(cmd) |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
cmd := "" |
|||
for { |
|||
prompt() |
|||
cmd = readLine() |
|||
execCmd(cmd) |
|||
} |
|||
return true |
|||
} |
@ -0,0 +1,6 @@ |
|||
package topology |
|||
|
|||
import ( |
|||
|
|||
) |
|||
|
@ -0,0 +1,16 @@ |
|||
package topology |
|||
|
|||
import ( |
|||
|
|||
) |
|||
|
|||
type VolumeInfo struct { |
|||
Id uint32 |
|||
Size int64 |
|||
} |
|||
type Node struct { |
|||
volumes map[uint64]VolumeInfo |
|||
volumeLimit int |
|||
Port int |
|||
PublicUrl string |
|||
} |
@ -0,0 +1,10 @@ |
|||
package topology |
|||
|
|||
import ( |
|||
|
|||
) |
|||
|
|||
type Rack struct { |
|||
nodes map[uint64]*Node |
|||
IpRanges []string |
|||
} |
@ -0,0 +1,128 @@ |
|||
// Copyright 2011 Numerotron Inc.
|
|||
// Use of this source code is governed by an MIT-style license
|
|||
// that can be found in the LICENSE file.
|
|||
//
|
|||
// Developed at www.stathat.com by Patrick Crosby
|
|||
// Contact us on twitter with any questions: twitter.com/stat_hat
|
|||
|
|||
// The jconfig package provides a simple, basic configuration file parser using JSON.
|
|||
package util |
|||
|
|||
import ( |
|||
"bytes" |
|||
"encoding/json" |
|||
"log" |
|||
"os" |
|||
) |
|||
|
|||
type Config struct { |
|||
data map[string]interface{} |
|||
filename string |
|||
} |
|||
|
|||
func newConfig() *Config { |
|||
result := new(Config) |
|||
result.data = make(map[string]interface{}) |
|||
return result |
|||
} |
|||
|
|||
// Loads config information from a JSON file
|
|||
func LoadConfig(filename string) *Config { |
|||
result := newConfig() |
|||
result.filename = filename |
|||
err := result.parse() |
|||
if err != nil { |
|||
log.Fatalf("error loading config file %s: %s", filename, err) |
|||
} |
|||
return result |
|||
} |
|||
|
|||
// Loads config information from a JSON string
|
|||
func LoadConfigString(s string) *Config { |
|||
result := newConfig() |
|||
err := json.Unmarshal([]byte(s), &result.data) |
|||
if err != nil { |
|||
log.Fatalf("error parsing config string %s: %s", s, err) |
|||
} |
|||
return result |
|||
} |
|||
|
|||
func (c *Config) StringMerge(s string) { |
|||
next := LoadConfigString(s) |
|||
c.merge(next.data) |
|||
} |
|||
|
|||
func (c *Config) LoadMerge(filename string) { |
|||
next := LoadConfig(filename) |
|||
c.merge(next.data) |
|||
} |
|||
|
|||
func (c *Config) merge(ndata map[string]interface{}) { |
|||
for k, v := range ndata { |
|||
c.data[k] = v |
|||
} |
|||
} |
|||
|
|||
func (c *Config) parse() error { |
|||
f, err := os.Open(c.filename) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
defer f.Close() |
|||
b := new(bytes.Buffer) |
|||
_, err = b.ReadFrom(f) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
err = json.Unmarshal(b.Bytes(), &c.data) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
// Returns a string for the config variable key
|
|||
func (c *Config) GetString(key string) string { |
|||
result, present := c.data[key] |
|||
if !present { |
|||
return "" |
|||
} |
|||
return result.(string) |
|||
} |
|||
|
|||
// Returns an int for the config variable key
|
|||
func (c *Config) GetInt(key string) int { |
|||
x, ok := c.data[key] |
|||
if !ok { |
|||
return -1 |
|||
} |
|||
return int(x.(float64)) |
|||
} |
|||
|
|||
// Returns a float for the config variable key
|
|||
func (c *Config) GetFloat(key string) float64 { |
|||
x, ok := c.data[key] |
|||
if !ok { |
|||
return -1 |
|||
} |
|||
return x.(float64) |
|||
} |
|||
|
|||
// Returns a bool for the config variable key
|
|||
func (c *Config) GetBool(key string) bool { |
|||
x, ok := c.data[key] |
|||
if !ok { |
|||
return false |
|||
} |
|||
return x.(bool) |
|||
} |
|||
|
|||
// Returns an array for the config variable key
|
|||
func (c *Config) GetArray(key string) []interface{} { |
|||
result, present := c.data[key] |
|||
if !present { |
|||
return []interface{}(nil) |
|||
} |
|||
return result.([]interface{}) |
|||
} |
Write
Preview
Loading…
Cancel
Save
Reference in new issue