Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge pull request #121 from stuartpb/seaweedfs-unify 

More consistency with SeaweedFS name
pull/122/head
chrislusf 10 years ago
parent
d960e94290 1a194a578c
commit
b9e8b5a816
14 changed files with 53 additions and 55 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 5
      README.md
  2. 2
      go/filer/filer.go
  3. 2
      go/images/preprocess.go
  4. 6
      go/weed/benchmark.go
  5. 2
      go/weed/fix.go
  6. 8
      go/weed/mount.go
  7. 2
      go/weed/mount_std.go
  8. 4
      go/weed/version.go
  9. 2
      go/weed/weed.go
  10. 4
      go/weed/weed_server/master_ui/templates.go
  11. 4
      go/weed/weed_server/volume_server_ui/templates.go
  12. 57
      note/memory_usage.txt
  13. 2
      note/security.txt
  14. 8
      note/weedfs.txt

5
README.md
View File

@ -1,5 +1,4 @@
Seaweed File System
=======
# SeaweedFS
[![Build Status](https://travis-ci.org/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.org/chrislusf/seaweedfs)
[![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/go?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/go)
@ -26,7 +25,7 @@ SeaweedFS costs only 40 bytes disk storage for each file's metadata. It is so si
![](https://api.bintray.com/packages/chrislusf/Weed-FS/seaweed/images/download.png)
https://bintray.com/chrislusf/Weed-FS/seaweed Download latest compiled binaries for different platforms here.
http://groups.google.com/group/weed-file-system Seaweed File System Discussion Group
http://groups.google.com/group/weed-file-system SeaweedFS Discussion Group
## Additional Features
* Can choose no replication or different replication level, rack and data center aware

2
go/filer/filer.go
View File

@ -1,6 +1,6 @@
package filer
type FileId string //file id on weedfs
type FileId string //file id in SeaweedFS
type FileEntry struct {
Name string `json:"name,omitempty"` //file name without path

2
go/images/preprocess.go
View File

@ -10,7 +10,7 @@ import (
* 1. possibly adjust the orientation
* 2. resize the image to a width or height limit
* 3. remove the exif data
* Call this function on any file uploaded to weedfs
* Call this function on any file uploaded to SeaweedFS
*
*/
func MaybePreprocessImage(filename string, data []byte, width, height int) (resized []byte, w int, h int) {

6
go/weed/benchmark.go
View File

@ -46,7 +46,7 @@ var (
func init() {
cmdBenchmark.Run = runbenchmark // break init cycle
cmdBenchmark.IsDebug = cmdBenchmark.Flag.Bool("debug", false, "verbose debug information")
b.server = cmdBenchmark.Flag.String("server", "localhost:9333", "weedfs master location")
b.server = cmdBenchmark.Flag.String("server", "localhost:9333", "SeaweedFS master location")
b.concurrency = cmdBenchmark.Flag.Int("c", 16, "number of concurrent write or read processes")
b.fileSize = cmdBenchmark.Flag.Int("size", 1024, "simulated file size in bytes, with random(0~63) bytes padding")
b.numberOfFiles = cmdBenchmark.Flag.Int("n", 1024*1024, "number of files to write for each thread")
@ -66,7 +66,7 @@ func init() {
var cmdBenchmark = &Command{
UsageLine: "benchmark -server=localhost:9333 -c=10 -n=100000",
Short: "benchmark on writing millions of files and read out",
Long: `benchmark on an empty weed file system.
Long: `benchmark on an empty SeaweedFS file system.
Two tests during benchmark:
1) write lots of small files to the system
@ -102,7 +102,7 @@ var (
)
func runbenchmark(cmd *Command, args []string) bool {
fmt.Printf("This is Seaweed File System version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
if *b.maxCpu < 1 {
*b.maxCpu = runtime.NumCPU()
}

2
go/weed/fix.go
View File

@ -16,7 +16,7 @@ func init() {
var cmdFix = &Command{
UsageLine: "fix -dir=/tmp -volumeId=234",
Short: "run weed tool fix on index file if corrupted",
Long: `Fix runs the SeeweedFS fix command to re-create the index .idx file.
Long: `Fix runs the SeaweedFS fix command to re-create the index .idx file.
`,
}

8
go/weed/mount.go
View File

@ -12,17 +12,17 @@ var (
func init() {
cmdMount.Run = runMount // break init cycle
cmdMount.IsDebug = cmdMount.Flag.Bool("debug", false, "verbose debug information")
mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weedfs filer location")
mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location")
mountOptions.dir = cmdMount.Flag.String("dir", "", "mount weed filer to this directory")
}
var cmdMount = &Command{
UsageLine: "mount -filer=localhost:8888 -dir=/some/dir",
Short: "mount weed filer to a directory as file system in userspace(FUSE)",
Long: `mount weed file system to userspace.
Long: `mount weed filer to userspace.
Pre-requisites:
1) have a weed file system running
1) have SeaweedFS master and volume servers running
2) have a "weed filer" running
These 2 requirements can be achieved with one command "weed server -filer=true"

2
go/weed/mount_std.go
View File

@ -16,7 +16,7 @@ import (
)
func runMount(cmd *Command, args []string) bool {
fmt.Printf("This is Seaweed File System version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
if *mountOptions.dir == "" {
fmt.Printf("Please specify the mount directory via \"-dir\"")
return false

4
go/weed/version.go
View File

@ -10,8 +10,8 @@ import (
var cmdVersion = &Command{
Run: runVersion,
UsageLine: "version",
Short: "print Seaweed File System version",
Long: `Version prints the Seaweed File System version`,
Short: "print SeaweedFS version",
Long: `Version prints the SeaweedFS version`,
}
func runVersion(cmd *Command, args []string) bool {

2
go/weed/weed.go
View File

@ -91,7 +91,7 @@ func main() {
}
var usageTemplate = `
Seaweed File System : store billions of files and serve them fast!
SeaweedFS: store billions of files and serve them fast!
Usage:

4
go/weed/weed_server/master_ui/templates.go
View File

@ -7,13 +7,13 @@ import (
var StatusTpl = template.Must(template.New("status").Parse(`<!DOCTYPE html>
<html>
<head>
<title>Seaweed File System {{ .Version }}</title>
<title>SeaweedFS {{ .Version }}</title>
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css">
</head>
<body>
<div class="container">
<div class="page-header">
<h1>Seaweed File System <small>{{ .Version }}</small></h1>
<h1>SeaweedFS <small>{{ .Version }}</small></h1>
</div>
<div class="row">

4
go/weed/weed_server/volume_server_ui/templates.go
View File

@ -21,7 +21,7 @@ var funcMap = template.FuncMap{
var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`<!DOCTYPE html>
<html>
<head>
<title>Seaweed File System {{ .Version }}</title>
<title>SeaweedFS {{ .Version }}</title>
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css">
<script type="text/javascript" src="https://code.jquery.com/jquery-2.1.3.min.js"></script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/jquery-sparklines/2.1.2/jquery.sparkline.min.js"></script>
@ -48,7 +48,7 @@ var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`<!DOC
<body>
<div class="container">
<div class="page-header">
<h1>Seaweed File System <small>{{ .Version }}</small></h1>
<h1>SeaweedFS <small>{{ .Version }}</small></h1>
</div>
<div class="row">

57
note/memory_usage.txt
View File

@ -3,44 +3,44 @@ Each volume has 25M index, so each cost 160MB memory
Things happened when I use lots of threads ( almost 120 ) keeping read file from Weed-FS.
Things happened when I use lots of threads ( almost 120 ) keeping read file from SeaweedFS.
But I'm not so familiar with linux so I can't tell you exactly what happened.
Next I'll show you things I know , if you need more info , contact me
My weed-fs version is about 0.12
My SeaweedFS version is about 0.12
1. top
top - 12:07:37 up 1 day, 3:17, 2 users, load average: 0.00, 0.00, 0.00
Tasks: 152 total, 1 running, 151 sleeping, 0 stopped, 0 zombie
Cpu(s): 0.0%us, 0.0%sy, 0.0%ni, 99.8%id, 0.1%wa, 0.0%hi, 0.0%si, 0.0%st
Mem: 16269880k total, 16192364k used, 77516k free, 58172k buffers
Swap: 2064376k total, 12324k used, 2052052k free, 2827520k cached
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
1499 root 20 0 11.6g 10g 1424 S 0.0 65.6 7:32.53 weedvolume
1498 root 20 0 3204m 2.1g 1428 S 0.0 13.5 4:36.59 weedvolume
1737 root 20 0 98868 4932 2920 S 0.0 0.0 0:00.56 sshd
1497 root 20 0 151m 4404 1152 S 0.0 0.0 1:21.40 weedmaster
1335 root 20 0 97816 3044 2896 S 0.0 0.0 0:00.76 sshd
After system became steady , weedvolume used 65.6% memory .
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
1499 root 20 0 11.6g 10g 1424 S 0.0 65.6 7:32.53 weedvolume
1498 root 20 0 3204m 2.1g 1428 S 0.0 13.5 4:36.59 weedvolume
1737 root 20 0 98868 4932 2920 S 0.0 0.0 0:00.56 sshd
1497 root 20 0 151m 4404 1152 S 0.0 0.0 1:21.40 weedmaster
1335 root 20 0 97816 3044 2896 S 0.0 0.0 0:00.76 sshd
After system became steady , weedvolume used 65.6% memory .
2. free -m
total used free shared buffers cached
Mem: 15888 15809 79 0 56 2758
-/+ buffers/cache: 12994 2894
Swap: 2015 12 2003
3. startup cmd
screen -d -m /opt/weed/weedmaster -mdir /data/weeddata/ > /data/logs/weed/master.log &
screen -d -m /opt/weed/weedvolume -volumes=0-64 -dir /data/weeddata/ -pulseSeconds=20 -publicUrl="x.y.z:9334" -port 9334 > /data/logs/weed/s01.log &
screen -d -m /opt/weed/weedvolume -volumes=65-107 -dir /data/weeddata/ -pulseSeconds=20 -publicUrl="x.y.z:9335" -port 9335 > /data/logs/weed/s02.log &
4. du -sh .
32G 0.dat
26M 0.idx
8.2G 100.dat
@ -258,22 +258,21 @@ screen -d -m /opt/weed/weedvolume -volumes=65-107 -dir /data/weeddata/ -pulseSec
32G 9.dat
25M 9.idx
4.0K directory.seq
You can see the volume 1-64 is now full.
5. more log
see logs.zip
In messages you can see these lines: (Line 51095)
Sep 26 06:14:31 wedb-01 kernel: auditd: page allocation failure. order:0, mode:0x20
Sep 26 06:14:31 wedb-01 kernel: Pid: 1009, comm: auditd Not tainted 2.6.32-220.el6.x86_64 #1
Sep 26 06:14:31 wedb-01 kernel: Call Trace:
After those lines , the system deny any new network connect request
6. /dir/status
{"Machines":[{"Server":{"Url":"127.0.0.1:9335","PublicUrl":"x.y.z:9335"},"Volumes":[{"Id":106,"Size":8728909632},{"Id":66,"Size":8729852744},{"Id":90,"Size":8747834896},{"Id":103,"Size":8718106024},{"Id":87,"Size":8732133512},{"Id":96,"Size":8737251904},{"Id":80,"Size":8704130712},{"Id":77,"Size":8717989496},{"Id":70,"Size":8731474744},{"Id":94,"Size":8758656144},{"Id":107,"Size":8729599232},{"Id":67,"Size":8736848088},{"Id":91,"Size":8665847760},{"Id":100,"Size":8703272552},{"Id":84,"Size":8745121528},{"Id":97,"Size":8713031744},{"Id":81,"Size":8726088872},{"Id":74,"Size":8738588152},{"Id":71,"Size":8729349920},{"Id":95,"Size":8741526896},{"Id":104,"Size":8699374736},{"Id":88,"Size":8740362880},{"Id":101,"Size":8711832992},{"Id":85,"Size":8723479552},{"Id":78,"Size":8700345400},{"Id":75,"Size":8727796912},{"Id":68,"Size":8698607440},{"Id":92,"Size":8682683056},{"Id":105,"Size":8741226152},{"Id":65,"Size":8725365752},{"Id":89,"Size":8703062600},{"Id":98,"Size":8742331560},{"Id":82,"Size":8762554952},{"Id":79,"Size":8696300376},{"Id":72,"Size":8708217304},{"Id":69,"Size":8740268144},{"Id":93,"Size":8685060320},{"Id":102,"Size":8708695352},{"Id":86,"Size":8783247776},{"Id":99,"Size":8753463608},{"Id":83,"Size":8725963952},{"Id":76,"Size":8694693536},{"Id":73,"Size":8733560832}]},{"Server":{"Url":"127.0.0.1:9334","PublicUrl":"x.y.z:9334"},"Volumes":[{"Id":34,"Size":33415706800},{"Id":58,"Size":33569224784},{"Id":18,"Size":33474649968},{"Id":55,"Size":33542422680},{"Id":15,"Size":33517247576},{"Id":48,"Size":33574860328},{"Id":8,"Size":33511257144},{"Id":45,"Size":33463948408},{"Id":5,"Size":34317702920},{"Id":29,"Size":33465695776},{"Id":38,"Size":33553119624},{"Id":62,"Size":33448316736},{"Id":22,"Size":33566586296},{"Id":35,"Size":33493733728},{"Id":59,"Size":33498554904},{"Id":19,"Size":33493313784},{"Id":52,"Size":33552978448},{"Id":12,"Size":33505183752},{"Id":49,"Size":33603029896},{"Id":9,"Size":33515778064},{"Id":42,"Size":33500402248},{"Id":2,"Size":34223232992},{"Id":26,"Size":33526519600},{"Id":39,"Size":33580414336},{"Id":63,"Size":33476332456},{"Id":23,"Size":33543872592},{"Id":32,"Size":33515290168},{"Id":56,"Size":33499171184},{"Id":16,"Size":33556591168},{"Id":64,"Size":33495148616},{"Id":53,"Size":33467738560},{"Id":13,"Size":33596873960},{"Id":46,"Size":33508120448},{"Id":6,"Size":33417470256},{"Id":30,"Size":33532933992},{"Id":43,"Size":33591802008},{"Id":3,"Size":34270682080},{"Id":27,"Size":33525736944},{"Id":36,"Size":33443597824},{"Id":60,"Size":33427931336},{"Id":20,"Size":33499083096},{"Id":33,"Size":33531396280},{"Id":57,"Size":33578015104},{"Id":17,"Size":33510525480},{"Id":50,"Size":33503123704},{"Id":10,"Size":33502391608},{"Id":47,"Size":33521868568},{"Id":7,"Size":33497101664},{"Id":31,"Size":33426905232},{"Id":40,"Size":33472978696},{"Id":0,"Size":34337344304},{"Id":24,"Size":33550157192},{"Id":37,"Size":33477162720},{"Id":61,"Size":33537175080},{"Id":21,"Size":33517192456},{"Id":54,"Size":33480720288},{"Id":14,"Size":33513192896},{"Id":51,"Size":33531336080},{"Id":11,"Size":33562385088},{"Id":44,"Size":33554479104},{"Id":4,"Size":34333127520},{"Id":28,"Size":33510503000},{"Id":41,"Size":33574922928},{"Id":1,"Size":34307181368},{"Id":25,"Size":33542834568}]}],"Writers":[106,66,90,103,87,96,80,77,70,94,107,67,91,100,84,97,81,74,71,95,104,88,101,85,78,75,68,92,105,65,89,98,82,79,72,69,93,102,86,99,83,76,73,34,58,18,55,15,48,8,45,5,29,38,62,22,35,59,19,52,12,49,9,42,2,26,39,63,23,32,56,16,64,53,13,46,6,30,43,3,27,36,60,20,33,57,17,50,10,47,7,31,40,0,24,37,61,21,54,14,51,11,44,4,28,41,1,25],"FileIdSequence":110250000}

2
note/security.txt
View File

@ -1,4 +1,4 @@
Design for Seaweed-FS security
Design for SeaweedFS security
Design Objectives
Security can mean many different things. The original vision is that: if you have one machine lying around

8
note/weedfs.txt
View File

@ -3,7 +3,7 @@ How to submit a content
1.c Create a hash value
1.d find a write logic volume id, and return [logic volume id, {physical volume ids}]
2. submit to physical volumes
2.c
2.c
generate the cookie
generate a unique id as key
choose the right altKey
@ -26,7 +26,7 @@ How to retrieve a content
How to submit a content
1. send bytes to weedfs, got <volume id, key uint64, cookie code>
1. send bytes to SeaweedFS, got <volume id, key uint64, cookie code>
store <key uint64, volume id uint32, cookie code uint32, ext>, and other information
To read a content
@ -40,7 +40,7 @@ The directory server
find a free logic volume id, start sending content to 3 machines
if all of them finishes, return <logic volume id, key, cookie code>
2.on read content
based on logic volume id, pick a machine with less load,
based on logic volume id, pick a machine with less load,
return <machine id>
Write Preview
Loading…
Cancel
Save