Chris Lu
6 years ago
committed by
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with
124 additions and
24 deletions
README.md
util/gostd
weed/command/server.go
weed/server/master_grpc_server.go
weed/server/volume_grpc_client_to_master.go
weed/shell/command_volume_unmount.go
weed/storage/needle/needle.go
weed/storage/needle/needle_read_write.go
weed/storage/needle/volume_id_test.go
weed/storage/volume_checking.go
weed/topology/topology_test.go
weed/util/compression.go
@ -388,6 +388,12 @@ Other key features include: Erasure Encoding, JWT security.
This is a super exciting project! And we need helpers and [support ](https://www.patreon.com/seaweedfs )!
BTW, We suggest run the code style check script `util/gostd` before you push your branch to remote, it will make SeaweedFS easy to review, maintain and develop:
```
$ ./util/gostd
```
[Back to TOC ](#table-of-contents )
## Installation Guide ##
@ -0,0 +1,98 @@
#!/usr/bin/env bash
############################ GLOBAL VARIABLES
regex=' '
branch="master"
max_length=150
REGEX_SUFFIX_GO=".+\.go$"
############################ FUNCTIONS
msg() {
printf '%b' "$1" >&2
}
die() {
msg "\33[31m[✘]\33[0m ${1}${2}"
exit 1
}
succ() {
msg "\33[34m[√]\33[0m ${1}${2}"
}
gostd() {
local branch=$1
local reg4exclude=$2
local max_length=$3
for file in `git diff $branch --name-only`
do
if ! [[ $file =~ $REGEX_SUFFIX_GO ]] || [[ $file =~ $reg4exclude ]]; then
continue
fi
error=`go fmt $file 2>&1`
if ! [ $? -eq 0 ]; then
die "go fmt $file:" "$error"
fi
succ "$file\n"
grep -n -E --color=always ".{$max_length}" $file | awk '{ printf ("%4s %s\n", "", $0) }'
done
}
get_options() {
while getopts "b:e:hl:" opts
do
case $opts in
b)
branch=$OPTARG
;;
e)
regex=$OPTARG
;;
h)
usage
exit 0
;;
l)
max_length=$OPTARG
;;
\?)
usage
exit 1
;;
esac
done
}
usage () {
cat << _EOC_
Usage:
gostd [options]
Options:
-b <branch/commit> Specify the git diff branch or commit.
(default: master)
-e <regex> Regex for excluding file or directory.
-h Print this usage.
-l <length> Show files that exceed the limit line length.
(default: 150)
Examples:
gostd
gostd -b master -l 100
gostd -b 59d532a -e weed/pb -l 100
_EOC_
}
main() {
get_options "$@"
gostd "$branch" "$regex" "$max_length"
}
############################ MAIN()
main "$@"
@ -1,10 +1,10 @@
package command
import (
"fmt"
"github.com/chrislusf/raft/protobuf"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/spf13/viper"
"fmt"
"net/http"
"os"
"runtime"
@ -79,10 +79,10 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
}
if len ( heartbeat . NewVolumes ) > 0 || len ( heartbeat . DeletedVolumes ) > 0 {
// process delta volume ids if exists for fast volume id updates
for _ , volInfo := range heartbeat . NewVolumes {
for _ , volInfo := range heartbeat . NewVolumes {
message . NewVids = append ( message . NewVids , volInfo . Id )
}
for _ , volInfo := range heartbeat . DeletedVolumes {
for _ , volInfo := range heartbeat . DeletedVolumes {
message . DeletedVids = append ( message . DeletedVids , volInfo . Id )
}
// update master internal volume layouts
@ -99,7 +99,7 @@ func (vs *VolumeServer) doHeartbeat(ctx context.Context, masterNode, masterGrpcA
select {
case volumeMessage := <- vs . store . NewVolumesChan :
deltaBeat := & master_pb . Heartbeat {
NewVolumes : [ ] * master_pb . VolumeShortInformationMessage {
NewVolumes : [ ] * master_pb . VolumeShortInformationMessage {
& volumeMessage ,
} ,
}
@ -110,7 +110,7 @@ func (vs *VolumeServer) doHeartbeat(ctx context.Context, masterNode, masterGrpcA
}
case volumeMessage := <- vs . store . DeletedVolumesChan :
deltaBeat := & master_pb . Heartbeat {
DeletedVolumes : [ ] * master_pb . VolumeShortInformationMessage {
DeletedVolumes : [ ] * master_pb . VolumeShortInformationMessage {
& volumeMessage ,
} ,
}
@ -50,7 +50,6 @@ func (c *commandVolumeUnmount) Do(args []string, commandEnv *commandEnv, writer
}
func unmountVolume ( ctx context . Context , grpcDialOption grpc . DialOption , volumeId needle . VolumeId , sourceVolumeServer string ) ( err error ) {
return operation . WithVolumeServerClient ( sourceVolumeServer , grpcDialOption , func ( volumeServerClient volume_server_pb . VolumeServerClient ) error {
_ , unmountErr := volumeServerClient . VolumeUnmount ( ctx , & volume_server_pb . VolumeUnmountRequest {
@ -188,4 +188,3 @@ func ParseNeedleIdCookie(key_hash_string string) (NeedleId, Cookie, error) {
func ( n * Needle ) LastModifiedString ( ) string {
return time . Unix ( int64 ( n . LastModified ) , 0 ) . Format ( "2006-01-02T15:04:05" )
}
@ -185,7 +185,7 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version
case Version2 , Version3 :
err = n . readNeedleDataVersion2 ( bytes [ NeedleHeaderSize : NeedleHeaderSize + int ( n . Size ) ] )
}
if err != nil && err != io . EOF {
if err != nil && err != io . EOF {
return err
}
if size > 0 {
@ -390,4 +390,3 @@ func (n *Needle) SetHasPairs() {
func getActualSize ( size uint32 , version Version ) int64 {
return NeedleHeaderSize + NeedleBodyLength ( size , version )
}
@ -3,11 +3,11 @@ package needle
import "testing"
func TestNewVolumeId ( t * testing . T ) {
if _ , err := NewVolumeId ( "1" ) ; err != nil {
if _ , err := NewVolumeId ( "1" ) ; err != nil {
t . Error ( err )
}
if _ , err := NewVolumeId ( "a" ) ; err != nil {
if _ , err := NewVolumeId ( "a" ) ; err != nil {
t . Logf ( "a is not legal volume id, %v" , err )
}
}
@ -15,7 +15,7 @@ func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (lastAppendAtNs uin
return 0 , fmt . Errorf ( "verifyIndexFileIntegrity %s failed: %v" , indexFile . Name ( ) , e )
}
if indexSize == 0 {
return 0 , nil
return 0 , nil
}
var lastIdxEntry [ ] byte
if lastIdxEntry , e = readIndexEntryAtOffset ( indexFile , indexSize - NeedleMapEntrySize ) ; e != nil {
@ -23,7 +23,7 @@ func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (lastAppendAtNs uin
}
key , offset , size := IdxFileEntry ( lastIdxEntry )
if offset . IsZero ( ) {
return 0 , nil
return 0 , nil
}
if size == TombstoneFileSize {
size = 0
@ -114,10 +114,10 @@ func TestHandlingVolumeServerHeartbeat(t *testing.T) {
nil ,
dn )
for vid , _ := range layout . vid2location {
for vid , _ := range layout . vid2location {
println ( "after add volume id" , vid )
}
for _ , vid := range layout . writables {
for _ , vid := range layout . writables {
println ( "after add writable volume id" , vid )
}
@ -91,4 +91,3 @@ func UnGzipData(input []byte) ([]byte, error) {
return false , false
}
xxxxxxxxxx