316014408
9 years ago
committed by
GitHub
GitHub
67 changed files with 1665 additions and 870 deletions
-
1.gitignore
-
41.travis.yml
-
93Makefile
-
7README.md
-
19docker/Dockerfile
-
8docker/entrypoint.sh
-
29unmaintained/change_superblock/change_superblock.go
-
2unmaintained/fix_dat/fix_dat.go
-
2weed/command/backup.go
-
3weed/command/compact.go
-
2weed/command/filer_copy.go
-
2weed/command/fix.go
-
33weed/command/master.go
-
34weed/command/server.go
-
8weed/command/upload.go
-
2weed/command/volume.go
-
439weed/filer/postgres_store/postgres_store.go
-
3weed/images/orientation_test.go
-
78weed/operation/submit.go
-
203weed/operation/system_message.pb.go
-
59weed/operation/system_message_test.go
-
21weed/operation/upload_content.go
-
6weed/pb/Makefile
-
375weed/pb/seaweed.pb.go
-
40weed/pb/seaweed.proto
-
4weed/proto/Makefile
-
27weed/proto/system_message.proto
-
4weed/server/common.go
-
7weed/server/filer_server.go
-
2weed/server/filer_server_handlers_read.go
-
8weed/server/filer_server_handlers_write.go
-
69weed/server/master_grpc_server.go
-
9weed/server/master_server.go
-
44weed/server/master_server_handlers_admin.go
-
88weed/server/volume_grpc_client.go
-
34weed/server/volume_server.go
-
23weed/server/volume_server_handlers_admin.go
-
13weed/server/volume_server_handlers_read.go
-
9weed/storage/disk_location.go
-
47weed/storage/needle.go
-
2weed/storage/needle_map.go
-
6weed/storage/needle_map_boltdb.go
-
6weed/storage/needle_map_leveldb.go
-
6weed/storage/needle_map_memory.go
-
44weed/storage/needle_read_write.go
-
111weed/storage/store.go
-
5weed/storage/volume.go
-
22weed/storage/volume_checking.go
-
17weed/storage/volume_create.go
-
19weed/storage/volume_create_linux.go
-
25weed/storage/volume_info.go
-
9weed/storage/volume_loading.go
-
30weed/storage/volume_read_write.go
-
1weed/storage/volume_super_block.go
-
2weed/storage/volume_sync.go
-
18weed/storage/volume_vacuum.go
-
1weed/topology/allocate_volume.go
-
3weed/topology/data_node.go
-
8weed/topology/node.go
-
5weed/topology/rack.go
-
15weed/topology/store_replicate.go
-
38weed/topology/topology.go
-
16weed/topology/topology_event_handling.go
-
1weed/topology/volume_growth.go
-
2weed/util/constants.go
-
5weed/util/http_util.go
-
4weed/util/net_timeout.go
@ -1,15 +1,44 @@ |
|||
sudo: false |
|||
language: go |
|||
go: |
|||
- 1.5 |
|||
- 1.6 |
|||
- tip |
|||
- 1.5 |
|||
- 1.6 |
|||
- tip |
|||
|
|||
before_install: |
|||
- export PATH=/home/travis/gopath/bin:$PATH |
|||
- export PATH=/home/travis/gopath/bin:$PATH |
|||
|
|||
install: |
|||
- go get ./weed/... |
|||
- go get ./weed/... |
|||
|
|||
script: |
|||
- go test ./weed/... |
|||
- go test ./weed/... |
|||
|
|||
before_deploy: |
|||
- make release |
|||
deploy: |
|||
provider: releases |
|||
skip_cleanup: true |
|||
api_key: |
|||
secure: ERL986+ncQ8lwAJUYDrQ8s2/FxF/cyNIwJIFCqspnWxQgGNNyokET9HapmlPSxjpFRF0q6L2WCg9OY3mSVRq4oI6hg1igOQ12KlLyN71XSJ3c8w0Ay5ho48TQ9l3f3Iu97mntBCe9l0R9pnT8wj1VI8YJxloXwUMG2yeTjA9aBI= |
|||
file: |
|||
- build/linux_arm.tar.gz |
|||
- build/linux_arm64.tar.gz |
|||
- build/linux_386.tar.gz |
|||
- build/linux_amd64.tar.gz |
|||
- build/darwin_amd64.tar.gz |
|||
- build/windows_386.zip |
|||
- build/windows_amd64.zip |
|||
- build/freebsd_arm.tar.gz |
|||
- build/freebsd_amd64.tar.gz |
|||
- build/freebsd_386.tar.gz |
|||
- build/netbsd_arm.tar.gz |
|||
- build/netbsd_amd64.tar.gz |
|||
- build/netbsd_386.tar.gz |
|||
- build/openbsd_arm.tar.gz |
|||
- build/openbsd_amd64.tar.gz |
|||
- build/openbsd_386.tar.gz |
|||
on: |
|||
tags: true |
|||
repo: chrislusf/seaweedfs |
|||
go: tip |
|||
@ -1,18 +1,17 @@ |
|||
FROM progrium/busybox |
|||
FROM frolvlad/alpine-glibc:alpine-3.4 |
|||
|
|||
COPY entrypoint.sh /entrypoint.sh |
|||
COPY Dockerfile /etc/Dockerfile |
|||
|
|||
RUN opkg-install curl |
|||
RUN echo tlsv1 >> ~/.curlrc |
|||
|
|||
RUN curl -Lks https://bintray.com$(curl -Lk http://bintray.com/chrislusf/seaweedfs/seaweedfs/_latestVersion | grep linux_amd64.tar.gz | sed -n "/href/ s/.*href=['\"]\([^'\"]*\)['\"].*/\1/gp") | gunzip | tar -xf - && \ |
|||
mv go_*amd64/weed /usr/bin/ && \ |
|||
rm -r go_*amd64 |
|||
RUN apk add --no-cache --virtual=build-dependencies --update curl wget ca-certificates && \ |
|||
wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/latest | egrep -o '/chrislusf/seaweedfs/releases/download/.*/linux_amd64.tar.gz') && \ |
|||
tar -C /usr/bin/ -xzvf /tmp/linux_amd64.tar.gz && \ |
|||
apk del curl wget ca-certificates build-dependencies && \ |
|||
rm -rf /tmp/* |
|||
|
|||
EXPOSE 8080 |
|||
EXPOSE 9333 |
|||
|
|||
VOLUME /data |
|||
|
|||
COPY entrypoint.sh /entrypoint.sh |
|||
RUN chmod +x /entrypoint.sh |
|||
|
|||
ENTRYPOINT ["/entrypoint.sh"] |
|||
@ -1,203 +0,0 @@ |
|||
// Code generated by protoc-gen-go.
|
|||
// source: system_message.proto
|
|||
// DO NOT EDIT!
|
|||
|
|||
/* |
|||
Package operation is a generated protocol buffer package. |
|||
|
|||
It is generated from these files: |
|||
system_message.proto |
|||
|
|||
It has these top-level messages: |
|||
VolumeInformationMessage |
|||
JoinMessage |
|||
*/ |
|||
package operation |
|||
|
|||
import proto "github.com/golang/protobuf/proto" |
|||
import math "math" |
|||
|
|||
// Reference imports to suppress errors if they are not otherwise used.
|
|||
var _ = proto.Marshal |
|||
var _ = math.Inf |
|||
|
|||
type VolumeInformationMessage struct { |
|||
Id *uint32 `protobuf:"varint,1,req,name=id" json:"id,omitempty"` |
|||
Size *uint64 `protobuf:"varint,2,req,name=size" json:"size,omitempty"` |
|||
Collection *string `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"` |
|||
FileCount *uint64 `protobuf:"varint,4,req,name=file_count" json:"file_count,omitempty"` |
|||
DeleteCount *uint64 `protobuf:"varint,5,req,name=delete_count" json:"delete_count,omitempty"` |
|||
DeletedByteCount *uint64 `protobuf:"varint,6,req,name=deleted_byte_count" json:"deleted_byte_count,omitempty"` |
|||
ReadOnly *bool `protobuf:"varint,7,opt,name=read_only" json:"read_only,omitempty"` |
|||
ReplicaPlacement *uint32 `protobuf:"varint,8,req,name=replica_placement" json:"replica_placement,omitempty"` |
|||
Version *uint32 `protobuf:"varint,9,opt,name=version,def=2" json:"version,omitempty"` |
|||
Ttl *uint32 `protobuf:"varint,10,opt,name=ttl" json:"ttl,omitempty"` |
|||
XXX_unrecognized []byte `json:"-"` |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) Reset() { *m = VolumeInformationMessage{} } |
|||
func (m *VolumeInformationMessage) String() string { return proto.CompactTextString(m) } |
|||
func (*VolumeInformationMessage) ProtoMessage() {} |
|||
|
|||
const Default_VolumeInformationMessage_Version uint32 = 2 |
|||
|
|||
func (m *VolumeInformationMessage) GetId() uint32 { |
|||
if m != nil && m.Id != nil { |
|||
return *m.Id |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetSize() uint64 { |
|||
if m != nil && m.Size != nil { |
|||
return *m.Size |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetCollection() string { |
|||
if m != nil && m.Collection != nil { |
|||
return *m.Collection |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetFileCount() uint64 { |
|||
if m != nil && m.FileCount != nil { |
|||
return *m.FileCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetDeleteCount() uint64 { |
|||
if m != nil && m.DeleteCount != nil { |
|||
return *m.DeleteCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetDeletedByteCount() uint64 { |
|||
if m != nil && m.DeletedByteCount != nil { |
|||
return *m.DeletedByteCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetReadOnly() bool { |
|||
if m != nil && m.ReadOnly != nil { |
|||
return *m.ReadOnly |
|||
} |
|||
return false |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetReplicaPlacement() uint32 { |
|||
if m != nil && m.ReplicaPlacement != nil { |
|||
return *m.ReplicaPlacement |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetVersion() uint32 { |
|||
if m != nil && m.Version != nil { |
|||
return *m.Version |
|||
} |
|||
return Default_VolumeInformationMessage_Version |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetTtl() uint32 { |
|||
if m != nil && m.Ttl != nil { |
|||
return *m.Ttl |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
type JoinMessage struct { |
|||
IsInit *bool `protobuf:"varint,1,opt,name=is_init" json:"is_init,omitempty"` |
|||
Ip *string `protobuf:"bytes,2,req,name=ip" json:"ip,omitempty"` |
|||
Port *uint32 `protobuf:"varint,3,req,name=port" json:"port,omitempty"` |
|||
PublicUrl *string `protobuf:"bytes,4,opt,name=public_url" json:"public_url,omitempty"` |
|||
MaxVolumeCount *uint32 `protobuf:"varint,5,req,name=max_volume_count" json:"max_volume_count,omitempty"` |
|||
MaxFileKey *uint64 `protobuf:"varint,6,req,name=max_file_key" json:"max_file_key,omitempty"` |
|||
DataCenter *string `protobuf:"bytes,7,opt,name=data_center" json:"data_center,omitempty"` |
|||
Rack *string `protobuf:"bytes,8,opt,name=rack" json:"rack,omitempty"` |
|||
Volumes []*VolumeInformationMessage `protobuf:"bytes,9,rep,name=volumes" json:"volumes,omitempty"` |
|||
AdminPort *uint32 `protobuf:"varint,10,opt,name=admin_port" json:"admin_port,omitempty"` |
|||
XXX_unrecognized []byte `json:"-"` |
|||
} |
|||
|
|||
func (m *JoinMessage) Reset() { *m = JoinMessage{} } |
|||
func (m *JoinMessage) String() string { return proto.CompactTextString(m) } |
|||
func (*JoinMessage) ProtoMessage() {} |
|||
|
|||
func (m *JoinMessage) GetIsInit() bool { |
|||
if m != nil && m.IsInit != nil { |
|||
return *m.IsInit |
|||
} |
|||
return false |
|||
} |
|||
|
|||
func (m *JoinMessage) GetIp() string { |
|||
if m != nil && m.Ip != nil { |
|||
return *m.Ip |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *JoinMessage) GetPort() uint32 { |
|||
if m != nil && m.Port != nil { |
|||
return *m.Port |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *JoinMessage) GetPublicUrl() string { |
|||
if m != nil && m.PublicUrl != nil { |
|||
return *m.PublicUrl |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *JoinMessage) GetMaxVolumeCount() uint32 { |
|||
if m != nil && m.MaxVolumeCount != nil { |
|||
return *m.MaxVolumeCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *JoinMessage) GetMaxFileKey() uint64 { |
|||
if m != nil && m.MaxFileKey != nil { |
|||
return *m.MaxFileKey |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *JoinMessage) GetDataCenter() string { |
|||
if m != nil && m.DataCenter != nil { |
|||
return *m.DataCenter |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *JoinMessage) GetRack() string { |
|||
if m != nil && m.Rack != nil { |
|||
return *m.Rack |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *JoinMessage) GetVolumes() []*VolumeInformationMessage { |
|||
if m != nil { |
|||
return m.Volumes |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (m *JoinMessage) GetAdminPort() uint32 { |
|||
if m != nil && m.AdminPort != nil { |
|||
return *m.AdminPort |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func init() { |
|||
} |
|||
@ -1,59 +0,0 @@ |
|||
package operation |
|||
|
|||
import ( |
|||
"encoding/json" |
|||
"log" |
|||
"testing" |
|||
|
|||
"github.com/golang/protobuf/proto" |
|||
) |
|||
|
|||
func TestSerialDeserial(t *testing.T) { |
|||
volumeMessage := &VolumeInformationMessage{ |
|||
Id: proto.Uint32(12), |
|||
Size: proto.Uint64(2341234), |
|||
Collection: proto.String("benchmark"), |
|||
FileCount: proto.Uint64(2341234), |
|||
DeleteCount: proto.Uint64(234), |
|||
DeletedByteCount: proto.Uint64(21234), |
|||
ReadOnly: proto.Bool(false), |
|||
ReplicaPlacement: proto.Uint32(210), |
|||
Version: proto.Uint32(2), |
|||
} |
|||
var volumeMessages []*VolumeInformationMessage |
|||
volumeMessages = append(volumeMessages, volumeMessage) |
|||
|
|||
joinMessage := &JoinMessage{ |
|||
IsInit: proto.Bool(true), |
|||
Ip: proto.String("127.0.3.12"), |
|||
Port: proto.Uint32(34546), |
|||
PublicUrl: proto.String("localhost:2342"), |
|||
MaxVolumeCount: proto.Uint32(210), |
|||
MaxFileKey: proto.Uint64(324234423), |
|||
DataCenter: proto.String("dc1"), |
|||
Rack: proto.String("rack2"), |
|||
Volumes: volumeMessages, |
|||
} |
|||
|
|||
data, err := proto.Marshal(joinMessage) |
|||
if err != nil { |
|||
log.Fatal("marshaling error: ", err) |
|||
} |
|||
newMessage := &JoinMessage{} |
|||
err = proto.Unmarshal(data, newMessage) |
|||
if err != nil { |
|||
log.Fatal("unmarshaling error: ", err) |
|||
} |
|||
log.Println("The pb data size is", len(data)) |
|||
|
|||
jsonData, jsonError := json.Marshal(joinMessage) |
|||
if jsonError != nil { |
|||
log.Fatal("json marshaling error: ", jsonError) |
|||
} |
|||
log.Println("The json data size is", len(jsonData), string(jsonData)) |
|||
|
|||
// Now test and newTest contain the same data.
|
|||
if *joinMessage.PublicUrl != *newMessage.PublicUrl { |
|||
log.Fatalf("data mismatch %q != %q", *joinMessage.PublicUrl, *newMessage.PublicUrl) |
|||
} |
|||
} |
|||
@ -0,0 +1,6 @@ |
|||
all: gen |
|||
|
|||
.PHONY : gen |
|||
|
|||
gen: |
|||
protoc seaweed.proto --go_out=plugins=grpc:. |
|||
@ -0,0 +1,375 @@ |
|||
// Code generated by protoc-gen-go.
|
|||
// source: seaweed.proto
|
|||
// DO NOT EDIT!
|
|||
|
|||
/* |
|||
Package pb is a generated protocol buffer package. |
|||
|
|||
It is generated from these files: |
|||
seaweed.proto |
|||
|
|||
It has these top-level messages: |
|||
Heartbeat |
|||
HeartbeatResponse |
|||
VolumeInformationMessage |
|||
*/ |
|||
package pb |
|||
|
|||
import proto "github.com/golang/protobuf/proto" |
|||
import fmt "fmt" |
|||
import math "math" |
|||
|
|||
import ( |
|||
context "golang.org/x/net/context" |
|||
grpc "google.golang.org/grpc" |
|||
) |
|||
|
|||
// Reference imports to suppress errors if they are not otherwise used.
|
|||
var _ = proto.Marshal |
|||
var _ = fmt.Errorf |
|||
var _ = math.Inf |
|||
|
|||
// This is a compile-time assertion to ensure that this generated file
|
|||
// is compatible with the proto package it is being compiled against.
|
|||
// A compilation error at this line likely means your copy of the
|
|||
// proto package needs to be updated.
|
|||
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
|
|||
|
|||
type Heartbeat struct { |
|||
Ip string `protobuf:"bytes,1,opt,name=ip" json:"ip,omitempty"` |
|||
Port uint32 `protobuf:"varint,2,opt,name=port" json:"port,omitempty"` |
|||
PublicUrl string `protobuf:"bytes,3,opt,name=public_url,json=publicUrl" json:"public_url,omitempty"` |
|||
MaxVolumeCount uint32 `protobuf:"varint,4,opt,name=max_volume_count,json=maxVolumeCount" json:"max_volume_count,omitempty"` |
|||
MaxFileKey uint64 `protobuf:"varint,5,opt,name=max_file_key,json=maxFileKey" json:"max_file_key,omitempty"` |
|||
DataCenter string `protobuf:"bytes,6,opt,name=data_center,json=dataCenter" json:"data_center,omitempty"` |
|||
Rack string `protobuf:"bytes,7,opt,name=rack" json:"rack,omitempty"` |
|||
AdminPort uint32 `protobuf:"varint,8,opt,name=admin_port,json=adminPort" json:"admin_port,omitempty"` |
|||
Volumes []*VolumeInformationMessage `protobuf:"bytes,9,rep,name=volumes" json:"volumes,omitempty"` |
|||
} |
|||
|
|||
func (m *Heartbeat) Reset() { *m = Heartbeat{} } |
|||
func (m *Heartbeat) String() string { return proto.CompactTextString(m) } |
|||
func (*Heartbeat) ProtoMessage() {} |
|||
func (*Heartbeat) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } |
|||
|
|||
func (m *Heartbeat) GetIp() string { |
|||
if m != nil { |
|||
return m.Ip |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *Heartbeat) GetPort() uint32 { |
|||
if m != nil { |
|||
return m.Port |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *Heartbeat) GetPublicUrl() string { |
|||
if m != nil { |
|||
return m.PublicUrl |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *Heartbeat) GetMaxVolumeCount() uint32 { |
|||
if m != nil { |
|||
return m.MaxVolumeCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *Heartbeat) GetMaxFileKey() uint64 { |
|||
if m != nil { |
|||
return m.MaxFileKey |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *Heartbeat) GetDataCenter() string { |
|||
if m != nil { |
|||
return m.DataCenter |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *Heartbeat) GetRack() string { |
|||
if m != nil { |
|||
return m.Rack |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *Heartbeat) GetAdminPort() uint32 { |
|||
if m != nil { |
|||
return m.AdminPort |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *Heartbeat) GetVolumes() []*VolumeInformationMessage { |
|||
if m != nil { |
|||
return m.Volumes |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
type HeartbeatResponse struct { |
|||
VolumeSizeLimit uint64 `protobuf:"varint,1,opt,name=volumeSizeLimit" json:"volumeSizeLimit,omitempty"` |
|||
SecretKey string `protobuf:"bytes,2,opt,name=secretKey" json:"secretKey,omitempty"` |
|||
} |
|||
|
|||
func (m *HeartbeatResponse) Reset() { *m = HeartbeatResponse{} } |
|||
func (m *HeartbeatResponse) String() string { return proto.CompactTextString(m) } |
|||
func (*HeartbeatResponse) ProtoMessage() {} |
|||
func (*HeartbeatResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } |
|||
|
|||
func (m *HeartbeatResponse) GetVolumeSizeLimit() uint64 { |
|||
if m != nil { |
|||
return m.VolumeSizeLimit |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *HeartbeatResponse) GetSecretKey() string { |
|||
if m != nil { |
|||
return m.SecretKey |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
type VolumeInformationMessage struct { |
|||
Id uint32 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"` |
|||
Size uint64 `protobuf:"varint,2,opt,name=size" json:"size,omitempty"` |
|||
Collection string `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"` |
|||
FileCount uint64 `protobuf:"varint,4,opt,name=file_count,json=fileCount" json:"file_count,omitempty"` |
|||
DeleteCount uint64 `protobuf:"varint,5,opt,name=delete_count,json=deleteCount" json:"delete_count,omitempty"` |
|||
DeletedByteCount uint64 `protobuf:"varint,6,opt,name=deleted_byte_count,json=deletedByteCount" json:"deleted_byte_count,omitempty"` |
|||
ReadOnly bool `protobuf:"varint,7,opt,name=read_only,json=readOnly" json:"read_only,omitempty"` |
|||
ReplicaPlacement uint32 `protobuf:"varint,8,opt,name=replica_placement,json=replicaPlacement" json:"replica_placement,omitempty"` |
|||
Version uint32 `protobuf:"varint,9,opt,name=version" json:"version,omitempty"` |
|||
Ttl uint32 `protobuf:"varint,10,opt,name=ttl" json:"ttl,omitempty"` |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) Reset() { *m = VolumeInformationMessage{} } |
|||
func (m *VolumeInformationMessage) String() string { return proto.CompactTextString(m) } |
|||
func (*VolumeInformationMessage) ProtoMessage() {} |
|||
func (*VolumeInformationMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } |
|||
|
|||
func (m *VolumeInformationMessage) GetId() uint32 { |
|||
if m != nil { |
|||
return m.Id |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetSize() uint64 { |
|||
if m != nil { |
|||
return m.Size |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetCollection() string { |
|||
if m != nil { |
|||
return m.Collection |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetFileCount() uint64 { |
|||
if m != nil { |
|||
return m.FileCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetDeleteCount() uint64 { |
|||
if m != nil { |
|||
return m.DeleteCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetDeletedByteCount() uint64 { |
|||
if m != nil { |
|||
return m.DeletedByteCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetReadOnly() bool { |
|||
if m != nil { |
|||
return m.ReadOnly |
|||
} |
|||
return false |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetReplicaPlacement() uint32 { |
|||
if m != nil { |
|||
return m.ReplicaPlacement |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetVersion() uint32 { |
|||
if m != nil { |
|||
return m.Version |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetTtl() uint32 { |
|||
if m != nil { |
|||
return m.Ttl |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func init() { |
|||
proto.RegisterType((*Heartbeat)(nil), "pb.Heartbeat") |
|||
proto.RegisterType((*HeartbeatResponse)(nil), "pb.HeartbeatResponse") |
|||
proto.RegisterType((*VolumeInformationMessage)(nil), "pb.VolumeInformationMessage") |
|||
} |
|||
|
|||
// Reference imports to suppress errors if they are not otherwise used.
|
|||
var _ context.Context |
|||
var _ grpc.ClientConn |
|||
|
|||
// This is a compile-time assertion to ensure that this generated file
|
|||
// is compatible with the grpc package it is being compiled against.
|
|||
const _ = grpc.SupportPackageIsVersion4 |
|||
|
|||
// Client API for Seaweed service
|
|||
|
|||
type SeaweedClient interface { |
|||
SendHeartbeat(ctx context.Context, opts ...grpc.CallOption) (Seaweed_SendHeartbeatClient, error) |
|||
} |
|||
|
|||
type seaweedClient struct { |
|||
cc *grpc.ClientConn |
|||
} |
|||
|
|||
func NewSeaweedClient(cc *grpc.ClientConn) SeaweedClient { |
|||
return &seaweedClient{cc} |
|||
} |
|||
|
|||
func (c *seaweedClient) SendHeartbeat(ctx context.Context, opts ...grpc.CallOption) (Seaweed_SendHeartbeatClient, error) { |
|||
stream, err := grpc.NewClientStream(ctx, &_Seaweed_serviceDesc.Streams[0], c.cc, "/pb.Seaweed/SendHeartbeat", opts...) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
x := &seaweedSendHeartbeatClient{stream} |
|||
return x, nil |
|||
} |
|||
|
|||
type Seaweed_SendHeartbeatClient interface { |
|||
Send(*Heartbeat) error |
|||
Recv() (*HeartbeatResponse, error) |
|||
grpc.ClientStream |
|||
} |
|||
|
|||
type seaweedSendHeartbeatClient struct { |
|||
grpc.ClientStream |
|||
} |
|||
|
|||
func (x *seaweedSendHeartbeatClient) Send(m *Heartbeat) error { |
|||
return x.ClientStream.SendMsg(m) |
|||
} |
|||
|
|||
func (x *seaweedSendHeartbeatClient) Recv() (*HeartbeatResponse, error) { |
|||
m := new(HeartbeatResponse) |
|||
if err := x.ClientStream.RecvMsg(m); err != nil { |
|||
return nil, err |
|||
} |
|||
return m, nil |
|||
} |
|||
|
|||
// Server API for Seaweed service
|
|||
|
|||
type SeaweedServer interface { |
|||
SendHeartbeat(Seaweed_SendHeartbeatServer) error |
|||
} |
|||
|
|||
func RegisterSeaweedServer(s *grpc.Server, srv SeaweedServer) { |
|||
s.RegisterService(&_Seaweed_serviceDesc, srv) |
|||
} |
|||
|
|||
func _Seaweed_SendHeartbeat_Handler(srv interface{}, stream grpc.ServerStream) error { |
|||
return srv.(SeaweedServer).SendHeartbeat(&seaweedSendHeartbeatServer{stream}) |
|||
} |
|||
|
|||
type Seaweed_SendHeartbeatServer interface { |
|||
Send(*HeartbeatResponse) error |
|||
Recv() (*Heartbeat, error) |
|||
grpc.ServerStream |
|||
} |
|||
|
|||
type seaweedSendHeartbeatServer struct { |
|||
grpc.ServerStream |
|||
} |
|||
|
|||
func (x *seaweedSendHeartbeatServer) Send(m *HeartbeatResponse) error { |
|||
return x.ServerStream.SendMsg(m) |
|||
} |
|||
|
|||
func (x *seaweedSendHeartbeatServer) Recv() (*Heartbeat, error) { |
|||
m := new(Heartbeat) |
|||
if err := x.ServerStream.RecvMsg(m); err != nil { |
|||
return nil, err |
|||
} |
|||
return m, nil |
|||
} |
|||
|
|||
var _Seaweed_serviceDesc = grpc.ServiceDesc{ |
|||
ServiceName: "pb.Seaweed", |
|||
HandlerType: (*SeaweedServer)(nil), |
|||
Methods: []grpc.MethodDesc{}, |
|||
Streams: []grpc.StreamDesc{ |
|||
{ |
|||
StreamName: "SendHeartbeat", |
|||
Handler: _Seaweed_SendHeartbeat_Handler, |
|||
ServerStreams: true, |
|||
ClientStreams: true, |
|||
}, |
|||
}, |
|||
Metadata: "seaweed.proto", |
|||
} |
|||
|
|||
func init() { proto.RegisterFile("seaweed.proto", fileDescriptor0) } |
|||
|
|||
var fileDescriptor0 = []byte{ |
|||
// 489 bytes of a gzipped FileDescriptorProto
|
|||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x74, 0x93, 0xdf, 0x8a, 0xd3, 0x40, |
|||
0x14, 0xc6, 0x4d, 0x1a, 0xdb, 0xe6, 0x74, 0xbb, 0x76, 0x07, 0x84, 0x41, 0x57, 0x8d, 0xbd, 0x0a, |
|||
0x28, 0x45, 0x56, 0xf0, 0xc6, 0x3b, 0x17, 0x16, 0x65, 0x15, 0x97, 0x29, 0x7a, 0xe3, 0x45, 0x98, |
|||
0x24, 0x67, 0x65, 0xd8, 0xc9, 0x1f, 0x26, 0xd3, 0xb5, 0xd9, 0x07, 0xf2, 0x49, 0x7c, 0x30, 0x99, |
|||
0x33, 0x4d, 0xab, 0x82, 0x77, 0xe7, 0xfc, 0xce, 0x97, 0xe4, 0x9b, 0xf3, 0x4d, 0x60, 0xde, 0xa1, |
|||
0xfc, 0x81, 0x58, 0xae, 0x5a, 0xd3, 0xd8, 0x86, 0x85, 0x6d, 0xbe, 0xfc, 0x19, 0x42, 0xfc, 0x1e, |
|||
0xa5, 0xb1, 0x39, 0x4a, 0xcb, 0x8e, 0x21, 0x54, 0x2d, 0x0f, 0x92, 0x20, 0x8d, 0x45, 0xa8, 0x5a, |
|||
0xc6, 0x20, 0x6a, 0x1b, 0x63, 0x79, 0x98, 0x04, 0xe9, 0x5c, 0x50, 0xcd, 0x9e, 0x00, 0xb4, 0x9b, |
|||
0x5c, 0xab, 0x22, 0xdb, 0x18, 0xcd, 0x47, 0xa4, 0x8d, 0x3d, 0xf9, 0x62, 0x34, 0x4b, 0x61, 0x51, |
|||
0xc9, 0x6d, 0x76, 0xdb, 0xe8, 0x4d, 0x85, 0x59, 0xd1, 0x6c, 0x6a, 0xcb, 0x23, 0x7a, 0xfc, 0xb8, |
|||
0x92, 0xdb, 0xaf, 0x84, 0xcf, 0x1d, 0x65, 0x09, 0x1c, 0x39, 0xe5, 0xb5, 0xd2, 0x98, 0xdd, 0x60, |
|||
0xcf, 0xef, 0x27, 0x41, 0x1a, 0x09, 0xa8, 0xe4, 0xf6, 0x42, 0x69, 0xbc, 0xc4, 0x9e, 0x3d, 0x83, |
|||
0x59, 0x29, 0xad, 0xcc, 0x0a, 0xac, 0x2d, 0x1a, 0x3e, 0xa6, 0x6f, 0x81, 0x43, 0xe7, 0x44, 0x9c, |
|||
0x3f, 0x23, 0x8b, 0x1b, 0x3e, 0xa1, 0x09, 0xd5, 0xce, 0x9f, 0x2c, 0x2b, 0x55, 0x67, 0xe4, 0x7c, |
|||
0x4a, 0x9f, 0x8e, 0x89, 0x5c, 0x39, 0xfb, 0x6f, 0x60, 0xe2, 0xbd, 0x75, 0x3c, 0x4e, 0x46, 0xe9, |
|||
0xec, 0xec, 0x74, 0xd5, 0xe6, 0x2b, 0xef, 0xeb, 0x43, 0x7d, 0xdd, 0x98, 0x4a, 0x5a, 0xd5, 0xd4, |
|||
0x9f, 0xb0, 0xeb, 0xe4, 0x77, 0x14, 0x83, 0x78, 0xf9, 0x0d, 0x4e, 0xf6, 0x7b, 0x12, 0xd8, 0xb5, |
|||
0x4d, 0xdd, 0x21, 0x4b, 0xe1, 0x81, 0x9f, 0xaf, 0xd5, 0x1d, 0x7e, 0x54, 0x95, 0xb2, 0xb4, 0xbc, |
|||
0x48, 0xfc, 0x8b, 0xd9, 0x29, 0xc4, 0x1d, 0x16, 0x06, 0xed, 0x25, 0xf6, 0xb4, 0xce, 0x58, 0x1c, |
|||
0xc0, 0xf2, 0x57, 0x08, 0xfc, 0x7f, 0x16, 0x28, 0x94, 0x92, 0xde, 0x3b, 0x17, 0xa1, 0x2a, 0xdd, |
|||
0xa1, 0x3b, 0x75, 0x87, 0xf4, 0x96, 0x48, 0x50, 0xcd, 0x9e, 0x02, 0x14, 0x8d, 0xd6, 0x58, 0xb8, |
|||
0x07, 0x77, 0xa1, 0xfc, 0x41, 0xdc, 0x52, 0x68, 0xcf, 0x87, 0x3c, 0x22, 0x11, 0x3b, 0xe2, 0xa3, |
|||
0x78, 0x0e, 0x47, 0x25, 0x6a, 0xb4, 0x83, 0xc0, 0x47, 0x31, 0xf3, 0xcc, 0x4b, 0x5e, 0x02, 0xf3, |
|||
0x6d, 0x99, 0xe5, 0xfd, 0x5e, 0x38, 0x26, 0xe1, 0x62, 0x37, 0x79, 0xd7, 0x0f, 0xea, 0xc7, 0x10, |
|||
0x1b, 0x94, 0x65, 0xd6, 0xd4, 0xba, 0xa7, 0x74, 0xa6, 0x62, 0xea, 0xc0, 0xe7, 0x5a, 0xf7, 0xec, |
|||
0x05, 0x9c, 0x18, 0x6c, 0xb5, 0x2a, 0x64, 0xd6, 0x6a, 0x59, 0x60, 0x85, 0xf5, 0x10, 0xd4, 0x62, |
|||
0x37, 0xb8, 0x1a, 0x38, 0xe3, 0x30, 0xb9, 0x45, 0xd3, 0xb9, 0x63, 0xc5, 0x24, 0x19, 0x5a, 0xb6, |
|||
0x80, 0x91, 0xb5, 0x9a, 0x03, 0x51, 0x57, 0x9e, 0x5d, 0xc0, 0x64, 0xed, 0x6f, 0x38, 0x7b, 0x0b, |
|||
0xf3, 0x35, 0xd6, 0xe5, 0xe1, 0x6a, 0xcf, 0x5d, 0xcc, 0xfb, 0xf6, 0xd1, 0xc3, 0xbf, 0xda, 0x21, |
|||
0xd0, 0xe5, 0xbd, 0x34, 0x78, 0x15, 0xe4, 0x63, 0xfa, 0x3f, 0x5e, 0xff, 0x0e, 0x00, 0x00, 0xff, |
|||
0xff, 0xd5, 0x08, 0xa6, 0xf2, 0x30, 0x03, 0x00, 0x00, |
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
syntax = "proto3"; |
|||
|
|||
package pb; |
|||
|
|||
////////////////////////////////////////////////// |
|||
|
|||
service Seaweed { |
|||
rpc SendHeartbeat(stream Heartbeat) returns (stream HeartbeatResponse) {} |
|||
} |
|||
|
|||
////////////////////////////////////////////////// |
|||
|
|||
message Heartbeat { |
|||
string ip = 1; |
|||
uint32 port = 2; |
|||
string public_url = 3; |
|||
uint32 max_volume_count = 4; |
|||
uint64 max_file_key = 5; |
|||
string data_center = 6; |
|||
string rack = 7; |
|||
uint32 admin_port = 8; |
|||
repeated VolumeInformationMessage volumes = 9; |
|||
} |
|||
message HeartbeatResponse { |
|||
uint64 volumeSizeLimit = 1; |
|||
string secretKey = 2; |
|||
} |
|||
|
|||
message VolumeInformationMessage { |
|||
uint32 id = 1; |
|||
uint64 size = 2; |
|||
string collection = 3; |
|||
uint64 file_count = 4; |
|||
uint64 delete_count = 5; |
|||
uint64 deleted_byte_count = 6; |
|||
bool read_only = 7; |
|||
uint32 replica_placement = 8; |
|||
uint32 version = 9; |
|||
uint32 ttl = 10; |
|||
} |
|||
@ -1,4 +0,0 @@ |
|||
TARG=../operation |
|||
|
|||
all: |
|||
protoc --go_out=$(TARG) system_message.proto |
|||
@ -1,27 +0,0 @@ |
|||
package operation; |
|||
|
|||
message VolumeInformationMessage { |
|||
required uint32 id = 1; |
|||
required uint64 size = 2; |
|||
optional string collection = 3; |
|||
required uint64 file_count = 4; |
|||
required uint64 delete_count = 5; |
|||
required uint64 deleted_byte_count = 6; |
|||
optional bool read_only = 7; |
|||
required uint32 replica_placement = 8; |
|||
optional uint32 version = 9 [default=2]; |
|||
optional uint32 ttl = 10; |
|||
} |
|||
|
|||
message JoinMessage { |
|||
optional bool is_init = 1; |
|||
required string ip = 2; |
|||
required uint32 port = 3; |
|||
optional string public_url = 4; |
|||
required uint32 max_volume_count = 5; |
|||
required uint64 max_file_key = 6; |
|||
optional string data_center = 7; |
|||
optional string rack = 8; |
|||
repeated VolumeInformationMessage volumes = 9; |
|||
optional uint32 admin_port = 10; |
|||
} |
|||
@ -0,0 +1,69 @@ |
|||
package weed_server |
|||
|
|||
import ( |
|||
"net" |
|||
"strings" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/pb" |
|||
"github.com/chrislusf/seaweedfs/weed/storage" |
|||
"github.com/chrislusf/seaweedfs/weed/topology" |
|||
"google.golang.org/grpc/peer" |
|||
) |
|||
|
|||
func (ms MasterServer) SendHeartbeat(stream pb.Seaweed_SendHeartbeatServer) error { |
|||
var dn *topology.DataNode |
|||
t := ms.Topo |
|||
for { |
|||
heartbeat, err := stream.Recv() |
|||
if err == nil { |
|||
if dn == nil { |
|||
t.Sequence.SetMax(heartbeat.MaxFileKey) |
|||
if heartbeat.Ip == "" { |
|||
if pr, ok := peer.FromContext(stream.Context()); ok { |
|||
if pr.Addr != net.Addr(nil) { |
|||
heartbeat.Ip = pr.Addr.String()[0:strings.LastIndex(pr.Addr.String(), ":")] |
|||
glog.V(0).Infof("remote IP address is detected as %v", heartbeat.Ip) |
|||
} |
|||
} |
|||
} |
|||
dcName, rackName := t.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack) |
|||
dc := t.GetOrCreateDataCenter(dcName) |
|||
rack := dc.GetOrCreateRack(rackName) |
|||
dn = rack.GetOrCreateDataNode(heartbeat.Ip, |
|||
int(heartbeat.Port), heartbeat.PublicUrl, |
|||
int(heartbeat.MaxVolumeCount)) |
|||
glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort()) |
|||
if err := stream.Send(&pb.HeartbeatResponse{ |
|||
VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024, |
|||
SecretKey: string(ms.guard.SecretKey), |
|||
}); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
var volumeInfos []storage.VolumeInfo |
|||
for _, v := range heartbeat.Volumes { |
|||
if vi, err := storage.NewVolumeInfo(v); err == nil { |
|||
volumeInfos = append(volumeInfos, vi) |
|||
} else { |
|||
glog.V(0).Infof("Fail to convert joined volume information: %v", err) |
|||
} |
|||
} |
|||
deletedVolumes := dn.UpdateVolumes(volumeInfos) |
|||
for _, v := range volumeInfos { |
|||
t.RegisterVolumeLayout(v, dn) |
|||
} |
|||
for _, v := range deletedVolumes { |
|||
t.UnRegisterVolumeLayout(v, dn) |
|||
} |
|||
|
|||
} else { |
|||
glog.V(0).Infof("lost volume server %s:%d", dn.Ip, dn.Port) |
|||
if dn != nil { |
|||
t.UnRegisterDataNode(dn) |
|||
} |
|||
return err |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,88 @@ |
|||
package weed_server |
|||
|
|||
import ( |
|||
"fmt" |
|||
"time" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/pb" |
|||
"github.com/chrislusf/seaweedfs/weed/security" |
|||
"github.com/chrislusf/seaweedfs/weed/storage" |
|||
"golang.org/x/net/context" |
|||
"google.golang.org/grpc" |
|||
) |
|||
|
|||
func (vs *VolumeServer) heartbeat() { |
|||
|
|||
glog.V(0).Infof("Volume server bootstraps with master %s", vs.GetMasterNode()) |
|||
vs.masterNodes = storage.NewMasterNodes(vs.masterNode) |
|||
vs.store.SetDataCenter(vs.dataCenter) |
|||
vs.store.SetRack(vs.rack) |
|||
|
|||
for { |
|||
err := vs.doHeartbeat(time.Duration(vs.pulseSeconds) * time.Second) |
|||
if err != nil { |
|||
glog.V(0).Infof("heartbeat error: %v", err) |
|||
time.Sleep(time.Duration(3*vs.pulseSeconds) * time.Second) |
|||
} |
|||
} |
|||
} |
|||
|
|||
func (vs *VolumeServer) doHeartbeat(sleepInterval time.Duration) error { |
|||
|
|||
masterNode, err := vs.masterNodes.FindMaster() |
|||
if err != nil { |
|||
return fmt.Errorf("No master found: %v", err) |
|||
} |
|||
|
|||
grpcConection, err := grpc.Dial(masterNode, grpc.WithInsecure()) |
|||
if err != nil { |
|||
return fmt.Errorf("fail to dial: %v", err) |
|||
} |
|||
defer grpcConection.Close() |
|||
|
|||
client := pb.NewSeaweedClient(grpcConection) |
|||
stream, err := client.SendHeartbeat(context.Background()) |
|||
if err != nil { |
|||
glog.V(0).Infof("%v.SendHeartbeat(_) = _, %v", client, err) |
|||
return err |
|||
} |
|||
vs.SetMasterNode(masterNode) |
|||
glog.V(0).Infof("Heartbeat to %s", masterNode) |
|||
|
|||
vs.store.Client = stream |
|||
defer func() { vs.store.Client = nil }() |
|||
|
|||
doneChan := make(chan error, 1) |
|||
|
|||
go func() { |
|||
for { |
|||
in, err := stream.Recv() |
|||
if err != nil { |
|||
doneChan <- err |
|||
return |
|||
} |
|||
vs.store.VolumeSizeLimit = in.GetVolumeSizeLimit() |
|||
vs.guard.SecretKey = security.Secret(in.GetSecretKey()) |
|||
} |
|||
}() |
|||
|
|||
if err = stream.Send(vs.store.CollectHeartbeat()); err != nil { |
|||
glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err) |
|||
return err |
|||
} |
|||
|
|||
tickChan := time.NewTimer(sleepInterval).C |
|||
|
|||
for { |
|||
select { |
|||
case <-tickChan: |
|||
if err = stream.Send(vs.store.CollectHeartbeat()); err != nil { |
|||
glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err) |
|||
return err |
|||
} |
|||
case err := <-doneChan: |
|||
return err |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,17 @@ |
|||
// +build !linux
|
|||
|
|||
package storage |
|||
|
|||
import ( |
|||
"os" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
) |
|||
|
|||
func createVolumeFile(fileName string, preallocate int64) (file *os.File, e error) { |
|||
file, e = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644) |
|||
if preallocate > 0 { |
|||
glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName) |
|||
} |
|||
return file, e |
|||
} |
|||
@ -0,0 +1,19 @@ |
|||
// +build linux
|
|||
|
|||
package storage |
|||
|
|||
import ( |
|||
"os" |
|||
"syscall" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
) |
|||
|
|||
func createVolumeFile(fileName string, preallocate int64) (file *os.File, e error) { |
|||
file, e = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644) |
|||
if preallocate != 0 { |
|||
syscall.Fallocate(int(file.Fd()), 1, 0, preallocate) |
|||
glog.V(0).Infof("Preallocated %d bytes disk space for %s", preallocate, fileName) |
|||
} |
|||
return file, e |
|||
} |
|||
@ -1,5 +1,5 @@ |
|||
package util |
|||
|
|||
const ( |
|||
VERSION = "0.71 beta" |
|||
VERSION = "0.72" |
|||
) |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue