Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

filer: able to tail meta data changes

pull/1287/head
Chris Lu 5 years ago
parent
2a2d92d06e
commit
bf270d9e8c
11 changed files with 366 additions and 179 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      other/java/client/src/main/proto/filer.proto
  2. 15
      weed/command/command.go
  3. 63
      weed/command/tail.go
  4. 4
      weed/filer2/filer.go
  5. 37
      weed/filer2/filer_notify.go
  6. 19
      weed/operation/grpc_client.go
  7. 2
      weed/pb/filer.proto
  8. 242
      weed/pb/filer_pb/filer.pb.go
  9. 82
      weed/queue/log_buffer.go
  10. 71
      weed/server/filer_grpc_server_listen.go
  11. 8
      weed/server/filer_server.go

2
other/java/client/src/main/proto/filer.proto
View File

@ -237,7 +237,7 @@ message GetFilerConfigurationResponse {
message ListenForEventsRequest {
string client_name = 1;
string directory = 2;
int64 since_sec = 3;
int64 since_ns = 3;
}
message FullEventNotification {
string directory = 1;

15
weed/command/command.go
View File

@ -12,21 +12,22 @@ var Commands = []*Command{
cmdBackup,
cmdCompact,
cmdCopy,
cmdFix,
cmdDownload,
cmdExport,
cmdFiler,
cmdFilerReplicate,
cmdServer,
cmdFix,
cmdMaster,
cmdFiler,
cmdMount,
cmdS3,
cmdUpload,
cmdDownload,
cmdMsgBroker,
cmdScaffold,
cmdServer,
cmdShell,
cmdTail,
cmdUpload,
cmdVersion,
cmdVolume,
cmdExport,
cmdMount,
cmdWebDav,
}

63
weed/command/tail.go
View File

@ -0,0 +1,63 @@
package command
import (
"context"
"fmt"
"io"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
)
func init() {
cmdTail.Run = runTail // break init cycle
}
var cmdTail = &Command{
UsageLine: "tail <wip> [-filer=localhost:8888]",
Short: "see recent changes on a filer",
Long: `See recent changes on a filer.
`,
}
var (
tailFiler = cmdTail.Flag.String("filer", "localhost:8888", "filer hostname:port")
tailTarget = cmdTail.Flag.String("target", "/", "a folder or file on filer")
)
func runTail(cmd *Command, args []string) bool {
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
tailErr := pb.WithFilerClient(*tailFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
stream, err := client.ListenForEvents(context.Background(), &filer_pb.ListenForEventsRequest{
ClientName: "tail",
Directory: *tailTarget,
SinceNs: 0,
})
if err != nil {
return fmt.Errorf("listen: %v", err)
}
for {
resp, listenErr := stream.Recv()
if listenErr == io.EOF {
return nil
}
if listenErr != nil {
return listenErr
}
fmt.Printf("events: %+v\n", resp.EventNotification)
}
})
if tailErr != nil {
fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)
}
return true
}

4
weed/filer2/filer.go
View File

@ -39,14 +39,14 @@ type Filer struct {
metaLogBuffer *queue.LogBuffer
}
func NewFiler(masters []string, grpcDialOption grpc.DialOption, filerGrpcPort uint32) *Filer {
func NewFiler(masters []string, grpcDialOption grpc.DialOption, filerGrpcPort uint32, notifyFn func()) *Filer {
f := &Filer{
directoryCache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
MasterClient: wdclient.NewMasterClient(grpcDialOption, "filer", filerGrpcPort, masters),
fileIdDeletionQueue: util.NewUnboundedQueue(),
GrpcDialOption: grpcDialOption,
}
f.metaLogBuffer = queue.NewLogBuffer(time.Minute, f.logFlushFunc)
f.metaLogBuffer = queue.NewLogBuffer(time.Minute, f.logFlushFunc, notifyFn)
go f.loopProcessingDeletion()

37
weed/filer2/filer_notify.go
View File

@ -77,3 +77,40 @@ func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) {
}
}
func (f *Filer) ReadLogBuffer(lastReadTime time.Time, eachEventFn func(fullpath string, eventNotification *filer_pb.EventNotification) error) (newLastReadTime time.Time, err error) {
var buf []byte
newLastReadTime, buf = f.metaLogBuffer.ReadFromBuffer(lastReadTime)
for pos := 0; pos+4 < len(buf); {
size := util.BytesToUint32(buf[pos : pos+4])
entryData := buf[pos+4 : pos+4+int(size)]
logEntry := &filer_pb.LogEntry{}
err = proto.Unmarshal(entryData, logEntry)
if err != nil {
glog.Errorf("unexpected unmarshal filer_pb.LogEntry: %v", err)
return lastReadTime, fmt.Errorf("unexpected unmarshal filer_pb.LogEntry: %v", err)
}
event := &filer_pb.FullEventNotification{}
err = proto.Unmarshal(logEntry.Data, event)
if err != nil {
glog.Errorf("unexpected unmarshal filer_pb.FullEventNotification: %v", err)
return lastReadTime, fmt.Errorf("unexpected unmarshal filer_pb.FullEventNotification: %v", err)
}
err = eachEventFn(event.Directory, event.EventNotification)
if err != nil {
return
}
pos += 4 + int(size)
}
return
}

19
weed/operation/grpc_client.go
View File

@ -9,6 +9,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
)
@ -17,7 +18,7 @@ func WithVolumeServerClient(volumeServer string, grpcDialOption grpc.DialOption,
grpcAddress, err := toVolumeServerGrpcAddress(volumeServer)
if err != nil {
return err
return fmt.Errorf("failed to parse volume server %v: %v", volumeServer, err)
}
return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
@ -41,7 +42,7 @@ func WithMasterServerClient(masterServer string, grpcDialOption grpc.DialOption,
masterGrpcAddress, parseErr := pb.ParseServerToGrpcAddress(masterServer)
if parseErr != nil {
return fmt.Errorf("failed to parse master grpc %v: %v", masterServer, parseErr)
return fmt.Errorf("failed to parse master %v: %v", masterServer, parseErr)
}
return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
@ -50,3 +51,17 @@ func WithMasterServerClient(masterServer string, grpcDialOption grpc.DialOption,
}, masterGrpcAddress, grpcDialOption)
}
func WithFilerServerClient(filerServer string, grpcDialOption grpc.DialOption, fn func(masterClient filer_pb.SeaweedFilerClient) error) error {
filerGrpcAddress, parseErr := pb.ParseServerToGrpcAddress(filerServer)
if parseErr != nil {
return fmt.Errorf("failed to parse filer %v: %v", filerGrpcAddress, parseErr)
}
return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
return fn(client)
}, filerGrpcAddress, grpcDialOption)
}

2
weed/pb/filer.proto
View File

@ -237,7 +237,7 @@ message GetFilerConfigurationResponse {
message ListenForEventsRequest {
string client_name = 1;
string directory = 2;
int64 since_sec = 3;
int64 since_ns = 3;
}
message FullEventNotification {
string directory = 1;

242
weed/pb/filer_pb/filer.pb.go
View File

@ -1093,7 +1093,7 @@ func (m *GetFilerConfigurationResponse) GetCipher() bool {
type ListenForEventsRequest struct {
ClientName string `protobuf:"bytes,1,opt,name=client_name,json=clientName" json:"client_name,omitempty"`
Directory string `protobuf:"bytes,2,opt,name=directory" json:"directory,omitempty"`
SinceSec int64 `protobuf:"varint,3,opt,name=since_sec,json=sinceSec" json:"since_sec,omitempty"`
SinceNs int64 `protobuf:"varint,3,opt,name=since_ns,json=sinceNs" json:"since_ns,omitempty"`
}
func (m *ListenForEventsRequest) Reset() { *m = ListenForEventsRequest{} }
@ -1115,9 +1115,9 @@ func (m *ListenForEventsRequest) GetDirectory() string {
return ""
}
func (m *ListenForEventsRequest) GetSinceSec() int64 {
func (m *ListenForEventsRequest) GetSinceNs() int64 {
if m != nil {
return m.SinceSec
return m.SinceNs
}
return 0
}
@ -1709,122 +1709,122 @@ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 1901 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x5f, 0x6f, 0xdc, 0xc6,
0x11, 0x37, 0xef, 0x9f, 0x8e, 0x73, 0x77, 0xb6, 0xb4, 0x27, 0x39, 0xe7, 0x93, 0x64, 0x2b, 0x74,
0x9d, 0xba, 0xb0, 0xa1, 0x1a, 0x6a, 0x1e, 0x92, 0xa6, 0x7d, 0xb0, 0x65, 0x29, 0x75, 0x63, 0x2b,
0x2e, 0x65, 0x17, 0x29, 0x0a, 0x94, 0xa0, 0xc8, 0xd5, 0x69, 0x2b, 0x1e, 0xc9, 0xec, 0x2e, 0xf5,
0x27, 0x6f, 0xfd, 0x1a, 0x05, 0xfa, 0xd0, 0xef, 0xd0, 0xc7, 0xa2, 0x2f, 0x45, 0x81, 0x7e, 0x8e,
0x3e, 0xf6, 0xa1, 0x9f, 0xa1, 0xd8, 0x59, 0x92, 0xb7, 0x3c, 0x9e, 0xa4, 0x04, 0x41, 0xde, 0x76,
0x67, 0x66, 0x67, 0x67, 0xe7, 0xcf, 0x6f, 0x86, 0x84, 0xde, 0x31, 0x8b, 0x28, 0xdf, 0x4e, 0x79,
0x22, 0x13, 0xd2, 0xc5, 0x8d, 0x97, 0x1e, 0x39, 0x5f, 0xc2, 0xfa, 0xeb, 0x24, 0x39, 0xcd, 0xd2,
0x97, 0x8c, 0xd3, 0x40, 0x26, 0xfc, 0x72, 0x2f, 0x96, 0xfc, 0xd2, 0xa5, 0x5f, 0x67, 0x54, 0x48,
0xb2, 0x01, 0x76, 0x58, 0x30, 0x46, 0xd6, 0x96, 0xf5, 0xd8, 0x76, 0x67, 0x04, 0x42, 0xa0, 0x15,
0xfb, 0x53, 0x3a, 0x6a, 0x20, 0x03, 0xd7, 0xce, 0x1e, 0x6c, 0x2c, 0x56, 0x28, 0xd2, 0x24, 0x16,
0x94, 0x3c, 0x82, 0x36, 0x55, 0x04, 0xd4, 0xd6, 0xdb, 0xb9, 0xb3, 0x5d, 0x98, 0xb2, 0xad, 0xe5,
0x34, 0xd7, 0xf9, 0x87, 0x05, 0xe4, 0x35, 0x13, 0x52, 0x11, 0x19, 0x15, 0xdf, 0xce, 0x9e, 0xbb,
0xd0, 0x49, 0x39, 0x3d, 0x66, 0x17, 0xb9, 0x45, 0xf9, 0x8e, 0x3c, 0x85, 0x15, 0x21, 0x7d, 0x2e,
0xf7, 0x79, 0x32, 0xdd, 0x67, 0x11, 0x3d, 0x50, 0x46, 0x37, 0x51, 0xa4, 0xce, 0x20, 0xdb, 0x40,
0x58, 0x1c, 0x44, 0x99, 0x60, 0x67, 0xf4, 0xb0, 0xe0, 0x8e, 0x5a, 0x5b, 0xd6, 0xe3, 0xae, 0xbb,
0x80, 0x43, 0x56, 0xa1, 0x1d, 0xb1, 0x29, 0x93, 0xa3, 0xf6, 0x96, 0xf5, 0x78, 0xe0, 0xea, 0x8d,
0xf3, 0x0b, 0x18, 0x56, 0xec, 0xff, 0x6e, 0xcf, 0xff, 0x4b, 0x03, 0xda, 0x48, 0x28, 0x7d, 0x6c,
0xcd, 0x7c, 0x4c, 0x3e, 0x84, 0x3e, 0x13, 0xde, 0xcc, 0x11, 0x0d, 0xb4, 0xad, 0xc7, 0x44, 0xe9,
0x73, 0xf2, 0x04, 0x3a, 0xc1, 0x49, 0x16, 0x9f, 0x8a, 0x51, 0x73, 0xab, 0xf9, 0xb8, 0xb7, 0x33,
0x9c, 0x5d, 0xa4, 0x1e, 0xba, 0xab, 0x78, 0x6e, 0x2e, 0x42, 0x3e, 0x01, 0xf0, 0xa5, 0xe4, 0xec,
0x28, 0x93, 0x54, 0xe0, 0x4b, 0x7b, 0x3b, 0x23, 0xe3, 0x40, 0x26, 0xe8, 0xf3, 0x92, 0xef, 0x1a,
0xb2, 0xe4, 0x53, 0xe8, 0xd2, 0x0b, 0x49, 0xe3, 0x90, 0x86, 0xa3, 0x36, 0x5e, 0xb4, 0x39, 0xf7,
0xa2, 0xed, 0xbd, 0x9c, 0xaf, 0xdf, 0x57, 0x8a, 0x8f, 0x3f, 0x83, 0x41, 0x85, 0x45, 0x96, 0xa1,
0x79, 0x4a, 0x8b, 0xa8, 0xaa, 0xa5, 0xf2, 0xec, 0x99, 0x1f, 0x65, 0x3a, 0xc1, 0xfa, 0xae, 0xde,
0xfc, 0xbc, 0xf1, 0x89, 0xe5, 0xbc, 0x04, 0x7b, 0x3f, 0x8b, 0xa2, 0xf2, 0x60, 0xc8, 0x78, 0x71,
0x30, 0x64, 0x7c, 0xe6, 0xe5, 0xc6, 0xb5, 0x5e, 0xfe, 0xbb, 0x05, 0x2b, 0x7b, 0x67, 0x34, 0x96,
0x07, 0x89, 0x64, 0xc7, 0x2c, 0xf0, 0x25, 0x4b, 0x62, 0xf2, 0x14, 0xec, 0x24, 0x0a, 0xbd, 0x6b,
0xc3, 0xd4, 0x4d, 0xa2, 0xdc, 0xea, 0xa7, 0x60, 0xc7, 0xf4, 0xdc, 0xbb, 0xf6, 0xba, 0x6e, 0x4c,
0xcf, 0xb5, 0xf4, 0x43, 0x18, 0x84, 0x34, 0xa2, 0x92, 0x7a, 0x65, 0x74, 0x54, 0xe8, 0xfa, 0x9a,
0xb8, 0xab, 0xc3, 0xf1, 0x11, 0xdc, 0x51, 0x2a, 0x53, 0x9f, 0xd3, 0x58, 0x7a, 0xa9, 0x2f, 0x4f,
0x30, 0x26, 0xb6, 0x3b, 0x88, 0xe9, 0xf9, 0x5b, 0xa4, 0xbe, 0xf5, 0xe5, 0x89, 0xf3, 0xb7, 0x06,
0xd8, 0x65, 0x30, 0xc9, 0x07, 0xb0, 0xa4, 0xae, 0xf5, 0x58, 0x98, 0x7b, 0xa2, 0xa3, 0xb6, 0xaf,
0x42, 0x55, 0x15, 0xc9, 0xf1, 0xb1, 0xa0, 0x12, 0xcd, 0x6b, 0xba, 0xf9, 0x4e, 0x65, 0x96, 0x60,
0xdf, 0xe8, 0x42, 0x68, 0xb9, 0xb8, 0x56, 0x1e, 0x9f, 0x4a, 0x36, 0xa5, 0x78, 0x61, 0xd3, 0xd5,
0x1b, 0x32, 0x84, 0x36, 0xf5, 0xa4, 0x3f, 0xc1, 0x0c, 0xb7, 0xdd, 0x16, 0x7d, 0xe7, 0x4f, 0xc8,
0x8f, 0xe0, 0xb6, 0x48, 0x32, 0x1e, 0x50, 0xaf, 0xb8, 0xb6, 0x83, 0xdc, 0xbe, 0xa6, 0xee, 0xeb,
0xcb, 0x1d, 0x68, 0x1e, 0xb3, 0x70, 0xb4, 0x84, 0x8e, 0x59, 0xae, 0x26, 0xe1, 0xab, 0xd0, 0x55,
0x4c, 0xf2, 0x53, 0x80, 0x52, 0x53, 0x38, 0xea, 0x5e, 0x21, 0x6a, 0x17, 0x7a, 0x43, 0xb2, 0x09,
0x10, 0xb0, 0xf4, 0x84, 0x72, 0x4f, 0x25, 0x8c, 0x8d, 0xc9, 0x61, 0x6b, 0xca, 0x17, 0xf4, 0x52,
0xb1, 0x99, 0xf0, 0x26, 0xdf, 0xb0, 0x34, 0xa5, 0xe1, 0x08, 0xd0, 0xc3, 0x36, 0x13, 0x9f, 0x6b,
0x82, 0xf3, 0x15, 0x74, 0x72, 0xe3, 0xd6, 0xc1, 0x3e, 0x4b, 0xa2, 0x6c, 0x5a, 0x3a, 0x6d, 0xe0,
0x76, 0x35, 0xe1, 0x55, 0x48, 0xee, 0x01, 0xa2, 0x24, 0x5e, 0xd1, 0x40, 0x17, 0xa1, 0x7f, 0xd5,
0x05, 0x77, 0xa1, 0x13, 0x24, 0xc9, 0x29, 0xd3, 0xbe, 0x5b, 0x72, 0xf3, 0x9d, 0xf3, 0xbf, 0x06,
0xdc, 0xae, 0x16, 0x8b, 0xba, 0x02, 0xb5, 0xa0, 0xa7, 0x2d, 0x54, 0x83, 0x6a, 0x0f, 0x2b, 0xde,
0x6e, 0x98, 0xde, 0x2e, 0x8e, 0x4c, 0x93, 0x50, 0x5f, 0x30, 0xd0, 0x47, 0xde, 0x24, 0x21, 0x55,
0xb9, 0x9e, 0xb1, 0x10, 0xc3, 0x33, 0x70, 0xd5, 0x52, 0x51, 0x26, 0x2c, 0xcc, 0xc1, 0x47, 0x2d,
0xd1, 0x3c, 0x8e, 0x7a, 0x3b, 0x3a, 0xe0, 0x7a, 0xa7, 0x02, 0x3e, 0x55, 0xd4, 0x25, 0x1d, 0x45,
0xb5, 0x26, 0x5b, 0xd0, 0xe3, 0x34, 0x8d, 0xf2, 0xdc, 0x47, 0xe7, 0xdb, 0xae, 0x49, 0x22, 0xf7,
0x01, 0x82, 0x24, 0x8a, 0x68, 0x80, 0x02, 0x36, 0x0a, 0x18, 0x14, 0x95, 0x77, 0x52, 0x46, 0x9e,
0xa0, 0x01, 0xba, 0xba, 0xed, 0x76, 0xa4, 0x8c, 0x0e, 0x69, 0xa0, 0xde, 0x91, 0x09, 0xca, 0x3d,
0x84, 0xaf, 0x1e, 0x9e, 0xeb, 0x2a, 0x02, 0x82, 0xec, 0x26, 0xc0, 0x84, 0x27, 0x59, 0xaa, 0xb9,
0xfd, 0xad, 0xa6, 0x42, 0x72, 0xa4, 0x20, 0xfb, 0x11, 0xdc, 0x16, 0x97, 0xd3, 0x88, 0xc5, 0xa7,
0x9e, 0xf4, 0xf9, 0x84, 0xca, 0xd1, 0x40, 0x57, 0x40, 0x4e, 0x7d, 0x87, 0x44, 0x27, 0x05, 0xb2,
0xcb, 0xa9, 0x2f, 0xe9, 0x77, 0x68, 0x5a, 0xdf, 0x0e, 0x1b, 0xc8, 0x1a, 0x74, 0x12, 0x8f, 0x5e,
0x04, 0x51, 0x5e, 0xa2, 0xed, 0x64, 0xef, 0x22, 0x88, 0x9c, 0x27, 0x30, 0xac, 0xdc, 0x98, 0xc3,
0xfa, 0x2a, 0xb4, 0x29, 0xe7, 0x49, 0x01, 0x42, 0x7a, 0xe3, 0xfc, 0x0e, 0xc8, 0xfb, 0x34, 0xfc,
0x21, 0xcc, 0x73, 0xd6, 0x60, 0x58, 0x51, 0xad, 0xed, 0x70, 0xfe, 0x65, 0x01, 0x79, 0x89, 0x58,
0xf2, 0xfd, 0xda, 0xb8, 0xaa, 0x6e, 0xd5, 0x62, 0x34, 0x56, 0x85, 0xbe, 0xf4, 0xf3, 0x06, 0xd8,
0x67, 0x42, 0xeb, 0x7f, 0xe9, 0x4b, 0x3f, 0x6f, 0x44, 0x9c, 0x06, 0x19, 0x57, 0x3d, 0x11, 0x93,
0x10, 0x1b, 0x91, 0x5b, 0x90, 0xc8, 0xc7, 0x70, 0x97, 0x4d, 0xe2, 0x84, 0xd3, 0x99, 0x98, 0xa7,
0x5d, 0xd5, 0x41, 0xe1, 0x55, 0xcd, 0x2d, 0x0f, 0xec, 0xa1, 0xe7, 0x9e, 0xc0, 0xb0, 0xf2, 0x8c,
0x6b, 0xdd, 0xfc, 0x67, 0x0b, 0x46, 0xcf, 0x65, 0x32, 0x65, 0x81, 0x4b, 0x95, 0xf1, 0x95, 0xa7,
0x3f, 0x84, 0x81, 0x42, 0xf3, 0xf9, 0xe7, 0xf7, 0x93, 0x28, 0x9c, 0x75, 0xcb, 0x7b, 0xa0, 0x00,
0xdd, 0x33, 0xbc, 0xb0, 0x94, 0x44, 0x21, 0x66, 0xe2, 0x43, 0x50, 0xa8, 0x6b, 0x9c, 0xd7, 0x73,
0x43, 0x3f, 0xa6, 0xe7, 0x95, 0xf3, 0x4a, 0x08, 0xcf, 0x6b, 0xa8, 0x5e, 0x8a, 0xe9, 0xb9, 0x3a,
0xef, 0xac, 0xc3, 0xbd, 0x05, 0xb6, 0xe5, 0xe1, 0xfa, 0xb7, 0x05, 0xc3, 0xe7, 0x42, 0xb0, 0x49,
0xfc, 0x5b, 0x84, 0x9d, 0xc2, 0xe8, 0x55, 0x68, 0x07, 0x49, 0x16, 0x4b, 0x34, 0xb6, 0xed, 0xea,
0xcd, 0x5c, 0x25, 0x36, 0x6a, 0x95, 0x38, 0x57, 0xcb, 0xcd, 0x7a, 0x2d, 0x1b, 0xb5, 0xda, 0xaa,
0xd4, 0xea, 0x03, 0xe8, 0xa9, 0x20, 0x7b, 0x01, 0x8d, 0x25, 0xe5, 0x39, 0xce, 0x83, 0x22, 0xed,
0x22, 0x45, 0x09, 0x98, 0xfd, 0x48, 0x43, 0x3d, 0xa4, 0xb3, 0x66, 0xf4, 0x1f, 0x0b, 0x56, 0xab,
0x4f, 0xc9, 0x63, 0x76, 0x65, 0x5f, 0x52, 0x50, 0xc6, 0xa3, 0xfc, 0x1d, 0x6a, 0xa9, 0x40, 0x21,
0xcd, 0x8e, 0x22, 0x16, 0x78, 0x8a, 0xa1, 0xed, 0xb7, 0x35, 0xe5, 0x3d, 0x8f, 0x66, 0x5e, 0x69,
0x99, 0x5e, 0x21, 0xd0, 0xf2, 0x33, 0x79, 0x52, 0xf4, 0x26, 0xb5, 0x9e, 0xf3, 0x54, 0xe7, 0x26,
0x4f, 0x2d, 0xd5, 0x3d, 0x55, 0x66, 0x5a, 0xd7, 0xcc, 0xb4, 0x8f, 0x61, 0xa8, 0x87, 0xdb, 0x6a,
0xb8, 0x36, 0x01, 0xca, 0x3e, 0x22, 0x46, 0x96, 0x06, 0xb3, 0xa2, 0x91, 0x08, 0xe7, 0x97, 0x60,
0xbf, 0x4e, 0xb4, 0x5e, 0x41, 0x9e, 0x81, 0x1d, 0x15, 0x1b, 0x14, 0xed, 0xed, 0x90, 0x59, 0x8d,
0x17, 0x72, 0xee, 0x4c, 0xc8, 0xf9, 0x0c, 0xba, 0x05, 0xb9, 0xf0, 0x99, 0x75, 0x95, 0xcf, 0x1a,
0x73, 0x3e, 0x73, 0xfe, 0x69, 0xc1, 0x6a, 0xd5, 0xe4, 0x3c, 0x2c, 0xef, 0x61, 0x50, 0x5e, 0xe1,
0x4d, 0xfd, 0x34, 0xb7, 0xe5, 0x99, 0x69, 0x4b, 0xfd, 0x58, 0x69, 0xa0, 0x78, 0xe3, 0xa7, 0x3a,
0x97, 0xfb, 0x91, 0x41, 0x1a, 0xbf, 0x83, 0x95, 0x9a, 0xc8, 0x82, 0xc9, 0xee, 0x27, 0xe6, 0x64,
0x57, 0x99, 0x4e, 0xcb, 0xd3, 0xe6, 0xb8, 0xf7, 0x29, 0x7c, 0xa0, 0xe1, 0x60, 0xb7, 0x8c, 0x61,
0xe1, 0xfb, 0x6a, 0xa8, 0xad, 0xf9, 0x50, 0x3b, 0x63, 0x18, 0xd5, 0x8f, 0xe6, 0xe5, 0x37, 0x81,
0x95, 0x43, 0xe9, 0x4b, 0x26, 0x24, 0x0b, 0xca, 0x4f, 0x8c, 0xb9, 0xdc, 0xb0, 0x6e, 0xea, 0x88,
0xf5, 0x3a, 0x5c, 0x86, 0xa6, 0x94, 0x45, 0xfe, 0xaa, 0xa5, 0x8a, 0x02, 0x31, 0x6f, 0xca, 0x63,
0xf0, 0x03, 0x5c, 0xa5, 0xf2, 0x41, 0x26, 0xd2, 0x8f, 0xf4, 0xc4, 0xd1, 0xc2, 0x89, 0xc3, 0x46,
0x0a, 0x8e, 0x1c, 0xba, 0x29, 0x87, 0x9a, 0xdb, 0xd6, 0xf3, 0x88, 0x22, 0x20, 0x73, 0x13, 0x00,
0x4b, 0x55, 0x57, 0x59, 0x47, 0x9f, 0x55, 0x94, 0x5d, 0x45, 0x70, 0xee, 0xc3, 0xc6, 0xe7, 0x54,
0xaa, 0xd9, 0x89, 0xef, 0x26, 0xf1, 0x31, 0x9b, 0x64, 0xdc, 0x37, 0x42, 0xe1, 0xfc, 0xd7, 0x82,
0xcd, 0x2b, 0x04, 0xf2, 0x07, 0x8f, 0x60, 0x69, 0xea, 0x0b, 0x49, 0x79, 0x51, 0x25, 0xc5, 0x76,
0xde, 0x15, 0x8d, 0x9b, 0x5c, 0xd1, 0xac, 0xb9, 0x62, 0x0d, 0x3a, 0x53, 0xff, 0xc2, 0x9b, 0x1e,
0xe5, 0xc3, 0x51, 0x7b, 0xea, 0x5f, 0xbc, 0x39, 0x42, 0x64, 0x63, 0xdc, 0x3b, 0xca, 0x82, 0x53,
0x2a, 0x45, 0x89, 0x6c, 0x8c, 0xbf, 0xd0, 0x14, 0xf5, 0x68, 0x25, 0xf0, 0x75, 0x46, 0x33, 0x2a,
0x72, 0xac, 0x50, 0xcd, 0xf1, 0x37, 0x48, 0xc0, 0x61, 0x0a, 0x27, 0x4b, 0x44, 0x89, 0xae, 0x9b,
0xef, 0x1c, 0x09, 0x77, 0xd5, 0xf7, 0x1d, 0x8d, 0xf7, 0x13, 0x8e, 0xdf, 0x10, 0x65, 0x02, 0x3d,
0x80, 0x5e, 0x10, 0x31, 0x05, 0x95, 0xc6, 0x87, 0x1b, 0x68, 0x12, 0xb6, 0x94, 0x4a, 0x37, 0x6e,
0xcc, 0x77, 0xe3, 0x75, 0xb0, 0x05, 0x8b, 0x03, 0x8a, 0x28, 0xdd, 0xc4, 0x01, 0xae, 0x8b, 0x84,
0x43, 0x1a, 0x38, 0x7f, 0xb2, 0x60, 0x0d, 0x3f, 0x7c, 0x6a, 0x5f, 0x2d, 0xd7, 0xb7, 0xf8, 0x5f,
0x03, 0xa1, 0x67, 0x68, 0x92, 0x71, 0x26, 0x2f, 0xbe, 0x75, 0x63, 0xc4, 0x98, 0x57, 0xeb, 0xae,
0xd0, 0x79, 0x92, 0xe3, 0x2b, 0x3c, 0x9a, 0xe8, 0xca, 0x1e, 0x42, 0x5b, 0x0a, 0x0f, 0x91, 0x4c,
0x19, 0xda, 0x92, 0xe2, 0x40, 0x90, 0xa7, 0x40, 0x52, 0x9f, 0x4b, 0xa6, 0xa4, 0xd5, 0xf8, 0xec,
0x9d, 0xf8, 0xe2, 0x04, 0x2f, 0x6b, 0xbb, 0xcb, 0x25, 0xe7, 0x0b, 0x7a, 0xf9, 0x2b, 0x5f, 0x9c,
0x28, 0xfc, 0xc6, 0xf9, 0xa2, 0x89, 0x63, 0x3c, 0xae, 0x77, 0xfe, 0xda, 0x85, 0xfe, 0x21, 0xf5,
0xcf, 0x29, 0x0d, 0x31, 0x9b, 0xc8, 0xa4, 0x40, 0xb1, 0xea, 0x5f, 0x05, 0xf2, 0x68, 0x1e, 0xae,
0x16, 0xfe, 0xc6, 0x18, 0x7f, 0x74, 0x93, 0x58, 0x0e, 0x08, 0xb7, 0xc8, 0x01, 0xf4, 0x8c, 0xcf,
0x76, 0xb2, 0x61, 0x1c, 0xac, 0xfd, 0x8d, 0x18, 0x6f, 0x5e, 0xc1, 0x2d, 0xb4, 0x3d, 0xb3, 0xc8,
0x6b, 0xe8, 0x19, 0xf3, 0xa2, 0xa9, 0xaf, 0x3e, 0xb8, 0x9a, 0xfa, 0x16, 0x0c, 0x99, 0xce, 0x2d,
0xa5, 0xcd, 0x98, 0xfa, 0x4c, 0x6d, 0xf5, 0x39, 0xd3, 0xd4, 0xb6, 0x68, 0x54, 0x44, 0x6d, 0xc6,
0x90, 0x65, 0x6a, 0xab, 0x8f, 0x90, 0xa6, 0xb6, 0x05, 0x93, 0x99, 0x73, 0x8b, 0xfc, 0x01, 0x56,
0x6a, 0x83, 0x0e, 0x71, 0x66, 0xa7, 0xae, 0x9a, 0xd0, 0xc6, 0x0f, 0xaf, 0x95, 0x29, 0xf5, 0x7f,
0x09, 0x7d, 0x73, 0xbe, 0x20, 0x86, 0x41, 0x0b, 0x46, 0xa8, 0xf1, 0xfd, 0xab, 0xd8, 0xa6, 0x42,
0xb3, 0xc5, 0x99, 0x0a, 0x17, 0x34, 0x79, 0x53, 0xe1, 0xa2, 0xce, 0xe8, 0xdc, 0x22, 0xbf, 0x87,
0xe5, 0xf9, 0x56, 0x43, 0x3e, 0x9c, 0x77, 0x5b, 0xad, 0x83, 0x8d, 0x9d, 0xeb, 0x44, 0x4a, 0xe5,
0xaf, 0x00, 0x66, 0x1d, 0x84, 0x18, 0x35, 0x5b, 0xeb, 0x60, 0xe3, 0x8d, 0xc5, 0xcc, 0x52, 0xd5,
0x1f, 0x61, 0x6d, 0x21, 0x4c, 0x13, 0xa3, 0x4c, 0xae, 0x03, 0xfa, 0xf1, 0x8f, 0x6f, 0x94, 0x2b,
0xef, 0xfa, 0x0a, 0xee, 0xcc, 0xc1, 0x24, 0xd9, 0xaa, 0x56, 0x4d, 0x1d, 0x41, 0xc7, 0x0f, 0xcc,
0x7f, 0x4f, 0x0b, 0xc0, 0x4e, 0x55, 0xd6, 0x8b, 0xfb, 0xb0, 0x2c, 0x34, 0x44, 0x1c, 0x8b, 0x6d,
0x8d, 0xae, 0x2f, 0x00, 0x6d, 0x79, 0xcb, 0x13, 0x99, 0x1c, 0x75, 0xf0, 0x57, 0xe7, 0xcf, 0xfe,
0x1f, 0x00, 0x00, 0xff, 0xff, 0xbe, 0x11, 0xf3, 0xf2, 0xf9, 0x14, 0x00, 0x00,
0x11, 0x37, 0xef, 0x3f, 0xe7, 0xee, 0x6c, 0x69, 0x4f, 0x76, 0xce, 0x67, 0xc9, 0x56, 0xe8, 0x3a,
0x75, 0x61, 0x43, 0x35, 0xd4, 0x3c, 0x24, 0x4d, 0xfb, 0x60, 0xcb, 0x52, 0xea, 0xc6, 0x56, 0x5c,
0xca, 0x2e, 0x52, 0x14, 0x28, 0x41, 0x91, 0xab, 0xd3, 0x56, 0x3c, 0x92, 0xd9, 0x5d, 0xea, 0x4f,
0xde, 0xfa, 0x35, 0x0a, 0xf4, 0xa1, 0xdf, 0xa1, 0x8f, 0x45, 0x5f, 0x8a, 0x02, 0xfd, 0x1c, 0x7d,
0xec, 0x43, 0x3f, 0x43, 0xb1, 0xb3, 0x24, 0x6f, 0x79, 0x3c, 0x49, 0x09, 0x82, 0xbc, 0xed, 0xce,
0xcc, 0xce, 0xce, 0xce, 0x9f, 0xdf, 0x0c, 0x09, 0xfd, 0x23, 0x16, 0x51, 0xbe, 0x95, 0xf2, 0x44,
0x26, 0xa4, 0x87, 0x1b, 0x2f, 0x3d, 0x74, 0xbe, 0x84, 0x7b, 0xaf, 0x93, 0xe4, 0x24, 0x4b, 0x5f,
0x32, 0x4e, 0x03, 0x99, 0xf0, 0x8b, 0xdd, 0x58, 0xf2, 0x0b, 0x97, 0x7e, 0x9d, 0x51, 0x21, 0xc9,
0x3a, 0xd8, 0x61, 0xc1, 0x18, 0x5b, 0x9b, 0xd6, 0x63, 0xdb, 0x9d, 0x13, 0x08, 0x81, 0x56, 0xec,
0xcf, 0xe8, 0xb8, 0x81, 0x0c, 0x5c, 0x3b, 0xbb, 0xb0, 0xbe, 0x5c, 0xa1, 0x48, 0x93, 0x58, 0x50,
0xf2, 0x08, 0xda, 0x54, 0x11, 0x50, 0x5b, 0x7f, 0xfb, 0xd6, 0x56, 0x61, 0xca, 0x96, 0x96, 0xd3,
0x5c, 0xe7, 0x1f, 0x16, 0x90, 0xd7, 0x4c, 0x48, 0x45, 0x64, 0x54, 0x7c, 0x3b, 0x7b, 0xee, 0x40,
0x27, 0xe5, 0xf4, 0x88, 0x9d, 0xe7, 0x16, 0xe5, 0x3b, 0xf2, 0x14, 0x56, 0x85, 0xf4, 0xb9, 0xdc,
0xe3, 0xc9, 0x6c, 0x8f, 0x45, 0x74, 0x5f, 0x19, 0xdd, 0x44, 0x91, 0x3a, 0x83, 0x6c, 0x01, 0x61,
0x71, 0x10, 0x65, 0x82, 0x9d, 0xd2, 0x83, 0x82, 0x3b, 0x6e, 0x6d, 0x5a, 0x8f, 0x7b, 0xee, 0x12,
0x0e, 0x59, 0x83, 0x76, 0xc4, 0x66, 0x4c, 0x8e, 0xdb, 0x9b, 0xd6, 0xe3, 0xa1, 0xab, 0x37, 0xce,
0x2f, 0x60, 0x54, 0xb1, 0xff, 0xbb, 0x3d, 0xff, 0x2f, 0x0d, 0x68, 0x23, 0xa1, 0xf4, 0xb1, 0x35,
0xf7, 0x31, 0xf9, 0x10, 0x06, 0x4c, 0x78, 0x73, 0x47, 0x34, 0xd0, 0xb6, 0x3e, 0x13, 0xa5, 0xcf,
0xc9, 0x13, 0xe8, 0x04, 0xc7, 0x59, 0x7c, 0x22, 0xc6, 0xcd, 0xcd, 0xe6, 0xe3, 0xfe, 0xf6, 0x68,
0x7e, 0x91, 0x7a, 0xe8, 0x8e, 0xe2, 0xb9, 0xb9, 0x08, 0xf9, 0x04, 0xc0, 0x97, 0x92, 0xb3, 0xc3,
0x4c, 0x52, 0x81, 0x2f, 0xed, 0x6f, 0x8f, 0x8d, 0x03, 0x99, 0xa0, 0xcf, 0x4b, 0xbe, 0x6b, 0xc8,
0x92, 0x4f, 0xa1, 0x47, 0xcf, 0x25, 0x8d, 0x43, 0x1a, 0x8e, 0xdb, 0x78, 0xd1, 0xc6, 0xc2, 0x8b,
0xb6, 0x76, 0x73, 0xbe, 0x7e, 0x5f, 0x29, 0x3e, 0xf9, 0x0c, 0x86, 0x15, 0x16, 0x59, 0x81, 0xe6,
0x09, 0x2d, 0xa2, 0xaa, 0x96, 0xca, 0xb3, 0xa7, 0x7e, 0x94, 0xe9, 0x04, 0x1b, 0xb8, 0x7a, 0xf3,
0xf3, 0xc6, 0x27, 0x96, 0xf3, 0x12, 0xec, 0xbd, 0x2c, 0x8a, 0xca, 0x83, 0x21, 0xe3, 0xc5, 0xc1,
0x90, 0xf1, 0xb9, 0x97, 0x1b, 0x57, 0x7a, 0xf9, 0xef, 0x16, 0xac, 0xee, 0x9e, 0xd2, 0x58, 0xee,
0x27, 0x92, 0x1d, 0xb1, 0xc0, 0x97, 0x2c, 0x89, 0xc9, 0x53, 0xb0, 0x93, 0x28, 0xf4, 0xae, 0x0c,
0x53, 0x2f, 0x89, 0x72, 0xab, 0x9f, 0x82, 0x1d, 0xd3, 0x33, 0xef, 0xca, 0xeb, 0x7a, 0x31, 0x3d,
0xd3, 0xd2, 0x0f, 0x61, 0x18, 0xd2, 0x88, 0x4a, 0xea, 0x95, 0xd1, 0x51, 0xa1, 0x1b, 0x68, 0xe2,
0x8e, 0x0e, 0xc7, 0x47, 0x70, 0x4b, 0xa9, 0x4c, 0x7d, 0x4e, 0x63, 0xe9, 0xa5, 0xbe, 0x3c, 0xc6,
0x98, 0xd8, 0xee, 0x30, 0xa6, 0x67, 0x6f, 0x91, 0xfa, 0xd6, 0x97, 0xc7, 0xce, 0xdf, 0x1a, 0x60,
0x97, 0xc1, 0x24, 0x1f, 0x40, 0x57, 0x5d, 0xeb, 0xb1, 0x30, 0xf7, 0x44, 0x47, 0x6d, 0x5f, 0x85,
0xaa, 0x2a, 0x92, 0xa3, 0x23, 0x41, 0x25, 0x9a, 0xd7, 0x74, 0xf3, 0x9d, 0xca, 0x2c, 0xc1, 0xbe,
0xd1, 0x85, 0xd0, 0x72, 0x71, 0xad, 0x3c, 0x3e, 0x93, 0x6c, 0x46, 0xf1, 0xc2, 0xa6, 0xab, 0x37,
0x64, 0x04, 0x6d, 0xea, 0x49, 0x7f, 0x8a, 0x19, 0x6e, 0xbb, 0x2d, 0xfa, 0xce, 0x9f, 0x92, 0x1f,
0xc1, 0x4d, 0x91, 0x64, 0x3c, 0xa0, 0x5e, 0x71, 0x6d, 0x07, 0xb9, 0x03, 0x4d, 0xdd, 0xd3, 0x97,
0x3b, 0xd0, 0x3c, 0x62, 0xe1, 0xb8, 0x8b, 0x8e, 0x59, 0xa9, 0x26, 0xe1, 0xab, 0xd0, 0x55, 0x4c,
0xf2, 0x53, 0x80, 0x52, 0x53, 0x38, 0xee, 0x5d, 0x22, 0x6a, 0x17, 0x7a, 0x43, 0xb2, 0x01, 0x10,
0xb0, 0xf4, 0x98, 0x72, 0x4f, 0x25, 0x8c, 0x8d, 0xc9, 0x61, 0x6b, 0xca, 0x17, 0xf4, 0x42, 0xb1,
0x99, 0xf0, 0xa6, 0xdf, 0xb0, 0x34, 0xa5, 0xe1, 0x18, 0xd0, 0xc3, 0x36, 0x13, 0x9f, 0x6b, 0x82,
0xf3, 0x15, 0x74, 0x72, 0xe3, 0xee, 0x81, 0x7d, 0x9a, 0x44, 0xd9, 0xac, 0x74, 0xda, 0xd0, 0xed,
0x69, 0xc2, 0xab, 0x90, 0xdc, 0x05, 0x44, 0x49, 0xbc, 0xa2, 0x81, 0x2e, 0x42, 0xff, 0xaa, 0x0b,
0xee, 0x40, 0x27, 0x48, 0x92, 0x13, 0xa6, 0x7d, 0xd7, 0x75, 0xf3, 0x9d, 0xf3, 0xbf, 0x06, 0xdc,
0xac, 0x16, 0x8b, 0xba, 0x02, 0xb5, 0xa0, 0xa7, 0x2d, 0x54, 0x83, 0x6a, 0x0f, 0x2a, 0xde, 0x6e,
0x98, 0xde, 0x2e, 0x8e, 0xcc, 0x92, 0x50, 0x5f, 0x30, 0xd4, 0x47, 0xde, 0x24, 0x21, 0x55, 0xb9,
0x9e, 0xb1, 0x10, 0xc3, 0x33, 0x74, 0xd5, 0x52, 0x51, 0xa6, 0x2c, 0xcc, 0xc1, 0x47, 0x2d, 0xd1,
0x3c, 0x8e, 0x7a, 0x3b, 0x3a, 0xe0, 0x7a, 0xa7, 0x02, 0x3e, 0x53, 0xd4, 0xae, 0x8e, 0xa2, 0x5a,
0x93, 0x4d, 0xe8, 0x73, 0x9a, 0x46, 0x79, 0xee, 0xa3, 0xf3, 0x6d, 0xd7, 0x24, 0x91, 0xfb, 0x00,
0x41, 0x12, 0x45, 0x34, 0x40, 0x01, 0x1b, 0x05, 0x0c, 0x8a, 0xca, 0x3b, 0x29, 0x23, 0x4f, 0xd0,
0x00, 0x5d, 0xdd, 0x76, 0x3b, 0x52, 0x46, 0x07, 0x34, 0x50, 0xef, 0xc8, 0x04, 0xe5, 0x1e, 0xc2,
0x57, 0x1f, 0xcf, 0xf5, 0x14, 0x01, 0x41, 0x76, 0x03, 0x60, 0xca, 0x93, 0x2c, 0xd5, 0xdc, 0xc1,
0x66, 0x53, 0x21, 0x39, 0x52, 0x90, 0xfd, 0x08, 0x6e, 0x8a, 0x8b, 0x59, 0xc4, 0xe2, 0x13, 0x4f,
0xfa, 0x7c, 0x4a, 0xe5, 0x78, 0xa8, 0x2b, 0x20, 0xa7, 0xbe, 0x43, 0xa2, 0x93, 0x02, 0xd9, 0xe1,
0xd4, 0x97, 0xf4, 0x3b, 0x34, 0xad, 0x6f, 0x87, 0x0d, 0xe4, 0x36, 0x74, 0x12, 0x8f, 0x9e, 0x07,
0x51, 0x5e, 0xa2, 0xed, 0x64, 0xf7, 0x3c, 0x88, 0x9c, 0x27, 0x30, 0xaa, 0xdc, 0x98, 0xc3, 0xfa,
0x1a, 0xb4, 0x29, 0xe7, 0x49, 0x01, 0x42, 0x7a, 0xe3, 0xfc, 0x0e, 0xc8, 0xfb, 0x34, 0xfc, 0x21,
0xcc, 0x73, 0x6e, 0xc3, 0xa8, 0xa2, 0x5a, 0xdb, 0xe1, 0xfc, 0xcb, 0x02, 0xf2, 0x12, 0xb1, 0xe4,
0xfb, 0xb5, 0x71, 0x55, 0xdd, 0xaa, 0xc5, 0x68, 0xac, 0x0a, 0x7d, 0xe9, 0xe7, 0x0d, 0x70, 0xc0,
0x84, 0xd6, 0xff, 0xd2, 0x97, 0x7e, 0xde, 0x88, 0x38, 0x0d, 0x32, 0xae, 0x7a, 0x22, 0x26, 0x21,
0x36, 0x22, 0xb7, 0x20, 0x91, 0x8f, 0xe1, 0x0e, 0x9b, 0xc6, 0x09, 0xa7, 0x73, 0x31, 0x4f, 0xbb,
0xaa, 0x83, 0xc2, 0x6b, 0x9a, 0x5b, 0x1e, 0xd8, 0x45, 0xcf, 0x3d, 0x81, 0x51, 0xe5, 0x19, 0x57,
0xba, 0xf9, 0xcf, 0x16, 0x8c, 0x9f, 0xcb, 0x64, 0xc6, 0x02, 0x97, 0x2a, 0xe3, 0x2b, 0x4f, 0x7f,
0x08, 0x43, 0x85, 0xe6, 0x8b, 0xcf, 0x1f, 0x24, 0x51, 0x38, 0xef, 0x96, 0x77, 0x41, 0x01, 0xba,
0x67, 0x78, 0xa1, 0x9b, 0x44, 0x21, 0x66, 0xe2, 0x43, 0x50, 0xa8, 0x6b, 0x9c, 0xd7, 0x73, 0xc3,
0x20, 0xa6, 0x67, 0x95, 0xf3, 0x4a, 0x08, 0xcf, 0x6b, 0xa8, 0xee, 0xc6, 0xf4, 0x4c, 0x9d, 0x77,
0xee, 0xc1, 0xdd, 0x25, 0xb6, 0xe5, 0xe1, 0xfa, 0xb7, 0x05, 0xa3, 0xe7, 0x42, 0xb0, 0x69, 0xfc,
0x5b, 0x84, 0x9d, 0xc2, 0xe8, 0x35, 0x68, 0x07, 0x49, 0x16, 0x4b, 0x34, 0xb6, 0xed, 0xea, 0xcd,
0x42, 0x25, 0x36, 0x6a, 0x95, 0xb8, 0x50, 0xcb, 0xcd, 0x7a, 0x2d, 0x1b, 0xb5, 0xda, 0xaa, 0xd4,
0xea, 0x03, 0xe8, 0xab, 0x20, 0x7b, 0x01, 0x8d, 0x25, 0xe5, 0x39, 0xce, 0x83, 0x22, 0xed, 0x20,
0x45, 0x09, 0x98, 0xfd, 0x48, 0x43, 0x3d, 0xa4, 0xf3, 0x66, 0xf4, 0x1f, 0x0b, 0xd6, 0xaa, 0x4f,
0xc9, 0x63, 0x76, 0x69, 0x5f, 0x52, 0x50, 0xc6, 0xa3, 0xfc, 0x1d, 0x6a, 0xa9, 0x40, 0x21, 0xcd,
0x0e, 0x23, 0x16, 0x78, 0x8a, 0xa1, 0xed, 0xb7, 0x35, 0xe5, 0x3d, 0x8f, 0xe6, 0x5e, 0x69, 0x99,
0x5e, 0x21, 0xd0, 0xf2, 0x33, 0x79, 0x5c, 0xf4, 0x26, 0xb5, 0x5e, 0xf0, 0x54, 0xe7, 0x3a, 0x4f,
0x75, 0xeb, 0x9e, 0x2a, 0x33, 0xad, 0x67, 0x66, 0xda, 0xc7, 0x30, 0xd2, 0xc3, 0x6d, 0x35, 0x5c,
0x1b, 0x00, 0x65, 0x1f, 0x11, 0x63, 0x4b, 0x83, 0x59, 0xd1, 0x48, 0x84, 0xf3, 0x4b, 0xb0, 0x5f,
0x27, 0x5a, 0xaf, 0x20, 0xcf, 0xc0, 0x8e, 0x8a, 0x0d, 0x8a, 0xf6, 0xb7, 0xc9, 0xbc, 0xc6, 0x0b,
0x39, 0x77, 0x2e, 0xe4, 0x7c, 0x06, 0xbd, 0x82, 0x5c, 0xf8, 0xcc, 0xba, 0xcc, 0x67, 0x8d, 0x05,
0x9f, 0x39, 0xff, 0xb4, 0x60, 0xad, 0x6a, 0x72, 0x1e, 0x96, 0xf7, 0x30, 0x2c, 0xaf, 0xf0, 0x66,
0x7e, 0x9a, 0xdb, 0xf2, 0xcc, 0xb4, 0xa5, 0x7e, 0xac, 0x34, 0x50, 0xbc, 0xf1, 0x53, 0x9d, 0xcb,
0x83, 0xc8, 0x20, 0x4d, 0xde, 0xc1, 0x6a, 0x4d, 0x64, 0xc9, 0x64, 0xf7, 0x13, 0x73, 0xb2, 0xab,
0x4c, 0xa7, 0xe5, 0x69, 0x73, 0xdc, 0xfb, 0x14, 0x3e, 0xd0, 0x70, 0xb0, 0x53, 0xc6, 0xb0, 0xf0,
0x7d, 0x35, 0xd4, 0xd6, 0x62, 0xa8, 0x9d, 0x09, 0x8c, 0xeb, 0x47, 0xf3, 0xf2, 0x9b, 0xc2, 0xea,
0x81, 0xf4, 0x25, 0x13, 0x92, 0x05, 0xe5, 0x27, 0xc6, 0x42, 0x6e, 0x58, 0xd7, 0x75, 0xc4, 0x7a,
0x1d, 0xae, 0x40, 0x53, 0xca, 0x22, 0x7f, 0xd5, 0x52, 0x45, 0x81, 0x98, 0x37, 0xe5, 0x31, 0xf8,
0x01, 0xae, 0x52, 0xf9, 0x20, 0x13, 0xe9, 0x47, 0x7a, 0xe2, 0x68, 0xe1, 0xc4, 0x61, 0x23, 0x05,
0x47, 0x0e, 0xdd, 0x94, 0x43, 0xcd, 0x6d, 0xeb, 0x79, 0x44, 0x11, 0x90, 0xb9, 0x01, 0x80, 0xa5,
0xaa, 0xab, 0xac, 0xa3, 0xcf, 0x2a, 0xca, 0x8e, 0x22, 0x38, 0xf7, 0x61, 0xfd, 0x73, 0x2a, 0xd5,
0xec, 0xc4, 0x77, 0x92, 0xf8, 0x88, 0x4d, 0x33, 0xee, 0x1b, 0xa1, 0x70, 0xfe, 0x6b, 0xc1, 0xc6,
0x25, 0x02, 0xf9, 0x83, 0xc7, 0xd0, 0x9d, 0xf9, 0x42, 0x52, 0x5e, 0x54, 0x49, 0xb1, 0x5d, 0x74,
0x45, 0xe3, 0x3a, 0x57, 0x34, 0x6b, 0xae, 0xb8, 0x0d, 0x9d, 0x99, 0x7f, 0xee, 0xcd, 0x0e, 0xf3,
0xe1, 0xa8, 0x3d, 0xf3, 0xcf, 0xdf, 0x1c, 0x22, 0xb2, 0x31, 0xee, 0x1d, 0x66, 0xc1, 0x09, 0x95,
0xa2, 0x44, 0x36, 0xc6, 0x5f, 0x68, 0x8a, 0x7a, 0xb4, 0x12, 0xf8, 0x3a, 0xa3, 0x19, 0x15, 0x39,
0x56, 0xa8, 0xe6, 0xf8, 0x1b, 0x24, 0xe0, 0x30, 0x85, 0x93, 0x25, 0xa2, 0x44, 0xcf, 0xcd, 0x77,
0x0e, 0x87, 0x3b, 0xea, 0xfb, 0x8e, 0xc6, 0x7b, 0x09, 0xc7, 0x6f, 0x88, 0x32, 0x81, 0x1e, 0x40,
0x3f, 0x88, 0x98, 0x82, 0x4a, 0xe3, 0xc3, 0x0d, 0x34, 0x09, 0x5b, 0x4a, 0xa5, 0x1b, 0x37, 0x16,
0xbb, 0xf1, 0x5d, 0xe8, 0x09, 0x16, 0x07, 0xd4, 0x8b, 0xf5, 0xd7, 0x41, 0xd3, 0xed, 0xe2, 0x7e,
0x5f, 0x38, 0x7f, 0xb2, 0xe0, 0x36, 0x7e, 0xf6, 0xd4, 0xbe, 0x59, 0xae, 0x6e, 0xf0, 0xbf, 0x06,
0x42, 0x4f, 0xd1, 0x20, 0xe3, 0x4c, 0x5e, 0x7a, 0xf7, 0x8c, 0x01, 0x63, 0x51, 0xad, 0xbb, 0x4a,
0x17, 0x49, 0x8e, 0xaf, 0xd0, 0x68, 0xaa, 0xeb, 0x7a, 0x04, 0x6d, 0x29, 0x3c, 0xc4, 0x31, 0x65,
0x67, 0x4b, 0x8a, 0x7d, 0x41, 0x9e, 0x02, 0x49, 0x7d, 0x2e, 0x99, 0x92, 0x56, 0xc3, 0xb3, 0x77,
0xec, 0x8b, 0x63, 0xbc, 0xac, 0xed, 0xae, 0x94, 0x9c, 0x2f, 0xe8, 0xc5, 0xaf, 0x7c, 0x71, 0xac,
0xd0, 0x1b, 0xa7, 0x8b, 0x26, 0x0e, 0xf1, 0xb8, 0xde, 0xfe, 0x6b, 0x0f, 0x06, 0x07, 0xd4, 0x3f,
0xa3, 0x34, 0xc4, 0x5c, 0x22, 0xd3, 0x02, 0xc3, 0xaa, 0xff, 0x14, 0xc8, 0xa3, 0x45, 0xb0, 0x5a,
0xfa, 0x13, 0x63, 0xf2, 0xd1, 0x75, 0x62, 0x39, 0x1c, 0xdc, 0x20, 0xfb, 0xd0, 0x37, 0x3e, 0xda,
0xc9, 0xba, 0x71, 0xb0, 0xf6, 0x2f, 0x62, 0xb2, 0x71, 0x09, 0xb7, 0xd0, 0xf6, 0xcc, 0x22, 0xaf,
0xa1, 0x6f, 0x4c, 0x8b, 0xa6, 0xbe, 0xfa, 0xd8, 0x6a, 0xea, 0x5b, 0x32, 0x62, 0x3a, 0x37, 0x94,
0x36, 0x63, 0xe6, 0x33, 0xb5, 0xd5, 0xa7, 0x4c, 0x53, 0xdb, 0xb2, 0x41, 0x11, 0xb5, 0x19, 0x23,
0x96, 0xa9, 0xad, 0x3e, 0x40, 0x9a, 0xda, 0x96, 0xcc, 0x65, 0xce, 0x0d, 0xf2, 0x07, 0x58, 0xad,
0x8d, 0x39, 0xc4, 0x99, 0x9f, 0xba, 0x6c, 0x3e, 0x9b, 0x3c, 0xbc, 0x52, 0xa6, 0xd4, 0xff, 0x25,
0x0c, 0xcc, 0xe9, 0x82, 0x18, 0x06, 0x2d, 0x19, 0xa0, 0x26, 0xf7, 0x2f, 0x63, 0x9b, 0x0a, 0xcd,
0x06, 0x67, 0x2a, 0x5c, 0xd2, 0xe2, 0x4d, 0x85, 0xcb, 0xfa, 0xa2, 0x73, 0x83, 0xfc, 0x1e, 0x56,
0x16, 0x1b, 0x0d, 0xf9, 0x70, 0xd1, 0x6d, 0xb5, 0xfe, 0x35, 0x71, 0xae, 0x12, 0x29, 0x95, 0xbf,
0x02, 0x98, 0xf7, 0x0f, 0x62, 0xd4, 0x6c, 0xad, 0x7f, 0x4d, 0xd6, 0x97, 0x33, 0x4b, 0x55, 0x7f,
0x84, 0xdb, 0x4b, 0x41, 0x9a, 0x18, 0x65, 0x72, 0x15, 0xcc, 0x4f, 0x7e, 0x7c, 0xad, 0x5c, 0x79,
0xd7, 0x57, 0x70, 0x6b, 0x01, 0x24, 0xc9, 0x66, 0xb5, 0x6a, 0xea, 0xf8, 0x39, 0x79, 0x60, 0xfe,
0x79, 0x5a, 0x02, 0x76, 0xaa, 0xb2, 0x5e, 0xdc, 0x87, 0x15, 0xa1, 0x21, 0xe2, 0x48, 0x6c, 0x69,
0x6c, 0x7d, 0x01, 0x68, 0xcb, 0x5b, 0x9e, 0xc8, 0xe4, 0xb0, 0x83, 0x3f, 0x3a, 0x7f, 0xf6, 0xff,
0x00, 0x00, 0x00, 0xff, 0xff, 0x2f, 0x77, 0xbb, 0xcb, 0xf7, 0x14, 0x00, 0x00,
}

82
weed/queue/log_buffer.go
View File

@ -6,28 +6,32 @@ import (
"github.com/golang/protobuf/proto"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
)
type LogBuffer struct {
buf []byte
idx []int
pos int
startTime time.Time
stopTime time.Time
sizeBuf []byte
flushInterval time.Duration
flushFn func(startTime, stopTime time.Time, buf []byte)
notifyFn func()
isStopping bool
sync.Mutex
sync.RWMutex
}
func NewLogBuffer(flushInterval time.Duration, flushFn func(startTime, stopTime time.Time, buf []byte)) *LogBuffer {
func NewLogBuffer(flushInterval time.Duration, flushFn func(startTime, stopTime time.Time, buf []byte), notifyFn func()) *LogBuffer {
lb := &LogBuffer{
buf: make([]byte, 4*0124*1024),
sizeBuf: make([]byte, 4),
flushInterval: flushInterval,
flushFn: flushFn,
notifyFn: notifyFn,
}
go lb.loopFlush()
return lb
@ -46,7 +50,12 @@ func (m *LogBuffer) AddToBuffer(ts time.Time, key, data []byte) {
size := len(logEntryData)
m.Lock()
defer m.Unlock()
defer func() {
m.Unlock()
if m.notifyFn != nil {
m.notifyFn()
}
}()
if m.pos == 0 {
m.startTime = ts
@ -55,12 +64,15 @@ func (m *LogBuffer) AddToBuffer(ts time.Time, key, data []byte) {
if m.startTime.Add(m.flushInterval).Before(ts) || len(m.buf)-m.pos < size+4 {
m.flush()
m.startTime = ts
if len(m.buf) < size+4 {
m.buf = make([]byte, 2*size+4)
}
}
m.stopTime = ts
m.idx = append(m.idx, m.pos)
util.Uint32toBytes(m.sizeBuf, uint32(size))
copy(m.buf[m.pos:m.pos+4], m.sizeBuf)
copy(m.buf[m.pos+4:m.pos+4+size], logEntryData)
m.pos += size + 4
}
@ -88,5 +100,67 @@ func (m *LogBuffer) flush() {
if m.flushFn != nil && m.pos > 0 {
m.flushFn(m.startTime, m.stopTime, m.buf[:m.pos])
m.pos = 0
m.idx = m.idx[:0]
}
}
func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (ts time.Time, bufferCopy []byte) {
m.RLock()
defer m.RUnlock()
// fmt.Printf("read from buffer: %v\n", lastReadTime)
if lastReadTime.Equal(m.stopTime) {
return lastReadTime, nil
}
if lastReadTime.After(m.stopTime) {
// glog.Fatalf("unexpected last read time %v, older than latest %v", lastReadTime, m.stopTime)
return lastReadTime, nil
}
if lastReadTime.Before(m.startTime) {
return m.stopTime, copiedBytes(m.buf[:m.pos])
}
lastTs := lastReadTime.UnixNano()
l, h := 0, len(m.idx)-1
// fmt.Printf("l=%d, h=%d\n", l, h)
for {
mid := (l + h) / 2
pos := m.idx[mid]
t := readTs(m.buf, m.idx[mid])
if t <= lastTs {
l = mid + 1
} else if lastTs < t {
var prevT int64
if mid > 0 {
prevT = readTs(m.buf, m.idx[mid-1])
}
if prevT <= lastTs {
return time.Unix(0, t), copiedBytes(m.buf[pos:m.pos])
}
h = mid - 1
}
// fmt.Printf("l=%d, h=%d\n", l, h)
}
}
func copiedBytes(buf []byte) (copied []byte) {
copied = make([]byte, len(buf))
copy(copied, buf)
return
}
func readTs(buf []byte, pos int) int64 {
size := util.BytesToUint32(buf[pos : pos+4])
entryData := buf[pos+4 : pos+4+int(size)]
logEntry := &filer_pb.LogEntry{}
err := proto.Unmarshal(entryData, logEntry)
if err != nil {
glog.Fatalf("unexpected unmarshal filer_pb.LogEntry: %v", err)
}
return logEntry.TsNs
}

71
weed/server/filer_grpc_server_listen.go
View File

@ -1,6 +1,9 @@
package weed_server
import (
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)
@ -9,62 +12,56 @@ func (fs *FilerServer) ListenForEvents(req *filer_pb.ListenForEventsRequest, str
peerAddress := findClientAddress(stream.Context(), 0)
clientName, messageChan := fs.addClient(req.ClientName, peerAddress)
clientName := fs.addClient(req.ClientName, peerAddress)
defer fs.deleteClient(clientName, messageChan)
defer fs.deleteClient(clientName)
// ts := time.Unix(req.SinceSec, 0)
lastReadTime := time.Now()
if req.SinceNs > 0 {
lastReadTime = time.Unix(0, req.SinceNs)
}
var readErr error
for {
// iterate through old messages
/*
for _, message := range ms.Topo.ToVolumeLocations() {
if err := stream.Send(message); err != nil {
return err
lastReadTime, readErr = fs.filer.ReadLogBuffer(lastReadTime, func(fullpath string, eventNotification *filer_pb.EventNotification) error {
if strings.HasPrefix(fullpath, "/.meta") {
return nil
}
if !strings.HasPrefix(fullpath, req.Directory) {
return nil
}
message := &filer_pb.FullEventNotification{
Directory: fullpath,
EventNotification: eventNotification,
}
*/
// need to add a buffer here to avoid slow clients
// also needs to support millions of clients
for message := range messageChan {
if err := stream.Send(message); err != nil {
glog.V(0).Infof("=> client %v: %+v", clientName, message)
return err
}
return nil
})
if readErr != nil {
glog.V(0).Infof("=> client %v: %+v", clientName, readErr)
return readErr
}
fs.listenersLock.Lock()
fs.listenersCond.Wait()
fs.listenersLock.Unlock()
}
return nil
}
func (fs *FilerServer) addClient(clientType string, clientAddress string) (clientName string, messageChan chan *filer_pb.FullEventNotification) {
func (fs *FilerServer) addClient(clientType string, clientAddress string) (clientName string) {
clientName = clientType + "@" + clientAddress
glog.V(0).Infof("+ listener %v", clientName)
messageChan = make(chan *filer_pb.FullEventNotification, 10)
fs.clientChansLock.Lock()
fs.clientChans[clientName] = messageChan
fs.clientChansLock.Unlock()
return
}
func (fs *FilerServer) deleteClient(clientName string, messageChan chan *filer_pb.FullEventNotification) {
func (fs *FilerServer) deleteClient(clientName string) {
glog.V(0).Infof("- listener %v", clientName)
close(messageChan)
fs.clientChansLock.Lock()
delete(fs.clientChans, clientName)
fs.clientChansLock.Unlock()
}
func (fs *FilerServer) sendMessageToClients(dir string, eventNotification *filer_pb.EventNotification) {
message := &filer_pb.FullEventNotification{
Directory: dir,
EventNotification: eventNotification,
}
fs.clientChansLock.RLock()
for _, ch := range fs.clientChans {
ch <- message
}
fs.clientChansLock.RUnlock()
func (fs *FilerServer) notifyMetaListeners() {
fs.listenersCond.Broadcast()
}

8
weed/server/filer_server.go
View File

@ -12,7 +12,6 @@ import (
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util"
@ -57,8 +56,8 @@ type FilerServer struct {
grpcDialOption grpc.DialOption
// notifying clients
clientChansLock sync.RWMutex
clientChans map[string]chan *filer_pb.FullEventNotification
listenersLock sync.Mutex
listenersCond *sync.Cond
}
func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) (fs *FilerServer, err error) {
@ -67,12 +66,13 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
option: option,
grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"),
}
fs.listenersCond = sync.NewCond(&fs.listenersLock)
if len(option.Masters) == 0 {
glog.Fatal("master list is required!")
}
fs.filer = filer2.NewFiler(option.Masters, fs.grpcDialOption, option.Port+10000)
fs.filer = filer2.NewFiler(option.Masters, fs.grpcDialOption, option.Port+10000, fs.notifyMetaListeners)
fs.filer.Cipher = option.Cipher
maybeStartMetrics(fs, option)

Write Preview
Loading…
Cancel
Save