From bf270d9e8c01052409464193b693d50fa09a70a9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 5 Apr 2020 00:51:16 -0700 Subject: [PATCH] filer: able to tail meta data changes --- other/java/client/src/main/proto/filer.proto | 2 +- weed/command/command.go | 15 +- weed/command/tail.go | 63 +++++ weed/filer2/filer.go | 4 +- weed/filer2/filer_notify.go | 37 +++ weed/operation/grpc_client.go | 19 +- weed/pb/filer.proto | 2 +- weed/pb/filer_pb/filer.pb.go | 242 +++++++++---------- weed/queue/log_buffer.go | 82 ++++++- weed/server/filer_grpc_server_listen.go | 71 +++--- weed/server/filer_server.go | 8 +- 11 files changed, 366 insertions(+), 179 deletions(-) create mode 100644 weed/command/tail.go diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 3975b517d..12f1cd85b 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -237,7 +237,7 @@ message GetFilerConfigurationResponse { message ListenForEventsRequest { string client_name = 1; string directory = 2; - int64 since_sec = 3; + int64 since_ns = 3; } message FullEventNotification { string directory = 1; diff --git a/weed/command/command.go b/weed/command/command.go index 9dc51e922..a82f3cdd0 100644 --- a/weed/command/command.go +++ b/weed/command/command.go @@ -12,21 +12,22 @@ var Commands = []*Command{ cmdBackup, cmdCompact, cmdCopy, - cmdFix, + cmdDownload, + cmdExport, + cmdFiler, cmdFilerReplicate, - cmdServer, + cmdFix, cmdMaster, - cmdFiler, + cmdMount, cmdS3, - cmdUpload, - cmdDownload, cmdMsgBroker, cmdScaffold, + cmdServer, cmdShell, + cmdTail, + cmdUpload, cmdVersion, cmdVolume, - cmdExport, - cmdMount, cmdWebDav, } diff --git a/weed/command/tail.go b/weed/command/tail.go new file mode 100644 index 000000000..e81a4cfe2 --- /dev/null +++ b/weed/command/tail.go @@ -0,0 +1,63 @@ +package command + +import ( + "context" + "fmt" + "io" + + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/util" +) + +func init() { + cmdTail.Run = runTail // break init cycle +} + +var cmdTail = &Command{ + UsageLine: "tail [-filer=localhost:8888]", + Short: "see recent changes on a filer", + Long: `See recent changes on a filer. + + `, +} + +var ( + tailFiler = cmdTail.Flag.String("filer", "localhost:8888", "filer hostname:port") + tailTarget = cmdTail.Flag.String("target", "/", "a folder or file on filer") +) + +func runTail(cmd *Command, args []string) bool { + + grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") + + tailErr := pb.WithFilerClient(*tailFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + + stream, err := client.ListenForEvents(context.Background(), &filer_pb.ListenForEventsRequest{ + ClientName: "tail", + Directory: *tailTarget, + SinceNs: 0, + }) + if err != nil { + return fmt.Errorf("listen: %v", err) + } + + for { + resp, listenErr := stream.Recv() + if listenErr == io.EOF { + return nil + } + if listenErr != nil { + return listenErr + } + fmt.Printf("events: %+v\n", resp.EventNotification) + } + + }) + if tailErr != nil { + fmt.Printf("tail %s: %v\n", *tailFiler, tailErr) + } + + return true +} diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index 014eb19e9..a2689f39f 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -39,14 +39,14 @@ type Filer struct { metaLogBuffer *queue.LogBuffer } -func NewFiler(masters []string, grpcDialOption grpc.DialOption, filerGrpcPort uint32) *Filer { +func NewFiler(masters []string, grpcDialOption grpc.DialOption, filerGrpcPort uint32, notifyFn func()) *Filer { f := &Filer{ directoryCache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)), MasterClient: wdclient.NewMasterClient(grpcDialOption, "filer", filerGrpcPort, masters), fileIdDeletionQueue: util.NewUnboundedQueue(), GrpcDialOption: grpcDialOption, } - f.metaLogBuffer = queue.NewLogBuffer(time.Minute, f.logFlushFunc) + f.metaLogBuffer = queue.NewLogBuffer(time.Minute, f.logFlushFunc, notifyFn) go f.loopProcessingDeletion() diff --git a/weed/filer2/filer_notify.go b/weed/filer2/filer_notify.go index e808e45f0..402d87313 100644 --- a/weed/filer2/filer_notify.go +++ b/weed/filer2/filer_notify.go @@ -77,3 +77,40 @@ func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) { } } +func (f *Filer) ReadLogBuffer(lastReadTime time.Time, eachEventFn func(fullpath string, eventNotification *filer_pb.EventNotification) error) (newLastReadTime time.Time, err error) { + + var buf []byte + newLastReadTime, buf = f.metaLogBuffer.ReadFromBuffer(lastReadTime) + + for pos := 0; pos+4 < len(buf); { + + size := util.BytesToUint32(buf[pos : pos+4]) + entryData := buf[pos+4 : pos+4+int(size)] + + logEntry := &filer_pb.LogEntry{} + err = proto.Unmarshal(entryData, logEntry) + if err != nil { + glog.Errorf("unexpected unmarshal filer_pb.LogEntry: %v", err) + return lastReadTime, fmt.Errorf("unexpected unmarshal filer_pb.LogEntry: %v", err) + } + + event := &filer_pb.FullEventNotification{} + err = proto.Unmarshal(logEntry.Data, event) + if err != nil { + glog.Errorf("unexpected unmarshal filer_pb.FullEventNotification: %v", err) + return lastReadTime, fmt.Errorf("unexpected unmarshal filer_pb.FullEventNotification: %v", err) + } + + err = eachEventFn(event.Directory, event.EventNotification) + + if err != nil { + return + } + + pos += 4 + int(size) + + } + + return + +} diff --git a/weed/operation/grpc_client.go b/weed/operation/grpc_client.go index dccf85da4..025a65b38 100644 --- a/weed/operation/grpc_client.go +++ b/weed/operation/grpc_client.go @@ -9,6 +9,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" ) @@ -17,7 +18,7 @@ func WithVolumeServerClient(volumeServer string, grpcDialOption grpc.DialOption, grpcAddress, err := toVolumeServerGrpcAddress(volumeServer) if err != nil { - return err + return fmt.Errorf("failed to parse volume server %v: %v", volumeServer, err) } return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { @@ -41,7 +42,7 @@ func WithMasterServerClient(masterServer string, grpcDialOption grpc.DialOption, masterGrpcAddress, parseErr := pb.ParseServerToGrpcAddress(masterServer) if parseErr != nil { - return fmt.Errorf("failed to parse master grpc %v: %v", masterServer, parseErr) + return fmt.Errorf("failed to parse master %v: %v", masterServer, parseErr) } return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { @@ -50,3 +51,17 @@ func WithMasterServerClient(masterServer string, grpcDialOption grpc.DialOption, }, masterGrpcAddress, grpcDialOption) } + +func WithFilerServerClient(filerServer string, grpcDialOption grpc.DialOption, fn func(masterClient filer_pb.SeaweedFilerClient) error) error { + + filerGrpcAddress, parseErr := pb.ParseServerToGrpcAddress(filerServer) + if parseErr != nil { + return fmt.Errorf("failed to parse filer %v: %v", filerGrpcAddress, parseErr) + } + + return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { + client := filer_pb.NewSeaweedFilerClient(grpcConnection) + return fn(client) + }, filerGrpcAddress, grpcDialOption) + +} diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index 3975b517d..12f1cd85b 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -237,7 +237,7 @@ message GetFilerConfigurationResponse { message ListenForEventsRequest { string client_name = 1; string directory = 2; - int64 since_sec = 3; + int64 since_ns = 3; } message FullEventNotification { string directory = 1; diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 461a575b5..c927c0f87 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -1093,7 +1093,7 @@ func (m *GetFilerConfigurationResponse) GetCipher() bool { type ListenForEventsRequest struct { ClientName string `protobuf:"bytes,1,opt,name=client_name,json=clientName" json:"client_name,omitempty"` Directory string `protobuf:"bytes,2,opt,name=directory" json:"directory,omitempty"` - SinceSec int64 `protobuf:"varint,3,opt,name=since_sec,json=sinceSec" json:"since_sec,omitempty"` + SinceNs int64 `protobuf:"varint,3,opt,name=since_ns,json=sinceNs" json:"since_ns,omitempty"` } func (m *ListenForEventsRequest) Reset() { *m = ListenForEventsRequest{} } @@ -1115,9 +1115,9 @@ func (m *ListenForEventsRequest) GetDirectory() string { return "" } -func (m *ListenForEventsRequest) GetSinceSec() int64 { +func (m *ListenForEventsRequest) GetSinceNs() int64 { if m != nil { - return m.SinceSec + return m.SinceNs } return 0 } @@ -1709,122 +1709,122 @@ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ // 1901 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x5f, 0x6f, 0xdc, 0xc6, - 0x11, 0x37, 0xef, 0x9f, 0x8e, 0x73, 0x77, 0xb6, 0xb4, 0x27, 0x39, 0xe7, 0x93, 0x64, 0x2b, 0x74, - 0x9d, 0xba, 0xb0, 0xa1, 0x1a, 0x6a, 0x1e, 0x92, 0xa6, 0x7d, 0xb0, 0x65, 0x29, 0x75, 0x63, 0x2b, - 0x2e, 0x65, 0x17, 0x29, 0x0a, 0x94, 0xa0, 0xc8, 0xd5, 0x69, 0x2b, 0x1e, 0xc9, 0xec, 0x2e, 0xf5, - 0x27, 0x6f, 0xfd, 0x1a, 0x05, 0xfa, 0xd0, 0xef, 0xd0, 0xc7, 0xa2, 0x2f, 0x45, 0x81, 0x7e, 0x8e, - 0x3e, 0xf6, 0xa1, 0x9f, 0xa1, 0xd8, 0x59, 0x92, 0xb7, 0x3c, 0x9e, 0xa4, 0x04, 0x41, 0xde, 0x76, - 0x67, 0x66, 0x67, 0x67, 0xe7, 0xcf, 0x6f, 0x86, 0x84, 0xde, 0x31, 0x8b, 0x28, 0xdf, 0x4e, 0x79, - 0x22, 0x13, 0xd2, 0xc5, 0x8d, 0x97, 0x1e, 0x39, 0x5f, 0xc2, 0xfa, 0xeb, 0x24, 0x39, 0xcd, 0xd2, - 0x97, 0x8c, 0xd3, 0x40, 0x26, 0xfc, 0x72, 0x2f, 0x96, 0xfc, 0xd2, 0xa5, 0x5f, 0x67, 0x54, 0x48, - 0xb2, 0x01, 0x76, 0x58, 0x30, 0x46, 0xd6, 0x96, 0xf5, 0xd8, 0x76, 0x67, 0x04, 0x42, 0xa0, 0x15, - 0xfb, 0x53, 0x3a, 0x6a, 0x20, 0x03, 0xd7, 0xce, 0x1e, 0x6c, 0x2c, 0x56, 0x28, 0xd2, 0x24, 0x16, - 0x94, 0x3c, 0x82, 0x36, 0x55, 0x04, 0xd4, 0xd6, 0xdb, 0xb9, 0xb3, 0x5d, 0x98, 0xb2, 0xad, 0xe5, - 0x34, 0xd7, 0xf9, 0x87, 0x05, 0xe4, 0x35, 0x13, 0x52, 0x11, 0x19, 0x15, 0xdf, 0xce, 0x9e, 0xbb, - 0xd0, 0x49, 0x39, 0x3d, 0x66, 0x17, 0xb9, 0x45, 0xf9, 0x8e, 0x3c, 0x85, 0x15, 0x21, 0x7d, 0x2e, - 0xf7, 0x79, 0x32, 0xdd, 0x67, 0x11, 0x3d, 0x50, 0x46, 0x37, 0x51, 0xa4, 0xce, 0x20, 0xdb, 0x40, - 0x58, 0x1c, 0x44, 0x99, 0x60, 0x67, 0xf4, 0xb0, 0xe0, 0x8e, 0x5a, 0x5b, 0xd6, 0xe3, 0xae, 0xbb, - 0x80, 0x43, 0x56, 0xa1, 0x1d, 0xb1, 0x29, 0x93, 0xa3, 0xf6, 0x96, 0xf5, 0x78, 0xe0, 0xea, 0x8d, - 0xf3, 0x0b, 0x18, 0x56, 0xec, 0xff, 0x6e, 0xcf, 0xff, 0x4b, 0x03, 0xda, 0x48, 0x28, 0x7d, 0x6c, - 0xcd, 0x7c, 0x4c, 0x3e, 0x84, 0x3e, 0x13, 0xde, 0xcc, 0x11, 0x0d, 0xb4, 0xad, 0xc7, 0x44, 0xe9, - 0x73, 0xf2, 0x04, 0x3a, 0xc1, 0x49, 0x16, 0x9f, 0x8a, 0x51, 0x73, 0xab, 0xf9, 0xb8, 0xb7, 0x33, - 0x9c, 0x5d, 0xa4, 0x1e, 0xba, 0xab, 0x78, 0x6e, 0x2e, 0x42, 0x3e, 0x01, 0xf0, 0xa5, 0xe4, 0xec, - 0x28, 0x93, 0x54, 0xe0, 0x4b, 0x7b, 0x3b, 0x23, 0xe3, 0x40, 0x26, 0xe8, 0xf3, 0x92, 0xef, 0x1a, - 0xb2, 0xe4, 0x53, 0xe8, 0xd2, 0x0b, 0x49, 0xe3, 0x90, 0x86, 0xa3, 0x36, 0x5e, 0xb4, 0x39, 0xf7, - 0xa2, 0xed, 0xbd, 0x9c, 0xaf, 0xdf, 0x57, 0x8a, 0x8f, 0x3f, 0x83, 0x41, 0x85, 0x45, 0x96, 0xa1, - 0x79, 0x4a, 0x8b, 0xa8, 0xaa, 0xa5, 0xf2, 0xec, 0x99, 0x1f, 0x65, 0x3a, 0xc1, 0xfa, 0xae, 0xde, - 0xfc, 0xbc, 0xf1, 0x89, 0xe5, 0xbc, 0x04, 0x7b, 0x3f, 0x8b, 0xa2, 0xf2, 0x60, 0xc8, 0x78, 0x71, - 0x30, 0x64, 0x7c, 0xe6, 0xe5, 0xc6, 0xb5, 0x5e, 0xfe, 0xbb, 0x05, 0x2b, 0x7b, 0x67, 0x34, 0x96, - 0x07, 0x89, 0x64, 0xc7, 0x2c, 0xf0, 0x25, 0x4b, 0x62, 0xf2, 0x14, 0xec, 0x24, 0x0a, 0xbd, 0x6b, - 0xc3, 0xd4, 0x4d, 0xa2, 0xdc, 0xea, 0xa7, 0x60, 0xc7, 0xf4, 0xdc, 0xbb, 0xf6, 0xba, 0x6e, 0x4c, - 0xcf, 0xb5, 0xf4, 0x43, 0x18, 0x84, 0x34, 0xa2, 0x92, 0x7a, 0x65, 0x74, 0x54, 0xe8, 0xfa, 0x9a, - 0xb8, 0xab, 0xc3, 0xf1, 0x11, 0xdc, 0x51, 0x2a, 0x53, 0x9f, 0xd3, 0x58, 0x7a, 0xa9, 0x2f, 0x4f, - 0x30, 0x26, 0xb6, 0x3b, 0x88, 0xe9, 0xf9, 0x5b, 0xa4, 0xbe, 0xf5, 0xe5, 0x89, 0xf3, 0xb7, 0x06, - 0xd8, 0x65, 0x30, 0xc9, 0x07, 0xb0, 0xa4, 0xae, 0xf5, 0x58, 0x98, 0x7b, 0xa2, 0xa3, 0xb6, 0xaf, - 0x42, 0x55, 0x15, 0xc9, 0xf1, 0xb1, 0xa0, 0x12, 0xcd, 0x6b, 0xba, 0xf9, 0x4e, 0x65, 0x96, 0x60, - 0xdf, 0xe8, 0x42, 0x68, 0xb9, 0xb8, 0x56, 0x1e, 0x9f, 0x4a, 0x36, 0xa5, 0x78, 0x61, 0xd3, 0xd5, - 0x1b, 0x32, 0x84, 0x36, 0xf5, 0xa4, 0x3f, 0xc1, 0x0c, 0xb7, 0xdd, 0x16, 0x7d, 0xe7, 0x4f, 0xc8, - 0x8f, 0xe0, 0xb6, 0x48, 0x32, 0x1e, 0x50, 0xaf, 0xb8, 0xb6, 0x83, 0xdc, 0xbe, 0xa6, 0xee, 0xeb, - 0xcb, 0x1d, 0x68, 0x1e, 0xb3, 0x70, 0xb4, 0x84, 0x8e, 0x59, 0xae, 0x26, 0xe1, 0xab, 0xd0, 0x55, - 0x4c, 0xf2, 0x53, 0x80, 0x52, 0x53, 0x38, 0xea, 0x5e, 0x21, 0x6a, 0x17, 0x7a, 0x43, 0xb2, 0x09, - 0x10, 0xb0, 0xf4, 0x84, 0x72, 0x4f, 0x25, 0x8c, 0x8d, 0xc9, 0x61, 0x6b, 0xca, 0x17, 0xf4, 0x52, - 0xb1, 0x99, 0xf0, 0x26, 0xdf, 0xb0, 0x34, 0xa5, 0xe1, 0x08, 0xd0, 0xc3, 0x36, 0x13, 0x9f, 0x6b, - 0x82, 0xf3, 0x15, 0x74, 0x72, 0xe3, 0xd6, 0xc1, 0x3e, 0x4b, 0xa2, 0x6c, 0x5a, 0x3a, 0x6d, 0xe0, - 0x76, 0x35, 0xe1, 0x55, 0x48, 0xee, 0x01, 0xa2, 0x24, 0x5e, 0xd1, 0x40, 0x17, 0xa1, 0x7f, 0xd5, - 0x05, 0x77, 0xa1, 0x13, 0x24, 0xc9, 0x29, 0xd3, 0xbe, 0x5b, 0x72, 0xf3, 0x9d, 0xf3, 0xbf, 0x06, - 0xdc, 0xae, 0x16, 0x8b, 0xba, 0x02, 0xb5, 0xa0, 0xa7, 0x2d, 0x54, 0x83, 0x6a, 0x0f, 0x2b, 0xde, - 0x6e, 0x98, 0xde, 0x2e, 0x8e, 0x4c, 0x93, 0x50, 0x5f, 0x30, 0xd0, 0x47, 0xde, 0x24, 0x21, 0x55, - 0xb9, 0x9e, 0xb1, 0x10, 0xc3, 0x33, 0x70, 0xd5, 0x52, 0x51, 0x26, 0x2c, 0xcc, 0xc1, 0x47, 0x2d, - 0xd1, 0x3c, 0x8e, 0x7a, 0x3b, 0x3a, 0xe0, 0x7a, 0xa7, 0x02, 0x3e, 0x55, 0xd4, 0x25, 0x1d, 0x45, - 0xb5, 0x26, 0x5b, 0xd0, 0xe3, 0x34, 0x8d, 0xf2, 0xdc, 0x47, 0xe7, 0xdb, 0xae, 0x49, 0x22, 0xf7, - 0x01, 0x82, 0x24, 0x8a, 0x68, 0x80, 0x02, 0x36, 0x0a, 0x18, 0x14, 0x95, 0x77, 0x52, 0x46, 0x9e, - 0xa0, 0x01, 0xba, 0xba, 0xed, 0x76, 0xa4, 0x8c, 0x0e, 0x69, 0xa0, 0xde, 0x91, 0x09, 0xca, 0x3d, - 0x84, 0xaf, 0x1e, 0x9e, 0xeb, 0x2a, 0x02, 0x82, 0xec, 0x26, 0xc0, 0x84, 0x27, 0x59, 0xaa, 0xb9, - 0xfd, 0xad, 0xa6, 0x42, 0x72, 0xa4, 0x20, 0xfb, 0x11, 0xdc, 0x16, 0x97, 0xd3, 0x88, 0xc5, 0xa7, - 0x9e, 0xf4, 0xf9, 0x84, 0xca, 0xd1, 0x40, 0x57, 0x40, 0x4e, 0x7d, 0x87, 0x44, 0x27, 0x05, 0xb2, - 0xcb, 0xa9, 0x2f, 0xe9, 0x77, 0x68, 0x5a, 0xdf, 0x0e, 0x1b, 0xc8, 0x1a, 0x74, 0x12, 0x8f, 0x5e, - 0x04, 0x51, 0x5e, 0xa2, 0xed, 0x64, 0xef, 0x22, 0x88, 0x9c, 0x27, 0x30, 0xac, 0xdc, 0x98, 0xc3, - 0xfa, 0x2a, 0xb4, 0x29, 0xe7, 0x49, 0x01, 0x42, 0x7a, 0xe3, 0xfc, 0x0e, 0xc8, 0xfb, 0x34, 0xfc, - 0x21, 0xcc, 0x73, 0xd6, 0x60, 0x58, 0x51, 0xad, 0xed, 0x70, 0xfe, 0x65, 0x01, 0x79, 0x89, 0x58, - 0xf2, 0xfd, 0xda, 0xb8, 0xaa, 0x6e, 0xd5, 0x62, 0x34, 0x56, 0x85, 0xbe, 0xf4, 0xf3, 0x06, 0xd8, - 0x67, 0x42, 0xeb, 0x7f, 0xe9, 0x4b, 0x3f, 0x6f, 0x44, 0x9c, 0x06, 0x19, 0x57, 0x3d, 0x11, 0x93, - 0x10, 0x1b, 0x91, 0x5b, 0x90, 0xc8, 0xc7, 0x70, 0x97, 0x4d, 0xe2, 0x84, 0xd3, 0x99, 0x98, 0xa7, - 0x5d, 0xd5, 0x41, 0xe1, 0x55, 0xcd, 0x2d, 0x0f, 0xec, 0xa1, 0xe7, 0x9e, 0xc0, 0xb0, 0xf2, 0x8c, - 0x6b, 0xdd, 0xfc, 0x67, 0x0b, 0x46, 0xcf, 0x65, 0x32, 0x65, 0x81, 0x4b, 0x95, 0xf1, 0x95, 0xa7, - 0x3f, 0x84, 0x81, 0x42, 0xf3, 0xf9, 0xe7, 0xf7, 0x93, 0x28, 0x9c, 0x75, 0xcb, 0x7b, 0xa0, 0x00, - 0xdd, 0x33, 0xbc, 0xb0, 0x94, 0x44, 0x21, 0x66, 0xe2, 0x43, 0x50, 0xa8, 0x6b, 0x9c, 0xd7, 0x73, - 0x43, 0x3f, 0xa6, 0xe7, 0x95, 0xf3, 0x4a, 0x08, 0xcf, 0x6b, 0xa8, 0x5e, 0x8a, 0xe9, 0xb9, 0x3a, - 0xef, 0xac, 0xc3, 0xbd, 0x05, 0xb6, 0xe5, 0xe1, 0xfa, 0xb7, 0x05, 0xc3, 0xe7, 0x42, 0xb0, 0x49, - 0xfc, 0x5b, 0x84, 0x9d, 0xc2, 0xe8, 0x55, 0x68, 0x07, 0x49, 0x16, 0x4b, 0x34, 0xb6, 0xed, 0xea, - 0xcd, 0x5c, 0x25, 0x36, 0x6a, 0x95, 0x38, 0x57, 0xcb, 0xcd, 0x7a, 0x2d, 0x1b, 0xb5, 0xda, 0xaa, - 0xd4, 0xea, 0x03, 0xe8, 0xa9, 0x20, 0x7b, 0x01, 0x8d, 0x25, 0xe5, 0x39, 0xce, 0x83, 0x22, 0xed, - 0x22, 0x45, 0x09, 0x98, 0xfd, 0x48, 0x43, 0x3d, 0xa4, 0xb3, 0x66, 0xf4, 0x1f, 0x0b, 0x56, 0xab, - 0x4f, 0xc9, 0x63, 0x76, 0x65, 0x5f, 0x52, 0x50, 0xc6, 0xa3, 0xfc, 0x1d, 0x6a, 0xa9, 0x40, 0x21, - 0xcd, 0x8e, 0x22, 0x16, 0x78, 0x8a, 0xa1, 0xed, 0xb7, 0x35, 0xe5, 0x3d, 0x8f, 0x66, 0x5e, 0x69, - 0x99, 0x5e, 0x21, 0xd0, 0xf2, 0x33, 0x79, 0x52, 0xf4, 0x26, 0xb5, 0x9e, 0xf3, 0x54, 0xe7, 0x26, - 0x4f, 0x2d, 0xd5, 0x3d, 0x55, 0x66, 0x5a, 0xd7, 0xcc, 0xb4, 0x8f, 0x61, 0xa8, 0x87, 0xdb, 0x6a, - 0xb8, 0x36, 0x01, 0xca, 0x3e, 0x22, 0x46, 0x96, 0x06, 0xb3, 0xa2, 0x91, 0x08, 0xe7, 0x97, 0x60, - 0xbf, 0x4e, 0xb4, 0x5e, 0x41, 0x9e, 0x81, 0x1d, 0x15, 0x1b, 0x14, 0xed, 0xed, 0x90, 0x59, 0x8d, - 0x17, 0x72, 0xee, 0x4c, 0xc8, 0xf9, 0x0c, 0xba, 0x05, 0xb9, 0xf0, 0x99, 0x75, 0x95, 0xcf, 0x1a, - 0x73, 0x3e, 0x73, 0xfe, 0x69, 0xc1, 0x6a, 0xd5, 0xe4, 0x3c, 0x2c, 0xef, 0x61, 0x50, 0x5e, 0xe1, - 0x4d, 0xfd, 0x34, 0xb7, 0xe5, 0x99, 0x69, 0x4b, 0xfd, 0x58, 0x69, 0xa0, 0x78, 0xe3, 0xa7, 0x3a, - 0x97, 0xfb, 0x91, 0x41, 0x1a, 0xbf, 0x83, 0x95, 0x9a, 0xc8, 0x82, 0xc9, 0xee, 0x27, 0xe6, 0x64, - 0x57, 0x99, 0x4e, 0xcb, 0xd3, 0xe6, 0xb8, 0xf7, 0x29, 0x7c, 0xa0, 0xe1, 0x60, 0xb7, 0x8c, 0x61, - 0xe1, 0xfb, 0x6a, 0xa8, 0xad, 0xf9, 0x50, 0x3b, 0x63, 0x18, 0xd5, 0x8f, 0xe6, 0xe5, 0x37, 0x81, - 0x95, 0x43, 0xe9, 0x4b, 0x26, 0x24, 0x0b, 0xca, 0x4f, 0x8c, 0xb9, 0xdc, 0xb0, 0x6e, 0xea, 0x88, - 0xf5, 0x3a, 0x5c, 0x86, 0xa6, 0x94, 0x45, 0xfe, 0xaa, 0xa5, 0x8a, 0x02, 0x31, 0x6f, 0xca, 0x63, - 0xf0, 0x03, 0x5c, 0xa5, 0xf2, 0x41, 0x26, 0xd2, 0x8f, 0xf4, 0xc4, 0xd1, 0xc2, 0x89, 0xc3, 0x46, - 0x0a, 0x8e, 0x1c, 0xba, 0x29, 0x87, 0x9a, 0xdb, 0xd6, 0xf3, 0x88, 0x22, 0x20, 0x73, 0x13, 0x00, - 0x4b, 0x55, 0x57, 0x59, 0x47, 0x9f, 0x55, 0x94, 0x5d, 0x45, 0x70, 0xee, 0xc3, 0xc6, 0xe7, 0x54, - 0xaa, 0xd9, 0x89, 0xef, 0x26, 0xf1, 0x31, 0x9b, 0x64, 0xdc, 0x37, 0x42, 0xe1, 0xfc, 0xd7, 0x82, - 0xcd, 0x2b, 0x04, 0xf2, 0x07, 0x8f, 0x60, 0x69, 0xea, 0x0b, 0x49, 0x79, 0x51, 0x25, 0xc5, 0x76, - 0xde, 0x15, 0x8d, 0x9b, 0x5c, 0xd1, 0xac, 0xb9, 0x62, 0x0d, 0x3a, 0x53, 0xff, 0xc2, 0x9b, 0x1e, - 0xe5, 0xc3, 0x51, 0x7b, 0xea, 0x5f, 0xbc, 0x39, 0x42, 0x64, 0x63, 0xdc, 0x3b, 0xca, 0x82, 0x53, - 0x2a, 0x45, 0x89, 0x6c, 0x8c, 0xbf, 0xd0, 0x14, 0xf5, 0x68, 0x25, 0xf0, 0x75, 0x46, 0x33, 0x2a, - 0x72, 0xac, 0x50, 0xcd, 0xf1, 0x37, 0x48, 0xc0, 0x61, 0x0a, 0x27, 0x4b, 0x44, 0x89, 0xae, 0x9b, - 0xef, 0x1c, 0x09, 0x77, 0xd5, 0xf7, 0x1d, 0x8d, 0xf7, 0x13, 0x8e, 0xdf, 0x10, 0x65, 0x02, 0x3d, - 0x80, 0x5e, 0x10, 0x31, 0x05, 0x95, 0xc6, 0x87, 0x1b, 0x68, 0x12, 0xb6, 0x94, 0x4a, 0x37, 0x6e, - 0xcc, 0x77, 0xe3, 0x75, 0xb0, 0x05, 0x8b, 0x03, 0x8a, 0x28, 0xdd, 0xc4, 0x01, 0xae, 0x8b, 0x84, - 0x43, 0x1a, 0x38, 0x7f, 0xb2, 0x60, 0x0d, 0x3f, 0x7c, 0x6a, 0x5f, 0x2d, 0xd7, 0xb7, 0xf8, 0x5f, - 0x03, 0xa1, 0x67, 0x68, 0x92, 0x71, 0x26, 0x2f, 0xbe, 0x75, 0x63, 0xc4, 0x98, 0x57, 0xeb, 0xae, - 0xd0, 0x79, 0x92, 0xe3, 0x2b, 0x3c, 0x9a, 0xe8, 0xca, 0x1e, 0x42, 0x5b, 0x0a, 0x0f, 0x91, 0x4c, - 0x19, 0xda, 0x92, 0xe2, 0x40, 0x90, 0xa7, 0x40, 0x52, 0x9f, 0x4b, 0xa6, 0xa4, 0xd5, 0xf8, 0xec, - 0x9d, 0xf8, 0xe2, 0x04, 0x2f, 0x6b, 0xbb, 0xcb, 0x25, 0xe7, 0x0b, 0x7a, 0xf9, 0x2b, 0x5f, 0x9c, - 0x28, 0xfc, 0xc6, 0xf9, 0xa2, 0x89, 0x63, 0x3c, 0xae, 0x77, 0xfe, 0xda, 0x85, 0xfe, 0x21, 0xf5, - 0xcf, 0x29, 0x0d, 0x31, 0x9b, 0xc8, 0xa4, 0x40, 0xb1, 0xea, 0x5f, 0x05, 0xf2, 0x68, 0x1e, 0xae, - 0x16, 0xfe, 0xc6, 0x18, 0x7f, 0x74, 0x93, 0x58, 0x0e, 0x08, 0xb7, 0xc8, 0x01, 0xf4, 0x8c, 0xcf, - 0x76, 0xb2, 0x61, 0x1c, 0xac, 0xfd, 0x8d, 0x18, 0x6f, 0x5e, 0xc1, 0x2d, 0xb4, 0x3d, 0xb3, 0xc8, - 0x6b, 0xe8, 0x19, 0xf3, 0xa2, 0xa9, 0xaf, 0x3e, 0xb8, 0x9a, 0xfa, 0x16, 0x0c, 0x99, 0xce, 0x2d, - 0xa5, 0xcd, 0x98, 0xfa, 0x4c, 0x6d, 0xf5, 0x39, 0xd3, 0xd4, 0xb6, 0x68, 0x54, 0x44, 0x6d, 0xc6, - 0x90, 0x65, 0x6a, 0xab, 0x8f, 0x90, 0xa6, 0xb6, 0x05, 0x93, 0x99, 0x73, 0x8b, 0xfc, 0x01, 0x56, - 0x6a, 0x83, 0x0e, 0x71, 0x66, 0xa7, 0xae, 0x9a, 0xd0, 0xc6, 0x0f, 0xaf, 0x95, 0x29, 0xf5, 0x7f, - 0x09, 0x7d, 0x73, 0xbe, 0x20, 0x86, 0x41, 0x0b, 0x46, 0xa8, 0xf1, 0xfd, 0xab, 0xd8, 0xa6, 0x42, - 0xb3, 0xc5, 0x99, 0x0a, 0x17, 0x34, 0x79, 0x53, 0xe1, 0xa2, 0xce, 0xe8, 0xdc, 0x22, 0xbf, 0x87, - 0xe5, 0xf9, 0x56, 0x43, 0x3e, 0x9c, 0x77, 0x5b, 0xad, 0x83, 0x8d, 0x9d, 0xeb, 0x44, 0x4a, 0xe5, - 0xaf, 0x00, 0x66, 0x1d, 0x84, 0x18, 0x35, 0x5b, 0xeb, 0x60, 0xe3, 0x8d, 0xc5, 0xcc, 0x52, 0xd5, - 0x1f, 0x61, 0x6d, 0x21, 0x4c, 0x13, 0xa3, 0x4c, 0xae, 0x03, 0xfa, 0xf1, 0x8f, 0x6f, 0x94, 0x2b, - 0xef, 0xfa, 0x0a, 0xee, 0xcc, 0xc1, 0x24, 0xd9, 0xaa, 0x56, 0x4d, 0x1d, 0x41, 0xc7, 0x0f, 0xcc, - 0x7f, 0x4f, 0x0b, 0xc0, 0x4e, 0x55, 0xd6, 0x8b, 0xfb, 0xb0, 0x2c, 0x34, 0x44, 0x1c, 0x8b, 0x6d, - 0x8d, 0xae, 0x2f, 0x00, 0x6d, 0x79, 0xcb, 0x13, 0x99, 0x1c, 0x75, 0xf0, 0x57, 0xe7, 0xcf, 0xfe, - 0x1f, 0x00, 0x00, 0xff, 0xff, 0xbe, 0x11, 0xf3, 0xf2, 0xf9, 0x14, 0x00, 0x00, + 0x11, 0x37, 0xef, 0x3f, 0xe7, 0xee, 0x6c, 0x69, 0x4f, 0x76, 0xce, 0x67, 0xc9, 0x56, 0xe8, 0x3a, + 0x75, 0x61, 0x43, 0x35, 0xd4, 0x3c, 0x24, 0x4d, 0xfb, 0x60, 0xcb, 0x52, 0xea, 0xc6, 0x56, 0x5c, + 0xca, 0x2e, 0x52, 0x14, 0x28, 0x41, 0x91, 0xab, 0xd3, 0x56, 0x3c, 0x92, 0xd9, 0x5d, 0xea, 0x4f, + 0xde, 0xfa, 0x35, 0x0a, 0xf4, 0xa1, 0xdf, 0xa1, 0x8f, 0x45, 0x5f, 0x8a, 0x02, 0xfd, 0x1c, 0x7d, + 0xec, 0x43, 0x3f, 0x43, 0xb1, 0xb3, 0x24, 0x6f, 0x79, 0x3c, 0x49, 0x09, 0x82, 0xbc, 0xed, 0xce, + 0xcc, 0xce, 0xce, 0xce, 0x9f, 0xdf, 0x0c, 0x09, 0xfd, 0x23, 0x16, 0x51, 0xbe, 0x95, 0xf2, 0x44, + 0x26, 0xa4, 0x87, 0x1b, 0x2f, 0x3d, 0x74, 0xbe, 0x84, 0x7b, 0xaf, 0x93, 0xe4, 0x24, 0x4b, 0x5f, + 0x32, 0x4e, 0x03, 0x99, 0xf0, 0x8b, 0xdd, 0x58, 0xf2, 0x0b, 0x97, 0x7e, 0x9d, 0x51, 0x21, 0xc9, + 0x3a, 0xd8, 0x61, 0xc1, 0x18, 0x5b, 0x9b, 0xd6, 0x63, 0xdb, 0x9d, 0x13, 0x08, 0x81, 0x56, 0xec, + 0xcf, 0xe8, 0xb8, 0x81, 0x0c, 0x5c, 0x3b, 0xbb, 0xb0, 0xbe, 0x5c, 0xa1, 0x48, 0x93, 0x58, 0x50, + 0xf2, 0x08, 0xda, 0x54, 0x11, 0x50, 0x5b, 0x7f, 0xfb, 0xd6, 0x56, 0x61, 0xca, 0x96, 0x96, 0xd3, + 0x5c, 0xe7, 0x1f, 0x16, 0x90, 0xd7, 0x4c, 0x48, 0x45, 0x64, 0x54, 0x7c, 0x3b, 0x7b, 0xee, 0x40, + 0x27, 0xe5, 0xf4, 0x88, 0x9d, 0xe7, 0x16, 0xe5, 0x3b, 0xf2, 0x14, 0x56, 0x85, 0xf4, 0xb9, 0xdc, + 0xe3, 0xc9, 0x6c, 0x8f, 0x45, 0x74, 0x5f, 0x19, 0xdd, 0x44, 0x91, 0x3a, 0x83, 0x6c, 0x01, 0x61, + 0x71, 0x10, 0x65, 0x82, 0x9d, 0xd2, 0x83, 0x82, 0x3b, 0x6e, 0x6d, 0x5a, 0x8f, 0x7b, 0xee, 0x12, + 0x0e, 0x59, 0x83, 0x76, 0xc4, 0x66, 0x4c, 0x8e, 0xdb, 0x9b, 0xd6, 0xe3, 0xa1, 0xab, 0x37, 0xce, + 0x2f, 0x60, 0x54, 0xb1, 0xff, 0xbb, 0x3d, 0xff, 0x2f, 0x0d, 0x68, 0x23, 0xa1, 0xf4, 0xb1, 0x35, + 0xf7, 0x31, 0xf9, 0x10, 0x06, 0x4c, 0x78, 0x73, 0x47, 0x34, 0xd0, 0xb6, 0x3e, 0x13, 0xa5, 0xcf, + 0xc9, 0x13, 0xe8, 0x04, 0xc7, 0x59, 0x7c, 0x22, 0xc6, 0xcd, 0xcd, 0xe6, 0xe3, 0xfe, 0xf6, 0x68, + 0x7e, 0x91, 0x7a, 0xe8, 0x8e, 0xe2, 0xb9, 0xb9, 0x08, 0xf9, 0x04, 0xc0, 0x97, 0x92, 0xb3, 0xc3, + 0x4c, 0x52, 0x81, 0x2f, 0xed, 0x6f, 0x8f, 0x8d, 0x03, 0x99, 0xa0, 0xcf, 0x4b, 0xbe, 0x6b, 0xc8, + 0x92, 0x4f, 0xa1, 0x47, 0xcf, 0x25, 0x8d, 0x43, 0x1a, 0x8e, 0xdb, 0x78, 0xd1, 0xc6, 0xc2, 0x8b, + 0xb6, 0x76, 0x73, 0xbe, 0x7e, 0x5f, 0x29, 0x3e, 0xf9, 0x0c, 0x86, 0x15, 0x16, 0x59, 0x81, 0xe6, + 0x09, 0x2d, 0xa2, 0xaa, 0x96, 0xca, 0xb3, 0xa7, 0x7e, 0x94, 0xe9, 0x04, 0x1b, 0xb8, 0x7a, 0xf3, + 0xf3, 0xc6, 0x27, 0x96, 0xf3, 0x12, 0xec, 0xbd, 0x2c, 0x8a, 0xca, 0x83, 0x21, 0xe3, 0xc5, 0xc1, + 0x90, 0xf1, 0xb9, 0x97, 0x1b, 0x57, 0x7a, 0xf9, 0xef, 0x16, 0xac, 0xee, 0x9e, 0xd2, 0x58, 0xee, + 0x27, 0x92, 0x1d, 0xb1, 0xc0, 0x97, 0x2c, 0x89, 0xc9, 0x53, 0xb0, 0x93, 0x28, 0xf4, 0xae, 0x0c, + 0x53, 0x2f, 0x89, 0x72, 0xab, 0x9f, 0x82, 0x1d, 0xd3, 0x33, 0xef, 0xca, 0xeb, 0x7a, 0x31, 0x3d, + 0xd3, 0xd2, 0x0f, 0x61, 0x18, 0xd2, 0x88, 0x4a, 0xea, 0x95, 0xd1, 0x51, 0xa1, 0x1b, 0x68, 0xe2, + 0x8e, 0x0e, 0xc7, 0x47, 0x70, 0x4b, 0xa9, 0x4c, 0x7d, 0x4e, 0x63, 0xe9, 0xa5, 0xbe, 0x3c, 0xc6, + 0x98, 0xd8, 0xee, 0x30, 0xa6, 0x67, 0x6f, 0x91, 0xfa, 0xd6, 0x97, 0xc7, 0xce, 0xdf, 0x1a, 0x60, + 0x97, 0xc1, 0x24, 0x1f, 0x40, 0x57, 0x5d, 0xeb, 0xb1, 0x30, 0xf7, 0x44, 0x47, 0x6d, 0x5f, 0x85, + 0xaa, 0x2a, 0x92, 0xa3, 0x23, 0x41, 0x25, 0x9a, 0xd7, 0x74, 0xf3, 0x9d, 0xca, 0x2c, 0xc1, 0xbe, + 0xd1, 0x85, 0xd0, 0x72, 0x71, 0xad, 0x3c, 0x3e, 0x93, 0x6c, 0x46, 0xf1, 0xc2, 0xa6, 0xab, 0x37, + 0x64, 0x04, 0x6d, 0xea, 0x49, 0x7f, 0x8a, 0x19, 0x6e, 0xbb, 0x2d, 0xfa, 0xce, 0x9f, 0x92, 0x1f, + 0xc1, 0x4d, 0x91, 0x64, 0x3c, 0xa0, 0x5e, 0x71, 0x6d, 0x07, 0xb9, 0x03, 0x4d, 0xdd, 0xd3, 0x97, + 0x3b, 0xd0, 0x3c, 0x62, 0xe1, 0xb8, 0x8b, 0x8e, 0x59, 0xa9, 0x26, 0xe1, 0xab, 0xd0, 0x55, 0x4c, + 0xf2, 0x53, 0x80, 0x52, 0x53, 0x38, 0xee, 0x5d, 0x22, 0x6a, 0x17, 0x7a, 0x43, 0xb2, 0x01, 0x10, + 0xb0, 0xf4, 0x98, 0x72, 0x4f, 0x25, 0x8c, 0x8d, 0xc9, 0x61, 0x6b, 0xca, 0x17, 0xf4, 0x42, 0xb1, + 0x99, 0xf0, 0xa6, 0xdf, 0xb0, 0x34, 0xa5, 0xe1, 0x18, 0xd0, 0xc3, 0x36, 0x13, 0x9f, 0x6b, 0x82, + 0xf3, 0x15, 0x74, 0x72, 0xe3, 0xee, 0x81, 0x7d, 0x9a, 0x44, 0xd9, 0xac, 0x74, 0xda, 0xd0, 0xed, + 0x69, 0xc2, 0xab, 0x90, 0xdc, 0x05, 0x44, 0x49, 0xbc, 0xa2, 0x81, 0x2e, 0x42, 0xff, 0xaa, 0x0b, + 0xee, 0x40, 0x27, 0x48, 0x92, 0x13, 0xa6, 0x7d, 0xd7, 0x75, 0xf3, 0x9d, 0xf3, 0xbf, 0x06, 0xdc, + 0xac, 0x16, 0x8b, 0xba, 0x02, 0xb5, 0xa0, 0xa7, 0x2d, 0x54, 0x83, 0x6a, 0x0f, 0x2a, 0xde, 0x6e, + 0x98, 0xde, 0x2e, 0x8e, 0xcc, 0x92, 0x50, 0x5f, 0x30, 0xd4, 0x47, 0xde, 0x24, 0x21, 0x55, 0xb9, + 0x9e, 0xb1, 0x10, 0xc3, 0x33, 0x74, 0xd5, 0x52, 0x51, 0xa6, 0x2c, 0xcc, 0xc1, 0x47, 0x2d, 0xd1, + 0x3c, 0x8e, 0x7a, 0x3b, 0x3a, 0xe0, 0x7a, 0xa7, 0x02, 0x3e, 0x53, 0xd4, 0xae, 0x8e, 0xa2, 0x5a, + 0x93, 0x4d, 0xe8, 0x73, 0x9a, 0x46, 0x79, 0xee, 0xa3, 0xf3, 0x6d, 0xd7, 0x24, 0x91, 0xfb, 0x00, + 0x41, 0x12, 0x45, 0x34, 0x40, 0x01, 0x1b, 0x05, 0x0c, 0x8a, 0xca, 0x3b, 0x29, 0x23, 0x4f, 0xd0, + 0x00, 0x5d, 0xdd, 0x76, 0x3b, 0x52, 0x46, 0x07, 0x34, 0x50, 0xef, 0xc8, 0x04, 0xe5, 0x1e, 0xc2, + 0x57, 0x1f, 0xcf, 0xf5, 0x14, 0x01, 0x41, 0x76, 0x03, 0x60, 0xca, 0x93, 0x2c, 0xd5, 0xdc, 0xc1, + 0x66, 0x53, 0x21, 0x39, 0x52, 0x90, 0xfd, 0x08, 0x6e, 0x8a, 0x8b, 0x59, 0xc4, 0xe2, 0x13, 0x4f, + 0xfa, 0x7c, 0x4a, 0xe5, 0x78, 0xa8, 0x2b, 0x20, 0xa7, 0xbe, 0x43, 0xa2, 0x93, 0x02, 0xd9, 0xe1, + 0xd4, 0x97, 0xf4, 0x3b, 0x34, 0xad, 0x6f, 0x87, 0x0d, 0xe4, 0x36, 0x74, 0x12, 0x8f, 0x9e, 0x07, + 0x51, 0x5e, 0xa2, 0xed, 0x64, 0xf7, 0x3c, 0x88, 0x9c, 0x27, 0x30, 0xaa, 0xdc, 0x98, 0xc3, 0xfa, + 0x1a, 0xb4, 0x29, 0xe7, 0x49, 0x01, 0x42, 0x7a, 0xe3, 0xfc, 0x0e, 0xc8, 0xfb, 0x34, 0xfc, 0x21, + 0xcc, 0x73, 0x6e, 0xc3, 0xa8, 0xa2, 0x5a, 0xdb, 0xe1, 0xfc, 0xcb, 0x02, 0xf2, 0x12, 0xb1, 0xe4, + 0xfb, 0xb5, 0x71, 0x55, 0xdd, 0xaa, 0xc5, 0x68, 0xac, 0x0a, 0x7d, 0xe9, 0xe7, 0x0d, 0x70, 0xc0, + 0x84, 0xd6, 0xff, 0xd2, 0x97, 0x7e, 0xde, 0x88, 0x38, 0x0d, 0x32, 0xae, 0x7a, 0x22, 0x26, 0x21, + 0x36, 0x22, 0xb7, 0x20, 0x91, 0x8f, 0xe1, 0x0e, 0x9b, 0xc6, 0x09, 0xa7, 0x73, 0x31, 0x4f, 0xbb, + 0xaa, 0x83, 0xc2, 0x6b, 0x9a, 0x5b, 0x1e, 0xd8, 0x45, 0xcf, 0x3d, 0x81, 0x51, 0xe5, 0x19, 0x57, + 0xba, 0xf9, 0xcf, 0x16, 0x8c, 0x9f, 0xcb, 0x64, 0xc6, 0x02, 0x97, 0x2a, 0xe3, 0x2b, 0x4f, 0x7f, + 0x08, 0x43, 0x85, 0xe6, 0x8b, 0xcf, 0x1f, 0x24, 0x51, 0x38, 0xef, 0x96, 0x77, 0x41, 0x01, 0xba, + 0x67, 0x78, 0xa1, 0x9b, 0x44, 0x21, 0x66, 0xe2, 0x43, 0x50, 0xa8, 0x6b, 0x9c, 0xd7, 0x73, 0xc3, + 0x20, 0xa6, 0x67, 0x95, 0xf3, 0x4a, 0x08, 0xcf, 0x6b, 0xa8, 0xee, 0xc6, 0xf4, 0x4c, 0x9d, 0x77, + 0xee, 0xc1, 0xdd, 0x25, 0xb6, 0xe5, 0xe1, 0xfa, 0xb7, 0x05, 0xa3, 0xe7, 0x42, 0xb0, 0x69, 0xfc, + 0x5b, 0x84, 0x9d, 0xc2, 0xe8, 0x35, 0x68, 0x07, 0x49, 0x16, 0x4b, 0x34, 0xb6, 0xed, 0xea, 0xcd, + 0x42, 0x25, 0x36, 0x6a, 0x95, 0xb8, 0x50, 0xcb, 0xcd, 0x7a, 0x2d, 0x1b, 0xb5, 0xda, 0xaa, 0xd4, + 0xea, 0x03, 0xe8, 0xab, 0x20, 0x7b, 0x01, 0x8d, 0x25, 0xe5, 0x39, 0xce, 0x83, 0x22, 0xed, 0x20, + 0x45, 0x09, 0x98, 0xfd, 0x48, 0x43, 0x3d, 0xa4, 0xf3, 0x66, 0xf4, 0x1f, 0x0b, 0xd6, 0xaa, 0x4f, + 0xc9, 0x63, 0x76, 0x69, 0x5f, 0x52, 0x50, 0xc6, 0xa3, 0xfc, 0x1d, 0x6a, 0xa9, 0x40, 0x21, 0xcd, + 0x0e, 0x23, 0x16, 0x78, 0x8a, 0xa1, 0xed, 0xb7, 0x35, 0xe5, 0x3d, 0x8f, 0xe6, 0x5e, 0x69, 0x99, + 0x5e, 0x21, 0xd0, 0xf2, 0x33, 0x79, 0x5c, 0xf4, 0x26, 0xb5, 0x5e, 0xf0, 0x54, 0xe7, 0x3a, 0x4f, + 0x75, 0xeb, 0x9e, 0x2a, 0x33, 0xad, 0x67, 0x66, 0xda, 0xc7, 0x30, 0xd2, 0xc3, 0x6d, 0x35, 0x5c, + 0x1b, 0x00, 0x65, 0x1f, 0x11, 0x63, 0x4b, 0x83, 0x59, 0xd1, 0x48, 0x84, 0xf3, 0x4b, 0xb0, 0x5f, + 0x27, 0x5a, 0xaf, 0x20, 0xcf, 0xc0, 0x8e, 0x8a, 0x0d, 0x8a, 0xf6, 0xb7, 0xc9, 0xbc, 0xc6, 0x0b, + 0x39, 0x77, 0x2e, 0xe4, 0x7c, 0x06, 0xbd, 0x82, 0x5c, 0xf8, 0xcc, 0xba, 0xcc, 0x67, 0x8d, 0x05, + 0x9f, 0x39, 0xff, 0xb4, 0x60, 0xad, 0x6a, 0x72, 0x1e, 0x96, 0xf7, 0x30, 0x2c, 0xaf, 0xf0, 0x66, + 0x7e, 0x9a, 0xdb, 0xf2, 0xcc, 0xb4, 0xa5, 0x7e, 0xac, 0x34, 0x50, 0xbc, 0xf1, 0x53, 0x9d, 0xcb, + 0x83, 0xc8, 0x20, 0x4d, 0xde, 0xc1, 0x6a, 0x4d, 0x64, 0xc9, 0x64, 0xf7, 0x13, 0x73, 0xb2, 0xab, + 0x4c, 0xa7, 0xe5, 0x69, 0x73, 0xdc, 0xfb, 0x14, 0x3e, 0xd0, 0x70, 0xb0, 0x53, 0xc6, 0xb0, 0xf0, + 0x7d, 0x35, 0xd4, 0xd6, 0x62, 0xa8, 0x9d, 0x09, 0x8c, 0xeb, 0x47, 0xf3, 0xf2, 0x9b, 0xc2, 0xea, + 0x81, 0xf4, 0x25, 0x13, 0x92, 0x05, 0xe5, 0x27, 0xc6, 0x42, 0x6e, 0x58, 0xd7, 0x75, 0xc4, 0x7a, + 0x1d, 0xae, 0x40, 0x53, 0xca, 0x22, 0x7f, 0xd5, 0x52, 0x45, 0x81, 0x98, 0x37, 0xe5, 0x31, 0xf8, + 0x01, 0xae, 0x52, 0xf9, 0x20, 0x13, 0xe9, 0x47, 0x7a, 0xe2, 0x68, 0xe1, 0xc4, 0x61, 0x23, 0x05, + 0x47, 0x0e, 0xdd, 0x94, 0x43, 0xcd, 0x6d, 0xeb, 0x79, 0x44, 0x11, 0x90, 0xb9, 0x01, 0x80, 0xa5, + 0xaa, 0xab, 0xac, 0xa3, 0xcf, 0x2a, 0xca, 0x8e, 0x22, 0x38, 0xf7, 0x61, 0xfd, 0x73, 0x2a, 0xd5, + 0xec, 0xc4, 0x77, 0x92, 0xf8, 0x88, 0x4d, 0x33, 0xee, 0x1b, 0xa1, 0x70, 0xfe, 0x6b, 0xc1, 0xc6, + 0x25, 0x02, 0xf9, 0x83, 0xc7, 0xd0, 0x9d, 0xf9, 0x42, 0x52, 0x5e, 0x54, 0x49, 0xb1, 0x5d, 0x74, + 0x45, 0xe3, 0x3a, 0x57, 0x34, 0x6b, 0xae, 0xb8, 0x0d, 0x9d, 0x99, 0x7f, 0xee, 0xcd, 0x0e, 0xf3, + 0xe1, 0xa8, 0x3d, 0xf3, 0xcf, 0xdf, 0x1c, 0x22, 0xb2, 0x31, 0xee, 0x1d, 0x66, 0xc1, 0x09, 0x95, + 0xa2, 0x44, 0x36, 0xc6, 0x5f, 0x68, 0x8a, 0x7a, 0xb4, 0x12, 0xf8, 0x3a, 0xa3, 0x19, 0x15, 0x39, + 0x56, 0xa8, 0xe6, 0xf8, 0x1b, 0x24, 0xe0, 0x30, 0x85, 0x93, 0x25, 0xa2, 0x44, 0xcf, 0xcd, 0x77, + 0x0e, 0x87, 0x3b, 0xea, 0xfb, 0x8e, 0xc6, 0x7b, 0x09, 0xc7, 0x6f, 0x88, 0x32, 0x81, 0x1e, 0x40, + 0x3f, 0x88, 0x98, 0x82, 0x4a, 0xe3, 0xc3, 0x0d, 0x34, 0x09, 0x5b, 0x4a, 0xa5, 0x1b, 0x37, 0x16, + 0xbb, 0xf1, 0x5d, 0xe8, 0x09, 0x16, 0x07, 0xd4, 0x8b, 0xf5, 0xd7, 0x41, 0xd3, 0xed, 0xe2, 0x7e, + 0x5f, 0x38, 0x7f, 0xb2, 0xe0, 0x36, 0x7e, 0xf6, 0xd4, 0xbe, 0x59, 0xae, 0x6e, 0xf0, 0xbf, 0x06, + 0x42, 0x4f, 0xd1, 0x20, 0xe3, 0x4c, 0x5e, 0x7a, 0xf7, 0x8c, 0x01, 0x63, 0x51, 0xad, 0xbb, 0x4a, + 0x17, 0x49, 0x8e, 0xaf, 0xd0, 0x68, 0xaa, 0xeb, 0x7a, 0x04, 0x6d, 0x29, 0x3c, 0xc4, 0x31, 0x65, + 0x67, 0x4b, 0x8a, 0x7d, 0x41, 0x9e, 0x02, 0x49, 0x7d, 0x2e, 0x99, 0x92, 0x56, 0xc3, 0xb3, 0x77, + 0xec, 0x8b, 0x63, 0xbc, 0xac, 0xed, 0xae, 0x94, 0x9c, 0x2f, 0xe8, 0xc5, 0xaf, 0x7c, 0x71, 0xac, + 0xd0, 0x1b, 0xa7, 0x8b, 0x26, 0x0e, 0xf1, 0xb8, 0xde, 0xfe, 0x6b, 0x0f, 0x06, 0x07, 0xd4, 0x3f, + 0xa3, 0x34, 0xc4, 0x5c, 0x22, 0xd3, 0x02, 0xc3, 0xaa, 0xff, 0x14, 0xc8, 0xa3, 0x45, 0xb0, 0x5a, + 0xfa, 0x13, 0x63, 0xf2, 0xd1, 0x75, 0x62, 0x39, 0x1c, 0xdc, 0x20, 0xfb, 0xd0, 0x37, 0x3e, 0xda, + 0xc9, 0xba, 0x71, 0xb0, 0xf6, 0x2f, 0x62, 0xb2, 0x71, 0x09, 0xb7, 0xd0, 0xf6, 0xcc, 0x22, 0xaf, + 0xa1, 0x6f, 0x4c, 0x8b, 0xa6, 0xbe, 0xfa, 0xd8, 0x6a, 0xea, 0x5b, 0x32, 0x62, 0x3a, 0x37, 0x94, + 0x36, 0x63, 0xe6, 0x33, 0xb5, 0xd5, 0xa7, 0x4c, 0x53, 0xdb, 0xb2, 0x41, 0x11, 0xb5, 0x19, 0x23, + 0x96, 0xa9, 0xad, 0x3e, 0x40, 0x9a, 0xda, 0x96, 0xcc, 0x65, 0xce, 0x0d, 0xf2, 0x07, 0x58, 0xad, + 0x8d, 0x39, 0xc4, 0x99, 0x9f, 0xba, 0x6c, 0x3e, 0x9b, 0x3c, 0xbc, 0x52, 0xa6, 0xd4, 0xff, 0x25, + 0x0c, 0xcc, 0xe9, 0x82, 0x18, 0x06, 0x2d, 0x19, 0xa0, 0x26, 0xf7, 0x2f, 0x63, 0x9b, 0x0a, 0xcd, + 0x06, 0x67, 0x2a, 0x5c, 0xd2, 0xe2, 0x4d, 0x85, 0xcb, 0xfa, 0xa2, 0x73, 0x83, 0xfc, 0x1e, 0x56, + 0x16, 0x1b, 0x0d, 0xf9, 0x70, 0xd1, 0x6d, 0xb5, 0xfe, 0x35, 0x71, 0xae, 0x12, 0x29, 0x95, 0xbf, + 0x02, 0x98, 0xf7, 0x0f, 0x62, 0xd4, 0x6c, 0xad, 0x7f, 0x4d, 0xd6, 0x97, 0x33, 0x4b, 0x55, 0x7f, + 0x84, 0xdb, 0x4b, 0x41, 0x9a, 0x18, 0x65, 0x72, 0x15, 0xcc, 0x4f, 0x7e, 0x7c, 0xad, 0x5c, 0x79, + 0xd7, 0x57, 0x70, 0x6b, 0x01, 0x24, 0xc9, 0x66, 0xb5, 0x6a, 0xea, 0xf8, 0x39, 0x79, 0x60, 0xfe, + 0x79, 0x5a, 0x02, 0x76, 0xaa, 0xb2, 0x5e, 0xdc, 0x87, 0x15, 0xa1, 0x21, 0xe2, 0x48, 0x6c, 0x69, + 0x6c, 0x7d, 0x01, 0x68, 0xcb, 0x5b, 0x9e, 0xc8, 0xe4, 0xb0, 0x83, 0x3f, 0x3a, 0x7f, 0xf6, 0xff, + 0x00, 0x00, 0x00, 0xff, 0xff, 0x2f, 0x77, 0xbb, 0xcb, 0xf7, 0x14, 0x00, 0x00, } diff --git a/weed/queue/log_buffer.go b/weed/queue/log_buffer.go index d6ccdf2a6..6ed2a719c 100644 --- a/weed/queue/log_buffer.go +++ b/weed/queue/log_buffer.go @@ -6,28 +6,32 @@ import ( "github.com/golang/protobuf/proto" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" ) type LogBuffer struct { buf []byte + idx []int pos int startTime time.Time stopTime time.Time sizeBuf []byte flushInterval time.Duration flushFn func(startTime, stopTime time.Time, buf []byte) + notifyFn func() isStopping bool - sync.Mutex + sync.RWMutex } -func NewLogBuffer(flushInterval time.Duration, flushFn func(startTime, stopTime time.Time, buf []byte)) *LogBuffer { +func NewLogBuffer(flushInterval time.Duration, flushFn func(startTime, stopTime time.Time, buf []byte), notifyFn func()) *LogBuffer { lb := &LogBuffer{ buf: make([]byte, 4*0124*1024), sizeBuf: make([]byte, 4), flushInterval: flushInterval, flushFn: flushFn, + notifyFn: notifyFn, } go lb.loopFlush() return lb @@ -46,7 +50,12 @@ func (m *LogBuffer) AddToBuffer(ts time.Time, key, data []byte) { size := len(logEntryData) m.Lock() - defer m.Unlock() + defer func() { + m.Unlock() + if m.notifyFn != nil { + m.notifyFn() + } + }() if m.pos == 0 { m.startTime = ts @@ -55,12 +64,15 @@ func (m *LogBuffer) AddToBuffer(ts time.Time, key, data []byte) { if m.startTime.Add(m.flushInterval).Before(ts) || len(m.buf)-m.pos < size+4 { m.flush() m.startTime = ts + if len(m.buf) < size+4 { + m.buf = make([]byte, 2*size+4) + } } m.stopTime = ts + m.idx = append(m.idx, m.pos) util.Uint32toBytes(m.sizeBuf, uint32(size)) copy(m.buf[m.pos:m.pos+4], m.sizeBuf) - copy(m.buf[m.pos+4:m.pos+4+size], logEntryData) m.pos += size + 4 } @@ -88,5 +100,67 @@ func (m *LogBuffer) flush() { if m.flushFn != nil && m.pos > 0 { m.flushFn(m.startTime, m.stopTime, m.buf[:m.pos]) m.pos = 0 + m.idx = m.idx[:0] + } +} + +func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (ts time.Time, bufferCopy []byte) { + m.RLock() + defer m.RUnlock() + + // fmt.Printf("read from buffer: %v\n", lastReadTime) + + if lastReadTime.Equal(m.stopTime) { + return lastReadTime, nil + } + if lastReadTime.After(m.stopTime) { + // glog.Fatalf("unexpected last read time %v, older than latest %v", lastReadTime, m.stopTime) + return lastReadTime, nil } + if lastReadTime.Before(m.startTime) { + return m.stopTime, copiedBytes(m.buf[:m.pos]) + } + + lastTs := lastReadTime.UnixNano() + l, h := 0, len(m.idx)-1 + + // fmt.Printf("l=%d, h=%d\n", l, h) + for { + mid := (l + h) / 2 + pos := m.idx[mid] + t := readTs(m.buf, m.idx[mid]) + if t <= lastTs { + l = mid + 1 + } else if lastTs < t { + var prevT int64 + if mid > 0 { + prevT = readTs(m.buf, m.idx[mid-1]) + } + if prevT <= lastTs { + return time.Unix(0, t), copiedBytes(m.buf[pos:m.pos]) + } + h = mid - 1 + } + // fmt.Printf("l=%d, h=%d\n", l, h) + } + +} +func copiedBytes(buf []byte) (copied []byte) { + copied = make([]byte, len(buf)) + copy(copied, buf) + return +} + +func readTs(buf []byte, pos int) int64 { + + size := util.BytesToUint32(buf[pos : pos+4]) + entryData := buf[pos+4 : pos+4+int(size)] + logEntry := &filer_pb.LogEntry{} + + err := proto.Unmarshal(entryData, logEntry) + if err != nil { + glog.Fatalf("unexpected unmarshal filer_pb.LogEntry: %v", err) + } + return logEntry.TsNs + } diff --git a/weed/server/filer_grpc_server_listen.go b/weed/server/filer_grpc_server_listen.go index 37d643ce2..81b0a9254 100644 --- a/weed/server/filer_grpc_server_listen.go +++ b/weed/server/filer_grpc_server_listen.go @@ -1,6 +1,9 @@ package weed_server import ( + "strings" + "time" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) @@ -9,62 +12,56 @@ func (fs *FilerServer) ListenForEvents(req *filer_pb.ListenForEventsRequest, str peerAddress := findClientAddress(stream.Context(), 0) - clientName, messageChan := fs.addClient(req.ClientName, peerAddress) + clientName := fs.addClient(req.ClientName, peerAddress) - defer fs.deleteClient(clientName, messageChan) + defer fs.deleteClient(clientName) - // ts := time.Unix(req.SinceSec, 0) + lastReadTime := time.Now() + if req.SinceNs > 0 { + lastReadTime = time.Unix(0, req.SinceNs) + } + var readErr error + for { - // iterate through old messages - /* - for _, message := range ms.Topo.ToVolumeLocations() { + lastReadTime, readErr = fs.filer.ReadLogBuffer(lastReadTime, func(fullpath string, eventNotification *filer_pb.EventNotification) error { + if strings.HasPrefix(fullpath, "/.meta") { + return nil + } + if !strings.HasPrefix(fullpath, req.Directory) { + return nil + } + message := &filer_pb.FullEventNotification{ + Directory: fullpath, + EventNotification: eventNotification, + } if err := stream.Send(message); err != nil { return err } + return nil + }) + if readErr != nil { + glog.V(0).Infof("=> client %v: %+v", clientName, readErr) + return readErr } - */ - - // need to add a buffer here to avoid slow clients - // also needs to support millions of clients - for message := range messageChan { - if err := stream.Send(message); err != nil { - glog.V(0).Infof("=> client %v: %+v", clientName, message) - return err - } + fs.listenersLock.Lock() + fs.listenersCond.Wait() + fs.listenersLock.Unlock() } return nil } -func (fs *FilerServer) addClient(clientType string, clientAddress string) (clientName string, messageChan chan *filer_pb.FullEventNotification) { +func (fs *FilerServer) addClient(clientType string, clientAddress string) (clientName string) { clientName = clientType + "@" + clientAddress glog.V(0).Infof("+ listener %v", clientName) - - messageChan = make(chan *filer_pb.FullEventNotification, 10) - - fs.clientChansLock.Lock() - fs.clientChans[clientName] = messageChan - fs.clientChansLock.Unlock() return } -func (fs *FilerServer) deleteClient(clientName string, messageChan chan *filer_pb.FullEventNotification) { +func (fs *FilerServer) deleteClient(clientName string) { glog.V(0).Infof("- listener %v", clientName) - close(messageChan) - fs.clientChansLock.Lock() - delete(fs.clientChans, clientName) - fs.clientChansLock.Unlock() } -func (fs *FilerServer) sendMessageToClients(dir string, eventNotification *filer_pb.EventNotification) { - message := &filer_pb.FullEventNotification{ - Directory: dir, - EventNotification: eventNotification, - } - fs.clientChansLock.RLock() - for _, ch := range fs.clientChans { - ch <- message - } - fs.clientChansLock.RUnlock() +func (fs *FilerServer) notifyMetaListeners() { + fs.listenersCond.Broadcast() } diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index ba7b0ed03..22bbd2a03 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -12,7 +12,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/util" @@ -57,8 +56,8 @@ type FilerServer struct { grpcDialOption grpc.DialOption // notifying clients - clientChansLock sync.RWMutex - clientChans map[string]chan *filer_pb.FullEventNotification + listenersLock sync.Mutex + listenersCond *sync.Cond } func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) (fs *FilerServer, err error) { @@ -67,12 +66,13 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) option: option, grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"), } + fs.listenersCond = sync.NewCond(&fs.listenersLock) if len(option.Masters) == 0 { glog.Fatal("master list is required!") } - fs.filer = filer2.NewFiler(option.Masters, fs.grpcDialOption, option.Port+10000) + fs.filer = filer2.NewFiler(option.Masters, fs.grpcDialOption, option.Port+10000, fs.notifyMetaListeners) fs.filer.Cipher = option.Cipher maybeStartMetrics(fs, option)