You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

169 lines
5.0 KiB

6 years ago
5 years ago
5 years ago
5 years ago
5 years ago
6 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. package filer2
  2. import (
  3. "context"
  4. "fmt"
  5. "io"
  6. "strings"
  7. "time"
  8. "github.com/golang/protobuf/proto"
  9. "github.com/chrislusf/seaweedfs/weed/glog"
  10. "github.com/chrislusf/seaweedfs/weed/notification"
  11. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  12. "github.com/chrislusf/seaweedfs/weed/util"
  13. )
  14. func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) {
  15. var fullpath string
  16. if oldEntry != nil {
  17. fullpath = string(oldEntry.FullPath)
  18. } else if newEntry != nil {
  19. fullpath = string(newEntry.FullPath)
  20. } else {
  21. return
  22. }
  23. // println("fullpath:", fullpath)
  24. if strings.HasPrefix(fullpath, SystemLogDir) {
  25. return
  26. }
  27. newParentPath := ""
  28. if newEntry != nil {
  29. newParentPath, _ = newEntry.FullPath.DirAndName()
  30. }
  31. eventNotification := &filer_pb.EventNotification{
  32. OldEntry: oldEntry.ToProtoEntry(),
  33. NewEntry: newEntry.ToProtoEntry(),
  34. DeleteChunks: deleteChunks,
  35. NewParentPath: newParentPath,
  36. IsFromOtherCluster: isFromOtherCluster,
  37. Signatures: append(signatures, f.Signature),
  38. }
  39. if notification.Queue != nil {
  40. glog.V(3).Infof("notifying entry update %v", fullpath)
  41. notification.Queue.SendMessage(fullpath, eventNotification)
  42. }
  43. f.logMetaEvent(ctx, fullpath, eventNotification)
  44. }
  45. func (f *Filer) logMetaEvent(ctx context.Context, fullpath string, eventNotification *filer_pb.EventNotification) {
  46. dir, _ := util.FullPath(fullpath).DirAndName()
  47. event := &filer_pb.SubscribeMetadataResponse{
  48. Directory: dir,
  49. EventNotification: eventNotification,
  50. TsNs: time.Now().UnixNano(),
  51. }
  52. data, err := proto.Marshal(event)
  53. if err != nil {
  54. glog.Errorf("failed to marshal filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
  55. return
  56. }
  57. f.LocalMetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)
  58. }
  59. func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) {
  60. startTime, stopTime = startTime.UTC(), stopTime.UTC()
  61. targetFile := fmt.Sprintf("%s/%04d-%02d-%02d/%02d-%02d.segment", SystemLogDir,
  62. startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(),
  63. // startTime.Second(), startTime.Nanosecond(),
  64. )
  65. for {
  66. if err := f.appendToFile(targetFile, buf); err != nil {
  67. glog.V(1).Infof("log write failed %s: %v", targetFile, err)
  68. time.Sleep(737 * time.Millisecond)
  69. } else {
  70. break
  71. }
  72. }
  73. }
  74. func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error) {
  75. startTime = startTime.UTC()
  76. startDate := fmt.Sprintf("%04d-%02d-%02d", startTime.Year(), startTime.Month(), startTime.Day())
  77. startHourMinute := fmt.Sprintf("%02d-%02d.segment", startTime.Hour(), startTime.Minute())
  78. sizeBuf := make([]byte, 4)
  79. startTsNs := startTime.UnixNano()
  80. dayEntries, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "")
  81. if listDayErr != nil {
  82. return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr)
  83. }
  84. for _, dayEntry := range dayEntries {
  85. // println("checking day", dayEntry.FullPath)
  86. hourMinuteEntries, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "")
  87. if listHourMinuteErr != nil {
  88. return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr)
  89. }
  90. for _, hourMinuteEntry := range hourMinuteEntries {
  91. // println("checking hh-mm", hourMinuteEntry.FullPath)
  92. if dayEntry.Name() == startDate {
  93. if strings.Compare(hourMinuteEntry.Name(), startHourMinute) < 0 {
  94. continue
  95. }
  96. }
  97. // println("processing", hourMinuteEntry.FullPath)
  98. chunkedFileReader := NewChunkStreamReaderFromFiler(f.MasterClient, hourMinuteEntry.Chunks)
  99. if lastTsNs, err = ReadEachLogEntry(chunkedFileReader, sizeBuf, startTsNs, eachLogEntryFn); err != nil {
  100. chunkedFileReader.Close()
  101. if err == io.EOF {
  102. continue
  103. }
  104. return lastTsNs, fmt.Errorf("reading %s: %v", hourMinuteEntry.FullPath, err)
  105. }
  106. chunkedFileReader.Close()
  107. }
  108. }
  109. return lastTsNs, nil
  110. }
  111. func ReadEachLogEntry(r io.Reader, sizeBuf []byte, ns int64, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error) {
  112. for {
  113. n, err := r.Read(sizeBuf)
  114. if err != nil {
  115. return lastTsNs, err
  116. }
  117. if n != 4 {
  118. return lastTsNs, fmt.Errorf("size %d bytes, expected 4 bytes", n)
  119. }
  120. size := util.BytesToUint32(sizeBuf)
  121. // println("entry size", size)
  122. entryData := make([]byte, size)
  123. n, err = r.Read(entryData)
  124. if err != nil {
  125. return lastTsNs, err
  126. }
  127. if n != int(size) {
  128. return lastTsNs, fmt.Errorf("entry data %d bytes, expected %d bytes", n, size)
  129. }
  130. logEntry := &filer_pb.LogEntry{}
  131. if err = proto.Unmarshal(entryData, logEntry); err != nil {
  132. return lastTsNs, err
  133. }
  134. if logEntry.TsNs <= ns {
  135. return lastTsNs, nil
  136. }
  137. // println("each log: ", logEntry.TsNs)
  138. if err := eachLogEntryFn(logEntry); err != nil {
  139. return lastTsNs, err
  140. } else {
  141. lastTsNs = logEntry.TsNs
  142. }
  143. }
  144. }