You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

209 lines
5.8 KiB

5 years ago
5 years ago
5 years ago
5 years ago
  1. package filesys
  2. import (
  3. "bytes"
  4. "io"
  5. )
  6. type IntervalNode struct {
  7. Data []byte
  8. Offset int64
  9. Size int64
  10. Next *IntervalNode
  11. }
  12. type IntervalLinkedList struct {
  13. Head *IntervalNode
  14. Tail *IntervalNode
  15. }
  16. type ContinuousIntervals struct {
  17. lists []*IntervalLinkedList
  18. }
  19. func (list *IntervalLinkedList) Offset() int64 {
  20. return list.Head.Offset
  21. }
  22. func (list *IntervalLinkedList) Size() int64 {
  23. return list.Tail.Offset + list.Tail.Size - list.Head.Offset
  24. }
  25. func (list *IntervalLinkedList) addNodeToTail(node *IntervalNode) {
  26. // glog.V(4).Infof("add to tail [%d,%d) + [%d,%d) => [%d,%d)", list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, node.Offset+node.Size, list.Head.Offset, node.Offset+node.Size)
  27. list.Tail.Next = node
  28. list.Tail = node
  29. }
  30. func (list *IntervalLinkedList) addNodeToHead(node *IntervalNode) {
  31. // glog.V(4).Infof("add to head [%d,%d) + [%d,%d) => [%d,%d)", node.Offset, node.Offset+node.Size, list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, list.Tail.Offset+list.Tail.Size)
  32. node.Next = list.Head
  33. list.Head = node
  34. }
  35. func (list *IntervalLinkedList) ReadData(buf []byte, start, stop int64) {
  36. t := list.Head
  37. for {
  38. nodeStart, nodeStop := max(start, t.Offset), min(stop, t.Offset+t.Size)
  39. if nodeStart < nodeStop {
  40. // glog.V(0).Infof("copying start=%d stop=%d t=[%d,%d) t.data=%d => bufSize=%d nodeStart=%d, nodeStop=%d", start, stop, t.Offset, t.Offset+t.Size, len(t.Data), len(buf), nodeStart, nodeStop)
  41. copy(buf[nodeStart-start:], t.Data[nodeStart-t.Offset:nodeStop-t.Offset])
  42. }
  43. if t.Next == nil {
  44. break
  45. }
  46. t = t.Next
  47. }
  48. }
  49. func (c *ContinuousIntervals) TotalSize() (total int64) {
  50. for _, list := range c.lists {
  51. total += list.Size()
  52. }
  53. return
  54. }
  55. func subList(list *IntervalLinkedList, start, stop int64) *IntervalLinkedList {
  56. var nodes []*IntervalNode
  57. for t := list.Head; t != nil; t = t.Next {
  58. nodeStart, nodeStop := max(start, t.Offset), min(stop, t.Offset+t.Size)
  59. if nodeStart >= nodeStop {
  60. // skip non overlapping IntervalNode
  61. continue
  62. }
  63. nodes = append(nodes, &IntervalNode{
  64. Data: t.Data[nodeStart-t.Offset : nodeStop-t.Offset],
  65. Offset: nodeStart,
  66. Size: nodeStop - nodeStart,
  67. Next: nil,
  68. })
  69. }
  70. for i := 1; i < len(nodes); i++ {
  71. nodes[i-1].Next = nodes[i]
  72. }
  73. return &IntervalLinkedList{
  74. Head: nodes[0],
  75. Tail: nodes[len(nodes)-1],
  76. }
  77. }
  78. func (c *ContinuousIntervals) AddInterval(data []byte, offset int64) {
  79. interval := &IntervalNode{Data: data, Offset: offset, Size: int64(len(data))}
  80. var newLists []*IntervalLinkedList
  81. for _, list := range c.lists {
  82. // if list is to the left of new interval, add to the new list
  83. if list.Tail.Offset+list.Tail.Size <= interval.Offset {
  84. newLists = append(newLists, list)
  85. }
  86. // if list is to the right of new interval, add to the new list
  87. if interval.Offset+interval.Size <= list.Head.Offset {
  88. newLists = append(newLists, list)
  89. }
  90. // if new interval overwrite the right part of the list
  91. if list.Head.Offset < interval.Offset && interval.Offset < list.Tail.Offset+list.Tail.Size {
  92. // create a new list of the left part of existing list
  93. newLists = append(newLists, subList(list, list.Offset(), interval.Offset))
  94. }
  95. // if new interval overwrite the left part of the list
  96. if list.Head.Offset < interval.Offset+interval.Size && interval.Offset+interval.Size < list.Tail.Offset+list.Tail.Size {
  97. // create a new list of the right part of existing list
  98. newLists = append(newLists, subList(list, interval.Offset+interval.Size, list.Tail.Offset+list.Tail.Size))
  99. }
  100. // skip anything that is fully overwritten by the new interval
  101. }
  102. c.lists = newLists
  103. // add the new interval to the lists, connecting neighbor lists
  104. var prevList, nextList *IntervalLinkedList
  105. for _, list := range c.lists {
  106. if list.Head.Offset == interval.Offset+interval.Size {
  107. nextList = list
  108. break
  109. }
  110. }
  111. for _, list := range c.lists {
  112. if list.Head.Offset+list.Size() == offset {
  113. list.addNodeToTail(interval)
  114. prevList = list
  115. break
  116. }
  117. }
  118. if prevList != nil && nextList != nil {
  119. // glog.V(4).Infof("connecting [%d,%d) + [%d,%d) => [%d,%d)", prevList.Head.Offset, prevList.Tail.Offset+prevList.Tail.Size, nextList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size, prevList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size)
  120. prevList.Tail.Next = nextList.Head
  121. prevList.Tail = nextList.Tail
  122. c.removeList(nextList)
  123. } else if nextList != nil {
  124. // add to head was not done when checking
  125. nextList.addNodeToHead(interval)
  126. }
  127. if prevList == nil && nextList == nil {
  128. c.lists = append(c.lists, &IntervalLinkedList{
  129. Head: interval,
  130. Tail: interval,
  131. })
  132. }
  133. return
  134. }
  135. func (c *ContinuousIntervals) RemoveLargestIntervalLinkedList() *IntervalLinkedList {
  136. var maxSize int64
  137. maxIndex := -1
  138. for k, list := range c.lists {
  139. if maxSize <= list.Size() {
  140. maxSize = list.Size()
  141. maxIndex = k
  142. }
  143. }
  144. if maxSize <= 0 {
  145. return nil
  146. }
  147. t := c.lists[maxIndex]
  148. c.lists = append(c.lists[0:maxIndex], c.lists[maxIndex+1:]...)
  149. return t
  150. }
  151. func (c *ContinuousIntervals) removeList(target *IntervalLinkedList) {
  152. index := -1
  153. for k, list := range c.lists {
  154. if list.Offset() == target.Offset() {
  155. index = k
  156. }
  157. }
  158. if index < 0 {
  159. return
  160. }
  161. c.lists = append(c.lists[0:index], c.lists[index+1:]...)
  162. }
  163. func (c *ContinuousIntervals) ReadData(data []byte, startOffset int64) (maxStop int64) {
  164. for _, list := range c.lists {
  165. start := max(startOffset, list.Offset())
  166. stop := min(startOffset+int64(len(data)), list.Offset()+list.Size())
  167. if start < stop {
  168. list.ReadData(data[start-startOffset:], start, stop)
  169. maxStop = max(maxStop, stop)
  170. }
  171. }
  172. return
  173. }
  174. func (l *IntervalLinkedList) ToReader() io.Reader {
  175. var readers []io.Reader
  176. t := l.Head
  177. readers = append(readers, bytes.NewReader(t.Data))
  178. for t.Next != nil {
  179. t = t.Next
  180. readers = append(readers, bytes.NewReader(t.Data))
  181. }
  182. return io.MultiReader(readers...)
  183. }