You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

211 lines
4.5 KiB

12 years ago
10 years ago
  1. package storage
  2. import (
  3. "strconv"
  4. "sync"
  5. )
  6. type NeedleValue struct {
  7. Key Key
  8. Offset uint32 `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
  9. Size uint32 `comment:"Size of the data portion"`
  10. }
  11. const (
  12. batch = 100000
  13. )
  14. type Key uint64
  15. func (k Key) String() string {
  16. return strconv.FormatUint(uint64(k), 10)
  17. }
  18. type CompactSection struct {
  19. sync.RWMutex
  20. values []NeedleValue
  21. overflow map[Key]NeedleValue
  22. start Key
  23. end Key
  24. counter int
  25. }
  26. func NewCompactSection(start Key) *CompactSection {
  27. return &CompactSection{
  28. values: make([]NeedleValue, batch),
  29. overflow: make(map[Key]NeedleValue),
  30. start: start,
  31. }
  32. }
  33. //return old entry size
  34. func (cs *CompactSection) Set(key Key, offset uint32, size uint32) uint32 {
  35. ret := uint32(0)
  36. if key > cs.end {
  37. cs.end = key
  38. }
  39. cs.Lock()
  40. if i := cs.binarySearchValues(key); i >= 0 {
  41. ret = cs.values[i].Size
  42. //println("key", key, "old size", ret)
  43. cs.values[i].Offset, cs.values[i].Size = offset, size
  44. } else {
  45. needOverflow := cs.counter >= batch
  46. needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > key
  47. if needOverflow {
  48. //println("start", cs.start, "counter", cs.counter, "key", key)
  49. if oldValue, found := cs.overflow[key]; found {
  50. ret = oldValue.Size
  51. }
  52. cs.overflow[key] = NeedleValue{Key: key, Offset: offset, Size: size}
  53. } else {
  54. p := &cs.values[cs.counter]
  55. p.Key, p.Offset, p.Size = key, offset, size
  56. //println("added index", cs.counter, "key", key, cs.values[cs.counter].Key)
  57. cs.counter++
  58. }
  59. }
  60. cs.Unlock()
  61. return ret
  62. }
  63. //return old entry size
  64. func (cs *CompactSection) Delete(key Key) uint32 {
  65. cs.Lock()
  66. ret := uint32(0)
  67. if i := cs.binarySearchValues(key); i >= 0 {
  68. if cs.values[i].Size > 0 {
  69. ret = cs.values[i].Size
  70. cs.values[i].Size = 0
  71. }
  72. }
  73. if v, found := cs.overflow[key]; found {
  74. delete(cs.overflow, key)
  75. ret = v.Size
  76. }
  77. cs.Unlock()
  78. return ret
  79. }
  80. func (cs *CompactSection) Get(key Key) (*NeedleValue, bool) {
  81. cs.RLock()
  82. if v, ok := cs.overflow[key]; ok {
  83. cs.RUnlock()
  84. return &v, true
  85. }
  86. if i := cs.binarySearchValues(key); i >= 0 {
  87. cs.RUnlock()
  88. return &cs.values[i], true
  89. }
  90. cs.RUnlock()
  91. return nil, false
  92. }
  93. func (cs *CompactSection) binarySearchValues(key Key) int {
  94. l, h := 0, cs.counter-1
  95. if h >= 0 && cs.values[h].Key < key {
  96. return -2
  97. }
  98. //println("looking for key", key)
  99. for l <= h {
  100. m := (l + h) / 2
  101. //println("mid", m, "key", cs.values[m].Key, cs.values[m].Offset, cs.values[m].Size)
  102. if cs.values[m].Key < key {
  103. l = m + 1
  104. } else if key < cs.values[m].Key {
  105. h = m - 1
  106. } else {
  107. //println("found", m)
  108. return m
  109. }
  110. }
  111. return -1
  112. }
  113. //This map assumes mostly inserting increasing keys
  114. type CompactMap struct {
  115. list []*CompactSection
  116. }
  117. func NewCompactMap() CompactMap {
  118. return CompactMap{}
  119. }
  120. func (cm *CompactMap) Set(key Key, offset uint32, size uint32) uint32 {
  121. x := cm.binarySearchCompactSection(key)
  122. if x < 0 {
  123. //println(x, "creating", len(cm.list), "section, starting", key)
  124. cm.list = append(cm.list, NewCompactSection(key))
  125. x = len(cm.list) - 1
  126. //keep compact section sorted by start
  127. for x > 0 {
  128. if cm.list[x-1].start > cm.list[x].start {
  129. cm.list[x-1], cm.list[x] = cm.list[x], cm.list[x-1]
  130. x = x - 1
  131. } else {
  132. break
  133. }
  134. }
  135. }
  136. return cm.list[x].Set(key, offset, size)
  137. }
  138. func (cm *CompactMap) Delete(key Key) uint32 {
  139. x := cm.binarySearchCompactSection(key)
  140. if x < 0 {
  141. return uint32(0)
  142. }
  143. return cm.list[x].Delete(key)
  144. }
  145. func (cm *CompactMap) Get(key Key) (*NeedleValue, bool) {
  146. x := cm.binarySearchCompactSection(key)
  147. if x < 0 {
  148. return nil, false
  149. }
  150. return cm.list[x].Get(key)
  151. }
  152. func (cm *CompactMap) binarySearchCompactSection(key Key) int {
  153. l, h := 0, len(cm.list)-1
  154. if h < 0 {
  155. return -5
  156. }
  157. if cm.list[h].start <= key {
  158. if cm.list[h].counter < batch || key <= cm.list[h].end {
  159. return h
  160. }
  161. return -4
  162. }
  163. for l <= h {
  164. m := (l + h) / 2
  165. if key < cm.list[m].start {
  166. h = m - 1
  167. } else { // cm.list[m].start <= key
  168. if cm.list[m+1].start <= key {
  169. l = m + 1
  170. } else {
  171. return m
  172. }
  173. }
  174. }
  175. return -3
  176. }
  177. // Visit visits all entries or stop if any error when visiting
  178. func (cm *CompactMap) Visit(visit func(NeedleValue) error) error {
  179. for _, cs := range cm.list {
  180. cs.RLock()
  181. for _, v := range cs.overflow {
  182. if err := visit(v); err != nil {
  183. cs.RUnlock()
  184. return err
  185. }
  186. }
  187. for _, v := range cs.values {
  188. if _, found := cs.overflow[v.Key]; !found {
  189. if err := visit(v); err != nil {
  190. cs.RUnlock()
  191. return err
  192. }
  193. }
  194. }
  195. cs.RUnlock()
  196. }
  197. return nil
  198. }