You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

93 lines
2.0 KiB

6 years ago
6 years ago
7 years ago
7 years ago
  1. package needle
  2. import (
  3. "fmt"
  4. "log"
  5. "os"
  6. "runtime"
  7. "testing"
  8. "time"
  9. . "github.com/chrislusf/seaweedfs/weed/storage/types"
  10. "github.com/chrislusf/seaweedfs/weed/util"
  11. )
  12. /*
  13. To see the memory usage:
  14. go test -run TestMemoryUsage
  15. The TotalAlloc section shows the memory increase for each iteration.
  16. go test -run TestMemoryUsage -memprofile=mem.out
  17. go tool pprof --alloc_space needle.test mem.out
  18. */
  19. func TestMemoryUsage(t *testing.T) {
  20. var maps []*CompactMap
  21. totalRowCount := uint64(0)
  22. startTime := time.Now()
  23. for i := 0; i < 10; i++ {
  24. indexFile, ie := os.OpenFile("../../../test/sample.idx", os.O_RDWR|os.O_RDONLY, 0644)
  25. if ie != nil {
  26. log.Fatalln(ie)
  27. }
  28. m, rowCount := loadNewNeedleMap(indexFile)
  29. maps = append(maps, m)
  30. totalRowCount += rowCount
  31. indexFile.Close()
  32. PrintMemUsage(totalRowCount)
  33. now := time.Now()
  34. fmt.Printf("\tTaken = %v\n", now.Sub(startTime))
  35. startTime = now
  36. }
  37. }
  38. func loadNewNeedleMap(file *os.File) (*CompactMap, uint64) {
  39. m := NewCompactMap()
  40. bytes := make([]byte, NeedleEntrySize)
  41. rowCount := uint64(0)
  42. count, e := file.Read(bytes)
  43. for count > 0 && e == nil {
  44. for i := 0; i < count; i += NeedleEntrySize {
  45. rowCount++
  46. key := BytesToNeedleId(bytes[i : i+NeedleIdSize])
  47. offset := BytesToOffset(bytes[i+NeedleIdSize : i+NeedleIdSize+OffsetSize])
  48. size := util.BytesToUint32(bytes[i+NeedleIdSize+OffsetSize : i+NeedleIdSize+OffsetSize+SizeSize])
  49. if offset > 0 {
  50. m.Set(NeedleId(key), offset, size)
  51. } else {
  52. m.Delete(key)
  53. }
  54. }
  55. count, e = file.Read(bytes)
  56. }
  57. return m, rowCount
  58. }
  59. func PrintMemUsage(totalRowCount uint64) {
  60. runtime.GC()
  61. var m runtime.MemStats
  62. runtime.ReadMemStats(&m)
  63. // For info on each, see: https://golang.org/pkg/runtime/#MemStats
  64. fmt.Printf("Each %.2f Bytes", float64(m.TotalAlloc)/float64(totalRowCount))
  65. fmt.Printf("\tAlloc = %v MiB", bToMb(m.Alloc))
  66. fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
  67. fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
  68. fmt.Printf("\tNumGC = %v", m.NumGC)
  69. }
  70. func bToMb(b uint64) uint64 {
  71. return b / 1024 / 1024
  72. }