You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

468 lines
13 KiB

  1. package main
  2. import (
  3. "bufio"
  4. "code.google.com/p/weed-fs/go/glog"
  5. "code.google.com/p/weed-fs/go/operation"
  6. "code.google.com/p/weed-fs/go/util"
  7. "fmt"
  8. "io"
  9. "math"
  10. "math/rand"
  11. "os"
  12. "runtime"
  13. "runtime/pprof"
  14. "sort"
  15. "strings"
  16. "sync"
  17. "time"
  18. )
  19. type BenchmarkOptions struct {
  20. server *string
  21. concurrency *int
  22. numberOfFiles *int
  23. fileSize *int
  24. idListFile *string
  25. write *bool
  26. read *bool
  27. sequentialRead *bool
  28. collection *string
  29. cpuprofile *string
  30. vid2server map[string]string //cache for vid locations
  31. }
  32. var (
  33. b BenchmarkOptions
  34. )
  35. func init() {
  36. cmdBenchmark.Run = runbenchmark // break init cycle
  37. cmdBenchmark.IsDebug = cmdBenchmark.Flag.Bool("debug", false, "verbose debug information")
  38. b.server = cmdBenchmark.Flag.String("server", "localhost:9333", "weedfs master location")
  39. b.concurrency = cmdBenchmark.Flag.Int("c", 16, "number of concurrent write or read processes")
  40. b.fileSize = cmdBenchmark.Flag.Int("size", 1024, "simulated file size in bytes")
  41. b.numberOfFiles = cmdBenchmark.Flag.Int("n", 1024*1024, "number of files to write for each thread")
  42. b.idListFile = cmdBenchmark.Flag.String("list", os.TempDir()+"/benchmark_list.txt", "list of uploaded file ids")
  43. b.write = cmdBenchmark.Flag.Bool("write", true, "enable write")
  44. b.read = cmdBenchmark.Flag.Bool("read", true, "enable read")
  45. b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file")
  46. b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection")
  47. b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "write cpu profile to file")
  48. b.vid2server = make(map[string]string)
  49. }
  50. var cmdBenchmark = &Command{
  51. UsageLine: "benchmark -server=localhost:9333 -c=10 -n=100000",
  52. Short: "benchmark on writing millions of files and read out",
  53. Long: `benchmark on an empty weed file system.
  54. Two tests during benchmark:
  55. 1) write lots of small files to the system
  56. 2) read the files out
  57. The file content is mostly zero, but no compression is done.
  58. You can choose to only benchmark read or write.
  59. During write, the list of uploaded file ids is stored in "-list" specified file.
  60. You can also use your own list of file ids to run read test.
  61. Write speed and read speed will be collected.
  62. The numbers are used to get a sense of the system.
  63. Usually your network or the hard drive is the real bottleneck.
  64. Another thing to watch is whether the volumes are evenly distributed
  65. to each volume server. Because the 7 more benchmark volumes are randomly distributed
  66. to servers with free slots, it's highly possible some servers have uneven amount of
  67. benchmark volumes. To remedy this, you can use this to grow the benchmark volumes
  68. before starting the benchmark command:
  69. http://localhost:9333/vol/grow?collection=benchmark&count=5
  70. After benchmarking, you can clean up the written data by deleting the benchmark collection
  71. http://localhost:9333/col/delete?collection=benchmark
  72. `,
  73. }
  74. var (
  75. wait sync.WaitGroup
  76. writeStats *stats
  77. readStats *stats
  78. )
  79. func runbenchmark(cmd *Command, args []string) bool {
  80. fmt.Printf("This is Weed File System version %s %s %s\n", VERSION, runtime.GOOS, runtime.GOARCH)
  81. if *b.cpuprofile != "" {
  82. f, err := os.Create(*b.cpuprofile)
  83. if err != nil {
  84. glog.Fatal(err)
  85. }
  86. pprof.StartCPUProfile(f)
  87. defer pprof.StopCPUProfile()
  88. }
  89. if *b.write {
  90. bench_write()
  91. }
  92. if *b.read {
  93. bench_read()
  94. }
  95. return true
  96. }
  97. func bench_write() {
  98. fileIdLineChan := make(chan string)
  99. finishChan := make(chan bool)
  100. writeStats = newStats()
  101. idChan := make(chan int)
  102. wait.Add(*b.concurrency)
  103. go writeFileIds(*b.idListFile, fileIdLineChan, finishChan)
  104. for i := 0; i < *b.concurrency; i++ {
  105. go writeFiles(idChan, fileIdLineChan, writeStats)
  106. }
  107. writeStats.start = time.Now()
  108. go writeStats.checkProgress("Writing Benchmark", finishChan)
  109. for i := 0; i < *b.numberOfFiles; i++ {
  110. idChan <- i
  111. }
  112. close(idChan)
  113. wait.Wait()
  114. writeStats.end = time.Now()
  115. wait.Add(1)
  116. finishChan <- true
  117. finishChan <- true
  118. close(finishChan)
  119. wait.Wait()
  120. writeStats.printStats()
  121. }
  122. func bench_read() {
  123. fileIdLineChan := make(chan string)
  124. finishChan := make(chan bool)
  125. readStats = newStats()
  126. wait.Add(*b.concurrency)
  127. go readFileIds(*b.idListFile, fileIdLineChan)
  128. readStats.start = time.Now()
  129. go readStats.checkProgress("Randomly Reading Benchmark", finishChan)
  130. for i := 0; i < *b.concurrency; i++ {
  131. go readFiles(fileIdLineChan, readStats)
  132. }
  133. wait.Wait()
  134. finishChan <- true
  135. close(finishChan)
  136. readStats.end = time.Now()
  137. readStats.printStats()
  138. }
  139. func writeFiles(idChan chan int, fileIdLineChan chan string, s *stats) {
  140. serverLimitChan := make(map[string]chan bool)
  141. for {
  142. if id, ok := <-idChan; ok {
  143. start := time.Now()
  144. fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: int64(*b.fileSize)}, FileSize: int64(*b.fileSize)}
  145. if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection); err == nil {
  146. fp.Server, fp.Fid, fp.Collection = assignResult.PublicUrl, assignResult.Fid, *b.collection
  147. if _, ok := serverLimitChan[fp.Server]; !ok {
  148. serverLimitChan[fp.Server] = make(chan bool, 7)
  149. }
  150. serverLimitChan[fp.Server] <- true
  151. if _, err := fp.Upload(0, *b.server); err == nil {
  152. fileIdLineChan <- fp.Fid
  153. s.completed++
  154. s.transferred += int64(*b.fileSize)
  155. } else {
  156. s.failed++
  157. }
  158. writeStats.addSample(time.Now().Sub(start))
  159. <-serverLimitChan[fp.Server]
  160. if *cmdBenchmark.IsDebug {
  161. fmt.Printf("writing %d file %s\n", id, fp.Fid)
  162. }
  163. } else {
  164. s.failed++
  165. println("writing file error:", err.Error())
  166. }
  167. } else {
  168. break
  169. }
  170. }
  171. wait.Done()
  172. }
  173. func readFiles(fileIdLineChan chan string, s *stats) {
  174. serverLimitChan := make(map[string]chan bool)
  175. masterLimitChan := make(chan bool, 7)
  176. for {
  177. if fid, ok := <-fileIdLineChan; ok {
  178. if len(fid) == 0 {
  179. continue
  180. }
  181. if fid[0] == '#' {
  182. continue
  183. }
  184. if *cmdBenchmark.IsDebug {
  185. fmt.Printf("reading file %s\n", fid)
  186. }
  187. parts := strings.SplitN(fid, ",", 2)
  188. vid := parts[0]
  189. start := time.Now()
  190. if server, ok := b.vid2server[vid]; !ok {
  191. masterLimitChan <- true
  192. if ret, err := operation.Lookup(*b.server, vid); err == nil {
  193. if len(ret.Locations) > 0 {
  194. server = ret.Locations[0].PublicUrl
  195. b.vid2server[vid] = server
  196. }
  197. }
  198. <-masterLimitChan
  199. }
  200. if server, ok := b.vid2server[vid]; ok {
  201. if _, ok := serverLimitChan[server]; !ok {
  202. serverLimitChan[server] = make(chan bool, 7)
  203. }
  204. serverLimitChan[server] <- true
  205. url := "http://" + server + "/" + fid
  206. if bytesRead, err := util.Get(url); err == nil {
  207. s.completed++
  208. s.transferred += int64(len(bytesRead))
  209. readStats.addSample(time.Now().Sub(start))
  210. } else {
  211. s.failed++
  212. println("!!!! Failed to read from ", url, " !!!!!")
  213. }
  214. <-serverLimitChan[server]
  215. } else {
  216. s.failed++
  217. println("!!!! volume id ", vid, " location not found!!!!!")
  218. }
  219. } else {
  220. break
  221. }
  222. }
  223. wait.Done()
  224. }
  225. func writeFileIds(fileName string, fileIdLineChan chan string, finishChan chan bool) {
  226. file, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
  227. if err != nil {
  228. glog.Fatalf("File to create file %s: %s\n", fileName, err)
  229. }
  230. defer file.Close()
  231. for {
  232. select {
  233. case <-finishChan:
  234. wait.Done()
  235. return
  236. case line := <-fileIdLineChan:
  237. file.Write([]byte(line))
  238. file.Write([]byte("\n"))
  239. }
  240. }
  241. }
  242. func readFileIds(fileName string, fileIdLineChan chan string) {
  243. file, err := os.Open(fileName) // For read access.
  244. if err != nil {
  245. glog.Fatalf("File to read file %s: %s\n", fileName, err)
  246. }
  247. defer file.Close()
  248. r := bufio.NewReader(file)
  249. if *b.sequentialRead {
  250. for {
  251. if line, err := Readln(r); err == nil {
  252. fileIdLineChan <- string(line)
  253. } else {
  254. break
  255. }
  256. }
  257. } else {
  258. lines := make([]string, 0, *b.numberOfFiles)
  259. for {
  260. if line, err := Readln(r); err == nil {
  261. lines = append(lines, string(line))
  262. } else {
  263. break
  264. }
  265. }
  266. for i := 0; i < *b.numberOfFiles; i++ {
  267. fileIdLineChan <- lines[rand.Intn(len(lines))]
  268. }
  269. }
  270. close(fileIdLineChan)
  271. }
  272. const (
  273. benchResolution = 10000 //0.1 microsecond
  274. benchBucket = 1000000000 / benchResolution
  275. )
  276. // An efficient statics collecting and rendering
  277. type stats struct {
  278. data []int
  279. overflow []int
  280. completed int
  281. failed int
  282. transferred int64
  283. start time.Time
  284. end time.Time
  285. }
  286. var percentages = []int{50, 66, 75, 80, 90, 95, 98, 99, 100}
  287. func newStats() *stats {
  288. return &stats{data: make([]int, benchResolution), overflow: make([]int, 0)}
  289. }
  290. func (s *stats) addSample(d time.Duration) {
  291. index := int(d / benchBucket)
  292. if index < 0 {
  293. fmt.Printf("This request takes %3.1f seconds, skipping!\n", float64(index)/10000)
  294. } else if index < len(s.data) {
  295. s.data[int(d/benchBucket)]++
  296. } else {
  297. s.overflow = append(s.overflow, index)
  298. }
  299. }
  300. func (s *stats) checkProgress(testName string, finishChan chan bool) {
  301. fmt.Printf("\n------------ %s ----------\n", testName)
  302. ticker := time.Tick(time.Second)
  303. lastCompleted, lastTransferred, lastTime := 0, int64(0), time.Now()
  304. for {
  305. select {
  306. case <-finishChan:
  307. return
  308. case t := <-ticker:
  309. completed, transferred, taken := s.completed-lastCompleted, s.transferred-lastTransferred, t.Sub(lastTime)
  310. fmt.Printf("Completed %d of %d requests, %3.1f%% %3.1f/s %3.1fMB/s\n",
  311. s.completed, *b.numberOfFiles, float64(s.completed)*100/float64(*b.numberOfFiles),
  312. float64(completed)*float64(int64(time.Second))/float64(int64(taken)),
  313. float64(transferred)*float64(int64(time.Second))/float64(int64(taken))/float64(1024*1024),
  314. )
  315. lastCompleted, lastTransferred, lastTime = s.completed, s.transferred, t
  316. }
  317. }
  318. }
  319. func (s *stats) printStats() {
  320. timeTaken := float64(int64(s.end.Sub(s.start))) / 1000000000
  321. fmt.Printf("\nConcurrency Level: %d\n", *b.concurrency)
  322. fmt.Printf("Time taken for tests: %.3f seconds\n", timeTaken)
  323. fmt.Printf("Complete requests: %d\n", s.completed)
  324. fmt.Printf("Failed requests: %d\n", s.failed)
  325. fmt.Printf("Total transferred: %d bytes\n", s.transferred)
  326. fmt.Printf("Requests per second: %.2f [#/sec]\n", float64(s.completed)/timeTaken)
  327. fmt.Printf("Transfer rate: %.2f [Kbytes/sec]\n", float64(s.transferred)/1024/timeTaken)
  328. n, sum := 0, 0
  329. min, max := 10000000, 0
  330. for i := 0; i < len(s.data); i++ {
  331. n += s.data[i]
  332. sum += s.data[i] * i
  333. if s.data[i] > 0 {
  334. if min > i {
  335. min = i
  336. }
  337. if max < i {
  338. max = i
  339. }
  340. }
  341. }
  342. n += len(s.overflow)
  343. for i := 0; i < len(s.overflow); i++ {
  344. sum += s.overflow[i]
  345. if min > s.overflow[i] {
  346. min = s.overflow[i]
  347. }
  348. if max < s.overflow[i] {
  349. max = s.overflow[i]
  350. }
  351. }
  352. avg := float64(sum) / float64(n)
  353. varianceSum := 0.0
  354. for i := 0; i < len(s.data); i++ {
  355. if s.data[i] > 0 {
  356. d := float64(i) - avg
  357. varianceSum += d * d * float64(s.data[i])
  358. }
  359. }
  360. for i := 0; i < len(s.overflow); i++ {
  361. d := float64(s.overflow[i]) - avg
  362. varianceSum += d * d
  363. }
  364. std := math.Sqrt(varianceSum / float64(n))
  365. fmt.Printf("\nConnection Times (ms)\n")
  366. fmt.Printf(" min avg max std\n")
  367. fmt.Printf("Total: %2.1f %3.1f %3.1f %3.1f\n", float32(min)/10, float32(avg)/10, float32(max)/10, std/10)
  368. //printing percentiles
  369. fmt.Printf("\nPercentage of the requests served within a certain time (ms)\n")
  370. percentiles := make([]int, len(percentages))
  371. for i := 0; i < len(percentages); i++ {
  372. percentiles[i] = n * percentages[i] / 100
  373. }
  374. percentiles[len(percentiles)-1] = n
  375. percentileIndex := 0
  376. currentSum := 0
  377. for i := 0; i < len(s.data); i++ {
  378. currentSum += s.data[i]
  379. if s.data[i] > 0 && percentileIndex < len(percentiles) && currentSum >= percentiles[percentileIndex] {
  380. fmt.Printf(" %3d%% %5.1f ms\n", percentages[percentileIndex], float32(i)/10.0)
  381. percentileIndex++
  382. for percentileIndex < len(percentiles) && currentSum >= percentiles[percentileIndex] {
  383. percentileIndex++
  384. }
  385. }
  386. }
  387. sort.Ints(s.overflow)
  388. for i := 0; i < len(s.overflow); i++ {
  389. currentSum++
  390. if percentileIndex < len(percentiles) && currentSum >= percentiles[percentileIndex] {
  391. fmt.Printf(" %3d%% %5.1f ms\n", percentages[percentileIndex], float32(s.overflow[i])/10.0)
  392. percentileIndex++
  393. for percentileIndex < len(percentiles) && currentSum >= percentiles[percentileIndex] {
  394. percentileIndex++
  395. }
  396. }
  397. }
  398. }
  399. // a fake reader to generate content to upload
  400. type FakeReader struct {
  401. id uint64 // an id number
  402. size int64 // max bytes
  403. }
  404. func (l *FakeReader) Read(p []byte) (n int, err error) {
  405. if l.size <= 0 {
  406. return 0, io.EOF
  407. }
  408. if int64(len(p)) > l.size {
  409. n = int(l.size)
  410. } else {
  411. n = len(p)
  412. }
  413. for i := 0; i < n-8; i += 8 {
  414. for s := uint(0); s < 8; s++ {
  415. p[i] = byte(l.id >> (s * 8))
  416. }
  417. }
  418. l.size -= int64(n)
  419. return
  420. }
  421. func Readln(r *bufio.Reader) ([]byte, error) {
  422. var (
  423. isPrefix bool = true
  424. err error = nil
  425. line, ln []byte
  426. )
  427. for isPrefix && err == nil {
  428. line, isPrefix, err = r.ReadLine()
  429. ln = append(ln, line...)
  430. }
  431. return ln, err
  432. }