You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

560 lines
15 KiB

10 years ago
  1. package command
  2. import (
  3. "bufio"
  4. "context"
  5. "fmt"
  6. "github.com/chrislusf/seaweedfs/weed/server"
  7. "github.com/spf13/viper"
  8. "google.golang.org/grpc"
  9. "io"
  10. "math"
  11. "math/rand"
  12. "os"
  13. "runtime"
  14. "runtime/pprof"
  15. "sort"
  16. "strings"
  17. "sync"
  18. "time"
  19. "github.com/chrislusf/seaweedfs/weed/glog"
  20. "github.com/chrislusf/seaweedfs/weed/operation"
  21. "github.com/chrislusf/seaweedfs/weed/security"
  22. "github.com/chrislusf/seaweedfs/weed/util"
  23. "github.com/chrislusf/seaweedfs/weed/wdclient"
  24. )
  25. type BenchmarkOptions struct {
  26. masters *string
  27. concurrency *int
  28. numberOfFiles *int
  29. fileSize *int
  30. idListFile *string
  31. write *bool
  32. deletePercentage *int
  33. read *bool
  34. sequentialRead *bool
  35. collection *string
  36. cpuprofile *string
  37. maxCpu *int
  38. grpcDialOption grpc.DialOption
  39. }
  40. var (
  41. b BenchmarkOptions
  42. sharedBytes []byte
  43. masterClient *wdclient.MasterClient
  44. isSecure bool
  45. )
  46. func init() {
  47. cmdBenchmark.Run = runBenchmark // break init cycle
  48. cmdBenchmark.IsDebug = cmdBenchmark.Flag.Bool("debug", false, "verbose debug information")
  49. b.masters = cmdBenchmark.Flag.String("master", "localhost:9333", "SeaweedFS master location")
  50. b.concurrency = cmdBenchmark.Flag.Int("c", 16, "number of concurrent write or read processes")
  51. b.fileSize = cmdBenchmark.Flag.Int("size", 1024, "simulated file size in bytes, with random(0~63) bytes padding")
  52. b.numberOfFiles = cmdBenchmark.Flag.Int("n", 1024*1024, "number of files to write for each thread")
  53. b.idListFile = cmdBenchmark.Flag.String("list", os.TempDir()+"/benchmark_list.txt", "list of uploaded file ids")
  54. b.write = cmdBenchmark.Flag.Bool("write", true, "enable write")
  55. b.deletePercentage = cmdBenchmark.Flag.Int("deletePercent", 0, "the percent of writes that are deletes")
  56. b.read = cmdBenchmark.Flag.Bool("read", true, "enable read")
  57. b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file")
  58. b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection")
  59. b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file")
  60. b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
  61. sharedBytes = make([]byte, 1024)
  62. }
  63. var cmdBenchmark = &Command{
  64. UsageLine: "benchmark -server=localhost:9333 -c=10 -n=100000",
  65. Short: "benchmark on writing millions of files and read out",
  66. Long: `benchmark on an empty SeaweedFS file system.
  67. Two tests during benchmark:
  68. 1) write lots of small files to the system
  69. 2) read the files out
  70. The file content is mostly zero, but no compression is done.
  71. You can choose to only benchmark read or write.
  72. During write, the list of uploaded file ids is stored in "-list" specified file.
  73. You can also use your own list of file ids to run read test.
  74. Write speed and read speed will be collected.
  75. The numbers are used to get a sense of the system.
  76. Usually your network or the hard drive is the real bottleneck.
  77. Another thing to watch is whether the volumes are evenly distributed
  78. to each volume server. Because the 7 more benchmark volumes are randomly distributed
  79. to servers with free slots, it's highly possible some servers have uneven amount of
  80. benchmark volumes. To remedy this, you can use this to grow the benchmark volumes
  81. before starting the benchmark command:
  82. http://localhost:9333/vol/grow?collection=benchmark&count=5
  83. After benchmarking, you can clean up the written data by deleting the benchmark collection
  84. http://localhost:9333/col/delete?collection=benchmark
  85. `,
  86. }
  87. var (
  88. wait sync.WaitGroup
  89. writeStats *stats
  90. readStats *stats
  91. )
  92. func runBenchmark(cmd *Command, args []string) bool {
  93. weed_server.LoadConfiguration("security", false)
  94. b.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
  95. fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
  96. if *b.maxCpu < 1 {
  97. *b.maxCpu = runtime.NumCPU()
  98. }
  99. runtime.GOMAXPROCS(*b.maxCpu)
  100. if *b.cpuprofile != "" {
  101. f, err := os.Create(*b.cpuprofile)
  102. if err != nil {
  103. glog.Fatal(err)
  104. }
  105. pprof.StartCPUProfile(f)
  106. defer pprof.StopCPUProfile()
  107. }
  108. masterClient = wdclient.NewMasterClient(context.Background(), b.grpcDialOption, "client", strings.Split(*b.masters, ","))
  109. go masterClient.KeepConnectedToMaster()
  110. masterClient.WaitUntilConnected()
  111. if *b.write {
  112. benchWrite()
  113. }
  114. if *b.read {
  115. benchRead()
  116. }
  117. return true
  118. }
  119. func benchWrite() {
  120. fileIdLineChan := make(chan string)
  121. finishChan := make(chan bool)
  122. writeStats = newStats(*b.concurrency)
  123. idChan := make(chan int)
  124. go writeFileIds(*b.idListFile, fileIdLineChan, finishChan)
  125. for i := 0; i < *b.concurrency; i++ {
  126. wait.Add(1)
  127. go writeFiles(idChan, fileIdLineChan, &writeStats.localStats[i])
  128. }
  129. writeStats.start = time.Now()
  130. writeStats.total = *b.numberOfFiles
  131. go writeStats.checkProgress("Writing Benchmark", finishChan)
  132. for i := 0; i < *b.numberOfFiles; i++ {
  133. idChan <- i
  134. }
  135. close(idChan)
  136. wait.Wait()
  137. writeStats.end = time.Now()
  138. wait.Add(2)
  139. finishChan <- true
  140. finishChan <- true
  141. wait.Wait()
  142. close(finishChan)
  143. writeStats.printStats()
  144. }
  145. func benchRead() {
  146. fileIdLineChan := make(chan string)
  147. finishChan := make(chan bool)
  148. readStats = newStats(*b.concurrency)
  149. go readFileIds(*b.idListFile, fileIdLineChan)
  150. readStats.start = time.Now()
  151. readStats.total = *b.numberOfFiles
  152. go readStats.checkProgress("Randomly Reading Benchmark", finishChan)
  153. for i := 0; i < *b.concurrency; i++ {
  154. wait.Add(1)
  155. go readFiles(fileIdLineChan, &readStats.localStats[i])
  156. }
  157. wait.Wait()
  158. wait.Add(1)
  159. finishChan <- true
  160. wait.Wait()
  161. close(finishChan)
  162. readStats.end = time.Now()
  163. readStats.printStats()
  164. }
  165. type delayedFile struct {
  166. enterTime time.Time
  167. fp *operation.FilePart
  168. }
  169. func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
  170. defer wait.Done()
  171. delayedDeleteChan := make(chan *delayedFile, 100)
  172. var waitForDeletions sync.WaitGroup
  173. for i := 0; i < 7; i++ {
  174. waitForDeletions.Add(1)
  175. go func() {
  176. defer waitForDeletions.Done()
  177. for df := range delayedDeleteChan {
  178. if df.enterTime.After(time.Now()) {
  179. time.Sleep(df.enterTime.Sub(time.Now()))
  180. }
  181. var jwtAuthorization security.EncodedJwt
  182. if isSecure {
  183. jwtAuthorization = operation.LookupJwt(masterClient.GetMaster(), df.fp.Fid)
  184. }
  185. if e := util.Delete(fmt.Sprintf("http://%s/%s", df.fp.Server, df.fp.Fid), string(jwtAuthorization)); e == nil {
  186. s.completed++
  187. } else {
  188. s.failed++
  189. }
  190. }
  191. }()
  192. }
  193. random := rand.New(rand.NewSource(time.Now().UnixNano()))
  194. for id := range idChan {
  195. start := time.Now()
  196. fileSize := int64(*b.fileSize + random.Intn(64))
  197. fp := &operation.FilePart{
  198. Reader: &FakeReader{id: uint64(id), size: fileSize},
  199. FileSize: fileSize,
  200. MimeType: "image/bench", // prevent gzip benchmark content
  201. }
  202. ar := &operation.VolumeAssignRequest{
  203. Count: 1,
  204. Collection: *b.collection,
  205. }
  206. if assignResult, err := operation.Assign(masterClient.GetMaster(), b.grpcDialOption, ar); err == nil {
  207. fp.Server, fp.Fid, fp.Collection = assignResult.Url, assignResult.Fid, *b.collection
  208. if !isSecure && assignResult.Auth != "" {
  209. isSecure = true
  210. }
  211. if _, err := fp.Upload(0, masterClient.GetMaster(), assignResult.Auth, b.grpcDialOption); err == nil {
  212. if random.Intn(100) < *b.deletePercentage {
  213. s.total++
  214. delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp}
  215. } else {
  216. fileIdLineChan <- fp.Fid
  217. }
  218. s.completed++
  219. s.transferred += fileSize
  220. } else {
  221. s.failed++
  222. fmt.Printf("Failed to write with error:%v\n", err)
  223. }
  224. writeStats.addSample(time.Now().Sub(start))
  225. if *cmdBenchmark.IsDebug {
  226. fmt.Printf("writing %d file %s\n", id, fp.Fid)
  227. }
  228. } else {
  229. s.failed++
  230. println("writing file error:", err.Error())
  231. }
  232. }
  233. close(delayedDeleteChan)
  234. waitForDeletions.Wait()
  235. }
  236. func readFiles(fileIdLineChan chan string, s *stat) {
  237. defer wait.Done()
  238. for fid := range fileIdLineChan {
  239. if len(fid) == 0 {
  240. continue
  241. }
  242. if fid[0] == '#' {
  243. continue
  244. }
  245. if *cmdBenchmark.IsDebug {
  246. fmt.Printf("reading file %s\n", fid)
  247. }
  248. start := time.Now()
  249. url, err := masterClient.LookupFileId(fid)
  250. if err != nil {
  251. s.failed++
  252. println("!!!! ", fid, " location not found!!!!!")
  253. continue
  254. }
  255. if bytesRead, err := util.Get(url); err == nil {
  256. s.completed++
  257. s.transferred += int64(len(bytesRead))
  258. readStats.addSample(time.Now().Sub(start))
  259. } else {
  260. s.failed++
  261. fmt.Printf("Failed to read %s error:%v\n", url, err)
  262. }
  263. }
  264. }
  265. func writeFileIds(fileName string, fileIdLineChan chan string, finishChan chan bool) {
  266. file, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
  267. if err != nil {
  268. glog.Fatalf("File to create file %s: %s\n", fileName, err)
  269. }
  270. defer file.Close()
  271. for {
  272. select {
  273. case <-finishChan:
  274. wait.Done()
  275. return
  276. case line := <-fileIdLineChan:
  277. file.Write([]byte(line))
  278. file.Write([]byte("\n"))
  279. }
  280. }
  281. }
  282. func readFileIds(fileName string, fileIdLineChan chan string) {
  283. file, err := os.Open(fileName) // For read access.
  284. if err != nil {
  285. glog.Fatalf("File to read file %s: %s\n", fileName, err)
  286. }
  287. defer file.Close()
  288. random := rand.New(rand.NewSource(time.Now().UnixNano()))
  289. r := bufio.NewReader(file)
  290. if *b.sequentialRead {
  291. for {
  292. if line, err := Readln(r); err == nil {
  293. fileIdLineChan <- string(line)
  294. } else {
  295. break
  296. }
  297. }
  298. } else {
  299. lines := make([]string, 0, readStats.total)
  300. for {
  301. if line, err := Readln(r); err == nil {
  302. lines = append(lines, string(line))
  303. } else {
  304. break
  305. }
  306. }
  307. if len(lines) > 0 {
  308. for i := 0; i < readStats.total; i++ {
  309. fileIdLineChan <- lines[random.Intn(len(lines))]
  310. }
  311. }
  312. }
  313. close(fileIdLineChan)
  314. }
  315. const (
  316. benchResolution = 10000 //0.1 microsecond
  317. benchBucket = 1000000000 / benchResolution
  318. )
  319. // An efficient statics collecting and rendering
  320. type stats struct {
  321. data []int
  322. overflow []int
  323. localStats []stat
  324. start time.Time
  325. end time.Time
  326. total int
  327. }
  328. type stat struct {
  329. completed int
  330. failed int
  331. total int
  332. transferred int64
  333. }
  334. var percentages = []int{50, 66, 75, 80, 90, 95, 98, 99, 100}
  335. func newStats(n int) *stats {
  336. return &stats{
  337. data: make([]int, benchResolution),
  338. overflow: make([]int, 0),
  339. localStats: make([]stat, n),
  340. }
  341. }
  342. func (s *stats) addSample(d time.Duration) {
  343. index := int(d / benchBucket)
  344. if index < 0 {
  345. fmt.Printf("This request takes %3.1f seconds, skipping!\n", float64(index)/10000)
  346. } else if index < len(s.data) {
  347. s.data[int(d/benchBucket)]++
  348. } else {
  349. s.overflow = append(s.overflow, index)
  350. }
  351. }
  352. func (s *stats) checkProgress(testName string, finishChan chan bool) {
  353. fmt.Printf("\n------------ %s ----------\n", testName)
  354. ticker := time.Tick(time.Second)
  355. lastCompleted, lastTransferred, lastTime := 0, int64(0), time.Now()
  356. for {
  357. select {
  358. case <-finishChan:
  359. wait.Done()
  360. return
  361. case t := <-ticker:
  362. completed, transferred, taken, total := 0, int64(0), t.Sub(lastTime), s.total
  363. for _, localStat := range s.localStats {
  364. completed += localStat.completed
  365. transferred += localStat.transferred
  366. total += localStat.total
  367. }
  368. fmt.Printf("Completed %d of %d requests, %3.1f%% %3.1f/s %3.1fMB/s\n",
  369. completed, total, float64(completed)*100/float64(total),
  370. float64(completed-lastCompleted)*float64(int64(time.Second))/float64(int64(taken)),
  371. float64(transferred-lastTransferred)*float64(int64(time.Second))/float64(int64(taken))/float64(1024*1024),
  372. )
  373. lastCompleted, lastTransferred, lastTime = completed, transferred, t
  374. }
  375. }
  376. }
  377. func (s *stats) printStats() {
  378. completed, failed, transferred, total := 0, 0, int64(0), s.total
  379. for _, localStat := range s.localStats {
  380. completed += localStat.completed
  381. failed += localStat.failed
  382. transferred += localStat.transferred
  383. total += localStat.total
  384. }
  385. timeTaken := float64(int64(s.end.Sub(s.start))) / 1000000000
  386. fmt.Printf("\nConcurrency Level: %d\n", *b.concurrency)
  387. fmt.Printf("Time taken for tests: %.3f seconds\n", timeTaken)
  388. fmt.Printf("Complete requests: %d\n", completed)
  389. fmt.Printf("Failed requests: %d\n", failed)
  390. fmt.Printf("Total transferred: %d bytes\n", transferred)
  391. fmt.Printf("Requests per second: %.2f [#/sec]\n", float64(completed)/timeTaken)
  392. fmt.Printf("Transfer rate: %.2f [Kbytes/sec]\n", float64(transferred)/1024/timeTaken)
  393. n, sum := 0, 0
  394. min, max := 10000000, 0
  395. for i := 0; i < len(s.data); i++ {
  396. n += s.data[i]
  397. sum += s.data[i] * i
  398. if s.data[i] > 0 {
  399. if min > i {
  400. min = i
  401. }
  402. if max < i {
  403. max = i
  404. }
  405. }
  406. }
  407. n += len(s.overflow)
  408. for i := 0; i < len(s.overflow); i++ {
  409. sum += s.overflow[i]
  410. if min > s.overflow[i] {
  411. min = s.overflow[i]
  412. }
  413. if max < s.overflow[i] {
  414. max = s.overflow[i]
  415. }
  416. }
  417. avg := float64(sum) / float64(n)
  418. varianceSum := 0.0
  419. for i := 0; i < len(s.data); i++ {
  420. if s.data[i] > 0 {
  421. d := float64(i) - avg
  422. varianceSum += d * d * float64(s.data[i])
  423. }
  424. }
  425. for i := 0; i < len(s.overflow); i++ {
  426. d := float64(s.overflow[i]) - avg
  427. varianceSum += d * d
  428. }
  429. std := math.Sqrt(varianceSum / float64(n))
  430. fmt.Printf("\nConnection Times (ms)\n")
  431. fmt.Printf(" min avg max std\n")
  432. fmt.Printf("Total: %2.1f %3.1f %3.1f %3.1f\n", float32(min)/10, float32(avg)/10, float32(max)/10, std/10)
  433. //printing percentiles
  434. fmt.Printf("\nPercentage of the requests served within a certain time (ms)\n")
  435. percentiles := make([]int, len(percentages))
  436. for i := 0; i < len(percentages); i++ {
  437. percentiles[i] = n * percentages[i] / 100
  438. }
  439. percentiles[len(percentiles)-1] = n
  440. percentileIndex := 0
  441. currentSum := 0
  442. for i := 0; i < len(s.data); i++ {
  443. currentSum += s.data[i]
  444. if s.data[i] > 0 && percentileIndex < len(percentiles) && currentSum >= percentiles[percentileIndex] {
  445. fmt.Printf(" %3d%% %5.1f ms\n", percentages[percentileIndex], float32(i)/10.0)
  446. percentileIndex++
  447. for percentileIndex < len(percentiles) && currentSum >= percentiles[percentileIndex] {
  448. percentileIndex++
  449. }
  450. }
  451. }
  452. sort.Ints(s.overflow)
  453. for i := 0; i < len(s.overflow); i++ {
  454. currentSum++
  455. if percentileIndex < len(percentiles) && currentSum >= percentiles[percentileIndex] {
  456. fmt.Printf(" %3d%% %5.1f ms\n", percentages[percentileIndex], float32(s.overflow[i])/10.0)
  457. percentileIndex++
  458. for percentileIndex < len(percentiles) && currentSum >= percentiles[percentileIndex] {
  459. percentileIndex++
  460. }
  461. }
  462. }
  463. }
  464. // a fake reader to generate content to upload
  465. type FakeReader struct {
  466. id uint64 // an id number
  467. size int64 // max bytes
  468. }
  469. func (l *FakeReader) Read(p []byte) (n int, err error) {
  470. if l.size <= 0 {
  471. return 0, io.EOF
  472. }
  473. if int64(len(p)) > l.size {
  474. n = int(l.size)
  475. } else {
  476. n = len(p)
  477. }
  478. if n >= 8 {
  479. for i := 0; i < 8; i++ {
  480. p[i] = byte(l.id >> uint(i*8))
  481. }
  482. }
  483. l.size -= int64(n)
  484. return
  485. }
  486. func (l *FakeReader) WriteTo(w io.Writer) (n int64, err error) {
  487. size := int(l.size)
  488. bufferSize := len(sharedBytes)
  489. for size > 0 {
  490. tempBuffer := sharedBytes
  491. if size < bufferSize {
  492. tempBuffer = sharedBytes[0:size]
  493. }
  494. count, e := w.Write(tempBuffer)
  495. if e != nil {
  496. return int64(size), e
  497. }
  498. size -= count
  499. }
  500. return l.size, nil
  501. }
  502. func Readln(r *bufio.Reader) ([]byte, error) {
  503. var (
  504. isPrefix = true
  505. err error
  506. line, ln []byte
  507. )
  508. for isPrefix && err == nil {
  509. line, isPrefix, err = r.ReadLine()
  510. ln = append(ln, line...)
  511. }
  512. return ln, err
  513. }