You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

401 lines
13 KiB

6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
4 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
  1. package stats
  2. import (
  3. "log"
  4. "net"
  5. "net/http"
  6. "os"
  7. "strconv"
  8. "strings"
  9. "time"
  10. "github.com/prometheus/client_golang/prometheus"
  11. "github.com/prometheus/client_golang/prometheus/collectors"
  12. "github.com/prometheus/client_golang/prometheus/promhttp"
  13. "github.com/prometheus/client_golang/prometheus/push"
  14. "github.com/seaweedfs/seaweedfs/weed/glog"
  15. )
  16. // Readonly volume types
  17. const (
  18. Namespace = "SeaweedFS"
  19. IsReadOnly = "IsReadOnly"
  20. NoWriteOrDelete = "noWriteOrDelete"
  21. NoWriteCanDelete = "noWriteCanDelete"
  22. IsDiskSpaceLow = "isDiskSpaceLow"
  23. )
  24. var readOnlyVolumeTypes = [4]string{IsReadOnly, NoWriteOrDelete, NoWriteCanDelete, IsDiskSpaceLow}
  25. var (
  26. Gather = prometheus.NewRegistry()
  27. MasterClientConnectCounter = prometheus.NewCounterVec(
  28. prometheus.CounterOpts{
  29. Namespace: Namespace,
  30. Subsystem: "wdclient",
  31. Name: "connect_updates",
  32. Help: "Counter of master client leader updates.",
  33. }, []string{"type"})
  34. MasterRaftIsleader = prometheus.NewGauge(
  35. prometheus.GaugeOpts{
  36. Namespace: Namespace,
  37. Subsystem: "master",
  38. Name: "is_leader",
  39. Help: "is leader",
  40. })
  41. MasterAdminLock = prometheus.NewGaugeVec(
  42. prometheus.GaugeOpts{
  43. Namespace: Namespace,
  44. Subsystem: "master",
  45. Name: "admin_lock",
  46. Help: "admin lock",
  47. }, []string{"client"})
  48. MasterReceivedHeartbeatCounter = prometheus.NewCounterVec(
  49. prometheus.CounterOpts{
  50. Namespace: Namespace,
  51. Subsystem: "master",
  52. Name: "received_heartbeats",
  53. Help: "Counter of master received heartbeat.",
  54. }, []string{"type"})
  55. MasterReplicaPlacementMismatch = prometheus.NewGaugeVec(
  56. prometheus.GaugeOpts{
  57. Namespace: Namespace,
  58. Subsystem: "master",
  59. Name: "replica_placement_mismatch",
  60. Help: "replica placement mismatch",
  61. }, []string{"collection", "id"})
  62. MasterVolumeLayoutWritable = prometheus.NewGaugeVec(
  63. prometheus.GaugeOpts{
  64. Namespace: Namespace,
  65. Subsystem: "master",
  66. Name: "volume_layout_writable",
  67. Help: "Number of writable volumes in volume layouts",
  68. }, []string{"collection", "disk", "rp", "ttl"})
  69. MasterVolumeLayoutCrowded = prometheus.NewGaugeVec(
  70. prometheus.GaugeOpts{
  71. Namespace: Namespace,
  72. Subsystem: "master",
  73. Name: "volume_layout_crowded",
  74. Help: "Number of crowded volumes in volume layouts",
  75. }, []string{"collection", "disk", "rp", "ttl"})
  76. MasterPickForWriteErrorCounter = prometheus.NewCounter(
  77. prometheus.CounterOpts{
  78. Namespace: Namespace,
  79. Subsystem: "master",
  80. Name: "pick_for_write_error",
  81. Help: "Counter of master pick for write error",
  82. })
  83. MasterLeaderChangeCounter = prometheus.NewCounterVec(
  84. prometheus.CounterOpts{
  85. Namespace: Namespace,
  86. Subsystem: "master",
  87. Name: "leader_changes",
  88. Help: "Counter of master leader changes.",
  89. }, []string{"type"})
  90. FilerRequestCounter = prometheus.NewCounterVec(
  91. prometheus.CounterOpts{
  92. Namespace: Namespace,
  93. Subsystem: "filer",
  94. Name: "request_total",
  95. Help: "Counter of filer requests.",
  96. }, []string{"type", "code"})
  97. FilerHandlerCounter = prometheus.NewCounterVec(
  98. prometheus.CounterOpts{
  99. Namespace: Namespace,
  100. Subsystem: "filer",
  101. Name: "handler_total",
  102. Help: "Counter of filer handlers.",
  103. }, []string{"type"})
  104. FilerRequestHistogram = prometheus.NewHistogramVec(
  105. prometheus.HistogramOpts{
  106. Namespace: Namespace,
  107. Subsystem: "filer",
  108. Name: "request_seconds",
  109. Help: "Bucketed histogram of filer request processing time.",
  110. Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
  111. }, []string{"type"})
  112. FilerInFlightRequestsGauge = prometheus.NewGaugeVec(
  113. prometheus.GaugeOpts{
  114. Namespace: Namespace,
  115. Subsystem: "filer",
  116. Name: "in_flight_requests",
  117. Help: "Current number of in-flight requests being handled by filer.",
  118. }, []string{"type"})
  119. FilerServerLastSendTsOfSubscribeGauge = prometheus.NewGaugeVec(
  120. prometheus.GaugeOpts{
  121. Namespace: Namespace,
  122. Subsystem: "filer",
  123. Name: "last_send_timestamp_of_subscribe",
  124. Help: "The last send timestamp of the filer subscription.",
  125. }, []string{"sourceFiler", "clientName", "path"})
  126. FilerStoreCounter = prometheus.NewCounterVec(
  127. prometheus.CounterOpts{
  128. Namespace: Namespace,
  129. Subsystem: "filerStore",
  130. Name: "request_total",
  131. Help: "Counter of filer store requests.",
  132. }, []string{"store", "type"})
  133. FilerStoreHistogram = prometheus.NewHistogramVec(
  134. prometheus.HistogramOpts{
  135. Namespace: Namespace,
  136. Subsystem: "filerStore",
  137. Name: "request_seconds",
  138. Help: "Bucketed histogram of filer store request processing time.",
  139. Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
  140. }, []string{"store", "type"})
  141. FilerSyncOffsetGauge = prometheus.NewGaugeVec(
  142. prometheus.GaugeOpts{
  143. Namespace: Namespace,
  144. Subsystem: "filerSync",
  145. Name: "sync_offset",
  146. Help: "The offset of the filer synchronization service.",
  147. }, []string{"sourceFiler", "targetFiler", "clientName", "path"})
  148. VolumeServerRequestCounter = prometheus.NewCounterVec(
  149. prometheus.CounterOpts{
  150. Namespace: Namespace,
  151. Subsystem: "volumeServer",
  152. Name: "request_total",
  153. Help: "Counter of volume server requests.",
  154. }, []string{"type", "code"})
  155. VolumeServerHandlerCounter = prometheus.NewCounterVec(
  156. prometheus.CounterOpts{
  157. Namespace: Namespace,
  158. Subsystem: "volumeServer",
  159. Name: "handler_total",
  160. Help: "Counter of volume server handlers.",
  161. }, []string{"type"})
  162. VolumeServerVacuumingCompactCounter = prometheus.NewCounterVec(
  163. prometheus.CounterOpts{
  164. Namespace: Namespace,
  165. Subsystem: "volumeServer",
  166. Name: "vacuuming_compact_count",
  167. Help: "Counter of volume vacuuming Compact counter",
  168. }, []string{"success"})
  169. VolumeServerVacuumingCommitCounter = prometheus.NewCounterVec(
  170. prometheus.CounterOpts{
  171. Namespace: Namespace,
  172. Subsystem: "volumeServer",
  173. Name: "vacuuming_commit_count",
  174. Help: "Counter of volume vacuuming commit counter",
  175. }, []string{"success"})
  176. VolumeServerVacuumingHistogram = prometheus.NewHistogramVec(
  177. prometheus.HistogramOpts{
  178. Namespace: Namespace,
  179. Subsystem: "volumeServer",
  180. Name: "vacuuming_seconds",
  181. Help: "Bucketed histogram of volume server vacuuming processing time.",
  182. Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
  183. }, []string{"type"})
  184. VolumeServerRequestHistogram = prometheus.NewHistogramVec(
  185. prometheus.HistogramOpts{
  186. Namespace: Namespace,
  187. Subsystem: "volumeServer",
  188. Name: "request_seconds",
  189. Help: "Bucketed histogram of volume server request processing time.",
  190. Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
  191. }, []string{"type"})
  192. VolumeServerInFlightRequestsGauge = prometheus.NewGaugeVec(
  193. prometheus.GaugeOpts{
  194. Namespace: Namespace,
  195. Subsystem: "volumeServer",
  196. Name: "in_flight_requests",
  197. Help: "Current number of in-flight requests being handled by volume server.",
  198. }, []string{"type"})
  199. VolumeServerVolumeGauge = prometheus.NewGaugeVec(
  200. prometheus.GaugeOpts{
  201. Namespace: Namespace,
  202. Subsystem: "volumeServer",
  203. Name: "volumes",
  204. Help: "Number of volumes or shards.",
  205. }, []string{"collection", "type"})
  206. VolumeServerReadOnlyVolumeGauge = prometheus.NewGaugeVec(
  207. prometheus.GaugeOpts{
  208. Namespace: Namespace,
  209. Subsystem: "volumeServer",
  210. Name: "read_only_volumes",
  211. Help: "Number of read only volumes.",
  212. }, []string{"collection", "type"})
  213. VolumeServerMaxVolumeCounter = prometheus.NewGauge(
  214. prometheus.GaugeOpts{
  215. Namespace: Namespace,
  216. Subsystem: "volumeServer",
  217. Name: "max_volumes",
  218. Help: "Maximum number of volumes.",
  219. })
  220. VolumeServerDiskSizeGauge = prometheus.NewGaugeVec(
  221. prometheus.GaugeOpts{
  222. Namespace: Namespace,
  223. Subsystem: "volumeServer",
  224. Name: "total_disk_size",
  225. Help: "Actual disk size used by volumes.",
  226. }, []string{"collection", "type"})
  227. VolumeServerResourceGauge = prometheus.NewGaugeVec(
  228. prometheus.GaugeOpts{
  229. Namespace: Namespace,
  230. Subsystem: "volumeServer",
  231. Name: "resource",
  232. Help: "Resource usage",
  233. }, []string{"name", "type"})
  234. S3RequestCounter = prometheus.NewCounterVec(
  235. prometheus.CounterOpts{
  236. Namespace: Namespace,
  237. Subsystem: "s3",
  238. Name: "request_total",
  239. Help: "Counter of s3 requests.",
  240. }, []string{"type", "code", "bucket"})
  241. S3HandlerCounter = prometheus.NewCounterVec(
  242. prometheus.CounterOpts{
  243. Namespace: Namespace,
  244. Subsystem: "s3",
  245. Name: "handler_total",
  246. Help: "Counter of s3 server handlers.",
  247. }, []string{"type"})
  248. S3RequestHistogram = prometheus.NewHistogramVec(
  249. prometheus.HistogramOpts{
  250. Namespace: Namespace,
  251. Subsystem: "s3",
  252. Name: "request_seconds",
  253. Help: "Bucketed histogram of s3 request processing time.",
  254. Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
  255. }, []string{"type", "bucket"})
  256. S3TimeToFirstByteHistogram = prometheus.NewHistogramVec(
  257. prometheus.HistogramOpts{
  258. Namespace: Namespace,
  259. Subsystem: "s3",
  260. Name: "time_to_first_byte_millisecond",
  261. Help: "Bucketed histogram of s3 time to first byte request processing time.",
  262. Buckets: prometheus.ExponentialBuckets(0.001, 2, 27),
  263. }, []string{"type", "bucket"})
  264. S3InFlightRequestsGauge = prometheus.NewGaugeVec(
  265. prometheus.GaugeOpts{
  266. Namespace: Namespace,
  267. Subsystem: "s3",
  268. Name: "in_flight_requests",
  269. Help: "Current number of in-flight requests being handled by s3.",
  270. }, []string{"type"})
  271. )
  272. func init() {
  273. Gather.MustRegister(MasterClientConnectCounter)
  274. Gather.MustRegister(MasterRaftIsleader)
  275. Gather.MustRegister(MasterAdminLock)
  276. Gather.MustRegister(MasterReceivedHeartbeatCounter)
  277. Gather.MustRegister(MasterLeaderChangeCounter)
  278. Gather.MustRegister(MasterReplicaPlacementMismatch)
  279. Gather.MustRegister(MasterVolumeLayoutWritable)
  280. Gather.MustRegister(MasterVolumeLayoutCrowded)
  281. Gather.MustRegister(FilerRequestCounter)
  282. Gather.MustRegister(FilerHandlerCounter)
  283. Gather.MustRegister(FilerRequestHistogram)
  284. Gather.MustRegister(FilerInFlightRequestsGauge)
  285. Gather.MustRegister(FilerStoreCounter)
  286. Gather.MustRegister(FilerStoreHistogram)
  287. Gather.MustRegister(FilerSyncOffsetGauge)
  288. Gather.MustRegister(FilerServerLastSendTsOfSubscribeGauge)
  289. Gather.MustRegister(collectors.NewGoCollector())
  290. Gather.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
  291. Gather.MustRegister(VolumeServerRequestCounter)
  292. Gather.MustRegister(VolumeServerHandlerCounter)
  293. Gather.MustRegister(VolumeServerRequestHistogram)
  294. Gather.MustRegister(VolumeServerInFlightRequestsGauge)
  295. Gather.MustRegister(VolumeServerVacuumingCompactCounter)
  296. Gather.MustRegister(VolumeServerVacuumingCommitCounter)
  297. Gather.MustRegister(VolumeServerVacuumingHistogram)
  298. Gather.MustRegister(VolumeServerVolumeGauge)
  299. Gather.MustRegister(VolumeServerMaxVolumeCounter)
  300. Gather.MustRegister(VolumeServerReadOnlyVolumeGauge)
  301. Gather.MustRegister(VolumeServerDiskSizeGauge)
  302. Gather.MustRegister(VolumeServerResourceGauge)
  303. Gather.MustRegister(S3RequestCounter)
  304. Gather.MustRegister(S3HandlerCounter)
  305. Gather.MustRegister(S3RequestHistogram)
  306. Gather.MustRegister(S3InFlightRequestsGauge)
  307. Gather.MustRegister(S3TimeToFirstByteHistogram)
  308. }
  309. func LoopPushingMetric(name, instance, addr string, intervalSeconds int) {
  310. if addr == "" || intervalSeconds == 0 {
  311. return
  312. }
  313. glog.V(0).Infof("%s server sends metrics to %s every %d seconds", name, addr, intervalSeconds)
  314. pusher := push.New(addr, name).Gatherer(Gather).Grouping("instance", instance)
  315. for {
  316. err := pusher.Push()
  317. if err != nil && !strings.HasPrefix(err.Error(), "unexpected status code 200") {
  318. glog.V(0).Infof("could not push metrics to prometheus push gateway %s: %v", addr, err)
  319. }
  320. if intervalSeconds <= 0 {
  321. intervalSeconds = 15
  322. }
  323. time.Sleep(time.Duration(intervalSeconds) * time.Second)
  324. }
  325. }
  326. func JoinHostPort(host string, port int) string {
  327. portStr := strconv.Itoa(port)
  328. if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") {
  329. return host + ":" + portStr
  330. }
  331. return net.JoinHostPort(host, portStr)
  332. }
  333. func StartMetricsServer(ip string, port int) {
  334. if port == 0 {
  335. return
  336. }
  337. http.Handle("/metrics", promhttp.HandlerFor(Gather, promhttp.HandlerOpts{}))
  338. log.Fatal(http.ListenAndServe(JoinHostPort(ip, port), nil))
  339. }
  340. func SourceName(port uint32) string {
  341. hostname, err := os.Hostname()
  342. if err != nil {
  343. return "unknown"
  344. }
  345. return net.JoinHostPort(hostname, strconv.Itoa(int(port)))
  346. }
  347. // todo - can be changed to DeletePartialMatch when https://github.com/prometheus/client_golang/pull/1013 gets released
  348. func DeleteCollectionMetrics(collection string) {
  349. VolumeServerDiskSizeGauge.DeleteLabelValues(collection, "normal")
  350. for _, volume_type := range readOnlyVolumeTypes {
  351. VolumeServerReadOnlyVolumeGauge.DeleteLabelValues(collection, volume_type)
  352. }
  353. VolumeServerVolumeGauge.DeleteLabelValues(collection, "volume")
  354. }