You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

432 lines
13 KiB

5 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. package command
  2. import (
  3. "io/ioutil"
  4. "path/filepath"
  5. )
  6. func init() {
  7. cmdScaffold.Run = runScaffold // break init cycle
  8. }
  9. var cmdScaffold = &Command{
  10. UsageLine: "scaffold -config=[filer|notification|replication|security|master]",
  11. Short: "generate basic configuration files",
  12. Long: `Generate filer.toml with all possible configurations for you to customize.
  13. The options can also be overwritten by environment variables.
  14. For example, the filer.toml mysql password can be overwritten by environment variable
  15. export WEED_MYSQL_PASSWORD=some_password
  16. Environment variable rules:
  17. * Prefix the variable name with "WEED_"
  18. * Upppercase the reset of variable name.
  19. * Replace '.' with '_'
  20. `,
  21. }
  22. var (
  23. outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory")
  24. config = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security|master] the configuration file to generate")
  25. )
  26. func runScaffold(cmd *Command, args []string) bool {
  27. content := ""
  28. switch *config {
  29. case "filer":
  30. content = FILER_TOML_EXAMPLE
  31. case "notification":
  32. content = NOTIFICATION_TOML_EXAMPLE
  33. case "replication":
  34. content = REPLICATION_TOML_EXAMPLE
  35. case "security":
  36. content = SECURITY_TOML_EXAMPLE
  37. case "master":
  38. content = MASTER_TOML_EXAMPLE
  39. }
  40. if content == "" {
  41. println("need a valid -config option")
  42. return false
  43. }
  44. if *outputPath != "" {
  45. ioutil.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
  46. } else {
  47. println(content)
  48. }
  49. return true
  50. }
  51. const (
  52. FILER_TOML_EXAMPLE = `
  53. # A sample TOML config file for SeaweedFS filer store
  54. # Used with "weed filer" or "weed server -filer"
  55. # Put this file to one of the location, with descending priority
  56. # ./filer.toml
  57. # $HOME/.seaweedfs/filer.toml
  58. # /etc/seaweedfs/filer.toml
  59. ####################################################
  60. # Customizable filer server options
  61. ####################################################
  62. [filer.options]
  63. # with http DELETE, by default the filer would check whether a folder is empty.
  64. # recursive_delete will delete all sub folders and files, similar to "rm -Rf"
  65. recursive_delete = false
  66. # directories under this folder will be automatically creating a separate bucket
  67. buckets_folder = "/buckets"
  68. buckets_fsync = [ # a list of buckets with all write requests fsync=true
  69. "important_bucket",
  70. "should_always_fsync",
  71. ]
  72. ####################################################
  73. # The following are filer store options
  74. ####################################################
  75. [leveldb2]
  76. # local on disk, mostly for simple single-machine setup, fairly scalable
  77. # faster than previous leveldb, recommended.
  78. enabled = true
  79. dir = "." # directory to store level db files
  80. [mysql] # or tidb
  81. # CREATE TABLE IF NOT EXISTS filemeta (
  82. # dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
  83. # name VARCHAR(1000) COMMENT 'directory or file name',
  84. # directory TEXT COMMENT 'full path to parent directory',
  85. # meta LONGBLOB,
  86. # PRIMARY KEY (dirhash, name)
  87. # ) DEFAULT CHARSET=utf8;
  88. enabled = false
  89. hostname = "localhost"
  90. port = 3306
  91. username = "root"
  92. password = ""
  93. database = "" # create or use an existing database
  94. connection_max_idle = 2
  95. connection_max_open = 100
  96. interpolateParams = false
  97. [postgres] # or cockroachdb
  98. # CREATE TABLE IF NOT EXISTS filemeta (
  99. # dirhash BIGINT,
  100. # name VARCHAR(65535),
  101. # directory VARCHAR(65535),
  102. # meta bytea,
  103. # PRIMARY KEY (dirhash, name)
  104. # );
  105. enabled = false
  106. hostname = "localhost"
  107. port = 5432
  108. username = "postgres"
  109. password = ""
  110. database = "" # create or use an existing database
  111. sslmode = "disable"
  112. connection_max_idle = 100
  113. connection_max_open = 100
  114. [cassandra]
  115. # CREATE TABLE filemeta (
  116. # directory varchar,
  117. # name varchar,
  118. # meta blob,
  119. # PRIMARY KEY (directory, name)
  120. # ) WITH CLUSTERING ORDER BY (name ASC);
  121. enabled = false
  122. keyspace="seaweedfs"
  123. hosts=[
  124. "localhost:9042",
  125. ]
  126. [redis2]
  127. enabled = false
  128. address = "localhost:6379"
  129. password = ""
  130. database = 0
  131. [redis_cluster2]
  132. enabled = false
  133. addresses = [
  134. "localhost:30001",
  135. "localhost:30002",
  136. "localhost:30003",
  137. "localhost:30004",
  138. "localhost:30005",
  139. "localhost:30006",
  140. ]
  141. password = ""
  142. # allows reads from slave servers or the master, but all writes still go to the master
  143. readOnly = true
  144. # automatically use the closest Redis server for reads
  145. routeByLatency = true
  146. [etcd]
  147. enabled = false
  148. servers = "localhost:2379"
  149. timeout = "3s"
  150. [mongodb]
  151. enabled = false
  152. uri = "mongodb://localhost:27017"
  153. option_pool_size = 0
  154. database = "seaweedfs"
  155. [elastic7]
  156. enabled = false
  157. servers = [
  158. "http://localhost1:9200",
  159. "http://localhost2:9200",
  160. "http://localhost3:9200",
  161. ]
  162. username = ""
  163. password = ""
  164. sniff_enabled = false
  165. healthcheck_enabled = false
  166. # increase the value is recommend, be sure the value in Elastic is greater or equal here
  167. index.max_result_window = 10000
  168. `
  169. NOTIFICATION_TOML_EXAMPLE = `
  170. # A sample TOML config file for SeaweedFS filer store
  171. # Used by both "weed filer" or "weed server -filer" and "weed filer.replicate"
  172. # Put this file to one of the location, with descending priority
  173. # ./notification.toml
  174. # $HOME/.seaweedfs/notification.toml
  175. # /etc/seaweedfs/notification.toml
  176. ####################################################
  177. # notification
  178. # send and receive filer updates for each file to an external message queue
  179. ####################################################
  180. [notification.log]
  181. # this is only for debugging perpose and does not work with "weed filer.replicate"
  182. enabled = false
  183. [notification.kafka]
  184. enabled = false
  185. hosts = [
  186. "localhost:9092"
  187. ]
  188. topic = "seaweedfs_filer"
  189. offsetFile = "./last.offset"
  190. offsetSaveIntervalSeconds = 10
  191. [notification.aws_sqs]
  192. # experimental, let me know if it works
  193. enabled = false
  194. aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  195. aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  196. region = "us-east-2"
  197. sqs_queue_name = "my_filer_queue" # an existing queue name
  198. [notification.google_pub_sub]
  199. # read credentials doc at https://cloud.google.com/docs/authentication/getting-started
  200. enabled = false
  201. google_application_credentials = "/path/to/x.json" # path to json credential file
  202. project_id = "" # an existing project id
  203. topic = "seaweedfs_filer_topic" # a topic, auto created if does not exists
  204. [notification.gocdk_pub_sub]
  205. # The Go Cloud Development Kit (https://gocloud.dev).
  206. # PubSub API (https://godoc.org/gocloud.dev/pubsub).
  207. # Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ.
  208. enabled = false
  209. # This URL will Dial the RabbitMQ server at the URL in the environment
  210. # variable RABBIT_SERVER_URL and open the exchange "myexchange".
  211. # The exchange must have already been created by some other means, like
  212. # the RabbitMQ management plugin.
  213. topic_url = "rabbit://myexchange"
  214. sub_url = "rabbit://myqueue"
  215. `
  216. REPLICATION_TOML_EXAMPLE = `
  217. # A sample TOML config file for replicating SeaweedFS filer
  218. # Used with "weed filer.replicate"
  219. # Put this file to one of the location, with descending priority
  220. # ./replication.toml
  221. # $HOME/.seaweedfs/replication.toml
  222. # /etc/seaweedfs/replication.toml
  223. [source.filer]
  224. enabled = true
  225. grpcAddress = "localhost:18888"
  226. # all files under this directory tree are replicated.
  227. # this is not a directory on your hard drive, but on your filer.
  228. # i.e., all files with this "prefix" are sent to notification message queue.
  229. directory = "/buckets"
  230. [sink.filer]
  231. enabled = false
  232. grpcAddress = "localhost:18888"
  233. # all replicated files are under this directory tree
  234. # this is not a directory on your hard drive, but on your filer.
  235. # i.e., all received files will be "prefixed" to this directory.
  236. directory = "/backup"
  237. replication = ""
  238. collection = ""
  239. ttlSec = 0
  240. [sink.s3]
  241. # read credentials doc at https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sessions.html
  242. # default loads credentials from the shared credentials file (~/.aws/credentials).
  243. enabled = false
  244. aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  245. aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  246. region = "us-east-2"
  247. bucket = "your_bucket_name" # an existing bucket
  248. directory = "/" # destination directory
  249. endpoint = ""
  250. [sink.google_cloud_storage]
  251. # read credentials doc at https://cloud.google.com/docs/authentication/getting-started
  252. enabled = false
  253. google_application_credentials = "/path/to/x.json" # path to json credential file
  254. bucket = "your_bucket_seaweedfs" # an existing bucket
  255. directory = "/" # destination directory
  256. [sink.azure]
  257. # experimental, let me know if it works
  258. enabled = false
  259. account_name = ""
  260. account_key = ""
  261. container = "mycontainer" # an existing container
  262. directory = "/" # destination directory
  263. [sink.backblaze]
  264. enabled = false
  265. b2_account_id = ""
  266. b2_master_application_key = ""
  267. bucket = "mybucket" # an existing bucket
  268. directory = "/" # destination directory
  269. `
  270. SECURITY_TOML_EXAMPLE = `
  271. # Put this file to one of the location, with descending priority
  272. # ./security.toml
  273. # $HOME/.seaweedfs/security.toml
  274. # /etc/seaweedfs/security.toml
  275. # this file is read by master, volume server, and filer
  276. # the jwt signing key is read by master and volume server.
  277. # a jwt defaults to expire after 10 seconds.
  278. [jwt.signing]
  279. key = ""
  280. expires_after_seconds = 10 # seconds
  281. # jwt for read is only supported with master+volume setup. Filer does not support this mode.
  282. [jwt.signing.read]
  283. key = ""
  284. expires_after_seconds = 10 # seconds
  285. # all grpc tls authentications are mutual
  286. # the values for the following ca, cert, and key are paths to the PERM files.
  287. # the host name is not checked, so the PERM files can be shared.
  288. [grpc]
  289. ca = ""
  290. [grpc.volume]
  291. cert = ""
  292. key = ""
  293. [grpc.master]
  294. cert = ""
  295. key = ""
  296. [grpc.filer]
  297. cert = ""
  298. key = ""
  299. [grpc.msg_broker]
  300. cert = ""
  301. key = ""
  302. # use this for any place needs a grpc client
  303. # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
  304. [grpc.client]
  305. cert = ""
  306. key = ""
  307. # volume server https options
  308. # Note: work in progress!
  309. # this does not work with other clients, e.g., "weed filer|mount" etc, yet.
  310. [https.client]
  311. enabled = true
  312. [https.volume]
  313. cert = ""
  314. key = ""
  315. `
  316. MASTER_TOML_EXAMPLE = `
  317. # Put this file to one of the location, with descending priority
  318. # ./master.toml
  319. # $HOME/.seaweedfs/master.toml
  320. # /etc/seaweedfs/master.toml
  321. # this file is read by master
  322. [master.maintenance]
  323. # periodically run these scripts are the same as running them from 'weed shell'
  324. scripts = """
  325. lock
  326. ec.encode -fullPercent=95 -quietFor=1h
  327. ec.rebuild -force
  328. ec.balance -force
  329. volume.balance -force
  330. volume.fix.replication
  331. unlock
  332. """
  333. sleep_minutes = 17 # sleep minutes between each script execution
  334. [master.filer]
  335. default = "localhost:8888" # used by maintenance scripts if the scripts needs to use fs related commands
  336. [master.sequencer]
  337. type = "raft" # Choose [raft|etcd] type for storing the file id sequence
  338. # when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence
  339. # example : http://127.0.0.1:2379,http://127.0.0.1:2389
  340. sequencer_etcd_urls = "http://127.0.0.1:2379"
  341. # configurations for tiered cloud storage
  342. # old volumes are transparently moved to cloud for cost efficiency
  343. [storage.backend]
  344. [storage.backend.s3.default]
  345. enabled = false
  346. aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  347. aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  348. region = "us-east-2"
  349. bucket = "your_bucket_name" # an existing bucket
  350. endpoint = ""
  351. # create this number of logical volumes if no more writable volumes
  352. # count_x means how many copies of data.
  353. # e.g.:
  354. # 000 has only one copy, copy_1
  355. # 010 and 001 has two copies, copy_2
  356. # 011 has only 3 copies, copy_3
  357. [master.volume_growth]
  358. copy_1 = 7 # create 1 x 7 = 7 actual volumes
  359. copy_2 = 6 # create 2 x 6 = 12 actual volumes
  360. copy_3 = 3 # create 3 x 3 = 9 actual volumes
  361. copy_other = 1 # create n x 1 = n actual volumes
  362. # configuration flags for replication
  363. [master.replication]
  364. # any replication counts should be considered minimums. If you specify 010 and
  365. # have 3 different racks, that's still considered writable. Writes will still
  366. # try to replicate to all available volumes. You should only use this option
  367. # if you are doing your own replication or periodic sync of volumes.
  368. treat_replication_as_minimums = false
  369. `
  370. )