You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

485 lines
14 KiB

5 years ago
4 years ago
4 years ago
4 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. package command
  2. import (
  3. "io/ioutil"
  4. "path/filepath"
  5. )
  6. func init() {
  7. cmdScaffold.Run = runScaffold // break init cycle
  8. }
  9. var cmdScaffold = &Command{
  10. UsageLine: "scaffold -config=[filer|notification|replication|security|master]",
  11. Short: "generate basic configuration files",
  12. Long: `Generate filer.toml with all possible configurations for you to customize.
  13. The options can also be overwritten by environment variables.
  14. For example, the filer.toml mysql password can be overwritten by environment variable
  15. export WEED_MYSQL_PASSWORD=some_password
  16. Environment variable rules:
  17. * Prefix the variable name with "WEED_"
  18. * Upppercase the reset of variable name.
  19. * Replace '.' with '_'
  20. `,
  21. }
  22. var (
  23. outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory")
  24. config = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security|master] the configuration file to generate")
  25. )
  26. func runScaffold(cmd *Command, args []string) bool {
  27. content := ""
  28. switch *config {
  29. case "filer":
  30. content = FILER_TOML_EXAMPLE
  31. case "notification":
  32. content = NOTIFICATION_TOML_EXAMPLE
  33. case "replication":
  34. content = REPLICATION_TOML_EXAMPLE
  35. case "security":
  36. content = SECURITY_TOML_EXAMPLE
  37. case "master":
  38. content = MASTER_TOML_EXAMPLE
  39. case "shell":
  40. content = SHELL_TOML_EXAMPLE
  41. }
  42. if content == "" {
  43. println("need a valid -config option")
  44. return false
  45. }
  46. if *outputPath != "" {
  47. ioutil.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
  48. } else {
  49. println(content)
  50. }
  51. return true
  52. }
  53. const (
  54. FILER_TOML_EXAMPLE = `
  55. # A sample TOML config file for SeaweedFS filer store
  56. # Used with "weed filer" or "weed server -filer"
  57. # Put this file to one of the location, with descending priority
  58. # ./filer.toml
  59. # $HOME/.seaweedfs/filer.toml
  60. # /etc/seaweedfs/filer.toml
  61. ####################################################
  62. # Customizable filer server options
  63. ####################################################
  64. [filer.options]
  65. # with http DELETE, by default the filer would check whether a folder is empty.
  66. # recursive_delete will delete all sub folders and files, similar to "rm -Rf"
  67. recursive_delete = false
  68. # directories under this folder will be automatically creating a separate bucket
  69. buckets_folder = "/buckets"
  70. ####################################################
  71. # The following are filer store options
  72. ####################################################
  73. [leveldb2]
  74. # local on disk, mostly for simple single-machine setup, fairly scalable
  75. # faster than previous leveldb, recommended.
  76. enabled = true
  77. dir = "./filerldb2" # directory to store level db files
  78. [rocksdb]
  79. # local on disk, similar to leveldb
  80. # since it is using a C wrapper, you need to install rocksdb and build it by yourself
  81. enabled = false
  82. dir = "./filerrdb" # directory to store rocksdb files
  83. [mysql] # or tidb
  84. # CREATE TABLE IF NOT EXISTS filemeta (
  85. # dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
  86. # name VARCHAR(1000) COMMENT 'directory or file name',
  87. # directory TEXT COMMENT 'full path to parent directory',
  88. # meta LONGBLOB,
  89. # PRIMARY KEY (dirhash, name)
  90. # ) DEFAULT CHARSET=utf8;
  91. enabled = false
  92. hostname = "localhost"
  93. port = 3306
  94. username = "root"
  95. password = ""
  96. database = "" # create or use an existing database
  97. connection_max_idle = 2
  98. connection_max_open = 100
  99. interpolateParams = false
  100. [postgres] # or cockroachdb
  101. # CREATE TABLE IF NOT EXISTS filemeta (
  102. # dirhash BIGINT,
  103. # name VARCHAR(65535),
  104. # directory VARCHAR(65535),
  105. # meta bytea,
  106. # PRIMARY KEY (dirhash, name)
  107. # );
  108. enabled = false
  109. hostname = "localhost"
  110. port = 5432
  111. username = "postgres"
  112. password = ""
  113. database = "" # create or use an existing database
  114. sslmode = "disable"
  115. connection_max_idle = 100
  116. connection_max_open = 100
  117. [cassandra]
  118. # CREATE TABLE filemeta (
  119. # directory varchar,
  120. # name varchar,
  121. # meta blob,
  122. # PRIMARY KEY (directory, name)
  123. # ) WITH CLUSTERING ORDER BY (name ASC);
  124. enabled = false
  125. keyspace="seaweedfs"
  126. hosts=[
  127. "localhost:9042",
  128. ]
  129. username=""
  130. password=""
  131. # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
  132. superLargeDirectories = []
  133. [hbase]
  134. enabled = false
  135. zkquorum = ""
  136. table = "seaweedfs"
  137. [redis2]
  138. enabled = false
  139. address = "localhost:6379"
  140. password = ""
  141. database = 0
  142. # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
  143. superLargeDirectories = []
  144. [redis_cluster2]
  145. enabled = false
  146. addresses = [
  147. "localhost:30001",
  148. "localhost:30002",
  149. "localhost:30003",
  150. "localhost:30004",
  151. "localhost:30005",
  152. "localhost:30006",
  153. ]
  154. password = ""
  155. # allows reads from slave servers or the master, but all writes still go to the master
  156. readOnly = true
  157. # automatically use the closest Redis server for reads
  158. routeByLatency = true
  159. # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
  160. superLargeDirectories = []
  161. [etcd]
  162. enabled = false
  163. servers = "localhost:2379"
  164. timeout = "3s"
  165. [mongodb]
  166. enabled = false
  167. uri = "mongodb://localhost:27017"
  168. option_pool_size = 0
  169. database = "seaweedfs"
  170. [elastic7]
  171. enabled = false
  172. servers = [
  173. "http://localhost1:9200",
  174. "http://localhost2:9200",
  175. "http://localhost3:9200",
  176. ]
  177. username = ""
  178. password = ""
  179. sniff_enabled = false
  180. healthcheck_enabled = false
  181. # increase the value is recommend, be sure the value in Elastic is greater or equal here
  182. index.max_result_window = 10000
  183. ##########################
  184. ##########################
  185. # To add path-specific filer store:
  186. #
  187. # 1. Add a name following the store type separated by a dot ".". E.g., cassandra.tmp
  188. # 2. Add a location configuraiton. E.g., location = "/tmp/"
  189. # 3. Copy and customize all other configurations.
  190. # Make sure they are not the same if using the same store type!
  191. # 4. Set enabled to true
  192. #
  193. # The following is just using cassandra as an example
  194. ##########################
  195. [redis2.tmp]
  196. enabled = false
  197. location = "/tmp/"
  198. address = "localhost:6379"
  199. password = ""
  200. database = 1
  201. `
  202. NOTIFICATION_TOML_EXAMPLE = `
  203. # A sample TOML config file for SeaweedFS filer store
  204. # Used by both "weed filer" or "weed server -filer" and "weed filer.replicate"
  205. # Put this file to one of the location, with descending priority
  206. # ./notification.toml
  207. # $HOME/.seaweedfs/notification.toml
  208. # /etc/seaweedfs/notification.toml
  209. ####################################################
  210. # notification
  211. # send and receive filer updates for each file to an external message queue
  212. ####################################################
  213. [notification.log]
  214. # this is only for debugging perpose and does not work with "weed filer.replicate"
  215. enabled = false
  216. [notification.kafka]
  217. enabled = false
  218. hosts = [
  219. "localhost:9092"
  220. ]
  221. topic = "seaweedfs_filer"
  222. offsetFile = "./last.offset"
  223. offsetSaveIntervalSeconds = 10
  224. [notification.aws_sqs]
  225. # experimental, let me know if it works
  226. enabled = false
  227. aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  228. aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  229. region = "us-east-2"
  230. sqs_queue_name = "my_filer_queue" # an existing queue name
  231. [notification.google_pub_sub]
  232. # read credentials doc at https://cloud.google.com/docs/authentication/getting-started
  233. enabled = false
  234. google_application_credentials = "/path/to/x.json" # path to json credential file
  235. project_id = "" # an existing project id
  236. topic = "seaweedfs_filer_topic" # a topic, auto created if does not exists
  237. [notification.gocdk_pub_sub]
  238. # The Go Cloud Development Kit (https://gocloud.dev).
  239. # PubSub API (https://godoc.org/gocloud.dev/pubsub).
  240. # Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ.
  241. enabled = false
  242. # This URL will Dial the RabbitMQ server at the URL in the environment
  243. # variable RABBIT_SERVER_URL and open the exchange "myexchange".
  244. # The exchange must have already been created by some other means, like
  245. # the RabbitMQ management plugin.
  246. topic_url = "rabbit://myexchange"
  247. sub_url = "rabbit://myqueue"
  248. `
  249. REPLICATION_TOML_EXAMPLE = `
  250. # A sample TOML config file for replicating SeaweedFS filer
  251. # Used with "weed filer.replicate"
  252. # Put this file to one of the location, with descending priority
  253. # ./replication.toml
  254. # $HOME/.seaweedfs/replication.toml
  255. # /etc/seaweedfs/replication.toml
  256. [source.filer]
  257. enabled = true
  258. grpcAddress = "localhost:18888"
  259. # all files under this directory tree are replicated.
  260. # this is not a directory on your hard drive, but on your filer.
  261. # i.e., all files with this "prefix" are sent to notification message queue.
  262. directory = "/buckets"
  263. [sink.filer]
  264. enabled = false
  265. grpcAddress = "localhost:18888"
  266. # all replicated files are under this directory tree
  267. # this is not a directory on your hard drive, but on your filer.
  268. # i.e., all received files will be "prefixed" to this directory.
  269. directory = "/backup"
  270. replication = ""
  271. collection = ""
  272. ttlSec = 0
  273. [sink.s3]
  274. # read credentials doc at https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sessions.html
  275. # default loads credentials from the shared credentials file (~/.aws/credentials).
  276. enabled = false
  277. aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  278. aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  279. region = "us-east-2"
  280. bucket = "your_bucket_name" # an existing bucket
  281. directory = "/" # destination directory
  282. endpoint = ""
  283. [sink.google_cloud_storage]
  284. # read credentials doc at https://cloud.google.com/docs/authentication/getting-started
  285. enabled = false
  286. google_application_credentials = "/path/to/x.json" # path to json credential file
  287. bucket = "your_bucket_seaweedfs" # an existing bucket
  288. directory = "/" # destination directory
  289. [sink.azure]
  290. # experimental, let me know if it works
  291. enabled = false
  292. account_name = ""
  293. account_key = ""
  294. container = "mycontainer" # an existing container
  295. directory = "/" # destination directory
  296. [sink.backblaze]
  297. enabled = false
  298. b2_account_id = ""
  299. b2_master_application_key = ""
  300. bucket = "mybucket" # an existing bucket
  301. directory = "/" # destination directory
  302. `
  303. SECURITY_TOML_EXAMPLE = `
  304. # Put this file to one of the location, with descending priority
  305. # ./security.toml
  306. # $HOME/.seaweedfs/security.toml
  307. # /etc/seaweedfs/security.toml
  308. # this file is read by master, volume server, and filer
  309. # the jwt signing key is read by master and volume server.
  310. # a jwt defaults to expire after 10 seconds.
  311. [jwt.signing]
  312. key = ""
  313. expires_after_seconds = 10 # seconds
  314. # jwt for read is only supported with master+volume setup. Filer does not support this mode.
  315. [jwt.signing.read]
  316. key = ""
  317. expires_after_seconds = 10 # seconds
  318. # all grpc tls authentications are mutual
  319. # the values for the following ca, cert, and key are paths to the PERM files.
  320. # the host name is not checked, so the PERM files can be shared.
  321. [grpc]
  322. ca = ""
  323. [grpc.volume]
  324. cert = ""
  325. key = ""
  326. [grpc.master]
  327. cert = ""
  328. key = ""
  329. [grpc.filer]
  330. cert = ""
  331. key = ""
  332. [grpc.msg_broker]
  333. cert = ""
  334. key = ""
  335. # use this for any place needs a grpc client
  336. # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
  337. [grpc.client]
  338. cert = ""
  339. key = ""
  340. # volume server https options
  341. # Note: work in progress!
  342. # this does not work with other clients, e.g., "weed filer|mount" etc, yet.
  343. [https.client]
  344. enabled = true
  345. [https.volume]
  346. cert = ""
  347. key = ""
  348. `
  349. MASTER_TOML_EXAMPLE = `
  350. # Put this file to one of the location, with descending priority
  351. # ./master.toml
  352. # $HOME/.seaweedfs/master.toml
  353. # /etc/seaweedfs/master.toml
  354. # this file is read by master
  355. [master.maintenance]
  356. # periodically run these scripts are the same as running them from 'weed shell'
  357. scripts = """
  358. lock
  359. ec.encode -fullPercent=95 -quietFor=1h
  360. ec.rebuild -force
  361. ec.balance -force
  362. volume.balance -force
  363. volume.fix.replication
  364. unlock
  365. """
  366. sleep_minutes = 17 # sleep minutes between each script execution
  367. [master.filer]
  368. default = "localhost:8888" # used by maintenance scripts if the scripts needs to use fs related commands
  369. [master.sequencer]
  370. type = "raft" # Choose [raft|etcd] type for storing the file id sequence
  371. # when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence
  372. # example : http://127.0.0.1:2379,http://127.0.0.1:2389
  373. sequencer_etcd_urls = "http://127.0.0.1:2379"
  374. # configurations for tiered cloud storage
  375. # old volumes are transparently moved to cloud for cost efficiency
  376. [storage.backend]
  377. [storage.backend.s3.default]
  378. enabled = false
  379. aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  380. aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  381. region = "us-east-2"
  382. bucket = "your_bucket_name" # an existing bucket
  383. endpoint = ""
  384. # create this number of logical volumes if no more writable volumes
  385. # count_x means how many copies of data.
  386. # e.g.:
  387. # 000 has only one copy, copy_1
  388. # 010 and 001 has two copies, copy_2
  389. # 011 has only 3 copies, copy_3
  390. [master.volume_growth]
  391. copy_1 = 7 # create 1 x 7 = 7 actual volumes
  392. copy_2 = 6 # create 2 x 6 = 12 actual volumes
  393. copy_3 = 3 # create 3 x 3 = 9 actual volumes
  394. copy_other = 1 # create n x 1 = n actual volumes
  395. # configuration flags for replication
  396. [master.replication]
  397. # any replication counts should be considered minimums. If you specify 010 and
  398. # have 3 different racks, that's still considered writable. Writes will still
  399. # try to replicate to all available volumes. You should only use this option
  400. # if you are doing your own replication or periodic sync of volumes.
  401. treat_replication_as_minimums = false
  402. `
  403. SHELL_TOML_EXAMPLE = `
  404. [cluster]
  405. default = "c1"
  406. [cluster.c1]
  407. master = "localhost:9333" # comma-separated master servers
  408. filer = "localhost:8888" # filer host and port
  409. [cluster.c2]
  410. master = ""
  411. filer = ""
  412. `
  413. )