You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

573 lines
17 KiB

5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
  1. package command
  2. import (
  3. "io/ioutil"
  4. "path/filepath"
  5. )
  6. func init() {
  7. cmdScaffold.Run = runScaffold // break init cycle
  8. }
  9. var cmdScaffold = &Command{
  10. UsageLine: "scaffold -config=[filer|notification|replication|security|master]",
  11. Short: "generate basic configuration files",
  12. Long: `Generate filer.toml with all possible configurations for you to customize.
  13. The options can also be overwritten by environment variables.
  14. For example, the filer.toml mysql password can be overwritten by environment variable
  15. export WEED_MYSQL_PASSWORD=some_password
  16. Environment variable rules:
  17. * Prefix the variable name with "WEED_"
  18. * Upppercase the reset of variable name.
  19. * Replace '.' with '_'
  20. `,
  21. }
  22. var (
  23. outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory")
  24. config = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security|master] the configuration file to generate")
  25. )
  26. func runScaffold(cmd *Command, args []string) bool {
  27. content := ""
  28. switch *config {
  29. case "filer":
  30. content = FILER_TOML_EXAMPLE
  31. case "notification":
  32. content = NOTIFICATION_TOML_EXAMPLE
  33. case "replication":
  34. content = REPLICATION_TOML_EXAMPLE
  35. case "security":
  36. content = SECURITY_TOML_EXAMPLE
  37. case "master":
  38. content = MASTER_TOML_EXAMPLE
  39. case "shell":
  40. content = SHELL_TOML_EXAMPLE
  41. }
  42. if content == "" {
  43. println("need a valid -config option")
  44. return false
  45. }
  46. if *outputPath != "" {
  47. ioutil.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
  48. } else {
  49. println(content)
  50. }
  51. return true
  52. }
  53. const (
  54. FILER_TOML_EXAMPLE = `
  55. # A sample TOML config file for SeaweedFS filer store
  56. # Used with "weed filer" or "weed server -filer"
  57. # Put this file to one of the location, with descending priority
  58. # ./filer.toml
  59. # $HOME/.seaweedfs/filer.toml
  60. # /etc/seaweedfs/filer.toml
  61. ####################################################
  62. # Customizable filer server options
  63. ####################################################
  64. [filer.options]
  65. # with http DELETE, by default the filer would check whether a folder is empty.
  66. # recursive_delete will delete all sub folders and files, similar to "rm -Rf"
  67. recursive_delete = false
  68. # directories under this folder will be automatically creating a separate bucket
  69. buckets_folder = "/buckets"
  70. ####################################################
  71. # The following are filer store options
  72. ####################################################
  73. [leveldb2]
  74. # local on disk, mostly for simple single-machine setup, fairly scalable
  75. # faster than previous leveldb, recommended.
  76. enabled = true
  77. dir = "./filerldb2" # directory to store level db files
  78. [leveldb3]
  79. # similar to leveldb2.
  80. # each bucket has its own meta store.
  81. enabled = false
  82. dir = "./filerldb3" # directory to store level db files
  83. [rocksdb]
  84. # local on disk, similar to leveldb
  85. # since it is using a C wrapper, you need to install rocksdb and build it by yourself
  86. enabled = false
  87. dir = "./filerrdb" # directory to store rocksdb files
  88. [sqlite]
  89. # local on disk, similar to leveldb
  90. enabled = false
  91. dbFile = "./filer.db" # sqlite db file
  92. [mysql] # or memsql, tidb
  93. # CREATE TABLE IF NOT EXISTS filemeta (
  94. # dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
  95. # name VARCHAR(1000) BINARY COMMENT 'directory or file name',
  96. # directory TEXT COMMENT 'full path to parent directory',
  97. # meta LONGBLOB,
  98. # PRIMARY KEY (dirhash, name)
  99. # ) DEFAULT CHARSET=utf8;
  100. enabled = false
  101. hostname = "localhost"
  102. port = 3306
  103. username = "root"
  104. password = ""
  105. database = "" # create or use an existing database
  106. connection_max_idle = 2
  107. connection_max_open = 100
  108. connection_max_lifetime_seconds = 0
  109. interpolateParams = false
  110. # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
  111. enableUpsert = true
  112. upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
  113. [mysql2] # or memsql, tidb
  114. enabled = false
  115. createTable = """
  116. CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` (
  117. dirhash BIGINT,
  118. name VARCHAR(1000) BINARY,
  119. directory TEXT,
  120. meta LONGBLOB,
  121. PRIMARY KEY (dirhash, name)
  122. ) DEFAULT CHARSET=utf8;
  123. """
  124. hostname = "localhost"
  125. port = 3306
  126. username = "root"
  127. password = ""
  128. database = "" # create or use an existing database
  129. connection_max_idle = 2
  130. connection_max_open = 100
  131. connection_max_lifetime_seconds = 0
  132. interpolateParams = false
  133. # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
  134. enableUpsert = true
  135. upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
  136. [postgres] # or cockroachdb, YugabyteDB
  137. # CREATE TABLE IF NOT EXISTS filemeta (
  138. # dirhash BIGINT,
  139. # name VARCHAR(65535),
  140. # directory VARCHAR(65535),
  141. # meta bytea,
  142. # PRIMARY KEY (dirhash, name)
  143. # );
  144. enabled = false
  145. hostname = "localhost"
  146. port = 5432
  147. username = "postgres"
  148. password = ""
  149. database = "postgres" # create or use an existing database
  150. schema = ""
  151. sslmode = "disable"
  152. connection_max_idle = 100
  153. connection_max_open = 100
  154. connection_max_lifetime_seconds = 0
  155. # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
  156. enableUpsert = true
  157. upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
  158. [postgres2]
  159. enabled = false
  160. createTable = """
  161. CREATE TABLE IF NOT EXISTS "%s" (
  162. dirhash BIGINT,
  163. name VARCHAR(65535),
  164. directory VARCHAR(65535),
  165. meta bytea,
  166. PRIMARY KEY (dirhash, name)
  167. );
  168. """
  169. hostname = "localhost"
  170. port = 5432
  171. username = "postgres"
  172. password = ""
  173. database = "postgres" # create or use an existing database
  174. schema = ""
  175. sslmode = "disable"
  176. connection_max_idle = 100
  177. connection_max_open = 100
  178. connection_max_lifetime_seconds = 0
  179. # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
  180. enableUpsert = true
  181. upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
  182. [cassandra]
  183. # CREATE TABLE filemeta (
  184. # directory varchar,
  185. # name varchar,
  186. # meta blob,
  187. # PRIMARY KEY (directory, name)
  188. # ) WITH CLUSTERING ORDER BY (name ASC);
  189. enabled = false
  190. keyspace="seaweedfs"
  191. hosts=[
  192. "localhost:9042",
  193. ]
  194. username=""
  195. password=""
  196. # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
  197. superLargeDirectories = []
  198. [hbase]
  199. enabled = false
  200. zkquorum = ""
  201. table = "seaweedfs"
  202. [redis2]
  203. enabled = false
  204. address = "localhost:6379"
  205. password = ""
  206. database = 0
  207. # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
  208. superLargeDirectories = []
  209. [redis_cluster2]
  210. enabled = false
  211. addresses = [
  212. "localhost:30001",
  213. "localhost:30002",
  214. "localhost:30003",
  215. "localhost:30004",
  216. "localhost:30005",
  217. "localhost:30006",
  218. ]
  219. password = ""
  220. # allows reads from slave servers or the master, but all writes still go to the master
  221. readOnly = false
  222. # automatically use the closest Redis server for reads
  223. routeByLatency = false
  224. # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
  225. superLargeDirectories = []
  226. [etcd]
  227. enabled = false
  228. servers = "localhost:2379"
  229. timeout = "3s"
  230. [mongodb]
  231. enabled = false
  232. uri = "mongodb://localhost:27017"
  233. option_pool_size = 0
  234. database = "seaweedfs"
  235. [elastic7]
  236. enabled = false
  237. servers = [
  238. "http://localhost1:9200",
  239. "http://localhost2:9200",
  240. "http://localhost3:9200",
  241. ]
  242. username = ""
  243. password = ""
  244. sniff_enabled = false
  245. healthcheck_enabled = false
  246. # increase the value is recommend, be sure the value in Elastic is greater or equal here
  247. index.max_result_window = 10000
  248. ##########################
  249. ##########################
  250. # To add path-specific filer store:
  251. #
  252. # 1. Add a name following the store type separated by a dot ".". E.g., cassandra.tmp
  253. # 2. Add a location configuraiton. E.g., location = "/tmp/"
  254. # 3. Copy and customize all other configurations.
  255. # Make sure they are not the same if using the same store type!
  256. # 4. Set enabled to true
  257. #
  258. # The following is just using redis as an example
  259. ##########################
  260. [redis2.tmp]
  261. enabled = false
  262. location = "/tmp/"
  263. address = "localhost:6379"
  264. password = ""
  265. database = 1
  266. `
  267. NOTIFICATION_TOML_EXAMPLE = `
  268. # A sample TOML config file for SeaweedFS filer store
  269. # Used by both "weed filer" or "weed server -filer" and "weed filer.replicate"
  270. # Put this file to one of the location, with descending priority
  271. # ./notification.toml
  272. # $HOME/.seaweedfs/notification.toml
  273. # /etc/seaweedfs/notification.toml
  274. ####################################################
  275. # notification
  276. # send and receive filer updates for each file to an external message queue
  277. ####################################################
  278. [notification.log]
  279. # this is only for debugging perpose and does not work with "weed filer.replicate"
  280. enabled = false
  281. [notification.kafka]
  282. enabled = false
  283. hosts = [
  284. "localhost:9092"
  285. ]
  286. topic = "seaweedfs_filer"
  287. offsetFile = "./last.offset"
  288. offsetSaveIntervalSeconds = 10
  289. [notification.aws_sqs]
  290. # experimental, let me know if it works
  291. enabled = false
  292. aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  293. aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  294. region = "us-east-2"
  295. sqs_queue_name = "my_filer_queue" # an existing queue name
  296. [notification.google_pub_sub]
  297. # read credentials doc at https://cloud.google.com/docs/authentication/getting-started
  298. enabled = false
  299. google_application_credentials = "/path/to/x.json" # path to json credential file
  300. project_id = "" # an existing project id
  301. topic = "seaweedfs_filer_topic" # a topic, auto created if does not exists
  302. [notification.gocdk_pub_sub]
  303. # The Go Cloud Development Kit (https://gocloud.dev).
  304. # PubSub API (https://godoc.org/gocloud.dev/pubsub).
  305. # Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ.
  306. enabled = false
  307. # This URL will Dial the RabbitMQ server at the URL in the environment
  308. # variable RABBIT_SERVER_URL and open the exchange "myexchange".
  309. # The exchange must have already been created by some other means, like
  310. # the RabbitMQ management plugin. Сreate myexchange of type fanout and myqueue then
  311. # create binding myexchange => myqueue
  312. topic_url = "rabbit://myexchange"
  313. sub_url = "rabbit://myqueue"
  314. `
  315. REPLICATION_TOML_EXAMPLE = `
  316. # A sample TOML config file for replicating SeaweedFS filer
  317. # Used with "weed filer.backup"
  318. # Using with "weed filer.replicate" is deprecated.
  319. # Put this file to one of the location, with descending priority
  320. # ./replication.toml
  321. # $HOME/.seaweedfs/replication.toml
  322. # /etc/seaweedfs/replication.toml
  323. [source.filer] # deprecated. Only useful with "weed filer.replicate"
  324. enabled = true
  325. grpcAddress = "localhost:18888"
  326. # all files under this directory tree are replicated.
  327. # this is not a directory on your hard drive, but on your filer.
  328. # i.e., all files with this "prefix" are sent to notification message queue.
  329. directory = "/buckets"
  330. [sink.local]
  331. enabled = false
  332. directory = "/data"
  333. # all replicated files are under modified time as yyyy-mm-dd directories
  334. # so each date directory contains all new and updated files.
  335. is_incremental = false
  336. [sink.filer]
  337. enabled = false
  338. grpcAddress = "localhost:18888"
  339. # all replicated files are under this directory tree
  340. # this is not a directory on your hard drive, but on your filer.
  341. # i.e., all received files will be "prefixed" to this directory.
  342. directory = "/backup"
  343. replication = ""
  344. collection = ""
  345. ttlSec = 0
  346. is_incremental = false
  347. [sink.s3]
  348. # read credentials doc at https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sessions.html
  349. # default loads credentials from the shared credentials file (~/.aws/credentials).
  350. enabled = false
  351. aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  352. aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  353. region = "us-east-2"
  354. bucket = "your_bucket_name" # an existing bucket
  355. directory = "/" # destination directory
  356. endpoint = ""
  357. is_incremental = false
  358. [sink.google_cloud_storage]
  359. # read credentials doc at https://cloud.google.com/docs/authentication/getting-started
  360. enabled = false
  361. google_application_credentials = "/path/to/x.json" # path to json credential file
  362. bucket = "your_bucket_seaweedfs" # an existing bucket
  363. directory = "/" # destination directory
  364. is_incremental = false
  365. [sink.azure]
  366. # experimental, let me know if it works
  367. enabled = false
  368. account_name = ""
  369. account_key = ""
  370. container = "mycontainer" # an existing container
  371. directory = "/" # destination directory
  372. is_incremental = false
  373. [sink.backblaze]
  374. enabled = false
  375. b2_account_id = ""
  376. b2_master_application_key = ""
  377. bucket = "mybucket" # an existing bucket
  378. directory = "/" # destination directory
  379. is_incremental = false
  380. `
  381. SECURITY_TOML_EXAMPLE = `
  382. # Put this file to one of the location, with descending priority
  383. # ./security.toml
  384. # $HOME/.seaweedfs/security.toml
  385. # /etc/seaweedfs/security.toml
  386. # this file is read by master, volume server, and filer
  387. # the jwt signing key is read by master and volume server.
  388. # a jwt defaults to expire after 10 seconds.
  389. [jwt.signing]
  390. key = ""
  391. expires_after_seconds = 10 # seconds
  392. # jwt for read is only supported with master+volume setup. Filer does not support this mode.
  393. [jwt.signing.read]
  394. key = ""
  395. expires_after_seconds = 10 # seconds
  396. # all grpc tls authentications are mutual
  397. # the values for the following ca, cert, and key are paths to the PERM files.
  398. # the host name is not checked, so the PERM files can be shared.
  399. [grpc]
  400. ca = ""
  401. # Set wildcard domain for enable TLS authentication by common names
  402. allowed_wildcard_domain = "" # .mycompany.com
  403. [grpc.volume]
  404. cert = ""
  405. key = ""
  406. allowed_commonNames = "" # comma-separated SSL certificate common names
  407. [grpc.master]
  408. cert = ""
  409. key = ""
  410. allowed_commonNames = "" # comma-separated SSL certificate common names
  411. [grpc.filer]
  412. cert = ""
  413. key = ""
  414. allowed_commonNames = "" # comma-separated SSL certificate common names
  415. [grpc.msg_broker]
  416. cert = ""
  417. key = ""
  418. allowed_commonNames = "" # comma-separated SSL certificate common names
  419. # use this for any place needs a grpc client
  420. # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
  421. [grpc.client]
  422. cert = ""
  423. key = ""
  424. # volume server https options
  425. # Note: work in progress!
  426. # this does not work with other clients, e.g., "weed filer|mount" etc, yet.
  427. [https.client]
  428. enabled = true
  429. [https.volume]
  430. cert = ""
  431. key = ""
  432. `
  433. MASTER_TOML_EXAMPLE = `
  434. # Put this file to one of the location, with descending priority
  435. # ./master.toml
  436. # $HOME/.seaweedfs/master.toml
  437. # /etc/seaweedfs/master.toml
  438. # this file is read by master
  439. [master.maintenance]
  440. # periodically run these scripts are the same as running them from 'weed shell'
  441. scripts = """
  442. lock
  443. ec.encode -fullPercent=95 -quietFor=1h
  444. ec.rebuild -force
  445. ec.balance -force
  446. volume.balance -force
  447. volume.fix.replication
  448. unlock
  449. """
  450. sleep_minutes = 17 # sleep minutes between each script execution
  451. [master.filer]
  452. default = "localhost:8888" # used by maintenance scripts if the scripts needs to use fs related commands
  453. [master.sequencer]
  454. type = "raft" # Choose [raft|etcd|snowflake] type for storing the file id sequence
  455. # when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence
  456. # example : http://127.0.0.1:2379,http://127.0.0.1:2389
  457. sequencer_etcd_urls = "http://127.0.0.1:2379"
  458. # configurations for tiered cloud storage
  459. # old volumes are transparently moved to cloud for cost efficiency
  460. [storage.backend]
  461. [storage.backend.s3.default]
  462. enabled = false
  463. aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  464. aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
  465. region = "us-east-2"
  466. bucket = "your_bucket_name" # an existing bucket
  467. endpoint = ""
  468. # create this number of logical volumes if no more writable volumes
  469. # count_x means how many copies of data.
  470. # e.g.:
  471. # 000 has only one copy, copy_1
  472. # 010 and 001 has two copies, copy_2
  473. # 011 has only 3 copies, copy_3
  474. [master.volume_growth]
  475. copy_1 = 7 # create 1 x 7 = 7 actual volumes
  476. copy_2 = 6 # create 2 x 6 = 12 actual volumes
  477. copy_3 = 3 # create 3 x 3 = 9 actual volumes
  478. copy_other = 1 # create n x 1 = n actual volumes
  479. # configuration flags for replication
  480. [master.replication]
  481. # any replication counts should be considered minimums. If you specify 010 and
  482. # have 3 different racks, that's still considered writable. Writes will still
  483. # try to replicate to all available volumes. You should only use this option
  484. # if you are doing your own replication or periodic sync of volumes.
  485. treat_replication_as_minimums = false
  486. `
  487. SHELL_TOML_EXAMPLE = `
  488. [cluster]
  489. default = "c1"
  490. [cluster.c1]
  491. master = "localhost:9333" # comma-separated master servers
  492. filer = "localhost:8888" # filer host and port
  493. [cluster.c2]
  494. master = ""
  495. filer = ""
  496. `
  497. )