diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 8e940f608..b48cf0959 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -1,6 +1,7 @@ package command import ( + "github.com/chrislusf/seaweedfs/weed/command/scaffold" "io/ioutil" "path/filepath" ) @@ -35,17 +36,17 @@ func runScaffold(cmd *Command, args []string) bool { content := "" switch *config { case "filer": - content = FILER_TOML_EXAMPLE + content = scaffold.Filer case "notification": - content = NOTIFICATION_TOML_EXAMPLE + content = scaffold.Notification case "replication": - content = REPLICATION_TOML_EXAMPLE + content = scaffold.Replication case "security": - content = SECURITY_TOML_EXAMPLE + content = scaffold.Security case "master": - content = MASTER_TOML_EXAMPLE + content = scaffold.Master case "shell": - content = SHELL_TOML_EXAMPLE + content = scaffold.Shell } if content == "" { println("need a valid -config option") @@ -59,517 +60,3 @@ func runScaffold(cmd *Command, args []string) bool { } return true } - -const ( - FILER_TOML_EXAMPLE = ` -# A sample TOML config file for SeaweedFS filer store -# Used with "weed filer" or "weed server -filer" -# Put this file to one of the location, with descending priority -# ./filer.toml -# $HOME/.seaweedfs/filer.toml -# /etc/seaweedfs/filer.toml - -#################################################### -# Customizable filer server options -#################################################### -[filer.options] -# with http DELETE, by default the filer would check whether a folder is empty. -# recursive_delete will delete all sub folders and files, similar to "rm -Rf" -recursive_delete = false -# directories under this folder will be automatically creating a separate bucket -buckets_folder = "/buckets" - -#################################################### -# The following are filer store options -#################################################### - -[leveldb2] -# local on disk, mostly for simple single-machine setup, fairly scalable -# faster than previous leveldb, recommended. -enabled = true -dir = "./filerldb2" # directory to store level db files - -[leveldb3] -# similar to leveldb2. -# each bucket has its own meta store. -enabled = false -dir = "./filerldb3" # directory to store level db files - -[rocksdb] -# local on disk, similar to leveldb -# since it is using a C wrapper, you need to install rocksdb and build it by yourself -enabled = false -dir = "./filerrdb" # directory to store rocksdb files - -[sqlite] -# local on disk, similar to leveldb -enabled = false -dbFile = "./filer.db" # sqlite db file - -[mysql] # or memsql, tidb -# CREATE TABLE IF NOT EXISTS filemeta ( -# dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field', -# name VARCHAR(1000) BINARY COMMENT 'directory or file name', -# directory TEXT COMMENT 'full path to parent directory', -# meta LONGBLOB, -# PRIMARY KEY (dirhash, name) -# ) DEFAULT CHARSET=utf8; - -enabled = false -hostname = "localhost" -port = 3306 -username = "root" -password = "" -database = "" # create or use an existing database -connection_max_idle = 2 -connection_max_open = 100 -connection_max_lifetime_seconds = 0 -interpolateParams = false -# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax: -enableUpsert = true -upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)""" - -[mysql2] # or memsql, tidb -enabled = false -createTable = """ - CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` ( - dirhash BIGINT, - name VARCHAR(1000) BINARY, - directory TEXT, - meta LONGBLOB, - PRIMARY KEY (dirhash, name) - ) DEFAULT CHARSET=utf8; -""" -hostname = "localhost" -port = 3306 -username = "root" -password = "" -database = "" # create or use an existing database -connection_max_idle = 2 -connection_max_open = 100 -connection_max_lifetime_seconds = 0 -interpolateParams = false -# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax: -enableUpsert = true -upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)""" - -[postgres] # or cockroachdb, YugabyteDB -# CREATE TABLE IF NOT EXISTS filemeta ( -# dirhash BIGINT, -# name VARCHAR(65535), -# directory VARCHAR(65535), -# meta bytea, -# PRIMARY KEY (dirhash, name) -# ); -enabled = false -hostname = "localhost" -port = 5432 -username = "postgres" -password = "" -database = "postgres" # create or use an existing database -schema = "" -sslmode = "disable" -connection_max_idle = 100 -connection_max_open = 100 -connection_max_lifetime_seconds = 0 -# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax: -enableUpsert = true -upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta""" - -[postgres2] -enabled = false -createTable = """ - CREATE TABLE IF NOT EXISTS "%s" ( - dirhash BIGINT, - name VARCHAR(65535), - directory VARCHAR(65535), - meta bytea, - PRIMARY KEY (dirhash, name) - ); -""" -hostname = "localhost" -port = 5432 -username = "postgres" -password = "" -database = "postgres" # create or use an existing database -schema = "" -sslmode = "disable" -connection_max_idle = 100 -connection_max_open = 100 -connection_max_lifetime_seconds = 0 -# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax: -enableUpsert = true -upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta""" - -[cassandra] -# CREATE TABLE filemeta ( -# directory varchar, -# name varchar, -# meta blob, -# PRIMARY KEY (directory, name) -# ) WITH CLUSTERING ORDER BY (name ASC); -enabled = false -keyspace="seaweedfs" -hosts=[ - "localhost:9042", -] -username="" -password="" -# This changes the data layout. Only add new directories. Removing/Updating will cause data loss. -superLargeDirectories = [] -# Name of the datacenter local to this filer, used as host selection fallback. -localDC="" - -[hbase] -enabled = false -zkquorum = "" -table = "seaweedfs" - -[redis2] -enabled = false -address = "localhost:6379" -password = "" -database = 0 -# This changes the data layout. Only add new directories. Removing/Updating will cause data loss. -superLargeDirectories = [] - -[redis_cluster2] -enabled = false -addresses = [ - "localhost:30001", - "localhost:30002", - "localhost:30003", - "localhost:30004", - "localhost:30005", - "localhost:30006", -] -password = "" -# allows reads from slave servers or the master, but all writes still go to the master -readOnly = false -# automatically use the closest Redis server for reads -routeByLatency = false -# This changes the data layout. Only add new directories. Removing/Updating will cause data loss. -superLargeDirectories = [] - -[etcd] -enabled = false -servers = "localhost:2379" -timeout = "3s" - -[mongodb] -enabled = false -uri = "mongodb://localhost:27017" -option_pool_size = 0 -database = "seaweedfs" - -[elastic7] -enabled = false -servers = [ - "http://localhost1:9200", - "http://localhost2:9200", - "http://localhost3:9200", -] -username = "" -password = "" -sniff_enabled = false -healthcheck_enabled = false -# increase the value is recommend, be sure the value in Elastic is greater or equal here -index.max_result_window = 10000 - - - -########################## -########################## -# To add path-specific filer store: -# -# 1. Add a name following the store type separated by a dot ".". E.g., cassandra.tmp -# 2. Add a location configuraiton. E.g., location = "/tmp/" -# 3. Copy and customize all other configurations. -# Make sure they are not the same if using the same store type! -# 4. Set enabled to true -# -# The following is just using redis as an example -########################## -[redis2.tmp] -enabled = false -location = "/tmp/" -address = "localhost:6379" -password = "" -database = 1 - -` - - NOTIFICATION_TOML_EXAMPLE = ` -# A sample TOML config file for SeaweedFS filer store -# Used by both "weed filer" or "weed server -filer" and "weed filer.replicate" -# Put this file to one of the location, with descending priority -# ./notification.toml -# $HOME/.seaweedfs/notification.toml -# /etc/seaweedfs/notification.toml - -#################################################### -# notification -# send and receive filer updates for each file to an external message queue -#################################################### -[notification.log] -# this is only for debugging perpose and does not work with "weed filer.replicate" -enabled = false - - -[notification.kafka] -enabled = false -hosts = [ - "localhost:9092" -] -topic = "seaweedfs_filer" -offsetFile = "./last.offset" -offsetSaveIntervalSeconds = 10 - - -[notification.aws_sqs] -# experimental, let me know if it works -enabled = false -aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials). -aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials). -region = "us-east-2" -sqs_queue_name = "my_filer_queue" # an existing queue name - - -[notification.google_pub_sub] -# read credentials doc at https://cloud.google.com/docs/authentication/getting-started -enabled = false -google_application_credentials = "/path/to/x.json" # path to json credential file -project_id = "" # an existing project id -topic = "seaweedfs_filer_topic" # a topic, auto created if does not exists - -[notification.gocdk_pub_sub] -# The Go Cloud Development Kit (https://gocloud.dev). -# PubSub API (https://godoc.org/gocloud.dev/pubsub). -# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ. -enabled = false -# This URL will Dial the RabbitMQ server at the URL in the environment -# variable RABBIT_SERVER_URL and open the exchange "myexchange". -# The exchange must have already been created by some other means, like -# the RabbitMQ management plugin. Сreate myexchange of type fanout and myqueue then -# create binding myexchange => myqueue -topic_url = "rabbit://myexchange" -sub_url = "rabbit://myqueue" -` - - REPLICATION_TOML_EXAMPLE = ` -# A sample TOML config file for replicating SeaweedFS filer -# Used with "weed filer.backup" -# Using with "weed filer.replicate" is deprecated. -# Put this file to one of the location, with descending priority -# ./replication.toml -# $HOME/.seaweedfs/replication.toml -# /etc/seaweedfs/replication.toml - -[source.filer] # deprecated. Only useful with "weed filer.replicate" -enabled = true -grpcAddress = "localhost:18888" -# all files under this directory tree are replicated. -# this is not a directory on your hard drive, but on your filer. -# i.e., all files with this "prefix" are sent to notification message queue. -directory = "/buckets" - -[sink.local] -enabled = false -directory = "/data" -# all replicated files are under modified time as yyyy-mm-dd directories -# so each date directory contains all new and updated files. -is_incremental = false - -[sink.filer] -enabled = false -grpcAddress = "localhost:18888" -# all replicated files are under this directory tree -# this is not a directory on your hard drive, but on your filer. -# i.e., all received files will be "prefixed" to this directory. -directory = "/backup" -replication = "" -collection = "" -ttlSec = 0 -is_incremental = false - -[sink.s3] -# read credentials doc at https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sessions.html -# default loads credentials from the shared credentials file (~/.aws/credentials). -enabled = false -aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials). -aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials). -region = "us-east-2" -bucket = "your_bucket_name" # an existing bucket -directory = "/" # destination directory -endpoint = "" -is_incremental = false - -[sink.google_cloud_storage] -# read credentials doc at https://cloud.google.com/docs/authentication/getting-started -enabled = false -google_application_credentials = "/path/to/x.json" # path to json credential file -bucket = "your_bucket_seaweedfs" # an existing bucket -directory = "/" # destination directory -is_incremental = false - -[sink.azure] -# experimental, let me know if it works -enabled = false -account_name = "" -account_key = "" -container = "mycontainer" # an existing container -directory = "/" # destination directory -is_incremental = false - -[sink.backblaze] -enabled = false -b2_account_id = "" -b2_master_application_key = "" -bucket = "mybucket" # an existing bucket -directory = "/" # destination directory -is_incremental = false - -` - - SECURITY_TOML_EXAMPLE = ` -# Put this file to one of the location, with descending priority -# ./security.toml -# $HOME/.seaweedfs/security.toml -# /etc/seaweedfs/security.toml -# this file is read by master, volume server, and filer - -# the jwt signing key is read by master and volume server. -# a jwt defaults to expire after 10 seconds. -[jwt.signing] -key = "" -expires_after_seconds = 10 # seconds - -# jwt for read is only supported with master+volume setup. Filer does not support this mode. -[jwt.signing.read] -key = "" -expires_after_seconds = 10 # seconds - -# all grpc tls authentications are mutual -# the values for the following ca, cert, and key are paths to the PERM files. -# the host name is not checked, so the PERM files can be shared. -[grpc] -ca = "" -# Set wildcard domain for enable TLS authentication by common names -allowed_wildcard_domain = "" # .mycompany.com - -[grpc.volume] -cert = "" -key = "" -allowed_commonNames = "" # comma-separated SSL certificate common names - -[grpc.master] -cert = "" -key = "" -allowed_commonNames = "" # comma-separated SSL certificate common names - -[grpc.filer] -cert = "" -key = "" -allowed_commonNames = "" # comma-separated SSL certificate common names - -[grpc.msg_broker] -cert = "" -key = "" -allowed_commonNames = "" # comma-separated SSL certificate common names - -# use this for any place needs a grpc client -# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload" -[grpc.client] -cert = "" -key = "" - -# volume server https options -# Note: work in progress! -# this does not work with other clients, e.g., "weed filer|mount" etc, yet. -[https.client] -enabled = true -[https.volume] -cert = "" -key = "" - - -` - - MASTER_TOML_EXAMPLE = ` -# Put this file to one of the location, with descending priority -# ./master.toml -# $HOME/.seaweedfs/master.toml -# /etc/seaweedfs/master.toml -# this file is read by master - -[master.maintenance] -# periodically run these scripts are the same as running them from 'weed shell' -scripts = """ - lock - ec.encode -fullPercent=95 -quietFor=1h - ec.rebuild -force - ec.balance -force - volume.balance -force - volume.fix.replication - unlock -""" -sleep_minutes = 17 # sleep minutes between each script execution - -[master.filer] -default = "localhost:8888" # used by maintenance scripts if the scripts needs to use fs related commands - - -[master.sequencer] -type = "raft" # Choose [raft|etcd|snowflake] type for storing the file id sequence -# when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence -# example : http://127.0.0.1:2379,http://127.0.0.1:2389 -sequencer_etcd_urls = "http://127.0.0.1:2379" - - -# configurations for tiered cloud storage -# old volumes are transparently moved to cloud for cost efficiency -[storage.backend] - [storage.backend.s3.default] - enabled = false - aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials). - aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials). - region = "us-east-2" - bucket = "your_bucket_name" # an existing bucket - endpoint = "" - -# create this number of logical volumes if no more writable volumes -# count_x means how many copies of data. -# e.g.: -# 000 has only one copy, copy_1 -# 010 and 001 has two copies, copy_2 -# 011 has only 3 copies, copy_3 -[master.volume_growth] -copy_1 = 7 # create 1 x 7 = 7 actual volumes -copy_2 = 6 # create 2 x 6 = 12 actual volumes -copy_3 = 3 # create 3 x 3 = 9 actual volumes -copy_other = 1 # create n x 1 = n actual volumes - -# configuration flags for replication -[master.replication] -# any replication counts should be considered minimums. If you specify 010 and -# have 3 different racks, that's still considered writable. Writes will still -# try to replicate to all available volumes. You should only use this option -# if you are doing your own replication or periodic sync of volumes. -treat_replication_as_minimums = false - -` - SHELL_TOML_EXAMPLE = ` - -[cluster] -default = "c1" - -[cluster.c1] -master = "localhost:9333" # comma-separated master servers -filer = "localhost:8888" # filer host and port - -[cluster.c2] -master = "" -filer = "" - -` -) diff --git a/weed/command/scaffold/example.go b/weed/command/scaffold/example.go new file mode 100644 index 000000000..6be6804e5 --- /dev/null +++ b/weed/command/scaffold/example.go @@ -0,0 +1,21 @@ +package scaffold + +import _ "embed" + +//go:embed filer.toml +var Filer string + +//go:embed notification.toml +var Notification string + +//go:embed replication.toml +var Replication string + +//go:embed security.toml +var Security string + +//go:embed master.toml +var Master string + +//go:embed shell.toml +var Shell string diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml new file mode 100644 index 000000000..1de7d02aa --- /dev/null +++ b/weed/command/scaffold/filer.toml @@ -0,0 +1,234 @@ +# A sample TOML config file for SeaweedFS filer store +# Used with "weed filer" or "weed server -filer" +# Put this file to one of the location, with descending priority +# ./filer.toml +# $HOME/.seaweedfs/filer.toml +# /etc/seaweedfs/filer.toml + +#################################################### +# Customizable filer server options +#################################################### +[filer.options] +# with http DELETE, by default the filer would check whether a folder is empty. +# recursive_delete will delete all sub folders and files, similar to "rm -Rf" +recursive_delete = false +# directories under this folder will be automatically creating a separate bucket +buckets_folder = "/buckets" + +#################################################### +# The following are filer store options +#################################################### + +[leveldb2] +# local on disk, mostly for simple single-machine setup, fairly scalable +# faster than previous leveldb, recommended. +enabled = true +dir = "./filerldb2" # directory to store level db files + +[leveldb3] +# similar to leveldb2. +# each bucket has its own meta store. +enabled = false +dir = "./filerldb3" # directory to store level db files + +[rocksdb] +# local on disk, similar to leveldb +# since it is using a C wrapper, you need to install rocksdb and build it by yourself +enabled = false +dir = "./filerrdb" # directory to store rocksdb files + +[sqlite] +# local on disk, similar to leveldb +enabled = false +dbFile = "./filer.db" # sqlite db file + +[mysql] # or memsql, tidb +# CREATE TABLE IF NOT EXISTS filemeta ( +# dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field', +# name VARCHAR(1000) BINARY COMMENT 'directory or file name', +# directory TEXT COMMENT 'full path to parent directory', +# meta LONGBLOB, +# PRIMARY KEY (dirhash, name) +# ) DEFAULT CHARSET=utf8; + +enabled = false +hostname = "localhost" +port = 3306 +username = "root" +password = "" +database = "" # create or use an existing database +connection_max_idle = 2 +connection_max_open = 100 +connection_max_lifetime_seconds = 0 +interpolateParams = false +# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax: +enableUpsert = true +upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)""" + +[mysql2] # or memsql, tidb +enabled = false +createTable = """ + CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` ( + dirhash BIGINT, + name VARCHAR(1000) BINARY, + directory TEXT, + meta LONGBLOB, + PRIMARY KEY (dirhash, name) + ) DEFAULT CHARSET=utf8; +""" +hostname = "localhost" +port = 3306 +username = "root" +password = "" +database = "" # create or use an existing database +connection_max_idle = 2 +connection_max_open = 100 +connection_max_lifetime_seconds = 0 +interpolateParams = false +# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax: +enableUpsert = true +upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)""" + +[postgres] # or cockroachdb, YugabyteDB +# CREATE TABLE IF NOT EXISTS filemeta ( +# dirhash BIGINT, +# name VARCHAR(65535), +# directory VARCHAR(65535), +# meta bytea, +# PRIMARY KEY (dirhash, name) +# ); +enabled = false +hostname = "localhost" +port = 5432 +username = "postgres" +password = "" +database = "postgres" # create or use an existing database +schema = "" +sslmode = "disable" +connection_max_idle = 100 +connection_max_open = 100 +connection_max_lifetime_seconds = 0 +# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax: +enableUpsert = true +upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta""" + +[postgres2] +enabled = false +createTable = """ + CREATE TABLE IF NOT EXISTS "%s" ( + dirhash BIGINT, + name VARCHAR(65535), + directory VARCHAR(65535), + meta bytea, + PRIMARY KEY (dirhash, name) + ); +""" +hostname = "localhost" +port = 5432 +username = "postgres" +password = "" +database = "postgres" # create or use an existing database +schema = "" +sslmode = "disable" +connection_max_idle = 100 +connection_max_open = 100 +connection_max_lifetime_seconds = 0 +# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax: +enableUpsert = true +upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta""" + +[cassandra] +# CREATE TABLE filemeta ( +# directory varchar, +# name varchar, +# meta blob, +# PRIMARY KEY (directory, name) +# ) WITH CLUSTERING ORDER BY (name ASC); +enabled = false +keyspace = "seaweedfs" +hosts = [ + "localhost:9042", +] +username = "" +password = "" +# This changes the data layout. Only add new directories. Removing/Updating will cause data loss. +superLargeDirectories = [] +# Name of the datacenter local to this filer, used as host selection fallback. +localDC = "" + +[hbase] +enabled = false +zkquorum = "" +table = "seaweedfs" + +[redis2] +enabled = false +address = "localhost:6379" +password = "" +database = 0 +# This changes the data layout. Only add new directories. Removing/Updating will cause data loss. +superLargeDirectories = [] + +[redis_cluster2] +enabled = false +addresses = [ + "localhost:30001", + "localhost:30002", + "localhost:30003", + "localhost:30004", + "localhost:30005", + "localhost:30006", +] +password = "" +# allows reads from slave servers or the master, but all writes still go to the master +readOnly = false +# automatically use the closest Redis server for reads +routeByLatency = false +# This changes the data layout. Only add new directories. Removing/Updating will cause data loss. +superLargeDirectories = [] + +[etcd] +enabled = false +servers = "localhost:2379" +timeout = "3s" + +[mongodb] +enabled = false +uri = "mongodb://localhost:27017" +option_pool_size = 0 +database = "seaweedfs" + +[elastic7] +enabled = false +servers = [ + "http://localhost1:9200", + "http://localhost2:9200", + "http://localhost3:9200", +] +username = "" +password = "" +sniff_enabled = false +healthcheck_enabled = false +# increase the value is recommend, be sure the value in Elastic is greater or equal here +index.max_result_window = 10000 + + + +########################## +########################## +# To add path-specific filer store: +# +# 1. Add a name following the store type separated by a dot ".". E.g., cassandra.tmp +# 2. Add a location configuraiton. E.g., location = "/tmp/" +# 3. Copy and customize all other configurations. +# Make sure they are not the same if using the same store type! +# 4. Set enabled to true +# +# The following is just using redis as an example +########################## +[redis2.tmp] +enabled = false +location = "/tmp/" +address = "localhost:6379" +password = "" +database = 1 diff --git a/weed/command/scaffold/master.toml b/weed/command/scaffold/master.toml new file mode 100644 index 000000000..f550f0ad6 --- /dev/null +++ b/weed/command/scaffold/master.toml @@ -0,0 +1,60 @@ +# Put this file to one of the location, with descending priority +# ./master.toml +# $HOME/.seaweedfs/master.toml +# /etc/seaweedfs/master.toml +# this file is read by master + +[master.maintenance] +# periodically run these scripts are the same as running them from 'weed shell' +scripts = """ + lock + ec.encode -fullPercent=95 -quietFor=1h + ec.rebuild -force + ec.balance -force + volume.balance -force + volume.fix.replication + unlock +""" +sleep_minutes = 17 # sleep minutes between each script execution + +[master.filer] +default = "localhost:8888" # used by maintenance scripts if the scripts needs to use fs related commands + + +[master.sequencer] +type = "raft" # Choose [raft|etcd|snowflake] type for storing the file id sequence +# when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence +# example : http://127.0.0.1:2379,http://127.0.0.1:2389 +sequencer_etcd_urls = "http://127.0.0.1:2379" + + +# configurations for tiered cloud storage +# old volumes are transparently moved to cloud for cost efficiency +[storage.backend] +[storage.backend.s3.default] +enabled = false +aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials). +aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials). +region = "us-east-2" +bucket = "your_bucket_name" # an existing bucket +endpoint = "" + +# create this number of logical volumes if no more writable volumes +# count_x means how many copies of data. +# e.g.: +# 000 has only one copy, copy_1 +# 010 and 001 has two copies, copy_2 +# 011 has only 3 copies, copy_3 +[master.volume_growth] +copy_1 = 7 # create 1 x 7 = 7 actual volumes +copy_2 = 6 # create 2 x 6 = 12 actual volumes +copy_3 = 3 # create 3 x 3 = 9 actual volumes +copy_other = 1 # create n x 1 = n actual volumes + +# configuration flags for replication +[master.replication] +# any replication counts should be considered minimums. If you specify 010 and +# have 3 different racks, that's still considered writable. Writes will still +# try to replicate to all available volumes. You should only use this option +# if you are doing your own replication or periodic sync of volumes. +treat_replication_as_minimums = false diff --git a/weed/command/scaffold/notification.toml b/weed/command/scaffold/notification.toml new file mode 100644 index 000000000..f35101edd --- /dev/null +++ b/weed/command/scaffold/notification.toml @@ -0,0 +1,54 @@ +# A sample TOML config file for SeaweedFS filer store +# Used by both "weed filer" or "weed server -filer" and "weed filer.replicate" +# Put this file to one of the location, with descending priority +# ./notification.toml +# $HOME/.seaweedfs/notification.toml +# /etc/seaweedfs/notification.toml + +#################################################### +# notification +# send and receive filer updates for each file to an external message queue +#################################################### +[notification.log] +# this is only for debugging perpose and does not work with "weed filer.replicate" +enabled = false + + +[notification.kafka] +enabled = false +hosts = [ + "localhost:9092" +] +topic = "seaweedfs_filer" +offsetFile = "./last.offset" +offsetSaveIntervalSeconds = 10 + + +[notification.aws_sqs] +# experimental, let me know if it works +enabled = false +aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials). +aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials). +region = "us-east-2" +sqs_queue_name = "my_filer_queue" # an existing queue name + + +[notification.google_pub_sub] +# read credentials doc at https://cloud.google.com/docs/authentication/getting-started +enabled = false +google_application_credentials = "/path/to/x.json" # path to json credential file +project_id = "" # an existing project id +topic = "seaweedfs_filer_topic" # a topic, auto created if does not exists + +[notification.gocdk_pub_sub] +# The Go Cloud Development Kit (https://gocloud.dev). +# PubSub API (https://godoc.org/gocloud.dev/pubsub). +# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ. +enabled = false +# This URL will Dial the RabbitMQ server at the URL in the environment +# variable RABBIT_SERVER_URL and open the exchange "myexchange". +# The exchange must have already been created by some other means, like +# the RabbitMQ management plugin. Сreate myexchange of type fanout and myqueue then +# create binding myexchange => myqueue +topic_url = "rabbit://myexchange" +sub_url = "rabbit://myqueue" diff --git a/weed/command/scaffold/replication.toml b/weed/command/scaffold/replication.toml new file mode 100644 index 000000000..c463c8077 --- /dev/null +++ b/weed/command/scaffold/replication.toml @@ -0,0 +1,71 @@ +# A sample TOML config file for replicating SeaweedFS filer +# Used with "weed filer.backup" +# Using with "weed filer.replicate" is deprecated. +# Put this file to one of the location, with descending priority +# ./replication.toml +# $HOME/.seaweedfs/replication.toml +# /etc/seaweedfs/replication.toml + +[source.filer] # deprecated. Only useful with "weed filer.replicate" +enabled = true +grpcAddress = "localhost:18888" +# all files under this directory tree are replicated. +# this is not a directory on your hard drive, but on your filer. +# i.e., all files with this "prefix" are sent to notification message queue. +directory = "/buckets" + +[sink.local] +enabled = false +directory = "/data" +# all replicated files are under modified time as yyyy-mm-dd directories +# so each date directory contains all new and updated files. +is_incremental = false + +[sink.filer] +enabled = false +grpcAddress = "localhost:18888" +# all replicated files are under this directory tree +# this is not a directory on your hard drive, but on your filer. +# i.e., all received files will be "prefixed" to this directory. +directory = "/backup" +replication = "" +collection = "" +ttlSec = 0 +is_incremental = false + +[sink.s3] +# read credentials doc at https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sessions.html +# default loads credentials from the shared credentials file (~/.aws/credentials). +enabled = false +aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials). +aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials). +region = "us-east-2" +bucket = "your_bucket_name" # an existing bucket +directory = "/" # destination directory +endpoint = "" +is_incremental = false + +[sink.google_cloud_storage] +# read credentials doc at https://cloud.google.com/docs/authentication/getting-started +enabled = false +google_application_credentials = "/path/to/x.json" # path to json credential file +bucket = "your_bucket_seaweedfs" # an existing bucket +directory = "/" # destination directory +is_incremental = false + +[sink.azure] +# experimental, let me know if it works +enabled = false +account_name = "" +account_key = "" +container = "mycontainer" # an existing container +directory = "/" # destination directory +is_incremental = false + +[sink.backblaze] +enabled = false +b2_account_id = "" +b2_master_application_key = "" +bucket = "mybucket" # an existing bucket +directory = "/" # destination directory +is_incremental = false diff --git a/weed/command/scaffold/security.toml b/weed/command/scaffold/security.toml new file mode 100644 index 000000000..0c69b2f24 --- /dev/null +++ b/weed/command/scaffold/security.toml @@ -0,0 +1,60 @@ +# Put this file to one of the location, with descending priority +# ./security.toml +# $HOME/.seaweedfs/security.toml +# /etc/seaweedfs/security.toml +# this file is read by master, volume server, and filer + +# the jwt signing key is read by master and volume server. +# a jwt defaults to expire after 10 seconds. +[jwt.signing] +key = "" +expires_after_seconds = 10 # seconds + +# jwt for read is only supported with master+volume setup. Filer does not support this mode. +[jwt.signing.read] +key = "" +expires_after_seconds = 10 # seconds + +# all grpc tls authentications are mutual +# the values for the following ca, cert, and key are paths to the PERM files. +# the host name is not checked, so the PERM files can be shared. +[grpc] +ca = "" +# Set wildcard domain for enable TLS authentication by common names +allowed_wildcard_domain = "" # .mycompany.com + +[grpc.volume] +cert = "" +key = "" +allowed_commonNames = "" # comma-separated SSL certificate common names + +[grpc.master] +cert = "" +key = "" +allowed_commonNames = "" # comma-separated SSL certificate common names + +[grpc.filer] +cert = "" +key = "" +allowed_commonNames = "" # comma-separated SSL certificate common names + +[grpc.msg_broker] +cert = "" +key = "" +allowed_commonNames = "" # comma-separated SSL certificate common names + +# use this for any place needs a grpc client +# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload" +[grpc.client] +cert = "" +key = "" + +# volume server https options +# Note: work in progress! +# this does not work with other clients, e.g., "weed filer|mount" etc, yet. +[https.client] +enabled = true +[https.volume] +cert = "" +key = "" + diff --git a/weed/command/scaffold/shell.toml b/weed/command/scaffold/shell.toml new file mode 100644 index 000000000..288ae2efe --- /dev/null +++ b/weed/command/scaffold/shell.toml @@ -0,0 +1,10 @@ +[cluster] +default = "c1" + +[cluster.c1] +master = "localhost:9333" # comma-separated master servers +filer = "localhost:8888" # filer host and port + +[cluster.c2] +master = "" +filer = ""