diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 52b547563..3296ebf21 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -102,6 +102,10 @@ enabled = false dir = "./filerrdb" # directory to store rocksdb files [mysql] # or memsql, tidb +# Empty insertQuery will use the default insert query +# example insertQuery can be for UPSERT syntax: +# insertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)""" +insertQuery = "" # CREATE TABLE IF NOT EXISTS filemeta ( # dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field', # name VARCHAR(1000) BINARY COMMENT 'directory or file name', @@ -123,6 +127,10 @@ interpolateParams = false [mysql2] # or memsql, tidb enabled = false +# Empty insertQuery will use the default insert query +# example insertQuery can be for UPSERT syntax: +# insertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)""" +insertQuery = "" createTable = """ CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` ( dirhash BIGINT, @@ -143,6 +151,10 @@ connection_max_lifetime_seconds = 0 interpolateParams = false [postgres] # or cockroachdb, YugabyteDB +# Empty insertQuery will use the default insert query +# example insertQuery can be for UPSERT syntax: +# insertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT ON CONSTRAINT "%[1]s_pkey" DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta""" +insertQuery = "" # CREATE TABLE IF NOT EXISTS filemeta ( # dirhash BIGINT, # name VARCHAR(65535), @@ -164,6 +176,10 @@ connection_max_lifetime_seconds = 0 [postgres2] enabled = false +# Empty insertQuery will use the default insert query +# example insertQuery can be for UPSERT syntax: +# insertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT ON CONSTRAINT "%[1]s_pkey" DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta""" +insertQuery = "" createTable = """ CREATE TABLE IF NOT EXISTS "%s" ( dirhash BIGINT, diff --git a/weed/filer/mysql/mysql_sql_gen.go b/weed/filer/mysql/mysql_sql_gen.go index 2f86a5798..105f3e62a 100644 --- a/weed/filer/mysql/mysql_sql_gen.go +++ b/weed/filer/mysql/mysql_sql_gen.go @@ -2,6 +2,7 @@ package mysql import ( "fmt" + "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" _ "github.com/go-sql-driver/mysql" ) @@ -9,6 +10,7 @@ import ( type SqlGenMysql struct { CreateTableSqlTemplate string DropTableSqlTemplate string + InsertQueryTemplate string } var ( @@ -16,7 +18,11 @@ var ( ) func (gen *SqlGenMysql) GetSqlInsert(tableName string) string { - return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", tableName) + if gen.InsertQueryTemplate != "" { + return fmt.Sprintf(gen.InsertQueryTemplate, tableName) + } else { + return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", tableName) + } } func (gen *SqlGenMysql) GetSqlUpdate(tableName string) string { diff --git a/weed/filer/mysql/mysql_store.go b/weed/filer/mysql/mysql_store.go index 501ab1d39..4b73faacf 100644 --- a/weed/filer/mysql/mysql_store.go +++ b/weed/filer/mysql/mysql_store.go @@ -3,9 +3,10 @@ package mysql import ( "database/sql" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer" "time" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" "github.com/chrislusf/seaweedfs/weed/util" _ "github.com/go-sql-driver/mysql" @@ -29,6 +30,7 @@ func (store *MysqlStore) GetName() string { func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) { return store.initialize( + configuration.GetString(prefix+"insertQuery"), configuration.GetString(prefix+"username"), configuration.GetString(prefix+"password"), configuration.GetString(prefix+"hostname"), @@ -41,13 +43,14 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str ) } -func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen, +func (store *MysqlStore) initialize(insertQuery, user, password, hostname string, port int, database string, maxIdle, maxOpen, maxLifetimeSeconds int, interpolateParams bool) (err error) { store.SupportBucketTable = false store.SqlGenerator = &SqlGenMysql{ CreateTableSqlTemplate: "", DropTableSqlTemplate: "drop table `%s`", + InsertQueryTemplate: insertQuery, } sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) diff --git a/weed/filer/mysql2/mysql2_store.go b/weed/filer/mysql2/mysql2_store.go index c796cd6aa..3c51f9363 100644 --- a/weed/filer/mysql2/mysql2_store.go +++ b/weed/filer/mysql2/mysql2_store.go @@ -32,6 +32,7 @@ func (store *MysqlStore2) GetName() string { func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) { return store.initialize( configuration.GetString(prefix+"createTable"), + configuration.GetString(prefix+"insertQuery"), configuration.GetString(prefix+"username"), configuration.GetString(prefix+"password"), configuration.GetString(prefix+"hostname"), @@ -44,13 +45,14 @@ func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix st ) } -func (store *MysqlStore2) initialize(createTable, user, password, hostname string, port int, database string, maxIdle, maxOpen, +func (store *MysqlStore2) initialize(createTable, insertQuery, user, password, hostname string, port int, database string, maxIdle, maxOpen, maxLifetimeSeconds int, interpolateParams bool) (err error) { store.SupportBucketTable = true store.SqlGenerator = &mysql.SqlGenMysql{ CreateTableSqlTemplate: createTable, DropTableSqlTemplate: "drop table `%s`", + InsertQueryTemplate: insertQuery, } sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) diff --git a/weed/filer/postgres/postgres_sql_gen.go b/weed/filer/postgres/postgres_sql_gen.go index 0d1c343c5..21a87ef5a 100644 --- a/weed/filer/postgres/postgres_sql_gen.go +++ b/weed/filer/postgres/postgres_sql_gen.go @@ -10,6 +10,7 @@ import ( type SqlGenPostgres struct { CreateTableSqlTemplate string DropTableSqlTemplate string + InsertQueryTemplate string } var ( @@ -17,7 +18,11 @@ var ( ) func (gen *SqlGenPostgres) GetSqlInsert(tableName string) string { - return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, tableName) + if gen.InsertQueryTemplate != "" { + return fmt.Sprintf(gen.InsertQueryTemplate, tableName) + } else { + return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, tableName) + } } func (gen *SqlGenPostgres) GetSqlUpdate(tableName string) string { diff --git a/weed/filer/postgres/postgres_store.go b/weed/filer/postgres/postgres_store.go index 9e4ff7c32..ea9b1c71e 100644 --- a/weed/filer/postgres/postgres_store.go +++ b/weed/filer/postgres/postgres_store.go @@ -29,6 +29,7 @@ func (store *PostgresStore) GetName() string { func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) { return store.initialize( + configuration.GetString(prefix+"insertQuery"), configuration.GetString(prefix+"username"), configuration.GetString(prefix+"password"), configuration.GetString(prefix+"hostname"), @@ -42,12 +43,13 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix ) } -func (store *PostgresStore) initialize(user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) { +func (store *PostgresStore) initialize(insertQuery, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) { store.SupportBucketTable = false store.SqlGenerator = &SqlGenPostgres{ CreateTableSqlTemplate: "", DropTableSqlTemplate: `drop table "%s"`, + InsertQueryTemplate: insertQuery, } sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode) diff --git a/weed/filer/postgres2/postgres2_store.go b/weed/filer/postgres2/postgres2_store.go index 92893bf7a..b83945db6 100644 --- a/weed/filer/postgres2/postgres2_store.go +++ b/weed/filer/postgres2/postgres2_store.go @@ -32,6 +32,7 @@ func (store *PostgresStore2) GetName() string { func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) { return store.initialize( configuration.GetString(prefix+"createTable"), + configuration.GetString(prefix+"insertQuery"), configuration.GetString(prefix+"username"), configuration.GetString(prefix+"password"), configuration.GetString(prefix+"hostname"), @@ -45,12 +46,13 @@ func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix ) } -func (store *PostgresStore2) initialize(createTable, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) { +func (store *PostgresStore2) initialize(createTable, insertQuery, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) { store.SupportBucketTable = true store.SqlGenerator = &postgres.SqlGenPostgres{ CreateTableSqlTemplate: createTable, DropTableSqlTemplate: `drop table "%s"`, + InsertQueryTemplate: insertQuery, } sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)