Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge pull request #1946 from LazyDBA247-Anyvision/master 

Adding Customisation - insertQuery for postgres/mysql
pull/1948/head
Chris Lu 4 years ago
committed by GitHub
parent
6b7aa9633f 96c62bd34d
commit
1ebdbad222
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 43 additions and 7 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 16
      weed/command/scaffold.go
  2. 8
      weed/filer/mysql/mysql_sql_gen.go
  3. 7
      weed/filer/mysql/mysql_store.go
  4. 4
      weed/filer/mysql2/mysql2_store.go
  5. 7
      weed/filer/postgres/postgres_sql_gen.go
  6. 4
      weed/filer/postgres/postgres_store.go
  7. 4
      weed/filer/postgres2/postgres2_store.go

16
weed/command/scaffold.go
View File

@ -102,6 +102,10 @@ enabled = false
dir = "./filerrdb" # directory to store rocksdb files
[mysql] # or memsql, tidb
# Empty insertQuery will use the default insert query
# example insertQuery can be for UPSERT syntax:
# insertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
insertQuery = ""
# CREATE TABLE IF NOT EXISTS filemeta (
# dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
# name VARCHAR(1000) BINARY COMMENT 'directory or file name',
@ -123,6 +127,10 @@ interpolateParams = false
[mysql2] # or memsql, tidb
enabled = false
# Empty insertQuery will use the default insert query
# example insertQuery can be for UPSERT syntax:
# insertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
insertQuery = ""
createTable = """
CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` (
dirhash BIGINT,
@ -143,6 +151,10 @@ connection_max_lifetime_seconds = 0
interpolateParams = false
[postgres] # or cockroachdb, YugabyteDB
# Empty insertQuery will use the default insert query
# example insertQuery can be for UPSERT syntax:
# insertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT ON CONSTRAINT "%[1]s_pkey" DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
insertQuery = ""
# CREATE TABLE IF NOT EXISTS filemeta (
# dirhash BIGINT,
# name VARCHAR(65535),
@ -164,6 +176,10 @@ connection_max_lifetime_seconds = 0
[postgres2]
enabled = false
# Empty insertQuery will use the default insert query
# example insertQuery can be for UPSERT syntax:
# insertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT ON CONSTRAINT "%[1]s_pkey" DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
insertQuery = ""
createTable = """
CREATE TABLE IF NOT EXISTS "%s" (
dirhash BIGINT,

8
weed/filer/mysql/mysql_sql_gen.go
View File

@ -2,6 +2,7 @@ package mysql
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
_ "github.com/go-sql-driver/mysql"
)
@ -9,6 +10,7 @@ import (
type SqlGenMysql struct {
CreateTableSqlTemplate string
DropTableSqlTemplate string
InsertQueryTemplate string
}
var (
@ -16,7 +18,11 @@ var (
)
func (gen *SqlGenMysql) GetSqlInsert(tableName string) string {
return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", tableName)
if gen.InsertQueryTemplate != "" {
return fmt.Sprintf(gen.InsertQueryTemplate, tableName)
} else {
return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", tableName)
}
}
func (gen *SqlGenMysql) GetSqlUpdate(tableName string) string {

7
weed/filer/mysql/mysql_store.go
View File

@ -3,9 +3,10 @@ package mysql
import (
"database/sql"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
"github.com/chrislusf/seaweedfs/weed/util"
_ "github.com/go-sql-driver/mysql"
@ -29,6 +30,7 @@ func (store *MysqlStore) GetName() string {
func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize(
configuration.GetString(prefix+"insertQuery"),
configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"),
@ -41,13 +43,14 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str
)
}
func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen,
func (store *MysqlStore) initialize(insertQuery, user, password, hostname string, port int, database string, maxIdle, maxOpen,
maxLifetimeSeconds int, interpolateParams bool) (err error) {
store.SupportBucketTable = false
store.SqlGenerator = &SqlGenMysql{
CreateTableSqlTemplate: "",
DropTableSqlTemplate: "drop table `%s`",
InsertQueryTemplate: insertQuery,
}
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)

4
weed/filer/mysql2/mysql2_store.go
View File

@ -32,6 +32,7 @@ func (store *MysqlStore2) GetName() string {
func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize(
configuration.GetString(prefix+"createTable"),
configuration.GetString(prefix+"insertQuery"),
configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"),
@ -44,13 +45,14 @@ func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix st
)
}
func (store *MysqlStore2) initialize(createTable, user, password, hostname string, port int, database string, maxIdle, maxOpen,
func (store *MysqlStore2) initialize(createTable, insertQuery, user, password, hostname string, port int, database string, maxIdle, maxOpen,
maxLifetimeSeconds int, interpolateParams bool) (err error) {
store.SupportBucketTable = true
store.SqlGenerator = &mysql.SqlGenMysql{
CreateTableSqlTemplate: createTable,
DropTableSqlTemplate: "drop table `%s`",
InsertQueryTemplate: insertQuery,
}
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)

7
weed/filer/postgres/postgres_sql_gen.go
View File

@ -10,6 +10,7 @@ import (
type SqlGenPostgres struct {
CreateTableSqlTemplate string
DropTableSqlTemplate string
InsertQueryTemplate string
}
var (
@ -17,7 +18,11 @@ var (
)
func (gen *SqlGenPostgres) GetSqlInsert(tableName string) string {
return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, tableName)
if gen.InsertQueryTemplate != "" {
return fmt.Sprintf(gen.InsertQueryTemplate, tableName)
} else {
return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, tableName)
}
}
func (gen *SqlGenPostgres) GetSqlUpdate(tableName string) string {

4
weed/filer/postgres/postgres_store.go
View File

@ -29,6 +29,7 @@ func (store *PostgresStore) GetName() string {
func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize(
configuration.GetString(prefix+"insertQuery"),
configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"),
@ -42,12 +43,13 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix
)
}
func (store *PostgresStore) initialize(user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
func (store *PostgresStore) initialize(insertQuery, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = false
store.SqlGenerator = &SqlGenPostgres{
CreateTableSqlTemplate: "",
DropTableSqlTemplate: `drop table "%s"`,
InsertQueryTemplate: insertQuery,
}
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)

4
weed/filer/postgres2/postgres2_store.go
View File

@ -32,6 +32,7 @@ func (store *PostgresStore2) GetName() string {
func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize(
configuration.GetString(prefix+"createTable"),
configuration.GetString(prefix+"insertQuery"),
configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"),
@ -45,12 +46,13 @@ func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix
)
}
func (store *PostgresStore2) initialize(createTable, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
func (store *PostgresStore2) initialize(createTable, insertQuery, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = true
store.SqlGenerator = &postgres.SqlGenPostgres{
CreateTableSqlTemplate: createTable,
DropTableSqlTemplate: `drop table "%s"`,
InsertQueryTemplate: insertQuery,
}
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)

Write Preview
Loading…
Cancel
Save