Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge pull request #1946 from LazyDBA247-Anyvision/master 

Adding Customisation - insertQuery for postgres/mysql
pull/1948/head
Chris Lu 4 years ago
committed by GitHub
parent
6b7aa9633f 96c62bd34d
commit
1ebdbad222
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 43 additions and 7 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 16
      weed/command/scaffold.go
  2. 6
      weed/filer/mysql/mysql_sql_gen.go
  3. 7
      weed/filer/mysql/mysql_store.go
  4. 4
      weed/filer/mysql2/mysql2_store.go
  5. 5
      weed/filer/postgres/postgres_sql_gen.go
  6. 4
      weed/filer/postgres/postgres_store.go
  7. 4
      weed/filer/postgres2/postgres2_store.go

16
weed/command/scaffold.go
View File

@ -102,6 +102,10 @@ enabled = false
dir = "./filerrdb" # directory to store rocksdb files dir = "./filerrdb" # directory to store rocksdb files
[mysql] # or memsql, tidb [mysql] # or memsql, tidb
# Empty insertQuery will use the default insert query
# example insertQuery can be for UPSERT syntax:
# insertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
insertQuery = ""
# CREATE TABLE IF NOT EXISTS filemeta ( # CREATE TABLE IF NOT EXISTS filemeta (
# dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field', # dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
# name VARCHAR(1000) BINARY COMMENT 'directory or file name', # name VARCHAR(1000) BINARY COMMENT 'directory or file name',
@ -123,6 +127,10 @@ interpolateParams = false
[mysql2] # or memsql, tidb [mysql2] # or memsql, tidb
enabled = false enabled = false
# Empty insertQuery will use the default insert query
# example insertQuery can be for UPSERT syntax:
# insertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
insertQuery = ""
createTable = """ createTable = """
CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` ( CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` (
dirhash BIGINT, dirhash BIGINT,
@ -143,6 +151,10 @@ connection_max_lifetime_seconds = 0
interpolateParams = false interpolateParams = false
[postgres] # or cockroachdb, YugabyteDB [postgres] # or cockroachdb, YugabyteDB
# Empty insertQuery will use the default insert query
# example insertQuery can be for UPSERT syntax:
# insertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT ON CONSTRAINT "%[1]s_pkey" DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
insertQuery = ""
# CREATE TABLE IF NOT EXISTS filemeta ( # CREATE TABLE IF NOT EXISTS filemeta (
# dirhash BIGINT, # dirhash BIGINT,
# name VARCHAR(65535), # name VARCHAR(65535),
@ -164,6 +176,10 @@ connection_max_lifetime_seconds = 0
[postgres2] [postgres2]
enabled = false enabled = false
# Empty insertQuery will use the default insert query
# example insertQuery can be for UPSERT syntax:
# insertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT ON CONSTRAINT "%[1]s_pkey" DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
insertQuery = ""
createTable = """ createTable = """
CREATE TABLE IF NOT EXISTS "%s" ( CREATE TABLE IF NOT EXISTS "%s" (
dirhash BIGINT, dirhash BIGINT,

6
weed/filer/mysql/mysql_sql_gen.go
View File

@ -2,6 +2,7 @@ package mysql
import ( import (
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
_ "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql"
) )
@ -9,6 +10,7 @@ import (
type SqlGenMysql struct { type SqlGenMysql struct {
CreateTableSqlTemplate string CreateTableSqlTemplate string
DropTableSqlTemplate string DropTableSqlTemplate string
InsertQueryTemplate string
} }
var ( var (
@ -16,8 +18,12 @@ var (
) )
func (gen *SqlGenMysql) GetSqlInsert(tableName string) string { func (gen *SqlGenMysql) GetSqlInsert(tableName string) string {
if gen.InsertQueryTemplate != "" {
return fmt.Sprintf(gen.InsertQueryTemplate, tableName)
} else {
return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", tableName) return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", tableName)
} }
}
func (gen *SqlGenMysql) GetSqlUpdate(tableName string) string { func (gen *SqlGenMysql) GetSqlUpdate(tableName string) string {
return fmt.Sprintf("UPDATE `%s` SET meta=? WHERE dirhash=? AND name=? AND directory=?", tableName) return fmt.Sprintf("UPDATE `%s` SET meta=? WHERE dirhash=? AND name=? AND directory=?", tableName)

7
weed/filer/mysql/mysql_store.go
View File

@ -3,9 +3,10 @@ package mysql
import ( import (
"database/sql" "database/sql"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
_ "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql"
@ -29,6 +30,7 @@ func (store *MysqlStore) GetName() string {
func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) { func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize( return store.initialize(
configuration.GetString(prefix+"insertQuery"),
configuration.GetString(prefix+"username"), configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"), configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"), configuration.GetString(prefix+"hostname"),
@ -41,13 +43,14 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str
) )
} }
func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen,
func (store *MysqlStore) initialize(insertQuery, user, password, hostname string, port int, database string, maxIdle, maxOpen,
maxLifetimeSeconds int, interpolateParams bool) (err error) { maxLifetimeSeconds int, interpolateParams bool) (err error) {
store.SupportBucketTable = false store.SupportBucketTable = false
store.SqlGenerator = &SqlGenMysql{ store.SqlGenerator = &SqlGenMysql{
CreateTableSqlTemplate: "", CreateTableSqlTemplate: "",
DropTableSqlTemplate: "drop table `%s`", DropTableSqlTemplate: "drop table `%s`",
InsertQueryTemplate: insertQuery,
} }
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)

4
weed/filer/mysql2/mysql2_store.go
View File

@ -32,6 +32,7 @@ func (store *MysqlStore2) GetName() string {
func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) { func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize( return store.initialize(
configuration.GetString(prefix+"createTable"), configuration.GetString(prefix+"createTable"),
configuration.GetString(prefix+"insertQuery"),
configuration.GetString(prefix+"username"), configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"), configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"), configuration.GetString(prefix+"hostname"),
@ -44,13 +45,14 @@ func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix st
) )
} }
func (store *MysqlStore2) initialize(createTable, user, password, hostname string, port int, database string, maxIdle, maxOpen,
func (store *MysqlStore2) initialize(createTable, insertQuery, user, password, hostname string, port int, database string, maxIdle, maxOpen,
maxLifetimeSeconds int, interpolateParams bool) (err error) { maxLifetimeSeconds int, interpolateParams bool) (err error) {
store.SupportBucketTable = true store.SupportBucketTable = true
store.SqlGenerator = &mysql.SqlGenMysql{ store.SqlGenerator = &mysql.SqlGenMysql{
CreateTableSqlTemplate: createTable, CreateTableSqlTemplate: createTable,
DropTableSqlTemplate: "drop table `%s`", DropTableSqlTemplate: "drop table `%s`",
InsertQueryTemplate: insertQuery,
} }
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)

5
weed/filer/postgres/postgres_sql_gen.go
View File

@ -10,6 +10,7 @@ import (
type SqlGenPostgres struct { type SqlGenPostgres struct {
CreateTableSqlTemplate string CreateTableSqlTemplate string
DropTableSqlTemplate string DropTableSqlTemplate string
InsertQueryTemplate string
} }
var ( var (
@ -17,8 +18,12 @@ var (
) )
func (gen *SqlGenPostgres) GetSqlInsert(tableName string) string { func (gen *SqlGenPostgres) GetSqlInsert(tableName string) string {
if gen.InsertQueryTemplate != "" {
return fmt.Sprintf(gen.InsertQueryTemplate, tableName)
} else {
return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, tableName) return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, tableName)
} }
}
func (gen *SqlGenPostgres) GetSqlUpdate(tableName string) string { func (gen *SqlGenPostgres) GetSqlUpdate(tableName string) string {
return fmt.Sprintf(`UPDATE "%s" SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4`, tableName) return fmt.Sprintf(`UPDATE "%s" SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4`, tableName)

4
weed/filer/postgres/postgres_store.go
View File

@ -29,6 +29,7 @@ func (store *PostgresStore) GetName() string {
func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) { func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize( return store.initialize(
configuration.GetString(prefix+"insertQuery"),
configuration.GetString(prefix+"username"), configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"), configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"), configuration.GetString(prefix+"hostname"),
@ -42,12 +43,13 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix
) )
} }
func (store *PostgresStore) initialize(user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
func (store *PostgresStore) initialize(insertQuery, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = false store.SupportBucketTable = false
store.SqlGenerator = &SqlGenPostgres{ store.SqlGenerator = &SqlGenPostgres{
CreateTableSqlTemplate: "", CreateTableSqlTemplate: "",
DropTableSqlTemplate: `drop table "%s"`, DropTableSqlTemplate: `drop table "%s"`,
InsertQueryTemplate: insertQuery,
} }
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode) sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)

4
weed/filer/postgres2/postgres2_store.go
View File

@ -32,6 +32,7 @@ func (store *PostgresStore2) GetName() string {
func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) { func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize( return store.initialize(
configuration.GetString(prefix+"createTable"), configuration.GetString(prefix+"createTable"),
configuration.GetString(prefix+"insertQuery"),
configuration.GetString(prefix+"username"), configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"), configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"), configuration.GetString(prefix+"hostname"),
@ -45,12 +46,13 @@ func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix
) )
} }
func (store *PostgresStore2) initialize(createTable, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
func (store *PostgresStore2) initialize(createTable, insertQuery, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = true store.SupportBucketTable = true
store.SqlGenerator = &postgres.SqlGenPostgres{ store.SqlGenerator = &postgres.SqlGenPostgres{
CreateTableSqlTemplate: createTable, CreateTableSqlTemplate: createTable,
DropTableSqlTemplate: `drop table "%s"`, DropTableSqlTemplate: `drop table "%s"`,
InsertQueryTemplate: insertQuery,
} }
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode) sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)

Write Preview
Loading…
Cancel
Save