lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-05-03 04:37:48 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #1946 from LazyDBA247-Anyvision/master

Browse Source 
Adding Customisation - insertQuery for postgres/mysql
This commit is contained in:
Chris Lu 2021-03-29 00:42:48 -07:00 committed by GitHub
commit 1ebdbad222
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 43 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
weed/command/scaffold.go
View File

@ -102,6 +102,10 @@ enabled = false
dir = "./filerrdb" # directory to store rocksdb files
[mysql] # or memsql, tidb
# Empty insertQuery will use the default insert query
# example insertQuery can be for UPSERT syntax:
# insertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
insertQuery = ""
# CREATE TABLE IF NOT EXISTS filemeta (
# dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
# name VARCHAR(1000) BINARY COMMENT 'directory or file name',
@ -123,6 +127,10 @@ interpolateParams = false
[mysql2] # or memsql, tidb
enabled = false
# Empty insertQuery will use the default insert query
# example insertQuery can be for UPSERT syntax:
# insertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
insertQuery = ""
createTable = """
CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` (
dirhash BIGINT,
@ -143,6 +151,10 @@ connection_max_lifetime_seconds = 0
interpolateParams = false
[postgres] # or cockroachdb, YugabyteDB
# Empty insertQuery will use the default insert query
# example insertQuery can be for UPSERT syntax:
# insertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT ON CONSTRAINT "%[1]s_pkey" DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
insertQuery = ""
# CREATE TABLE IF NOT EXISTS filemeta (
# dirhash BIGINT,
# name VARCHAR(65535),
@ -164,6 +176,10 @@ connection_max_lifetime_seconds = 0
[postgres2]
enabled = false
# Empty insertQuery will use the default insert query
# example insertQuery can be for UPSERT syntax:
# insertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT ON CONSTRAINT "%[1]s_pkey" DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
insertQuery = ""
createTable = """
CREATE TABLE IF NOT EXISTS "%s" (
dirhash BIGINT,

8
weed/filer/mysql/mysql_sql_gen.go
View File

@ -2,6 +2,7 @@ package mysql
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
_ "github.com/go-sql-driver/mysql"
)
@ -9,6 +10,7 @@ import (
type SqlGenMysql struct {
CreateTableSqlTemplate string
DropTableSqlTemplate string
InsertQueryTemplate string
}
var (
@ -16,7 +18,11 @@ var (
)
func (gen *SqlGenMysql) GetSqlInsert(tableName string) string {
return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", tableName)
if gen.InsertQueryTemplate != "" {
return fmt.Sprintf(gen.InsertQueryTemplate, tableName)
} else {
return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", tableName)
}
}
func (gen *SqlGenMysql) GetSqlUpdate(tableName string) string {

7
weed/filer/mysql/mysql_store.go
View File

@ -3,9 +3,10 @@ package mysql
import (
"database/sql"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
"github.com/chrislusf/seaweedfs/weed/util"
_ "github.com/go-sql-driver/mysql"
@ -29,6 +30,7 @@ func (store *MysqlStore) GetName() string {
func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize(
configuration.GetString(prefix+"insertQuery"),
configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"),
@ -41,13 +43,14 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str
)
}
func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen,
func (store *MysqlStore) initialize(insertQuery, user, password, hostname string, port int, database string, maxIdle, maxOpen,
maxLifetimeSeconds int, interpolateParams bool) (err error) {
store.SupportBucketTable = false
store.SqlGenerator = &SqlGenMysql{
CreateTableSqlTemplate: "",
DropTableSqlTemplate: "drop table `%s`",
InsertQueryTemplate: insertQuery,
}
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)

4
weed/filer/mysql2/mysql2_store.go
View File

@ -32,6 +32,7 @@ func (store *MysqlStore2) GetName() string {
func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize(
configuration.GetString(prefix+"createTable"),
configuration.GetString(prefix+"insertQuery"),
configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"),
@ -44,13 +45,14 @@ func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix st
)
}
func (store *MysqlStore2) initialize(createTable, user, password, hostname string, port int, database string, maxIdle, maxOpen,
func (store *MysqlStore2) initialize(createTable, insertQuery, user, password, hostname string, port int, database string, maxIdle, maxOpen,
maxLifetimeSeconds int, interpolateParams bool) (err error) {
store.SupportBucketTable = true
store.SqlGenerator = &mysql.SqlGenMysql{
CreateTableSqlTemplate: createTable,
DropTableSqlTemplate: "drop table `%s`",
InsertQueryTemplate: insertQuery,
}
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)

7
weed/filer/postgres/postgres_sql_gen.go
View File

@ -10,6 +10,7 @@ import (
type SqlGenPostgres struct {
CreateTableSqlTemplate string
DropTableSqlTemplate string
InsertQueryTemplate string
}
var (
@ -17,7 +18,11 @@ var (
)
func (gen *SqlGenPostgres) GetSqlInsert(tableName string) string {
return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, tableName)
if gen.InsertQueryTemplate != "" {
return fmt.Sprintf(gen.InsertQueryTemplate, tableName)
} else {
return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, tableName)
}
}
func (gen *SqlGenPostgres) GetSqlUpdate(tableName string) string {

4
weed/filer/postgres/postgres_store.go
View File

@ -29,6 +29,7 @@ func (store *PostgresStore) GetName() string {
func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize(
configuration.GetString(prefix+"insertQuery"),
configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"),
@ -42,12 +43,13 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix
)
}
func (store *PostgresStore) initialize(user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
func (store *PostgresStore) initialize(insertQuery, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = false
store.SqlGenerator = &SqlGenPostgres{
CreateTableSqlTemplate: "",
DropTableSqlTemplate: `drop table "%s"`,
InsertQueryTemplate: insertQuery,
}
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)

4
weed/filer/postgres2/postgres2_store.go
View File

@ -32,6 +32,7 @@ func (store *PostgresStore2) GetName() string {
func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize(
configuration.GetString(prefix+"createTable"),
configuration.GetString(prefix+"insertQuery"),
configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"),
@ -45,12 +46,13 @@ func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix
)
}
func (store *PostgresStore2) initialize(createTable, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
func (store *PostgresStore2) initialize(createTable, insertQuery, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = true
store.SqlGenerator = &postgres.SqlGenPostgres{
CreateTableSqlTemplate: createTable,
DropTableSqlTemplate: `drop table "%s"`,
InsertQueryTemplate: insertQuery,
}
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)