lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-08-20 04:46:42 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Filer Store: postgres backend support pgbouncer (#7077)

Browse Source 
support pgbouncer
This commit is contained in:
Chris Lu 2025-08-03 11:56:04 -07:00 committed by GitHub
parent d49b44f2a4
commit 4fb7bbb215
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 73 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
go.mod
View File

@ -45,6 +45,7 @@ require (
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/jackc/pgx/v5 v5.7.5
github.com/jcmturner/gofork v1.7.6 // indirect
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
github.com/jinzhu/copier v0.4.0
@ -54,7 +55,6 @@ require (
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/reedsolomon v1.12.5
github.com/kurin/blazer v0.5.3
github.com/lib/pq v1.10.9
github.com/linxGnu/grocksdb v1.10.1
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-ieproxy v0.0.11 // indirect
@ -164,6 +164,9 @@ require github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // ind
require (
github.com/cenkalti/backoff/v3 v3.2.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
)

10
go.sum
View File

@ -1198,6 +1198,14 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs=
github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
@ -1291,8 +1299,6 @@ github.com/lanrat/extsort v1.0.2 h1:p3MLVpQEPwEGPzeLBb+1eSErzRl6Bgjgr+qnIs2RxrU=
github.com/lanrat/extsort v1.0.2/go.mod h1:ivzsdLm8Tv+88qbdpMElV6Z15StlzPUtZSKsGb51hnQ=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/linxGnu/grocksdb v1.10.1 h1:YX6gUcKvSC3d0s9DaqgbU+CRkZHzlELgHu1Z/kmtslg=
github.com/linxGnu/grocksdb v1.10.1/go.mod h1:C3CNe9UYc9hlEM2pC82AqiGS3LRW537u9LFV4wIZuHk=
github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8=

4
weed/command/scaffold/filer.toml
View File

@ -120,6 +120,8 @@ sslmode = "disable"
connection_max_idle = 100
connection_max_open = 100
connection_max_lifetime_seconds = 0
# Set to true when using PgBouncer connection pooler
pgbouncer_compatible = false
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
enableUpsert = true
upsertQuery = """
@ -157,6 +159,8 @@ sslmode = "disable"
connection_max_idle = 100
connection_max_open = 100
connection_max_lifetime_seconds = 0
# Set to true when using PgBouncer connection pooler
pgbouncer_compatible = false
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
enableUpsert = true
upsertQuery = """

7
weed/credential/postgres/postgres_store.go
View File

@ -8,7 +8,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/credential"
"github.com/seaweedfs/seaweedfs/weed/util"
_ "github.com/lib/pq"
_ "github.com/jackc/pgx/v5/stdlib"
)
func init() {
@ -52,11 +52,12 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix
sslmode = "disable"
}
// Build connection string
// Build pgx-optimized connection string
// Note: prefer_simple_protocol=true is only needed for PgBouncer, not direct PostgreSQL connections
connStr := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s search_path=%s",
hostname, port, username, password, database, sslmode, schema)
db, err := sql.Open("postgres", connStr)
db, err := sql.Open("pgx", connStr)
if err != nil {
return fmt.Errorf("failed to open database: %w", err)
}

2
weed/filer/postgres/postgres_sql_gen.go
View File

@ -3,7 +3,7 @@ package postgres
import (
"fmt"
_ "github.com/lib/pq"
_ "github.com/jackc/pgx/v5/stdlib"
"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
)

32
weed/filer/postgres/postgres_store.go
View File

@ -1,3 +1,10 @@
// Package postgres provides PostgreSQL filer store implementation
// Migrated from github.com/lib/pq to github.com/jackc/pgx for:
// - Active development and support
// - Better performance and PostgreSQL-specific features
// - Improved error handling (no more panics)
// - Built-in logging capabilities
// - Superior SSL certificate support
package postgres
import (
@ -6,7 +13,7 @@ import (
"strconv"
"time"
_ "github.com/lib/pq"
_ "github.com/jackc/pgx/v5/stdlib"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
"github.com/seaweedfs/seaweedfs/weed/util"
@ -39,13 +46,14 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix
configuration.GetString(prefix+"sslkey"),
configuration.GetString(prefix+"sslrootcert"),
configuration.GetString(prefix+"sslcrl"),
configuration.GetBool(prefix+"pgbouncer_compatible"),
configuration.GetInt(prefix+"connection_max_idle"),
configuration.GetInt(prefix+"connection_max_open"),
configuration.GetInt(prefix+"connection_max_lifetime_seconds"),
)
}
func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = false
if !enableUpsert {
@ -57,13 +65,23 @@ func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, us
UpsertQueryTemplate: upsertQuery,
}
// pgx-optimized connection string with better timeouts and connection handling
sqlUrl := "connect_timeout=30"
// PgBouncer compatibility: add prefer_simple_protocol=true when needed
// This avoids prepared statement issues with PgBouncer's transaction pooling mode
if pgbouncerCompatible {
sqlUrl += " prefer_simple_protocol=true"
}
if hostname != "" {
sqlUrl += " host=" + hostname
}
if port != 0 {
sqlUrl += " port=" + strconv.Itoa(port)
}
// SSL configuration - pgx provides better SSL support than lib/pq
if sslmode != "" {
sqlUrl += " sslmode=" + sslmode
}
@ -91,16 +109,18 @@ func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, us
sqlUrl += " dbname=" + database
adaptedSqlUrl += " dbname=" + database
}
if schema != "" {
if schema != "" && !pgbouncerCompatible {
sqlUrl += " search_path=" + schema
adaptedSqlUrl += " search_path=" + schema
}
var dbErr error
store.DB, dbErr = sql.Open("postgres", sqlUrl)
store.DB, dbErr = sql.Open("pgx", sqlUrl)
if dbErr != nil {
store.DB.Close()
if store.DB != nil {
store.DB.Close()
}
store.DB = nil
return fmt.Errorf("can not connect to %s error:%v", adaptedSqlUrl, err)
return fmt.Errorf("can not connect to %s error:%v", adaptedSqlUrl, dbErr)
}
store.DB.SetMaxIdleConns(maxIdle)

32
weed/filer/postgres2/postgres2_store.go
View File

@ -1,3 +1,10 @@
// Package postgres2 provides PostgreSQL filer store implementation with bucket support
// Migrated from github.com/lib/pq to github.com/jackc/pgx for:
// - Active development and support
// - Better performance and PostgreSQL-specific features
// - Improved error handling (no more panics)
// - Built-in logging capabilities
// - Superior SSL certificate support
package postgres2
import (
@ -7,7 +14,7 @@ import (
"strconv"
"time"
_ "github.com/lib/pq"
_ "github.com/jackc/pgx/v5/stdlib"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
"github.com/seaweedfs/seaweedfs/weed/filer/postgres"
@ -44,13 +51,14 @@ func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix
configuration.GetString(prefix+"sslkey"),
configuration.GetString(prefix+"sslrootcert"),
configuration.GetString(prefix+"sslcrl"),
configuration.GetBool(prefix+"pgbouncer_compatible"),
configuration.GetInt(prefix+"connection_max_idle"),
configuration.GetInt(prefix+"connection_max_open"),
configuration.GetInt(prefix+"connection_max_lifetime_seconds"),
)
}
func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = true
if !enableUpsert {
@ -62,13 +70,23 @@ func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableU
UpsertQueryTemplate: upsertQuery,
}
// pgx-optimized connection string with better timeouts and connection handling
sqlUrl := "connect_timeout=30"
// PgBouncer compatibility: add prefer_simple_protocol=true when needed
// This avoids prepared statement issues with PgBouncer's transaction pooling mode
if pgbouncerCompatible {
sqlUrl += " prefer_simple_protocol=true"
}
if hostname != "" {
sqlUrl += " host=" + hostname
}
if port != 0 {
sqlUrl += " port=" + strconv.Itoa(port)
}
// SSL configuration - pgx provides better SSL support than lib/pq
if sslmode != "" {
sqlUrl += " sslmode=" + sslmode
}
@ -96,16 +114,18 @@ func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableU
sqlUrl += " dbname=" + database
adaptedSqlUrl += " dbname=" + database
}
if schema != "" {
if schema != "" && !pgbouncerCompatible {
sqlUrl += " search_path=" + schema
adaptedSqlUrl += " search_path=" + schema
}
var dbErr error
store.DB, dbErr = sql.Open("postgres", sqlUrl)
store.DB, dbErr = sql.Open("pgx", sqlUrl)
if dbErr != nil {
store.DB.Close()
if store.DB != nil {
store.DB.Close()
}
store.DB = nil
return fmt.Errorf("can not connect to %s error:%v", adaptedSqlUrl, err)
return fmt.Errorf("can not connect to %s error:%v", adaptedSqlUrl, dbErr)
}
store.DB.SetMaxIdleConns(maxIdle)