diff --git a/weed/command/iam.go b/weed/command/iam.go index f67173389..1839bba2c 100644 --- a/weed/command/iam.go +++ b/weed/command/iam.go @@ -21,7 +21,6 @@ import ( _ "github.com/seaweedfs/seaweedfs/weed/credential/filer_etc" _ "github.com/seaweedfs/seaweedfs/weed/credential/memory" _ "github.com/seaweedfs/seaweedfs/weed/credential/postgres" - _ "github.com/seaweedfs/seaweedfs/weed/credential/sqlite" ) var ( diff --git a/weed/command/scaffold/credential.toml b/weed/command/scaffold/credential.toml index 380867800..d217786d6 100644 --- a/weed/command/scaffold/credential.toml +++ b/weed/command/scaffold/credential.toml @@ -12,12 +12,6 @@ enabled = true # filer address and grpc_dial_option will be automatically configured by the server -# SQLite credential store (recommended for single-node deployments) -[credential.sqlite] -enabled = false -file = "/var/lib/seaweedfs/credentials.db" -# Optional: table name prefix (default: "sw_") -table_prefix = "sw_" # PostgreSQL credential store (recommended for multi-node deployments) [credential.postgres] @@ -49,7 +43,5 @@ enabled = false # # Examples: # export WEED_CREDENTIAL_POSTGRES_PASSWORD=secret -# export WEED_CREDENTIAL_SQLITE_FILE=/custom/path/credentials.db # export WEED_CREDENTIAL_POSTGRES_HOSTNAME=db.example.com # export WEED_CREDENTIAL_FILER_ETC_ENABLED=true -# export WEED_CREDENTIAL_SQLITE_ENABLED=false \ No newline at end of file diff --git a/weed/credential/README.md b/weed/credential/README.md index a08bc914e..dc3dc04c4 100644 --- a/weed/credential/README.md +++ b/weed/credential/README.md @@ -20,7 +20,6 @@ This document shows how the credential store has been integrated into SeaweedFS' The credential store provides a pluggable backend for storing S3 identities and credentials, supporting: - **Filer-based storage** (filer_etc) - Uses existing filer storage (default) -- **SQLite** - Local database storage - **PostgreSQL** - Shared database for multiple servers - **Memory** - In-memory storage for testing @@ -40,10 +39,6 @@ This creates a `credential.toml` file with all available options. The filer_etc [credential.filer_etc] enabled = true -# SQLite credential store (recommended for single-node deployments) -[credential.sqlite] -enabled = false -file = "/var/lib/seaweedfs/credentials.db" # PostgreSQL credential store (recommended for multi-node deployments) [credential.postgres] @@ -79,14 +74,7 @@ enabled = true This uses the existing filer storage and is compatible with current deployments. -### SQLite Store -```toml -[credential.sqlite] -enabled = true -file = "/var/lib/seaweedfs/credentials.db" -table_prefix = "sw_" -``` ### PostgreSQL Store @@ -121,15 +109,12 @@ All credential configuration can be overridden with environment variables: # Override PostgreSQL password export WEED_CREDENTIAL_POSTGRES_PASSWORD=secret -# Override SQLite file path -export WEED_CREDENTIAL_SQLITE_FILE=/custom/path/credentials.db # Override PostgreSQL hostname export WEED_CREDENTIAL_POSTGRES_HOSTNAME=db.example.com # Enable/disable stores export WEED_CREDENTIAL_FILER_ETC_ENABLED=true -export WEED_CREDENTIAL_SQLITE_ENABLED=false ``` Rules: @@ -159,7 +144,7 @@ if credConfig, err := credential.LoadCredentialConfiguration(); err == nil && cr ## Benefits 1. **Easy Configuration** - Generate template with `weed scaffold -config=credential` -2. **Pluggable Storage** - Switch between filer_etc, SQLite, PostgreSQL without code changes +2. **Pluggable Storage** - Switch between filer_etc, PostgreSQL without code changes 3. **Backward Compatibility** - Filer-based storage works with existing deployments 4. **Scalability** - Database stores support multiple concurrent servers 5. **Performance** - Database access can be faster than file-based storage diff --git a/weed/credential/credential_store.go b/weed/credential/credential_store.go index 60a86cfda..cd36263dc 100644 --- a/weed/credential/credential_store.go +++ b/weed/credential/credential_store.go @@ -23,7 +23,6 @@ const ( StoreTypeMemory CredentialStoreTypeName = "memory" StoreTypeFilerEtc CredentialStoreTypeName = "filer_etc" StoreTypePostgres CredentialStoreTypeName = "postgres" - StoreTypeSQLite CredentialStoreTypeName = "sqlite" ) // CredentialStore defines the interface for user credential storage and retrieval diff --git a/weed/credential/credential_test.go b/weed/credential/credential_test.go index 70eeb7b0c..dd1449fa5 100644 --- a/weed/credential/credential_test.go +++ b/weed/credential/credential_test.go @@ -19,10 +19,10 @@ func TestCredentialStoreInterface(t *testing.T) { storeNames := GetAvailableStores() expectedStores := []string{string(StoreTypeFilerEtc), string(StoreTypeMemory)} - // Add SQLite and PostgreSQL if they're available (build tags dependent) + // Add PostgreSQL if it's available (build tags dependent) for _, storeName := range storeNames { found := false - for _, expected := range append(expectedStores, string(StoreTypeSQLite), string(StoreTypePostgres)) { + for _, expected := range append(expectedStores, string(StoreTypePostgres)) { if string(storeName) == expected { found = true break @@ -319,10 +319,10 @@ func TestGetAvailableStores(t *testing.T) { // We expect at least memory and filer_etc stores to be available expectedStores := []string{string(StoreTypeFilerEtc), string(StoreTypeMemory)} - // Add SQLite and PostgreSQL if they're available (build tags dependent) + // Add PostgreSQL if it's available (build tags dependent) for _, storeName := range storeNames { found := false - for _, expected := range append(expectedStores, string(StoreTypeSQLite), string(StoreTypePostgres)) { + for _, expected := range append(expectedStores, string(StoreTypePostgres)) { if storeName == expected { found = true break diff --git a/weed/credential/sqlite/sqlite_store.go b/weed/credential/sqlite/sqlite_store.go deleted file mode 100644 index 70d015fc9..000000000 --- a/weed/credential/sqlite/sqlite_store.go +++ /dev/null @@ -1,557 +0,0 @@ -package sqlite - -import ( - "context" - "database/sql" - "encoding/json" - "fmt" - "os" - "path/filepath" - - "github.com/seaweedfs/seaweedfs/weed/credential" - "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" - "github.com/seaweedfs/seaweedfs/weed/util" - - _ "modernc.org/sqlite" -) - -func init() { - credential.Stores = append(credential.Stores, &SqliteStore{}) -} - -// SqliteStore implements CredentialStore using SQLite -type SqliteStore struct { - db *sql.DB - configured bool -} - -func (store *SqliteStore) GetName() credential.CredentialStoreTypeName { - return credential.StoreTypeSQLite -} - -func (store *SqliteStore) Initialize(configuration util.Configuration, prefix string) error { - if store.configured { - return nil - } - - dbFile := configuration.GetString(prefix + "dbFile") - if dbFile == "" { - dbFile = "seaweedfs_credentials.db" - } - - // Create directory if it doesn't exist - dir := filepath.Dir(dbFile) - if dir != "." { - if err := os.MkdirAll(dir, 0755); err != nil { - return fmt.Errorf("failed to create directory %s: %v", dir, err) - } - } - - db, err := sql.Open("sqlite", dbFile) - if err != nil { - return fmt.Errorf("failed to open database: %v", err) - } - - // Test connection - if err := db.Ping(); err != nil { - db.Close() - return fmt.Errorf("failed to ping database: %v", err) - } - - store.db = db - - // Create tables if they don't exist - if err := store.createTables(); err != nil { - db.Close() - return fmt.Errorf("failed to create tables: %v", err) - } - - store.configured = true - return nil -} - -func (store *SqliteStore) createTables() error { - // Create users table - usersTable := ` - CREATE TABLE IF NOT EXISTS users ( - username TEXT PRIMARY KEY, - email TEXT, - account_data TEXT, - actions TEXT, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - updated_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - CREATE INDEX IF NOT EXISTS idx_users_email ON users(email); - ` - - // Create credentials table - credentialsTable := ` - CREATE TABLE IF NOT EXISTS credentials ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - username TEXT REFERENCES users(username) ON DELETE CASCADE, - access_key TEXT UNIQUE NOT NULL, - secret_key TEXT NOT NULL, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - CREATE INDEX IF NOT EXISTS idx_credentials_username ON credentials(username); - CREATE INDEX IF NOT EXISTS idx_credentials_access_key ON credentials(access_key); - ` - - // Execute table creation - if _, err := store.db.Exec(usersTable); err != nil { - return fmt.Errorf("failed to create users table: %v", err) - } - - if _, err := store.db.Exec(credentialsTable); err != nil { - return fmt.Errorf("failed to create credentials table: %v", err) - } - - return nil -} - -func (store *SqliteStore) LoadConfiguration(ctx context.Context) (*iam_pb.S3ApiConfiguration, error) { - if !store.configured { - return nil, fmt.Errorf("store not configured") - } - - config := &iam_pb.S3ApiConfiguration{} - - // Query all users - rows, err := store.db.QueryContext(ctx, "SELECT username, email, account_data, actions FROM users") - if err != nil { - return nil, fmt.Errorf("failed to query users: %v", err) - } - defer rows.Close() - - for rows.Next() { - var username, email, accountDataJSON, actionsJSON string - - if err := rows.Scan(&username, &email, &accountDataJSON, &actionsJSON); err != nil { - return nil, fmt.Errorf("failed to scan user row: %v", err) - } - - identity := &iam_pb.Identity{ - Name: username, - } - - // Parse account data - if accountDataJSON != "" { - if err := json.Unmarshal([]byte(accountDataJSON), &identity.Account); err != nil { - return nil, fmt.Errorf("failed to unmarshal account data for user %s: %v", username, err) - } - } - - // Parse actions - if actionsJSON != "" { - if err := json.Unmarshal([]byte(actionsJSON), &identity.Actions); err != nil { - return nil, fmt.Errorf("failed to unmarshal actions for user %s: %v", username, err) - } - } - - // Query credentials for this user - credRows, err := store.db.QueryContext(ctx, "SELECT access_key, secret_key FROM credentials WHERE username = ?", username) - if err != nil { - return nil, fmt.Errorf("failed to query credentials for user %s: %v", username, err) - } - - for credRows.Next() { - var accessKey, secretKey string - if err := credRows.Scan(&accessKey, &secretKey); err != nil { - credRows.Close() - return nil, fmt.Errorf("failed to scan credential row for user %s: %v", username, err) - } - - identity.Credentials = append(identity.Credentials, &iam_pb.Credential{ - AccessKey: accessKey, - SecretKey: secretKey, - }) - } - credRows.Close() - - config.Identities = append(config.Identities, identity) - } - - return config, nil -} - -func (store *SqliteStore) SaveConfiguration(ctx context.Context, config *iam_pb.S3ApiConfiguration) error { - if !store.configured { - return fmt.Errorf("store not configured") - } - - // Start transaction - tx, err := store.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %v", err) - } - defer tx.Rollback() - - // Clear existing data - if _, err := tx.ExecContext(ctx, "DELETE FROM credentials"); err != nil { - return fmt.Errorf("failed to clear credentials: %v", err) - } - if _, err := tx.ExecContext(ctx, "DELETE FROM users"); err != nil { - return fmt.Errorf("failed to clear users: %v", err) - } - - // Insert all identities - for _, identity := range config.Identities { - // Marshal account data - var accountDataJSON string - if identity.Account != nil { - data, err := json.Marshal(identity.Account) - if err != nil { - return fmt.Errorf("failed to marshal account data for user %s: %v", identity.Name, err) - } - accountDataJSON = string(data) - } - - // Marshal actions - var actionsJSON string - if identity.Actions != nil { - data, err := json.Marshal(identity.Actions) - if err != nil { - return fmt.Errorf("failed to marshal actions for user %s: %v", identity.Name, err) - } - actionsJSON = string(data) - } - - // Insert user - _, err := tx.ExecContext(ctx, - "INSERT INTO users (username, email, account_data, actions) VALUES (?, ?, ?, ?)", - identity.Name, "", accountDataJSON, actionsJSON) - if err != nil { - return fmt.Errorf("failed to insert user %s: %v", identity.Name, err) - } - - // Insert credentials - for _, cred := range identity.Credentials { - _, err := tx.ExecContext(ctx, - "INSERT INTO credentials (username, access_key, secret_key) VALUES (?, ?, ?)", - identity.Name, cred.AccessKey, cred.SecretKey) - if err != nil { - return fmt.Errorf("failed to insert credential for user %s: %v", identity.Name, err) - } - } - } - - return tx.Commit() -} - -func (store *SqliteStore) CreateUser(ctx context.Context, identity *iam_pb.Identity) error { - if !store.configured { - return fmt.Errorf("store not configured") - } - - // Check if user already exists - var count int - err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM users WHERE username = ?", identity.Name).Scan(&count) - if err != nil { - return fmt.Errorf("failed to check user existence: %v", err) - } - if count > 0 { - return credential.ErrUserAlreadyExists - } - - // Start transaction - tx, err := store.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %v", err) - } - defer tx.Rollback() - - // Marshal account data - var accountDataJSON string - if identity.Account != nil { - data, err := json.Marshal(identity.Account) - if err != nil { - return fmt.Errorf("failed to marshal account data: %v", err) - } - accountDataJSON = string(data) - } - - // Marshal actions - var actionsJSON string - if identity.Actions != nil { - data, err := json.Marshal(identity.Actions) - if err != nil { - return fmt.Errorf("failed to marshal actions: %v", err) - } - actionsJSON = string(data) - } - - // Insert user - _, err = tx.ExecContext(ctx, - "INSERT INTO users (username, email, account_data, actions) VALUES (?, ?, ?, ?)", - identity.Name, "", accountDataJSON, actionsJSON) - if err != nil { - return fmt.Errorf("failed to insert user: %v", err) - } - - // Insert credentials - for _, cred := range identity.Credentials { - _, err = tx.ExecContext(ctx, - "INSERT INTO credentials (username, access_key, secret_key) VALUES (?, ?, ?)", - identity.Name, cred.AccessKey, cred.SecretKey) - if err != nil { - return fmt.Errorf("failed to insert credential: %v", err) - } - } - - return tx.Commit() -} - -func (store *SqliteStore) GetUser(ctx context.Context, username string) (*iam_pb.Identity, error) { - if !store.configured { - return nil, fmt.Errorf("store not configured") - } - - var email, accountDataJSON, actionsJSON string - - err := store.db.QueryRowContext(ctx, - "SELECT email, account_data, actions FROM users WHERE username = ?", - username).Scan(&email, &accountDataJSON, &actionsJSON) - if err != nil { - if err == sql.ErrNoRows { - return nil, credential.ErrUserNotFound - } - return nil, fmt.Errorf("failed to query user: %v", err) - } - - identity := &iam_pb.Identity{ - Name: username, - } - - // Parse account data - if accountDataJSON != "" { - if err := json.Unmarshal([]byte(accountDataJSON), &identity.Account); err != nil { - return nil, fmt.Errorf("failed to unmarshal account data: %v", err) - } - } - - // Parse actions - if actionsJSON != "" { - if err := json.Unmarshal([]byte(actionsJSON), &identity.Actions); err != nil { - return nil, fmt.Errorf("failed to unmarshal actions: %v", err) - } - } - - // Query credentials - rows, err := store.db.QueryContext(ctx, "SELECT access_key, secret_key FROM credentials WHERE username = ?", username) - if err != nil { - return nil, fmt.Errorf("failed to query credentials: %v", err) - } - defer rows.Close() - - for rows.Next() { - var accessKey, secretKey string - if err := rows.Scan(&accessKey, &secretKey); err != nil { - return nil, fmt.Errorf("failed to scan credential: %v", err) - } - - identity.Credentials = append(identity.Credentials, &iam_pb.Credential{ - AccessKey: accessKey, - SecretKey: secretKey, - }) - } - - return identity, nil -} - -func (store *SqliteStore) UpdateUser(ctx context.Context, username string, identity *iam_pb.Identity) error { - if !store.configured { - return fmt.Errorf("store not configured") - } - - // Start transaction - tx, err := store.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %v", err) - } - defer tx.Rollback() - - // Check if user exists - var count int - err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM users WHERE username = ?", username).Scan(&count) - if err != nil { - return fmt.Errorf("failed to check user existence: %v", err) - } - if count == 0 { - return credential.ErrUserNotFound - } - - // Marshal account data - var accountDataJSON string - if identity.Account != nil { - data, err := json.Marshal(identity.Account) - if err != nil { - return fmt.Errorf("failed to marshal account data: %v", err) - } - accountDataJSON = string(data) - } - - // Marshal actions - var actionsJSON string - if identity.Actions != nil { - data, err := json.Marshal(identity.Actions) - if err != nil { - return fmt.Errorf("failed to marshal actions: %v", err) - } - actionsJSON = string(data) - } - - // Update user - _, err = tx.ExecContext(ctx, - "UPDATE users SET email = ?, account_data = ?, actions = ?, updated_at = CURRENT_TIMESTAMP WHERE username = ?", - "", accountDataJSON, actionsJSON, username) - if err != nil { - return fmt.Errorf("failed to update user: %v", err) - } - - // Delete existing credentials - _, err = tx.ExecContext(ctx, "DELETE FROM credentials WHERE username = ?", username) - if err != nil { - return fmt.Errorf("failed to delete existing credentials: %v", err) - } - - // Insert new credentials - for _, cred := range identity.Credentials { - _, err = tx.ExecContext(ctx, - "INSERT INTO credentials (username, access_key, secret_key) VALUES (?, ?, ?)", - username, cred.AccessKey, cred.SecretKey) - if err != nil { - return fmt.Errorf("failed to insert credential: %v", err) - } - } - - return tx.Commit() -} - -func (store *SqliteStore) DeleteUser(ctx context.Context, username string) error { - if !store.configured { - return fmt.Errorf("store not configured") - } - - result, err := store.db.ExecContext(ctx, "DELETE FROM users WHERE username = ?", username) - if err != nil { - return fmt.Errorf("failed to delete user: %v", err) - } - - rowsAffected, err := result.RowsAffected() - if err != nil { - return fmt.Errorf("failed to get rows affected: %v", err) - } - - if rowsAffected == 0 { - return credential.ErrUserNotFound - } - - return nil -} - -func (store *SqliteStore) ListUsers(ctx context.Context) ([]string, error) { - if !store.configured { - return nil, fmt.Errorf("store not configured") - } - - rows, err := store.db.QueryContext(ctx, "SELECT username FROM users ORDER BY username") - if err != nil { - return nil, fmt.Errorf("failed to query users: %v", err) - } - defer rows.Close() - - var usernames []string - for rows.Next() { - var username string - if err := rows.Scan(&username); err != nil { - return nil, fmt.Errorf("failed to scan username: %v", err) - } - usernames = append(usernames, username) - } - - return usernames, nil -} - -func (store *SqliteStore) GetUserByAccessKey(ctx context.Context, accessKey string) (*iam_pb.Identity, error) { - if !store.configured { - return nil, fmt.Errorf("store not configured") - } - - var username string - err := store.db.QueryRowContext(ctx, "SELECT username FROM credentials WHERE access_key = ?", accessKey).Scan(&username) - if err != nil { - if err == sql.ErrNoRows { - return nil, credential.ErrAccessKeyNotFound - } - return nil, fmt.Errorf("failed to query access key: %v", err) - } - - return store.GetUser(ctx, username) -} - -func (store *SqliteStore) CreateAccessKey(ctx context.Context, username string, cred *iam_pb.Credential) error { - if !store.configured { - return fmt.Errorf("store not configured") - } - - // Check if user exists - var count int - err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM users WHERE username = ?", username).Scan(&count) - if err != nil { - return fmt.Errorf("failed to check user existence: %v", err) - } - if count == 0 { - return credential.ErrUserNotFound - } - - // Insert credential - _, err = store.db.ExecContext(ctx, - "INSERT INTO credentials (username, access_key, secret_key) VALUES (?, ?, ?)", - username, cred.AccessKey, cred.SecretKey) - if err != nil { - return fmt.Errorf("failed to insert credential: %v", err) - } - - return nil -} - -func (store *SqliteStore) DeleteAccessKey(ctx context.Context, username string, accessKey string) error { - if !store.configured { - return fmt.Errorf("store not configured") - } - - result, err := store.db.ExecContext(ctx, - "DELETE FROM credentials WHERE username = ? AND access_key = ?", - username, accessKey) - if err != nil { - return fmt.Errorf("failed to delete access key: %v", err) - } - - rowsAffected, err := result.RowsAffected() - if err != nil { - return fmt.Errorf("failed to get rows affected: %v", err) - } - - if rowsAffected == 0 { - // Check if user exists - var count int - err = store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM users WHERE username = ?", username).Scan(&count) - if err != nil { - return fmt.Errorf("failed to check user existence: %v", err) - } - if count == 0 { - return credential.ErrUserNotFound - } - return credential.ErrAccessKeyNotFound - } - - return nil -} - -func (store *SqliteStore) Shutdown() { - if store.db != nil { - store.db.Close() - store.db = nil - } - store.configured = false -} diff --git a/weed/credential/test/integration_test.go b/weed/credential/test/integration_test.go index 53cd80bc0..c1e55ecf8 100644 --- a/weed/credential/test/integration_test.go +++ b/weed/credential/test/integration_test.go @@ -12,7 +12,6 @@ import ( _ "github.com/seaweedfs/seaweedfs/weed/credential/filer_etc" _ "github.com/seaweedfs/seaweedfs/weed/credential/memory" _ "github.com/seaweedfs/seaweedfs/weed/credential/postgres" - _ "github.com/seaweedfs/seaweedfs/weed/credential/sqlite" ) func TestStoreRegistration(t *testing.T) { @@ -22,7 +21,7 @@ func TestStoreRegistration(t *testing.T) { t.Fatal("No credential stores registered") } - expectedStores := []string{string(credential.StoreTypeFilerEtc), string(credential.StoreTypeMemory), string(credential.StoreTypeSQLite), string(credential.StoreTypePostgres)} + expectedStores := []string{string(credential.StoreTypeFilerEtc), string(credential.StoreTypeMemory), string(credential.StoreTypePostgres)} // Verify all expected stores are present for _, expected := range expectedStores {