pass backend config from master to volume servers

weed/storage/backend/backend.go

@@ -3,10 +3,11 @@ package backend
 import (
 	"io"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/spf13/viper"
 )
 
@@ -21,14 +22,17 @@ type BackendStorageFile interface {
 }
 
 type BackendStorage interface {
-	Name() string
+	ToProperties() map[string]string
 	NewStorageFile(key string) BackendStorageFile
 }
 
+type StringProperties interface {
+	GetString(key string) string
+}
 type StorageType string
 type BackendStorageFactory interface {
 	StorageType() StorageType
-	BuildStorage(configuration util.Configuration, id string) (BackendStorage, error)
+	BuildStorage(configuration StringProperties, id string) (BackendStorage, error)
 }
 
 var (
@@ -49,6 +53,9 @@ func LoadConfiguration(config *viper.Viper) {
 		}
 		backendTypeSub := backendSub.Sub(backendTypeName)
 		for backendStorageId, _ := range backendSub.GetStringMap(backendTypeName) {
+			if !backendTypeSub.GetBool(backendStorageId + ".enabled") {
+				continue
+			}
 			backendStorage, buildErr := backendStorageFactory.BuildStorage(backendTypeSub.Sub(backendStorageId), backendStorageId)
 			if buildErr != nil {
 				glog.Fatalf("fail to create backend storage %s.%s", backendTypeName, backendStorageId)
@@ -61,3 +68,54 @@ func LoadConfiguration(config *viper.Viper) {
 	}
 
 }
+
+func LoadFromPbStorageBackends(storageBackends []*master_pb.StorageBackend) {
+
+	for _, storageBackend := range storageBackends {
+		backendStorageFactory, found := BackendStorageFactories[StorageType(storageBackend.Type)]
+		if !found {
+			glog.Warningf("storage type %s not found", storageBackend.Type)
+			continue
+		}
+		backendStorage, buildErr := backendStorageFactory.BuildStorage(newProperties(storageBackend.Properties), storageBackend.Id)
+		if buildErr != nil {
+			glog.Fatalf("fail to create backend storage %s.%s", storageBackend.Type, storageBackend.Id)
+		}
+		BackendStorages[storageBackend.Type+"."+storageBackend.Id] = backendStorage
+		if storageBackend.Id == "default" {
+			BackendStorages[storageBackend.Type] = backendStorage
+		}
+	}
+}
+
+type Properties struct {
+	m map[string]string
+}
+
+func newProperties(m map[string]string) *Properties {
+	return &Properties{m: m}
+}
+
+func (p *Properties) GetString(key string) string {
+	if v, found := p.m[key]; found {
+		return v
+	}
+	return ""
+}
+
+func ToPbStorageBackends() (backends []*master_pb.StorageBackend) {
+	for sName, s := range BackendStorages {
+		parts := strings.Split(sName, ".")
+		if len(parts) != 2 {
+			continue
+		}
+
+		sType, sId := parts[0], parts[1]
+		backends = append(backends, &master_pb.StorageBackend{
+			Type:       sType,
+			Id:         sId,
+			Properties: s.ToProperties(),
+		})
+	}
+	return
+}

weed/storage/backend/s3_backend/s3_backend.go

@@ -10,7 +10,6 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3/s3iface"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
-	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 func init() {
@@ -23,33 +22,39 @@ type S3BackendFactory struct {
 func (factory *S3BackendFactory) StorageType() backend.StorageType {
 	return backend.StorageType("s3")
 }
-func (factory *S3BackendFactory) BuildStorage(configuration util.Configuration, id string) (backend.BackendStorage, error) {
+func (factory *S3BackendFactory) BuildStorage(configuration backend.StringProperties, id string) (backend.BackendStorage, error) {
 	return newS3BackendStorage(configuration, id)
 }
 
 type S3BackendStorage struct {
-	id     string
-	conn   s3iface.S3API
-	region string
-	bucket string
+	id                    string
+	aws_access_key_id     string
+	aws_secret_access_key string
+	region                string
+	bucket                string
+	conn                  s3iface.S3API
 }
 
-func newS3BackendStorage(configuration util.Configuration, id string) (s *S3BackendStorage, err error) {
+func newS3BackendStorage(configuration backend.StringProperties, id string) (s *S3BackendStorage, err error) {
 	s = &S3BackendStorage{}
 	s.id = id
-	s.conn, err = createSession(
-		configuration.GetString("aws_access_key_id"),
-		configuration.GetString("aws_secret_access_key"),
-		configuration.GetString("region"))
+	s.aws_access_key_id = configuration.GetString("aws_access_key_id")
+	s.aws_secret_access_key = configuration.GetString("aws_secret_access_key")
 	s.region = configuration.GetString("region")
 	s.bucket = configuration.GetString("bucket")
+	s.conn, err = createSession(s.aws_access_key_id, s.aws_secret_access_key, s.region)
 
-	glog.V(0).Infof("created s3 backend storage %s for region %s bucket %s", s.Name(), s.region, s.bucket)
+	glog.V(0).Infof("created backend storage s3.%s for region %s bucket %s", s.id, s.region, s.bucket)
 	return
 }
 
-func (s *S3BackendStorage) Name() string {
-	return "s3." + s.id
+func (s *S3BackendStorage) ToProperties() map[string]string {
+	m := make(map[string]string)
+	m["aws_access_key_id"] = s.aws_access_key_id
+	m["aws_secret_access_key"] = s.aws_secret_access_key
+	m["region"] = s.region
+	m["bucket"] = s.bucket
+	return m
 }
 
 func (s *S3BackendStorage) NewStorageFile(key string) backend.BackendStorageFile {