support env variables to overwrite toml file

This commit is contained in:
Chris Lu
2020-01-29 09:09:55 -08:00
parent 27b94cb65b
commit d335f04de6
60 changed files with 268 additions and 247 deletions

View File

@@ -35,7 +35,7 @@ type StringProperties interface {
type StorageType string
type BackendStorageFactory interface {
StorageType() StorageType
BuildStorage(configuration StringProperties, id string) (BackendStorage, error)
BuildStorage(configuration StringProperties, configPrefix string, id string) (BackendStorage, error)
}
var (
@@ -48,19 +48,17 @@ func LoadConfiguration(config *viper.Viper) {
StorageBackendPrefix := "storage.backend"
backendSub := config.Sub(StorageBackendPrefix)
for backendTypeName := range config.GetStringMap(StorageBackendPrefix) {
backendStorageFactory, found := BackendStorageFactories[StorageType(backendTypeName)]
if !found {
glog.Fatalf("backend storage type %s not found", backendTypeName)
}
backendTypeSub := backendSub.Sub(backendTypeName)
for backendStorageId := range backendSub.GetStringMap(backendTypeName) {
if !backendTypeSub.GetBool(backendStorageId + ".enabled") {
for backendStorageId := range config.GetStringMap(StorageBackendPrefix + "." + backendTypeName) {
if !config.GetBool(StorageBackendPrefix + "." + backendTypeName + "." + backendStorageId + ".enabled") {
continue
}
backendStorage, buildErr := backendStorageFactory.BuildStorage(backendTypeSub.Sub(backendStorageId), backendStorageId)
backendStorage, buildErr := backendStorageFactory.BuildStorage(config,
StorageBackendPrefix+"."+backendTypeName+"."+backendStorageId+".", backendStorageId)
if buildErr != nil {
glog.Fatalf("fail to create backend storage %s.%s", backendTypeName, backendStorageId)
}
@@ -82,7 +80,7 @@ func LoadFromPbStorageBackends(storageBackends []*master_pb.StorageBackend) {
glog.Warningf("storage type %s not found", storageBackend.Type)
continue
}
backendStorage, buildErr := backendStorageFactory.BuildStorage(newProperties(storageBackend.Properties), storageBackend.Id)
backendStorage, buildErr := backendStorageFactory.BuildStorage(newProperties(storageBackend.Properties), "", storageBackend.Id)
if buildErr != nil {
glog.Fatalf("fail to create backend storage %s.%s", storageBackend.Type, storageBackend.Id)
}

View File

@@ -26,8 +26,8 @@ type S3BackendFactory struct {
func (factory *S3BackendFactory) StorageType() backend.StorageType {
return backend.StorageType("s3")
}
func (factory *S3BackendFactory) BuildStorage(configuration backend.StringProperties, id string) (backend.BackendStorage, error) {
return newS3BackendStorage(configuration, id)
func (factory *S3BackendFactory) BuildStorage(configuration backend.StringProperties, configPrefix string, id string) (backend.BackendStorage, error) {
return newS3BackendStorage(configuration, configPrefix, id)
}
type S3BackendStorage struct {
@@ -39,13 +39,13 @@ type S3BackendStorage struct {
conn s3iface.S3API
}
func newS3BackendStorage(configuration backend.StringProperties, id string) (s *S3BackendStorage, err error) {
func newS3BackendStorage(configuration backend.StringProperties, configPrefix string, id string) (s *S3BackendStorage, err error) {
s = &S3BackendStorage{}
s.id = id
s.aws_access_key_id = configuration.GetString("aws_access_key_id")
s.aws_secret_access_key = configuration.GetString("aws_secret_access_key")
s.region = configuration.GetString("region")
s.bucket = configuration.GetString("bucket")
s.aws_access_key_id = configuration.GetString(configPrefix + "aws_access_key_id")
s.aws_secret_access_key = configuration.GetString(configPrefix + "aws_secret_access_key")
s.region = configuration.GetString(configPrefix + "region")
s.bucket = configuration.GetString(configPrefix + "bucket")
s.conn, err = createSession(s.aws_access_key_id, s.aws_secret_access_key, s.region)
glog.V(0).Infof("created backend storage s3.%s for region %s bucket %s", s.id, s.region, s.bucket)

View File

@@ -356,7 +356,7 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, prealloca
func copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName string, sb super_block.SuperBlock, version needle.Version, preallocate int64) (err error) {
var (
srcDatBackend, dstDatBackend backend.BackendStorageFile
dataFile *os.File
dataFile *os.File
)
if dstDatBackend, err = createVolumeFile(dstDatName, preallocate, 0); err != nil {
return