add s3 replication sink

weed/command/filer_replication.go

@@ -6,6 +6,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/spf13/viper"
 	"strings"
+	"github.com/chrislusf/seaweedfs/weed/replication/sink"
 )
 
 func init() {
@@ -57,7 +58,21 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 		}
 	}
 
-	replicator := replication.NewReplicator(config.Sub("source.filer"), config.Sub("sink.filer"))
+	var dataSink sink.ReplicationSink
+	for _, sk := range sink.Sinks {
+		if config.GetBool("sink." + sk.GetName() + ".enabled") {
+			viperSub := config.Sub("sink." + sk.GetName())
+			if err := sk.Initialize(viperSub); err != nil {
+				glog.Fatalf("Failed to initialize sink for %s: %+v",
+					sk.GetName(), err)
+			}
+			glog.V(0).Infof("Configure sink to %s", sk.GetName())
+			dataSink = sk
+			break
+		}
+	}
+
+	replicator := replication.NewReplicator(config.Sub("source.filer"), dataSink)
 
 	for {
 		key, m, err := notificationInput.ReceiveMessage()

weed/command/scaffold.go

@@ -161,7 +161,7 @@ grpcAddress = "localhost:18888"
 directory = "/buckets"    # all files under this directory tree are replicated
 
 [notification.kafka]
-enabled = true
+enabled = false
 hosts = [
   "localhost:9092"
 ]
@@ -170,12 +170,22 @@ offsetFile = "./last.offset"
 offsetSaveIntervalSeconds = 10
 
 [sink.filer]
-enabled = true
+enabled = false
 grpcAddress = "localhost:18888"
 directory = "/backup"    # all replicated files are under this directory tree
 replication = ""
 collection = ""
 ttlSec = 0
 
+[sink.s3]
+# See https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sessions.html
+# default loads credentials from the shared credentials file (~/.aws/credentials). 
+enabled = false
+aws_access_key_id     = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
+aws_secret_access_key = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
+region = "us-east-2"
+bucket = "your_bucket_name"    # an existing bucket
+directory = ""                 # destination directory (do not prefix or suffix with "/")
+
 `
 )