tier storage: support downloading the remote dat files

weed/storage/backend/backend.go

@@ -25,6 +25,8 @@ type BackendStorage interface {
 	ToProperties() map[string]string
 	NewStorageFile(key string, tierInfo *volume_server_pb.VolumeTierInfo) BackendStorageFile
 	CopyFile(f *os.File, fn func(progressed int64, percentage float32) error) (key string, size int64, err error)
+	DownloadFile(fileName string, key string, fn func(progressed int64, percentage float32) error) (size int64, err error)
+	DeleteFile(key string) (err error)
 }
 
 type StringProperties interface {
@@ -41,6 +43,7 @@ var (
 	BackendStorages         = make(map[string]BackendStorage)
 )
 
+// used by master to load remote storage configurations
 func LoadConfiguration(config *viper.Viper) {
 
 	StorageBackendPrefix := "storage.backend"
@@ -70,6 +73,7 @@ func LoadConfiguration(config *viper.Viper) {
 
 }
 
+// used by volume server to receive remote storage configurations from master
 func LoadFromPbStorageBackends(storageBackends []*master_pb.StorageBackend) {
 
 	for _, storageBackend := range storageBackends {

weed/storage/backend/s3_backend/s3_backend.go

@@ -9,10 +9,11 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/google/uuid"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
-	"github.com/google/uuid"
 )
 
 func init() {
@@ -85,6 +86,24 @@ func (s *S3BackendStorage) CopyFile(f *os.File, fn func(progressed int64, percen
 	return
 }
 
+func (s *S3BackendStorage) DownloadFile(fileName string, key string, fn func(progressed int64, percentage float32) error) (size int64, err error) {
+
+	glog.V(1).Infof("download dat file of %s from remote s3.%s as %s", fileName, s.id, key)
+
+	size, err = downloadFromS3(s.conn, fileName, s.bucket, key, fn)
+
+	return
+}
+
+func (s *S3BackendStorage) DeleteFile(key string) (err error) {
+
+	glog.V(1).Infof("delete dat file %s from remote", key)
+
+	err = deleteFromS3(s.conn, s.bucket, key)
+
+	return
+}
+
 type S3BackendStorageFile struct {
 	backendStorage *S3BackendStorage
 	key            string

weed/storage/backend/s3_backend/s3_download.go

@@ -0,0 +1,98 @@
+package s3_backend
+
+import (
+	"fmt"
+	"os"
+	"sync/atomic"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+)
+
+func downloadFromS3(sess s3iface.S3API, destFileName string, sourceBucket string, sourceKey string,
+	fn func(progressed int64, percentage float32) error) (fileSize int64, err error) {
+
+	fileSize, err = getFileSize(sess, sourceBucket, sourceKey)
+	if err != nil {
+		return
+	}
+
+	//open the file
+	f, err := os.OpenFile(destFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return 0, fmt.Errorf("failed to open file %q, %v", destFileName, err)
+	}
+	defer f.Close()
+
+	// Create a downloader with the session and custom options
+	downloader := s3manager.NewDownloaderWithClient(sess, func(u *s3manager.Downloader) {
+		u.PartSize = int64(64 * 1024 * 1024)
+		u.Concurrency = 5
+	})
+
+	fileWriter := &s3DownloadProgressedWriter{
+		fp:      f,
+		size:    fileSize,
+		written: 0,
+		fn:      fn,
+	}
+
+	// Download the file from S3.
+	fileSize, err = downloader.Download(fileWriter,&s3.GetObjectInput{
+		Bucket: aws.String(sourceBucket),
+		Key:    aws.String(sourceKey),
+	})
+	if err != nil {
+		return fileSize, fmt.Errorf("failed to download file %s: %v", destFileName, err)
+	}
+
+	glog.V(1).Infof("downloaded file %s\n", destFileName)
+
+	return
+}
+
+// adapted from https://github.com/aws/aws-sdk-go/pull/1868
+// and https://petersouter.xyz/s3-download-progress-bar-in-golang/
+type s3DownloadProgressedWriter struct {
+	fp      *os.File
+	size    int64
+	written int64
+	fn      func(progressed int64, percentage float32) error
+}
+
+func (w *s3DownloadProgressedWriter) WriteAt(p []byte, off int64) (int, error) {
+	n, err := w.fp.WriteAt(p, off)
+	if err != nil {
+		return n, err
+	}
+
+	// Got the length have read( or means has uploaded), and you can construct your message
+	atomic.AddInt64(&w.written, int64(n))
+
+	if w.fn != nil {
+		written := w.written
+		if err := w.fn(written, float32(written*100)/float32(w.size)); err != nil {
+			return n, err
+		}
+	}
+
+	return n, err
+}
+
+func getFileSize(svc s3iface.S3API, bucket string, key string) (filesize int64, error error) {
+	params := &s3.HeadObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	}
+
+	resp, err := svc.HeadObject(params)
+	if err != nil {
+		return 0, err
+	}
+
+	return *resp.ContentLength, nil
+}

weed/storage/backend/s3_backend/s3_sessions.go

@@ -52,3 +52,11 @@ func createSession(awsAccessKeyId, awsSecretAccessKey, region string) (s3iface.S
 	return t, nil
 
 }
+
+func deleteFromS3(sess s3iface.S3API, sourceBucket string, sourceKey string) (err error) {
+	_, err = sess.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String(sourceBucket),
+		Key:    aws.String(sourceKey),
+	})
+	return err
+}