add "weed copy" command to copy files to filer

weed/command/command.go

@@ -11,6 +11,7 @@ var Commands = []*Command{
 	cmdBenchmark,
 	cmdBackup,
 	cmdCompact,
+	cmdCopy,
 	cmdFix,
 	cmdServer,
 	cmdMaster,

weed/command/copy.go

@@ -0,0 +1,147 @@
+package command
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	filer_operation "github.com/chrislusf/seaweedfs/weed/operation/filer"
+	"github.com/chrislusf/seaweedfs/weed/security"
+)
+
+var (
+	copy CopyOptions
+)
+
+type CopyOptions struct {
+	master      *string
+	include     *string
+	replication *string
+	collection  *string
+	ttl         *string
+	maxMB       *int
+	secretKey   *string
+
+	secret security.Secret
+}
+
+func init() {
+	cmdCopy.Run = runCopy // break init cycle
+	cmdCopy.IsDebug = cmdCopy.Flag.Bool("debug", false, "verbose debug information")
+	copy.master = cmdCopy.Flag.String("master", "localhost:9333", "SeaweedFS master location")
+	copy.include = cmdCopy.Flag.String("include", "", "pattens of files to copy, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
+	copy.replication = cmdCopy.Flag.String("replication", "", "replication type")
+	copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
+	copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
+	copy.maxMB = cmdCopy.Flag.Int("maxMB", 0, "split files larger than the limit")
+	copy.secretKey = cmdCopy.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
+}
+
+var cmdCopy = &Command{
+	UsageLine: "copy file_or_dir1 [file_or_dir2 file_or_dir3] http://localhost:8888/path/to/a/folder/",
+	Short:     "copy one or a list of files to a filer folder",
+	Long: `copy one or a list of files, or batch copy one whole folder recursively, to a filer folder
+
+  It can copy one or a list of files or folders.
+
+  If copying a whole folder recursively:
+  All files under the folder and subfolders will be copyed.
+  Optional parameter "-include" allows you to specify the file name patterns.
+
+  If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is.
+  This can save volume server's gzipped processing and allow customizable gzip compression level.
+  The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js".
+
+  If "maxMB" is set to a positive number, files larger than it would be split into chunks and copyed separatedly.
+  The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned.
+
+  `,
+}
+
+func runCopy(cmd *Command, args []string) bool {
+	copy.secret = security.Secret(*copy.secretKey)
+	if len(args) <= 1 {
+		return false
+	}
+	filerDestination := args[len(args)-1]
+	fileOrDirs := args[0 : len(args)-1]
+
+	filerUrl, err := url.Parse(filerDestination)
+	if err != nil {
+		fmt.Printf("The last argument should be a URL on filer: %v\n", err)
+		return false
+	}
+	if len(fileOrDirs) > 1 && !strings.HasSuffix(filerUrl.Path, "/") {
+		fmt.Println("Can not copy multiple items to a file. The last argument must be a folder ended with \"/\"")
+		return false
+	}
+
+	for _, fileOrDir := range fileOrDirs {
+		if !doEachCopy(fileOrDir, filerUrl.Host, filerUrl.Path) {
+			return false
+		}
+	}
+	return true
+}
+
+func doEachCopy(fileOrDir string, host string, path string) bool {
+	f, err := os.Open(fileOrDir)
+	if err != nil {
+		fmt.Printf("Failed to open file %s: %v", fileOrDir, err)
+		return false
+	}
+	defer f.Close()
+
+	fi, err := f.Stat()
+	if err != nil {
+		fmt.Printf("Failed to get stat for file %s: %v", fileOrDir, err)
+		return false
+	}
+
+	mode := fi.Mode()
+	if mode.IsDir() {
+		files, _ := ioutil.ReadDir(fileOrDir)
+		for _, subFileOrDir := range files {
+			if !doEachCopy(fileOrDir+"/"+subFileOrDir.Name(), host, path+fi.Name()+"/") {
+				return false
+			}
+		}
+		return true
+	}
+
+	// this is a regular file
+	if *copy.include != "" {
+		if ok, _ := filepath.Match(*copy.include, filepath.Base(fileOrDir)); !ok {
+			return true
+		}
+	}
+
+	parts, err := operation.NewFileParts([]string{fileOrDir})
+	if err != nil {
+		fmt.Printf("Failed to read file %s: %v", fileOrDir, err)
+	}
+
+	results, err := operation.SubmitFiles(*copy.master, parts,
+		*copy.replication, *copy.collection,
+		*copy.ttl, *copy.maxMB, copy.secret)
+	if err != nil {
+		fmt.Printf("Failed to submit file %s: %v", fileOrDir, err)
+	}
+
+	if strings.HasSuffix(path, "/") {
+		path = path + fi.Name()
+	}
+
+	if err = filer_operation.RegisterFile(host, path, results[0].Fid, copy.secret); err != nil {
+		fmt.Printf("Failed to register file %s on %s: %v", fileOrDir, host, err)
+		return false
+	}
+
+	fmt.Printf("Copy %s => http://%s/%s\n", fileOrDir, host, path)
+
+	return true
+}

weed/command/server.go

@@ -82,7 +82,7 @@ func init() {
 	filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server")
 	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
 	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
-	filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified")
+	filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -dir is specified")
 	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
 	filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
 	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
@@ -164,6 +164,7 @@ func runServer(cmd *Command, args []string) bool {
 
 	if *isStartingFiler {
 		go func() {
+			time.Sleep(1 * time.Second)
 			r := http.NewServeMux()
 			_, nfs_err := weed_server.NewFilerServer(r, *serverBindIp, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection,
 				*filerOptions.defaultReplicaPlacement,

weed/command/upload.go

@@ -15,7 +15,7 @@ var (
 )
 
 type UploadOptions struct {
-	server      *string
+	master      *string
 	dir         *string
 	include     *string
 	replication *string
@@ -28,7 +28,7 @@ type UploadOptions struct {
 func init() {
 	cmdUpload.Run = runUpload // break init cycle
 	cmdUpload.IsDebug = cmdUpload.Flag.Bool("debug", false, "verbose debug information")
-	upload.server = cmdUpload.Flag.String("server", "localhost:9333", "SeaweedFS master location")
+	upload.master = cmdUpload.Flag.String("master", "localhost:9333", "SeaweedFS master location")
 	upload.dir = cmdUpload.Flag.String("dir", "", "Upload the whole folder recursively if specified.")
 	upload.include = cmdUpload.Flag.String("include", "", "pattens of files to upload, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
 	upload.replication = cmdUpload.Flag.String("replication", "", "replication type")
@@ -39,7 +39,7 @@ func init() {
 }
 
 var cmdUpload = &Command{
-	UsageLine: "upload -server=localhost:9333 file1 [file2 file3]\n         weed upload -server=localhost:9333 -dir=one_directory -include=*.pdf",
+	UsageLine: "upload -master=localhost:9333 file1 [file2 file3]\n         weed upload -server=localhost:9333 -dir=one_directory -include=*.pdf",
 	Short:     "upload one or a list of files",
 	Long: `upload one or a list of files, or batch upload one whole folder recursively.
 
@@ -79,7 +79,7 @@ func runUpload(cmd *Command, args []string) bool {
 					if e != nil {
 						return e
 					}
-					results, e := operation.SubmitFiles(*upload.server, parts,
+					results, e := operation.SubmitFiles(*upload.master, parts,
 						*upload.replication, *upload.collection,
 						*upload.ttl, *upload.maxMB, secret)
 					bytes, _ := json.Marshal(results)
@@ -98,7 +98,7 @@ func runUpload(cmd *Command, args []string) bool {
 		if e != nil {
 			fmt.Println(e.Error())
 		}
-		results, _ := operation.SubmitFiles(*upload.server, parts,
+		results, _ := operation.SubmitFiles(*upload.master, parts,
 			*upload.replication, *upload.collection,
 			*upload.ttl, *upload.maxMB, secret)
 		bytes, _ := json.Marshal(results)