Merge branch 'master' into add_remote_storage

weed/command/autocomplete.go

@@ -0,0 +1,109 @@
+package command
+
+import (
+	"fmt"
+	flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
+	"github.com/posener/complete"
+	completeinstall "github.com/posener/complete/cmd/install"
+	"runtime"
+)
+
+func AutocompleteMain(commands []*Command) bool {
+	subCommands := make(map[string]complete.Command)
+	helpSubCommands := make(map[string]complete.Command)
+	for _, cmd := range commands {
+		flags := make(map[string]complete.Predictor)
+		cmd.Flag.VisitAll(func(flag *flag.Flag) {
+			flags["-"+flag.Name] = complete.PredictAnything
+		})
+
+		subCommands[cmd.Name()] = complete.Command{
+			Flags: flags,
+		}
+		helpSubCommands[cmd.Name()] = complete.Command{}
+	}
+	subCommands["help"] = complete.Command{Sub: helpSubCommands}
+
+	globalFlags := make(map[string]complete.Predictor)
+	flag.VisitAll(func(flag *flag.Flag) {
+		globalFlags["-"+flag.Name] = complete.PredictAnything
+	})
+
+	weedCmd := complete.Command{
+		Sub:         subCommands,
+		Flags:       globalFlags,
+		GlobalFlags: complete.Flags{"-h": complete.PredictNothing},
+	}
+	cmp := complete.New("weed", weedCmd)
+
+	return cmp.Complete()
+}
+
+func installAutoCompletion() bool {
+	if runtime.GOOS == "windows" {
+		fmt.Println("windows is not supported")
+		return false
+	}
+
+	err := completeinstall.Install("weed")
+	if err != nil {
+		fmt.Printf("install failed! %s\n", err)
+		return false
+	}
+	fmt.Printf("autocompletion is enabled. Please restart your shell.\n")
+	return true
+}
+
+func uninstallAutoCompletion() bool {
+	if runtime.GOOS == "windows" {
+		fmt.Println("windows is not supported")
+		return false
+	}
+
+	err := completeinstall.Uninstall("weed")
+	if err != nil {
+		fmt.Printf("uninstall failed! %s\n", err)
+		return false
+	}
+	fmt.Printf("autocompletion is disable. Please restart your shell.\n")
+	return true
+}
+
+var cmdAutocomplete = &Command{
+	Run:       runAutocomplete,
+	UsageLine: "autocomplete",
+	Short:     "install autocomplete",
+	Long: `weed autocomplete is installed in the shell.
+
+    Supported shells are bash, zsh, and fish.
+    Windows is not supported.
+
+`,
+}
+
+func runAutocomplete(cmd *Command, args []string) bool {
+	if len(args) != 0 {
+		cmd.Usage()
+	}
+
+	return installAutoCompletion()
+}
+
+var cmdUnautocomplete = &Command{
+	Run:       runUnautocomplete,
+	UsageLine: "autocomplete.uninstall",
+	Short:     "uninstall autocomplete",
+	Long: `weed autocomplete is uninstalled in the shell.
+
+    Windows is not supported.
+
+`,
+}
+
+func runUnautocomplete(cmd *Command, args []string) bool {
+	if len(args) != 0 {
+		cmd.Usage()
+	}
+
+	return uninstallAutoCompletion()
+}

weed/command/command.go

@@ -8,6 +8,8 @@ import (
 )
 
 var Commands = []*Command{
+	cmdAutocomplete,
+	cmdUnautocomplete,
 	cmdBackup,
 	cmdBenchmark,
 	cmdCompact,

weed/command/filer_backup.go

@@ -52,11 +52,11 @@ var cmdFilerBackup = &Command{
 
 func runFilerBackup(cmd *Command, args []string) bool {
 
-	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
-
 	util.LoadConfiguration("security", false)
 	util.LoadConfiguration("replication", true)
 
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
 	for {
 		err := doFilerBackup(grpcDialOption, &filerBackupOptions)
 		if err != nil {

weed/command/filer_meta_backup.go

@@ -53,6 +53,7 @@ The backup writes to another filer store specified in a backup_filer.toml.
 
 func runFilerMetaBackup(cmd *Command, args []string) bool {
 
+	util.LoadConfiguration("security", false)
 	metaBackup.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	// load backup_filer.toml

weed/command/filer_meta_tail.go

@@ -45,6 +45,7 @@ var (
 
 func runFilerMetaTail(cmd *Command, args []string) bool {
 
+	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	var filterFunc func(dir, fname string) bool

weed/command/filer_sync.go

@@ -89,6 +89,7 @@ var cmdFilerSynchronize = &Command{
 
 func runFilerSynchronize(cmd *Command, args []string) bool {
 
+	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	grace.SetupProfiling(*syncCpuProfile, *syncMemProfile)

weed/command/iam.go

@@ -49,6 +49,7 @@ func (iamopt *IamOptions) startIamServer() bool {
 		return false
 	}
 
+	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 	for {
 		err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {

weed/command/scaffold/master.toml

@@ -11,7 +11,7 @@ scripts = """
   ec.encode -fullPercent=95 -quietFor=1h
   ec.rebuild -force
   ec.balance -force
-  volume.deleteEmpty -quietFor=24h
+  volume.deleteEmpty -quietFor=24h -force
   volume.balance -force
   volume.fix.replication
   unlock

weed/command/upload.go

@@ -71,13 +71,13 @@ func runUpload(cmd *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
-	defaultCollection, err := readMasterConfiguration(grpcDialOption, *upload.master)
+	defaultReplication, err := readMasterConfiguration(grpcDialOption, *upload.master)
 	if err != nil {
 		fmt.Printf("upload: %v", err)
 		return false
 	}
 	if *upload.replication == "" {
-		*upload.replication = defaultCollection
+		*upload.replication = defaultReplication
 	}
 
 	if len(args) == 0 {

weed/shell/command_volume_balance.go

@@ -120,6 +120,21 @@ func balanceVolumeServers(commandEnv *CommandEnv, diskTypes []types.DiskType, vo
 
 func balanceVolumeServersByDiskType(commandEnv *CommandEnv, diskType types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, volumeSizeLimit uint64, collection string, applyBalancing bool) error {
 
+	// balance read only volumes
+	for _, n := range nodes {
+		n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool {
+			if collection != "ALL_COLLECTIONS" {
+				if v.Collection != collection {
+					return false
+				}
+			}
+			return v.DiskType == string(diskType) && (v.ReadOnly || v.Size >= volumeSizeLimit)
+		})
+	}
+	if err := balanceSelectedVolume(commandEnv, volumeReplicas, nodes, capacityByMaxVolumeCount(diskType), sortReadOnlyVolumes, applyBalancing); err != nil {
+		return err
+	}
+
 	// balance writable volumes
 	for _, n := range nodes {
 		n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool {
@@ -135,21 +150,6 @@ func balanceVolumeServersByDiskType(commandEnv *CommandEnv, diskType types.DiskT
 		return err
 	}
 
-	// balance readable volumes
-	for _, n := range nodes {
-		n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool {
-			if collection != "ALL_COLLECTIONS" {
-				if v.Collection != collection {
-					return false
-				}
-			}
-			return v.DiskType == string(diskType) && (v.ReadOnly || v.Size >= volumeSizeLimit)
-		})
-	}
-	if err := balanceSelectedVolume(commandEnv, volumeReplicas, nodes, capacityByMaxVolumeCount(diskType), sortReadOnlyVolumes, applyBalancing); err != nil {
-		return err
-	}
-
 	return nil
 }
 

weed/shell/command_volume_delete_empty.go

@@ -3,11 +3,10 @@ package shell
 import (
 	"flag"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"io"
 	"log"
 	"time"
-
-	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 )
 
 func init() {
@@ -24,7 +23,7 @@ func (c *commandVolumeDeleteEmpty) Name() string {
 func (c *commandVolumeDeleteEmpty) Help() string {
 	return `delete empty volumes from all volume servers
 
-	volume.deleteEmpty -quietFor=24h
+	volume.deleteEmpty -quietFor=24h -force
 
 	This command deletes all empty volumes from one volume server.
 
@@ -39,6 +38,7 @@ func (c *commandVolumeDeleteEmpty) Do(args []string, commandEnv *CommandEnv, wri
 
 	volDeleteCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
 	quietPeriod := volDeleteCommand.Duration("quietFor", 24*time.Hour, "select empty volumes with no recent writes, avoid newly created ones")
+	applyBalancing := volDeleteCommand.Bool("force", false, "apply to delete empty volumes")
 	if err = volDeleteCommand.Parse(args); err != nil {
 		return nil
 	}
@@ -55,10 +55,15 @@ func (c *commandVolumeDeleteEmpty) Do(args []string, commandEnv *CommandEnv, wri
 	eachDataNode(topologyInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) {
 		for _, diskInfo := range dn.DiskInfos {
 			for _, v := range diskInfo.VolumeInfos {
-				if v.Size <= 8 && v.ModifiedAtSecond + quietSeconds < nowUnixSeconds {
-					log.Printf("deleting empty volume %d from %s", v.Id, dn.Id)
-					if deleteErr := deleteVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(v.Id), dn.Id); deleteErr != nil {
-						err = deleteErr
+				if v.Size <= 8 && v.ModifiedAtSecond+quietSeconds < nowUnixSeconds {
+					if *applyBalancing {
+						log.Printf("deleting empty volume %d from %s", v.Id, dn.Id)
+						if deleteErr := deleteVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(v.Id), dn.Id); deleteErr != nil {
+							err = deleteErr
+						}
+						continue
+					} else {
+						log.Printf("empty volume %d from %s", v.Id, dn.Id)
 					}
 				}
 			}

weed/shell/command_volume_fix_replication.go

@@ -160,7 +160,7 @@ func (c *commandVolumeFixReplication) fixUnderReplicatedVolumes(commandEnv *Comm
 	for _, vid := range underReplicatedVolumeIds {
 		for i := 0; i < retryCount+1; i++ {
 			if err = c.fixOneUnderReplicatedVolume(commandEnv, writer, takeAction, volumeReplicas, vid, allLocations); err == nil {
-				continue
+				break
 			}
 		}
 	}

weed/util/constants.go

@@ -5,7 +5,7 @@ import (
 )
 
 var (
-	VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 60)
+	VERSION = fmt.Sprintf("%s %.02f", sizeLimit, 2.61)
 	COMMIT  = ""
 )
 

weed/weed.go

@@ -46,6 +46,11 @@ func main() {
 	glog.MaxSize = 1024 * 1024 * 32
 	rand.Seed(time.Now().UnixNano())
 	flag.Usage = usage
+
+	if command.AutocompleteMain(commands) {
+		return
+	}
+
 	flag.Parse()
 
 	args := flag.Args()