shell: fs.meta.save add option to export all fileIds for all files

weed/shell/command_fs_meta_notify.go

@@ -43,7 +43,7 @@ func (c *commandFsMetaNotify) Do(args []string, commandEnv *CommandEnv, writer i
 
 	var dirCount, fileCount uint64
 
-	err = doTraverseBFS(writer, commandEnv, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) {
+	err = doTraverseBfs(writer, commandEnv, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) {
 
 		if entry.IsDirectory {
 			dirCount++

weed/shell/command_fs_meta_save.go

@@ -48,6 +48,7 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.
 	fsMetaSaveCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
 	verbose := fsMetaSaveCommand.Bool("v", false, "print out each processed files")
 	outputFileName := fsMetaSaveCommand.String("o", "", "output the meta data to this file")
+	chunksFileName := fsMetaSaveCommand.String("chunks", "", "output all the chunks to this file")
 	if err = fsMetaSaveCommand.Parse(args); err != nil {
 		return nil
 	}
@@ -57,13 +58,58 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.
 		return parseErr
 	}
 
-	t := time.Now()
+	if *outputFileName != "" {
-	fileName := *outputFileName
+		fileName := *outputFileName
-	if fileName == "" {
+		if fileName == "" {
-		fileName = fmt.Sprintf("%s-%d-%4d%02d%02d-%02d%02d%02d.meta",
+			t := time.Now()
-			commandEnv.option.FilerHost, commandEnv.option.FilerPort, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
+			fileName = fmt.Sprintf("%s-%d-%4d%02d%02d-%02d%02d%02d.meta",
+				commandEnv.option.FilerHost, commandEnv.option.FilerPort, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
+		}
+		return doTraverseBfsAndSaving(fileName, commandEnv, writer, path, *verbose, func(dst io.Writer, outputChan chan []byte) {
+			sizeBuf := make([]byte, 4)
+			for b := range outputChan {
+				util.Uint32toBytes(sizeBuf, uint32(len(b)))
+				dst.Write(sizeBuf)
+				dst.Write(b)
+			}
+		}, func(entry *filer_pb.FullEntry, outputChan chan []byte) (err error) {
+			bytes, err := proto.Marshal(entry)
+			if err != nil {
+				fmt.Fprintf(writer, "marshall error: %v\n", err)
+				return
+			}
+
+			outputChan <- bytes
+			return nil
+		})
 	}
 
+	if *chunksFileName != "" {
+		return doTraverseBfsAndSaving(*chunksFileName, commandEnv, writer, path, *verbose, func(dst io.Writer, outputChan chan []byte) {
+			for b := range outputChan {
+				dst.Write(b)
+			}
+		}, func(entry *filer_pb.FullEntry, outputChan chan []byte) (err error) {
+			for _, chunk := range entry.Entry.Chunks {
+				dir := entry.Dir
+				if dir == "/" {
+					dir = ""
+				}
+				outputLine := fmt.Sprintf("%d\t%s\t%s/%s\n", chunk.Fid.FileKey, chunk.FileId, dir, entry.Entry.Name)
+				outputChan <- []byte(outputLine)
+			}
+			return nil
+		})
+	}
+
+	return err
+
+}
+
+func doTraverseBfsAndSaving(fileName string, commandEnv *CommandEnv, writer io.Writer, path string, verbose bool,
+	saveFn func(dst io.Writer, outputChan chan []byte),
+	genFn func(entry *filer_pb.FullEntry, outputChan chan []byte) error) error {
+
 	dst, openErr := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
 	if openErr != nil {
 		return fmt.Errorf("failed to create file %s: %v", fileName, openErr)
@@ -74,39 +120,31 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.
 	wg.Add(1)
 	outputChan := make(chan []byte, 1024)
 	go func() {
-		sizeBuf := make([]byte, 4)
+		saveFn(dst, outputChan)
-		for b := range outputChan {
-			util.Uint32toBytes(sizeBuf, uint32(len(b)))
-			dst.Write(sizeBuf)
-			dst.Write(b)
-		}
 		wg.Done()
 	}()
 
 	var dirCount, fileCount uint64
 
-	err = doTraverseBFS(writer, commandEnv, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) {
+	err := doTraverseBfs(writer, commandEnv, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) {
 
 		protoMessage := &filer_pb.FullEntry{
 			Dir:   string(parentPath),
 			Entry: entry,
 		}
 
-		bytes, err := proto.Marshal(protoMessage)
+		if err := genFn(protoMessage, outputChan); err != nil {
-		if err != nil {
 			fmt.Fprintf(writer, "marshall error: %v\n", err)
 			return
 		}
 
-		outputChan <- bytes
-
 		if entry.IsDirectory {
 			atomic.AddUint64(&dirCount, 1)
 		} else {
 			atomic.AddUint64(&fileCount, 1)
 		}
 
-		if *verbose {
+		if verbose {
 			println(parentPath.Child(entry.Name))
 		}
 
@@ -118,13 +156,11 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.
 
 	if err == nil {
 		fmt.Fprintf(writer, "total %d directories, %d files\n", dirCount, fileCount)
-		fmt.Fprintf(writer, "meta data for %s is saved to %s\n", path, fileName)
 	}
-
 	return err
-
 }
-func doTraverseBFS(writer io.Writer, filerClient filer_pb.FilerClient, parentPath util.FullPath, fn func(parentPath util.FullPath, entry *filer_pb.Entry)) (err error) {
+
+func doTraverseBfs(writer io.Writer, filerClient filer_pb.FilerClient, parentPath util.FullPath, fn func(parentPath util.FullPath, entry *filer_pb.Entry)) (err error) {
 
 	K := 5