refactoring code

reusable code by go clients

go/operation/assign_file_id.go

@@ -0,0 +1,40 @@
+package operation
+
+import (
+	"code.google.com/p/weed-fs/go/glog"
+	"code.google.com/p/weed-fs/go/util"
+	"encoding/json"
+	"errors"
+	"net/url"
+	"strconv"
+)
+
+type AssignResult struct {
+	Fid       string `json:"fid"`
+	Url       string `json:"url"`
+	PublicUrl string `json:"publicUrl"`
+	Count     int
+	Error     string `json:"error"`
+}
+
+func Assign(server string, count int, replication string) (*AssignResult, error) {
+	values := make(url.Values)
+	values.Add("count", strconv.Itoa(count))
+	if replication != "" {
+		values.Add("replication", replication)
+	}
+	jsonBlob, err := util.Post("http://"+server+"/dir/assign", values)
+	glog.V(2).Info("assign result :", string(jsonBlob))
+	if err != nil {
+		return nil, err
+	}
+	var ret AssignResult
+	err = json.Unmarshal(jsonBlob, &ret)
+	if err != nil {
+		return nil, err
+	}
+	if ret.Count <= 0 {
+		return nil, errors.New(ret.Error)
+	}
+	return &ret, nil
+}

go/operation/submit.go

@@ -0,0 +1,91 @@
+package operation
+
+import (
+	"code.google.com/p/weed-fs/go/glog"
+	"io"
+	"mime"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+type SubmitResult struct {
+	FileName string `json:"fileName"`
+	FileUrl  string `json:"fileUrl"`
+	Fid      string `json:"fid"`
+	Size     int    `json:"size"`
+	Error    string `json:"error"`
+}
+
+func Submit(master string, reader io.Reader, replication string) (result SubmitResult, err error) {
+	assignResult, assignError := Assign(master, 1, replication)
+	if assignError != nil {
+		result.Error = assignError.Error()
+		return
+	}
+	url := "http://" + assignResult.PublicUrl + "/" + assignResult.Fid
+	uploadResult, uploadError := Upload(url, "", reader, false, "")
+	if uploadError != nil {
+		result.Error = uploadError.Error()
+		return
+	}
+	result.Size = uploadResult.Size
+	result.FileUrl = url
+	result.Fid = assignResult.Fid
+	return result, nil
+}
+
+func SubmitFiles(master string, files []string, replication string) ([]SubmitResult, error) {
+	results := make([]SubmitResult, len(files))
+	for index, file := range files {
+		results[index].FileName = file
+	}
+	ret, err := Assign(master, len(files), replication)
+	if err != nil {
+		for index, _ := range files {
+			results[index].Error = err.Error()
+		}
+		return results, err
+	}
+	for index, file := range files {
+		fid := ret.Fid
+		if index > 0 {
+			fid = fid + "_" + strconv.Itoa(index)
+		}
+		results[index].Size, err = upload(file, ret.PublicUrl, fid)
+		if err != nil {
+			fid = ""
+			results[index].Error = err.Error()
+		}
+		results[index].Fid = fid
+		results[index].FileUrl = ret.PublicUrl + "/" + fid
+	}
+	return results, nil
+}
+
+func upload(filename string, server string, fid string) (int, error) {
+	glog.V(2).Info("Start uploading file:", filename)
+	fh, err := os.Open(filename)
+	if err != nil {
+		glog.V(0).Info("Failed to open file: ", filename)
+		return 0, err
+	}
+	fi, fiErr := fh.Stat()
+	if fiErr != nil {
+		glog.V(0).Info("Failed to stat file:", filename)
+		return 0, fiErr
+	}
+	filename = path.Base(filename)
+	isGzipped := path.Ext(filename) == ".gz"
+	if isGzipped {
+		filename = filename[0 : len(filename)-3]
+	}
+	mtype := mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
+	ret, e := Upload("http://"+server+"/"+fid+"?ts="+strconv.Itoa(int(fi.ModTime().Unix())), filename, fh, isGzipped, mtype)
+	if e != nil {
+		return 0, e
+	}
+	return ret.Size, e
+}

go/operation/upload_content.go

@@ -2,12 +2,12 @@ package operation
 
 import (
 	"bytes"
+	"code.google.com/p/weed-fs/go/glog"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"code.google.com/p/weed-fs/go/glog"
 	"mime"
 	"mime/multipart"
 	"net/http"
@@ -24,6 +24,12 @@ type UploadResult struct {
 var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
 
 func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string) (*UploadResult, error) {
+	return upload_content(uploadUrl, func(w io.Writer) (err error) {
+		_, err = io.Copy(w, reader)
+		return
+	}, filename, isGzipped, mtype)
+}
+func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string) (*UploadResult, error) {
 	body_buf := bytes.NewBufferString("")
 	body_writer := multipart.NewWriter(body_buf)
 	h := make(textproto.MIMEHeader)
@@ -31,7 +37,9 @@ func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool,
 	if mtype == "" {
 		mtype = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
 	}
-	h.Set("Content-Type", mtype)
+	if mtype != "" {
+		h.Set("Content-Type", mtype)
+	}
 	if isGzipped {
 		h.Set("Content-Encoding", "gzip")
 	}
@@ -40,7 +48,7 @@ func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool,
 		glog.V(0).Infoln("error creating form file", err)
 		return nil, err
 	}
-	if _, err = io.Copy(file_writer, reader); err != nil {
+	if err = fillBufferFunction(file_writer); err != nil {
 		glog.V(0).Infoln("error copying data", err)
 		return nil, err
 	}