a correct implementation of filer

go/weed/weed_server/filer_server.go

@@ -1,28 +1,17 @@
 package weed_server
 
 import (
-	"errors"
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/util"
+	"code.google.com/p/weed-fs/go/filer"
+	"code.google.com/p/weed-fs/go/glog"
 	"net/http"
 	"strconv"
-	"strings"
 )
 
-/*
-1. level db is only for local instance
-2. db stores two types of pairs
- <path/to/dir, sub folder names>
- <path/to/file, file id>
- So, to list a directory, just get the directory entry, and iterate the current directory files
- Care must be taken to maintain the <dir, sub dirs> and <file, fileid> pairs.
-3.
-*/
 type FilerServer struct {
 	port       string
 	master     string
 	collection string
-	db         *leveldb.DB
+	filer      filer.Filer
 }
 
 func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string) (fs *FilerServer, err error) {
@@ -32,7 +21,8 @@ func NewFilerServer(r *http.ServeMux, port int, master string, dir string, colle
 		port:       ":" + strconv.Itoa(port),
 	}
 
-	if fs.db, err = leveldb.OpenFile(dir, nil); err != nil {
+	if fs.filer, err = filer.NewFilerEmbedded(dir); err != nil {
+		glog.Fatal("Can not start filer in dir:", dir)
 		return
 	}
 
@@ -40,127 +30,3 @@ func NewFilerServer(r *http.ServeMux, port int, master string, dir string, colle
 
 	return fs, nil
 }
-
-func (fs *FilerServer) CreateFile(fullFileName string, fid string) (err error) {
-	fs.ensureFileFolder(fullFileName)
-	return fs.db.Put([]byte(fullFileName), []byte(fid), nil)
-}
-
-func (fs *FilerServer) FindFile(fullFileName string) (fid string, err error) {
-	return fs.findEntry(fullFileName)
-}
-
-func (fs *FilerServer) ListDirectories(fullpath string) (dirs []string, err error) {
-	data, e := fs.db.Get([]byte(fullpath), nil)
-	if e != nil {
-		return nil, e
-	}
-	val := string(data)
-	if val == "" {
-		return nil, nil
-	}
-	return strings.Split(val, ":"), nil
-}
-
-func (fs *FilerServer) ListFiles(fullpath string, start, limit int) (files []string) {
-	if !strings.HasSuffix(fullpath, "/") {
-		fullpath += "/"
-	}
-	iter := fs.db.NewIterator(&util.Range{Start: []byte(fullpath)}, nil)
-	startCounter, limitCounter := -1, 0
-	for iter.Next() {
-		startCounter++
-		if startCounter < start {
-			continue
-		}
-		limitCounter++
-		if limit > 0 {
-			if limitCounter > limit {
-				break
-			}
-		}
-		key := string(iter.Key())
-		if !strings.HasPrefix(key, fullpath) {
-			break
-		}
-		fileName := key[len(fullpath):]
-		if fileName == "" {
-			continue //skip the directory entry
-		}
-		if strings.Contains(fileName, "/") {
-			break
-		}
-		files = append(files, fileName)
-	}
-	iter.Release()
-	return
-}
-
-func (fs *FilerServer) Delete(fullpath string, isForceDirectoryRemoval bool) (fid string, isFile bool, err error) {
-	val, e := fs.findEntry(fullpath)
-	if e != nil {
-		return "", false, e
-	}
-	if strings.Contains(val, ",") {
-		return val, true, fs.db.Delete([]byte(fullpath), nil)
-	}
-	// deal with directory
-	if !strings.HasSuffix(fullpath, "/") {
-		fullpath += "/"
-	}
-	iter := fs.db.NewIterator(&util.Range{Start: []byte(fullpath)}, nil)
-	counter := 0
-	for iter.Next() {
-		counter++
-		if counter > 0 {
-			break
-		}
-	}
-	iter.Release()
-	if counter > 0 {
-		return "", false, errors.New("Force Deletion Not Supported Yet")
-	}
-	return "", false, fs.db.Delete([]byte(fullpath), nil)
-}
-
-func (fs *FilerServer) findEntry(fullpath string) (value string, err error) {
-	data, e := fs.db.Get([]byte(fullpath), nil)
-	if e != nil {
-		return "", e
-	}
-	return string(data), nil
-}
-
-func (fs *FilerServer) ensureFileFolder(fullFileName string) (err error) {
-	parts := strings.Split(fullFileName, "/")
-	path := "/"
-	for i := 1; i < len(parts); i++ {
-		sub := parts[i]
-		if i == len(parts)-1 {
-			sub = ""
-		}
-		if err = fs.ensureFolderHasEntry(path, sub); err != nil {
-			return
-		}
-		if sub != "" {
-			path = path + sub + "/"
-		}
-	}
-	return nil
-}
-func (fs *FilerServer) ensureFolderHasEntry(path string, sub string) (err error) {
-	val, e := fs.findEntry(path)
-	if e == leveldb.ErrNotFound {
-		return fs.db.Put([]byte(path), []byte(sub), nil)
-	} else if e != nil {
-		return e
-	}
-	list := strings.Split(val, ":")
-	for _, v := range list {
-		if v == sub {
-			return nil
-		}
-	}
-	list = append(list, sub)
-	return fs.db.Put([]byte(path), []byte(strings.Join(list, ":")), nil)
-}

go/weed/weed_server/filer_server_handlers.go

@@ -34,7 +34,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
 	if !strings.HasSuffix(r.URL.Path, "/") {
 		return
 	}
-	dirlist, err := fs.ListDirectories(r.URL.Path)
+	dirlist, err := fs.filer.ListDirectories(r.URL.Path)
 	if err == leveldb.ErrNotFound {
 		glog.V(3).Infoln("Directory Not Found in db", r.URL.Path)
 		w.WriteHeader(http.StatusNotFound)
@@ -43,12 +43,12 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
 	m := make(map[string]interface{})
 	m["Directory"] = r.URL.Path
 	m["Subdirectories"] = dirlist
-	start, _ := strconv.Atoi(r.FormValue("start"))
+	lastFile := r.FormValue("lastFile")
 	limit, limit_err := strconv.Atoi(r.FormValue("limit"))
 	if limit_err != nil {
 		limit = 100
 	}
-	m["Files"] = fs.ListFiles(r.URL.Path, start, limit)
+	m["Files"], _ = fs.filer.ListFiles(r.URL.Path, lastFile, limit)
 	writeJsonQuiet(w, r, m)
 }
 func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) {
@@ -56,7 +56,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 		fs.listDirectoryHandler(w, r)
 		return
 	}
-	fileId, err := fs.FindFile(r.URL.Path)
+	fileId, err := fs.filer.FindFile(r.URL.Path)
 	if err == leveldb.ErrNotFound {
 		glog.V(3).Infoln("Not found in db", r.URL.Path)
 		w.WriteHeader(http.StatusNotFound)
@@ -115,6 +115,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	u, _ := url.Parse("http://" + assignResult.PublicUrl + "/" + assignResult.Fid)
+	glog.V(4).Infoln("post to", u)
 	request := &http.Request{
 		Method:        r.Method,
 		URL:           u,
@@ -141,6 +142,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 		writeJsonError(w, r, ra_err)
 		return
 	}
+	glog.V(4).Infoln("post result", string(resp_body))
 	var ret operation.UploadResult
 	unmarshal_err := json.Unmarshal(resp_body, &ret)
 	if unmarshal_err != nil {
@@ -167,7 +169,8 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
-	if db_err := fs.CreateFile(path, assignResult.Fid); db_err != nil {
+	glog.V(4).Infoln("saving", path, "=>", assignResult.Fid)
+	if db_err := fs.filer.CreateFile(path, assignResult.Fid); db_err != nil {
 		operation.DeleteFile(fs.master, assignResult.Fid) //clean up
 		glog.V(0).Infoln("failing to write to filer server", db_err.Error())
 		w.WriteHeader(http.StatusInternalServerError)
@@ -177,10 +180,13 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
-	isForceDirectoryRemoval := r.FormValue("force") == "true" // force remove for directories
-	fid, isFile, err := fs.Delete(r.URL.Path, isForceDirectoryRemoval)
-	if err == nil {
-		if isFile {
+	var err error
+	var fid string
+	if strings.HasSuffix(r.URL.Path, "/") {
+		err = fs.filer.DeleteDirectory(r.URL.Path)
+	} else {
+		fid, err = fs.filer.DeleteFile(r.URL.Path)
+		if err == nil {
 			err = operation.DeleteFile(fs.master, fid)
 		}
 	}