mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2025-08-24 08:15:28 +08:00
refactoring
This commit is contained in:
parent
2f15e93466
commit
1b0bfbaf59
@ -25,7 +25,7 @@ type FilerClient interface {
|
|||||||
WithFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error
|
WithFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error
|
||||||
}
|
}
|
||||||
|
|
||||||
func ReadIntoBuffer(ctx context.Context, filerClient FilerClient, fullFilePath string, buff []byte, chunkViews []*ChunkView, baseOffset int64) (totalRead int64, err error) {
|
func ReadIntoBuffer(ctx context.Context, filerClient FilerClient, fullFilePath FullPath, buff []byte, chunkViews []*ChunkView, baseOffset int64) (totalRead int64, err error) {
|
||||||
var vids []string
|
var vids []string
|
||||||
for _, chunkView := range chunkViews {
|
for _, chunkView := range chunkViews {
|
||||||
vids = append(vids, VolumeId(chunkView.FileId))
|
vids = append(vids, VolumeId(chunkView.FileId))
|
||||||
@ -93,9 +93,9 @@ func ReadIntoBuffer(ctx context.Context, filerClient FilerClient, fullFilePath s
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetEntry(ctx context.Context, filerClient FilerClient, fullFilePath string) (entry *filer_pb.Entry, err error) {
|
func GetEntry(ctx context.Context, filerClient FilerClient, fullFilePath FullPath) (entry *filer_pb.Entry, err error) {
|
||||||
|
|
||||||
dir, name := FullPath(fullFilePath).DirAndName()
|
dir, name := fullFilePath.DirAndName()
|
||||||
|
|
||||||
err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
|
err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
|
||||||
|
|
||||||
@ -126,14 +126,14 @@ func GetEntry(ctx context.Context, filerClient FilerClient, fullFilePath string)
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath, prefix string, fn func(entry *filer_pb.Entry, isLast bool)) (err error) {
|
func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath FullPath, prefix string, fn func(entry *filer_pb.Entry, isLast bool)) (err error) {
|
||||||
|
|
||||||
err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
|
err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
|
||||||
|
|
||||||
lastEntryName := ""
|
lastEntryName := ""
|
||||||
|
|
||||||
request := &filer_pb.ListEntriesRequest{
|
request := &filer_pb.ListEntriesRequest{
|
||||||
Directory: fullDirPath,
|
Directory: string(fullDirPath),
|
||||||
Prefix: prefix,
|
Prefix: prefix,
|
||||||
StartFromFileName: lastEntryName,
|
StartFromFileName: lastEntryName,
|
||||||
Limit: math.MaxUint32,
|
Limit: math.MaxUint32,
|
||||||
|
@ -3,6 +3,8 @@ package filer2
|
|||||||
import (
|
import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/chrislusf/seaweedfs/weed/util"
|
||||||
)
|
)
|
||||||
|
|
||||||
type FullPath string
|
type FullPath string
|
||||||
@ -34,3 +36,7 @@ func (fp FullPath) Child(name string) FullPath {
|
|||||||
}
|
}
|
||||||
return FullPath(dir + "/" + name)
|
return FullPath(dir + "/" + name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (fp FullPath) AsInode() uint64 {
|
||||||
|
return uint64(util.HashStringToLong(string(fp)))
|
||||||
|
}
|
||||||
|
@ -9,7 +9,6 @@ import (
|
|||||||
"github.com/chrislusf/seaweedfs/weed/filer2"
|
"github.com/chrislusf/seaweedfs/weed/filer2"
|
||||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||||
"github.com/chrislusf/seaweedfs/weed/util"
|
|
||||||
"github.com/seaweedfs/fuse"
|
"github.com/seaweedfs/fuse"
|
||||||
"github.com/seaweedfs/fuse/fs"
|
"github.com/seaweedfs/fuse/fs"
|
||||||
)
|
)
|
||||||
@ -51,7 +50,7 @@ func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
attr.Inode = uint64(util.HashStringToLong(dir.Path))
|
attr.Inode = filer2.FullPath(dir.Path).AsInode()
|
||||||
attr.Mode = os.FileMode(dir.entry.Attributes.FileMode) | os.ModeDir
|
attr.Mode = os.FileMode(dir.entry.Attributes.FileMode) | os.ModeDir
|
||||||
attr.Mtime = time.Unix(dir.entry.Attributes.Mtime, 0)
|
attr.Mtime = time.Unix(dir.entry.Attributes.Mtime, 0)
|
||||||
attr.Ctime = time.Unix(dir.entry.Attributes.Crtime, 0)
|
attr.Ctime = time.Unix(dir.entry.Attributes.Crtime, 0)
|
||||||
@ -75,7 +74,7 @@ func (dir *Dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *f
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (dir *Dir) setRootDirAttributes(attr *fuse.Attr) {
|
func (dir *Dir) setRootDirAttributes(attr *fuse.Attr) {
|
||||||
attr.Inode = uint64(util.HashStringToLong(dir.Path))
|
attr.Inode = 1 // filer2.FullPath(dir.Path).AsInode()
|
||||||
attr.Valid = time.Hour
|
attr.Valid = time.Hour
|
||||||
attr.Uid = dir.wfs.option.MountUid
|
attr.Uid = dir.wfs.option.MountUid
|
||||||
attr.Gid = dir.wfs.option.MountGid
|
attr.Gid = dir.wfs.option.MountGid
|
||||||
@ -181,13 +180,8 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
|
|||||||
|
|
||||||
glog.V(4).Infof("dir Lookup %s: %s", dir.Path, req.Name)
|
glog.V(4).Infof("dir Lookup %s: %s", dir.Path, req.Name)
|
||||||
|
|
||||||
var entry *filer_pb.Entry
|
fullFilePath := filer2.NewFullPath(dir.Path, req.Name)
|
||||||
fullFilePath := string(filer2.NewFullPath(dir.Path, req.Name))
|
entry := dir.wfs.cacheGet(fullFilePath)
|
||||||
|
|
||||||
item := dir.wfs.listDirectoryEntriesCache.Get(fullFilePath)
|
|
||||||
if item != nil && !item.Expired() {
|
|
||||||
entry = item.Value().(*filer_pb.Entry)
|
|
||||||
}
|
|
||||||
|
|
||||||
if entry == nil {
|
if entry == nil {
|
||||||
glog.V(3).Infof("dir Lookup cache miss %s", fullFilePath)
|
glog.V(3).Infof("dir Lookup cache miss %s", fullFilePath)
|
||||||
@ -196,22 +190,20 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
|
|||||||
glog.V(1).Infof("dir GetEntry %s: %v", fullFilePath, err)
|
glog.V(1).Infof("dir GetEntry %s: %v", fullFilePath, err)
|
||||||
return nil, fuse.ENOENT
|
return nil, fuse.ENOENT
|
||||||
}
|
}
|
||||||
if entry != nil {
|
dir.wfs.cacheSet(fullFilePath, entry, 5*time.Minute)
|
||||||
dir.wfs.listDirectoryEntriesCache.Set(fullFilePath, entry, 5*time.Minute)
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
glog.V(4).Infof("dir Lookup cache hit %s", fullFilePath)
|
glog.V(4).Infof("dir Lookup cache hit %s", fullFilePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
if entry != nil {
|
if entry != nil {
|
||||||
if entry.IsDirectory {
|
if entry.IsDirectory {
|
||||||
node = &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs, entry: entry}
|
node = &Dir{Path: string(fullFilePath), wfs: dir.wfs, entry: entry}
|
||||||
} else {
|
} else {
|
||||||
node = dir.newFile(req.Name, entry)
|
node = dir.newFile(req.Name, entry)
|
||||||
}
|
}
|
||||||
|
|
||||||
resp.EntryValid = time.Second
|
resp.EntryValid = time.Second
|
||||||
resp.Attr.Inode = uint64(util.HashStringToLong(fullFilePath))
|
resp.Attr.Inode = fullFilePath.AsInode()
|
||||||
resp.Attr.Valid = time.Second
|
resp.Attr.Valid = time.Second
|
||||||
resp.Attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
|
resp.Attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
|
||||||
resp.Attr.Ctime = time.Unix(entry.Attributes.Crtime, 0)
|
resp.Attr.Ctime = time.Unix(entry.Attributes.Crtime, 0)
|
||||||
@ -232,9 +224,9 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
|
|||||||
|
|
||||||
cacheTtl := 5 * time.Minute
|
cacheTtl := 5 * time.Minute
|
||||||
|
|
||||||
readErr := filer2.ReadDirAllEntries(ctx, dir.wfs, dir.Path, "", func(entry *filer_pb.Entry, isLast bool) {
|
readErr := filer2.ReadDirAllEntries(ctx, dir.wfs, filer2.FullPath(dir.Path), "", func(entry *filer_pb.Entry, isLast bool) {
|
||||||
fullpath := string(filer2.NewFullPath(dir.Path, entry.Name))
|
fullpath := filer2.NewFullPath(dir.Path, entry.Name)
|
||||||
inode := uint64(util.HashStringToLong(fullpath))
|
inode := fullpath.AsInode()
|
||||||
if entry.IsDirectory {
|
if entry.IsDirectory {
|
||||||
dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: fuse.DT_Dir}
|
dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: fuse.DT_Dir}
|
||||||
ret = append(ret, dirent)
|
ret = append(ret, dirent)
|
||||||
@ -242,7 +234,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
|
|||||||
dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: fuse.DT_File}
|
dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: fuse.DT_File}
|
||||||
ret = append(ret, dirent)
|
ret = append(ret, dirent)
|
||||||
}
|
}
|
||||||
dir.wfs.listDirectoryEntriesCache.Set(fullpath, entry, cacheTtl)
|
dir.wfs.cacheSet(fullpath, entry, cacheTtl)
|
||||||
})
|
})
|
||||||
if readErr != nil {
|
if readErr != nil {
|
||||||
glog.V(0).Infof("list %s: %v", dir.Path, err)
|
glog.V(0).Infof("list %s: %v", dir.Path, err)
|
||||||
@ -264,14 +256,15 @@ func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
|
|||||||
|
|
||||||
func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) error {
|
func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) error {
|
||||||
|
|
||||||
entry, err := filer2.GetEntry(ctx, dir.wfs, path.Join(dir.Path, req.Name))
|
filePath := filer2.NewFullPath(dir.Path, req.Name)
|
||||||
|
entry, err := filer2.GetEntry(ctx, dir.wfs, filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
dir.wfs.deleteFileChunks(ctx, entry.Chunks)
|
dir.wfs.deleteFileChunks(ctx, entry.Chunks)
|
||||||
|
|
||||||
dir.wfs.listDirectoryEntriesCache.Delete(path.Join(dir.Path, req.Name))
|
dir.wfs.cacheDelete(filePath)
|
||||||
|
|
||||||
return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
|
return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
|
||||||
|
|
||||||
@ -295,7 +288,7 @@ func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) erro
|
|||||||
|
|
||||||
func (dir *Dir) removeFolder(ctx context.Context, req *fuse.RemoveRequest) error {
|
func (dir *Dir) removeFolder(ctx context.Context, req *fuse.RemoveRequest) error {
|
||||||
|
|
||||||
dir.wfs.listDirectoryEntriesCache.Delete(string(filer2.NewFullPath(dir.Path, req.Name)))
|
dir.wfs.cacheDelete(filer2.NewFullPath(dir.Path, req.Name))
|
||||||
|
|
||||||
return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
|
return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
|
||||||
|
|
||||||
@ -340,7 +333,7 @@ func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fus
|
|||||||
dir.entry.Attributes.Mtime = req.Mtime.Unix()
|
dir.entry.Attributes.Mtime = req.Mtime.Unix()
|
||||||
}
|
}
|
||||||
|
|
||||||
dir.wfs.listDirectoryEntriesCache.Delete(dir.Path)
|
dir.wfs.cacheDelete(filer2.FullPath(dir.Path))
|
||||||
|
|
||||||
return dir.saveEntry(ctx)
|
return dir.saveEntry(ctx)
|
||||||
|
|
||||||
@ -358,7 +351,7 @@ func (dir *Dir) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
dir.wfs.listDirectoryEntriesCache.Delete(dir.Path)
|
dir.wfs.cacheDelete(filer2.FullPath(dir.Path))
|
||||||
|
|
||||||
return dir.saveEntry(ctx)
|
return dir.saveEntry(ctx)
|
||||||
|
|
||||||
@ -376,7 +369,7 @@ func (dir *Dir) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) e
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
dir.wfs.listDirectoryEntriesCache.Delete(dir.Path)
|
dir.wfs.cacheDelete(filer2.FullPath(dir.Path))
|
||||||
|
|
||||||
return dir.saveEntry(ctx)
|
return dir.saveEntry(ctx)
|
||||||
|
|
||||||
|
@ -2,7 +2,6 @@ package filesys
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/chrislusf/seaweedfs/weed/filer2"
|
"github.com/chrislusf/seaweedfs/weed/filer2"
|
||||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||||
@ -16,7 +15,10 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
|
|||||||
newDir := newDirectory.(*Dir)
|
newDir := newDirectory.(*Dir)
|
||||||
glog.V(4).Infof("dir Rename %s/%s => %s/%s", dir.Path, req.OldName, newDir.Path, req.NewName)
|
glog.V(4).Infof("dir Rename %s/%s => %s/%s", dir.Path, req.OldName, newDir.Path, req.NewName)
|
||||||
|
|
||||||
err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
|
dir.wfs.cacheDelete(filer2.NewFullPath(newDir.Path, req.NewName))
|
||||||
|
dir.wfs.cacheDelete(filer2.NewFullPath(dir.Path, req.OldName))
|
||||||
|
|
||||||
|
return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
|
||||||
|
|
||||||
request := &filer_pb.AtomicRenameEntryRequest{
|
request := &filer_pb.AtomicRenameEntryRequest{
|
||||||
OldDirectory: dir.Path,
|
OldDirectory: dir.Path,
|
||||||
@ -27,19 +29,11 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
|
|||||||
|
|
||||||
_, err := client.AtomicRenameEntry(ctx, request)
|
_, err := client.AtomicRenameEntry(ctx, request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("renaming %s/%s => %s/%s: %v", dir.Path, req.OldName, newDir.Path, req.NewName, err)
|
glog.V(0).Infof("dir Rename %s/%s => %s/%s : %v", dir.Path, req.OldName, newDir.Path, req.NewName, err)
|
||||||
|
return fuse.EIO
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|
||||||
})
|
})
|
||||||
|
|
||||||
if err == nil {
|
|
||||||
oldpath := string(filer2.NewFullPath(dir.Path, req.OldName))
|
|
||||||
newpath := string(filer2.NewFullPath(newDir.Path, req.NewName))
|
|
||||||
dir.wfs.listDirectoryEntriesCache.Delete(oldpath)
|
|
||||||
dir.wfs.listDirectoryEntriesCache.Delete(newpath)
|
|
||||||
}
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
@ -3,14 +3,12 @@ package filesys
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
"sort"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/chrislusf/seaweedfs/weed/filer2"
|
"github.com/chrislusf/seaweedfs/weed/filer2"
|
||||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||||
"github.com/chrislusf/seaweedfs/weed/util"
|
|
||||||
"github.com/seaweedfs/fuse"
|
"github.com/seaweedfs/fuse"
|
||||||
"github.com/seaweedfs/fuse/fs"
|
"github.com/seaweedfs/fuse/fs"
|
||||||
)
|
)
|
||||||
@ -35,19 +33,22 @@ type File struct {
|
|||||||
isOpen bool
|
isOpen bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (file *File) fullpath() string {
|
func (file *File) fullpath() filer2.FullPath {
|
||||||
return filepath.Join(file.dir.Path, file.Name)
|
return filer2.NewFullPath(file.dir.Path, file.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error {
|
func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error {
|
||||||
|
|
||||||
glog.V(4).Infof("file Attr %s, open:%v, existing attr: %+v", file.fullpath(), file.isOpen, attr)
|
glog.V(4).Infof("file Attr %s, open:%v, existing attr: %+v", file.fullpath(), file.isOpen, attr)
|
||||||
|
|
||||||
if err := file.maybeLoadEntry(ctx); err != nil {
|
if !file.isOpen {
|
||||||
return err
|
if err := file.maybeLoadEntry(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
attr.Inode = uint64(util.HashStringToLong(file.fullpath()))
|
attr.Inode = file.fullpath().AsInode()
|
||||||
|
attr.Valid = time.Second
|
||||||
attr.Mode = os.FileMode(file.entry.Attributes.FileMode)
|
attr.Mode = os.FileMode(file.entry.Attributes.FileMode)
|
||||||
attr.Size = filer2.TotalSize(file.entry.Chunks)
|
attr.Size = filer2.TotalSize(file.entry.Chunks)
|
||||||
if file.isOpen {
|
if file.isOpen {
|
||||||
@ -132,7 +133,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
file.wfs.listDirectoryEntriesCache.Delete(file.fullpath())
|
file.wfs.cacheDelete(file.fullpath())
|
||||||
|
|
||||||
return file.saveEntry(ctx)
|
return file.saveEntry(ctx)
|
||||||
|
|
||||||
@ -150,7 +151,7 @@ func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
file.wfs.listDirectoryEntriesCache.Delete(file.fullpath())
|
file.wfs.cacheDelete(file.fullpath())
|
||||||
|
|
||||||
return file.saveEntry(ctx)
|
return file.saveEntry(ctx)
|
||||||
|
|
||||||
@ -168,7 +169,7 @@ func (file *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest)
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
file.wfs.listDirectoryEntriesCache.Delete(file.fullpath())
|
file.wfs.cacheDelete(file.fullpath())
|
||||||
|
|
||||||
return file.saveEntry(ctx)
|
return file.saveEntry(ctx)
|
||||||
|
|
||||||
|
@ -11,6 +11,7 @@ import (
|
|||||||
"github.com/karlseguin/ccache"
|
"github.com/karlseguin/ccache"
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
|
|
||||||
|
"github.com/chrislusf/seaweedfs/weed/filer2"
|
||||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||||
"github.com/chrislusf/seaweedfs/weed/util"
|
"github.com/chrislusf/seaweedfs/weed/util"
|
||||||
@ -47,7 +48,7 @@ type WFS struct {
|
|||||||
|
|
||||||
// contains all open handles
|
// contains all open handles
|
||||||
handles []*FileHandle
|
handles []*FileHandle
|
||||||
pathToHandleIndex map[string]int
|
pathToHandleIndex map[filer2.FullPath]int
|
||||||
pathToHandleLock sync.Mutex
|
pathToHandleLock sync.Mutex
|
||||||
bufPool sync.Pool
|
bufPool sync.Pool
|
||||||
|
|
||||||
@ -62,7 +63,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
|
|||||||
wfs := &WFS{
|
wfs := &WFS{
|
||||||
option: option,
|
option: option,
|
||||||
listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(option.DirListCacheLimit * 3).ItemsToPrune(100)),
|
listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(option.DirListCacheLimit * 3).ItemsToPrune(100)),
|
||||||
pathToHandleIndex: make(map[string]int),
|
pathToHandleIndex: make(map[filer2.FullPath]int),
|
||||||
bufPool: sync.Pool{
|
bufPool: sync.Pool{
|
||||||
New: func() interface{} {
|
New: func() interface{} {
|
||||||
return make([]byte, option.ChunkSizeLimit)
|
return make([]byte, option.ChunkSizeLimit)
|
||||||
@ -117,7 +118,7 @@ func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHand
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (wfs *WFS) ReleaseHandle(fullpath string, handleId fuse.HandleID) {
|
func (wfs *WFS) ReleaseHandle(fullpath filer2.FullPath, handleId fuse.HandleID) {
|
||||||
wfs.pathToHandleLock.Lock()
|
wfs.pathToHandleLock.Lock()
|
||||||
defer wfs.pathToHandleLock.Unlock()
|
defer wfs.pathToHandleLock.Unlock()
|
||||||
|
|
||||||
@ -191,3 +192,21 @@ func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (wfs *WFS) cacheGet(path filer2.FullPath) *filer_pb.Entry {
|
||||||
|
item := wfs.listDirectoryEntriesCache.Get(string(path))
|
||||||
|
if item != nil && !item.Expired() {
|
||||||
|
return item.Value().(*filer_pb.Entry)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (wfs *WFS) cacheSet(path filer2.FullPath, entry *filer_pb.Entry, ttl time.Duration) {
|
||||||
|
if entry == nil {
|
||||||
|
wfs.listDirectoryEntriesCache.Delete(string(path))
|
||||||
|
}else{
|
||||||
|
wfs.listDirectoryEntriesCache.Set(string(path), entry, ttl)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (wfs *WFS) cacheDelete(path filer2.FullPath) {
|
||||||
|
wfs.listDirectoryEntriesCache.Delete(string(path))
|
||||||
|
}
|
||||||
|
@ -2,6 +2,7 @@ package filesys
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/chrislusf/seaweedfs/weed/filer2"
|
"github.com/chrislusf/seaweedfs/weed/filer2"
|
||||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||||
@ -109,13 +110,12 @@ func listxattr(entry *filer_pb.Entry, req *fuse.ListxattrRequest, resp *fuse.Lis
|
|||||||
|
|
||||||
func (wfs *WFS) maybeLoadEntry(ctx context.Context, dir, name string) (entry *filer_pb.Entry, err error) {
|
func (wfs *WFS) maybeLoadEntry(ctx context.Context, dir, name string) (entry *filer_pb.Entry, err error) {
|
||||||
|
|
||||||
fullpath := string(filer2.NewFullPath(dir, name))
|
fullpath := filer2.NewFullPath(dir, name)
|
||||||
item := wfs.listDirectoryEntriesCache.Get(fullpath)
|
entry = wfs.cacheGet(fullpath)
|
||||||
if item != nil && !item.Expired() {
|
if entry != nil {
|
||||||
entry = item.Value().(*filer_pb.Entry)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// glog.V(3).Infof("read entry cache miss %s", fullpath)
|
glog.V(3).Infof("read entry cache miss %s", fullpath)
|
||||||
|
|
||||||
err = wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
|
err = wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
|
||||||
|
|
||||||
@ -126,16 +126,16 @@ func (wfs *WFS) maybeLoadEntry(ctx context.Context, dir, name string) (entry *fi
|
|||||||
|
|
||||||
resp, err := client.LookupDirectoryEntry(ctx, request)
|
resp, err := client.LookupDirectoryEntry(ctx, request)
|
||||||
if err != nil || resp == nil || resp.Entry == nil {
|
if err != nil || resp == nil || resp.Entry == nil {
|
||||||
if err == filer2.ErrNotFound {
|
if err == filer2.ErrNotFound || strings.Contains(err.Error(), filer2.ErrNotFound.Error()) {
|
||||||
glog.V(3).Infof("file attr read not found file %v: %v", request, err)
|
glog.V(3).Infof("file attr read not found file %v: %v", request, err)
|
||||||
return fuse.ENOENT
|
return fuse.ENOENT
|
||||||
}
|
}
|
||||||
glog.V(3).Infof("file attr read file %v: %v", request, err)
|
glog.V(3).Infof("file attr read file %v: %v", request, err)
|
||||||
return fuse.ENOENT
|
return fuse.EIO
|
||||||
}
|
}
|
||||||
|
|
||||||
entry = resp.Entry
|
entry = resp.Entry
|
||||||
wfs.listDirectoryEntriesCache.Set(fullpath, entry, wfs.option.EntryCacheTtl)
|
wfs.cacheSet(fullpath, entry, wfs.option.EntryCacheTtl)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
@ -335,8 +335,10 @@ func (fs *WebDavFileSystem) stat(ctx context.Context, fullFilePath string) (os.F
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fullpath := filer2.FullPath(fullFilePath)
|
||||||
|
|
||||||
var fi FileInfo
|
var fi FileInfo
|
||||||
entry, err := filer2.GetEntry(ctx, fs, fullFilePath)
|
entry, err := filer2.GetEntry(ctx, fs, fullpath)
|
||||||
if entry == nil {
|
if entry == nil {
|
||||||
return nil, os.ErrNotExist
|
return nil, os.ErrNotExist
|
||||||
}
|
}
|
||||||
@ -344,14 +346,12 @@ func (fs *WebDavFileSystem) stat(ctx context.Context, fullFilePath string) (os.F
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
fi.size = int64(filer2.TotalSize(entry.GetChunks()))
|
fi.size = int64(filer2.TotalSize(entry.GetChunks()))
|
||||||
fi.name = fullFilePath
|
fi.name = string(fullpath)
|
||||||
fi.mode = os.FileMode(entry.Attributes.FileMode)
|
fi.mode = os.FileMode(entry.Attributes.FileMode)
|
||||||
fi.modifiledTime = time.Unix(entry.Attributes.Mtime, 0)
|
fi.modifiledTime = time.Unix(entry.Attributes.Mtime, 0)
|
||||||
fi.isDirectory = entry.IsDirectory
|
fi.isDirectory = entry.IsDirectory
|
||||||
|
|
||||||
_, fi.name = path.Split(path.Clean(fi.name))
|
if fi.name == "/" {
|
||||||
if fi.name == "" {
|
|
||||||
fi.name = "/"
|
|
||||||
fi.modifiledTime = time.Now()
|
fi.modifiledTime = time.Now()
|
||||||
fi.isDirectory = true
|
fi.isDirectory = true
|
||||||
}
|
}
|
||||||
@ -372,7 +372,7 @@ func (f *WebDavFile) Write(buf []byte) (int, error) {
|
|||||||
var err error
|
var err error
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
if f.entry == nil {
|
if f.entry == nil {
|
||||||
f.entry, err = filer2.GetEntry(ctx, f.fs, f.name)
|
f.entry, err = filer2.GetEntry(ctx, f.fs, filer2.FullPath(f.name))
|
||||||
}
|
}
|
||||||
|
|
||||||
if f.entry == nil {
|
if f.entry == nil {
|
||||||
@ -470,7 +470,7 @@ func (f *WebDavFile) Read(p []byte) (readSize int, err error) {
|
|||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
if f.entry == nil {
|
if f.entry == nil {
|
||||||
f.entry, err = filer2.GetEntry(ctx, f.fs, f.name)
|
f.entry, err = filer2.GetEntry(ctx, f.fs, filer2.FullPath(f.name))
|
||||||
}
|
}
|
||||||
if f.entry == nil {
|
if f.entry == nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
@ -486,7 +486,7 @@ func (f *WebDavFile) Read(p []byte) (readSize int, err error) {
|
|||||||
}
|
}
|
||||||
chunkViews := filer2.ViewFromVisibleIntervals(f.entryViewCache, f.off, len(p))
|
chunkViews := filer2.ViewFromVisibleIntervals(f.entryViewCache, f.off, len(p))
|
||||||
|
|
||||||
totalRead, err := filer2.ReadIntoBuffer(ctx, f.fs, f.name, p, chunkViews, f.off)
|
totalRead, err := filer2.ReadIntoBuffer(ctx, f.fs, filer2.FullPath(f.name), p, chunkViews, f.off)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
@ -507,12 +507,9 @@ func (f *WebDavFile) Readdir(count int) (ret []os.FileInfo, err error) {
|
|||||||
glog.V(2).Infof("WebDavFileSystem.Readdir %v count %d", f.name, count)
|
glog.V(2).Infof("WebDavFileSystem.Readdir %v count %d", f.name, count)
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
dir := f.name
|
dir, _ := filer2.FullPath(f.name).DirAndName()
|
||||||
if dir != "/" && strings.HasSuffix(dir, "/") {
|
|
||||||
dir = dir[:len(dir)-1]
|
|
||||||
}
|
|
||||||
|
|
||||||
err = filer2.ReadDirAllEntries(ctx, f.fs, dir, "", func(entry *filer_pb.Entry, isLast bool) {
|
err = filer2.ReadDirAllEntries(ctx, f.fs, filer2.FullPath(dir), "", func(entry *filer_pb.Entry, isLast bool) {
|
||||||
fi := FileInfo{
|
fi := FileInfo{
|
||||||
size: int64(filer2.TotalSize(entry.GetChunks())),
|
size: int64(filer2.TotalSize(entry.GetChunks())),
|
||||||
name: entry.Name,
|
name: entry.Name,
|
||||||
|
@ -59,7 +59,7 @@ func (c *commandFsDu) Do(args []string, commandEnv *CommandEnv, writer io.Writer
|
|||||||
|
|
||||||
func duTraverseDirectory(ctx context.Context, writer io.Writer, filerClient filer2.FilerClient, dir, name string) (blockCount uint64, byteCount uint64, err error) {
|
func duTraverseDirectory(ctx context.Context, writer io.Writer, filerClient filer2.FilerClient, dir, name string) (blockCount uint64, byteCount uint64, err error) {
|
||||||
|
|
||||||
err = filer2.ReadDirAllEntries(ctx, filerClient, dir, name, func(entry *filer_pb.Entry, isLast bool) {
|
err = filer2.ReadDirAllEntries(ctx, filerClient, filer2.FullPath(dir), name, func(entry *filer_pb.Entry, isLast bool) {
|
||||||
if entry.IsDirectory {
|
if entry.IsDirectory {
|
||||||
subDir := fmt.Sprintf("%s/%s", dir, entry.Name)
|
subDir := fmt.Sprintf("%s/%s", dir, entry.Name)
|
||||||
if dir == "/" {
|
if dir == "/" {
|
||||||
|
@ -69,7 +69,7 @@ func (c *commandFsLs) Do(args []string, commandEnv *CommandEnv, writer io.Writer
|
|||||||
dir, name := filer2.FullPath(path).DirAndName()
|
dir, name := filer2.FullPath(path).DirAndName()
|
||||||
entryCount := 0
|
entryCount := 0
|
||||||
|
|
||||||
err = filer2.ReadDirAllEntries(ctx, commandEnv.getFilerClient(filerServer, filerPort), dir, name, func(entry *filer_pb.Entry, isLast bool) {
|
err = filer2.ReadDirAllEntries(ctx, commandEnv.getFilerClient(filerServer, filerPort), filer2.FullPath(dir), name, func(entry *filer_pb.Entry, isLast bool) {
|
||||||
|
|
||||||
if !showHidden && strings.HasPrefix(entry.Name, ".") {
|
if !showHidden && strings.HasPrefix(entry.Name, ".") {
|
||||||
return
|
return
|
||||||
|
@ -168,7 +168,7 @@ func processOneDirectory(ctx context.Context, writer io.Writer, filerClient file
|
|||||||
parentPath filer2.FullPath, queue *util.Queue, jobQueueWg *sync.WaitGroup,
|
parentPath filer2.FullPath, queue *util.Queue, jobQueueWg *sync.WaitGroup,
|
||||||
fn func(parentPath filer2.FullPath, entry *filer_pb.Entry)) (err error) {
|
fn func(parentPath filer2.FullPath, entry *filer_pb.Entry)) (err error) {
|
||||||
|
|
||||||
return filer2.ReadDirAllEntries(ctx, filerClient, string(parentPath), "", func(entry *filer_pb.Entry, isLast bool) {
|
return filer2.ReadDirAllEntries(ctx, filerClient, parentPath, "", func(entry *filer_pb.Entry, isLast bool) {
|
||||||
|
|
||||||
fn(parentPath, entry)
|
fn(parentPath, entry)
|
||||||
|
|
||||||
|
@ -39,7 +39,7 @@ func (c *commandFsTree) Do(args []string, commandEnv *CommandEnv, writer io.Writ
|
|||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
dirCount, fCount, terr := treeTraverseDirectory(ctx, writer, commandEnv.getFilerClient(filerServer, filerPort), dir, name, newPrefix(), -1)
|
dirCount, fCount, terr := treeTraverseDirectory(ctx, writer, commandEnv.getFilerClient(filerServer, filerPort), filer2.FullPath(dir), name, newPrefix(), -1)
|
||||||
|
|
||||||
if terr == nil {
|
if terr == nil {
|
||||||
fmt.Fprintf(writer, "%d directories, %d files\n", dirCount, fCount)
|
fmt.Fprintf(writer, "%d directories, %d files\n", dirCount, fCount)
|
||||||
@ -49,7 +49,7 @@ func (c *commandFsTree) Do(args []string, commandEnv *CommandEnv, writer io.Writ
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func treeTraverseDirectory(ctx context.Context, writer io.Writer, filerClient filer2.FilerClient, dir, name string, prefix *Prefix, level int) (directoryCount, fileCount int64, err error) {
|
func treeTraverseDirectory(ctx context.Context, writer io.Writer, filerClient filer2.FilerClient, dir filer2.FullPath, name string, prefix *Prefix, level int) (directoryCount, fileCount int64, err error) {
|
||||||
|
|
||||||
prefix.addMarker(level)
|
prefix.addMarker(level)
|
||||||
|
|
||||||
@ -64,10 +64,7 @@ func treeTraverseDirectory(ctx context.Context, writer io.Writer, filerClient fi
|
|||||||
|
|
||||||
if entry.IsDirectory {
|
if entry.IsDirectory {
|
||||||
directoryCount++
|
directoryCount++
|
||||||
subDir := fmt.Sprintf("%s/%s", dir, entry.Name)
|
subDir := dir.Child(entry.Name)
|
||||||
if dir == "/" {
|
|
||||||
subDir = "/" + entry.Name
|
|
||||||
}
|
|
||||||
dirCount, fCount, terr := treeTraverseDirectory(ctx, writer, filerClient, subDir, "", prefix, level+1)
|
dirCount, fCount, terr := treeTraverseDirectory(ctx, writer, filerClient, subDir, "", prefix, level+1)
|
||||||
directoryCount += dirCount
|
directoryCount += dirCount
|
||||||
fileCount += fCount
|
fileCount += fCount
|
||||||
|
Loading…
Reference in New Issue
Block a user