caching visible intervals for read

speeds up 4x in single thread mode
speeds up 30% in 32 threads mode

weed/filesys/dir.go

@@ -7,11 +7,11 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/seaweedfs/fuse"
-	"github.com/seaweedfs/fuse/fs"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
 )
 
 type Dir struct {
@@ -101,10 +101,11 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 
 func (dir *Dir) newFile(name string, entry *filer_pb.Entry) *File {
 	return &File{
-		Name:  name,
-		dir:   dir,
-		wfs:   dir.wfs,
-		entry: entry,
+		Name:           name,
+		dir:            dir,
+		wfs:            dir.wfs,
+		entry:          entry,
+		entryViewCache: nil,
 	}
 }
 

weed/filesys/file.go

@@ -1,12 +1,12 @@
 package filesys
 
 import (
-	"github.com/seaweedfs/fuse"
-	"github.com/seaweedfs/fuse/fs"
 	"context"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
 	"os"
 	"path/filepath"
 	"time"
@@ -20,11 +20,12 @@ var _ = fs.NodeFsyncer(&File{})
 var _ = fs.NodeSetattrer(&File{})
 
 type File struct {
-	Name   string
-	dir    *Dir
-	wfs    *WFS
-	entry  *filer_pb.Entry
-	isOpen bool
+	Name           string
+	dir            *Dir
+	wfs            *WFS
+	entry          *filer_pb.Entry
+	entryViewCache []*filer2.VisibleInterval
+	isOpen         bool
 }
 
 func (file *File) fullpath() string {
@@ -82,6 +83,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 		if req.Size == 0 {
 			// fmt.Printf("truncate %v \n", fullPath)
 			file.entry.Chunks = nil
+			file.entryViewCache = nil
 		}
 		file.entry.Attributes.FileSize = req.Size
 	}
@@ -138,6 +140,7 @@ func (file *File) maybeLoadAttributes(ctx context.Context) error {
 		if item != nil && !item.Expired() {
 			entry := item.Value().(*filer_pb.Entry)
 			file.entry = entry
+			file.entryViewCache = nil
 			// glog.V(1).Infof("file attr read cached %v attributes", file.Name)
 		} else {
 			err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
@@ -154,6 +157,7 @@ func (file *File) maybeLoadAttributes(ctx context.Context) error {
 				}
 
 				file.entry = resp.Entry
+				file.entryViewCache = nil
 
 				glog.V(3).Infof("file attr %v %+v: %d", file.fullpath(), file.entry.Attributes, filer2.TotalSize(file.entry.Chunks))
 

weed/filesys/filehandle.go

@@ -1,14 +1,14 @@
 package filesys
 
 import (
-	"github.com/seaweedfs/fuse"
-	"github.com/seaweedfs/fuse/fs"
 	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
 	"net/http"
 	"strings"
 	"sync"
@@ -58,7 +58,11 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus
 
 	buff := make([]byte, req.Size)
 
-	chunkViews := filer2.ViewFromChunks(fh.f.entry.Chunks, req.Offset, req.Size)
+	if fh.f.entryViewCache == nil {
+		fh.f.entryViewCache = filer2.NonOverlappingVisibleIntervals(fh.f.entry.Chunks)
+	}
+
+	chunkViews := filer2.ViewFromVisibleIntervals(fh.f.entryViewCache, req.Offset, req.Size)
 
 	var vids []string
 	for _, chunkView := range chunkViews {
@@ -154,6 +158,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
 
 	for _, chunk := range chunks {
 		fh.f.entry.Chunks = append(fh.f.entry.Chunks, chunk)
+		fh.f.entryViewCache = nil
 		glog.V(1).Infof("uploaded %s/%s to %s [%d,%d)", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
 		fh.dirtyMetadata = true
 	}
@@ -188,6 +193,7 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 	}
 	if chunk != nil {
 		fh.f.entry.Chunks = append(fh.f.entry.Chunks, chunk)
+		fh.f.entryViewCache = nil
 	}
 
 	if !fh.dirtyMetadata {