lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-21 00:58:51 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

remote storage location changed to struct

Browse Source
This commit is contained in:
Chris Lu
2021-07-29 02:08:55 -07:00
parent c090d6bb25
commit 899963ac20
9 changed files with 320 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
weed/filer/filer_remote_storage.go
View File

@@ -70,11 +70,11 @@ func (rs *FilerRemoteStorage) loadRemoteStorageMountMapping(data []byte) (err er
return nil
}
func (rs *FilerRemoteStorage) mapDirectoryToRemoteStorage(dir util.FullPath, remoteStorageName string) {
rs.rules.Put([]byte(dir+"/"), remoteStorageName)
func (rs *FilerRemoteStorage) mapDirectoryToRemoteStorage(dir util.FullPath, loc *filer_pb.RemoteStorageLocation) {
rs.rules.Put([]byte(dir+"/"), loc)
}
func (rs *FilerRemoteStorage) FindMountDirectory(p util.FullPath) (mountDir util.FullPath, remoteLocation remote_storage.RemoteStorageLocation) {
func (rs *FilerRemoteStorage) FindMountDirectory(p util.FullPath) (mountDir util.FullPath, remoteLocation *filer_pb.RemoteStorageLocation) {
var storageLocation string
rs.rules.MatchPrefix([]byte(p), func(key []byte, value interface{}) bool {
mountDir = util.FullPath(string(key))
@@ -84,7 +84,7 @@ func (rs *FilerRemoteStorage) FindMountDirectory(p util.FullPath) (mountDir util
if storageLocation == "" {
return
}
remoteLocation = remote_storage.RemoteStorageLocation(storageLocation)
remoteLocation = remote_storage.ParseLocation(storageLocation)
return
}
@@ -99,9 +99,9 @@ func (rs *FilerRemoteStorage) FindRemoteStorageClient(p util.FullPath) (client r
return
}
storageName, _, _ := remote_storage.RemoteStorageLocation(storageLocation).NameBucketPath()
loc := remote_storage.ParseLocation(storageLocation)
return rs.GetRemoteStorageClient(storageName)
return rs.GetRemoteStorageClient(loc.Name)
}
func (rs *FilerRemoteStorage) GetRemoteStorageClient(storageName string) (client remote_storage.RemoteStorageClient, remoteConf *filer_pb.RemoteConf, found bool) {
@@ -118,21 +118,21 @@ func (rs *FilerRemoteStorage) GetRemoteStorageClient(storageName string) (client
return
}
func AddMapping(oldContent []byte, dir string, storageLocation remote_storage.RemoteStorageLocation) (newContent []byte, err error) {
func AddMapping(oldContent []byte, dir string, storageLocation *filer_pb.RemoteStorageLocation) (newContent []byte, err error) {
mappings := &filer_pb.RemoteStorageMapping{
Mappings: make(map[string]string),
Mappings: make(map[string]*filer_pb.RemoteStorageLocation),
}
if len(oldContent) > 0 {
if err = proto.Unmarshal(oldContent, mappings); err != nil {
return oldContent, fmt.Errorf("unmarshal existing mappings: %v", err)
glog.Warningf("unmarshal existing mappings: %v", err)
}
}
// set the new mapping
mappings.Mappings[dir] = string(storageLocation)
mappings.Mappings[dir] = storageLocation
if newContent, err = proto.Marshal(mappings); err != nil {
return oldContent, fmt.Errorf("unmarshal existing mappings: %v", err)
return oldContent, fmt.Errorf("marshal mappings: %v", err)
}
return

5
weed/filer/filer_search.go
View File

@@ -3,6 +3,7 @@ package filer
import (
"context"
"github.com/chrislusf/seaweedfs/weed/util"
"math"
"path/filepath"
"strings"
)
@@ -27,6 +28,10 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start
return true
})
if limit == math.MaxInt64 {
limit = math.MaxInt64 - 1
}
hasMore = int64(len(entries)) >= limit+1
if hasMore {
entries = entries[:limit]

14
weed/filer/read_remote.go
View File

@@ -2,6 +2,7 @@ package filer
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"io"
)
@@ -10,18 +11,23 @@ func (entry *Entry) IsRemoteOnly() bool {
}
func (f *Filer) ReadRemote(w io.Writer, entry *Entry, offset int64, size int64) error {
client, _, found := f.RemoteStorage.GetRemoteStorageClient(remoteEntry.Remote.StorageName)
client, _, found := f.RemoteStorage.GetRemoteStorageClient(entry.Remote.StorageName)
if !found {
return fmt.Errorf("remote storage %v not found", entry.Remote.StorageName)
}
mountDir, remoteLoation := f.RemoteStorage.FindMountDirectory(entry.FullPath)
_, bucket, path := remoteLoation.NameBucketPath()
remoteFullPath := path + string(entry.FullPath[len(mountDir):])
remoteFullPath := remoteLoation.Path + string(entry.FullPath[len(mountDir):])
client.ReadFile(bucket, remoteFullPath[1:], offset, size, func(w io.Writer) error {
sourceLoc := &filer_pb.RemoteStorageLocation{
Name: remoteLoation.Name,
Bucket: remoteLoation.Bucket,
Path: remoteFullPath,
}
client.ReadFile(sourceLoc, offset, size, func(w io.Writer) error {
return nil
})
return nil
}