lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-09-18 19:07:55 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

cloud mount: remote storage support hdfs

Browse Source
This commit is contained in:
Chris Lu
2021-08-29 18:41:29 -07:00
parent 05d2774117
commit 001a472057
19 changed files with 539 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
weed/shell/command_remote_configure.go
View File

@@ -58,7 +58,7 @@ func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, write
isDelete := remoteConfigureCommand.Bool("delete", false, "delete one remote storage by its name")
remoteConfigureCommand.StringVar(&conf.Name, "name", "", "a short name to identify the remote storage")
remoteConfigureCommand.StringVar(&conf.Type, "type", "s3", "[s3|gcs|azure|b2|aliyun|tencent|baidu|wasabi] storage type")
remoteConfigureCommand.StringVar(&conf.Type, "type", "s3", "[s3|gcs|azure|b2|aliyun|tencent|baidu|wasabi|hdfs] storage type")
remoteConfigureCommand.StringVar(&conf.S3AccessKey, "s3.access_key", "", "s3 access key")
remoteConfigureCommand.StringVar(&conf.S3SecretKey, "s3.secret_key", "", "s3 secret key")
@@ -95,10 +95,27 @@ func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, write
remoteConfigureCommand.StringVar(&conf.WasabiEndpoint, "wasabi.endpoint", "", "Wasabi endpoint, see https://wasabi.com/wp-content/themes/wasabi/docs/API_Guide/index.html#t=topics%2Fapidiff-intro.htm")
remoteConfigureCommand.StringVar(&conf.WasabiRegion, "wasabi.region", "", "Wasabi region")
var namenodes arrayFlags
remoteConfigureCommand.Var(&namenodes, "hdfs.namenodes", "hdfs name node and port, example: namenode1:8020,namenode2:8020")
remoteConfigureCommand.StringVar(&conf.HdfsUsername, "hdfs.username", "", "hdfs user name")
remoteConfigureCommand.StringVar(&conf.HdfsServicePrincipalName, "hdfs.servicePrincipalName", "", `Kerberos service principal name for the namenode
Example: hdfs/namenode.hadoop.docker
Namenode running as service 'hdfs' with FQDN 'namenode.hadoop.docker'.
`)
remoteConfigureCommand.StringVar(&conf.HdfsDataTransferProtection, "hdfs.dataTransferProtection", "", "[authentication|integrity|privacy] Kerberos data transfer protection")
if err = remoteConfigureCommand.Parse(args); err != nil {
return nil
}
if conf.Type != "s3" {
// clear out the default values
conf.S3Region = ""
conf.S3ForcePathStyle = false
}
if conf.Name == "" {
return c.listExistingRemoteStorages(commandEnv, writer)
}
@@ -187,3 +204,14 @@ func (c *commandRemoteConfigure) saveRemoteStorage(commandEnv *CommandEnv, write
return nil
}
type arrayFlags []string
func (i *arrayFlags) String() string {
return "my string representation"
}
func (i *arrayFlags) Set(value string) error {
*i = append(*i, value)
return nil
}

6
weed/shell/command_remote_meta_sync.go
View File

@@ -55,7 +55,7 @@ func (c *commandRemoteMetaSync) Do(args []string, commandEnv *CommandEnv, writer
}
mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, detectErr := detectMountInfo(commandEnv, writer, *dir)
if detectErr != nil{
if detectErr != nil {
jsonPrintln(writer, mappings)
return detectErr
}
@@ -106,7 +106,7 @@ func detectMountInfo(commandEnv *CommandEnv, writer io.Writer, dir string) (*rem
If entry.RemoteEntry.RemoteTag != remoteEntry.RemoteTag {
the remote version is updated, need to pull meta
}
*/
*/
func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, dirToCache util.FullPath, remoteConf *remote_pb.RemoteConf) error {
// visit remote storage
@@ -158,7 +158,7 @@ func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util
fmt.Fprintln(writer, " (skip)")
return nil
}
if existingEntry.RemoteEntry.RemoteETag != remoteEntry.RemoteETag {
if existingEntry.RemoteEntry.RemoteETag != remoteEntry.RemoteETag || existingEntry.RemoteEntry.RemoteMtime < remoteEntry.RemoteMtime {
// the remote version is updated, need to pull meta
fmt.Fprintln(writer, " (update)")
return doSaveRemoteEntry(client, string(localDir), existingEntry, remoteEntry)

10
weed/shell/command_remote_mount.go
View File

@@ -60,15 +60,17 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io
return err
}
remoteStorageLocation := remote_storage.ParseLocation(*remote)
// find configuration for remote storage
// remotePath is /<bucket>/path/to/dir
remoteConf, err := c.findRemoteStorageConfiguration(commandEnv, writer, remoteStorageLocation)
remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remote_storage.ParseLocationName(*remote))
if err != nil {
return fmt.Errorf("find configuration for %s: %v", *remote, err)
}
remoteStorageLocation, err := remote_storage.ParseRemoteLocation(remoteConf.Type, *remote)
if err != nil {
return err
}
// sync metadata from remote
if err = c.syncMetadata(commandEnv, writer, *dir, *nonEmpty, remoteConf, remoteStorageLocation); err != nil {
return fmt.Errorf("pull metadata: %v", err)