HCFS: use latest grpc versions, separate hadoop2 and hadoop3

2025-09-19 09:47:56 +08:00 · 2019-09-01 03:46:51 -07:00
parent 5f283498c0
commit cb299dfaa2
14 changed files with 439 additions and 102 deletions
--- a/other/java/client/pom.xml
+++ b/other/java/client/pom.xml
@@ -4,7 +4,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.1.0</version>
+    <version>1.1.3</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -13,12 +13,18 @@
    </parent>

    <properties>
-        <protobuf.version>3.5.1</protobuf.version>
-        <grpc.version>1.16.1</grpc.version>
-        <guava.version>26.0-jre</guava.version>
+        <protobuf.version>3.9.1</protobuf.version>
+        <!-- follow https://github.com/grpc/grpc-java -->
+        <grpc.version>1.23.0</grpc.version>
+        <guava.version>28.0-jre</guava.version>
    </properties>

    <dependencies>
+        <dependency>
+            <groupId>com.moandjiezana.toml</groupId>
+            <artifactId>toml4j</artifactId>
+            <version>0.7.2</version>
+        </dependency>
        <!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
        <dependency>
            <groupId>com.google.protobuf</groupId>
@@ -74,7 +80,7 @@
            <extension>
                <groupId>kr.motd.maven</groupId>
                <artifactId>os-maven-plugin</artifactId>
-                <version>1.5.0.Final</version>
+                <version>1.6.2</version>
            </extension>
        </extensions>
        <plugins>
@@ -89,9 +95,9 @@
            <plugin>
                <groupId>org.xolstice.maven.plugins</groupId>
                <artifactId>protobuf-maven-plugin</artifactId>
-                <version>0.5.1</version>
+                <version>0.6.1</version>
                <configuration>
-                    <protocArtifact>com.google.protobuf:protoc:${protobuf.version}-1:exe:${os.detected.classifier}</protocArtifact>
+                    <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
                    <pluginId>grpc-java</pluginId>
                    <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
                </configuration>
--- a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java
+++ b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java
@@ -2,39 +2,46 @@ package seaweedfs.client;

 import io.grpc.ManagedChannel;
 import io.grpc.ManagedChannelBuilder;
-import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
 import io.grpc.netty.shaded.io.grpc.netty.NegotiationType;
 import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
 import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
-import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;

 import javax.net.ssl.SSLException;
-import java.io.File;
 import java.util.concurrent.TimeUnit;
-import java.util.logging.Logger;

 public class FilerGrpcClient {

-    private static final Logger logger = Logger.getLogger(FilerGrpcClient.class.getName());
+    private static final Logger logger = LoggerFactory.getLogger(FilerGrpcClient.class);

    private final ManagedChannel channel;
    private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub;
    private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub;
    private final SeaweedFilerGrpc.SeaweedFilerFutureStub futureStub;

+    static SslContext sslContext;

-    public FilerGrpcClient(String host, int grpcPort) {
-        this(ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext());
+    static {
+        try {
+            sslContext = FilerSslContext.loadSslContext();
+        } catch (SSLException e) {
+            logger.warn("failed to load ssl context", e);
+        }
    }

-    public FilerGrpcClient(String host, int grpcPort,
-                           String caFilePath,
-                           String clientCertFilePath,
-                           String clientPrivateKeyFilePath) throws SSLException {
+    public FilerGrpcClient(String host, int grpcPort) {
+        this(host, grpcPort, sslContext);
+    }
+
+    public FilerGrpcClient(String host, int grpcPort, SslContext sslContext) {
+
+        this(sslContext == null ?
+                ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext() :
+                NettyChannelBuilder.forAddress(host, grpcPort)
+                        .negotiationType(NegotiationType.TLS)
+                        .sslContext(sslContext));

-        this(NettyChannelBuilder.forAddress(host, grpcPort)
-                .negotiationType(NegotiationType.TLS)
-                .sslContext(buildSslContext(caFilePath,clientCertFilePath,clientPrivateKeyFilePath)));
    }

    public FilerGrpcClient(ManagedChannelBuilder<?> channelBuilder) {
@@ -60,17 +67,4 @@ public class FilerGrpcClient {
        return futureStub;
    }

-    private static SslContext buildSslContext(String trustCertCollectionFilePath,
-                                              String clientCertChainFilePath,
-                                              String clientPrivateKeyFilePath) throws SSLException {
-        SslContextBuilder builder = GrpcSslContexts.forClient();
-        if (trustCertCollectionFilePath != null) {
-            builder.trustManager(new File(trustCertCollectionFilePath));
-        }
-        if (clientCertChainFilePath != null && clientPrivateKeyFilePath != null) {
-            builder.keyManager(new File(clientCertChainFilePath), new File(clientPrivateKeyFilePath));
-        }
-        return builder.build();
-    }
-
 }
--- a/other/java/client/src/main/java/seaweedfs/client/FilerSslContext.java
+++ b/other/java/client/src/main/java/seaweedfs/client/FilerSslContext.java
@@ -0,0 +1,64 @@
+package seaweedfs.client;
+
+import com.google.common.base.Strings;
+import com.moandjiezana.toml.Toml;
+import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
+import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.SSLException;
+import java.io.File;
+
+public class FilerSslContext {
+
+    private static final Logger logger = LoggerFactory.getLogger(FilerSslContext.class);
+
+    public static SslContext loadSslContext() throws SSLException {
+        String securityFileName = "security.toml";
+        String home = System.getProperty("user.home");
+        File f1 = new File("./"+securityFileName);
+        File f2 = new File(home + "/.seaweedfs/"+securityFileName);
+        File f3 = new File(home + "/etc/seaweedfs/"+securityFileName);
+
+        File securityFile = f1.exists()? f1 : f2.exists() ? f2 : f3.exists()? f3 : null;
+
+        if (securityFile==null){
+            return null;
+        }
+
+        Toml toml = new Toml().read(securityFile);
+        logger.debug("reading ssl setup from {}", securityFile);
+
+        String trustCertCollectionFilePath = toml.getString("grpc.ca");
+        logger.debug("loading ca from {}", trustCertCollectionFilePath);
+        String clientCertChainFilePath = toml.getString("grpc.client.cert");
+        logger.debug("loading client ca from {}", clientCertChainFilePath);
+        String clientPrivateKeyFilePath = toml.getString("grpc.client.key");
+        logger.debug("loading client key from {}", clientPrivateKeyFilePath);
+
+        if (Strings.isNullOrEmpty(clientPrivateKeyFilePath) && Strings.isNullOrEmpty(clientPrivateKeyFilePath)){
+            return null;
+        }
+
+        // possibly fix the format https://netty.io/wiki/sslcontextbuilder-and-private-key.html
+
+        return buildSslContext(trustCertCollectionFilePath, clientCertChainFilePath, clientPrivateKeyFilePath);
+    }
+
+
+    private static SslContext buildSslContext(String trustCertCollectionFilePath,
+                                             String clientCertChainFilePath,
+                                             String clientPrivateKeyFilePath) throws SSLException {
+        SslContextBuilder builder = GrpcSslContexts.forClient();
+        if (trustCertCollectionFilePath != null) {
+            builder.trustManager(new File(trustCertCollectionFilePath));
+        }
+        if (clientCertChainFilePath != null && clientPrivateKeyFilePath != null) {
+            builder.keyManager(new File(clientCertChainFilePath), new File(clientPrivateKeyFilePath));
+        }
+        return builder.trustManager(InsecureTrustManagerFactory.INSTANCE).build();
+    }
+}
--- a/other/java/client/src/test/java/seaweedfs/client/SeaweedFilerTest.java
+++ b/other/java/client/src/test/java/seaweedfs/client/SeaweedFilerTest.java
@@ -0,0 +1,17 @@
+package seaweedfs.client;
+
+import java.util.List;
+
+public class SeaweedFilerTest {
+    public static void main(String[] args){
+
+        FilerClient filerClient = new FilerClient("localhost", 18888);
+
+        List<FilerProto.Entry> entries = filerClient.listEntries("/");
+
+        for (FilerProto.Entry entry : entries) {
+            System.out.println(entry.toString());
+        }
+
+    }
+}
--- a/other/java/hdfs2/dependency-reduced-pom.xml
+++ b/other/java/hdfs2/dependency-reduced-pom.xml
@@ -8,7 +8,7 @@
  </parent>
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.github.chrislusf</groupId>
-  <artifactId>seaweedfs-hadoop-client</artifactId>
+  <artifactId>seaweedfs-hadoop2-client</artifactId>
  <version>${seaweedfs.client.version}</version>
  <build>
    <plugins>
@@ -123,7 +123,7 @@
    </snapshotRepository>
  </distributionManagement>
  <properties>
-    <seaweedfs.client.version>1.1.0</seaweedfs.client.version>
-    <hadoop.version>2.7.4</hadoop.version>
+    <seaweedfs.client.version>1.1.3</seaweedfs.client.version>
+    <hadoop.version>2.9.2</hadoop.version>
  </properties>
 </project>
--- a/other/java/hdfs2/pom.xml
+++ b/other/java/hdfs2/pom.xml
@@ -5,12 +5,12 @@
    <modelVersion>4.0.0</modelVersion>

    <properties>
-        <seaweedfs.client.version>1.1.0</seaweedfs.client.version>
-        <hadoop.version>2.7.4</hadoop.version>
+        <seaweedfs.client.version>1.1.3</seaweedfs.client.version>
+        <hadoop.version>2.9.2</hadoop.version>
    </properties>

    <groupId>com.github.chrislusf</groupId>
-    <artifactId>seaweedfs-hadoop-client</artifactId>
+    <artifactId>seaweedfs-hadoop2-client</artifactId>
    <version>${seaweedfs.client.version}</version>

    <parent>
--- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
+++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
@@ -34,9 +34,6 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
    public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
    public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
    public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
-    public static final String FS_SEAWEED_GRPC_CA = "fs.seaweed.ca";
-    public static final String FS_SEAWEED_GRPC_CLIENT_KEY = "fs.seaweed.client.key";
-    public static final String FS_SEAWEED_GRPC_CLIENT_CERT = "fs.seaweed.client.cert";

    private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
    private static int BUFFER_SIZE = 16 * 1024 * 1024;
@@ -75,16 +72,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
        setConf(conf);
        this.uri = uri;

-        if (conf.get(FS_SEAWEED_GRPC_CA) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CA).length() != 0
-                && conf.get(FS_SEAWEED_GRPC_CLIENT_CERT) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_CERT).length() != 0
-                && conf.get(FS_SEAWEED_GRPC_CLIENT_KEY) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_KEY).length() != 0) {
-            seaweedFileSystemStore = new SeaweedFileSystemStore(host, port,
-                    conf.get(FS_SEAWEED_GRPC_CA),
-                    conf.get(FS_SEAWEED_GRPC_CLIENT_CERT),
-                    conf.get(FS_SEAWEED_GRPC_CLIENT_KEY));
-        } else {
-            seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
-        }
+        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);

    }

--- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
+++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
@@ -12,7 +12,6 @@ import seaweedfs.client.FilerGrpcClient;
 import seaweedfs.client.FilerProto;
 import seaweedfs.client.SeaweedRead;

-import javax.net.ssl.SSLException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -34,13 +33,6 @@ public class SeaweedFileSystemStore {
        filerClient = new FilerClient(filerGrpcClient);
    }

-    public SeaweedFileSystemStore(String host, int port,
-                                  String caFile, String clientCertFile, String clientKeyFile) throws SSLException {
-        int grpcPort = 10000 + port;
-        filerGrpcClient = new FilerGrpcClient(host, grpcPort, caFile, clientCertFile, clientKeyFile);
-        filerClient = new FilerClient(filerGrpcClient);
-    }
-
    public static String getParentDirectory(Path path) {
        return path.isRoot() ? "/" : path.getParent().toUri().getPath();
    }
--- a/other/java/hdfs3/dependency-reduced-pom.xml
+++ b/other/java/hdfs3/dependency-reduced-pom.xml
@@ -8,7 +8,7 @@
  </parent>
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.github.chrislusf</groupId>
-  <artifactId>seaweedfs-hadoop-client</artifactId>
+  <artifactId>seaweedfs-hadoop3-client</artifactId>
  <version>${seaweedfs.client.version}</version>
  <build>
    <plugins>
@@ -123,7 +123,7 @@
    </snapshotRepository>
  </distributionManagement>
  <properties>
-    <seaweedfs.client.version>1.1.0</seaweedfs.client.version>
+    <seaweedfs.client.version>1.1.3</seaweedfs.client.version>
    <hadoop.version>3.1.1</hadoop.version>
  </properties>
 </project>
--- a/other/java/hdfs3/pom.xml
+++ b/other/java/hdfs3/pom.xml
@@ -5,12 +5,12 @@
    <modelVersion>4.0.0</modelVersion>

    <properties>
-        <seaweedfs.client.version>1.1.0</seaweedfs.client.version>
+        <seaweedfs.client.version>1.1.3</seaweedfs.client.version>
        <hadoop.version>3.1.1</hadoop.version>
    </properties>

    <groupId>com.github.chrislusf</groupId>
-    <artifactId>seaweedfs-hadoop-client</artifactId>
+    <artifactId>seaweedfs-hadoop3-client</artifactId>
    <version>${seaweedfs.client.version}</version>

    <parent>
--- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
+++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
@@ -34,9 +34,6 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
    public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
    public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
    public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
-    public static final String FS_SEAWEED_GRPC_CA = "fs.seaweed.ca";
-    public static final String FS_SEAWEED_GRPC_CLIENT_KEY = "fs.seaweed.client.key";
-    public static final String FS_SEAWEED_GRPC_CLIENT_CERT = "fs.seaweed.client.cert";

    private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
    private static int BUFFER_SIZE = 16 * 1024 * 1024;
@@ -75,16 +72,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
        setConf(conf);
        this.uri = uri;

-        if (conf.get(FS_SEAWEED_GRPC_CA) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CA).length() != 0
-                && conf.get(FS_SEAWEED_GRPC_CLIENT_CERT) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_CERT).length() != 0
-                && conf.get(FS_SEAWEED_GRPC_CLIENT_KEY) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_KEY).length() != 0) {
-            seaweedFileSystemStore = new SeaweedFileSystemStore(host, port,
-                    conf.get(FS_SEAWEED_GRPC_CA),
-                    conf.get(FS_SEAWEED_GRPC_CLIENT_CERT),
-                    conf.get(FS_SEAWEED_GRPC_CLIENT_KEY));
-        } else {
-            seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
-        }
+        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);

    }

--- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
+++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
@@ -12,7 +12,6 @@ import seaweedfs.client.FilerGrpcClient;
 import seaweedfs.client.FilerProto;
 import seaweedfs.client.SeaweedRead;

-import javax.net.ssl.SSLException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -34,13 +33,6 @@ public class SeaweedFileSystemStore {
        filerClient = new FilerClient(filerGrpcClient);
    }

-    public SeaweedFileSystemStore(String host, int port,
-                                  String caFile, String clientCertFile, String clientKeyFile) throws SSLException {
-        int grpcPort = 10000 + port;
-        filerGrpcClient = new FilerGrpcClient(host, grpcPort, caFile, clientCertFile, clientKeyFile);
-        filerClient = new FilerClient(filerGrpcClient);
-    }
-
    public static String getParentDirectory(Path path) {
        return path.isRoot() ? "/" : path.getParent().toUri().getPath();
    }