lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-07-31 20:53:37 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Hadoop: more accurate block size

Browse Source
This commit is contained in:
Chris Lu 2020-10-18 14:13:46 -07:00
parent c0ab458671
commit 459de70a77
4 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
View File

@ -61,7 +61,7 @@ public class SeaweedFileSystem extends FileSystem {
setConf(conf); setConf(conf);
this.uri = uri; this.uri = uri;
seaweedFileSystemStore = new SeaweedFileSystemStore(host, port); seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, conf);
} }

10
other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
View File

@ -1,5 +1,6 @@
package seaweed.hdfs; package seaweed.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
@ -17,17 +18,22 @@ import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import static seaweed.hdfs.SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE;
import static seaweed.hdfs.SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE;
public class SeaweedFileSystemStore { public class SeaweedFileSystemStore {
private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystemStore.class); private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystemStore.class);
private FilerGrpcClient filerGrpcClient; private FilerGrpcClient filerGrpcClient;
private FilerClient filerClient; private FilerClient filerClient;
private Configuration conf;
public SeaweedFileSystemStore(String host, int port) { public SeaweedFileSystemStore(String host, int port, Configuration conf) {
int grpcPort = 10000 + port; int grpcPort = 10000 + port;
filerGrpcClient = new FilerGrpcClient(host, grpcPort); filerGrpcClient = new FilerGrpcClient(host, grpcPort);
filerClient = new FilerClient(filerGrpcClient); filerClient = new FilerClient(filerGrpcClient);
this.conf = conf;
} }
public static String getParentDirectory(Path path) { public static String getParentDirectory(Path path) {
@ -123,7 +129,7 @@ public class SeaweedFileSystemStore {
long length = SeaweedRead.fileSize(entry); long length = SeaweedRead.fileSize(entry);
boolean isDir = entry.getIsDirectory(); boolean isDir = entry.getIsDirectory();
int block_replication = 1; int block_replication = 1;
int blocksize = 512; int blocksize = this.conf.getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
long modification_time = attributes.getMtime() * 1000; // milliseconds long modification_time = attributes.getMtime() * 1000; // milliseconds
long access_time = 0; long access_time = 0;
FsPermission permission = FsPermission.createImmutable((short) attributes.getFileMode()); FsPermission permission = FsPermission.createImmutable((short) attributes.getFileMode());

2
other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
View File

@ -61,7 +61,7 @@ public class SeaweedFileSystem extends FileSystem {
setConf(conf); setConf(conf);
this.uri = uri; this.uri = uri;
seaweedFileSystemStore = new SeaweedFileSystemStore(host, port); seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, conf);
} }

10
other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
View File

@ -1,5 +1,6 @@
package seaweed.hdfs; package seaweed.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
@ -17,17 +18,22 @@ import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import static seaweed.hdfs.SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE;
import static seaweed.hdfs.SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE;
public class SeaweedFileSystemStore { public class SeaweedFileSystemStore {
private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystemStore.class); private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystemStore.class);
private FilerGrpcClient filerGrpcClient; private FilerGrpcClient filerGrpcClient;
private FilerClient filerClient; private FilerClient filerClient;
private Configuration conf;
public SeaweedFileSystemStore(String host, int port) { public SeaweedFileSystemStore(String host, int port, Configuration conf) {
int grpcPort = 10000 + port; int grpcPort = 10000 + port;
filerGrpcClient = new FilerGrpcClient(host, grpcPort); filerGrpcClient = new FilerGrpcClient(host, grpcPort);
filerClient = new FilerClient(filerGrpcClient); filerClient = new FilerClient(filerGrpcClient);
this.conf = conf;
} }
public static String getParentDirectory(Path path) { public static String getParentDirectory(Path path) {
@ -123,7 +129,7 @@ public class SeaweedFileSystemStore {
long length = SeaweedRead.fileSize(entry); long length = SeaweedRead.fileSize(entry);
boolean isDir = entry.getIsDirectory(); boolean isDir = entry.getIsDirectory();
int block_replication = 1; int block_replication = 1;
int blocksize = 512; int blocksize = this.conf.getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
long modification_time = attributes.getMtime() * 1000; // milliseconds long modification_time = attributes.getMtime() * 1000; // milliseconds
long access_time = 0; long access_time = 0;
FsPermission permission = FsPermission.createImmutable((short) attributes.getFileMode()); FsPermission permission = FsPermission.createImmutable((short) attributes.getFileMode());