lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-21 22:17:25 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

add leveldb store

Browse Source 
1. switch to viper for filer store configuration
2. simplify FindEntry() return values, removing “found”
3. add leveldb store
This commit is contained in:
Chris Lu
2018-05-26 03:49:46 -07:00
parent c34feca59c
commit 9e77563c99
17 changed files with 382 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
weed/command/filer.go
View File

@@ -2,7 +2,6 @@ package command
import (
"net/http"
"os"
"strconv"
"time"
@@ -13,6 +12,7 @@ import (
"google.golang.org/grpc/reflection"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/filer2"
)
var (
@@ -26,17 +26,10 @@ type FilerOptions struct {
publicPort *int
collection *string
defaultReplicaPlacement *string
dir *string
redirectOnRead *bool
disableDirListing *bool
confFile *string
maxMB *int
secretKey *string
cassandra_server *string
cassandra_keyspace *string
redis_server *string
redis_password *string
redis_database *int
}
func init() {
@@ -46,23 +39,15 @@ func init() {
f.ip = cmdFiler.Flag.String("ip", "", "filer server http listen ip address")
f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
f.publicPort = cmdFiler.Flag.Int("port.public", 0, "port opened to public")
f.dir = cmdFiler.Flag.String("dir", os.TempDir(), "directory to store meta data")
f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified")
f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
f.confFile = cmdFiler.Flag.String("confFile", "", "json encoded filer conf file")
f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server")
f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server")
f.redis_server = cmdFiler.Flag.String("redis.server", "", "comma separated host:port[,host2:port2]* of the redis server, e.g., 127.0.0.1:6379")
f.redis_password = cmdFiler.Flag.String("redis.password", "", "password in clear text")
f.redis_database = cmdFiler.Flag.Int("redis.database", 0, "the database on the redis server")
f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
}
var cmdFiler = &Command{
UsageLine: "filer -port=8888 -dir=/tmp -master=<ip:port>",
UsageLine: "filer -port=8888 -master=<ip:port>",
Short: "start a file server that points to a master server",
Long: `start a file server which accepts REST operation for any files.
@@ -75,20 +60,15 @@ var cmdFiler = &Command{
//return a json format subdirectory and files listing
GET /path/to/
Current <fullpath~fileid> mapping metadata store is local embedded leveldb.
It should be highly scalable to hundreds of millions of files on a modest machine.
The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", or "/etc/seaweedfs/", in that order.
Future we will ensure it can avoid of being SPOF.
The following are example filer.toml configuration file.
`,
` + filer2.FILER_TOML_EXAMPLE + "\n",
}
func runFiler(cmd *Command, args []string) bool {
if err := util.TestFolderWritable(*f.dir); err != nil {
glog.Fatalf("Check Meta Folder (-dir) Writable %s : %s", *f.dir, err)
}
f.start()
return true
@@ -104,13 +84,10 @@ func (fo *FilerOptions) start() {
}
fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux,
*fo.ip, *fo.port, *fo.master, *fo.dir, *fo.collection,
*fo.ip, *fo.port, *fo.master, *fo.collection,
*fo.defaultReplicaPlacement, *fo.redirectOnRead, *fo.disableDirListing,
*fo.confFile,
*fo.maxMB,
*fo.secretKey,
*fo.cassandra_server, *fo.cassandra_keyspace,
*fo.redis_server, *fo.redis_password, *fo.redis_database,
)
if nfs_err != nil {
glog.Fatalf("Filer startup error: %v", nfs_err)

16
weed/command/server.go
View File

@@ -87,17 +87,10 @@ func init() {
filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -dir is specified")
filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
filerOptions.confFile = cmdServer.Flag.String("filer.confFile", "", "json encoded filer conf file")
filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
filerOptions.cassandra_server = cmdServer.Flag.String("filer.cassandra.server", "", "host[:port] of the cassandra server")
filerOptions.cassandra_keyspace = cmdServer.Flag.String("filer.cassandra.keyspace", "seaweed", "keyspace of the cassandra server")
filerOptions.redis_server = cmdServer.Flag.String("filer.redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379")
filerOptions.redis_password = cmdServer.Flag.String("filer.redis.password", "", "redis password in clear text")
filerOptions.redis_database = cmdServer.Flag.Int("filer.redis.database", 0, "the database on the redis server")
}
func runServer(cmd *Command, args []string) bool {
@@ -157,15 +150,6 @@ func runServer(cmd *Command, args []string) bool {
if *masterMetaFolder == "" {
*masterMetaFolder = folders[0]
}
if *isStartingFiler {
if *filerOptions.dir == "" {
*filerOptions.dir = *masterMetaFolder + "/filer"
os.MkdirAll(*filerOptions.dir, 0700)
}
if err := util.TestFolderWritable(*filerOptions.dir); err != nil {
glog.Fatalf("Check Mapping Meta Folder (-filer.dir=\"%s\") Writable: %s", *filerOptions.dir, err)
}
}
if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err)
}