lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-21 13:28:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

filer, volume: add concurrent upload size limit to avoid OOM

Browse Source 
add some back pressure when writes are slow
This commit is contained in:
Chris Lu
2021-03-30 02:10:50 -07:00
parent a1e18a1384
commit ac875976c0
7 changed files with 138 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
weed/server/filer_server.go
View File

@@ -45,22 +45,23 @@ import (
)
type FilerOption struct {
Masters []string
Collection string
DefaultReplication string
DisableDirListing bool
MaxMB int
DirListingLimit int
DataCenter string
Rack string
DefaultLevelDbDir string
DisableHttp bool
Host string
Port uint32
recursiveDelete bool
Cipher bool
SaveToFilerLimit int
Filers []string
Masters []string
Collection string
DefaultReplication string
DisableDirListing bool
MaxMB int
DirListingLimit int
DataCenter string
Rack string
DefaultLevelDbDir string
DisableHttp bool
Host string
Port uint32
recursiveDelete bool
Cipher bool
SaveToFilerLimit int
Filers []string
ConcurrentUploadLimit int64
}
type FilerServer struct {
@@ -79,14 +80,18 @@ type FilerServer struct {
brokers map[string]map[string]bool
brokersLock sync.Mutex
inFlightDataSize int64
inFlightDataLimitCond *sync.Cond
}
func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) (fs *FilerServer, err error) {
fs = &FilerServer{
option: option,
grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"),
brokers: make(map[string]map[string]bool),
option: option,
grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"),
brokers: make(map[string]map[string]bool),
inFlightDataLimitCond: sync.NewCond(new(sync.Mutex)),
}
fs.listenersCond = sync.NewCond(&fs.listenersLock)

39
weed/server/filer_server_handlers.go
View File

@@ -4,6 +4,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/util"
"net/http"
"strings"
"sync/atomic"
"time"
"github.com/chrislusf/seaweedfs/weed/stats"
@@ -47,18 +48,34 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
fs.DeleteHandler(w, r)
}
stats.FilerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds())
case "PUT":
stats.FilerRequestCounter.WithLabelValues("put").Inc()
if _, ok := r.URL.Query()["tagging"]; ok {
fs.PutTaggingHandler(w, r)
} else {
fs.PostHandler(w, r)
case "POST", "PUT":
// wait until in flight data is less than the limit
contentLength := getContentLength(r)
fs.inFlightDataLimitCond.L.Lock()
for atomic.LoadInt64(&fs.inFlightDataSize) > fs.option.ConcurrentUploadLimit {
fs.inFlightDataLimitCond.Wait()
}
atomic.AddInt64(&fs.inFlightDataSize, contentLength)
fs.inFlightDataLimitCond.L.Unlock()
defer func() {
atomic.AddInt64(&fs.inFlightDataSize, -contentLength)
fs.inFlightDataLimitCond.Signal()
}()
if r.Method == "PUT" {
stats.FilerRequestCounter.WithLabelValues("put").Inc()
if _, ok := r.URL.Query()["tagging"]; ok {
fs.PutTaggingHandler(w, r)
} else {
fs.PostHandler(w, r)
}
stats.FilerRequestHistogram.WithLabelValues("put").Observe(time.Since(start).Seconds())
} else { // method == "POST"
stats.FilerRequestCounter.WithLabelValues("post").Inc()
fs.PostHandler(w, r)
stats.FilerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds())
}
stats.FilerRequestHistogram.WithLabelValues("put").Observe(time.Since(start).Seconds())
case "POST":
stats.FilerRequestCounter.WithLabelValues("post").Inc()
fs.PostHandler(w, r)
stats.FilerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds())
case "OPTIONS":
stats.FilerRequestCounter.WithLabelValues("options").Inc()
OptionsHandler(w, r, false)

8
weed/server/volume_server.go
View File

@@ -4,6 +4,7 @@ import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"net/http"
"sync"
"google.golang.org/grpc"
@@ -34,6 +35,10 @@ type VolumeServer struct {
fileSizeLimitBytes int64
isHeartbeating bool
stopChan chan bool
inFlightDataSize int64
inFlightDataLimitCond *sync.Cond
concurrentUploadLimit int64
}
func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
@@ -48,6 +53,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
readRedirect bool,
compactionMBPerSecond int,
fileSizeLimitMB int,
concurrentUploadLimit int64,
) *VolumeServer {
v := util.GetViper()
@@ -72,6 +78,8 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
fileSizeLimitBytes: int64(fileSizeLimitMB) * 1024 * 1024,
isHeartbeating: true,
stopChan: make(chan bool),
inFlightDataLimitCond: sync.NewCond(new(sync.Mutex)),
concurrentUploadLimit: concurrentUploadLimit,
}
vs.SeedMasterNodes = masterNodes

30
weed/server/volume_server_handlers.go
View File

@@ -2,7 +2,9 @@ package weed_server
import (
"net/http"
"strconv"
"strings"
"sync/atomic"
"github.com/chrislusf/seaweedfs/weed/util"
@@ -40,8 +42,24 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
stats.DeleteRequest()
vs.guard.WhiteList(vs.DeleteHandler)(w, r)
case "PUT", "POST":
// wait until in flight data is less than the limit
contentLength := getContentLength(r)
vs.inFlightDataLimitCond.L.Lock()
for atomic.LoadInt64(&vs.inFlightDataSize) > vs.concurrentUploadLimit {
vs.inFlightDataLimitCond.Wait()
}
atomic.AddInt64(&vs.inFlightDataSize, contentLength)
vs.inFlightDataLimitCond.L.Unlock()
defer func() {
atomic.AddInt64(&vs.inFlightDataSize, -contentLength)
vs.inFlightDataLimitCond.Signal()
}()
// processs uploads
stats.WriteRequest()
vs.guard.WhiteList(vs.PostHandler)(w, r)
case "OPTIONS":
stats.ReadRequest()
w.Header().Add("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
@@ -49,6 +67,18 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
}
}
func getContentLength(r *http.Request) int64 {
contentLength := r.Header.Get("Content-Length")
if contentLength != "" {
length, err := strconv.ParseInt(contentLength, 10, 64)
if err != nil {
return 0
}
return length
}
return 0
}
func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION)
if r.Header.Get("Origin") != "" {