use bytes.Buffer to reduce memory allocation and gc

weed/server/common.go

@@ -1,6 +1,7 @@
 package weed_server
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -104,7 +105,9 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterFn ope
 	}
 
 	debug("parsing upload file...")
-	pu, pe := needle.ParseUpload(r, 256*1024*1024)
+	bytesBuffer := bufPool.Get().(*bytes.Buffer)
+	defer bufPool.Put(bytesBuffer)
+	pu, pe := needle.ParseUpload(r, 256*1024*1024, bytesBuffer)
 	if pe != nil {
 		writeJsonError(w, r, http.StatusBadRequest, pe)
 		return

weed/server/filer_server_handlers_write_cipher.go

@@ -1,6 +1,7 @@
 package weed_server
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
@@ -30,7 +31,10 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht
 
 	sizeLimit := int64(fs.option.MaxMB) * 1024 * 1024
 
-	pu, err := needle.ParseUpload(r, sizeLimit)
+	bytesBuffer := bufPool.Get().(*bytes.Buffer)
+	defer bufPool.Put(bytesBuffer)
+
+	pu, err := needle.ParseUpload(r, sizeLimit, bytesBuffer)
 	uncompressedData := pu.Data
 	if pu.IsGzipped {
 		uncompressedData = pu.UncompressedData

weed/server/filer_server_handlers_write_upload.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
@@ -19,6 +20,12 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
+var bufPool = sync.Pool{
+	New: func() interface{} {
+		return new(bytes.Buffer)
+	},
+}
+
 func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, contentLength int64, so *operation.StorageOption) ([]*filer_pb.FileChunk, hash.Hash, int64, error, []byte) {
 	var fileChunks []*filer_pb.FileChunk
 
@@ -28,21 +35,28 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque
 	chunkOffset := int64(0)
 	var smallContent []byte
 
+	bytesBuffer := bufPool.Get().(*bytes.Buffer)
+	defer bufPool.Put(bytesBuffer)
 	for {
 		limitedReader := io.LimitReader(partReader, int64(chunkSize))
 
-		data, err := ioutil.ReadAll(limitedReader)
+		bytesBuffer.Reset()
+
+		dataSize, err := bytesBuffer.ReadFrom(limitedReader)
+
+		// data, err := ioutil.ReadAll(limitedReader)
 		if err != nil {
 			return nil, nil, 0, err, nil
 		}
 		if chunkOffset == 0 && !isAppend(r) {
-			if len(data) < int(fs.option.SaveToFilerLimit) || strings.HasPrefix(r.URL.Path, filer.DirectoryEtcRoot) && len(data) < 4*1024 {
-				smallContent = data
-				chunkOffset += int64(len(data))
+			if dataSize < fs.option.SaveToFilerLimit || strings.HasPrefix(r.URL.Path, filer.DirectoryEtcRoot) && dataSize < 4*1024 {
+				chunkOffset += dataSize
+				smallContent = make([]byte, dataSize)
+				bytesBuffer.Write(smallContent)
 				break
 			}
 		}
-		dataReader := util.NewBytesReader(data)
+		dataReader := util.NewBytesReader(bytesBuffer.Bytes())
 
 		// retry to assign a different file id
 		var fileId, urlLocation string

weed/server/volume_server_handlers_write.go

@@ -1,6 +1,7 @@
 package weed_server
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"net/http"
@@ -42,7 +43,10 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	reqNeedle, originalSize, contentMd5, ne := needle.CreateNeedleFromRequest(r, vs.FixJpgOrientation, vs.fileSizeLimitBytes)
+	bytesBuffer := bufPool.Get().(*bytes.Buffer)
+	defer bufPool.Put(bytesBuffer)
+
+	reqNeedle, originalSize, contentMd5, ne := needle.CreateNeedleFromRequest(r, vs.FixJpgOrientation, vs.fileSizeLimitBytes, bytesBuffer)
 	if ne != nil {
 		writeJsonError(w, r, http.StatusBadRequest, ne)
 		return