mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2025-10-21 06:57:23 +08:00
filer: processing all response headers, no pass through to volume server
* filer calculate MD5 etag * filer handle response headers, instread of pass it to volume servers
This commit is contained in:
Chris Lu
@@ -2,14 +2,10 @@ package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -17,7 +13,6 @@ import (
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/stats"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) {
|
||||
@@ -68,118 +63,30 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
|
||||
}
|
||||
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
if r.Method == "HEAD" {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(int64(filer2.TotalSize(entry.Chunks)), 10))
|
||||
w.Header().Set("Last-Modified", entry.Attr.Mtime.Format(http.TimeFormat))
|
||||
if entry.Attr.Mime != "" {
|
||||
w.Header().Set("Content-Type", entry.Attr.Mime)
|
||||
}
|
||||
setEtag(w, filer2.ETag(entry.Chunks))
|
||||
return
|
||||
}
|
||||
|
||||
if len(entry.Chunks) == 1 {
|
||||
fs.handleSingleChunk(w, r, entry)
|
||||
return
|
||||
}
|
||||
|
||||
fs.handleMultipleChunks(w, r, entry)
|
||||
|
||||
}
|
||||
|
||||
func (fs *FilerServer) handleSingleChunk(w http.ResponseWriter, r *http.Request, entry *filer2.Entry) {
|
||||
|
||||
fileId := entry.Chunks[0].GetFileIdString()
|
||||
|
||||
urlString, err := fs.filer.MasterClient.LookupFileId(fileId)
|
||||
if err != nil {
|
||||
glog.V(1).Infof("operation LookupFileId %s failed, err: %v", fileId, err)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
if fs.option.RedirectOnRead && entry.Chunks[0].CipherKey == nil {
|
||||
stats.FilerRequestCounter.WithLabelValues("redirect").Inc()
|
||||
http.Redirect(w, r, urlString, http.StatusFound)
|
||||
return
|
||||
}
|
||||
|
||||
u, _ := url.Parse(urlString)
|
||||
q := u.Query()
|
||||
for key, values := range r.URL.Query() {
|
||||
for _, value := range values {
|
||||
q.Add(key, value)
|
||||
}
|
||||
}
|
||||
u.RawQuery = q.Encode()
|
||||
request := &http.Request{
|
||||
Method: r.Method,
|
||||
URL: u,
|
||||
Proto: r.Proto,
|
||||
ProtoMajor: r.ProtoMajor,
|
||||
ProtoMinor: r.ProtoMinor,
|
||||
Header: r.Header,
|
||||
Body: r.Body,
|
||||
Host: r.Host,
|
||||
ContentLength: r.ContentLength,
|
||||
}
|
||||
glog.V(3).Infoln("retrieving from", u)
|
||||
resp, do_err := util.Do(request)
|
||||
if do_err != nil {
|
||||
glog.V(0).Infoln("failing to connect to volume server", do_err.Error())
|
||||
writeJsonError(w, r, http.StatusInternalServerError, do_err)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
io.Copy(ioutil.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
}()
|
||||
for k, v := range resp.Header {
|
||||
w.Header()[k] = v
|
||||
}
|
||||
if entry.Attr.Mime != "" {
|
||||
w.Header().Set("Content-Type", entry.Attr.Mime)
|
||||
}
|
||||
if entry.Chunks[0].CipherKey == nil {
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
io.Copy(w, resp.Body)
|
||||
} else {
|
||||
fs.writeEncryptedChunk(w, resp, entry)
|
||||
}
|
||||
}
|
||||
|
||||
func (fs *FilerServer) writeEncryptedChunk(w http.ResponseWriter, resp *http.Response, entry *filer2.Entry) {
|
||||
chunk := entry.Chunks[0]
|
||||
encryptedData, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
glog.V(1).Infof("read encrypted %s failed, err: %v", chunk.FileId, err)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
decryptedData, err := util.Decrypt(encryptedData, util.CipherKey(chunk.CipherKey))
|
||||
if err != nil {
|
||||
glog.V(1).Infof("decrypt %s failed, err: %v", chunk.FileId, err)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", chunk.Size))
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
w.Write(decryptedData)
|
||||
}
|
||||
|
||||
func (fs *FilerServer) handleMultipleChunks(w http.ResponseWriter, r *http.Request, entry *filer2.Entry) {
|
||||
w.Header().Set("Last-Modified", entry.Attr.Mtime.Format(http.TimeFormat))
|
||||
|
||||
// mime type
|
||||
mimeType := entry.Attr.Mime
|
||||
if mimeType == "" {
|
||||
if ext := path.Ext(entry.Name()); ext != "" {
|
||||
if ext := filepath.Ext(entry.Name()); ext != "" {
|
||||
mimeType = mime.TypeByExtension(ext)
|
||||
}
|
||||
}
|
||||
if mimeType != "" {
|
||||
w.Header().Set("Content-Type", mimeType)
|
||||
}
|
||||
|
||||
// set etag
|
||||
setEtag(w, filer2.ETag(entry.Chunks))
|
||||
|
||||
if r.Method == "HEAD" {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(int64(filer2.TotalSize(entry.Chunks)), 10))
|
||||
return
|
||||
}
|
||||
|
||||
filename := entry.Name()
|
||||
adjustHeadersAfterHEAD(w, r, filename)
|
||||
|
||||
totalSize := int64(filer2.TotalSize(entry.Chunks))
|
||||
|
||||
processRangeRequst(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
|
||||
@@ -188,94 +95,3 @@ func (fs *FilerServer) handleMultipleChunks(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
}
|
||||
|
||||
func processRangeRequst(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) {
|
||||
rangeReq := r.Header.Get("Range")
|
||||
|
||||
if rangeReq == "" {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
|
||||
if err := writeFn(w, 0, totalSize); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
//the rest is dealing with partial content request
|
||||
//mostly copy from src/pkg/net/http/fs.go
|
||||
ranges, err := parseRange(rangeReq, totalSize)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
|
||||
return
|
||||
}
|
||||
if sumRangesSize(ranges) > totalSize {
|
||||
// The total number of bytes in all the ranges
|
||||
// is larger than the size of the file by
|
||||
// itself, so this is probably an attack, or a
|
||||
// dumb client. Ignore the range request.
|
||||
return
|
||||
}
|
||||
if len(ranges) == 0 {
|
||||
return
|
||||
}
|
||||
if len(ranges) == 1 {
|
||||
// RFC 2616, Section 14.16:
|
||||
// "When an HTTP message includes the content of a single
|
||||
// range (for example, a response to a request for a
|
||||
// single range, or to a request for a set of ranges
|
||||
// that overlap without any holes), this content is
|
||||
// transmitted with a Content-Range header, and a
|
||||
// Content-Length header showing the number of bytes
|
||||
// actually transferred.
|
||||
// ...
|
||||
// A response to a request for a single range MUST NOT
|
||||
// be sent using the multipart/byteranges media type."
|
||||
ra := ranges[0]
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
|
||||
w.Header().Set("Content-Range", ra.contentRange(totalSize))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
|
||||
err = writeFn(w, ra.start, ra.length)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// process multiple ranges
|
||||
for _, ra := range ranges {
|
||||
if ra.start > totalSize {
|
||||
http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable)
|
||||
return
|
||||
}
|
||||
}
|
||||
sendSize := rangesMIMESize(ranges, mimeType, totalSize)
|
||||
pr, pw := io.Pipe()
|
||||
mw := multipart.NewWriter(pw)
|
||||
w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary())
|
||||
sendContent := pr
|
||||
defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish.
|
||||
go func() {
|
||||
for _, ra := range ranges {
|
||||
part, e := mw.CreatePart(ra.mimeHeader(mimeType, totalSize))
|
||||
if e != nil {
|
||||
pw.CloseWithError(e)
|
||||
return
|
||||
}
|
||||
if e = writeFn(part, ra.start, ra.length); e != nil {
|
||||
pw.CloseWithError(e)
|
||||
return
|
||||
}
|
||||
}
|
||||
mw.Close()
|
||||
pw.Close()
|
||||
}()
|
||||
if w.Header().Get("Content-Encoding") == "" {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10))
|
||||
}
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
if _, err := io.CopyN(w, sendContent, sendSize); err != nil {
|
||||
http.Error(w, "Internal Error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user