lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-21 17:27:23 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

go fmt

Browse Source
This commit is contained in:
Chris Lu
2018-07-24 01:38:08 -07:00
parent 4f317c7e3d
commit 77fadab560
4 changed files with 32 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
weed/s3api/s3api_bucket_handlers.go
View File

@@ -3,12 +3,13 @@ package s3api
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/gorilla/mux"
"net/http" "net/http"
"os" "os"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/gorilla/mux"
) )
var ( var (

11
weed/s3api/s3api_objects_list_handlers.go
View File

@@ -3,17 +3,18 @@ package s3api
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/gorilla/mux"
"net/http" "net/http"
"net/url" "net/url"
"path/filepath" "path/filepath"
"strconv" "strconv"
"time" "time"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/gorilla/mux"
) )
const ( const (

10
weed/storage/needle.go
View File

@@ -29,12 +29,12 @@ type Needle struct {
DataSize uint32 `comment:"Data size"` //version2 DataSize uint32 `comment:"Data size"` //version2
Data []byte `comment:"The actual file data"` Data []byte `comment:"The actual file data"`
Flags byte `comment:"boolean flags"` //version2 Flags byte `comment:"boolean flags"` //version2
NameSize uint8 //version2 NameSize uint8 //version2
Name []byte `comment:"maximum 256 characters"` //version2 Name []byte `comment:"maximum 256 characters"` //version2
MimeSize uint8 //version2 MimeSize uint8 //version2
Mime []byte `comment:"maximum 256 characters"` //version2 Mime []byte `comment:"maximum 256 characters"` //version2
PairsSize uint16 //version2 PairsSize uint16 //version2
Pairs []byte `comment:"additional name value pairs, json format, maximum 64kB"` Pairs []byte `comment:"additional name value pairs, json format, maximum 64kB"`
LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
Ttl *TTL Ttl *TTL
@@ -135,7 +135,7 @@ func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) {
dotSep := strings.LastIndex(r.URL.Path, ".") dotSep := strings.LastIndex(r.URL.Path, ".")
fid := r.URL.Path[commaSep+1:] fid := r.URL.Path[commaSep+1:]
if dotSep > 0 { if dotSep > 0 {
fid = r.URL.Path[commaSep+1: dotSep] fid = r.URL.Path[commaSep+1 : dotSep]
} }
e = n.ParsePath(fid) e = n.ParsePath(fid)

34
weed/storage/needle_read_write.go
View File

@@ -59,7 +59,7 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, actualSize i
actualSize = NeedleEntrySize + int64(n.Size) actualSize = NeedleEntrySize + int64(n.Size)
padding := PaddingLength(n.Size, version) padding := PaddingLength(n.Size, version)
util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value()) util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
_, err = w.Write(header[0: NeedleChecksumSize+padding]) _, err = w.Write(header[0 : NeedleChecksumSize+padding])
return return
case Version2, Version3: case Version2, Version3:
header := make([]byte, NeedleEntrySize+TimestampSize) // adding timestamp to reuse it and avoid extra allocation header := make([]byte, NeedleEntrySize+TimestampSize) // adding timestamp to reuse it and avoid extra allocation
@@ -123,7 +123,7 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, actualSize i
} }
if n.HasLastModifiedDate() { if n.HasLastModifiedDate() {
util.Uint64toBytes(header[0:8], n.LastModified) util.Uint64toBytes(header[0:8], n.LastModified)
if _, err = w.Write(header[8-LastModifiedBytesLength: 8]); err != nil { if _, err = w.Write(header[8-LastModifiedBytesLength : 8]); err != nil {
return return
} }
} }
@@ -146,11 +146,11 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, actualSize i
padding := PaddingLength(n.Size, version) padding := PaddingLength(n.Size, version)
util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value()) util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
if version == Version2 { if version == Version2 {
_, err = w.Write(header[0: NeedleChecksumSize+padding]) _, err = w.Write(header[0 : NeedleChecksumSize+padding])
} else { } else {
// version3 // version3
util.Uint64toBytes(header[NeedleChecksumSize:NeedleChecksumSize+TimestampSize], n.AppendAtNs) util.Uint64toBytes(header[NeedleChecksumSize:NeedleChecksumSize+TimestampSize], n.AppendAtNs)
_, err = w.Write(header[0: NeedleChecksumSize+TimestampSize+padding]) _, err = w.Write(header[0 : NeedleChecksumSize+TimestampSize+padding])
} }
return n.DataSize, getActualSize(n.Size, version), err return n.DataSize, getActualSize(n.Size, version), err
@@ -175,14 +175,14 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version
} }
switch version { switch version {
case Version1: case Version1:
n.Data = bytes[NeedleEntrySize: NeedleEntrySize+size] n.Data = bytes[NeedleEntrySize : NeedleEntrySize+size]
case Version2, Version3: case Version2, Version3:
n.readNeedleDataVersion2(bytes[NeedleEntrySize: NeedleEntrySize+int(n.Size)]) n.readNeedleDataVersion2(bytes[NeedleEntrySize : NeedleEntrySize+int(n.Size)])
} }
if size == 0 { if size == 0 {
return nil return nil
} }
checksum := util.BytesToUint32(bytes[NeedleEntrySize+size: NeedleEntrySize+size+NeedleChecksumSize]) checksum := util.BytesToUint32(bytes[NeedleEntrySize+size : NeedleEntrySize+size+NeedleChecksumSize])
newChecksum := NewCRC(n.Data) newChecksum := NewCRC(n.Data)
if checksum != newChecksum.Value() { if checksum != newChecksum.Value() {
return errors.New("CRC error! Data On Disk Corrupted") return errors.New("CRC error! Data On Disk Corrupted")
@@ -190,28 +190,28 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version
n.Checksum = newChecksum n.Checksum = newChecksum
if version == Version3 { if version == Version3 {
tsOffset := NeedleEntrySize + size + NeedleChecksumSize tsOffset := NeedleEntrySize + size + NeedleChecksumSize
n.AppendAtNs = util.BytesToUint64(bytes[tsOffset: tsOffset+TimestampSize]) n.AppendAtNs = util.BytesToUint64(bytes[tsOffset : tsOffset+TimestampSize])
} }
return nil return nil
} }
func (n *Needle) ParseNeedleHeader(bytes []byte) { func (n *Needle) ParseNeedleHeader(bytes []byte) {
n.Cookie = BytesToCookie(bytes[0:CookieSize]) n.Cookie = BytesToCookie(bytes[0:CookieSize])
n.Id = BytesToNeedleId(bytes[CookieSize: CookieSize+NeedleIdSize]) n.Id = BytesToNeedleId(bytes[CookieSize : CookieSize+NeedleIdSize])
n.Size = util.BytesToUint32(bytes[CookieSize+NeedleIdSize: NeedleEntrySize]) n.Size = util.BytesToUint32(bytes[CookieSize+NeedleIdSize : NeedleEntrySize])
} }
func (n *Needle) readNeedleDataVersion2(bytes []byte) { func (n *Needle) readNeedleDataVersion2(bytes []byte) {
index, lenBytes := 0, len(bytes) index, lenBytes := 0, len(bytes)
if index < lenBytes { if index < lenBytes {
n.DataSize = util.BytesToUint32(bytes[index: index+4]) n.DataSize = util.BytesToUint32(bytes[index : index+4])
index = index + 4 index = index + 4
if int(n.DataSize)+index > lenBytes { if int(n.DataSize)+index > lenBytes {
// this if clause is due to bug #87 and #93, fixed in v0.69 // this if clause is due to bug #87 and #93, fixed in v0.69
// remove this clause later // remove this clause later
return return
} }
n.Data = bytes[index: index+int(n.DataSize)] n.Data = bytes[index : index+int(n.DataSize)]
index = index + int(n.DataSize) index = index + int(n.DataSize)
n.Flags = bytes[index] n.Flags = bytes[index]
index = index + 1 index = index + 1
@@ -219,25 +219,25 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) {
if index < lenBytes && n.HasName() { if index < lenBytes && n.HasName() {
n.NameSize = uint8(bytes[index]) n.NameSize = uint8(bytes[index])
index = index + 1 index = index + 1
n.Name = bytes[index: index+int(n.NameSize)] n.Name = bytes[index : index+int(n.NameSize)]
index = index + int(n.NameSize) index = index + int(n.NameSize)
} }
if index < lenBytes && n.HasMime() { if index < lenBytes && n.HasMime() {
n.MimeSize = uint8(bytes[index]) n.MimeSize = uint8(bytes[index])
index = index + 1 index = index + 1
n.Mime = bytes[index: index+int(n.MimeSize)] n.Mime = bytes[index : index+int(n.MimeSize)]
index = index + int(n.MimeSize) index = index + int(n.MimeSize)
} }
if index < lenBytes && n.HasLastModifiedDate() { if index < lenBytes && n.HasLastModifiedDate() {
n.LastModified = util.BytesToUint64(bytes[index: index+LastModifiedBytesLength]) n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
index = index + LastModifiedBytesLength index = index + LastModifiedBytesLength
} }
if index < lenBytes && n.HasTtl() { if index < lenBytes && n.HasTtl() {
n.Ttl = LoadTTLFromBytes(bytes[index: index+TtlBytesLength]) n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
index = index + TtlBytesLength index = index + TtlBytesLength
} }
if index < lenBytes && n.HasPairs() { if index < lenBytes && n.HasPairs() {
n.PairsSize = util.BytesToUint16(bytes[index: index+2]) n.PairsSize = util.BytesToUint16(bytes[index : index+2])
index += 2 index += 2
end := index + int(n.PairsSize) end := index + int(n.PairsSize)
n.Pairs = bytes[index:end] n.Pairs = bytes[index:end]