lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-02-09 09:17:28 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

filer: remember content is gzipped or not

Browse Source
This commit is contained in:
Chris Lu
2020-03-08 21:39:33 -07:00
parent 5ac6297c68
commit 2e3f6ad3a9
25 changed files with 338 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
weed/util/http_util.go
View File

@@ -189,11 +189,11 @@ func NormalizeUrl(url string) string {
return "http://" + url
}
func ReadUrl(fileUrl string, cipherKey []byte, isFullChunk bool, offset int64, size int, buf []byte) (int64, error) {
func ReadUrl(fileUrl string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int, buf []byte) (int64, error) {
if cipherKey != nil {
var n int
err := readEncryptedUrl(fileUrl, cipherKey, offset, size, func(data []byte) {
err := readEncryptedUrl(fileUrl, cipherKey, isGzipped, offset, size, func(data []byte) {
n = copy(buf, data)
})
return int64(n), err
@@ -258,10 +258,10 @@ func ReadUrl(fileUrl string, cipherKey []byte, isFullChunk bool, offset int64, s
return n, err
}
func ReadUrlAsStream(fileUrl string, cipherKey []byte, isFullChunk bool, offset int64, size int, fn func(data []byte)) error {
func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) error {
if cipherKey != nil {
return readEncryptedUrl(fileUrl, cipherKey, offset, size, fn)
return readEncryptedUrl(fileUrl, cipherKey, isContentGzipped, offset, size, fn)
}
req, err := http.NewRequest("GET", fileUrl, nil)
@@ -300,7 +300,7 @@ func ReadUrlAsStream(fileUrl string, cipherKey []byte, isFullChunk bool, offset
}
func readEncryptedUrl(fileUrl string, cipherKey []byte, offset int64, size int, fn func(data []byte)) error {
func readEncryptedUrl(fileUrl string, cipherKey []byte, isContentGzipped bool, offset int64, size int, fn func(data []byte)) error {
encryptedData, err := Get(fileUrl)
if err != nil {
return fmt.Errorf("fetch %s: %v", fileUrl, err)
@@ -309,6 +309,12 @@ func readEncryptedUrl(fileUrl string, cipherKey []byte, offset int64, size int,
if err != nil {
return fmt.Errorf("decrypt %s: %v", fileUrl, err)
}
if isContentGzipped {
decryptedData, err = UnGzipData(decryptedData)
if err != nil {
return fmt.Errorf("unzip decrypt %s: %v", fileUrl, err)
}
}
if len(decryptedData) < int(offset)+size {
return fmt.Errorf("read decrypted %s size %d [%d, %d)", fileUrl, len(decryptedData), offset, int(offset)+size)
}