lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-09-19 23:10:37 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

prepare for zstd

Browse Source
This commit is contained in:
Chris Lu
2020-06-23 09:12:02 -07:00
parent a4e3cffe0a
commit fe60db404a
4 changed files with 38 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
weed/util/compression.go
View File

@@ -8,9 +8,8 @@ import (
"io/ioutil"
"strings"
"golang.org/x/tools/godoc/util"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/klauspost/compress/zstd"
)
func GzipData(input []byte) ([]byte, error) {
@@ -30,6 +29,9 @@ func DecompressData(input []byte) ([]byte, error) {
if IsGzippedContent(input) {
return ungzipData(input)
}
if IsZstdContent(input) {
return unzstdData(input)
}
return nil, fmt.Errorf("unsupported compression")
}
@@ -44,19 +46,10 @@ func ungzipData(input []byte) ([]byte, error) {
return output, err
}
/*
* Default more not to gzip since gzip can be done on client side.
*/
func IsGzippable(ext, mtype string, data []byte) bool {
var zstdEncoder, _ = zstd.NewWriter(nil)
shouldBeZipped, iAmSure := IsGzippableFileType(ext, mtype)
if iAmSure {
return shouldBeZipped
}
isMostlyText := util.IsText(data)
return isMostlyText
func unzstdData(input []byte) ([]byte, error) {
return zstdEncoder.EncodeAll(input, nil), nil
}
func IsGzippedContent(data []byte) bool {
@@ -66,9 +59,16 @@ func IsGzippedContent(data []byte) bool {
return data[0] == 31 && data[1] == 139
}
func IsZstdContent(data []byte) bool {
if len(data) < 4 {
return false
}
return data[0] == 0xFD && data[1] == 0x2F && data[2] == 0xB5 && data[3] == 0x28
}
/*
* Default more not to gzip since gzip can be done on client side.
*/func IsGzippableFileType(ext, mtype string) (shouldBeZipped, iAmSure bool) {
* Default not to compressed since compression can be done on client side.
*/func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool) {
// text
if strings.HasPrefix(mtype, "text/") {
@@ -86,7 +86,7 @@ func IsGzippedContent(data []byte) bool {
// by file name extension
switch ext {
case ".zip", ".rar", ".gz", ".bz2", ".xz":
case ".zip", ".rar", ".gz", ".bz2", ".xz", ".zst":
return false, true
case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
return true, true
@@ -98,13 +98,15 @@ func IsGzippedContent(data []byte) bool {
// by mime type
if strings.HasPrefix(mtype, "application/") {
if strings.HasSuffix(mtype, "zstd") {
return false, true
}
if strings.HasSuffix(mtype, "xml") {
return true, true
}
if strings.HasSuffix(mtype, "script") {
return true, true
}
}
if strings.HasPrefix(mtype, "audio/") {