lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-11-24 16:53:14 +08:00
Code Issues Actions 17 Packages Projects Releases Wiki Activity

Compare commits

base: lsm:2.14
Branches Tags
lsm:master lsm:java-client-replication-configuration lsm:also-delete-parent-directory-if-empty lsm:gh-pages lsm:pr-7412 lsm:enhance-erasure-coding lsm:add-admin-and-worker-to-helm-charts lsm:do-not-retry-if-error-is-NotFound lsm:improve-fuse-mount2 lsm:improve-fuse-mount lsm:add-ec-vacuum lsm:detect-and-plan-ec-tasks lsm:worker-execute-ec-tasks lsm:fix-versioning-listing-only lsm:fix-GetObjectLockConfigurationHandler lsm:testing-sdx-generation lsm:adding-message-queue-integration-tests lsm:refactor-volume-write lsm:refactor-needle-read-operations lsm:changing-to-zap lsm:collect-public-metrics lsm:revert-6434-bugfix-missing-s3-audit lsm:mq2 lsm:dependabot/maven/other/java/examples/org.apache.hadoop-hadoop-common-3.4.0 lsm:dependabot/maven/other/java/client/com.google.protobuf-protobuf-java-3.25.5 lsm:revert-5819-patch-1 lsm:mq-subscribe lsm:revert-5134-patch-1 lsm:create-table-snapshot-api-design lsm:sub lsm:test-reverting-lock-table lsm:track-mount-e2e lsm:data_query_pushdown lsm:testing lsm:message_send lsm:original_weed_mount lsm:mount2 lsm:avoid_releasing_temp_file_on_write lsm:add_remote_storage lsm:remote_overlay lsm:test_udp lsm:volume_buffered_writes lsm:ftp lsm:logrus lsm:random_access_file lsm:tikv lsm:tcp_read lsm:fasthttp lsm:add_fasthttp_client lsm:s3-select lsm:filer1_maintenance_branch
lsm:4.00 lsm:3.99 lsm:3.98 lsm:3.97 lsm:3.96 lsm:3.95 lsm:3.94 lsm:3.93 lsm:3.92 lsm:3.91 lsm:3.90 lsm:3.89 lsm:3.88 lsm:3.87 lsm:3.86 lsm:3.85 lsm:3.84 lsm:3.83 lsm:3.82 lsm:3.81 lsm:3.80 lsm:3.79 lsm:3.78 lsm:3.77 lsm:3.76 lsm:3.75 lsm:3.74 lsm:3.73 lsm:3.72 lsm:3.71 lsm:3.69 lsm:3.68 lsm:3.67 lsm:3.66 lsm:helm-3.65.1 lsm:3.65 lsm:3.64 lsm:3.63 lsm:3.62 lsm:3.61 lsm:3.60 lsm:3.59 lsm:3.58 lsm:3.57 lsm:3.56 lsm:3.55 lsm:3.54 lsm:3.53 lsm:3.52 lsm:3.51 lsm:3.50 lsm:3.48 lsm:3.47 lsm:3.46 lsm:3.45 lsm:3.44 lsm:3.43 lsm:3.42 lsm:3.41 lsm:3.40 lsm:3.39 lsm:3.38 lsm:3.37 lsm:3.36 lsm:3.35 lsm:3.34 lsm:3.33 lsm:v3.33 lsm:3.32 lsm:3.31 lsm:3.30 lsm:3.29 lsm:3.28 lsm:3.27 lsm:3.26 lsm:3.25 lsm:3.24 lsm:3.23 lsm:3.22 lsm:3.21 lsm:3.20 lsm:3.19 lsm:3.18 lsm:3.16 lsm:3.15 lsm:dev lsm:3.14 lsm:3.13 lsm:3.12 lsm:3.11 lsm:3.10 lsm:3.09 lsm:3.08 lsm:3.07 lsm:3.06 lsm:3.05 lsm:3.04 lsm:3.03 lsm:3.02 lsm:3.01 lsm:3.00 lsm:2.99 lsm:2.98 lsm:2.97 lsm:2.96 lsm:2.95 lsm:2.94 lsm:2.93 lsm:2.92 lsm:2.91 lsm:2.90 lsm:2.89 lsm:2.88 lsm:2.87 lsm:2.86 lsm:2.85 lsm:2.84 lsm:2.83 lsm:2.82 lsm:2.81 lsm:2.80 lsm:2.79 lsm:2.78 lsm:2.77 lsm:2.76 lsm:2.75 lsm:2.74 lsm:2.73 lsm:2.72 lsm:2.71 lsm:2.70 lsm:2.69 lsm:2.68 lsm:2.67 lsm:2.66 lsm:2.65 lsm:2.64 lsm:2.63 lsm:2.62 lsm:2.61 lsm:2.60 lsm:2.59 lsm:2.58 lsm:2.57 lsm:2.56 lsm:2.55 lsm:2.54 lsm:2.53 lsm:2.52 lsm:2.51 lsm:2.50 lsm:2.49 lsm:2.48 lsm:2.47 lsm:2.43 lsm:2.42 lsm:2.41 lsm:2.40 lsm:2.39 lsm:2.38 lsm:2.37 lsm:2.36 lsm:2.35 lsm:2.34 lsm:2.33 lsm:2.32 lsm:2.31 lsm:2.30 lsm:2.29 lsm:2.28 lsm:2.27 lsm:2.26 lsm:2.25 lsm:2.24 lsm:2.23 lsm:2.22 lsm:2.21 lsm:2.20 lsm:2.19 lsm:2.18 lsm:2.17 lsm:2.16 lsm:2.15 lsm:2.14 lsm:2.13 lsm:2.12 lsm:2.11 lsm:2.10 lsm:2.09 lsm:2.08 lsm:2.07 lsm:2.06 lsm:2.05 lsm:2.04 lsm:2.03 lsm:2.02 lsm:2.01 lsm:2.00 lsm:1.99 lsm:1.98 lsm:1.97 lsm:1.96 lsm:1.95 lsm:1.94 lsm:1.93 lsm:1.92 lsm:1.91 lsm:1.90 lsm:1.88 lsm:1.87 lsm:1.86 lsm:1.85 lsm:1.84 lsm:1.83 lsm:1.82 lsm:1.81 lsm:1.80 lsm:1.79 lsm:1.78 lsm:1.77 lsm:1.76 lsm:1.75 lsm:1.74 lsm:1.73 lsm:1.72 lsm:1.71 lsm:1.70 lsm:1;70 lsm:1.69 lsm:1.68 lsm:1.67 lsm:1.66 lsm:1.65 lsm:1.64 lsm:1.63 lsm:1.62 lsm:1.61 lsm:1.61RC lsm:1.60 lsm:1.59 lsm:1.58 lsm:1.57 lsm:1.56 lsm:1.55 lsm:1.54 lsm:1.53 lsm:1.52 lsm:1.51 lsm:1.50 lsm:1.49 lsm:1.48 lsm:1.47 lsm:1.46 lsm:1.45 lsm:1.44 lsm:1.43 lsm:1.42 lsm:1.41 lsm:1.40 lsm:1.38 lsm:1.37 lsm:1.36 lsm:1.35 lsm:1.34 lsm:1.33 lsm:1.32 lsm:1.31 lsm:1.30 lsm:1.29 lsm:1.28 lsm:1.27 lsm:1.26 lsm:1.25 lsm:1.24 lsm:1.23 lsm:1.22 lsm:1.21 lsm:1.20 lsm:1.19 lsm:1.18 lsm:1.17 lsm:1.16 lsm:1.15 lsm:1.14 lsm:1.12 lsm:1.11 lsm:1.10 lsm:1.09 lsm:1.08 lsm:1.07 lsm:1.06 lsm:1.05 lsm:1.04 lsm:1.03 lsm:1.02 lsm:1.01 lsm:1.00 lsm:0.99 lsm:0.98 lsm:0.97 lsm:0.96 lsm:0.95 lsm:0.94 lsm:0.93 lsm:0.92 lsm:0.91 lsm:0.90 lsm:0.77 lsm:0.76 lsm:0.75 lsm:0.74 lsm:0.73 lsm:0.72.release lsm:0.72 lsm:v0.69 lsm:v0.70beta
...
compare: lsm:s3-select
Branches Tags
lsm:java-client-replication-configuration lsm:master lsm:also-delete-parent-directory-if-empty lsm:gh-pages lsm:pr-7412 lsm:enhance-erasure-coding lsm:add-admin-and-worker-to-helm-charts lsm:do-not-retry-if-error-is-NotFound lsm:improve-fuse-mount2 lsm:improve-fuse-mount lsm:add-ec-vacuum lsm:detect-and-plan-ec-tasks lsm:worker-execute-ec-tasks lsm:fix-versioning-listing-only lsm:fix-GetObjectLockConfigurationHandler lsm:testing-sdx-generation lsm:adding-message-queue-integration-tests lsm:refactor-volume-write lsm:refactor-needle-read-operations lsm:changing-to-zap lsm:collect-public-metrics lsm:revert-6434-bugfix-missing-s3-audit lsm:mq2 lsm:dependabot/maven/other/java/examples/org.apache.hadoop-hadoop-common-3.4.0 lsm:dependabot/maven/other/java/client/com.google.protobuf-protobuf-java-3.25.5 lsm:revert-5819-patch-1 lsm:mq-subscribe lsm:revert-5134-patch-1 lsm:create-table-snapshot-api-design lsm:sub lsm:test-reverting-lock-table lsm:track-mount-e2e lsm:data_query_pushdown lsm:testing lsm:message_send lsm:original_weed_mount lsm:mount2 lsm:avoid_releasing_temp_file_on_write lsm:add_remote_storage lsm:remote_overlay lsm:test_udp lsm:volume_buffered_writes lsm:ftp lsm:logrus lsm:random_access_file lsm:tikv lsm:tcp_read lsm:fasthttp lsm:add_fasthttp_client lsm:s3-select lsm:filer1_maintenance_branch
lsm:4.00 lsm:3.99 lsm:3.98 lsm:3.97 lsm:3.96 lsm:3.95 lsm:3.94 lsm:3.93 lsm:3.92 lsm:3.91 lsm:3.90 lsm:3.89 lsm:3.88 lsm:3.87 lsm:3.86 lsm:3.85 lsm:3.84 lsm:3.83 lsm:3.82 lsm:3.81 lsm:3.80 lsm:3.79 lsm:3.78 lsm:3.77 lsm:3.76 lsm:3.75 lsm:3.74 lsm:3.73 lsm:3.72 lsm:3.71 lsm:3.69 lsm:3.68 lsm:3.67 lsm:3.66 lsm:helm-3.65.1 lsm:3.65 lsm:3.64 lsm:3.63 lsm:3.62 lsm:3.61 lsm:3.60 lsm:3.59 lsm:3.58 lsm:3.57 lsm:3.56 lsm:3.55 lsm:3.54 lsm:3.53 lsm:3.52 lsm:3.51 lsm:3.50 lsm:3.48 lsm:3.47 lsm:3.46 lsm:3.45 lsm:3.44 lsm:3.43 lsm:3.42 lsm:3.41 lsm:3.40 lsm:3.39 lsm:3.38 lsm:3.37 lsm:3.36 lsm:3.35 lsm:3.34 lsm:3.33 lsm:v3.33 lsm:3.32 lsm:3.31 lsm:3.30 lsm:3.29 lsm:3.28 lsm:3.27 lsm:3.26 lsm:3.25 lsm:3.24 lsm:3.23 lsm:3.22 lsm:3.21 lsm:3.20 lsm:3.19 lsm:3.18 lsm:3.16 lsm:3.15 lsm:dev lsm:3.14 lsm:3.13 lsm:3.12 lsm:3.11 lsm:3.10 lsm:3.09 lsm:3.08 lsm:3.07 lsm:3.06 lsm:3.05 lsm:3.04 lsm:3.03 lsm:3.02 lsm:3.01 lsm:3.00 lsm:2.99 lsm:2.98 lsm:2.97 lsm:2.96 lsm:2.95 lsm:2.94 lsm:2.93 lsm:2.92 lsm:2.91 lsm:2.90 lsm:2.89 lsm:2.88 lsm:2.87 lsm:2.86 lsm:2.85 lsm:2.84 lsm:2.83 lsm:2.82 lsm:2.81 lsm:2.80 lsm:2.79 lsm:2.78 lsm:2.77 lsm:2.76 lsm:2.75 lsm:2.74 lsm:2.73 lsm:2.72 lsm:2.71 lsm:2.70 lsm:2.69 lsm:2.68 lsm:2.67 lsm:2.66 lsm:2.65 lsm:2.64 lsm:2.63 lsm:2.62 lsm:2.61 lsm:2.60 lsm:2.59 lsm:2.58 lsm:2.57 lsm:2.56 lsm:2.55 lsm:2.54 lsm:2.53 lsm:2.52 lsm:2.51 lsm:2.50 lsm:2.49 lsm:2.48 lsm:2.47 lsm:2.43 lsm:2.42 lsm:2.41 lsm:2.40 lsm:2.39 lsm:2.38 lsm:2.37 lsm:2.36 lsm:2.35 lsm:2.34 lsm:2.33 lsm:2.32 lsm:2.31 lsm:2.30 lsm:2.29 lsm:2.28 lsm:2.27 lsm:2.26 lsm:2.25 lsm:2.24 lsm:2.23 lsm:2.22 lsm:2.21 lsm:2.20 lsm:2.19 lsm:2.18 lsm:2.17 lsm:2.16 lsm:2.15 lsm:2.14 lsm:2.13 lsm:2.12 lsm:2.11 lsm:2.10 lsm:2.09 lsm:2.08 lsm:2.07 lsm:2.06 lsm:2.05 lsm:2.04 lsm:2.03 lsm:2.02 lsm:2.01 lsm:2.00 lsm:1.99 lsm:1.98 lsm:1.97 lsm:1.96 lsm:1.95 lsm:1.94 lsm:1.93 lsm:1.92 lsm:1.91 lsm:1.90 lsm:1.88 lsm:1.87 lsm:1.86 lsm:1.85 lsm:1.84 lsm:1.83 lsm:1.82 lsm:1.81 lsm:1.80 lsm:1.79 lsm:1.78 lsm:1.77 lsm:1.76 lsm:1.75 lsm:1.74 lsm:1.73 lsm:1.72 lsm:1.71 lsm:1.70 lsm:1;70 lsm:1.69 lsm:1.68 lsm:1.67 lsm:1.66 lsm:1.65 lsm:1.64 lsm:1.63 lsm:1.62 lsm:1.61 lsm:1.61RC lsm:1.60 lsm:1.59 lsm:1.58 lsm:1.57 lsm:1.56 lsm:1.55 lsm:1.54 lsm:1.53 lsm:1.52 lsm:1.51 lsm:1.50 lsm:1.49 lsm:1.48 lsm:1.47 lsm:1.46 lsm:1.45 lsm:1.44 lsm:1.43 lsm:1.42 lsm:1.41 lsm:1.40 lsm:1.38 lsm:1.37 lsm:1.36 lsm:1.35 lsm:1.34 lsm:1.33 lsm:1.32 lsm:1.31 lsm:1.30 lsm:1.29 lsm:1.28 lsm:1.27 lsm:1.26 lsm:1.25 lsm:1.24 lsm:1.23 lsm:1.22 lsm:1.21 lsm:1.20 lsm:1.19 lsm:1.18 lsm:1.17 lsm:1.16 lsm:1.15 lsm:1.14 lsm:1.12 lsm:1.11 lsm:1.10 lsm:1.09 lsm:1.08 lsm:1.07 lsm:1.06 lsm:1.05 lsm:1.04 lsm:1.03 lsm:1.02 lsm:1.01 lsm:1.00 lsm:0.99 lsm:0.98 lsm:0.97 lsm:0.96 lsm:0.95 lsm:0.94 lsm:0.93 lsm:0.92 lsm:0.91 lsm:0.90 lsm:0.77 lsm:0.76 lsm:0.75 lsm:0.74 lsm:0.73 lsm:0.72.release lsm:0.72 lsm:v0.69 lsm:v0.70beta

6 Commits
2.14 ... s3-select

Author SHA1 Message Date
Chris Lu
a0a9f6c134 Merge pull request #1166 from BruceWangNo1/s3-select 
temporary s3-select feature documentation
 2019-12-23 11:23:56 -08:00
Bruce Wang
fa4f599ad4 temporary s3-select feature documentation 2019-12-23 17:34:34 +08:00
Chris Lu
8d068dc5d6 Merge pull request #1137 from BruceWangNo1/s3-select 
followed @chrislusf and @iliul 's suggestions
 2019-11-25 19:37:22 -08:00
Bruce Wang
86a71c3e24 turning off support for Go 1.11 and 1.12 in .travis.yml 2019-11-26 10:18:10 +08:00
Bruce Wang
922776151e followed @chrislusf and @iliul 's suggestions. Commits is moved to branch s3-select and proxyToFiler is modified to accommodate S3 Select 2019-11-25 22:37:30 +08:00
Bruce Wang
2e07aab7ea AWS S3 Select feature now available 2019-11-24 10:45:06 +08:00
6 changed files with 673 additions and 20 deletions
Expand all files Collapse all files

2
.travis.yml
View File

@@ -1,8 +1,6 @@
sudo: false
language: go
go:
- 1.11.x
- 1.12.x
- 1.13.x
before_install:

22
go.mod
View File

@@ -10,10 +10,9 @@ require (
contrib.go.opencensus.io/resource v0.1.2 // indirect
github.com/Azure/azure-amqp-common-go v1.1.4 // indirect
github.com/Azure/azure-pipeline-go v0.2.2 // indirect
github.com/Azure/azure-sdk-for-go v33.0.0+incompatible // indirect
github.com/Azure/azure-storage-blob-go v0.8.0
github.com/Azure/go-autorest v13.0.0+incompatible // indirect
github.com/Azure/go-autorest/tracing v0.5.0 // indirect
github.com/Azure/go-autorest/autorest v0.9.2 // indirect
github.com/DataDog/zstd v1.4.1 // indirect
github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190828224159-d93c53a4824c // indirect
github.com/Shopify/sarama v1.23.1
@@ -22,12 +21,10 @@ require (
github.com/aws/aws-sdk-go v1.23.13
github.com/chrislusf/raft v0.0.0-20190225081310-10d6e2182d92
github.com/coreos/etcd v3.3.15+incompatible // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/disintegration/imaging v1.6.1
github.com/dustin/go-humanize v1.0.0
github.com/eapache/go-resiliency v1.2.0 // indirect
github.com/gabriel-vasile/mimetype v0.3.17
github.com/go-kit/kit v0.9.0 // indirect
github.com/go-redis/redis v6.15.2+incompatible
@@ -39,28 +36,23 @@ require (
github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70 // indirect
github.com/google/uuid v1.1.1
github.com/gorilla/mux v1.7.3
github.com/gorilla/websocket v1.4.1 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f
github.com/jcmturner/gofork v1.0.0 // indirect
github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9 // indirect
github.com/karlseguin/ccache v2.0.3+incompatible
github.com/karlseguin/expect v1.0.1 // indirect
github.com/klauspost/cpuid v1.2.1 // indirect
github.com/klauspost/crc32 v1.2.0
github.com/klauspost/reedsolomon v1.9.2
github.com/klauspost/reedsolomon v1.9.3
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/kr/pty v1.1.8 // indirect
github.com/kurin/blazer v0.5.3
github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3
github.com/lib/pq v1.2.0
github.com/magiconair/properties v1.8.1 // indirect
github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect
github.com/mattn/go-isatty v0.0.9 // indirect
github.com/mattn/go-runewidth v0.0.4 // indirect
github.com/minio/minio v0.0.0-20191122214605-0a56e33ce1b5
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
github.com/nats-io/gnatsd v1.4.1 // indirect
github.com/nats-io/go-nats v1.7.2 // indirect
github.com/nats-io/nats-server/v2 v2.0.4 // indirect
github.com/onsi/ginkgo v1.10.1 // indirect
github.com/onsi/gomega v1.7.0 // indirect
github.com/opentracing/opentracing-go v1.1.0 // indirect
@@ -81,7 +73,6 @@ require (
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
github.com/satori/go.uuid v1.2.0
github.com/seaweedfs/fuse v0.0.0-20190510212405-310228904eff
github.com/sirupsen/logrus v1.4.2 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.2.2 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
@@ -107,13 +98,12 @@ require (
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979 // indirect
golang.org/x/image v0.0.0-20190829233526-b3c06291d021 // indirect
golang.org/x/mobile v0.0.0-20190830201351-c6da95954960 // indirect
golang.org/x/net v0.0.0-20190909003024-a7b16738d86b
golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b
golang.org/x/net v0.0.0-20190923162816-aa69164e4478
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69
golang.org/x/tools v0.0.0-20190911022129-16c5e0f7d110
google.golang.org/api v0.9.0
google.golang.org/appengine v1.6.2 // indirect
google.golang.org/grpc v1.23.0
gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
gopkg.in/karlseguin/expect.v1 v1.0.1 // indirect
honnef.co/go/tools v0.0.1-2019.2.2 // indirect

548
go.sum
View File

File diff suppressed because it is too large Load Diff

47
weed/s3api/S3SelectREADME.md Normal file
View File

@@ -0,0 +1,47 @@
# AWS S3 Select feature in Seaweedfs
## Start weed
```bash
git checkout s3-select
# Compile weed
weed server -s3
```
## Install And Configure AWSCLI
```bash
pip install awscli
aws configure // default all the way
aws configure set default.s3.signature_version s3v4
```
## Make a bucket in S3
```bash
aws --endpoint-url http://localhost:8333 s3 mb s3://newbucketgoogle
```
## Download googleplaystore.csv
Link: https://www.kaggle.com/lava18/google-play-store-apps
## Copy the file to the S3 bucket
```bash
aws --endpoint-url http://localhost:8333 s3 copy google-play-store-apps/googleplaystore.csv s3://newbucketgoogle
```
## Perform S3 Select Query
```bash
aws --endpoint-url http://localhost:8333 s3api select-object-content --bucket "newbucketgoogle" --key
"googleplaystore.csv" --expression "select * from s3object limit 5" --expression-type 'SQL' --input-serialization '{"CSV": {"FileHeaderInfo": "USE","RecordDelimiter": "\n","FieldDelimiter": ","}, "CompressionType": "NONE"}' --output-serialization '{"CSV": {"RecordDelimiter": "\n","FieldDelimiter": ","}}' "sample.csv"
```
## Check out result
```bash
vim sample.csv
```

72
weed/s3api/s3api_object_handlers.go
View File

@@ -4,6 +4,8 @@ import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/minio/minio/cmd"
"github.com/minio/minio/pkg/s3select"
"io"
"io/ioutil"
"net/http"
@@ -12,6 +14,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/server"
"github.com/gorilla/mux"
xhttp "github.com/minio/minio/cmd/http"
)
var (
@@ -115,10 +118,15 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
}
func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResonse *http.Response, w http.ResponseWriter)) {
var method string
method = r.Method
glog.V(2).Infof("s3 proxying %s to %s", r.Method, destUrl)
if r.URL.RawQuery == "select&select-type=2" {
method = "GET"
}
glog.V(2).Infof("s3 proxying %s to %s", method, destUrl)
proxyReq, err := http.NewRequest(r.Method, destUrl, r.Body)
proxyReq, err := http.NewRequest(method, destUrl, r.Body)
if err != nil {
glog.Errorf("NewRequest %s: %v", destUrl, err)
@@ -224,3 +232,63 @@ func getObject(vars map[string]string) string {
}
return object
}
func (s3a *S3ApiServer) SelectObjectContent(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
object := getObject(vars)
s3Select, err := s3select.NewS3Select(r.Body)
if err != nil {
if serr, ok := err.(s3select.SelectError); ok {
encodedErrorResponse := encodeResponse(cmd.APIErrorResponse{
Code: serr.ErrorCode(),
Message: serr.ErrorMessage(),
BucketName: bucket,
Key: object,
Resource: r.URL.Path,
RequestID: w.Header().Get(xhttp.AmzRequestID),
HostID: "",
})
writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, "application/xml")
} else {
writeErrorResponse(w, ErrInternalError, r.URL)
}
return
}
if strings.HasSuffix(r.URL.Path, "/") {
writeErrorResponse(w, ErrNotImplemented, r.URL)
return
}
destUrl := fmt.Sprintf("http://%s%s/%s%s",
s3a.option.Filer, s3a.option.BucketsPath, bucket, object)
passThroughResponseSelectObjectContent := func(proxyResonse *http.Response,
w http.ResponseWriter) {
getObject := func(offset, length int64) (io.ReadCloser, error) {
return proxyResonse.Body, nil
}
if err = s3Select.Open(getObject); err != nil {
if serr, ok := err.(s3select.SelectError); ok {
encodedErrorResponse := encodeResponse(cmd.APIErrorResponse{
Code: serr.ErrorCode(),
Message: serr.ErrorMessage(),
BucketName: bucket,
Key: object,
Resource: r.URL.Path,
RequestID: w.Header().Get(xhttp.AmzRequestID),
HostID: "",
})
writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML)
} else {
writeResponse(w, http.StatusInternalServerError, encodeResponse("not s3select.SelectError"), mimeXML)
}
return
}
s3Select.Evaluate(w)
s3Select.Close()
}
s3a.proxyToFiler(w, r, destUrl, passThroughResponseSelectObjectContent)
}

2
weed/s3api/s3api_server.go
View File

@@ -79,6 +79,8 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.GetObjectHandler)
// ListObjectsV1 (Legacy)
bucket.Methods("GET").HandlerFunc(s3a.ListObjectsV1Handler)
// SelectObjectContent
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(s3a.SelectObjectContent).Queries("select", "", "select-type", "2")
// DeleteMultipleObjects
bucket.Methods("POST").HandlerFunc(s3a.DeleteMultipleObjectsHandler).Queries("delete", "")