objstore : implement Aliyun OSS (#1573)

* add oss support

Signed-off-by: wujinhu <wujinhu920@126.com>

* fix docs

Signed-off-by: wujinhu <wujinhu920@126.com>

* fix Makefile

Signed-off-by: wujinhu <wujinhu920@126.com>

* review comments

Signed-off-by: wujinhu <wujinhu920@126.com>

* fix style

Signed-off-by: wujinhu <wujinhu920@126.com>

* review comments

Signed-off-by: wujinhu <wujinhu920@126.com>

* review comments

Signed-off-by: wujinhu <wujinhu920@126.com>

* review comments

Signed-off-by: wujinhu <wujinhu920@126.com>

* review comments

Signed-off-by: wujinhu <wujinhu920@126.com>
Signed-off-by: Giedrius Statkevičius <giedriuswork@gmail.com>

Author: wujinhu

CHANGELOG.md

@@ -14,6 +14,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 ### Added
 
 - [#1660](https://github.com/thanos-io/thanos/pull/1660) Add a new `--prometheus.ready_timeout` CLI option to the sidecar to set how long to wait until Prometheus starts up.
+- [#1573](https://github.com/thanos-io/thanos/pull/1573) `AliYun OSS` object storage, see [documents](docs/storage.md#aliyun-oss-configuration) for further information.
 
 ### Fixed
 

CONTRIBUTING.md

@@ -57,6 +57,7 @@ $ git push origin <your_branch_for_new_pr>
 - THANOS_SKIP_AZURE_TESTS to skip Azure tests.
 - THANOS_SKIP_SWIFT_TESTS to skip SWIFT tests.
 - THANOS_SKIP_TENCENT_COS_TESTS to skip Tencent COS tests.
+- THANOS_SKIP_ALIYUN_OSS_TESTS to skip Aliyun OSS tests.
 
 If you skip all of these, the store specific tests will be run against memory object storage only.
 CI runs GCS and inmem tests only for now. Not having these variables will produce auth errors against GCS, AWS, Azure or COS tests.

Makefile

@@ -160,8 +160,8 @@ docs: $(EMBEDMD) build
 .PHONY: check-docs
 check-docs: $(EMBEDMD) $(LICHE) build
 	@EMBEDMD_BIN="$(EMBEDMD)" scripts/genflagdocs.sh check
-	@$(LICHE) --recursive docs --exclude "cloud.tencent.com" --document-root .
-	@$(LICHE) --exclude "cloud.tencent.com|goreportcard.com" --document-root . *.md
+	@$(LICHE) --recursive docs --exclude "(cloud.tencent.com|alibabacloud.com)" --document-root .
+	@$(LICHE) --exclude "(cloud.tencent.com|goreportcard.com|alibabacloud.com)" --document-root . *.md
 
 # checks Go code comments if they have trailing period (excludes protobuffers and vendor files).
 # Comments with more than 3 spaces at beginning are omitted from the check, example: '//    - foo'.
@@ -203,17 +203,19 @@ test: check-git install-deps
 	@go install github.com/thanos-io/thanos/cmd/thanos
 	# Be careful on GOCACHE. Those tests are sometimes using built Thanos/Prometheus binaries directly. Don't cache those.
 	@rm -rf ${GOCACHE}
-	@echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS='true' or/and THANOS_SKIP_S3_AWS_TESTS='true' or/and THANOS_SKIP_AZURE_TESTS='true' and/or THANOS_SKIP_SWIFT_TESTS='true' and/or THANOS_SKIP_TENCENT_COS_TESTS='true' if you want to skip e2e tests against real store buckets"
+	@echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS='true' or/and THANOS_SKIP_S3_AWS_TESTS='true' or/and THANOS_SKIP_AZURE_TESTS='true' and/or THANOS_SKIP_SWIFT_TESTS='true' and/or THANOS_SKIP_ALIYUN_OSS_TESTS='true' and/or THANOS_SKIP_TENCENT_COS_TESTS='true' if you want to skip e2e tests against real store buckets"
 	@go test $(shell go list ./... | grep -v /vendor/);
 
 .PHONY: test-ci
 test-ci: export THANOS_SKIP_AZURE_TESTS = true
 test-ci: export THANOS_SKIP_SWIFT_TESTS = true
 test-ci: export THANOS_SKIP_TENCENT_COS_TESTS = true
+test-ci: export THANOS_SKIP_ALIYUN_OSS_TESTS = true
 test-ci:
 	@echo ">> Skipping AZURE tests"
 	@echo ">> Skipping SWIFT tests"
 	@echo ">> Skipping TENCENT tests"
+	@echo ">> Skipping ALIYUN tests"
 	$(MAKE) test
 
 .PHONY: test-local

docs/storage.md

@@ -60,6 +60,7 @@ Current object storage client implementations:
 | [Azure Storage Account](./storage.md#azure) | Stable  (production usage) | yes       | @vglafirov   |
 | [OpenStack Swift](./storage.md#openstack-swift)      | Beta  (working PoCs, testing usage)               | no        | @sudhi-vm   |
 | [Tencent COS](./storage.md#tencent-cos)          | Beta  (testing usage)                   | no        | @jojohappy          |
+| [AliYun OSS](./storage.md#aliyun-oss)           | Beta  (testing usage)                   | no        | @shaulboozhiao,@wujinhu      |
 
 NOTE: Currently Thanos requires strong consistency (write-read) for object store implementation.
 
@@ -336,3 +337,20 @@ config:
 ```
 
 Set the flags `--objstore.config-file` to reference to the configuration file.
+
+##  AliYun OSS Configuration
+In order to use AliYun OSS object storage, you should first create a bucket with proper Storage Class , ACLs and get the access key on the AliYun cloud. Go to [https://www.alibabacloud.com/product/oss](https://www.alibabacloud.com/product/oss) for more detail.
+
+To use AliYun OSS object storage, please specify following yaml configuration file in `objstore.config*` flag.
+
+[embedmd]:# (flags/config_bucket_aliyunoss.txt $)
+```$
+type: ALIYUNOSS
+config:
+  endpoint: ""
+  bucket: ""
+  access_key_id: ""
+  access_key_secret: ""
+```
+
+Use --objstore.config-file to reference to this configuration file.

go.mod

@@ -4,7 +4,9 @@ require (
 	cloud.google.com/go v0.44.1
 	github.com/Azure/azure-storage-blob-go v0.7.0
 	github.com/NYTimes/gziphandler v1.1.1
+	github.com/aliyun/aliyun-oss-go-sdk v2.0.1+incompatible
 	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878
+	github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
 	github.com/cespare/xxhash v1.1.0
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/dustin/go-humanize v1.0.0

go.sum

@@ -35,6 +35,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZq
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/aliyun/aliyun-oss-go-sdk v2.0.1+incompatible h1:/MzpJOMHn/uBtd1dkS7Q9PF2ZjT6xTQMXSvv1e6ydXc=
+github.com/aliyun/aliyun-oss-go-sdk v2.0.1+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -47,6 +49,8 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:l
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.23.12 h1:2UnxgNO6Y5J1OrkXS8XNp0UatDxD1bWHiDT62RDPggI=
 github.com/aws/aws-sdk-go v1.23.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
+github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=

pkg/objstore/client/factory.go

@@ -13,6 +13,7 @@ import (
 	"github.com/thanos-io/thanos/pkg/objstore/azure"
 	"github.com/thanos-io/thanos/pkg/objstore/cos"
 	"github.com/thanos-io/thanos/pkg/objstore/gcs"
+	"github.com/thanos-io/thanos/pkg/objstore/oss"
 	"github.com/thanos-io/thanos/pkg/objstore/s3"
 	"github.com/thanos-io/thanos/pkg/objstore/swift"
 	yaml "gopkg.in/yaml.v2"
@@ -21,11 +22,12 @@ import (
 type ObjProvider string
 
 const (
-	GCS   ObjProvider = "GCS"
-	S3    ObjProvider = "S3"
-	AZURE ObjProvider = "AZURE"
-	SWIFT ObjProvider = "SWIFT"
-	COS   ObjProvider = "COS"
+	GCS       ObjProvider = "GCS"
+	S3        ObjProvider = "S3"
+	AZURE     ObjProvider = "AZURE"
+	SWIFT     ObjProvider = "SWIFT"
+	COS       ObjProvider = "COS"
+	ALIYUNOSS ObjProvider = "ALIYUNOSS"
 )
 
 type BucketConfig struct {
@@ -59,6 +61,8 @@ func NewBucket(logger log.Logger, confContentYaml []byte, reg prometheus.Registe
 		bucket, err = swift.NewContainer(logger, config)
 	case string(COS):
 		bucket, err = cos.NewBucket(logger, config, component)
+	case string(ALIYUNOSS):
+		bucket, err = oss.NewBucket(logger, config, component)
 	default:
 		return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type)
 	}

pkg/objstore/objtesting/foreach.go

@@ -9,6 +9,7 @@ import (
 	"github.com/thanos-io/thanos/pkg/objstore/cos"
 	"github.com/thanos-io/thanos/pkg/objstore/gcs"
 	"github.com/thanos-io/thanos/pkg/objstore/inmem"
+	"github.com/thanos-io/thanos/pkg/objstore/oss"
 	"github.com/thanos-io/thanos/pkg/objstore/s3"
 	"github.com/thanos-io/thanos/pkg/objstore/swift"
 	"github.com/thanos-io/thanos/pkg/testutil"
@@ -114,4 +115,21 @@ func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket))
 	} else {
 		t.Log("THANOS_SKIP_TENCENT_COS_TESTS envvar present. Skipping test against Tencent COS.")
 	}
+
+	// Optional OSS.
+	if _, ok := os.LookupEnv("THANOS_SKIP_ALIYUN_OSS_TESTS"); !ok {
+		bkt, closeFn, err := oss.NewTestBucket(t)
+		testutil.Ok(t, err)
+
+		ok := t.Run("AliYun oss", func(t *testing.T) {
+			testFn(t, bkt)
+		})
+
+		closeFn()
+		if !ok {
+			return
+		}
+	} else {
+		t.Log("THANOS_SKIP_ALIYUN_OSS_TESTS envvar present. Skipping test against AliYun OSS.")
+	}
 }