diff --git a/.gitignore b/.gitignore index fb281a361fa..12f3a9109df 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ data/ website/public/ website/docs-pre-processed/ +!website/data diff --git a/CHANGELOG.md b/CHANGELOG.md index bb37f0fd25a..af54d0bd822 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,11 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased -## [v0.4.0-rc.1](https://github.com/improbable-eng/thanos/releases/tag/v0.4.0-rc.1) - 2019.04.26 +### Added + +- [#1094](https://github.com/improbable-eng/thanos/pull/1094) Allow configuring the response header timeout for the S3 client. + +## [v0.4.0](https://github.com/improbable-eng/thanos/releases/tag/v0.4.0) - 2019.05.3 :warning: **IMPORTANT** :warning: This is the last release that supports gossip. From Thanos v0.5.0, gossip will be completely removed. @@ -20,6 +24,8 @@ See [this](docs/proposals/approved/201809_gossip-removal.md) for more details. ### Added +- [thanos.io](https://thanos.io) website & automation :tada: +- [#1053](https://github.com/improbable-eng/thanos/pull/1053) compactor: Compactor & store gateway now handles incomplete uploads gracefully. Added hard limit on how long block upload can take (30m). - [#811](https://github.com/improbable-eng/thanos/pull/811) Remote write receiver component :heart: :heart: thanks to RedHat (@brancz) contribution. - [#910](https://github.com/improbable-eng/thanos/pull/910) Query's stores UI page is now sorted by type and old DNS or File SD stores are removed after 5 minutes (configurable via the new `--store.unhealthy-timeout=5m` flag). - [#905](https://github.com/improbable-eng/thanos/pull/905) Thanos support for Query API: /api/v1/labels. Notice that the API was added in Prometheus v2.6. @@ -79,7 +85,7 @@ Note that this is required to have SRV resolution working on [Golang 1.11+ with * Added `thanos_rule_evaluation_with_warnings_total` to Ruler. * DNS `thanos_ruler_query_apis*` are now `thanos_ruler_query_apis_*` for consistency. * DNS `thanos_querier_store_apis*` are now `thanos_querier_store_apis__*` for consistency. - * Query Gate `thanos_bucket_store_series*` are now `thanos_bucket_store_series___*` for consistency. + * Query Gate `thanos_bucket_store_series*` are now `thanos_bucket_store_series_*` for consistency. * Most of thanos ruler metris related to rule manager has `strategy` label. Ruler tracing spans: @@ -100,13 +106,15 @@ Note that this is required to have SRV resolution working on [Golang 1.11+ with * [BUGFIX] Fix sorting of rule groups. #5260 * store: [ENHANCEMENT] Fast path for EmptyPostings cases in Merge, Intersect and Without. * tooling: [FEATURE] New dump command to tsdb tool to dump all samples. - * compactor: [ENHANCEMENT] When closing the db any running compaction will be cancelled so it doesn't block. + * compactor: + * [ENHANCEMENT] When closing the db any running compaction will be cancelled so it doesn't block. + * [CHANGE] Renamed flag `--sync-delay` to `--consistency-delay` [#1053](https://github.com/improbable-eng/thanos/pull/1053) For ruler essentially whole TSDB CHANGELOG applies beween v0.4.0-v0.6.1: https://github.com/prometheus/tsdb/blob/master/CHANGELOG.md Note that this was added on TSDB and Prometheus: [FEATURE] Time-ovelapping blocks are now allowed. #370 Whoever due to nature of Thanos compaction (distributed systems), for safety reason this is disabled for Thanos compactor for now. - + - [#868](https://github.com/improbable-eng/thanos/pull/868) Go has been updated to 1.12. - [#1055](https://github.com/improbable-eng/thanos/pull/1055) Gossip flags are now disabled by default and deprecated. - [#964](https://github.com/improbable-eng/thanos/pull/964) repair: Repair process now sorts the series and labels within block. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4769428dd52..5bd5c88e1d5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,7 @@ CI runs GCS and inmem tests only for now. Not having these variables will produc ## Dependency management -The Thanos project uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies on external packages. This requires a working Go environment with version 1.11 or greater, git and [bzr](http://wiki.bazaar.canonical.com/Download) installed. +The Thanos project uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies on external packages. This requires a working Go environment with version 1.11 or greater and git installed. To add or update a new dependency, use the `go get` command: diff --git a/Makefile b/Makefile index 7eb4bce5d5a..522e0158b10 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,6 @@ HUGO ?= $(GOBIN)/hugo-$(HUGO_VERSION) GOBINDATA_VERSION ?= a9c83481b38ebb1c4eb8f0168fd4b10ca1d3c523 GOBINDATA ?= $(GOBIN)/go-bindata-$(GOBINDATA_VERSION) GIT ?= $(shell which git) -BZR ?= $(shell which bzr) WEB_DIR ?= website WEBSITE_BASE_URL ?= https://thanos.io @@ -104,7 +103,7 @@ assets: $(GOBINDATA) # build builds Thanos binary using `promu`. .PHONY: build -build: check-git check-bzr go-mod-tidy $(PROMU) +build: check-git go-mod-tidy $(PROMU) @echo ">> building binaries $(GOBIN)" @$(PROMU) build --prefix $(PREFIX) @@ -161,7 +160,7 @@ format: $(GOIMPORTS) # proto generates golang files from Thanos proto files. .PHONY: proto -proto: check-git check-bzr $(GOIMPORTS) $(PROTOC) +proto: check-git $(GOIMPORTS) $(PROTOC) @go install ./vendor/github.com/gogo/protobuf/protoc-gen-gogofast @GOIMPORTS_BIN="$(GOIMPORTS)" PROTOC_BIN="$(PROTOC)" scripts/genproto.sh @@ -183,7 +182,7 @@ tarballs-release: $(PROMU) # test runs all Thanos golang tests against each supported version of Prometheus. .PHONY: test -test: check-git check-bzr test-deps +test: check-git test-deps @echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS='true' or/and THANOS_SKIP_S3_AWS_TESTS='true' or/and THANOS_SKIP_AZURE_TESTS='true' and/or THANOS_SKIP_SWIFT_TESTS='true' and/or THANOS_SKIP_TENCENT_COS_TESTS='true' if you want to skip e2e tests against real store buckets" THANOS_TEST_PROMETHEUS_VERSIONS="$(PROM_VERSIONS)" THANOS_TEST_ALERTMANAGER_PATH="alertmanager-$(ALERTMANAGER_VERSION)" go test $(shell go list ./... | grep -v /vendor/ | grep -v /benchmark/); @@ -198,13 +197,13 @@ test-deps: # vet vets the code. .PHONY: vet -vet: check-git check-bzr +vet: check-git @echo ">> vetting code" @go vet ./... # go mod related .PHONY: go-mod-tidy -go-mod-tidy: check-git check-bzr +go-mod-tidy: check-git @go mod tidy .PHONY: check-go-mod @@ -220,13 +219,6 @@ else @echo >&2 "No git binary found."; exit 1 endif -.PHONY: check-bzr -check-bzr: -ifneq ($(BZR),) - @test -x $(BZR) || (echo >&2 "No bzr exectuable binary found at $(BZR)."; exit 1) -else - @echo >&2 "No bzr binary found."; exit 1 -endif .PHONY: web-pre-process web-pre-process: diff --git a/VERSION b/VERSION index cc73c6027e9..ead44d592e0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.0-rc.1 +0.4.0-master diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index bec9746c55a..365d211992d 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -82,7 +82,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri objStoreConfig := regCommonObjStoreFlags(cmd, "", true) - syncDelay := modelDuration(cmd.Flag("sync-delay", "Minimum age of fresh (non-compacted) blocks before they are being processed."). + consistencyDelay := modelDuration(cmd.Flag("consistency-delay", fmt.Sprintf("Minimum age of fresh (non-compacted) blocks before they are being processed. Malformed blocks older than the maximum of consistency-delay and %s will be removed.", compact.MinimumAgeForRemoval)). Default("30m")) retentionRaw := modelDuration(cmd.Flag("retention.resolution-raw", "How long to retain raw samples in bucket. 0d - disables this retention").Default("0d")) @@ -114,7 +114,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application, name stri *httpAddr, *dataDir, objStoreConfig, - time.Duration(*syncDelay), + time.Duration(*consistencyDelay), *haltOnError, *acceptMalformedIndex, *wait, @@ -140,7 +140,7 @@ func runCompact( httpBindAddr string, dataDir string, objStoreConfig *pathOrContent, - syncDelay time.Duration, + consistencyDelay time.Duration, haltOnError bool, acceptMalformedIndex bool, wait bool, @@ -182,7 +182,7 @@ func runCompact( } }() - sy, err := compact.NewSyncer(logger, reg, bkt, syncDelay, + sy, err := compact.NewSyncer(logger, reg, bkt, consistencyDelay, blockSyncConcurrency, acceptMalformedIndex) if err != nil { return errors.Wrap(err, "create syncer") diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 7f77e15412d..c135956403d 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -12,13 +12,14 @@ import ( "github.com/improbable-eng/thanos/pkg/objstore/client" "github.com/improbable-eng/thanos/pkg/runutil" "github.com/improbable-eng/thanos/pkg/store" + storecache "github.com/improbable-eng/thanos/pkg/store/cache" "github.com/improbable-eng/thanos/pkg/store/storepb" "github.com/oklog/run" - "github.com/opentracing/opentracing-go" + opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "google.golang.org/grpc" - "gopkg.in/alecthomas/kingpin.v2" + kingpin "gopkg.in/alecthomas/kingpin.v2" ) // registerStore registers a store command. @@ -120,12 +121,23 @@ func runStore( } }() + // TODO(bwplotka): Add as a flag? + maxItemSizeBytes := indexCacheSizeBytes / 2 + + indexCache, err := storecache.NewIndexCache(logger, reg, storecache.Opts{ + MaxSizeBytes: indexCacheSizeBytes, + MaxItemSizeBytes: maxItemSizeBytes, + }) + if err != nil { + return errors.Wrap(err, "create index cache") + } + bs, err := store.NewBucketStore( logger, reg, bkt, dataDir, - indexCacheSizeBytes, + indexCache, chunkPoolSizeBytes, maxSampleCount, maxConcurrent, diff --git a/docs/components/compact.md b/docs/components/compact.md index 373c6d94b9c..439adc520bf 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -60,8 +60,10 @@ Flags: --objstore.config= Alternative to 'objstore.config-file' flag. Object store configuration in YAML. - --sync-delay=30m Minimum age of fresh (non-compacted) blocks - before they are being processed. + --consistency-delay=30m Minimum age of fresh (non-compacted) blocks + before they are being processed. Malformed blocks + older than the maximum of consistency-delay and + 30m0s will be removed. --retention.resolution-raw=0d How long to retain raw samples in bucket. 0d - disables this retention diff --git a/docs/contributing/how-to-contribute-to-docs.md b/docs/contributing/how-to-contribute-to-docs.md index 1c7a8837e62..96206cac972 100644 --- a/docs/contributing/how-to-contribute-to-docs.md +++ b/docs/contributing/how-to-contribute-to-docs.md @@ -63,12 +63,33 @@ Keep `menu` the same as sub-directory the file is in. This will help to manage a Show new menu section in main page by changing `website/layouts/_default/baseof.html` file. +## Logos + +We'd love to showcase your company's logo on our main page and README! +Requirements for the company: +* it is using Thanos on production +* it is a legal registered company +* it is happy to announce that you use Thanos publicly + +If all those are met, add yourself in [`website/data/sponsors.yml`](/website/data/sponsors.yml) like so: +```yml +- name: My Awesome Company + url: https://wwww.company.com + logo: company.png +``` + +Copy your company's logo in [`/website/static/logos`](/website/static/logos), make sure it follows these rules: + +* Rectangle shape +* Greyscale is preferred but color is fine +* Keep it under 50KB + +and create PR against Thanos `master` branch. + ## Testing Every PR is building website and on success it shows the link to preview. -Alternatively you can run `make web-serve` to serve and preview the website locally. - ## Deployment. We use [Netlify](https://www.netlify.com/) for hosting. We are using Open Source license (PRO). Thanks Netlify for this! @@ -77,4 +98,4 @@ On every commit to `master` netlify runs CI that invokes `make web` (defined in NOTE: Check for status badge in README for build status on the page. -If master build for netlify succeed, the new content is published automatically. \ No newline at end of file +If master build for netlify succeed, the new content is published automatically. diff --git a/docs/getting-started.md b/docs/getting-started.md index d7d53d203eb..dcdaf28c217 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -30,8 +30,9 @@ Thanos will work in cloud native environments as well as more traditional ones. You can find the latest Thanos release [here](https://github.com/improbable-eng/thanos/releases). -If you want to build Thanos from source, make sure you have installed `bzr` and `git`. `bzr` is required, because `go` modules will use whatever VCS dependency use and in our case a single deps is using `bzr`. -And that you have a working installation of the Go [toolchain](https://github.com/golang/tools) (`GOPATH`, `PATH=${GOPATH}/bin:${PATH}`), Thanos can be downloaded and built by running: +If you want to build Thanos from source, make sure you have installed `git` and that you have a working installation of the Go [toolchain](https://github.com/golang/tools) (`GOPATH`, `PATH=${GOPATH}/bin:${PATH}`). + +Thanos can be downloaded and built by running: ```bash go get -d github.com/improbable-eng/thanos/... diff --git a/go.mod b/go.mod index 044c0d0441c..205c61553e2 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/NYTimes/gziphandler v1.1.1 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da github.com/fatih/structtag v1.0.0 - github.com/fortytw2/leaktest v1.2.0 + github.com/fortytw2/leaktest v1.3.0 github.com/fsnotify/fsnotify v1.4.7 github.com/go-kit/kit v0.8.0 github.com/gogo/protobuf v1.2.0 @@ -16,13 +16,13 @@ require ( github.com/gophercloud/gophercloud v0.0.0-20181206160319-9d88c34913a9 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20181025070259-68e3a13e4117 - github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 - github.com/hashicorp/go-version v1.1.0 + github.com/hashicorp/go-sockaddr v1.0.0 + github.com/hashicorp/go-version v1.2.0 github.com/hashicorp/golang-lru v0.5.1 - github.com/hashicorp/memberlist v0.1.0 + github.com/hashicorp/memberlist v0.1.3 github.com/julienschmidt/httprouter v1.1.0 // indirect github.com/lovoo/gcloud-opentracing v0.3.0 - github.com/miekg/dns v1.1.4 + github.com/miekg/dns v1.1.8 github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1 github.com/mozillazg/go-cos v0.11.0 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 diff --git a/go.sum b/go.sum index ca3bbf06649..8ae87f66a1c 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,8 @@ github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= github.com/fatih/structtag v1.0.0 h1:pTHj65+u3RKWYPSGaU290FpI/dXxTaHdVwVwbcPKmEc= github.com/fatih/structtag v1.0.0/go.mod h1:IKitwq45uXL/yqi5mYghiD3w9H6eTOvI9vnk8tXMphA= -github.com/fortytw2/leaktest v1.2.0 h1:cj6GCiwJDH7l3tMHLjZDo0QqPtrXJiWSI9JgpeQKw+Q= -github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/getsentry/raven-go v0.1.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= @@ -82,6 +82,8 @@ github.com/golang/snappy v0.0.0-20160529050041-d9eb7a3d35ec/go.mod h1:/XxbfmMg8l github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= @@ -116,21 +118,27 @@ github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxB github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c h1:BTAbnbegUIMB6xmQCwWE8yRzbA4XSpnZY5hvRJC188I= github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90/go.mod h1:o4zcYY1e0GEZI6eSEr+43QDYmuGglw1qSO6qdHUHCgg= github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 h1:7YOlAIO2YWnJZkQp7B5eFykaIY7C9JndqAFQyVV5BhM= github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0= -github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/memberlist v0.1.0 h1:qSsCiC0WYD39lbSitKNt40e30uorm2Ss/d4JGU1hzH8= github.com/hashicorp/memberlist v0.1.0/go.mod h1:ncdBp14cuox2iFOq3kDiquKU6fqsTBc3W6JvZwjxxsE= +github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.0.0-20161007004122-1d4fa605f6ff h1:3QdMaUEV3zE0VcBgBj+P3GQ26ZKmhb5gLea1hqAZ50U= github.com/hashicorp/serf v0.0.0-20161007004122-1d4fa605f6ff/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ= @@ -163,8 +171,11 @@ github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8Bz github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.4 h1:rCMZsU2ScVSYcAsOXgmC6+AKOK+6pmQTOcw03nfwYV0= github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.8 h1:1QYRAKU3lN5cRfLCkPU08hwvLJFhvjP6MqNMmQz6ZVI= +github.com/miekg/dns v1.1.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1 h1:jw16EimP5oAEM/2wt+SiEUov/YDyTCTDuPtIKgQIvk0= github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1/go.mod h1:vuvdOZLJuf5HmJAJrKV64MmozrSsk+or0PB5dzdfspg= github.com/mitchellh/go-homedir v0.0.0-20180523094522-3864e76763d9/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -260,6 +271,7 @@ go.opencensus.io v0.18.1-0.20181204023538-aab39bd6a98b/go.mod h1:vKdFvxhtzZ9onBp go.opencensus.io v0.19.0 h1:+jrnNy8MR4GZXvwF9PEuSyHxA4NaTf6601oNRwCSXq0= go.opencensus.io v0.19.0/go.mod h1:AYeH0+ZxYyghG8diqaaIq/9P3VgCCt5GF2ldCY4dkFg= golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b h1:Ib/yptP38nXZFMwqWSip+OKuMP9OkyDe3p+DssP8n9w= golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -268,6 +280,7 @@ golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -283,6 +296,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FY golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 150451185d5..c9c21cdeb02 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" "os" + "path" "path/filepath" "sort" "strings" @@ -31,6 +32,8 @@ const ( ResolutionLevelRaw = ResolutionLevel(downsample.ResLevel0) ResolutionLevel5m = ResolutionLevel(downsample.ResLevel1) ResolutionLevel1h = ResolutionLevel(downsample.ResLevel2) + + MinimumAgeForRemoval = time.Duration(30 * time.Minute) ) var blockTooFreshSentinelError = errors.New("Block too fresh") @@ -41,7 +44,7 @@ type Syncer struct { logger log.Logger reg prometheus.Registerer bkt objstore.Bucket - syncDelay time.Duration + consistencyDelay time.Duration mtx sync.Mutex blocks map[ulid.ULID]*metadata.Meta blocksMtx sync.Mutex @@ -132,14 +135,14 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics { // NewSyncer returns a new Syncer for the given Bucket and directory. // Blocks must be at least as old as the sync delay for being considered. -func NewSyncer(logger log.Logger, reg prometheus.Registerer, bkt objstore.Bucket, syncDelay time.Duration, blockSyncConcurrency int, acceptMalformedIndex bool) (*Syncer, error) { +func NewSyncer(logger log.Logger, reg prometheus.Registerer, bkt objstore.Bucket, consistencyDelay time.Duration, blockSyncConcurrency int, acceptMalformedIndex bool) (*Syncer, error) { if logger == nil { logger = log.NewNopLogger() } return &Syncer{ logger: logger, reg: reg, - syncDelay: syncDelay, + consistencyDelay: consistencyDelay, blocks: map[ulid.ULID]*metadata.Meta{}, bkt: bkt, metrics: newSyncerMetrics(reg), @@ -149,7 +152,8 @@ func NewSyncer(logger log.Logger, reg prometheus.Registerer, bkt objstore.Bucket } // SyncMetas synchronizes all meta files from blocks in the bucket into -// the memory. +// the memory. It removes any partial blocks older than the max of +// consistencyDelay and MinimumAgeForRemoval from the bucket. func (c *Syncer) SyncMetas(ctx context.Context) error { c.mtx.Lock() defer c.mtx.Unlock() @@ -194,6 +198,9 @@ func (c *Syncer) syncMetas(ctx context.Context) error { continue } if err != nil { + if removedOrIgnored := c.removeIfMetaMalformed(workCtx, id); removedOrIgnored { + continue + } errChan <- err return } @@ -250,6 +257,10 @@ func (c *Syncer) downloadMeta(ctx context.Context, id ulid.ULID) (*metadata.Meta meta, err := block.DownloadMeta(ctx, c.logger, c.bkt, id) if err != nil { + if ulid.Now()-id.Time() < uint64(c.consistencyDelay/time.Millisecond) { + level.Debug(c.logger).Log("msg", "block is too fresh for now", "block", id) + return nil, blockTooFreshSentinelError + } return nil, errors.Wrapf(err, "downloading meta.json for %s", id) } @@ -259,7 +270,7 @@ func (c *Syncer) downloadMeta(ctx context.Context, id ulid.ULID) (*metadata.Meta // - compactor created blocks // NOTE: It is not safe to miss "old" block (even that it is newly created) in sync step. Compactor needs to aware of ALL old blocks. // TODO(bplotka): https://github.com/improbable-eng/thanos/issues/377 - if ulid.Now()-id.Time() < uint64(c.syncDelay/time.Millisecond) && + if ulid.Now()-id.Time() < uint64(c.consistencyDelay/time.Millisecond) && meta.Thanos.Source != metadata.BucketRepairSource && meta.Thanos.Source != metadata.CompactorSource && meta.Thanos.Source != metadata.CompactorRepairSource { @@ -271,6 +282,33 @@ func (c *Syncer) downloadMeta(ctx context.Context, id ulid.ULID) (*metadata.Meta return &meta, nil } +// removeIfMalformed removes a block from the bucket if that block does not have a meta file. It ignores blocks that +// are younger than MinimumAgeForRemoval. +func (c *Syncer) removeIfMetaMalformed(ctx context.Context, id ulid.ULID) (removedOrIgnored bool) { + metaExists, err := c.bkt.Exists(ctx, path.Join(id.String(), block.MetaFilename)) + if err != nil { + level.Warn(c.logger).Log("msg", "failed to check meta exists for block", "block", id, "err", err) + return false + } + if metaExists { + // Meta exists, block is not malformed. + return false + } + + if ulid.Now()-id.Time() <= uint64(MinimumAgeForRemoval/time.Millisecond) { + // Minimum delay has not expired, ignore for now + return true + } + + if err := block.Delete(ctx, c.bkt, id); err != nil { + level.Warn(c.logger).Log("msg", "failed to delete malformed block", "block", id, "err", err) + return false + } + level.Info(c.logger).Log("msg", "deleted malformed block", "block", id) + + return true +} + // GroupKey returns a unique identifier for the group the block belongs to. It considers // the downsampling resolution and the block's labels. func GroupKey(meta metadata.Meta) string { diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index f175fcce931..a5fbe5d790b 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -1,7 +1,13 @@ package compact import ( + "bytes" + "context" + "github.com/improbable-eng/thanos/pkg/objstore/inmem" + "github.com/oklog/ulid" + "path" "testing" + "time" "github.com/improbable-eng/thanos/pkg/testutil" "github.com/pkg/errors" @@ -37,3 +43,37 @@ func TestRetryError(t *testing.T) { err = errors.Wrap(retry(errors.Wrap(halt(errors.New("test")), "something")), "something2") testutil.Assert(t, IsHaltError(err), "not a halt error. Retry should not hide halt error") } + +func TestSyncer_SyncMetas_HandlesMalformedBlocks(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + bkt := inmem.NewBucket() + sy, err := NewSyncer(nil, nil, bkt, 10*time.Second, 1, false) + testutil.Ok(t, err) + + // Generate 1 block which is older than MinimumAgeForRemoval which has chunk data but no meta. Compactor should delete it. + shouldDeleteId, err := ulid.New(uint64(time.Now().Add(-time.Hour).Unix()*1000), nil) + testutil.Ok(t, err) + + var fakeChunk bytes.Buffer + fakeChunk.Write([]byte{0,1,2,3}) + testutil.Ok(t, bkt.Upload(ctx, path.Join(shouldDeleteId.String(), "chunks", "000001"), &fakeChunk)) + + // Generate 1 block which is older than consistencyDelay but younger than MinimumAgeForRemoval, and which has chunk + // data but no meta. Compactor should ignore it. + shouldIgnoreId, err := ulid.New(uint64(time.Now().Unix()*1000), nil) + testutil.Ok(t, err) + + testutil.Ok(t, bkt.Upload(ctx, path.Join(shouldIgnoreId.String(), "chunks", "000001"), &fakeChunk)) + + testutil.Ok(t, sy.SyncMetas(ctx)) + + exists, err := bkt.Exists(ctx, path.Join(shouldDeleteId.String(), "chunks", "000001")) + testutil.Ok(t, err) + testutil.Equals(t, false, exists) + + exists, err = bkt.Exists(ctx, path.Join(shouldIgnoreId.String(), "chunks", "000001")) + testutil.Ok(t, err) + testutil.Equals(t, true, exists) +} diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 115b6c4c20a..cd58767b20a 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -37,7 +37,7 @@ const DirDelim = "/" type Config struct { Bucket string `yaml:"bucket"` Endpoint string `yaml:"endpoint"` - Region string `yaml:"region"` + Region string `yaml:"region"` AccessKey string `yaml:"access_key"` Insecure bool `yaml:"insecure"` SignatureV2 bool `yaml:"signature_version2"` @@ -54,8 +54,9 @@ type TraceConfig struct { // HTTPConfig stores the http.Transport configuration for the s3 minio client. type HTTPConfig struct { - IdleConnTimeout model.Duration `yaml:"idle_conn_timeout"` - InsecureSkipVerify bool `yaml:"insecure_skip_verify"` + IdleConnTimeout model.Duration `yaml:"idle_conn_timeout"` + ResponseHeaderTimeout model.Duration `yaml:"response_header_timeout"` + InsecureSkipVerify bool `yaml:"insecure_skip_verify"` } // Bucket implements the store.Bucket interface against s3-compatible APIs. @@ -69,7 +70,10 @@ type Bucket struct { // parseConfig unmarshals a buffer into a Config with default HTTPConfig values. func parseConfig(conf []byte) (Config, error) { - defaultHTTPConfig := HTTPConfig{IdleConnTimeout: model.Duration(90 * time.Second)} + defaultHTTPConfig := HTTPConfig{ + IdleConnTimeout: model.Duration(90 * time.Second), + ResponseHeaderTimeout: model.Duration(2 * time.Minute), + } config := Config{HTTPConfig: defaultHTTPConfig} if err := yaml.Unmarshal(conf, &config); err != nil { return Config{}, err @@ -100,6 +104,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B } if config.AccessKey != "" { signature := credentials.SignatureV4 + // TODO(bwplotka): Don't do flags, use actual v2, v4 params. if config.SignatureV2 { signature = credentials.SignatureV2 } @@ -139,10 +144,11 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B IdleConnTimeout: time.Duration(config.HTTPConfig.IdleConnTimeout), TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, - // The ResponseHeaderTimeout here is the only change from the - // default minio transport, it was introduced to cover cases - // where the tcp connection works but the server never answers - ResponseHeaderTimeout: 15 * time.Second, + // The ResponseHeaderTimeout here is the only change + // from the default minio transport, it was introduced + // to cover cases where the tcp connection works but + // the server never answers. Defaults to 2 minutes. + ResponseHeaderTimeout: time.Duration(config.HTTPConfig.ResponseHeaderTimeout), // Set this value so that the underlying transport round-tripper // doesn't try to auto decode the body of objects with // content-encoding set to `gzip`. diff --git a/pkg/objstore/s3/s3_test.go b/pkg/objstore/s3/s3_test.go index c8d8204a469..9f47f0c496f 100644 --- a/pkg/objstore/s3/s3_test.go +++ b/pkg/objstore/s3/s3_test.go @@ -7,12 +7,48 @@ import ( "github.com/improbable-eng/thanos/pkg/testutil" ) -func TestParseConfig_DefaultHTTPOpts(t *testing.T) { +func TestParseConfig(t *testing.T) { + input := []byte(`bucket: abcd +insecure: false`) + cfg, err := parseConfig(input) + testutil.Ok(t, err) + + if cfg.Bucket != "abcd" { + t.Errorf("parsing of bucket failed: got %v, expected %v", cfg.Bucket, "abcd") + } + if cfg.Insecure { + t.Errorf("parsing of insecure failed: got %v, expected %v", cfg.Insecure, false) + } +} + +func TestParseConfig_DefaultHTTPConfig(t *testing.T) { + input := []byte(`bucket: abcd +insecure: false`) + cfg, err := parseConfig(input) + testutil.Ok(t, err) + + if time.Duration(cfg.HTTPConfig.IdleConnTimeout) != time.Duration(90*time.Second) { + t.Errorf("parsing of idle_conn_timeout failed: got %v, expected %v", + time.Duration(cfg.HTTPConfig.IdleConnTimeout), time.Duration(90*time.Second)) + } + + if time.Duration(cfg.HTTPConfig.ResponseHeaderTimeout) != time.Duration(2*time.Minute) { + t.Errorf("parsing of response_header_timeout failed: got %v, expected %v", + time.Duration(cfg.HTTPConfig.IdleConnTimeout), time.Duration(2*time.Minute)) + } + + if cfg.HTTPConfig.InsecureSkipVerify { + t.Errorf("parsing of insecure_skip_verify failed: got %v, expected %v", cfg.HTTPConfig.InsecureSkipVerify, false) + } +} + +func TestParseConfig_CustomHTTPConfig(t *testing.T) { input := []byte(`bucket: abcd insecure: false http_config: insecure_skip_verify: true - idle_conn_timeout: 50s`) + idle_conn_timeout: 50s + response_header_timeout: 1m`) cfg, err := parseConfig(input) testutil.Ok(t, err) @@ -21,12 +57,11 @@ http_config: time.Duration(cfg.HTTPConfig.IdleConnTimeout), time.Duration(50*time.Second)) } - if cfg.Bucket != "abcd" { - t.Errorf("parsing of bucket failed: got %v, expected %v", cfg.Bucket, "abcd") - } - if cfg.Insecure { - t.Errorf("parsing of insecure failed: got %v, expected %v", cfg.Insecure, false) + if time.Duration(cfg.HTTPConfig.ResponseHeaderTimeout) != time.Duration(1*time.Minute) { + t.Errorf("parsing of response_header_timeout failed: got %v, expected %v", + time.Duration(cfg.HTTPConfig.IdleConnTimeout), time.Duration(1*time.Minute)) } + if !cfg.HTTPConfig.InsecureSkipVerify { t.Errorf("parsing of insecure_skip_verify failed: got %v, expected %v", cfg.HTTPConfig.InsecureSkipVerify, false) } diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 24b80ec5fab..3823d84225d 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -26,7 +26,6 @@ import ( "github.com/improbable-eng/thanos/pkg/objstore" "github.com/improbable-eng/thanos/pkg/pool" "github.com/improbable-eng/thanos/pkg/runutil" - storecache "github.com/improbable-eng/thanos/pkg/store/cache" "github.com/improbable-eng/thanos/pkg/store/storepb" "github.com/improbable-eng/thanos/pkg/strutil" "github.com/improbable-eng/thanos/pkg/tracing" @@ -176,6 +175,13 @@ func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics { return &m } +type indexCache interface { + SetPostings(b ulid.ULID, l labels.Label, v []byte) + Postings(b ulid.ULID, l labels.Label) ([]byte, bool) + SetSeries(b ulid.ULID, id uint64, v []byte) + Series(b ulid.ULID, id uint64) ([]byte, bool) +} + // BucketStore implements the store API backed by a bucket. It loads all index // files to local disk. type BucketStore struct { @@ -183,7 +189,7 @@ type BucketStore struct { metrics *bucketStoreMetrics bucket objstore.BucketReader dir string - indexCache *storecache.IndexCache + indexCache indexCache chunkPool *pool.BytesPool // Sets of blocks that have the same labels. They are indexed by a hash over their label set. @@ -211,7 +217,7 @@ func NewBucketStore( reg prometheus.Registerer, bucket objstore.BucketReader, dir string, - indexCacheSizeBytes uint64, + indexCache indexCache, maxChunkPoolBytes uint64, maxSampleCount uint64, maxConcurrent int, @@ -226,17 +232,6 @@ func NewBucketStore( return nil, errors.Errorf("max concurrency value cannot be lower than 0 (got %v)", maxConcurrent) } - // TODO(bwplotka): Add as a flag? - maxItemSizeBytes := indexCacheSizeBytes / 2 - - indexCache, err := storecache.NewIndexCache(logger, reg, storecache.Opts{ - MaxSizeBytes: indexCacheSizeBytes, - MaxItemSizeBytes: maxItemSizeBytes, - }) - if err != nil { - return nil, errors.Wrap(err, "create index cache") - } - chunkPool, err := pool.NewBytesPool(2e5, 50e6, 2, maxChunkPoolBytes) if err != nil { return nil, errors.Wrap(err, "create chunk pool") @@ -1066,7 +1061,7 @@ type bucketBlock struct { bucket objstore.BucketReader meta *metadata.Meta dir string - indexCache *storecache.IndexCache + indexCache indexCache chunkPool *pool.BytesPool indexVersion int @@ -1089,7 +1084,7 @@ func newBucketBlock( bkt objstore.BucketReader, id ulid.ULID, dir string, - indexCache *storecache.IndexCache, + indexCache indexCache, chunkPool *pool.BytesPool, p partitioner, ) (b *bucketBlock, err error) { @@ -1105,7 +1100,7 @@ func newBucketBlock( if err = b.loadMeta(ctx, id); err != nil { return nil, errors.Wrap(err, "load meta") } - if err = b.loadIndexCache(ctx); err != nil { + if err = b.loadIndexCacheFile(ctx); err != nil { return nil, errors.Wrap(err, "load index cache") } // Get object handles for all chunk files. @@ -1149,9 +1144,9 @@ func (b *bucketBlock) loadMeta(ctx context.Context, id ulid.ULID) error { return nil } -func (b *bucketBlock) loadIndexCache(ctx context.Context) (err error) { +func (b *bucketBlock) loadIndexCacheFile(ctx context.Context) (err error) { cachefn := filepath.Join(b.dir, block.IndexCacheFilename) - if err = b.loadIndexCacheFromFile(ctx, cachefn); err == nil { + if err = b.loadIndexCacheFileFromFile(ctx, cachefn); err == nil { return nil } if !os.IsNotExist(errors.Cause(err)) { @@ -1160,7 +1155,7 @@ func (b *bucketBlock) loadIndexCache(ctx context.Context) (err error) { // Try to download index cache file from object store. if err = objstore.DownloadFile(ctx, b.logger, b.bucket, b.indexCacheFilename(), cachefn); err == nil { - return b.loadIndexCacheFromFile(ctx, cachefn) + return b.loadIndexCacheFileFromFile(ctx, cachefn) } if !b.bucket.IsObjNotFoundErr(errors.Cause(err)) { @@ -1184,10 +1179,10 @@ func (b *bucketBlock) loadIndexCache(ctx context.Context) (err error) { return errors.Wrap(err, "write index cache") } - return errors.Wrap(b.loadIndexCacheFromFile(ctx, cachefn), "read index cache") + return errors.Wrap(b.loadIndexCacheFileFromFile(ctx, cachefn), "read index cache") } -func (b *bucketBlock) loadIndexCacheFromFile(ctx context.Context, cache string) (err error) { +func (b *bucketBlock) loadIndexCacheFileFromFile(ctx context.Context, cache string) (err error) { b.indexVersion, b.symbols, b.lvals, b.postings, err = block.ReadIndexCache(b.logger, cache) return err } @@ -1249,13 +1244,13 @@ type bucketIndexReader struct { block *bucketBlock dec *index.Decoder stats *queryStats - cache *storecache.IndexCache + cache indexCache mtx sync.Mutex loadedSeries map[uint64][]byte } -func newBucketIndexReader(ctx context.Context, logger log.Logger, block *bucketBlock, cache *storecache.IndexCache) *bucketIndexReader { +func newBucketIndexReader(ctx context.Context, logger log.Logger, block *bucketBlock, cache indexCache) *bucketIndexReader { r := &bucketIndexReader{ logger: logger, ctx: ctx, diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go index 02ad79781a5..c0fc42b97c7 100644 --- a/pkg/store/bucket_e2e_test.go +++ b/pkg/store/bucket_e2e_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "github.com/oklog/ulid" + "github.com/go-kit/kit/log" "github.com/improbable-eng/thanos/pkg/block" "github.com/improbable-eng/thanos/pkg/block/metadata" @@ -23,12 +25,46 @@ import ( "github.com/prometheus/tsdb/labels" ) +type noopCache struct{} + +func (noopCache) SetPostings(b ulid.ULID, l labels.Label, v []byte) {} +func (noopCache) Postings(b ulid.ULID, l labels.Label) ([]byte, bool) { return nil, false } +func (noopCache) SetSeries(b ulid.ULID, id uint64, v []byte) {} +func (noopCache) Series(b ulid.ULID, id uint64) ([]byte, bool) { return nil, false } + +type swappableCache struct { + ptr indexCache +} + +func (c *swappableCache) SwapWith(ptr2 indexCache) { + c.ptr = ptr2 +} + +func (c *swappableCache) SetPostings(b ulid.ULID, l labels.Label, v []byte) { + c.ptr.SetPostings(b, l, v) +} + +func (c *swappableCache) Postings(b ulid.ULID, l labels.Label) ([]byte, bool) { + return c.ptr.Postings(b, l) +} + +func (c *swappableCache) SetSeries(b ulid.ULID, id uint64, v []byte) { + c.ptr.SetSeries(b, id, v) +} + +func (c *swappableCache) Series(b ulid.ULID, id uint64) ([]byte, bool) { + return c.ptr.Series(b, id) +} + type storeSuite struct { cancel context.CancelFunc wg sync.WaitGroup store *BucketStore minTime, maxTime int64 + cache *swappableCache + + logger log.Logger } func (s *storeSuite) Close() { @@ -53,7 +89,11 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m now := start ctx, cancel := context.WithCancel(context.Background()) - s := &storeSuite{cancel: cancel} + s := &storeSuite{ + cancel: cancel, + logger: log.NewLogfmtLogger(os.Stderr), + cache: &swappableCache{}, + } blocks := 0 for i := 0; i < 3; i++ { mint := timestamp.FromTime(now) @@ -78,17 +118,17 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m meta, err := metadata.Read(dir2) testutil.Ok(t, err) meta.Thanos.Labels = map[string]string{"ext2": "value2"} - testutil.Ok(t, metadata.Write(log.NewNopLogger(), dir2, meta)) + testutil.Ok(t, metadata.Write(s.logger, dir2, meta)) - testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, dir1)) - testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, dir2)) + testutil.Ok(t, block.Upload(ctx, s.logger, bkt, dir1)) + testutil.Ok(t, block.Upload(ctx, s.logger, bkt, dir2)) blocks += 2 testutil.Ok(t, os.RemoveAll(dir1)) testutil.Ok(t, os.RemoveAll(dir2)) } - store, err := NewBucketStore(log.NewLogfmtLogger(os.Stderr), nil, bkt, dir, 100, 0, maxSampleCount, 20, false, 20) + store, err := NewBucketStore(s.logger, nil, bkt, dir, s.cache, 0, maxSampleCount, 20, false, 20) testutil.Ok(t, err) s.store = store @@ -310,13 +350,6 @@ func testBucketStore_e2e(t testing.TB, ctx context.Context, s *storeSuite) { } { t.Log("Run ", i) - // Always clean cache before each test. - s.store.indexCache, err = storecache.NewIndexCache(log.NewNopLogger(), nil, storecache.Opts{ - MaxSizeBytes: 100, - MaxItemSizeBytes: 100, - }) - testutil.Ok(t, err) - srv := newStoreSeriesServer(ctx) testutil.Ok(t, s.store.Series(tcase.req, srv)) @@ -341,6 +374,26 @@ func TestBucketStore_e2e(t *testing.T) { s := prepareStoreWithTestBlocks(t, dir, bkt, false, 0) defer s.Close() + t.Log("Test with no index cache") + s.cache.SwapWith(noopCache{}) + testBucketStore_e2e(t, ctx, s) + + t.Log("Test with large, sufficient index cache") + indexCache, err := storecache.NewIndexCache(s.logger, nil, storecache.Opts{ + MaxItemSizeBytes: 1e5, + MaxSizeBytes: 2e5, + }) + testutil.Ok(t, err) + s.cache.SwapWith(indexCache) + testBucketStore_e2e(t, ctx, s) + + t.Log("Test with small index cache") + indexCache2, err := storecache.NewIndexCache(s.logger, nil, storecache.Opts{ + MaxItemSizeBytes: 50, + MaxSizeBytes: 100, + }) + testutil.Ok(t, err) + s.cache.SwapWith(indexCache2) testBucketStore_e2e(t, ctx, s) }) } @@ -370,6 +423,13 @@ func TestBucketStore_ManyParts_e2e(t *testing.T) { s := prepareStoreWithTestBlocks(t, dir, bkt, true, 0) defer s.Close() + indexCache, err := storecache.NewIndexCache(s.logger, nil, storecache.Opts{ + MaxItemSizeBytes: 1e5, + MaxSizeBytes: 2e5, + }) + testutil.Ok(t, err) + s.cache.SwapWith(indexCache) + testBucketStore_e2e(t, ctx, s) }) } diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index 18f953c298e..4e9e5810144 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -283,7 +283,7 @@ func TestBucketStore_Info(t *testing.T) { dir, err := ioutil.TempDir("", "prometheus-test") testutil.Ok(t, err) - bucketStore, err := NewBucketStore(nil, nil, nil, dir, 2e5, 2e5, 0, 0, false, 20) + bucketStore, err := NewBucketStore(nil, nil, nil, dir, noopCache{}, 2e5, 0, 0, false, 20) testutil.Ok(t, err) resp, err := bucketStore.Info(ctx, &storepb.InfoRequest{}) diff --git a/website/data/sponsors.yml b/website/data/sponsors.yml new file mode 100644 index 00000000000..ba860b70889 --- /dev/null +++ b/website/data/sponsors.yml @@ -0,0 +1,20 @@ +--- +sponsors: +- name: Monzo + url: https://www.monzo.com + logo: monzo.png +- name: Utility Warehouse + url: https://www.utilitywarehouse.co.uk + logo: utilitywarehouse.png +- name: Adform + url: https://site.adform.com + logo: Adform_logo_RGB.png +- name: Seznam.cz + url: https://www.seznam.cz/ + logo: seznam.png +- name: tiket.com + url: https://www.tiket.com + logo: tiket.png +- name: uSwitch + url: https://www.uswitch.com + logo: uswitch.png diff --git a/website/hugo.yaml b/website/hugo.yaml index 472134ab6f1..890d9f63fe8 100644 --- a/website/hugo.yaml +++ b/website/hugo.yaml @@ -31,4 +31,5 @@ params: SlackInvite: "https://join.slack.com/t/improbable-eng/shared_invite/enQtMzQ1ODcyMzQ5MjM4LWY5ZWZmNGM2ODc5MmViNmQ3ZTA3ZTY3NzQwOTBlMTkzZmIxZTIxODk0OWU3YjZhNWVlNDU3MDlkZGViZjhkMjc" GithubUser: "improbable-eng" GithubProject: "thanos" - TwitterHandle: "ThanosMetrics" \ No newline at end of file + TwitterHandle: "ThanosMetrics" + Description: "Highly available Prometheus setup with long term storage capabilities." \ No newline at end of file diff --git a/website/layouts/_default/baseof.html b/website/layouts/_default/baseof.html index 9fd91250abf..83b2d2bcb18 100644 --- a/website/layouts/_default/baseof.html +++ b/website/layouts/_default/baseof.html @@ -13,10 +13,16 @@ {{ block "title" . }}{{ .Site.Title }}{{ end }} - + + + + + + + diff --git a/website/layouts/_default/single.html b/website/layouts/_default/single.html index ec7b15a64e7..d68c45c348f 100644 --- a/website/layouts/_default/single.html +++ b/website/layouts/_default/single.html @@ -2,10 +2,12 @@
- {{ template "_default/sidemenu.html" . }} + {{ partial "_default/sidemenu.html" . }}
- {{ replace .Content "" "
" | safeHTML }} + {{ $content := replace .Content "
" "
" }} + {{ $content = $content | replaceRE "()" `${1} # ${3}` }} + {{ $content | safeHTML}} diff --git a/website/layouts/index.html b/website/layouts/index.html index 3125dca1bf7..ba73f068a32 100644 --- a/website/layouts/index.html +++ b/website/layouts/index.html @@ -52,7 +52,34 @@

Downsampling & Compaction

-
+
+

Founded By

+
+
+
+ Improbable +
+
+
+

Used By

+
+ {{ range $sponsor := sort $.Site.Data.sponsors.sponsors "name" }} + {{ if $sponsor.logo }} +
+
+ {{ $sponsor.name }} +
+
+ {{ end }} + {{ end }} +
+
+ Your Company +
+
+
+
+

Join the community !

diff --git a/website/layouts/_default/sidemenu.html b/website/layouts/partials/_default/sidemenu.html similarity index 100% rename from website/layouts/_default/sidemenu.html rename to website/layouts/partials/_default/sidemenu.html diff --git a/website/layouts/proposal/single.html b/website/layouts/proposal/single.html index 507809d3b5f..1a86321c6db 100644 --- a/website/layouts/proposal/single.html +++ b/website/layouts/proposal/single.html @@ -2,7 +2,7 @@
- {{ template "_default/sidemenu.html" . }} + {{ partial "_default/sidemenu.html" . }}

{{ .Title }}

diff --git a/website/static/logos/Adform_logo_RGB.png b/website/static/logos/Adform_logo_RGB.png new file mode 100644 index 00000000000..78ceef9244d Binary files /dev/null and b/website/static/logos/Adform_logo_RGB.png differ diff --git a/website/static/logos/improbable.png b/website/static/logos/improbable.png new file mode 100644 index 00000000000..27db9c44162 Binary files /dev/null and b/website/static/logos/improbable.png differ diff --git a/website/static/logos/monzo.png b/website/static/logos/monzo.png new file mode 100644 index 00000000000..93d3264f712 Binary files /dev/null and b/website/static/logos/monzo.png differ diff --git a/website/static/logos/seznam.png b/website/static/logos/seznam.png new file mode 100644 index 00000000000..089621c5b1a Binary files /dev/null and b/website/static/logos/seznam.png differ diff --git a/website/static/logos/tiket.png b/website/static/logos/tiket.png new file mode 100644 index 00000000000..2a8fbc3cc32 Binary files /dev/null and b/website/static/logos/tiket.png differ diff --git a/website/static/logos/uswitch.png b/website/static/logos/uswitch.png new file mode 100644 index 00000000000..c3dedbedd10 Binary files /dev/null and b/website/static/logos/uswitch.png differ diff --git a/website/static/logos/utilitywarehouse.png b/website/static/logos/utilitywarehouse.png new file mode 100644 index 00000000000..95df50191ce Binary files /dev/null and b/website/static/logos/utilitywarehouse.png differ diff --git a/website/static/main.css b/website/static/main.css index 5311ca759ed..f8a9530a2e9 100644 --- a/website/static/main.css +++ b/website/static/main.css @@ -49,4 +49,25 @@ pre { .list-group-item-thanos:hover { color: #6D41FF; -} \ No newline at end of file +} + +.img-sponsor { + padding: 25px; + background-color: #fff; + box-shadow: 0 2px 4px rgba(0,0,0,.1); + border: 1px solid #dee2e6; + margin: 15px 0 15px; + border-radius: .25rem; + display: flex; + align-items: center; + justify-content: center; + height: 100px; +} + +.img-sponsor img { + max-width: 100%; + max-height: 100%; +} + +.header-anchor { font-size: 100%; visibility: hidden;} +h1:hover a, h2:hover a, h3:hover a, h4:hover a { visibility: visible} \ No newline at end of file diff --git a/website/static/thanos-twitter.png b/website/static/thanos-twitter.png new file mode 100644 index 00000000000..d2d5aed3622 Binary files /dev/null and b/website/static/thanos-twitter.png differ