Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
9c94918
store/bucket_test: add interleaved resolutions test for getFor()
May 15, 2019
cd3363c
store/bucket: include blocks in the middle as well
May 15, 2019
00b2c7c
store/bucket: add test cases with duplicated time ranges
May 16, 2019
8f22504
query/querier: send proper maxSourceResolution
May 16, 2019
e2c46bf
README: add entry
May 16, 2019
342aede
query/querier_test: add queryableCreator test
May 16, 2019
4e73b2f
store/bucket: do the iteration without sorting
May 16, 2019
fae1929
store/bucket: bsi->j in loop
May 16, 2019
bf12553
Merge remote-tracking branch 'origin/master' into getFor
May 17, 2019
de4b79a
store/bucket: fix according to review comments
May 17, 2019
c3abb09
query/querier_test: fix test
May 17, 2019
f22cffa
*: clarify everywhere that max source resolution is in millis
May 17, 2019
5e9d0e9
*: maxSourceResolutionMillis -> maxResolutionMillis
May 17, 2019
420ebe0
CHANGELOG: update
May 17, 2019
e8b3189
query/querier_test: fix
May 17, 2019
bacfd06
store/bucket: add gets all data in range property test
May 17, 2019
93a764c
store/bucket_test: add production property test
May 17, 2019
f4b0a66
store/bucket_test: fix
May 17, 2019
dac133e
store/bucket_test: add always gets property
May 17, 2019
dbe3727
query/querier_test: do not shrink
May 17, 2019
6dac954
store/bucket: revert change
May 17, 2019
55aa32d
store/bucket_test: remove more confusion
May 18, 2019
3fe4217
store/bucket: clean up tests
May 22, 2019
3bbc6cb
Simplified goFor implementation.
bwplotka May 27, 2019
3602678
Merge pull request #1 from improbable-eng/getFor
May 27, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
*: maxSourceResolutionMillis -> maxResolutionMillis
  • Loading branch information
Giedrius Statkevičius committed May 17, 2019
commit 5e9d0e9bf02ca5ab711b9be13c05b9d1ce490c64
2 changes: 1 addition & 1 deletion pkg/query/api/v1.go
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ func (api *API) parseEnableDedupParam(r *http.Request) (enableDeduplication bool
return enableDeduplication, nil
}

func (api *API) parseDownsamplingParamMillis(r *http.Request, step time.Duration) (maxSourceResolutionMillis int64, _ *ApiError) {
func (api *API) parseDownsamplingParamMillis(r *http.Request, step time.Duration) (maxResolutionMillis int64, _ *ApiError) {
const maxSourceResolutionParam = "max_source_resolution"
maxSourceResolution := 0 * time.Second

Expand Down
78 changes: 39 additions & 39 deletions pkg/query/api/v1_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -837,67 +837,67 @@ func BenchmarkQueryResultEncoding(b *testing.B) {

func TestParseDownsamplingParamMillis(t *testing.T) {
var tests = []struct {
maxSourceResolution string
result int64
step time.Duration
fail bool
enableAutodownsampling bool
maxSourceResolutionParam string
result int64
step time.Duration
fail bool
enableAutodownsampling bool
}{
{
maxSourceResolution: "0s",
enableAutodownsampling: false,
step: time.Hour,
result: int64(compact.ResolutionLevelRaw),
fail: false,
maxSourceResolutionParam: "0s",
enableAutodownsampling: false,
step: time.Hour,
result: int64(compact.ResolutionLevelRaw),
fail: false,
},
{
maxSourceResolution: "5m",
step: time.Hour,
enableAutodownsampling: false,
result: int64(compact.ResolutionLevel5m),
fail: false,
maxSourceResolutionParam: "5m",
step: time.Hour,
enableAutodownsampling: false,
result: int64(compact.ResolutionLevel5m),
fail: false,
},
{
maxSourceResolution: "1h",
step: time.Hour,
enableAutodownsampling: false,
result: int64(compact.ResolutionLevel1h),
fail: false,
maxSourceResolutionParam: "1h",
step: time.Hour,
enableAutodownsampling: false,
result: int64(compact.ResolutionLevel1h),
fail: false,
},
{
maxSourceResolution: "",
enableAutodownsampling: true,
step: time.Hour,
result: int64(time.Hour / (5 * 1000 * 1000)),
fail: false,
maxSourceResolutionParam: "",
enableAutodownsampling: true,
step: time.Hour,
result: int64(time.Hour / (5 * 1000 * 1000)),
fail: false,
},
{
maxSourceResolution: "",
enableAutodownsampling: true,
step: time.Hour,
result: int64((1 * time.Hour) / 6),
fail: true,
maxSourceResolutionParam: "",
enableAutodownsampling: true,
step: time.Hour,
result: int64((1 * time.Hour) / 6),
fail: true,
},
{
maxSourceResolution: "",
enableAutodownsampling: true,
step: time.Hour,
result: int64((1 * time.Hour) / 6),
fail: true,
maxSourceResolutionParam: "",
enableAutodownsampling: true,
step: time.Hour,
result: int64((1 * time.Hour) / 6),
fail: true,
},
}

for i, test := range tests {
api := API{enableAutodownsampling: test.enableAutodownsampling}
v := url.Values{}
v.Set("max_source_resolution", test.maxSourceResolution)
v.Set("max_source_resolution", test.maxSourceResolutionParam)
r := http.Request{PostForm: v}

maxSourceRes, _ := api.parseDownsamplingParamMillis(&r, test.step)
maxResMillis, _ := api.parseDownsamplingParamMillis(&r, test.step)
if test.fail == false {
testutil.Assert(t, maxSourceRes == test.result, "case %v: expected %v to be equal to %v", i, maxSourceRes, test.result)
testutil.Assert(t, maxResMillis == test.result, "case %v: expected %v to be equal to %v", i, maxResMillis, test.result)
} else {
testutil.Assert(t, maxSourceRes != test.result, "case %v: expected %v not to be equal to %v", i, maxSourceRes, test.result)
testutil.Assert(t, maxResMillis != test.result, "case %v: expected %v not to be equal to %v", i, maxResMillis, test.result)
}

}
Expand Down
82 changes: 41 additions & 41 deletions pkg/query/querier.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,51 +22,51 @@ type WarningReporter func(error)

// QueryableCreator returns implementation of promql.Queryable that fetches data from the proxy store API endpoints.
// If deduplication is enabled, all data retrieved from it will be deduplicated along the replicaLabel by default.
// maxSourceResolutionMillis controls downsampling resolution that is allowed (specified in milliseconds).
// maxResolutionMillis controls downsampling resolution that is allowed (specified in milliseconds).
// partialResponse controls `partialResponseDisabled` option of StoreAPI and partial response behaviour of proxy.
type QueryableCreator func(deduplicate bool, maxSourceResolutionMillis int64, partialResponse bool, r WarningReporter) storage.Queryable
type QueryableCreator func(deduplicate bool, maxResolutionMillis int64, partialResponse bool, r WarningReporter) storage.Queryable

// NewQueryableCreator creates QueryableCreator.
func NewQueryableCreator(logger log.Logger, proxy storepb.StoreServer, replicaLabel string) QueryableCreator {
return func(deduplicate bool, maxSourceResolutionMillis int64, partialResponse bool, r WarningReporter) storage.Queryable {
return func(deduplicate bool, maxResolutionMillis int64, partialResponse bool, r WarningReporter) storage.Queryable {
return &queryable{
logger: logger,
replicaLabel: replicaLabel,
proxy: proxy,
deduplicate: deduplicate,
maxSourceResolutionMillis: maxSourceResolutionMillis,
partialResponse: partialResponse,
warningReporter: r,
logger: logger,
replicaLabel: replicaLabel,
proxy: proxy,
deduplicate: deduplicate,
maxResolutionMillis: maxResolutionMillis,
partialResponse: partialResponse,
warningReporter: r,
}
}
}

type queryable struct {
logger log.Logger
replicaLabel string
proxy storepb.StoreServer
deduplicate bool
maxSourceResolutionMillis int64
partialResponse bool
warningReporter WarningReporter
logger log.Logger
replicaLabel string
proxy storepb.StoreServer
deduplicate bool
maxResolutionMillis int64
partialResponse bool
warningReporter WarningReporter
}

// Querier returns a new storage querier against the underlying proxy store API.
func (q *queryable) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) {
return newQuerier(ctx, q.logger, mint, maxt, q.replicaLabel, q.proxy, q.deduplicate, int64(q.maxSourceResolutionMillis), q.partialResponse, q.warningReporter), nil
return newQuerier(ctx, q.logger, mint, maxt, q.replicaLabel, q.proxy, q.deduplicate, int64(q.maxResolutionMillis), q.partialResponse, q.warningReporter), nil
}

type querier struct {
ctx context.Context
logger log.Logger
cancel func()
mint, maxt int64
replicaLabel string
proxy storepb.StoreServer
deduplicate bool
maxSourceResolutionMillis int64
partialResponse bool
warningReporter WarningReporter
ctx context.Context
logger log.Logger
cancel func()
mint, maxt int64
replicaLabel string
proxy storepb.StoreServer
deduplicate bool
maxResolutionMillis int64
partialResponse bool
warningReporter WarningReporter
}

// newQuerier creates implementation of storage.Querier that fetches data from the proxy
Expand All @@ -78,7 +78,7 @@ func newQuerier(
replicaLabel string,
proxy storepb.StoreServer,
deduplicate bool,
maxSourceResolutionMillis int64,
maxResolutionMillis int64,
partialResponse bool,
warningReporter WarningReporter,
) *querier {
Expand All @@ -90,17 +90,17 @@ func newQuerier(
}
ctx, cancel := context.WithCancel(ctx)
return &querier{
ctx: ctx,
logger: logger,
cancel: cancel,
mint: mint,
maxt: maxt,
replicaLabel: replicaLabel,
proxy: proxy,
deduplicate: deduplicate,
maxSourceResolutionMillis: maxSourceResolutionMillis,
partialResponse: partialResponse,
warningReporter: warningReporter,
ctx: ctx,
logger: logger,
cancel: cancel,
mint: mint,
maxt: maxt,
replicaLabel: replicaLabel,
proxy: proxy,
deduplicate: deduplicate,
maxResolutionMillis: maxResolutionMillis,
partialResponse: partialResponse,
warningReporter: warningReporter,
}
}

Expand Down Expand Up @@ -183,7 +183,7 @@ func (q *querier) Select(params *storage.SelectParams, ms ...*labels.Matcher) (s
MinTime: q.mint,
MaxTime: q.maxt,
Matchers: sms,
MaxResolutionWindow: q.maxSourceResolutionMillis,
MaxResolutionWindow: q.maxResolutionMillis,
Aggregates: queryAggrs,
PartialResponseDisabled: !q.partialResponse,
}, resp); err != nil {
Expand Down
8 changes: 4 additions & 4 deletions pkg/store/bucket.go
Original file line number Diff line number Diff line change
Expand Up @@ -675,7 +675,7 @@ func populateChunk(out *storepb.AggrChunk, in chunkenc.Chunk, aggrs []storepb.Ag
// labels and resolution. This is important because we allow mixed resolution results, so it is quite crucial
// to be aware what exactly resolution we see on query.
// TODO(bplotka): Consider adding resolution label to all results to propagate that info to UI and Query API.
func debugFoundBlockSetOverview(logger log.Logger, mint, maxt, maxSourceResolutionMillis int64, lset labels.Labels, bs []*bucketBlock) {
func debugFoundBlockSetOverview(logger log.Logger, mint, maxt, maxResolutionMillis int64, lset labels.Labels, bs []*bucketBlock) {
if len(bs) == 0 {
level.Debug(logger).Log("msg", "No block found", "mint", mint, "maxt", maxt, "lset", lset.String())
return
Expand Down Expand Up @@ -703,7 +703,7 @@ func debugFoundBlockSetOverview(logger log.Logger, mint, maxt, maxSourceResoluti

parts = append(parts, fmt.Sprintf("Range: %d-%d Resolution: %d", currMin, currMax, currRes))

level.Debug(logger).Log("msg", "Blocks source resolutions", "blocks", len(bs), "Maximum Resolution", maxSourceResolutionMillis, "mint", mint, "maxt", maxt, "lset", lset.String(), "spans", strings.Join(parts, "\n"))
level.Debug(logger).Log("msg", "Blocks source resolutions", "blocks", len(bs), "Maximum Resolution", maxResolutionMillis, "mint", mint, "maxt", maxt, "lset", lset.String(), "spans", strings.Join(parts, "\n"))
}

// Series implements the storepb.StoreServer interface.
Expand Down Expand Up @@ -997,7 +997,7 @@ func int64index(s []int64, x int64) int {

// getFor returns a time-ordered list of blocks that cover date between mint and maxt.
// Blocks with the lowest resolution possible but not lower than the given resolution are returned.
func (s *bucketBlockSet) getFor(mint, maxt, maxSourceResolutionMillis int64) (bs []*bucketBlock) {
func (s *bucketBlockSet) getFor(mint, maxt, maxResolutionMillis int64) (bs []*bucketBlock) {
if mint == maxt {
return nil
}
Expand All @@ -1007,7 +1007,7 @@ func (s *bucketBlockSet) getFor(mint, maxt, maxSourceResolutionMillis int64) (bs

// Find first matching resolution.
i := 0
for ; i < len(s.resolutions) && s.resolutions[i] > maxSourceResolutionMillis; i++ {
for ; i < len(s.resolutions) && s.resolutions[i] > maxResolutionMillis; i++ {
}

// Base case, we fill the given interval with the closest resolution.
Expand Down