diff --git a/.gitignore b/.gitignore index 87538edf291..051b2faf5af 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ website/docs-pre-processed/ # React build assets pkg/ui/static/react +/tmp tmp/bin examples/tmp/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e39620c956..4e2270be572 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6222](https://github.com/thanos-io/thanos/pull/6222) mixin(Receive): Fix tenant series received charts. - [#6218](https://github.com/thanos-io/thanos/pull/6218) mixin(Store): handle ResourceExhausted as a non-server error. As a consequence, this error won't contribute to Store's grpc errors alerts. - [#6271](https://github.com/thanos-io/thanos/pull/6271) Receive: Fix segfault in `LabelValues` during head compaction. +- [#6229](https://github.com/thanos-io/thanos/pull/6229) Compact: Fixed issue with no-compact-mark.json being ignored ### Changed - [#6168](https://github.com/thanos-io/thanos/pull/6168) Receiver: Make ketama hashring fail early when configured with number of nodes lower than the replication factor. diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 1368b56989a..baf640ea059 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -351,6 +351,7 @@ func runCompact( metadata.HashFunc(conf.hashFunc), conf.blockFilesConcurrency, conf.compactBlocksFetchConcurrency, + noCompactMarkerFilter, ) tsdbPlanner := compact.NewPlanner(logger, levels, noCompactMarkerFilter) planner := compact.WithLargeTotalIndexSizeFilter( diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index c6c36bf3cc1..b630e511ce9 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -232,6 +232,7 @@ type DefaultGrouper struct { hashFunc metadata.HashFunc blockFilesConcurrency int compactBlocksFetchConcurrency int + noCompactionMarkFilter *GatherNoCompactionMarkFilter } // NewDefaultGrouper makes a new DefaultGrouper. @@ -247,6 +248,7 @@ func NewDefaultGrouper( hashFunc metadata.HashFunc, blockFilesConcurrency int, compactBlocksFetchConcurrency int, + noCompactionMarkFilter *GatherNoCompactionMarkFilter, ) *DefaultGrouper { return &DefaultGrouper{ bkt: bkt, @@ -279,6 +281,7 @@ func NewDefaultGrouper( hashFunc: hashFunc, blockFilesConcurrency: blockFilesConcurrency, compactBlocksFetchConcurrency: compactBlocksFetchConcurrency, + noCompactionMarkFilter: noCompactionMarkFilter, } } @@ -286,7 +289,15 @@ func NewDefaultGrouper( // It creates all groups from the scratch on every call. func (g *DefaultGrouper) Groups(blocks map[ulid.ULID]*metadata.Meta) (res []*Group, err error) { groups := map[string]*Group{} + noCompactMarkedMap := g.noCompactionMarkFilter.NoCompactMarkedBlocks() for _, m := range blocks { + if _, ok := noCompactMarkedMap[m.ULID]; ok { + level.Debug(g.logger).Log( + "msg", "groupper: skipping block marked for no compaction", + "ulid", m.ULID.String()) + + continue + } groupKey := m.Thanos.GroupKey() group, ok := groups[groupKey] if !ok { diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index 9b40da16dcb..a88e86d2ed8 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -15,6 +15,7 @@ import ( "testing" "time" + "github.com/efficientgo/core/testutil" "github.com/go-kit/log" "github.com/oklog/ulid" "github.com/prometheus/client_golang/prometheus" @@ -25,8 +26,6 @@ import ( "github.com/prometheus/prometheus/tsdb" "github.com/thanos-io/objstore" "github.com/thanos-io/objstore/objtesting" - - "github.com/efficientgo/core/testutil" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/dedup" @@ -103,6 +102,7 @@ func TestSyncer_GarbageCollect_e2e(t *testing.T) { garbageCollectedBlocks := promauto.With(nil).NewCounter(prometheus.CounterOpts{}) blockMarkedForNoCompact := promauto.With(nil).NewCounter(prometheus.CounterOpts{}) ignoreDeletionMarkFilter := block.NewIgnoreDeletionMarkFilter(nil, nil, 48*time.Hour, fetcherConcurrency) + noCompactMarkerFilter := NewGatherNoCompactionMarkFilter(nil, objstore.WithNoopInstr(bkt), fetcherConcurrency) sy, err := NewMetaSyncer(nil, nil, bkt, metaFetcher, duplicateBlocksFilter, ignoreDeletionMarkFilter, blocksMarkedForDeletion, garbageCollectedBlocks) testutil.Ok(t, err) @@ -138,7 +138,7 @@ func TestSyncer_GarbageCollect_e2e(t *testing.T) { testutil.Ok(t, sy.GarbageCollect(ctx)) // Only the level 3 block, the last source block in both resolutions should be left. - grouper := NewDefaultGrouper(nil, bkt, false, false, nil, blocksMarkedForDeletion, garbageCollectedBlocks, blockMarkedForNoCompact, metadata.NoneFunc, 10, 10) + grouper := NewDefaultGrouper(nil, bkt, false, false, nil, blocksMarkedForDeletion, garbageCollectedBlocks, blockMarkedForNoCompact, metadata.NoneFunc, 10, 10, noCompactMarkerFilter) groups, err := grouper.Groups(sy.Metas()) testutil.Ok(t, err) @@ -211,7 +211,7 @@ func testGroupCompactE2e(t *testing.T, mergeFunc storage.VerticalChunkSeriesMerg testutil.Ok(t, err) planner := NewPlanner(logger, []int64{1000, 3000}, noCompactMarkerFilter) - grouper := NewDefaultGrouper(logger, bkt, false, false, reg, blocksMarkedForDeletion, garbageCollectedBlocks, blocksMaredForNoCompact, metadata.NoneFunc, 10, 10) + grouper := NewDefaultGrouper(logger, bkt, false, false, reg, blocksMarkedForDeletion, garbageCollectedBlocks, blocksMaredForNoCompact, metadata.NoneFunc, 10, 10, noCompactMarkerFilter) bComp, err := NewBucketCompactor(logger, sy, grouper, planner, comp, dir, bkt, 2, true) testutil.Ok(t, err) @@ -279,6 +279,12 @@ func testGroupCompactE2e(t *testing.T, mergeFunc storage.VerticalChunkSeriesMerg {{Name: "a", Value: "7"}}, }, }, + { + numSamples: 100, mint: 6000, maxt: 7000, extLset: extLabels, res: 124, + series: []labels.Labels{ + {{Name: "a", Value: "8"}}, + }, + }, // Second group (extLabels2). { numSamples: 100, mint: 2000, maxt: 3000, extLset: extLabels2, res: 124, @@ -317,7 +323,7 @@ func testGroupCompactE2e(t *testing.T, mergeFunc storage.VerticalChunkSeriesMerg }) groupKey1 := metas[0].Thanos.GroupKey() - groupKey2 := metas[6].Thanos.GroupKey() + groupKey2 := metas[7].Thanos.GroupKey() testutil.Ok(t, bComp.Compact(ctx)) testutil.Equals(t, 5.0, promtest.ToFloat64(sy.metrics.garbageCollectedBlocks)) @@ -409,7 +415,7 @@ func testGroupCompactE2e(t *testing.T, mergeFunc storage.VerticalChunkSeriesMerg testutil.Equals(t, uint64(5), meta.Stats.NumSeries) testutil.Equals(t, uint64(2*4*100-100), meta.Stats.NumSamples) testutil.Equals(t, 2, meta.Compaction.Level) - testutil.Equals(t, []ulid.ULID{metas[6].ULID, metas[7].ULID}, meta.Compaction.Sources) + testutil.Equals(t, []ulid.ULID{metas[7].ULID, metas[8].ULID}, meta.Compaction.Sources) // Check thanos meta. testutil.Assert(t, labels.Equal(extLabels2, labels.FromMap(meta.Thanos.Labels)), "ext labels does not match") diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index bde42c14206..a48c191d3d3 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -210,7 +210,8 @@ func TestRetentionProgressCalculate(t *testing.T) { var bkt objstore.Bucket temp := promauto.With(reg).NewCounter(prometheus.CounterOpts{Name: "test_metric_for_group", Help: "this is a test metric for compact progress tests"}) - grouper := NewDefaultGrouper(logger, bkt, false, false, reg, temp, temp, temp, "", 1, 1) + noCompactMarkerFilter := NewGatherNoCompactionMarkFilter(logger, objstore.WithNoopInstr(bkt), 2) + grouper := NewDefaultGrouper(logger, bkt, false, false, reg, temp, temp, temp, "", 1, 1, noCompactMarkerFilter) type groupedResult map[string]float64 @@ -376,7 +377,8 @@ func TestCompactProgressCalculate(t *testing.T) { var bkt objstore.Bucket temp := promauto.With(reg).NewCounter(prometheus.CounterOpts{Name: "test_metric_for_group", Help: "this is a test metric for compact progress tests"}) - grouper := NewDefaultGrouper(logger, bkt, false, false, reg, temp, temp, temp, "", 1, 1) + noCompactMarkerFilter := NewGatherNoCompactionMarkFilter(logger, objstore.WithNoopInstr(bkt), 2) + grouper := NewDefaultGrouper(logger, bkt, false, false, reg, temp, temp, temp, "", 1, 1, noCompactMarkerFilter) for _, tcase := range []struct { testName string @@ -498,7 +500,8 @@ func TestDownsampleProgressCalculate(t *testing.T) { var bkt objstore.Bucket temp := promauto.With(reg).NewCounter(prometheus.CounterOpts{Name: "test_metric_for_group", Help: "this is a test metric for downsample progress tests"}) - grouper := NewDefaultGrouper(logger, bkt, false, false, reg, temp, temp, temp, "", 1, 1) + noCompactMarkerFilter := NewGatherNoCompactionMarkFilter(logger, objstore.WithNoopInstr(bkt), 2) + grouper := NewDefaultGrouper(logger, bkt, false, false, reg, temp, temp, temp, "", 1, 1, noCompactMarkerFilter) for _, tcase := range []struct { testName string diff --git a/test/e2e/compact_test.go b/test/e2e/compact_test.go index 69577d1dcad..edf75619818 100644 --- a/test/e2e/compact_test.go +++ b/test/e2e/compact_test.go @@ -534,7 +534,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "2", "case": "compaction-ready-one-block-marked-for-no-compact"}}, {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "3", "case": "compaction-ready-one-block-marked-for-no-compact"}}, {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "4", "case": "compaction-ready-one-block-marked-for-no-compact", "replica": "1"}}, - {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "5", "case": "compaction-ready-one-block-marked-for-no-compact", "replica": "1"}}, + {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "5", "case": "compaction-ready-one-block-marked-for-no-compact"}}, {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "6", "case": "compaction-ready-one-block-marked-for-no-compact", "replica": "1"}}, {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "2", "case": "compaction-ready-after-dedup"}}, {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "3", "case": "compaction-ready-after-dedup"}}, @@ -574,7 +574,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "2", "case": "compaction-ready-one-block-marked-for-no-compact"}}, {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "3", "case": "compaction-ready-one-block-marked-for-no-compact"}}, {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "4", "case": "compaction-ready-one-block-marked-for-no-compact", "replica": "1"}}, - {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "5", "case": "compaction-ready-one-block-marked-for-no-compact", "replica": "1"}}, + {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "5", "case": "compaction-ready-one-block-marked-for-no-compact"}}, {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "6", "case": "compaction-ready-one-block-marked-for-no-compact", "replica": "1"}}, {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "2", "case": "compaction-ready-after-dedup"}}, {Value: 120, Metric: map[model.LabelName]model.LabelValue{"a": "1", "b": "3", "case": "compaction-ready-after-dedup"}}, @@ -697,7 +697,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { testutil.Ok(t, c.WaitSumMetrics(e2emon.Greater(0), "thanos_compact_iterations_total")) testutil.Ok(t, c.WaitSumMetrics(e2emon.Equals(0), "thanos_compact_blocks_cleaned_total")) testutil.Ok(t, c.WaitSumMetrics(e2emon.Equals(0), "thanos_compact_block_cleanup_failures_total")) - testutil.Ok(t, c.WaitSumMetrics(e2emon.Equals(2*4+2+2*3+2), "thanos_compact_blocks_marked_total")) // 18. + testutil.Ok(t, c.WaitSumMetrics(e2emon.Equals(2*4+2+2*3+2+1), "thanos_compact_blocks_marked_total")) // 19. testutil.Ok(t, c.WaitSumMetrics(e2emon.Equals(0), "thanos_compact_aborted_partial_uploads_deletion_attempts_total")) testutil.Ok(t, c.WaitSumMetrics(e2emon.Equals(6), "thanos_compact_group_compactions_total")) testutil.Ok(t, c.WaitSumMetrics(e2emon.Equals(3), "thanos_compact_group_vertical_compactions_total")) @@ -766,7 +766,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { // NOTE: We cannot assert on intermediate `thanos_blocks_meta_` metrics as those are gauge and change dynamically due to many // compaction groups. Wait for at least first compaction iteration (next is in 5m). testutil.Ok(t, c.WaitSumMetricsWithOptions(e2emon.Greater(0), []string{"thanos_compact_iterations_total"}, e2emon.WaitMissingMetrics())) - testutil.Ok(t, c.WaitSumMetricsWithOptions(e2emon.Equals(18), []string{"thanos_compact_blocks_cleaned_total"}, e2emon.WaitMissingMetrics())) + testutil.Ok(t, c.WaitSumMetricsWithOptions(e2emon.Equals(19), []string{"thanos_compact_blocks_cleaned_total"}, e2emon.WaitMissingMetrics())) testutil.Ok(t, c.WaitSumMetricsWithOptions(e2emon.Equals(0), []string{"thanos_compact_block_cleanup_failures_total"}, e2emon.WaitMissingMetrics())) testutil.Ok(t, c.WaitSumMetricsWithOptions(e2emon.Equals(0), []string{"thanos_compact_blocks_marked_total"}, e2emon.WaitMissingMetrics())) testutil.Ok(t, c.WaitSumMetricsWithOptions(e2emon.Equals(0), []string{"thanos_compact_aborted_partial_uploads_deletion_attempts_total"}, e2emon.WaitMissingMetrics())) @@ -779,7 +779,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { testutil.Ok(t, c.WaitSumMetricsWithOptions(e2emon.Equals(0), []string{"thanos_compact_downsample_total"}, e2emon.WaitMissingMetrics())) testutil.Ok(t, c.WaitSumMetricsWithOptions(e2emon.Equals(0), []string{"thanos_compact_downsample_failures_total"}, e2emon.WaitMissingMetrics())) - testutil.Ok(t, str.WaitSumMetricsWithOptions(e2emon.Equals(float64(len(rawBlockIDs)+8+6-18-2+2)), []string{"thanos_blocks_meta_synced"}, e2emon.WaitMissingMetrics())) + testutil.Ok(t, str.WaitSumMetricsWithOptions(e2emon.Equals(float64(len(rawBlockIDs)+8+6-18-2+2-1)), []string{"thanos_blocks_meta_synced"}, e2emon.WaitMissingMetrics())) testutil.Ok(t, str.WaitSumMetricsWithOptions(e2emon.Equals(0), []string{"thanos_blocks_meta_sync_failures_total"}, e2emon.WaitMissingMetrics())) testutil.Ok(t, c.WaitSumMetricsWithOptions(e2emon.Equals(0), []string{"thanos_compact_halted"}, e2emon.WaitMissingMetrics())) @@ -802,7 +802,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { ) // Store view: - testutil.Ok(t, str.WaitSumMetrics(e2emon.Equals(float64(len(rawBlockIDs)+8-18+6-2+2)), "thanos_blocks_meta_synced")) + testutil.Ok(t, str.WaitSumMetrics(e2emon.Equals(float64(len(rawBlockIDs)+8-18+6-2+2-1)), "thanos_blocks_meta_synced")) testutil.Ok(t, str.WaitSumMetrics(e2emon.Equals(0), "thanos_blocks_meta_sync_failures_total")) testutil.Ok(t, str.WaitSumMetrics(e2emon.Equals(0), "thanos_blocks_meta_modified")) } @@ -836,7 +836,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { testutil.Ok(t, str.Stop()) testutil.Ok(t, e2e.StartAndWaitReady(str)) testutil.Ok(t, runutil.Retry(time.Second, ctx.Done(), func() error { - return str.WaitSumMetrics(e2emon.Equals(float64(len(rawBlockIDs)+8-18+6-2+2-1)), "thanos_blocks_meta_synced") + return str.WaitSumMetrics(e2emon.Equals(float64(len(rawBlockIDs)+8-18+6-2+2-1-1)), "thanos_blocks_meta_synced") })) testutil.Ok(t, q.WaitSumMetricsWithOptions(e2emon.Equals(1), []string{"thanos_store_nodes_grpc_connections"}, e2emon.WaitMissingMetrics(), e2emon.WithLabelMatchers( matchers.MustNewMatcher(matchers.MatchEqual, "store_type", "store"),