From 0fb5a6b14d97f3f1dd47d5a8cab73c25914c48d8 Mon Sep 17 00:00:00 2001 From: John Fallows Date: Fri, 24 May 2024 13:01:25 -0700 Subject: [PATCH 1/8] Update CHANGELOG.md --- CHANGELOG.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index abacd7f8cf..3d23b7b327 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## [0.9.81](https://github.com/aklivity/zilla/tree/0.9.81) (2024-05-24) + +[Full Changelog](https://github.com/aklivity/zilla/compare/0.9.80...0.9.81) + +**Implemented enhancements:** + +- Improve Starting Zilla with the CLI [\#1016](https://github.com/aklivity/zilla/issues/1016) +- Generate `zilla dump` packet captures in timestamp order including across workers [\#959](https://github.com/aklivity/zilla/issues/959) +- Split protocol testing into separate ITs for `zilla dump` command [\#958](https://github.com/aklivity/zilla/issues/958) +- Add zilla context to MQTT consumer groups [\#886](https://github.com/aklivity/zilla/issues/886) + +**Fixed bugs:** + +- Telemetry attribute service.name doesn't get sent correctly [\#1007](https://github.com/aklivity/zilla/issues/1007) +- Streampay `zilla` instance crashes while trying to access `https://localhost:9090` [\#975](https://github.com/aklivity/zilla/issues/975) + +**Merged pull requests:** + +- Add service.name attribute to metrics [\#1048](https://github.com/aklivity/zilla/pull/1048) ([attilakreiner](https://github.com/attilakreiner)) +- Starting Zilla with the CLI improvement [\#1042](https://github.com/aklivity/zilla/pull/1042) ([aDaemonThread](https://github.com/aDaemonThread)) +- Sort frames by timestamp in dump command [\#1041](https://github.com/aklivity/zilla/pull/1041) ([attilakreiner](https://github.com/attilakreiner)) +- Ensure new mqtt subscriptions are not empty [\#1040](https://github.com/aklivity/zilla/pull/1040) ([jfallows](https://github.com/jfallows)) +- Add zilla context to MQTT consumer groups [\#1035](https://github.com/aklivity/zilla/pull/1035) ([bmaidics](https://github.com/bmaidics)) +- Split protocol testing into separate ITs for zilla dump command [\#989](https://github.com/aklivity/zilla/pull/989) ([attilakreiner](https://github.com/attilakreiner)) + ## [0.9.80](https://github.com/aklivity/zilla/tree/0.9.80) (2024-05-20) [Full Changelog](https://github.com/aklivity/zilla/compare/0.9.79...0.9.80) From 761dcfa40cae791838c70476830219258c1f92cf Mon Sep 17 00:00:00 2001 From: Ankit Kumar Date: Sat, 25 May 2024 04:06:40 +0530 Subject: [PATCH 2/8] =?UTF-8?q?Fix:=20http-kafka=20will=20fetch=20messages?= =?UTF-8?q?=20that=20have=20been=20deleted=20by=20a=20reten=E2=80=A6=20(#1?= =?UTF-8?q?033)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cache/KafkaCacheCursorFactory.java | 68 ++++++++++--------- .../stream/KafkaCacheServerFetchFactory.java | 20 +++--- 2 files changed, 47 insertions(+), 41 deletions(-) diff --git a/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/cache/KafkaCacheCursorFactory.java b/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/cache/KafkaCacheCursorFactory.java index 618d265fc7..c756aff33b 100644 --- a/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/cache/KafkaCacheCursorFactory.java +++ b/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/cache/KafkaCacheCursorFactory.java @@ -126,23 +126,26 @@ public void init( this.latestOffset = latestOffset; assert !segmentNode.sentinel(); - KafkaCacheSegment newSegment = null; - while (newSegment == null) + if (segmentNode.segment() != null) { - newSegment = segmentNode.segment().acquire(); - if (newSegment == null) + KafkaCacheSegment newSegment = null; + while (newSegment == null) { - segmentNode = segmentNode.next(); + newSegment = segmentNode.segment().acquire(); + if (newSegment == null) + { + segmentNode = segmentNode.next(); + } } - } - this.segmentNode = segmentNode; - this.segment = newSegment; + this.segmentNode = segmentNode; + this.segment = newSegment; - assert this.segmentNode != null; - assert this.segment != null; + assert this.segmentNode != null; + assert this.segment != null; - final int position = condition.reset(segment, offset, latestOffset, POSITION_UNSET); - this.position = position == RETRY_SEGMENT_VALUE || position == NEXT_SEGMENT_VALUE ? 0 : position; + final int position = condition.reset(segment, offset, latestOffset, POSITION_UNSET); + this.position = position == RETRY_SEGMENT_VALUE || position == NEXT_SEGMENT_VALUE ? 0 : position; + } } public KafkaCacheEntryFW next( @@ -151,7 +154,7 @@ public KafkaCacheEntryFW next( KafkaCacheEntryFW nextEntry = null; next: - while (nextEntry == null) + while (segmentNode != null && nextEntry == null) { final int positionNext = condition.next(position); if (positionNext == RETRY_SEGMENT_VALUE) @@ -345,30 +348,33 @@ public void advance( assert segmentNode != null; assert segment != null; - KafkaCacheSegment newSegment = segmentNode.segment(); - if (segment != newSegment) + if (segmentNode != null) { - segment.release(); - - Node newSegmentNode = segmentNode; - newSegment = newSegment.acquire(); - while (newSegment == null) + KafkaCacheSegment newSegment = segmentNode.segment(); + if (segment != newSegment) { - newSegment = newSegmentNode.segment().acquire(); - if (newSegment == null) + segment.release(); + + Node newSegmentNode = segmentNode; + newSegment = newSegment.acquire(); + while (newSegment == null) { - newSegmentNode = newSegmentNode.next(); + newSegment = newSegmentNode.segment().acquire(); + if (newSegment == null) + { + newSegmentNode = newSegmentNode.next(); + } } - } - this.segmentNode = newSegmentNode; - this.segment = newSegment; + this.segmentNode = newSegmentNode; + this.segment = newSegment; - assert segmentNode != null; - assert !segmentNode.sentinel(); - assert segment != null; + assert segmentNode != null; + assert !segmentNode.sentinel(); + assert segment != null; - final int position = condition.reset(segment, offset, latestOffset, POSITION_UNSET); - this.position = position == RETRY_SEGMENT_VALUE || position == NEXT_SEGMENT_VALUE ? 0 : position; + final int position = condition.reset(segment, offset, latestOffset, POSITION_UNSET); + this.position = position == RETRY_SEGMENT_VALUE || position == NEXT_SEGMENT_VALUE ? 0 : position; + } } } diff --git a/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaCacheServerFetchFactory.java b/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaCacheServerFetchFactory.java index 82079a3ae5..e1d35853ad 100644 --- a/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaCacheServerFetchFactory.java +++ b/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaCacheServerFetchFactory.java @@ -866,14 +866,13 @@ private void onServerFanoutReplyData( final long retainAt = partition.retainAt(nextHead.segment()); this.retainId = doServerFanoutInitialSignalAt(retainAt, traceId, SIGNAL_SEGMENT_RETAIN); + } - if (deleteId == NO_CANCEL_ID && - partition.cleanupPolicy().delete() && - !nextHead.previous().sentinel()) - { - final long deleteAt = partition.deleteAt(nextHead.previous().segment(), retentionMillisMax); - this.deleteId = doServerFanoutInitialSignalAt(deleteAt, traceId, SIGNAL_SEGMENT_DELETE); - } + if (deleteId == NO_CANCEL_ID && + partition.cleanupPolicy().delete()) + { + final long deleteAt = partition.deleteAt(nextHead.segment(), retentionMillisMax); + this.deleteId = doServerFanoutInitialSignalAt(deleteAt, traceId, SIGNAL_SEGMENT_DELETE); } final int entryFlags = (flags & FLAGS_SKIP) != 0x00 ? CACHE_ENTRY_FLAGS_ABORTED : 0x00; @@ -1187,15 +1186,16 @@ private void onServerFanoutInitialSignalSegmentDelete( final long now = currentTimeMillis(); Node segmentNode = partition.sentinel().next(); - while (segmentNode != partition.head() && - partition.deleteAt(segmentNode.segment(), retentionMillisMax) <= now) + while (!segmentNode.sentinel() && + partition.deleteAt(segmentNode.segment(), retentionMillisMax) <= now) { segmentNode.remove(); segmentNode = segmentNode.next(); } + assert segmentNode != null; - if (segmentNode != partition.head()) + if (segmentNode != partition.sentinel()) { final long deleteAt = partition.deleteAt(segmentNode.segment(), retentionMillisMax); this.deleteId = doServerFanoutInitialSignalAt(deleteAt, traceId, SIGNAL_SEGMENT_DELETE); From d22c0d056aac9ab7c67a4616abdfcede2908d8af Mon Sep 17 00:00:00 2001 From: John Fallows Date: Sat, 25 May 2024 08:49:36 -0700 Subject: [PATCH 3/8] Support kafka cache bootstrap with topic default offset live (#1052) --- .../stream/KafkaCacheBootstrapFactory.java | 2 +- .../internal/stream/CacheBootstrapIT.java | 17 ++ .../config/cache.options.bootstrap.live.yaml | 39 +++ .../client.rpt | 236 ++++++++++++++++ .../server.rpt | 251 ++++++++++++++++++ .../streams/application/BootstrapIT.java | 1 - .../kafka/streams/application/MergedIT.java | 12 + 7 files changed, 556 insertions(+), 2 deletions(-) create mode 100644 specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/config/cache.options.bootstrap.live.yaml create mode 100644 specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.fetch.message.values.live/client.rpt create mode 100644 specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.fetch.message.values.live/server.rpt diff --git a/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaCacheBootstrapFactory.java b/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaCacheBootstrapFactory.java index 9f17334f44..6fb2a405a5 100644 --- a/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaCacheBootstrapFactory.java +++ b/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaCacheBootstrapFactory.java @@ -415,7 +415,7 @@ private final class KafkaBootstrapStream this.describeStream = new KafkaBootstrapDescribeStream(this); this.metaStream = new KafkaBootstrapMetaStream(this); this.fetchStreams = new ArrayList<>(); - this.nextOffsetsById = new Long2LongHashMap(-1L); + this.nextOffsetsById = new Long2LongHashMap(-3L); this.defaultOffset = defaultOffset; this.consumers = new Int2ObjectHashMap<>(); this.leadersByPartitionId = new Int2IntHashMap(-1); diff --git a/runtime/binding-kafka/src/test/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/CacheBootstrapIT.java b/runtime/binding-kafka/src/test/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/CacheBootstrapIT.java index d679728ebe..d55e94a547 100644 --- a/runtime/binding-kafka/src/test/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/CacheBootstrapIT.java +++ b/runtime/binding-kafka/src/test/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/CacheBootstrapIT.java @@ -72,6 +72,23 @@ public void shouldReceiveMergedMessageValues() throws Exception k3po.finish(); } + @Test + @Configuration("cache.options.bootstrap.live.yaml") + @Specification({ + "${app}/unmerged.fetch.message.values.live/server"}) + public void shouldReceiveMergedMessageValuesLive() throws Exception + { + Thread.sleep(500); + k3po.start(); + k3po.awaitBarrier("CHANGING_PARTITION_COUNT"); + Thread.sleep(200); // allow A1, B1, A2, B2 to be merged + k3po.notifyBarrier("CHANGED_PARTITION_COUNT"); + k3po.awaitBarrier("SENT_MESSAGE_A2"); + k3po.awaitBarrier("SENT_MESSAGE_B2"); + k3po.awaitBarrier("SENT_MESSAGE_C2"); + k3po.finish(); + } + @Test @Configuration("cache.yaml") @Specification({ diff --git a/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/config/cache.options.bootstrap.live.yaml b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/config/cache.options.bootstrap.live.yaml new file mode 100644 index 0000000000..4172402b91 --- /dev/null +++ b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/config/cache.options.bootstrap.live.yaml @@ -0,0 +1,39 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +--- +name: test +bindings: + app0: + type: kafka + kind: cache_client + routes: + - exit: cache0 + when: + - topic: test + cache0: + type: kafka + kind: cache_server + options: + topics: + - name: test + defaultOffset: live + bootstrap: + - test + routes: + - exit: app1 + when: + - topic: test diff --git a/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.fetch.message.values.live/client.rpt b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.fetch.message.values.live/client.rpt new file mode 100644 index 0000000000..957d6f4c84 --- /dev/null +++ b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.fetch.message.values.live/client.rpt @@ -0,0 +1,236 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +connect "zilla://streams/app1" + option zilla:window 8192 + option zilla:transmission "half-duplex" + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .describe() + .topic("test") + .config("cleanup.policy") + .config("max.message.bytes") + .config("segment.bytes") + .config("segment.index.bytes") + .config("segment.ms") + .config("retention.bytes") + .config("retention.ms") + .config("delete.retention.ms") + .config("min.compaction.lag.ms") + .config("max.compaction.lag.ms") + .config("min.cleanable.dirty.ratio") + .build() + .build()} + +connected + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .describe() + .topic("test") + .config("cleanup.policy") + .config("max.message.bytes") + .config("segment.bytes") + .config("segment.index.bytes") + .config("segment.ms") + .config("retention.bytes") + .config("retention.ms") + .config("delete.retention.ms") + .config("min.compaction.lag.ms") + .config("max.compaction.lag.ms") + .config("min.cleanable.dirty.ratio") + .build() + .build()} + +read zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .describe() + .config("cleanup.policy", "delete") + .config("max.message.bytes", 1000012) + .config("segment.bytes", 1073741824) + .config("segment.index.bytes", 10485760) + .config("segment.ms", 604800000) + .config("retention.bytes", -1) + .config("retention.ms", 604800000) + .config("delete.retention.ms", 86400000) + .config("min.compaction.lag.ms", 0) + .config("max.compaction.lag.ms", 9223372036854775807) + .config("min.cleanable.dirty.ratio", 0.5) + .build() + .build()} + +read notify RECEIVED_CONFIG + +connect await RECEIVED_CONFIG + "zilla://streams/app1" + option zilla:window 8192 + option zilla:transmission "half-duplex" + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .meta() + .topic("test") + .build() + .build()} + +connected + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .meta() + .topic("test") + .build() + .build()} + +read zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .meta() + .partition(0, 1) + .partition(1, 2) + .build() + .build()} +read notify PARTITION_COUNT_2 + +read zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .meta() + .partition(0, 1) + .partition(1, 2) + .partition(2, 3) + .build() + .build()} +read notify PARTITION_COUNT_3 + +connect await PARTITION_COUNT_2 + "zilla://streams/app1" + option zilla:window 8192 + option zilla:transmission "half-duplex" + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(0, -1) + .build() + .build()} + +connected + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(0, 1, 2) + .build() + .build()} + +read zilla:data.ext ${kafka:matchDataEx() + .typeId(zilla:id("kafka")) + .fetch() + .partition(0, 1, 2) + .build() + .build()} +read "Hello, world #A1" + +read zilla:data.ext ${kafka:matchDataEx() + .typeId(zilla:id("kafka")) + .fetch() + .partition(0, 2, 2) + .build() + .build()} +read "Hello, world #A2" +read notify RECEIVED_MESSAGE_A2 + +connect await PARTITION_COUNT_2 + "zilla://streams/app1" + option zilla:window 8192 + option zilla:transmission "half-duplex" + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(1, -1) + .build() + .build()} + +connected + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(1, 1, 2) + .build() + .build()} + +read zilla:data.ext ${kafka:matchDataEx() + .typeId(zilla:id("kafka")) + .fetch() + .partition(1, 1, 2) + .build() + .build()} +read "Hello, world #B1" + +read zilla:data.ext ${kafka:matchDataEx() + .typeId(zilla:id("kafka")) + .fetch() + .partition(1, 2, 2) + .build() + .build()} +read "Hello, world #B2" +read notify RECEIVED_MESSAGE_B2 + +connect await PARTITION_COUNT_3 + "zilla://streams/app1" + option zilla:window 8192 + option zilla:transmission "half-duplex" + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(2, -1) + .build() + .build()} + +connected + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(2, 1, 2) + .build() + .build()} + +read zilla:data.ext ${kafka:matchDataEx() + .typeId(zilla:id("kafka")) + .fetch() + .partition(2, 1, 2) + .build() + .build()} +read "Hello, world #C1" + +read zilla:data.ext ${kafka:matchDataEx() + .typeId(zilla:id("kafka")) + .fetch() + .partition(2, 2, 2) + .build() + .build()} +read "Hello, world #C2" +read notify RECEIVED_MESSAGE_C2 diff --git a/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.fetch.message.values.live/server.rpt b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.fetch.message.values.live/server.rpt new file mode 100644 index 0000000000..32370d2698 --- /dev/null +++ b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.fetch.message.values.live/server.rpt @@ -0,0 +1,251 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +property deltaMillis 0L +property newTimestamp ${kafka:timestamp() + deltaMillis} + +accept "zilla://streams/app1" + option zilla:window 8192 + option zilla:transmission "half-duplex" + +accepted + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .describe() + .topic("test") + .config("cleanup.policy") + .config("max.message.bytes") + .config("segment.bytes") + .config("segment.index.bytes") + .config("segment.ms") + .config("retention.bytes") + .config("retention.ms") + .config("delete.retention.ms") + .config("min.compaction.lag.ms") + .config("max.compaction.lag.ms") + .config("min.cleanable.dirty.ratio") + .build() + .build()} + +connected + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .describe() + .topic("test") + .config("cleanup.policy") + .config("max.message.bytes") + .config("segment.bytes") + .config("segment.index.bytes") + .config("segment.ms") + .config("retention.bytes") + .config("retention.ms") + .config("delete.retention.ms") + .config("min.compaction.lag.ms") + .config("max.compaction.lag.ms") + .config("min.cleanable.dirty.ratio") + .build() + .build()} +write flush + +write zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .describe() + .config("cleanup.policy", "delete") + .config("max.message.bytes", 1000012) + .config("segment.bytes", 1073741824) + .config("segment.index.bytes", 10485760) + .config("segment.ms", 604800000) + .config("retention.bytes", -1) + .config("retention.ms", 604800000) + .config("delete.retention.ms", 86400000) + .config("min.compaction.lag.ms", 0) + .config("max.compaction.lag.ms", 9223372036854775807) + .config("min.cleanable.dirty.ratio", 0.5) + .build() + .build()} +write flush + +accepted + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .meta() + .topic("test") + .build() + .build()} + +connected + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .meta() + .topic("test") + .build() + .build()} +write flush + +write zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .meta() + .partition(0, 1) + .partition(1, 2) + .build() + .build()} +write flush + +write await SENT_MESSAGE_A2 +write await SENT_MESSAGE_B2 + +write notify CHANGING_PARTITION_COUNT +write await CHANGED_PARTITION_COUNT + +write zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .meta() + .partition(0, 1) + .partition(1, 2) + .partition(2, 3) + .build() + .build()} +write flush + +accepted + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(0, -1) + .build() + .build()} + +connected + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(0, 1, 2) + .build() + .build()} +write flush + +write zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .fetch() + .timestamp(newTimestamp) + .partition(0, 1, 2) + .build() + .build()} +write "Hello, world #A1" +write flush + +write zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .fetch() + .timestamp(newTimestamp) + .partition(0, 2, 2) + .build() + .build()} +write "Hello, world #A2" +write flush +write notify SENT_MESSAGE_A2 + +accepted + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(1, -1) + .build() + .build()} + +connected + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(1, 1, 2) + .build() + .build()} +write flush + +write zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .fetch() + .timestamp(newTimestamp) + .partition(1, 1, 2) + .build() + .build()} +write "Hello, world #B1" +write flush + +write zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .fetch() + .timestamp(newTimestamp) + .partition(1, 2, 2) + .build() + .build()} +write "Hello, world #B2" +write flush +write notify SENT_MESSAGE_B2 + +accepted + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(2, -1) + .build() + .build()} + +connected + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(2, 1, 2) + .build() + .build()} +write flush + +write zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .fetch() + .timestamp(newTimestamp) + .partition(2, 1, 2) + .build() + .build()} +write "Hello, world #C1" +write flush + +write zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .fetch() + .timestamp(newTimestamp) + .partition(2, 2, 2) + .build() + .build()} +write "Hello, world #C2" +write flush +write notify SENT_MESSAGE_C2 diff --git a/specs/binding-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/kafka/streams/application/BootstrapIT.java b/specs/binding-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/kafka/streams/application/BootstrapIT.java index bf01fda115..0a1ca5b38d 100644 --- a/specs/binding-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/kafka/streams/application/BootstrapIT.java +++ b/specs/binding-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/kafka/streams/application/BootstrapIT.java @@ -45,7 +45,6 @@ public void shouldRequestBootstrapPartitionOffsetsEarliest() throws Exception k3po.finish(); } - @Test @Specification({ "${app}/group.fetch.message.value/client", diff --git a/specs/binding-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/kafka/streams/application/MergedIT.java b/specs/binding-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/kafka/streams/application/MergedIT.java index 6ae0943d59..12a994c0fd 100644 --- a/specs/binding-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/kafka/streams/application/MergedIT.java +++ b/specs/binding-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/kafka/streams/application/MergedIT.java @@ -495,6 +495,18 @@ public void shouldFetchUnmergedMessageValues() throws Exception k3po.finish(); } + @Test + @Specification({ + "${app}/unmerged.fetch.message.values.live/client", + "${app}/unmerged.fetch.message.values.live/server"}) + public void shouldFetchUnmergedMessageValuesLive() throws Exception + { + k3po.start(); + k3po.awaitBarrier("CHANGING_PARTITION_COUNT"); + k3po.notifyBarrier("CHANGED_PARTITION_COUNT"); + k3po.finish(); + } + @Test @Specification({ "${app}/unmerged.fetch.partition.offsets.latest/client", From 6ca94d2a894474dbec07bcef916844bff803b329 Mon Sep 17 00:00:00 2001 From: Akram Yakubov Date: Sun, 26 May 2024 06:09:40 -0700 Subject: [PATCH 4/8] Queue as different kafka produce request if producerId or producerEpoch varies (#1053) --- .../stream/KafkaClientProduceFactory.java | 294 +++++++++--------- .../produce/message.value/client.rpt | 3 +- .../produce.v3/message.value/client.rpt | 6 +- .../produce.v3/message.value/server.rpt | 9 +- 4 files changed, 156 insertions(+), 156 deletions(-) diff --git a/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaClientProduceFactory.java b/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaClientProduceFactory.java index 70c89311fc..8f4b71cad3 100644 --- a/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaClientProduceFactory.java +++ b/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaClientProduceFactory.java @@ -32,7 +32,6 @@ import java.util.function.LongFunction; import java.util.zip.CRC32C; -import org.agrona.BitUtil; import org.agrona.DirectBuffer; import org.agrona.MutableDirectBuffer; import org.agrona.collections.LongLongConsumer; @@ -92,14 +91,12 @@ public final class KafkaClientProduceFactory extends KafkaClientSaslHandshaker i private static final int FLAGS_FIN = 0x01; private static final int FLAGS_INIT = 0x02; - private static final byte RECORD_BATCH_MAGIC = 2; - private static final short RECORD_BATCH_ATTRIBUTES_NONE = 0; - private static final short RECORD_BATCH_ATTRIBUTES_NO_TIMESTAMP = 0x08; private static final long RECORD_BATCH_PRODUCER_ID_NONE = -1; - private static final short RECORD_BATCH_PRODUCER_EPOCH_NONE = -1; private static final int RECORD_BATCH_BASE_SEQUENCE_NONE = -1; - private static final int RECORD_SEQUENCE_NONE = -1; + private static final byte RECORD_BATCH_MAGIC = 2; private static final byte RECORD_ATTRIBUTES_NONE = 0; + private static final short RECORD_BATCH_ATTRIBUTES_NONE = 0; + private static final short RECORD_BATCH_ATTRIBUTES_NO_TIMESTAMP = 0x08; private static final String TRANSACTION_ID_NONE = null; @@ -151,6 +148,7 @@ public final class KafkaClientProduceFactory extends KafkaClientSaslHandshaker i private final RecordHeaderFW.Builder recordHeaderRW = new RecordHeaderFW.Builder(); private final RecordTrailerFW.Builder recordTrailerRW = new RecordTrailerFW.Builder(); + private final RecordBatchFW recordBatchRO = new RecordBatchFW(); private final ResponseHeaderFW responseHeaderRO = new ResponseHeaderFW(); private final ProduceResponseFW produceResponseRO = new ProduceResponseFW(); private final ProduceTopicResponseFW produceTopicResponseRO = new ProduceTopicResponseFW(); @@ -253,7 +251,7 @@ public MessageConsumer newStream( final KafkaBindingConfig binding = supplyBinding.apply(routedId); final KafkaRouteConfig resolved = binding != null ? binding.resolve(authorization, topicName) : null; - if (resolved != null && kafkaBeginEx != null) + if (resolved != null) { final long resolvedId = resolved.id; final int partitionId = kafkaProduceBeginEx.partition().partitionId(); @@ -553,10 +551,7 @@ private int flushRecordInit( final int maxEncodeableBytes = client.encodeSlotLimit + client.valueCompleteSize + produceRecordFramingSize; if (client.encodeSlot != NO_SLOT && - (maxEncodeableBytes > encodePool.slotCapacity() || - client.producerId != producerId || - client.producerEpoch != producerEpoch || - sequence <= client.sequence && sequence != RECORD_BATCH_BASE_SEQUENCE_NONE)) + maxEncodeableBytes > encodePool.slotCapacity()) { client.doEncodeRequestIfNecessary(traceId, budgetId); } @@ -565,16 +560,7 @@ private int flushRecordInit( client.encodeableRecordBytesDeferred = deferred; client.valueCompleteSize = valueCompleteSize; - if (client.producerId == RECORD_BATCH_PRODUCER_ID_NONE) - { - client.baseSequence = sequence; - } - - client.producerId = producerId; - client.producerEpoch = producerEpoch; - client.sequence = sequence; - - client.doEncodeRecordInit(traceId, timestamp, ackMode, key, payload, headers); + client.doEncodeRecordInit(traceId, timestamp, producerId, producerEpoch, sequence, ackMode, key, payload, headers); if (client.encodeSlot != NO_SLOT) { client.flusher = flushRecordContFin; @@ -1237,14 +1223,9 @@ private final class KafkaProduceClient extends KafkaSaslClient private int encodeSlotLimit; private long encodeSlotTraceId; - private long encodeableRecordBatchTimestamp; - private long encodeableRecordBatchTimestampMax; - private long encodeableRecordTimestamp; private int flushableRecordHeadersBytes; - private int encodeableRecordCount; - private int encodeableRecordBytes; private int encodeableRecordBytesDeferred; - private int encodeableRecordValueBytes; + private int encodeableRecordBatchSlotOffset; private int flushableRequestBytes; private int decodeSlot = NO_SLOT; @@ -1264,11 +1245,6 @@ private final class KafkaProduceClient extends KafkaSaslClient private LongLongConsumer encoder; private boolean flushable; - private long producerId = RECORD_BATCH_PRODUCER_ID_NONE; - private short producerEpoch = RECORD_BATCH_PRODUCER_EPOCH_NONE; - private int baseSequence = RECORD_BATCH_BASE_SEQUENCE_NONE; - private int sequence = RECORD_SEQUENCE_NONE; - KafkaProduceClient( KafkaProduceStream stream, long resolvedId, @@ -1282,8 +1258,6 @@ private final class KafkaProduceClient extends KafkaSaslClient this.topic = requireNonNull(topic); this.partitionId = partitionId; this.flusher = flushRecord; - this.encodeableRecordBatchTimestamp = TIMESTAMP_NONE; - this.encodeableRecordBatchTimestampMax = TIMESTAMP_NONE; this.encodeableAckMode = KafkaAckMode.NONE; this.flushable = sasl == null; @@ -1651,88 +1625,126 @@ private void flush( private void doEncodeRecordInit( long traceId, long timestamp, + long producerId, + short producerEpoch, + int sequence, KafkaAckMode ackMode, KafkaKeyFW key, OctetsFW value, Array32FW headers) { - encodeableRecordTimestamp = timestamp; - if (encodeableRecordBatchTimestamp == TIMESTAMP_NONE) + encodeableAckMode = maxAckMode(encodeableAckMode, ackMode); + + if (encodeSlot == NO_SLOT) { - encodeableRecordBatchTimestamp = timestamp; + encodeSlot = encodePool.acquire(initialId); + encodeSlotOffset = PRODUCE_REQUEST_RECORDS_OFFSET_MAX; + encodeSlotLimit = encodeSlotOffset; + encodeableRecordBatchSlotOffset = encodeSlotLimit; } - final MutableDirectBuffer encodeBuffer = writeBuffer; - final int encodeLimit = writeBuffer.capacity(); - int encodeProgress = 0; + assert encodeSlot != NO_SLOT; + final int maxLimit = encodePool.slotCapacity(); + final MutableDirectBuffer encodeSlotBuffer = encodePool.buffer(encodeSlot); + + RecordBatchFW encodeableRecordBatch = recordBatchRO.tryWrap( + encodeSlotBuffer, encodeableRecordBatchSlotOffset, encodeSlotLimit); + + if (encodeableRecordBatch == null || + encodeableRecordBatch.producerId() != producerId || + encodeableRecordBatch.producerEpoch() != producerEpoch || + encodeableRecordBatch.baseSequence() >= sequence) + { + encodeableRecordBatch = recordBatchRW.wrap(encodeSlotBuffer, encodeSlotLimit, maxLimit) + .baseOffset(0) + .length(FIELD_OFFSET_RECORD_COUNT - FIELD_OFFSET_LENGTH) + .leaderEpoch(-1) + .magic(RECORD_BATCH_MAGIC) + .crc(0) + .attributes(RECORD_BATCH_ATTRIBUTES_NONE) + .lastOffsetDelta(0) + .firstTimestamp(timestamp) + .maxTimestamp(TIMESTAMP_NONE) + .producerId(producerId) + .producerEpoch(producerEpoch) + .baseSequence(producerId == RECORD_BATCH_PRODUCER_ID_NONE + ? RECORD_BATCH_BASE_SEQUENCE_NONE + : sequence) + .recordCount(0) + .build(); + encodeableRecordBatchSlotOffset = encodeSlotLimit; + encodeSlotLimit = encodeableRecordBatch.limit(); + } + + assert encodeableRecordBatch != null; + + final int offsetDelta = encodeableRecordBatch.recordCount(); final int headersCount = headers.fieldCount(); - final RecordTrailerFW recordTrailer = recordTrailerRW.wrap(encodeBuffer, 0, encodeLimit) + final RecordTrailerFW recordTrailer = recordTrailerRW.wrap(encodeSlotBuffer, encodeSlotLimit, maxLimit) .headerCount(headersCount) .build(); - final int recordTrailerSize = recordTrailer.limit(); + final int recordTrailerSize = recordTrailer.sizeof(); - final int timestampDelta = (int) (timestamp - encodeableRecordBatchTimestamp); - RecordHeaderFW recordHeader = recordHeaderRW.wrap(encodeBuffer, encodeProgress, encodeLimit) + final int timestampDelta = (int) (timestamp - encodeableRecordBatch.firstTimestamp()); + RecordHeaderFW recordHeader = recordHeaderRW.wrap(encodeSlotBuffer, encodeSlotLimit, maxLimit) .length(Integer.MAX_VALUE) .attributes(RECORD_ATTRIBUTES_NONE) .timestampDelta(timestampDelta) - .offsetDelta(encodeableRecordCount) + .offsetDelta(offsetDelta) .keyLength(key.length()) .key(key.value()) .valueLength(value != null ? value.sizeof() + encodeableRecordBytesDeferred : -1) .build(); final int valueSize = value != null ? value.sizeof() : 0; - final int recordHeaderSize = recordHeader.limit(); + final int recordHeaderSize = recordHeader.sizeof(); final int headerSize = headers.items().capacity(); final int recordSize = recordHeaderSize + valueSize + encodeableRecordBytesDeferred + recordTrailerSize + headerSize - RECORD_LENGTH_MAX; final int valueLength = value != null ? value.sizeof() + encodeableRecordBytesDeferred : -1; - recordHeader = recordHeaderRW.wrap(encodeBuffer, encodeProgress, encodeLimit) + recordHeader = recordHeaderRW.wrap(encodeSlotBuffer, encodeSlotLimit, maxLimit) .length(recordSize) .attributes(RECORD_ATTRIBUTES_NONE) .timestampDelta(timestampDelta) - .offsetDelta(encodeableRecordCount) + .offsetDelta(offsetDelta) .keyLength(key.length()) .key(key.value()) .valueLength(valueLength) .build(); - final int newRecordSize = recordHeader.limit(); - encodeProgress += newRecordSize; - encodeableRecordBytes += newRecordSize + valueSize + encodeableRecordBytesDeferred + - recordTrailerSize + headerSize; - - if (encodeSlot == NO_SLOT) - { - encodeSlot = encodePool.acquire(initialId); - encodeSlotOffset = PRODUCE_REQUEST_RECORDS_OFFSET_MAX; - encodeSlotLimit = encodeSlotOffset; - } + final int newRecordHeaderSize = recordHeader.sizeof(); + encodeSlotLimit += newRecordHeaderSize; + + int newRecordSize = newRecordHeaderSize + valueSize + encodeableRecordBytesDeferred + + recordTrailerSize + headerSize; + + final int recordBatchLength = encodeableRecordBatch.length(); + final long maxTimestamp = Math.max(encodeableRecordBatch.maxTimestamp(), timestamp); + final short attributes = maxTimestamp == 0L + ? RECORD_BATCH_ATTRIBUTES_NO_TIMESTAMP + : RECORD_BATCH_ATTRIBUTES_NONE; + final int recordCount = encodeableRecordBatch.recordCount(); + + encodeSlotBuffer.putInt(encodeableRecordBatchSlotOffset + RecordBatchFW.FIELD_OFFSET_LENGTH, + recordBatchLength + newRecordSize, BIG_ENDIAN); + encodeSlotBuffer.putShort(encodeableRecordBatchSlotOffset + RecordBatchFW.FIELD_OFFSET_ATTRIBUTES, + attributes, BIG_ENDIAN); + encodeSlotBuffer.putInt(encodeableRecordBatchSlotOffset + RecordBatchFW.FIELD_OFFSET_LAST_OFFSET_DELTA, + recordCount, BIG_ENDIAN); + encodeSlotBuffer.putLong(encodeableRecordBatchSlotOffset + RecordBatchFW.FIELD_OFFSET_MAX_TIMESTAMP, + maxTimestamp, BIG_ENDIAN); + encodeSlotBuffer.putInt(encodeableRecordBatchSlotOffset + RecordBatchFW.FIELD_OFFSET_RECORD_COUNT, + recordCount + 1, BIG_ENDIAN); - if (encodeSlot != NO_SLOT) + if (headersCount > 0) { - final MutableDirectBuffer encodeSlotBuffer = encodePool.buffer(encodeSlot); - - encodeSlotBuffer.putBytes(encodeSlotLimit, encodeBuffer, 0, encodeProgress); - encodeSlotLimit += encodeProgress; - encodeableRecordValueBytes = 0; + flushableRecordHeadersBytes = headers.sizeof(); - if (headersCount > 0) - { - flushableRecordHeadersBytes = headers.sizeof(); - - final int encodeSlotMaxLimit = encodePool.slotCapacity() - flushableRecordHeadersBytes; - encodeSlotBuffer.putBytes(encodeSlotMaxLimit, headers.buffer(), headers.offset(), - flushableRecordHeadersBytes); - } - - encodeableRecordBatchTimestampMax = Math.max(encodeableRecordBatchTimestamp, encodeableRecordTimestamp); - - encodeableRecordCount++; - encodeableAckMode = maxAckMode(encodeableAckMode, ackMode); + final int encodeSlotMaxLimit = encodePool.slotCapacity() - flushableRecordHeadersBytes; + encodeSlotBuffer.putBytes(encodeSlotMaxLimit, headers.buffer(), headers.offset(), + flushableRecordHeadersBytes); } } @@ -1758,7 +1770,6 @@ private void doEncodeRecordCont( encodeSlotBuffer.putBytes(encodeSlotLimit, value.buffer(), value.offset(), length); encodeSlotLimit += length; - encodeableRecordValueBytes += length; if ((flags & FLAGS_FIN) == 0) { @@ -1863,6 +1874,11 @@ private void doEncodeProduceRequest( final int encodeOffset = 0; final int encodeLimit = encodeBuffer.capacity(); + if (KafkaConfiguration.DEBUG) + { + System.out.format("[client] %s[%d] PRODUCE\n", topic, partitionId); + } + int encodeProgress = encodeOffset; final RequestHeaderFW requestHeader = requestHeaderRW.wrap(encodeBuffer, encodeProgress, encodeLimit) @@ -1891,50 +1907,31 @@ private void doEncodeProduceRequest( encodeProgress = topicRequest.limit(); - final int recordBatchLength = FIELD_OFFSET_RECORD_COUNT - FIELD_OFFSET_LENGTH + encodeableRecordBytes; - final int recordSetLength = FIELD_OFFSET_LENGTH + BitUtil.SIZE_OF_INT + recordBatchLength; + assert encodeSlot != NO_SLOT; + final MutableDirectBuffer encodeSlotBuffer = encodePool.buffer(encodeSlot); + + final RecordBatchFW recordBatch = recordBatchRO.wrap(encodeSlotBuffer, encodeSlotOffset, encodeSlotLimit); + final int recordBatchLength = RecordBatchFW.FIELD_OFFSET_LEADER_EPOCH + recordBatch.length(); final ProducePartitionRequestFW partitionRequest = partitionRequestRW.wrap(encodeBuffer, encodeProgress, encodeLimit) .partitionId(partitionId) - .recordSetLength(recordSetLength) + .recordSetLength(recordBatchLength) .build(); encodeProgress = partitionRequest.limit(); - final int crcOffset = encodeProgress - encodeOffset + RecordBatchFW.FIELD_OFFSET_CRC; - final int crcLimit = encodeProgress - encodeOffset + RecordBatchFW.FIELD_OFFSET_ATTRIBUTES; - - final short attributes = encodeableRecordBatchTimestampMax == 0L - ? RECORD_BATCH_ATTRIBUTES_NO_TIMESTAMP - : RECORD_BATCH_ATTRIBUTES_NONE; - - final int baseSequence = client.producerId == RECORD_BATCH_PRODUCER_ID_NONE ? RECORD_BATCH_BASE_SEQUENCE_NONE : - client.baseSequence; - - final RecordBatchFW recordBatch = recordBatchRW.wrap(encodeBuffer, encodeProgress, encodeLimit) - .baseOffset(0) - .length(recordBatchLength) - .leaderEpoch(-1) - .magic(RECORD_BATCH_MAGIC) - .crc(0) - .attributes(attributes) - .lastOffsetDelta(encodeableRecordCount - 1) - .firstTimestamp(encodeableRecordBatchTimestamp) - .maxTimestamp(encodeableRecordBatchTimestampMax) - .producerId(client.producerId) - .producerEpoch(client.producerEpoch) - .baseSequence(baseSequence) - .recordCount(encodeableRecordCount) - .build(); - - encodeProgress = recordBatch.limit(); - assert encodeProgress <= PRODUCE_REQUEST_RECORDS_OFFSET_MAX; final int encodeSizeOf = encodeProgress - encodeOffset; final int requestId = nextRequestId++; - final int requestSize = encodeSizeOf - FIELD_OFFSET_API_KEY + encodeableRecordBytes; + + final int requestSize = encodeSizeOf - FIELD_OFFSET_API_KEY + recordBatchLength; + + assert flushableRequestBytes == 0; + flushableRequestBytes = requestSize + FIELD_OFFSET_API_KEY; + encodedAckMode = encodeableAckMode; + encodeableAckMode = KafkaAckMode.NONE; requestHeaderRW.wrap(encodeBuffer, requestHeader.offset(), requestHeader.limit()) .length(requestSize) @@ -1944,49 +1941,56 @@ private void doEncodeProduceRequest( .clientId(requestHeader.clientId()) .build(); - if (KafkaConfiguration.DEBUG) - { - System.out.format("[client] %s[%d] PRODUCE\n", topic, partitionId); - } - - assert flushableRequestBytes == 0; - flushableRequestBytes = encodeSizeOf + encodeableRecordBytes; - encodeableRecordCount = 0; - encodeableRecordBytes = 0; - encodeableRecordBatchTimestamp = TIMESTAMP_NONE; - encodedAckMode = encodeableAckMode; - encodeableAckMode = KafkaAckMode.NONE; - client.producerId = RECORD_BATCH_PRODUCER_ID_NONE; - client.producerEpoch = RECORD_BATCH_PRODUCER_EPOCH_NONE; - client.baseSequence = RECORD_BATCH_BASE_SEQUENCE_NONE; - client.sequence = RECORD_SEQUENCE_NONE; - - assert encodeSlot != NO_SLOT; - final MutableDirectBuffer encodeSlotBuffer = encodePool.buffer(encodeSlot); + encodeCrc(); encodeSlotOffset -= encodeSizeOf; assert encodeSlotOffset >= 0; encodeSlotBuffer.putBytes(encodeSlotOffset, encodeBuffer, encodeOffset, encodeSizeOf); - final ByteBuffer encodeSlotByteBuffer = encodePool.byteBuffer(encodeSlot); - final int encodeSlotBytePosition = encodeSlotByteBuffer.position(); - final int partialValueSize = encodeableRecordBytesDeferred > 0 ? encodeableRecordValueBytes : 0; - encodeSlotByteBuffer.limit(encodeSlotBytePosition + encodeSlotLimit - partialValueSize); - encodeSlotByteBuffer.position(encodeSlotBytePosition + encodeSlotOffset + crcLimit); + doNetworkData(traceId, budgetId, EMPTY_BUFFER, 0, 0); - final CRC32C crc = crc32c; - crc.reset(); - crc.update(encodeSlotByteBuffer); + decoder = decodeProduceResponse; + } - long checksum = crc.getValue(); - if (encodeableRecordBytesDeferred > 0) + private void encodeCrc() + { + final MutableDirectBuffer encodeBuffer = writeBuffer; + final MutableDirectBuffer encodeSlotBuffer = encodePool.buffer(encodeSlot); + + final int encodeLimit = encodeSlotLimit; + int encodeProgress = encodeSlotOffset; + + while (encodeProgress < encodeLimit) { - checksum = computeChecksum(encodeBuffer, encodeLimit, encodeProgress, encodeSlotBuffer, checksum); - } - encodeSlotBuffer.putInt(encodeSlotOffset + crcOffset, (int) checksum, BIG_ENDIAN); - doNetworkData(traceId, budgetId, EMPTY_BUFFER, 0, 0); + final RecordBatchFW recordBatch = recordBatchRO.wrap(encodeSlotBuffer, encodeProgress, encodeLimit); - decoder = decodeProduceResponse; + final int recordBatchLength = RecordBatchFW.FIELD_OFFSET_LEADER_EPOCH + recordBatch.length(); + final int recordBatchLimit = recordBatch.offset() + recordBatchLength; + + final int crcOffset = recordBatch.offset() + RecordBatchFW.FIELD_OFFSET_CRC; + final int crcDataOffset = recordBatch.offset() + RecordBatchFW.FIELD_OFFSET_ATTRIBUTES; + final int crcDataLimit = recordBatchLimit <= encodeLimit ? recordBatchLimit : recordBatch.limit(); + + final ByteBuffer encodeSlotByteBuffer = encodePool.byteBuffer(encodeSlot); + final int encodePosition = encodeSlotByteBuffer.position(); + encodeSlotByteBuffer.position(encodePosition + crcDataOffset); + encodeSlotByteBuffer.limit(encodePosition + crcDataLimit); + + final CRC32C crc = crc32c; + crc.reset(); + crc.update(encodeSlotByteBuffer); + + long checksum = crc.getValue(); + + if (crcDataLimit == recordBatch.limit()) + { + checksum = computeChecksum(encodeBuffer, encodeLimit, encodeProgress, encodeSlotBuffer, checksum); + } + + encodeSlotBuffer.putInt(crcOffset, (int) checksum, BIG_ENDIAN); + + encodeProgress += recordBatchLength; + } } private long computeChecksum( diff --git a/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/produce/message.value/client.rpt b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/produce/message.value/client.rpt index 2ef11da015..b575b09d4b 100644 --- a/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/produce/message.value/client.rpt +++ b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/produce/message.value/client.rpt @@ -15,7 +15,6 @@ # property deltaMillis 0L -property newTimestamp ${kafka:timestamp() + deltaMillis} connect "zilla://streams/app0" option zilla:window 8192 @@ -73,7 +72,7 @@ read zilla:begin.ext ${kafka:beginEx() write zilla:data.ext ${kafka:dataEx() .typeId(zilla:id("kafka")) .produce() - .timestamp(newTimestamp) + .timestamp(1716424650323) .build() .build()} write "Hello, world" diff --git a/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/network/produce.v3/message.value/client.rpt b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/network/produce.v3/message.value/client.rpt index 28f774653d..c04f2d37a5 100644 --- a/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/network/produce.v3/message.value/client.rpt +++ b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/network/produce.v3/message.value/client.rpt @@ -98,11 +98,11 @@ write 125 # size 68 # length -1 [0x02] - 0x4e8723aa + 0x11d1815d 0s 0 # last offset delta - ${newTimestamp} # first timestamp - ${newTimestamp} # last timestamp + 1716424650323L # first timestamp + 1716424650323L # last timestamp -1L -1s -1 diff --git a/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/network/produce.v3/message.value/server.rpt b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/network/produce.v3/message.value/server.rpt index 4489e563ac..cd06fbdd02 100644 --- a/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/network/produce.v3/message.value/server.rpt +++ b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/network/produce.v3/message.value/server.rpt @@ -16,9 +16,6 @@ property networkAcceptWindow 8192 -property deltaMillis 0L -property newTimestamp ${kafka:timestamp() + deltaMillis} - accept "zilla://streams/net0" option zilla:window ${networkAcceptWindow} option zilla:transmission "duplex" @@ -94,11 +91,11 @@ read 125 68 # length -1 [0x02] - [0..4] + 0x11d1815d 0s 0 # last offset delta - (long:timestamp) # first timestamp - ${timestamp} # last timestamp + 1716424650323L # first timestamp + 1716424650323L # last timestamp -1L -1s -1 From 78878be83c68f9bed14054b2716ebbd508fdd399 Mon Sep 17 00:00:00 2001 From: bmaidics Date: Mon, 27 May 2024 20:31:59 +0200 Subject: [PATCH 5/8] Set decoder to ignoreAll after session is taken over by other MQTT client (#1045) --- .../internal/stream/MqttServerFactory.java | 7 +- .../internal/stream/server/v5/PublishIT.java | 10 +++ .../publish.session.takeover/client.rpt | 73 ++++++++++++++++++ .../publish.session.takeover/server.rpt | 74 +++++++++++++++++++ .../v5/publish.session.takeover/client.rpt | 54 ++++++++++++++ .../v5/publish.session.takeover/server.rpt | 56 ++++++++++++++ .../mqtt/streams/application/PublishIT.java | 9 +++ .../mqtt/streams/network/v5/PublishIT.java | 9 +++ 8 files changed, 289 insertions(+), 3 deletions(-) create mode 100644 specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/publish.session.takeover/client.rpt create mode 100644 specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/publish.session.takeover/server.rpt create mode 100644 specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/publish.session.takeover/client.rpt create mode 100644 specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/publish.session.takeover/server.rpt diff --git a/runtime/binding-mqtt/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/MqttServerFactory.java b/runtime/binding-mqtt/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/MqttServerFactory.java index 04e48899e3..5eac1db3ac 100644 --- a/runtime/binding-mqtt/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/MqttServerFactory.java +++ b/runtime/binding-mqtt/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/MqttServerFactory.java @@ -3172,7 +3172,7 @@ private MqttPublishStream resolvePublishStream( final long resolvedId = resolved.id; stream = publishes.computeIfAbsent(topicKey, s -> - new MqttPublishStream(routedId, resolvedId, topic, qos, binding.supplyModelConfig(topic))); + new MqttPublishStream(routedId, resolvedId, topic, topicKey, binding.supplyModelConfig(topic))); stream.doPublishBegin(traceId, affinity, qos); } else @@ -4890,6 +4890,7 @@ private void closeStreams( { session.cleanupEnd(traceId); } + decoder = decodeIgnoreAll; } private void cleanupBudgetCreditor() @@ -5669,7 +5670,7 @@ private class MqttPublishStream long originId, long routedId, String topic, - int qos, + long topicKey, ModelConfig config) { this.originId = originId; @@ -5677,7 +5678,7 @@ private class MqttPublishStream this.initialId = supplyInitialId.applyAsLong(routedId); this.replyId = supplyReplyId.applyAsLong(initialId); this.topic = topic; - this.topicKey = topicKey(topic, qos); + this.topicKey = topicKey; this.contentType = config != null ? supplyValidator.apply(config) : null; } diff --git a/runtime/binding-mqtt/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/server/v5/PublishIT.java b/runtime/binding-mqtt/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/server/v5/PublishIT.java index 3f1727c378..2b67a301b2 100644 --- a/runtime/binding-mqtt/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/server/v5/PublishIT.java +++ b/runtime/binding-mqtt/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/server/v5/PublishIT.java @@ -66,6 +66,16 @@ public void shouldPublishOneMessage() throws Exception k3po.finish(); } + @Test + @Configuration("server.yaml") + @Specification({ + "${net}/publish.session.takeover/client", + "${app}/publish.session.takeover/server"}) + public void shouldPublishAfterSessionTakeover() throws Exception + { + k3po.finish(); + } + @Test @Configuration("server.yaml") @Specification({ diff --git a/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/publish.session.takeover/client.rpt b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/publish.session.takeover/client.rpt new file mode 100644 index 0000000000..7d1bc307ab --- /dev/null +++ b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/publish.session.takeover/client.rpt @@ -0,0 +1,73 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +connect "zilla://streams/app0" + option zilla:window 8192 + option zilla:transmission "duplex" + +write zilla:begin.ext ${mqtt:beginEx() + .typeId(zilla:id("mqtt")) + .session() + .flags("CLEAN_START") + .clientId("client") + .build() + .build()} + +read zilla:begin.ext ${mqtt:matchBeginEx() + .typeId(zilla:id("mqtt")) + .session() + .flags("CLEAN_START") + .subscribeQosMax(2) + .publishQosMax(2) + .packetSizeMax(66560) + .capabilities("RETAIN", "WILDCARD", "SUBSCRIPTION_IDS", "SHARED_SUBSCRIPTIONS") + .clientId("client") + .build() + .build()} + +connected + +read zilla:data.empty +read notify RECEIVED_SESSION_STATE + +read notify SENT_FIRST_MESSAGE +read closed + + +connect await RECEIVED_SESSION_STATE + "zilla://streams/app0" + option zilla:window 8192 + option zilla:transmission "duplex" + +write zilla:begin.ext ${mqtt:beginEx() + .typeId(zilla:id("mqtt")) + .publish() + .clientId("client") + .topic("sensor/one") + .build() + .build()} + +connected + +write zilla:data.ext ${mqtt:dataEx() + .typeId(zilla:id("mqtt")) + .publish() + .build() + .build()} + +write "message1" + +write close diff --git a/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/publish.session.takeover/server.rpt b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/publish.session.takeover/server.rpt new file mode 100644 index 0000000000..edd9548c1a --- /dev/null +++ b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/publish.session.takeover/server.rpt @@ -0,0 +1,74 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +accept "zilla://streams/app0" + option zilla:window 8192 + option zilla:transmission "duplex" + +accepted + +read zilla:begin.ext ${mqtt:matchBeginEx() + .typeId(zilla:id("mqtt")) + .session() + .flags("CLEAN_START") + .clientId("client") + .build() + .build()} + +write zilla:begin.ext ${mqtt:beginEx() + .typeId(zilla:id("mqtt")) + .session() + .flags("CLEAN_START") + .subscribeQosMax(2) + .publishQosMax(2) + .packetSizeMax(66560) + .capabilities("RETAIN", "WILDCARD", "SUBSCRIPTION_IDS", "SHARED_SUBSCRIPTIONS") + .clientId("client") + .build() + .build()} + +connected + +write zilla:data.empty +write flush + +write await RECEIVED_FIRST_MESSAGE +write close +write notify SENT_CLOSE + + +accepted + +read zilla:begin.ext ${mqtt:matchBeginEx() + .typeId(zilla:id("mqtt")) + .publish() + .clientId("client") + .topic("sensor/one") + .build() + .build()} + +connected + +read zilla:data.ext ${mqtt:matchDataEx() + .typeId(zilla:id("mqtt")) + .publish() + .build() + .build()} + +read "message1" +read notify RECEIVED_FIRST_MESSAGE + +read closed diff --git a/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/publish.session.takeover/client.rpt b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/publish.session.takeover/client.rpt new file mode 100644 index 0000000000..d76049a711 --- /dev/null +++ b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/publish.session.takeover/client.rpt @@ -0,0 +1,54 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +connect "zilla://streams/net0" + option zilla:window 8192 + option zilla:transmission "duplex" + option zilla:byteorder "network" + +connected + +write [0x10 0x18] # CONNECT + [0x00 0x04] "MQTT" # protocol name + [0x05] # protocol version + [0x02] # flags = clean start + [0x00 0x3c] # keep alive = 60s + [0x05] # properties + [0x27] 66560 # maximum packet size = 66560 + [0x00 0x06] "client" # client id + +read [0x20 0x03] # CONNACK + [0x00] # flags = none + [0x00] # reason code + [0x00] # properties + +write [0x30 0x15] # PUBLISH + [0x00 0x0a] "sensor/one" # topic name + [0x00] # properties + "message1" # payload + +write await SENT_CLOSE +write [0x30 0x11] # PUBLISH + [0x00 0x0a] "sensor/one" # topic name + [0x00] # properties +write "test" # payload + +read [0xe0 0x02] # DISCONNECT + [0x8e] # session taken over + [0x00] # properties = none + +read closed +write close diff --git a/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/publish.session.takeover/server.rpt b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/publish.session.takeover/server.rpt new file mode 100644 index 0000000000..8d595ba2b2 --- /dev/null +++ b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/publish.session.takeover/server.rpt @@ -0,0 +1,56 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +accept "zilla://streams/net0" + option zilla:window 8192 + option zilla:transmission "duplex" + option zilla:byteorder "network" + +accepted +connected + +read [0x10 0x18] # CONNECT + [0x00 0x04] "MQTT" # protocol name + [0x05] # protocol version + [0x02] # flags = clean start + [0x00 0x3c] # keep alive = 60s + [0x05] # properties + [0x27] 66560 # maximum packet size = 66560 + [0x00 0x06] "client" # client id + +write [0x20 0x03] # CONNACK + [0x00] # flags = none + [0x00] # reason code + [0x00] # properties = none + +read [0x30 0x15] # PUBLISH + [0x00 0x0a] "sensor/one" # topic name + [0x00] # properties + "message1" # payload + +read notify SENT_CLOSE +read [0x30 0x11] # PUBLISH + [0x00 0x0a] "sensor/one" # topic name + [0x00] # properties + "test" # payload + +write [0xe0 0x02] # DISCONNECT + [0x8e] # session taken over + [0x00] # properties = none + +write close +read closed + diff --git a/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/application/PublishIT.java b/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/application/PublishIT.java index 69125918fc..ed026cb317 100644 --- a/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/application/PublishIT.java +++ b/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/application/PublishIT.java @@ -63,6 +63,15 @@ public void shouldSendMultipleMessages() throws Exception k3po.finish(); } + @Test + @Specification({ + "${app}/publish.session.takeover/client", + "${app}/publish.session.takeover/server"}) + public void shouldSendMessageAfterSessionTakeover() throws Exception + { + k3po.finish(); + } + @Test @Specification({ "${app}/publish.multiple.clients/client", diff --git a/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/PublishIT.java b/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/PublishIT.java index 39d82f3648..fa791d4288 100644 --- a/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/PublishIT.java +++ b/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/PublishIT.java @@ -63,6 +63,15 @@ public void shouldSendMultipleMessages() throws Exception k3po.finish(); } + @Test + @Specification({ + "${net}/publish.session.takeover/client", + "${net}/publish.session.takeover/server"}) + public void shouldSendMessageAfterSessionTakeover() throws Exception + { + k3po.finish(); + } + @Test @Specification({ "${net}/publish.multiple.clients/client", From 2429256627e8e461c0649d92ea6a47983d1ce1fe Mon Sep 17 00:00:00 2001 From: bmaidics Date: Tue, 28 May 2024 04:13:52 +0200 Subject: [PATCH 6/8] Add detection of non-compacted session topic (#1044) --- .../command/dump/internal/airline/zilla.lua | 5 + ...licationIT_shouldFetchMergedFilterSync.txt | 36 +++-- .../internal/stream/KafkaMergedFactory.java | 43 ++++- .../kafka/internal/stream/CacheMergedIT.java | 10 ++ .../kafka/internal/MqttKafkaEventContext.java | 74 +++++++++ .../internal/MqttKafkaEventFormatter.java | 65 ++++++++ .../MqttKafkaEventFormatterFactory.java | 34 ++++ .../stream/MqttKafkaSessionFactory.java | 68 +++++++- .../src/main/moditect/module-info.java | 2 + ...time.engine.event.EventFormatterFactorySpi | 1 + .../stream/MqttKafkaSessionProxyIT.java | 13 ++ .../internal/stream/server/v4/SessionIT.java | 10 ++ .../internal/stream/server/v5/SessionIT.java | 20 +++ .../kafka/internal/KafkaFunctions.java | 10 ++ .../main/resources/META-INF/zilla/kafka.idl | 1 + .../client.rpt | 42 +++++ .../server.rpt | 47 ++++++ .../client.rpt | 152 ++++++++++++++++++ .../server.rpt | 151 +++++++++++++++++ .../kafka/streams/application/MergedIT.java | 19 +++ .../resources/META-INF/zilla/mqtt_kafka.idl | 18 +++ .../mqtt/kafka/config/proxy.log.event.yaml | 37 +++++ .../client.rpt | 113 +++++++++++++ .../server.rpt | 108 +++++++++++++ .../client.rpt | 46 ++++++ .../server.rpt | 49 ++++++ .../binding/mqtt/kafka/streams/KafkaIT.java | 9 ++ .../binding/mqtt/kafka/streams/MqttIT.java | 9 ++ .../client.rpt | 44 +++++ .../server.rpt | 47 ++++++ .../client.rpt | 79 +++++++++ .../server.rpt | 80 +++++++++ .../client.rpt | 35 ++++ .../server.rpt | 36 +++++ .../client.rpt | 37 +++++ .../server.rpt | 38 +++++ .../mqtt/streams/application/SessionIT.java | 18 +++ .../mqtt/streams/network/v4/SessionIT.java | 9 ++ .../mqtt/streams/network/v5/SessionIT.java | 9 ++ 39 files changed, 1606 insertions(+), 18 deletions(-) create mode 100644 runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/MqttKafkaEventContext.java create mode 100644 runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/MqttKafkaEventFormatter.java create mode 100644 runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/MqttKafkaEventFormatterFactory.java create mode 100644 runtime/binding-mqtt-kafka/src/main/resources/META-INF/services/io.aklivity.zilla.runtime.engine.event.EventFormatterFactorySpi create mode 100644 specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/merged.produce.and.fetch.get.cleanup.policy/client.rpt create mode 100644 specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/merged.produce.and.fetch.get.cleanup.policy/server.rpt create mode 100644 specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.produce.and.fetch.get.cleanup.policy/client.rpt create mode 100644 specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.produce.and.fetch.get.cleanup.policy/server.rpt create mode 100644 specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/config/proxy.log.event.yaml create mode 100644 specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/kafka/session.reject.non.compacted.sessions.topic/client.rpt create mode 100644 specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/kafka/session.reject.non.compacted.sessions.topic/server.rpt create mode 100644 specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/mqtt/session.reject.non.compacted.sessions.topic/client.rpt create mode 100644 specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/mqtt/session.reject.non.compacted.sessions.topic/server.rpt create mode 100644 specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.reject.non.compacted.sessions.topic/client.rpt create mode 100644 specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.reject.non.compacted.sessions.topic/server.rpt create mode 100644 specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.subscribe.invalid.state/client.rpt create mode 100644 specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.subscribe.invalid.state/server.rpt create mode 100644 specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v4/session.reject.non.compacted.sessions.topic/client.rpt create mode 100644 specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v4/session.reject.non.compacted.sessions.topic/server.rpt create mode 100644 specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/session.reject.non.compacted.sessions.topic/client.rpt create mode 100644 specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/session.reject.non.compacted.sessions.topic/server.rpt diff --git a/incubator/command-dump/src/main/resources/io/aklivity/zilla/runtime/command/dump/internal/airline/zilla.lua b/incubator/command-dump/src/main/resources/io/aklivity/zilla/runtime/command/dump/internal/airline/zilla.lua index 9cf59d0b9f..60bf401234 100644 --- a/incubator/command-dump/src/main/resources/io/aklivity/zilla/runtime/command/dump/internal/airline/zilla.lua +++ b/incubator/command-dump/src/main/resources/io/aklivity/zilla/runtime/command/dump/internal/airline/zilla.lua @@ -2391,6 +2391,11 @@ function handle_kafka_begin_merged_extension(buffer, offset, ext_subtree) local ack_mode = kafka_ext_ack_modes[slice_ack_mode_id:le_int()] ext_subtree:add_le(fields.kafka_ext_ack_mode_id, slice_ack_mode_id) ext_subtree:add(fields.kafka_ext_ack_mode, ack_mode) + -- configs + local configs_offset = ack_mode_offset + ack_mode_length + local configs_length = resolve_length_of_array(buffer, configs_offset) + dissect_and_add_kafka_config_struct_array(buffer, configs_offset, ext_subtree, fields.kafka_ext_config_array_length, + fields.kafka_ext_config_array_size) end function handle_kafka_begin_init_producer_id_extension(buffer, offset, ext_subtree) diff --git a/incubator/command-dump/src/test/resources/io/aklivity/zilla/runtime/command/dump/internal/KafkaMergedApplicationIT_shouldFetchMergedFilterSync.txt b/incubator/command-dump/src/test/resources/io/aklivity/zilla/runtime/command/dump/internal/KafkaMergedApplicationIT_shouldFetchMergedFilterSync.txt index 4c91348e1e..f2a4c27876 100644 --- a/incubator/command-dump/src/test/resources/io/aklivity/zilla/runtime/command/dump/internal/KafkaMergedApplicationIT_shouldFetchMergedFilterSync.txt +++ b/incubator/command-dump/src/test/resources/io/aklivity/zilla/runtime/command/dump/internal/KafkaMergedApplicationIT_shouldFetchMergedFilterSync.txt @@ -1,7 +1,7 @@ -Frame 1: 303 bytes on wire (2424 bits), 303 bytes captured (2424 bits) +Frame 1: 311 bytes on wire (2488 bits), 311 bytes captured (2488 bits) Ethernet II, Src: Send_00 (20:53:45:4e:44:00), Dst: Receive_00 (20:52:45:43:56:00) Internet Protocol Version 6, Src: fe80::3f3f:0:0:2, Dst: fe80::3f3f:0:0:3 -Transmission Control Protocol, Src Port: 0, Dst Port: 7114, Seq: 0, Ack: 1, Len: 229 +Transmission Control Protocol, Src Port: 0, Dst Port: 7114, Seq: 0, Ack: 1, Len: 237 Zilla Frame Frame Type ID: 0x00000001 Frame Type: BEGIN @@ -68,18 +68,21 @@ Zilla Frame Delta Type: NONE (0) Ack Mode ID: -1 Ack Mode: IN_SYNC_REPLICAS + Configs (0 items) + Length: 4 + Size: 0 Frame 2: 211 bytes on wire (1688 bits), 211 bytes captured (1688 bits) Ethernet II, Src: Send_00 (20:53:45:4e:44:00), Dst: Receive_00 (20:52:45:43:56:00) Internet Protocol Version 6, Src: fe80::3f3f:0:0:2, Dst: fe80::3f3f:0:0:3 -Transmission Control Protocol, Src Port: 0, Dst Port: 7114, Seq: 229, Ack: 1, Len: 137 +Transmission Control Protocol, Src Port: 0, Dst Port: 7114, Seq: 237, Ack: 1, Len: 137 Zilla Frame Frame Type ID: 0x40000002 Frame Type: WINDOW Protocol Type ID: 0x00000000 Protocol Type: Worker: 0 - Offset: 0x000000c0 + Offset: 0x000000c8 Origin ID: 0x0000000100000002 Origin Namespace: test Origin Binding: app0 @@ -103,17 +106,17 @@ Zilla Frame Progress: 0 Progress/Maximum: 0/8192 -Frame 3: 303 bytes on wire (2424 bits), 303 bytes captured (2424 bits) +Frame 3: 311 bytes on wire (2488 bits), 311 bytes captured (2488 bits) Ethernet II, Src: Send_00 (20:53:45:4e:44:00), Dst: Receive_00 (20:52:45:43:56:00) Internet Protocol Version 6, Src: fe80::3f3f:0:0:3, Dst: fe80::3f3f:0:0:2 -Transmission Control Protocol, Src Port: 7114, Dst Port: 0, Seq: 1, Ack: 366, Len: 229 +Transmission Control Protocol, Src Port: 7114, Dst Port: 0, Seq: 1, Ack: 374, Len: 237 Zilla Frame Frame Type ID: 0x00000001 Frame Type: BEGIN Protocol Type ID: 0x00000000 Protocol Type: Worker: 0 - Offset: 0x00000120 + Offset: 0x00000128 Origin ID: 0x0000000100000002 Origin Namespace: test Origin Binding: app0 @@ -173,18 +176,21 @@ Zilla Frame Delta Type: NONE (0) Ack Mode ID: -1 Ack Mode: IN_SYNC_REPLICAS + Configs (0 items) + Length: 4 + Size: 0 Frame 4: 320 bytes on wire (2560 bits), 320 bytes captured (2560 bits) Ethernet II, Src: Send_00 (20:53:45:4e:44:00), Dst: Receive_00 (20:52:45:43:56:00) Internet Protocol Version 6, Src: fe80::3f3f:0:0:3, Dst: fe80::3f3f:0:0:2 -Transmission Control Protocol, Src Port: 7114, Dst Port: 0, Seq: 230, Ack: 366, Len: 246 +Transmission Control Protocol, Src Port: 7114, Dst Port: 0, Seq: 238, Ack: 374, Len: 246 Zilla Frame Frame Type ID: 0x00000005 Frame Type: FLUSH Protocol Type ID: 0x00000000 Protocol Type: Worker: 0 - Offset: 0x000001e0 + Offset: 0x000001f0 Origin ID: 0x0000000100000002 Origin Namespace: test Origin Binding: app0 @@ -247,14 +253,14 @@ Zilla Frame Frame 5: 211 bytes on wire (1688 bits), 211 bytes captured (1688 bits) Ethernet II, Src: Send_00 (20:53:45:4e:44:00), Dst: Receive_00 (20:52:45:43:56:00) Internet Protocol Version 6, Src: fe80::3f3f:0:0:3, Dst: fe80::3f3f:0:0:2 -Transmission Control Protocol, Src Port: 7114, Dst Port: 0, Seq: 476, Ack: 366, Len: 137 +Transmission Control Protocol, Src Port: 7114, Dst Port: 0, Seq: 484, Ack: 374, Len: 137 Zilla Frame Frame Type ID: 0x40000002 Frame Type: WINDOW Protocol Type ID: 0x00000000 Protocol Type: Worker: 0 - Offset: 0x000002b0 + Offset: 0x000002c0 Origin ID: 0x0000000100000002 Origin Namespace: test Origin Binding: app0 @@ -281,14 +287,14 @@ Zilla Frame Frame 6: 194 bytes on wire (1552 bits), 194 bytes captured (1552 bits) Ethernet II, Src: Send_00 (20:53:45:4e:44:00), Dst: Receive_00 (20:52:45:43:56:00) Internet Protocol Version 6, Src: fe80::3f3f:0:0:2, Dst: fe80::3f3f:0:0:3 -Transmission Control Protocol, Src Port: 0, Dst Port: 7114, Seq: 366, Ack: 613, Len: 120 +Transmission Control Protocol, Src Port: 0, Dst Port: 7114, Seq: 374, Ack: 621, Len: 120 Zilla Frame Frame Type ID: 0x00000003 Frame Type: END Protocol Type ID: 0x00000000 Protocol Type: Worker: 0 - Offset: 0x00000310 + Offset: 0x00000320 Origin ID: 0x0000000100000002 Origin Namespace: test Origin Binding: app0 @@ -309,14 +315,14 @@ Zilla Frame Frame 7: 194 bytes on wire (1552 bits), 194 bytes captured (1552 bits) Ethernet II, Src: Send_00 (20:53:45:4e:44:00), Dst: Receive_00 (20:52:45:43:56:00) Internet Protocol Version 6, Src: fe80::3f3f:0:0:3, Dst: fe80::3f3f:0:0:2 -Transmission Control Protocol, Src Port: 7114, Dst Port: 0, Seq: 613, Ack: 486, Len: 120 +Transmission Control Protocol, Src Port: 7114, Dst Port: 0, Seq: 621, Ack: 494, Len: 120 Zilla Frame Frame Type ID: 0x00000003 Frame Type: END Protocol Type ID: 0x00000000 Protocol Type: Worker: 0 - Offset: 0x00000360 + Offset: 0x00000370 Origin ID: 0x0000000100000002 Origin Namespace: test Origin Binding: app0 diff --git a/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaMergedFactory.java b/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaMergedFactory.java index 9280cfef77..21c91c6eb3 100644 --- a/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaMergedFactory.java +++ b/runtime/binding-kafka/src/main/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/KafkaMergedFactory.java @@ -16,6 +16,7 @@ package io.aklivity.zilla.runtime.binding.kafka.internal.stream; import static io.aklivity.zilla.runtime.binding.kafka.internal.types.KafkaCapabilities.FETCH_ONLY; +import static io.aklivity.zilla.runtime.binding.kafka.internal.types.KafkaCapabilities.PRODUCE_AND_FETCH; import static io.aklivity.zilla.runtime.binding.kafka.internal.types.KafkaCapabilities.PRODUCE_ONLY; import static io.aklivity.zilla.runtime.binding.kafka.internal.types.KafkaOffsetType.HISTORICAL; import static io.aklivity.zilla.runtime.binding.kafka.internal.types.KafkaOffsetType.LIVE; @@ -39,6 +40,7 @@ import org.agrona.collections.MutableInteger; import org.agrona.collections.MutableLong; import org.agrona.collections.MutableReference; +import org.agrona.collections.Object2ObjectHashMap; import org.agrona.concurrent.UnsafeBuffer; import io.aklivity.zilla.runtime.binding.kafka.internal.KafkaBinding; @@ -1033,6 +1035,7 @@ private final class KafkaMergedStream private final KafkaIsolation isolation; private final KafkaDeltaType deltaType; private final KafkaAckMode ackMode; + private final Object2ObjectHashMap configs; private KafkaOffsetType maximumOffset; private List filters; @@ -1107,6 +1110,7 @@ private final class KafkaMergedStream this.isolation = isolation; this.deltaType = deltaType; this.ackMode = ackMode; + this.configs = new Object2ObjectHashMap<>(); } private void onMergedMessage( @@ -1618,7 +1622,7 @@ else if (capabilities == PRODUCE_ONLY) else { doBegin(sender, originId, routedId, replyId, replySeq, replyAck, replyMax, - traceId, authorization, affinity, EMPTY_EXTENSION); + traceId, authorization, affinity, beginExToKafka(beginExToKafkaMergedProduceAndFetch())); } doUnmergedFetchReplyWindowsIfNecessary(traceId); @@ -1659,6 +1663,13 @@ private Consumer beginExToKafkaMergedFetchOnly() .latestOffset(v) .metadata(metadataRW.length() > 0 ? metadataRW.toString() : null)); }); + String cleanupPolicy = configs.get(CONFIG_NAME_CLEANUP_POLICY); + if (cleanupPolicy != null) + { + builder.configsItem(c -> + c.name(CONFIG_NAME_CLEANUP_POLICY) + .value(cleanupPolicy)); + } }; } @@ -1668,6 +1679,29 @@ private Consumer beginExToKafkaMergedProduceOnly() { builder.capabilities(c -> c.set(PRODUCE_ONLY)).topic(topic); leadersByPartitionId.intForEach((k, v) -> builder.partitionsItem(i -> i.partitionId(k))); + String cleanupPolicy = configs.get(CONFIG_NAME_CLEANUP_POLICY); + if (cleanupPolicy != null) + { + builder.configsItem(c -> + c.name(CONFIG_NAME_CLEANUP_POLICY) + .value(cleanupPolicy)); + } + }; + } + + private Consumer beginExToKafkaMergedProduceAndFetch() + { + return builder -> + { + builder.capabilities(c -> c.set(PRODUCE_AND_FETCH)) + .topic(topic); + String cleanupPolicy = configs.get(CONFIG_NAME_CLEANUP_POLICY); + if (cleanupPolicy != null) + { + builder.configsItem(c -> + c.name(CONFIG_NAME_CLEANUP_POLICY) + .value(cleanupPolicy)); + } }; } @@ -1952,6 +1986,13 @@ private void onTopicConfigChanged( long traceId, ArrayFW configs) { + configs.forEach(c -> + { + if (c.name().equals(CONFIG_NAME_CLEANUP_POLICY)) + { + this.configs.put(CONFIG_NAME_CLEANUP_POLICY, c.value().asString()); + } + }); metaStream.doMetaInitialBeginIfNecessary(traceId); } diff --git a/runtime/binding-kafka/src/test/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/CacheMergedIT.java b/runtime/binding-kafka/src/test/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/CacheMergedIT.java index 05fda6b413..9708c0ce06 100644 --- a/runtime/binding-kafka/src/test/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/CacheMergedIT.java +++ b/runtime/binding-kafka/src/test/java/io/aklivity/zilla/runtime/binding/kafka/internal/stream/CacheMergedIT.java @@ -80,6 +80,16 @@ public void shouldFetchMergedMessagesWithHeaderFilterAfterCompaction() throws Ex k3po.finish(); } + @Test + @Configuration("cache.options.merged.yaml") + @Specification({ + "${app}/merged.produce.and.fetch.get.cleanup.policy/client", + "${app}/unmerged.produce.and.fetch.get.cleanup.policy/server"}) + public void shouldProduceAndFetchMergedGetCompaction() throws Exception + { + k3po.finish(); + } + @Ignore("requires k3po parallel reads") @Test @Configuration("cache.options.merged.yaml") diff --git a/runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/MqttKafkaEventContext.java b/runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/MqttKafkaEventContext.java new file mode 100644 index 0000000000..f62228de03 --- /dev/null +++ b/runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/MqttKafkaEventContext.java @@ -0,0 +1,74 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.runtime.binding.mqtt.kafka.internal; + +import static io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.event.MqttKafkaEventType.NON_COMPACT_SESSIONS_TOPIC; + +import java.nio.ByteBuffer; +import java.time.Clock; + +import org.agrona.concurrent.AtomicBuffer; +import org.agrona.concurrent.UnsafeBuffer; + +import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.String16FW; +import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.event.EventFW; +import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.event.MqttKafkaEventExFW; +import io.aklivity.zilla.runtime.engine.EngineContext; +import io.aklivity.zilla.runtime.engine.binding.function.MessageConsumer; + +public class MqttKafkaEventContext +{ + private static final int EVENT_BUFFER_CAPACITY = 2048; + + private final AtomicBuffer eventBuffer = new UnsafeBuffer(ByteBuffer.allocate(EVENT_BUFFER_CAPACITY)); + private final AtomicBuffer extensionBuffer = new UnsafeBuffer(ByteBuffer.allocate(EVENT_BUFFER_CAPACITY)); + private final EventFW.Builder eventRW = new EventFW.Builder(); + private final MqttKafkaEventExFW.Builder mqttKafkaEventExRW = new MqttKafkaEventExFW.Builder(); + private final int mqttTypeId; + private final int nonCompactSessionsTopicEventId; + private final MessageConsumer eventWriter; + private final Clock clock; + + public MqttKafkaEventContext( + EngineContext context) + { + this.mqttTypeId = context.supplyTypeId(MqttKafkaBinding.NAME); + this.nonCompactSessionsTopicEventId = context.supplyEventId("binding.mqtt.kafka.non.compact.sessions.topic"); + this.eventWriter = context.supplyEventWriter(); + this.clock = context.clock(); + } + + public void onMqttConnectionReset( + long traceId, + long bindingId, + String16FW reason) + { + MqttKafkaEventExFW extension = mqttKafkaEventExRW + .wrap(extensionBuffer, 0, extensionBuffer.capacity()) + .nonCompactSessionsTopic(e -> e + .typeId(NON_COMPACT_SESSIONS_TOPIC.value()) + .reason(reason)) + .build(); + EventFW event = eventRW + .wrap(eventBuffer, 0, eventBuffer.capacity()) + .id(nonCompactSessionsTopicEventId) + .timestamp(clock.millis()) + .traceId(traceId) + .namespacedId(bindingId) + .extension(extension.buffer(), extension.offset(), extension.limit()) + .build(); + eventWriter.accept(mqttTypeId, event.buffer(), event.offset(), event.limit()); + } +} diff --git a/runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/MqttKafkaEventFormatter.java b/runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/MqttKafkaEventFormatter.java new file mode 100644 index 0000000000..8c66ebbe31 --- /dev/null +++ b/runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/MqttKafkaEventFormatter.java @@ -0,0 +1,65 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.runtime.binding.mqtt.kafka.internal; + +import org.agrona.DirectBuffer; + +import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.String16FW; +import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.event.EventFW; +import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.event.MqttKafkaEventExFW; +import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.event.MqttKafkaResetMqttConnectionExFW; +import io.aklivity.zilla.runtime.engine.Configuration; +import io.aklivity.zilla.runtime.engine.event.EventFormatterSpi; + +public final class MqttKafkaEventFormatter implements EventFormatterSpi +{ + private static final String NON_COMPACT_SESSIONS_TOPIC_FORMAT = "NON COMPACT SESSIONS TOPIC - %s"; + + private final EventFW eventRO = new EventFW(); + private final MqttKafkaEventExFW mqttKafkaEventExRO = new MqttKafkaEventExFW(); + + MqttKafkaEventFormatter( + Configuration config) + { + } + + public String format( + DirectBuffer buffer, + int index, + int length) + { + final EventFW event = eventRO.wrap(buffer, index, index + length); + final MqttKafkaEventExFW extension = mqttKafkaEventExRO + .wrap(event.extension().buffer(), event.extension().offset(), event.extension().limit()); + String result = null; + switch (extension.kind()) + { + case NON_COMPACT_SESSIONS_TOPIC: + { + MqttKafkaResetMqttConnectionExFW ex = extension.nonCompactSessionsTopic(); + result = String.format(NON_COMPACT_SESSIONS_TOPIC_FORMAT, asString(ex.reason())); + break; + } + } + return result; + } + + private static String asString( + String16FW stringFW) + { + String s = stringFW.asString(); + return s == null ? "" : s; + } +} diff --git a/runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/MqttKafkaEventFormatterFactory.java b/runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/MqttKafkaEventFormatterFactory.java new file mode 100644 index 0000000000..a5b625d1c8 --- /dev/null +++ b/runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/MqttKafkaEventFormatterFactory.java @@ -0,0 +1,34 @@ +/* + * Copyright 2021-2023 Aklivity Inc + * + * Licensed under the Aklivity Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://www.aklivity.io/aklivity-community-license/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.aklivity.zilla.runtime.binding.mqtt.kafka.internal; + +import io.aklivity.zilla.runtime.engine.Configuration; +import io.aklivity.zilla.runtime.engine.event.EventFormatterFactorySpi; + +public final class MqttKafkaEventFormatterFactory implements EventFormatterFactorySpi +{ + @Override + public MqttKafkaEventFormatter create( + Configuration config) + { + return new MqttKafkaEventFormatter(config); + } + + @Override + public String type() + { + return MqttKafkaBinding.NAME; + } +} diff --git a/runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/stream/MqttKafkaSessionFactory.java b/runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/stream/MqttKafkaSessionFactory.java index b7731c36d9..7ab987ddba 100644 --- a/runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/stream/MqttKafkaSessionFactory.java +++ b/runtime/binding-mqtt-kafka/src/main/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/stream/MqttKafkaSessionFactory.java @@ -53,6 +53,7 @@ import io.aklivity.zilla.runtime.binding.mqtt.kafka.config.MqttKafkaRouteConfig; import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.InstanceId; import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.MqttKafkaConfiguration; +import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.MqttKafkaEventContext; import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.config.MqttKafkaBindingConfig; import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.config.MqttKafkaHeaderHelper; import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.stream.MqttKafkaPublishMetadata.KafkaGroup; @@ -63,6 +64,7 @@ import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.Flyweight; import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.KafkaAckMode; import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.KafkaCapabilities; +import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.KafkaConfigFW; import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.KafkaEvaluation; import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.KafkaHeaderFW; import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.KafkaKeyFW; @@ -94,6 +96,7 @@ import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.KafkaGroupBeginExFW; import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.KafkaGroupFlushExFW; import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.KafkaInitProducerIdBeginExFW; +import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.KafkaMergedBeginExFW; import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.KafkaMergedDataExFW; import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.KafkaMergedFlushExFW; import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.KafkaMetaDataExFW; @@ -167,6 +170,12 @@ public class MqttKafkaSessionFactory implements MqttKafkaStreamFactory public static final int MQTT_NOT_AUTHORIZED = 0x87; public static final int MQTT_IMPLEMENTATION_SPECIFIC_ERROR = 0x83; public static final String MQTT_INVALID_SESSION_TIMEOUT_REASON = "Invalid session expiry interval"; + public static final String16FW MQTT_NON_COMPACT_SESSIONS_TOPIC = new String16FW("Sessions Kafka topic in non-compacted"); + private static final KafkaConfigFW CONFIG_COMPACT_CLEANUP_POLICY = new KafkaConfigFW.Builder() + .wrap(new UnsafeBuffer(new byte[25]), 0, 25) + .name("cleanup.policy") + .value("compact") + .build(); static { @@ -268,6 +277,7 @@ public class MqttKafkaSessionFactory implements MqttKafkaStreamFactory private final String groupIdPrefixFormat; private final Function supplyNamespace; private final Function supplyLocalName; + private final MqttKafkaEventContext events; private String serverRef; private int reconnectAttempt; @@ -320,6 +330,7 @@ public MqttKafkaSessionFactory( this.groupIdPrefixFormat = config.groupIdPrefixFormat(); this.supplyNamespace = context::supplyNamespace; this.supplyLocalName = context::supplyLocalName; + this.events = new MqttKafkaEventContext(context); } @Override @@ -2907,7 +2918,7 @@ private void onKafkaMessage( } } - private void onKafkaBegin( + protected void onKafkaBegin( BeginFW begin) { final long sequence = begin.sequence(); @@ -3156,7 +3167,7 @@ private void doKafkaEnd( } } - private void doKafkaAbort( + protected void doKafkaAbort( long traceId, long authorization) { @@ -3415,6 +3426,59 @@ private KafkaSessionStateProxy( super(originId, routedId, delegate); } + @Override + protected void onKafkaBegin( + BeginFW begin) + { + final long sequence = begin.sequence(); + final long acknowledge = begin.acknowledge(); + final int maximum = begin.maximum(); + final long traceId = begin.traceId(); + final long authorization = begin.authorization(); + final long affinity = begin.affinity(); + final OctetsFW extension = begin.extension(); + + assert acknowledge <= sequence; + assert sequence >= replySeq; + assert acknowledge >= replyAck; + + replySeq = sequence; + replyAck = acknowledge; + replyMax = maximum; + state = MqttKafkaState.openingReply(state); + + assert replyAck <= replySeq; + + final KafkaBeginExFW kafkaBeginEx = extension.get(kafkaBeginExRO::tryWrap); + + onKafkaBegin: + { + if (kafkaBeginEx != null) + { + assert kafkaBeginEx.kind() == KafkaBeginExFW.KIND_MERGED; + final KafkaMergedBeginExFW kafkaMergedBeginEx = kafkaBeginEx.merged(); + + final KafkaConfigFW cleanupPolicyConfig = + kafkaMergedBeginEx.configs().matchFirst(c -> c.equals(CONFIG_COMPACT_CLEANUP_POLICY)); + + if (cleanupPolicyConfig == null) + { + Flyweight mqttResetEx = mqttSessionResetExRW.wrap(sessionExtBuffer, 0, sessionExtBuffer.capacity()) + .typeId(mqttTypeId) + .reasonCode(MQTT_IMPLEMENTATION_SPECIFIC_ERROR) + .build(); + delegate.doMqttWindow(authorization, traceId, 0, 0, 0); + delegate.doMqttReset(traceId, mqttResetEx); + events.onMqttConnectionReset(traceId, routedId, MQTT_NON_COMPACT_SESSIONS_TOPIC); + doKafkaAbort(traceId, authorization); + break onKafkaBegin; + } + } + + super.onKafkaBegin(begin); + } + } + @Override protected void onKafkaDataImpl( DataFW data) diff --git a/runtime/binding-mqtt-kafka/src/main/moditect/module-info.java b/runtime/binding-mqtt-kafka/src/main/moditect/module-info.java index e08083a7ce..2648cb37d9 100644 --- a/runtime/binding-mqtt-kafka/src/main/moditect/module-info.java +++ b/runtime/binding-mqtt-kafka/src/main/moditect/module-info.java @@ -30,4 +30,6 @@ provides io.aklivity.zilla.runtime.engine.config.OptionsConfigAdapterSpi with io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.config.MqttKafkaOptionsConfigAdapter; + provides io.aklivity.zilla.runtime.engine.event.EventFormatterFactorySpi + with io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.MqttKafkaEventFormatterFactory; } diff --git a/runtime/binding-mqtt-kafka/src/main/resources/META-INF/services/io.aklivity.zilla.runtime.engine.event.EventFormatterFactorySpi b/runtime/binding-mqtt-kafka/src/main/resources/META-INF/services/io.aklivity.zilla.runtime.engine.event.EventFormatterFactorySpi new file mode 100644 index 0000000000..6bb2339795 --- /dev/null +++ b/runtime/binding-mqtt-kafka/src/main/resources/META-INF/services/io.aklivity.zilla.runtime.engine.event.EventFormatterFactorySpi @@ -0,0 +1 @@ +io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.MqttKafkaEventFormatterFactory diff --git a/runtime/binding-mqtt-kafka/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/stream/MqttKafkaSessionProxyIT.java b/runtime/binding-mqtt-kafka/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/stream/MqttKafkaSessionProxyIT.java index 945b86d372..6808a4587b 100644 --- a/runtime/binding-mqtt-kafka/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/stream/MqttKafkaSessionProxyIT.java +++ b/runtime/binding-mqtt-kafka/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/kafka/internal/stream/MqttKafkaSessionProxyIT.java @@ -143,6 +143,19 @@ public void shouldSubscribeSaveSubscriptionsInSession() throws Exception k3po.finish(); } + @Test + @Configuration("proxy.log.event.yaml") + @Configure(name = WILL_AVAILABLE_NAME, value = "false") + @Configure(name = PUBLISH_MAX_QOS_NAME, value = "1") + @Configure(name = PUBLISH_MAX_QOS_NAME, value = "0") + @Specification({ + "${mqtt}/session.reject.non.compacted.sessions.topic/client", + "${kafka}/session.reject.non.compacted.sessions.topic/server"}) + public void shouldRejectSessionNonCompactedSessionsTopic() throws Exception + { + k3po.finish(); + } + @Test @Configuration("proxy.yaml") @Configure(name = WILL_AVAILABLE_NAME, value = "false") diff --git a/runtime/binding-mqtt/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/server/v4/SessionIT.java b/runtime/binding-mqtt/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/server/v4/SessionIT.java index efb11c07a2..a8360ce722 100644 --- a/runtime/binding-mqtt/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/server/v4/SessionIT.java +++ b/runtime/binding-mqtt/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/server/v4/SessionIT.java @@ -189,6 +189,16 @@ public void shouldClientTakeOverSession() throws Exception k3po.finish(); } + @Test + @Configuration("server.yaml") + @Specification({ + "${net}/session.reject.non.compacted.sessions.topic/client", + "${app}/session.reject.non.compacted.sessions.topic/server"}) + public void shouldRejectSessionNonCompactedSessionsTopic() throws Exception + { + k3po.finish(); + } + @Test @Configuration("server.route.non.default.yaml") @Specification({ diff --git a/runtime/binding-mqtt/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/server/v5/SessionIT.java b/runtime/binding-mqtt/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/server/v5/SessionIT.java index 640d6bc3d7..19fdde8dd6 100644 --- a/runtime/binding-mqtt/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/server/v5/SessionIT.java +++ b/runtime/binding-mqtt/src/test/java/io/aklivity/zilla/runtime/binding/mqtt/internal/stream/server/v5/SessionIT.java @@ -84,6 +84,16 @@ public void shouldSubscribeSaveSubscriptionsInSession() throws Exception k3po.finish(); } + @Test + @Configuration("server.yaml") + @Specification({ + "${net}/session.subscribe/client", + "${app}/session.subscribe.invalid.state/server"}) + public void shouldSubscribeInvalidSessionState() throws Exception + { + k3po.finish(); + } + @Test @Configuration("server.yaml") @Specification({ @@ -266,6 +276,16 @@ public void shouldSubscribeAndPublishToNonDefaultRoute() throws Exception k3po.finish(); } + @Test + @Configuration("server.yaml") + @Specification({ + "${net}/session.reject.non.compacted.sessions.topic/client", + "${app}/session.reject.non.compacted.sessions.topic/server"}) + public void shouldRejectSessionNonCompactedSessionsTopic() throws Exception + { + k3po.finish(); + } + @Test @Configuration("server.yaml") @Specification({ diff --git a/specs/binding-kafka.spec/src/main/java/io/aklivity/zilla/specs/binding/kafka/internal/KafkaFunctions.java b/specs/binding-kafka.spec/src/main/java/io/aklivity/zilla/specs/binding/kafka/internal/KafkaFunctions.java index c2b215b83f..692cc2c6fd 100644 --- a/specs/binding-kafka.spec/src/main/java/io/aklivity/zilla/specs/binding/kafka/internal/KafkaFunctions.java +++ b/specs/binding-kafka.spec/src/main/java/io/aklivity/zilla/specs/binding/kafka/internal/KafkaFunctions.java @@ -36,6 +36,7 @@ import io.aklivity.zilla.specs.binding.kafka.internal.types.KafkaAckMode; import io.aklivity.zilla.specs.binding.kafka.internal.types.KafkaCapabilities; import io.aklivity.zilla.specs.binding.kafka.internal.types.KafkaConditionFW; +import io.aklivity.zilla.specs.binding.kafka.internal.types.KafkaConfigFW; import io.aklivity.zilla.specs.binding.kafka.internal.types.KafkaDeltaFW; import io.aklivity.zilla.specs.binding.kafka.internal.types.KafkaDeltaType; import io.aklivity.zilla.specs.binding.kafka.internal.types.KafkaDeltaTypeFW; @@ -1165,6 +1166,14 @@ public KafkaMergedBeginExBuilder ackMode( return this; } + public KafkaMergedBeginExBuilder config( + String name, + String value) + { + mergedBeginExRW.configsItem(c -> c.name(name).value(value)); + return this; + } + public KafkaBeginExBuilder build() { final KafkaMergedBeginExFW mergedBeginEx = mergedBeginExRW.build(); @@ -5802,6 +5811,7 @@ public final class KafkaMergedBeginExMatcherBuilder private KafkaEvaluation evaluation; private KafkaAckMode ackMode; private Array32FW.Builder filtersRW; + private Array32FW.Builder configsFW; private KafkaMergedBeginExMatcherBuilder() { diff --git a/specs/binding-kafka.spec/src/main/resources/META-INF/zilla/kafka.idl b/specs/binding-kafka.spec/src/main/resources/META-INF/zilla/kafka.idl index 0377305825..dd8c53577a 100644 --- a/specs/binding-kafka.spec/src/main/resources/META-INF/zilla/kafka.idl +++ b/specs/binding-kafka.spec/src/main/resources/META-INF/zilla/kafka.idl @@ -243,6 +243,7 @@ scope kafka KafkaIsolation isolation = READ_COMMITTED; KafkaDeltaType deltaType = NONE; KafkaAckMode ackMode = IN_SYNC_REPLICAS; + KafkaConfig[] configs; } union KafkaMergedDataEx switch (uint8) diff --git a/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/merged.produce.and.fetch.get.cleanup.policy/client.rpt b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/merged.produce.and.fetch.get.cleanup.policy/client.rpt new file mode 100644 index 0000000000..2b4c6b385c --- /dev/null +++ b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/merged.produce.and.fetch.get.cleanup.policy/client.rpt @@ -0,0 +1,42 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +connect "zilla://streams/app0" + option zilla:window 16 + option zilla:transmission "duplex" + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .merged() + .capabilities("PRODUCE_AND_FETCH") + .topic("test") + .partition(0, 1) + .filter() + .header("header1", "value1") + .build() + .build() + .build()} + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .merged() + .capabilities("PRODUCE_AND_FETCH") + .topic("test") + .config("cleanup.policy", "compact") + .build() + .build()} + +connected diff --git a/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/merged.produce.and.fetch.get.cleanup.policy/server.rpt b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/merged.produce.and.fetch.get.cleanup.policy/server.rpt new file mode 100644 index 0000000000..337b0932ee --- /dev/null +++ b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/merged.produce.and.fetch.get.cleanup.policy/server.rpt @@ -0,0 +1,47 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +property deltaMillis 0L +property newTimestamp ${kafka:timestamp() + deltaMillis} + +accept "zilla://streams/app0" + option zilla:window 8192 + option zilla:transmission "duplex" + +accepted + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .merged() + .capabilities("PRODUCE_AND_FETCH") + .topic("test") + .partition(0, 1) + .filter() + .header("header1", "value1") + .build() + .build() + .build()} + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .merged() + .capabilities("PRODUCE_AND_FETCH") + .topic("test") + .config("cleanup.policy", "compact") + .build() + .build()} + +connected diff --git a/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.produce.and.fetch.get.cleanup.policy/client.rpt b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.produce.and.fetch.get.cleanup.policy/client.rpt new file mode 100644 index 0000000000..a2652e56f7 --- /dev/null +++ b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.produce.and.fetch.get.cleanup.policy/client.rpt @@ -0,0 +1,152 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +connect "zilla://streams/app1" + option zilla:window 8192 + option zilla:transmission "half-duplex" + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .describe() + .topic("test") + .config("cleanup.policy") + .config("max.message.bytes") + .config("segment.bytes") + .config("segment.index.bytes") + .config("segment.ms") + .config("retention.bytes") + .config("retention.ms") + .config("delete.retention.ms") + .config("min.compaction.lag.ms") + .config("max.compaction.lag.ms") + .config("min.cleanable.dirty.ratio") + .build() + .build()} + +connected + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .describe() + .topic("test") + .config("cleanup.policy") + .config("max.message.bytes") + .config("segment.bytes") + .config("segment.index.bytes") + .config("segment.ms") + .config("retention.bytes") + .config("retention.ms") + .config("delete.retention.ms") + .config("min.compaction.lag.ms") + .config("max.compaction.lag.ms") + .config("min.cleanable.dirty.ratio") + .build() + .build()} + +read zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .describe() + .config("cleanup.policy", "compact") + .config("max.message.bytes", 16) + .config("segment.bytes", 1073741824) + .config("segment.index.bytes", 64) + .config("segment.ms", 604800000) + .config("retention.bytes", -1) + .config("retention.ms", 604800000) + .config("delete.retention.ms", 86400000) + .config("min.compaction.lag.ms", 0) + .config("max.compaction.lag.ms", 9223372036854775807) + .config("min.cleanable.dirty.ratio", 0.0) + .build() + .build()} + +read notify RECEIVED_CONFIG + +connect await RECEIVED_CONFIG + "zilla://streams/app1" + option zilla:window 8192 + option zilla:transmission "half-duplex" + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .meta() + .topic("test") + .build() + .build()} + +connected + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .meta() + .topic("test") + .build() + .build()} + +read zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .meta() + .partition(0, 1) + .build() + .build()} +read notify PARTITION_COUNT_1 + +connect await PARTITION_COUNT_1 + "zilla://streams/app1" + option zilla:window 8192 + option zilla:transmission "half-duplex" + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(0, -2) + .build() + .build()} + +connected + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(0, 1, 2) + .build() + .build()} + + +connect await PARTITION_COUNT_1 + "zilla://streams/app1" + option zilla:window 8192 + option zilla:transmission "half-duplex" + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .produce() + .topic("test") + .partition(0) + .build() + .build()} + +connected + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .produce() + .topic("test") + .partition(0) + .build() + .build()} diff --git a/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.produce.and.fetch.get.cleanup.policy/server.rpt b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.produce.and.fetch.get.cleanup.policy/server.rpt new file mode 100644 index 0000000000..93fb0ab0ba --- /dev/null +++ b/specs/binding-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/kafka/streams/application/merged/unmerged.produce.and.fetch.get.cleanup.policy/server.rpt @@ -0,0 +1,151 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +property deltaMillis 0L +property newTimestamp ${kafka:timestamp() + deltaMillis} + +accept "zilla://streams/app1" + option zilla:window 8192 + option zilla:transmission "half-duplex" + +accepted + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .describe() + .topic("test") + .config("cleanup.policy") + .config("max.message.bytes") + .config("segment.bytes") + .config("segment.index.bytes") + .config("segment.ms") + .config("retention.bytes") + .config("retention.ms") + .config("delete.retention.ms") + .config("min.compaction.lag.ms") + .config("max.compaction.lag.ms") + .config("min.cleanable.dirty.ratio") + .build() + .build()} + +connected + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .describe() + .topic("test") + .config("cleanup.policy") + .config("max.message.bytes") + .config("segment.bytes") + .config("segment.index.bytes") + .config("segment.ms") + .config("retention.bytes") + .config("retention.ms") + .config("delete.retention.ms") + .config("min.compaction.lag.ms") + .config("max.compaction.lag.ms") + .config("min.cleanable.dirty.ratio") + .build() + .build()} +write flush + +write zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .describe() + .config("cleanup.policy", "compact") + .config("max.message.bytes", 16) + .config("segment.bytes", 1073741824) + .config("segment.index.bytes", 64) + .config("segment.ms", 604800000) + .config("retention.bytes", -1) + .config("retention.ms", 604800000) + .config("delete.retention.ms", 86400000) + .config("min.compaction.lag.ms", 0) + .config("max.compaction.lag.ms", 9223372036854775807) + .config("min.cleanable.dirty.ratio", 0.0) + .build() + .build()} +write flush + +accepted + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .meta() + .topic("test") + .build() + .build()} + +connected + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .meta() + .topic("test") + .build() + .build()} +write flush + +write zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .meta() + .partition(0, 1) + .build() + .build()} +write flush + +accepted + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(0, -2) + .build() + .build()} + +connected + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .fetch() + .topic("test") + .partition(0, 1, 2) + .build() + .build()} +write flush + + +accepted + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .produce() + .topic("test") + .partition(0) + .build() + .build()} + +connected + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .produce() + .topic("test") + .partition(0) + .build() + .build()} +write flush diff --git a/specs/binding-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/kafka/streams/application/MergedIT.java b/specs/binding-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/kafka/streams/application/MergedIT.java index 12a994c0fd..6aa5454608 100644 --- a/specs/binding-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/kafka/streams/application/MergedIT.java +++ b/specs/binding-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/kafka/streams/application/MergedIT.java @@ -714,6 +714,16 @@ public void shouldUnmergedFetchServerSentResetAndAbortWithMessage() throws Excep k3po.finish(); } + @Test + @Specification({ + "${app}/unmerged.produce.and.fetch.get.cleanup.policy/client", + "${app}/unmerged.produce.and.fetch.get.cleanup.policy/server"}) + public void shouldProduceAndFetchUnmergedGetCompaction() throws Exception + { + k3po.finish(); + } + + @Test @Specification({ "${app}/merged.fetch.filter.not.header/client", @@ -903,6 +913,15 @@ public void shouldProduceUnmergedMessageValueByGettingPartitionId() throws Excep k3po.finish(); } + @Test + @Specification({ + "${app}/merged.produce.and.fetch.get.cleanup.policy/client", + "${app}/merged.produce.and.fetch.get.cleanup.policy/server"}) + public void shouldProduceAndFetchMergedGetCompaction() throws Exception + { + k3po.finish(); + } + @Test @Specification({ "${app}/merged.fetch.unsubscribe/client", diff --git a/specs/binding-mqtt-kafka.spec/src/main/resources/META-INF/zilla/mqtt_kafka.idl b/specs/binding-mqtt-kafka.spec/src/main/resources/META-INF/zilla/mqtt_kafka.idl index 74a792a173..5b5cd547f2 100644 --- a/specs/binding-mqtt-kafka.spec/src/main/resources/META-INF/zilla/mqtt_kafka.idl +++ b/specs/binding-mqtt-kafka.spec/src/main/resources/META-INF/zilla/mqtt_kafka.idl @@ -30,4 +30,22 @@ scope mqtt_kafka int8 length; int16[length] packetIds = null; } + + scope event + { + enum MqttKafkaEventType (uint8) + { + NON_COMPACT_SESSIONS_TOPIC (1) + } + + struct MqttKafkaResetMqttConnectionEx extends core::stream::Extension + { + string16 reason; + } + + union MqttKafkaEventEx switch (MqttKafkaEventType) + { + case NON_COMPACT_SESSIONS_TOPIC: MqttKafkaResetMqttConnectionEx nonCompactSessionsTopic; + } + } } \ No newline at end of file diff --git a/specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/config/proxy.log.event.yaml b/specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/config/proxy.log.event.yaml new file mode 100644 index 0000000000..f6c94af5cc --- /dev/null +++ b/specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/config/proxy.log.event.yaml @@ -0,0 +1,37 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +--- +name: test +telemetry: + exporters: + exporter0: + type: test + options: + events: + - qname: test.kafka0 + id: binding.mqtt.kafka.non.compact.sessions.topic + message: NON COMPACT SESSIONS TOPIC - Sessions Kafka topic in non-compacted +bindings: + mqtt0: + type: mqtt-kafka + kind: proxy + options: + server: mqtt-1.example.com:1883 + topics: + sessions: mqtt-sessions + messages: mqtt-messages + retained: mqtt-retained + exit: kafka0 diff --git a/specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/kafka/session.reject.non.compacted.sessions.topic/client.rpt b/specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/kafka/session.reject.non.compacted.sessions.topic/client.rpt new file mode 100644 index 0000000000..a8498c5c61 --- /dev/null +++ b/specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/kafka/session.reject.non.compacted.sessions.topic/client.rpt @@ -0,0 +1,113 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +connect "zilla://streams/kafka0" + option zilla:window 8192 + option zilla:transmission "duplex" + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .merged() + .capabilities("PRODUCE_AND_FETCH") + .topic("mqtt-sessions") + .groupId("mqtt-clients") + .filter() + .key("client-1#migrate") + .headerNot("sender-id", "sender-1") + .build() + .build() + .build()} + +connected + +write zilla:data.ext ${kafka:dataEx() + .typeId(zilla:id("kafka")) + .merged() + .produce() + .deferred(0) + .partition(-1, -1) + .key("client-1#migrate") + .hashKey("client-1") + .header("sender-id", "sender-1") + .build() + .build()} +write zilla:data.empty +write flush +write notify SENT_INIT_MIGRATE + +write close +read closed + + +connect await SENT_INIT_MIGRATE + "zilla://streams/kafka0" + option zilla:window 8192 + option zilla:transmission "duplex" + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .group() + .groupId("zilla:test-mqtt0-client-1-session") + .protocol("highlander") + .timeout(1000) + .build() + .build()} + +connected + +read advised zilla:flush ${kafka:matchFlushEx() + .typeId(zilla:id("kafka")) + .group() + .leaderId("consumer-1") + .memberId("consumer-1") + .members("consumer-1") + .build() + .build()} +read notify RECEIVED_LEADER + +write zilla:data.empty + + +connect await RECEIVED_LEADER + "zilla://streams/kafka0" + option zilla:window 8192 + option zilla:transmission "duplex" + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .merged() + .capabilities("PRODUCE_AND_FETCH") + .topic("mqtt-sessions") + .groupId("mqtt-clients") + .filter() + .key("client-1") + .build() + .filter() + .key("client-1#migrate") + .headerNot("sender-id", "sender-1") + .build() + .build() + .build()} + +read zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .merged() + .capabilities("PRODUCE_AND_FETCH") + .topic("mqtt-sessions") + .config("cleanup.policy", "delete") + .build() + .build()} + +connected \ No newline at end of file diff --git a/specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/kafka/session.reject.non.compacted.sessions.topic/server.rpt b/specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/kafka/session.reject.non.compacted.sessions.topic/server.rpt new file mode 100644 index 0000000000..f84db986bc --- /dev/null +++ b/specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/kafka/session.reject.non.compacted.sessions.topic/server.rpt @@ -0,0 +1,108 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +accept "zilla://streams/kafka0" + option zilla:window 8192 + option zilla:transmission "duplex" + +accepted + +read zilla:begin.ext ${kafka:matchBeginEx() + .typeId(zilla:id("kafka")) + .merged() + .capabilities("PRODUCE_AND_FETCH") + .topic("mqtt-sessions") + .groupId("mqtt-clients") + .filter() + .key("client-1#migrate") + .headerNot("sender-id", "sender-1") + .build() + .build() + .build()} + +connected + +read zilla:data.ext ${kafka:matchDataEx() + .typeId(zilla:id("kafka")) + .merged() + .produce() + .deferred(0) + .partition(-1, -1) + .key("client-1#migrate") + .hashKey("client-1") + .header("sender-id", "sender-1") + .build() + .build()} +read zilla:data.empty + +read closed +write close + + +accepted + +read zilla:begin.ext ${kafka:matchBeginEx() + .typeId(zilla:id("kafka")) + .group() + .groupId("zilla:test-mqtt0-client-1-session") + .protocol("highlander") + .timeout(1000) + .build() + .build()} + +connected + +# This is the second prerequisite +write advise zilla:flush ${kafka:flushEx() + .typeId(zilla:id("kafka")) + .group() + .leaderId("consumer-1") + .memberId("consumer-1") + .members("consumer-1") + .build() + .build()} +write flush + +read zilla:data.empty + + +accepted + +read zilla:begin.ext ${kafka:matchBeginEx() + .typeId(zilla:id("kafka")) + .merged() + .capabilities("PRODUCE_AND_FETCH") + .topic("mqtt-sessions") + .groupId("mqtt-clients") + .filter() + .key("client-1") + .build() + .filter() + .key("client-1#migrate") + .headerNot("sender-id", "sender-1") + .build() + .build() + .build()} + +write zilla:begin.ext ${kafka:beginEx() + .typeId(zilla:id("kafka")) + .merged() + .capabilities("PRODUCE_AND_FETCH") + .topic("mqtt-sessions") + .config("cleanup.policy", "delete") + .build() + .build()} + +connect abort diff --git a/specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/mqtt/session.reject.non.compacted.sessions.topic/client.rpt b/specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/mqtt/session.reject.non.compacted.sessions.topic/client.rpt new file mode 100644 index 0000000000..17138ad814 --- /dev/null +++ b/specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/mqtt/session.reject.non.compacted.sessions.topic/client.rpt @@ -0,0 +1,46 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +connect "zilla://streams/mqtt0" + option zilla:window 8192 + option zilla:transmission "half-duplex" + +write zilla:begin.ext ${mqtt:beginEx() + .typeId(zilla:id("mqtt")) + .session() + .expiry(1) + .capabilities("REDIRECT") + .clientId("client-1") + .build() + .build()} + +read zilla:begin.ext ${mqtt:matchBeginEx() + .typeId(zilla:id("mqtt")) + .session() + .expiry(1) + .subscribeQosMax(2) + .publishQosMax(2) + .capabilities("RETAIN", "SUBSCRIPTION_IDS", "WILDCARD") + .clientId("client-1") + .build() + .build()} + +connected + +read zilla:reset.ext ${mqtt:resetEx() + .typeId(zilla:id("mqtt")) + .reasonCode(131) + .build()} +write aborted diff --git a/specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/mqtt/session.reject.non.compacted.sessions.topic/server.rpt b/specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/mqtt/session.reject.non.compacted.sessions.topic/server.rpt new file mode 100644 index 0000000000..3b4ff464d6 --- /dev/null +++ b/specs/binding-mqtt-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/mqtt/session.reject.non.compacted.sessions.topic/server.rpt @@ -0,0 +1,49 @@ +# +# Copyright 2021-2023 Aklivity Inc +# +# Licensed under the Aklivity Community License (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://www.aklivity.io/aklivity-community-license/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# + +accept "zilla://streams/mqtt0" + option zilla:window 8192 + option zilla:transmission "duplex" + +accepted + +read zilla:begin.ext ${mqtt:matchBeginEx() + .typeId(zilla:id("mqtt")) + .session() + .expiry(1) + .capabilities("REDIRECT") + .clientId("client-1") + .build() + .build()} + +write zilla:begin.ext ${mqtt:beginEx() + .typeId(zilla:id("mqtt")) + .session() + .expiry(1) + .subscribeQosMax(2) + .publishQosMax(2) + .capabilities("RETAIN", "SUBSCRIPTION_IDS", "WILDCARD") + .clientId("client-1") + .build() + .build()} + +connected + +write zilla:reset.ext ${mqtt:resetEx() + .typeId(zilla:id("mqtt")) + .reasonCode(131) + .build()} + +read abort diff --git a/specs/binding-mqtt-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/KafkaIT.java b/specs/binding-mqtt-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/KafkaIT.java index ad86104769..76b2d69056 100644 --- a/specs/binding-mqtt-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/KafkaIT.java +++ b/specs/binding-mqtt-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/KafkaIT.java @@ -638,6 +638,15 @@ public void shouldSubscribeSaveSubscriptionsInSession() throws Exception k3po.finish(); } + @Test + @Specification({ + "${kafka}/session.reject.non.compacted.sessions.topic/client", + "${kafka}/session.reject.non.compacted.sessions.topic/server"}) + public void shouldRejectSessionNonCompactedSessionsTopic() throws Exception + { + k3po.finish(); + } + @Test @Specification({ "${kafka}/session.subscribe.via.session.state/client", diff --git a/specs/binding-mqtt-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/MqttIT.java b/specs/binding-mqtt-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/MqttIT.java index 15cfc5bd6b..006a056988 100644 --- a/specs/binding-mqtt-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/MqttIT.java +++ b/specs/binding-mqtt-kafka.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/kafka/streams/MqttIT.java @@ -523,6 +523,15 @@ public void shouldSubscribeSaveSubscriptionsInSession() throws Exception k3po.finish(); } + @Test + @Specification({ + "${mqtt}/session.reject.non.compacted.sessions.topic/client", + "${mqtt}/session.reject.non.compacted.sessions.topic/server"}) + public void shouldRejectSessionNonCompactedSessionsTopic() throws Exception + { + k3po.finish(); + } + @Test @Specification({ "${mqtt}/session.subscribe.via.session.state/client", diff --git a/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.reject.non.compacted.sessions.topic/client.rpt b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.reject.non.compacted.sessions.topic/client.rpt new file mode 100644 index 0000000000..3b56ea3017 --- /dev/null +++ b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.reject.non.compacted.sessions.topic/client.rpt @@ -0,0 +1,44 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +connect "zilla://streams/app0" + option zilla:window 8192 + option zilla:transmission "half-duplex" + +write zilla:begin.ext ${mqtt:beginEx() + .typeId(zilla:id("mqtt")) + .session() + .clientId("client") + .build() + .build()} + +read zilla:begin.ext ${mqtt:matchBeginEx() + .typeId(zilla:id("mqtt")) + .session() + .subscribeQosMax(2) + .publishQosMax(2) + .capabilities("RETAIN", "SUBSCRIPTION_IDS", "WILDCARD") + .clientId("client") + .build() + .build()} + +connected + +read zilla:reset.ext ${mqtt:resetEx() + .typeId(zilla:id("mqtt")) + .reasonCode(131) + .build()} +write aborted diff --git a/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.reject.non.compacted.sessions.topic/server.rpt b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.reject.non.compacted.sessions.topic/server.rpt new file mode 100644 index 0000000000..4d3496049c --- /dev/null +++ b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.reject.non.compacted.sessions.topic/server.rpt @@ -0,0 +1,47 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +accept "zilla://streams/app0" + option zilla:window 8192 + option zilla:transmission "duplex" + +accepted + +read zilla:begin.ext ${mqtt:matchBeginEx() + .typeId(zilla:id("mqtt")) + .session() + .clientId("client") + .build() + .build()} + +write zilla:begin.ext ${mqtt:beginEx() + .typeId(zilla:id("mqtt")) + .session() + .subscribeQosMax(2) + .publishQosMax(2) + .capabilities("RETAIN", "SUBSCRIPTION_IDS", "WILDCARD") + .clientId("client") + .build() + .build()} + +connected + +write zilla:reset.ext ${mqtt:resetEx() + .typeId(zilla:id("mqtt")) + .reasonCode(131) + .build()} + +read abort diff --git a/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.subscribe.invalid.state/client.rpt b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.subscribe.invalid.state/client.rpt new file mode 100644 index 0000000000..7b68fbd642 --- /dev/null +++ b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.subscribe.invalid.state/client.rpt @@ -0,0 +1,79 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +connect "zilla://streams/app0" + option zilla:window 8192 + option zilla:transmission "duplex" + +write zilla:begin.ext ${mqtt:beginEx() + .typeId(zilla:id("mqtt")) + .session() + .clientId("client") + .build() + .build()} + +read zilla:begin.ext ${mqtt:matchBeginEx() + .typeId(zilla:id("mqtt")) + .session() + .flags("CLEAN_START") + .subscribeQosMax(2) + .publishQosMax(2) + .packetSizeMax(66560) + .capabilities("RETAIN", "WILDCARD", "SUBSCRIPTION_IDS", "SHARED_SUBSCRIPTIONS") + .clientId("client") + .build() + .build()} + +connected + +read zilla:data.empty + +write zilla:data.ext ${mqtt:dataEx() + .typeId(zilla:id("mqtt")) + .session() + .kind("STATE") + .build() + .build()} + +write ${mqtt:session() + .subscription("sensor/one", 1, "AT_MOST_ONCE", "SEND_RETAINED") + .build()} +write flush + +read ${mqtt:session() + .subscription("sensor/one", 1, "AT_MOST_ONCE", "SEND_RETAINED") + .build()} + + +read ${mqtt:session() + .subscription("sensor/one", 1, "AT_MOST_ONCE", "SEND_RETAINED") + .subscription("sensor/one", 1, "AT_MOST_ONCE", "SEND_RETAINED") + .build()} + +connect "zilla://streams/app0" + option zilla:window 8192 + option zilla:transmission "duplex" + +write zilla:begin.ext ${mqtt:beginEx() + .typeId(zilla:id("mqtt")) + .subscribe() + .clientId("client") + .filter("sensor/one", 1, "AT_MOST_ONCE", "SEND_RETAINED") + .build() + .build()} + +connected + diff --git a/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.subscribe.invalid.state/server.rpt b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.subscribe.invalid.state/server.rpt new file mode 100644 index 0000000000..5aa5ffbd9a --- /dev/null +++ b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/application/session.subscribe.invalid.state/server.rpt @@ -0,0 +1,80 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +accept "zilla://streams/app0" + option zilla:window 8192 + option zilla:transmission "duplex" + +accepted + +read zilla:begin.ext ${mqtt:matchBeginEx() + .typeId(zilla:id("mqtt")) + .session() + .clientId("client") + .build() + .build()} + +write zilla:begin.ext ${mqtt:beginEx() + .typeId(zilla:id("mqtt")) + .session() + .flags("CLEAN_START") + .subscribeQosMax(2) + .publishQosMax(2) + .packetSizeMax(66560) + .capabilities("RETAIN", "WILDCARD", "SUBSCRIPTION_IDS", "SHARED_SUBSCRIPTIONS") + .clientId("client") + .build() + .build()} + +connected + +write zilla:data.empty +write flush + +read zilla:data.ext ${mqtt:dataEx() + .typeId(zilla:id("mqtt")) + .session() + .kind("STATE") + .build() + .build()} + +read ${mqtt:session() + .subscription("sensor/one", 1, "AT_MOST_ONCE", "SEND_RETAINED") + .build()} + +write ${mqtt:session() + .subscription("sensor/one", 1, "AT_MOST_ONCE", "SEND_RETAINED") + .build()} +write flush + + +write ${mqtt:session() + .subscription("sensor/one", 1, "AT_MOST_ONCE", "SEND_RETAINED") + .subscription("sensor/one", 1, "AT_MOST_ONCE", "SEND_RETAINED") + .build()} +write flush + +accepted + +read zilla:begin.ext ${mqtt:matchBeginEx() + .typeId(zilla:id("mqtt")) + .subscribe() + .clientId("client") + .filter("sensor/one", 1, "AT_MOST_ONCE", "SEND_RETAINED") + .build() + .build()} + +connected diff --git a/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v4/session.reject.non.compacted.sessions.topic/client.rpt b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v4/session.reject.non.compacted.sessions.topic/client.rpt new file mode 100644 index 0000000000..6619adae1c --- /dev/null +++ b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v4/session.reject.non.compacted.sessions.topic/client.rpt @@ -0,0 +1,35 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +connect "zilla://streams/net0" + option zilla:window 8192 + option zilla:transmission "duplex" + option zilla:byteorder "network" + +connected + +write [0x10 0x11] # CONNECT + [0x00 0x04] "MQTT" # protocol name + [0x04] # protocol version + [0x02] # flags = clean start + [0x00 0x3c] # keep alive = 60s + [0x00 0x06] "client" # client id + +read [0x20 0x02] # CONNACK + [0x00] # flags = none + [0x83] # reason code = = implementation specific error + +read closed diff --git a/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v4/session.reject.non.compacted.sessions.topic/server.rpt b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v4/session.reject.non.compacted.sessions.topic/server.rpt new file mode 100644 index 0000000000..f56fbe87b7 --- /dev/null +++ b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v4/session.reject.non.compacted.sessions.topic/server.rpt @@ -0,0 +1,36 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +accept "zilla://streams/net0" + option zilla:window 8192 + option zilla:transmission "duplex" + option zilla:byteorder "network" + +accepted +connected + +read [0x10 0x11] # CONNECT + [0x00 0x04] "MQTT" # protocol name + [0x04] # protocol version + [0x02] # flags = clean start + [0x00 0x3c] # keep alive = 60s + [0x00 0x06] "client" # client id + +write [0x20 0x02] # CONNACK + [0x00] # flags = none + [0x83] # reason code = = implementation specific error + +write close diff --git a/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/session.reject.non.compacted.sessions.topic/client.rpt b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/session.reject.non.compacted.sessions.topic/client.rpt new file mode 100644 index 0000000000..6d5075e5e6 --- /dev/null +++ b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/session.reject.non.compacted.sessions.topic/client.rpt @@ -0,0 +1,37 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +connect "zilla://streams/net0" + option zilla:window 8192 + option zilla:transmission "duplex" + option zilla:byteorder "network" + +connected + +write [0x10 0x13] # CONNECT + [0x00 0x04] "MQTT" # protocol name + [0x05] # protocol version + [0x02] # flags = clean start + [0x00 0x3c] # keep alive = 60s + [0x00] # properties = none + [0x00 0x06] "client" # client id + +read [0x20 0x03] # CONNACK + [0x00] # flags = none + [0x83] # reason code = implementation specific error + [0x00] # properties + +read closed diff --git a/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/session.reject.non.compacted.sessions.topic/server.rpt b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/session.reject.non.compacted.sessions.topic/server.rpt new file mode 100644 index 0000000000..c53c986d69 --- /dev/null +++ b/specs/binding-mqtt.spec/src/main/scripts/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/session.reject.non.compacted.sessions.topic/server.rpt @@ -0,0 +1,38 @@ +# +# Copyright 2021-2023 Aklivity Inc. +# +# Aklivity licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +accept "zilla://streams/net0" + option zilla:window 8192 + option zilla:transmission "duplex" + option zilla:byteorder "network" + +accepted +connected + +read [0x10 0x13] # CONNECT + [0x00 0x04] "MQTT" # protocol name + [0x05] # protocol version + [0x02] # flags = clean start + [0x00 0x3c] # keep alive = 60s + [0x00] # properties = none + [0x00 0x06] "client" # client id + +write [0x20 0x03] # CONNACK + [0x00] # flags = none + [0x83] # reason code = implementation specific error + [0x00] # properties + +write close diff --git a/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/application/SessionIT.java b/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/application/SessionIT.java index 46b6f84e6b..213924acba 100644 --- a/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/application/SessionIT.java +++ b/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/application/SessionIT.java @@ -209,6 +209,15 @@ public void shouldRedirectAfterConnack() throws Exception k3po.finish(); } + @Test + @Specification({ + "${app}/session.subscribe.invalid.state/client", + "${app}/session.subscribe.invalid.state/server"}) + public void shouldSubscribeInvalidSessionState() throws Exception + { + k3po.finish(); + } + @Test @Specification({ "${app}/session.subscribe.multiple.isolated/client", @@ -227,6 +236,15 @@ public void shouldSubscribeAndPublishToNonDefaultRoute() throws Exception k3po.finish(); } + @Test + @Specification({ + "${app}/session.reject.non.compacted.sessions.topic/client", + "${app}/session.reject.non.compacted.sessions.topic/server"}) + public void shouldRejectSessionNonCompactedSessionsTopic() throws Exception + { + k3po.finish(); + } + @Test @Specification({ "${app}/session.invalid.session.timeout.after.connack/client", diff --git a/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/network/v4/SessionIT.java b/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/network/v4/SessionIT.java index 844dccd4de..a9855be4c4 100644 --- a/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/network/v4/SessionIT.java +++ b/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/network/v4/SessionIT.java @@ -145,4 +145,13 @@ public void shouldSubscribeAndPublishToNonDefaultRoute() throws Exception { k3po.finish(); } + @Test + @Specification({ + "${net}/session.reject.non.compacted.sessions.topic/client", + "${net}/session.reject.non.compacted.sessions.topic/server"}) + public void shouldRejectSessionNonCompactedSessionsTopic() throws Exception + { + k3po.finish(); + } + } diff --git a/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/SessionIT.java b/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/SessionIT.java index a0da7b4841..c9dcfa4a66 100644 --- a/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/SessionIT.java +++ b/specs/binding-mqtt.spec/src/test/java/io/aklivity/zilla/specs/binding/mqtt/streams/network/v5/SessionIT.java @@ -192,6 +192,15 @@ public void shouldSubscribeAndPublishToNonDefaultRoute() throws Exception k3po.finish(); } + @Test + @Specification({ + "${net}/session.reject.non.compacted.sessions.topic/client", + "${net}/session.reject.non.compacted.sessions.topic/server"}) + public void shouldRejectSessionNonCompactedSessionsTopic() throws Exception + { + k3po.finish(); + } + @Test @Specification({ "${net}/session.invalid.session.timeout.after.connack/client", From e500419bc1781aa03816bfa33b1edac9a5b73f49 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Date: Tue, 28 May 2024 23:59:22 +0530 Subject: [PATCH 7/8] Update to handle catalog IT validation(resolve schema from subject) (#1055) --- .../test/internal/binding/TestBindingFactory.java | 11 +++++++++++ .../config/TestBindingOptionsConfigAdapter.java | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/internal/binding/TestBindingFactory.java b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/internal/binding/TestBindingFactory.java index 22c7112144..d6e8b7bb7b 100644 --- a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/internal/binding/TestBindingFactory.java +++ b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/internal/binding/TestBindingFactory.java @@ -15,8 +15,11 @@ */ package io.aklivity.zilla.runtime.engine.test.internal.binding; +import static io.aklivity.zilla.runtime.engine.test.internal.binding.config.TestBindingOptionsConfigAdapter.DEFAULT_ASSERTION_SCHEMA; + import java.util.LinkedList; import java.util.List; +import java.util.Objects; import org.agrona.DirectBuffer; import org.agrona.MutableDirectBuffer; @@ -275,6 +278,14 @@ private void onInitialBegin( { doInitialReset(traceId); } + if (DEFAULT_ASSERTION_SCHEMA != assertion.schema) + { + String schema = handler.resolve(id); + if (!Objects.equals(assertion.schema, schema)) + { + doInitialReset(traceId); + } + } } else { diff --git a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/internal/binding/config/TestBindingOptionsConfigAdapter.java b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/internal/binding/config/TestBindingOptionsConfigAdapter.java index 6840c32814..4575e6bf68 100644 --- a/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/internal/binding/config/TestBindingOptionsConfigAdapter.java +++ b/runtime/engine/src/test/java/io/aklivity/zilla/runtime/engine/test/internal/binding/config/TestBindingOptionsConfigAdapter.java @@ -33,6 +33,8 @@ public final class TestBindingOptionsConfigAdapter implements OptionsConfigAdapterSpi { + public static final String DEFAULT_ASSERTION_SCHEMA = new String(); + private static final String MODE_NAME = "mode"; private static final String CATALOG_NAME = "catalog"; private static final String AUTHORIZATION_NAME = "authorization"; @@ -174,7 +176,8 @@ public OptionsConfig adaptFromJson( JsonObject c = assertion.asJsonObject(); catalogAssertions.add(new TestBindingOptionsConfig.CatalogAssertion( c.containsKey(ID_NAME) ? c.getInt(ID_NAME) : 0, - c.containsKey(SCHEMA_NAME) ? !c.isNull(SCHEMA_NAME) ? c.getString(SCHEMA_NAME) : null : null, + c.containsKey(SCHEMA_NAME) ? !c.isNull(SCHEMA_NAME) ? c.getString(SCHEMA_NAME) + : null : DEFAULT_ASSERTION_SCHEMA, c.containsKey(DELAY_NAME) ? c.getJsonNumber(DELAY_NAME).longValue() : 0L)); } testOptions.catalogAssertions(catalogName, catalogAssertions); From ed2fdb2894ffc7e6c56511b8d4130e9177cc6104 Mon Sep 17 00:00:00 2001 From: John Fallows Date: Tue, 28 May 2024 11:44:03 -0700 Subject: [PATCH 8/8] Prepare release 0.9.82 --- CHANGELOG.md | 11 +++++++++++ build/flyweight-maven-plugin/pom.xml | 2 +- build/pom.xml | 2 +- cloud/docker-image/pom.xml | 2 +- cloud/helm-chart/pom.xml | 2 +- cloud/pom.xml | 2 +- conf/pom.xml | 2 +- incubator/binding-amqp.spec/pom.xml | 2 +- incubator/binding-amqp/pom.xml | 2 +- incubator/catalog-filesystem.spec/pom.xml | 2 +- incubator/catalog-filesystem/pom.xml | 2 +- incubator/command-dump/pom.xml | 2 +- incubator/command-log/pom.xml | 2 +- incubator/command-tune/pom.xml | 2 +- incubator/pom.xml | 2 +- manager/pom.xml | 2 +- pom.xml | 2 +- runtime/binding-asyncapi/pom.xml | 2 +- runtime/binding-echo/pom.xml | 2 +- runtime/binding-fan/pom.xml | 2 +- runtime/binding-filesystem/pom.xml | 2 +- runtime/binding-grpc-kafka/pom.xml | 2 +- runtime/binding-grpc/pom.xml | 2 +- runtime/binding-http-filesystem/pom.xml | 2 +- runtime/binding-http-kafka/pom.xml | 2 +- runtime/binding-http/pom.xml | 2 +- runtime/binding-kafka-grpc/pom.xml | 2 +- runtime/binding-kafka/pom.xml | 2 +- runtime/binding-mqtt-kafka/pom.xml | 2 +- runtime/binding-mqtt/pom.xml | 2 +- runtime/binding-openapi-asyncapi/pom.xml | 2 +- runtime/binding-openapi/pom.xml | 2 +- runtime/binding-proxy/pom.xml | 2 +- runtime/binding-sse-kafka/pom.xml | 2 +- runtime/binding-sse/pom.xml | 2 +- runtime/binding-tcp/pom.xml | 2 +- runtime/binding-tls/pom.xml | 2 +- runtime/binding-ws/pom.xml | 2 +- runtime/catalog-apicurio/pom.xml | 2 +- runtime/catalog-inline/pom.xml | 2 +- runtime/catalog-karapace/pom.xml | 2 +- runtime/command-metrics/pom.xml | 2 +- runtime/command-start/pom.xml | 2 +- runtime/command-stop/pom.xml | 2 +- runtime/command/pom.xml | 2 +- runtime/common/pom.xml | 2 +- runtime/engine/pom.xml | 2 +- runtime/exporter-otlp/pom.xml | 2 +- runtime/exporter-prometheus/pom.xml | 2 +- runtime/exporter-stdout/pom.xml | 2 +- runtime/guard-jwt/pom.xml | 2 +- runtime/metrics-grpc/pom.xml | 2 +- runtime/metrics-http/pom.xml | 2 +- runtime/metrics-stream/pom.xml | 2 +- runtime/model-avro/pom.xml | 2 +- runtime/model-core/pom.xml | 2 +- runtime/model-json/pom.xml | 2 +- runtime/model-protobuf/pom.xml | 2 +- runtime/pom.xml | 2 +- runtime/resolver-env/pom.xml | 2 +- runtime/vault-filesystem/pom.xml | 2 +- specs/binding-asyncapi.spec/pom.xml | 2 +- specs/binding-echo.spec/pom.xml | 2 +- specs/binding-fan.spec/pom.xml | 2 +- specs/binding-filesystem.spec/pom.xml | 2 +- specs/binding-grpc-kafka.spec/pom.xml | 2 +- specs/binding-grpc.spec/pom.xml | 2 +- specs/binding-http-filesystem.spec/pom.xml | 2 +- specs/binding-http-kafka.spec/pom.xml | 2 +- specs/binding-http.spec/pom.xml | 2 +- specs/binding-kafka-grpc.spec/pom.xml | 2 +- specs/binding-kafka.spec/pom.xml | 2 +- specs/binding-mqtt-kafka.spec/pom.xml | 2 +- specs/binding-mqtt.spec/pom.xml | 2 +- specs/binding-openapi-asyncapi.spec/pom.xml | 2 +- specs/binding-openapi.spec/pom.xml | 2 +- specs/binding-proxy.spec/pom.xml | 2 +- specs/binding-sse-kafka.spec/pom.xml | 2 +- specs/binding-sse.spec/pom.xml | 2 +- specs/binding-tcp.spec/pom.xml | 2 +- specs/binding-tls.spec/pom.xml | 2 +- specs/binding-ws.spec/pom.xml | 2 +- specs/catalog-apicurio.spec/pom.xml | 2 +- specs/catalog-inline.spec/pom.xml | 2 +- specs/catalog-karapace.spec/pom.xml | 2 +- specs/engine.spec/pom.xml | 2 +- specs/exporter-otlp.spec/pom.xml | 2 +- specs/exporter-prometheus.spec/pom.xml | 2 +- specs/exporter-stdout.spec/pom.xml | 2 +- specs/guard-jwt.spec/pom.xml | 2 +- specs/metrics-grpc.spec/pom.xml | 2 +- specs/metrics-http.spec/pom.xml | 2 +- specs/metrics-stream.spec/pom.xml | 2 +- specs/model-avro.spec/pom.xml | 2 +- specs/model-core.spec/pom.xml | 2 +- specs/model-json.spec/pom.xml | 2 +- specs/model-protobuf.spec/pom.xml | 2 +- specs/pom.xml | 2 +- specs/vault-filesystem.spec/pom.xml | 2 +- 99 files changed, 109 insertions(+), 98 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d23b7b327..0098da6a19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [Unreleased](https://github.com/aklivity/zilla/tree/HEAD) + +[Full Changelog](https://github.com/aklivity/zilla/compare/0.9.81...HEAD) + +**Fixed bugs:** + +- Zilla crashes with `IllegalArgumentException: cannot accept missingValue` when using `defaultOffset: live` [\#1051](https://github.com/aklivity/zilla/issues/1051) +- Zilla crashes on mqtt cli -T option [\#1039](https://github.com/aklivity/zilla/issues/1039) +- Running `emqtt_bench` both `sub` and `pub` triggers an exception [\#1000](https://github.com/aklivity/zilla/issues/1000) +- `http-kafka` will `fetch` messages that have been deleted by a retention policy [\#897](https://github.com/aklivity/zilla/issues/897) + ## [0.9.81](https://github.com/aklivity/zilla/tree/0.9.81) (2024-05-24) [Full Changelog](https://github.com/aklivity/zilla/compare/0.9.80...0.9.81) diff --git a/build/flyweight-maven-plugin/pom.xml b/build/flyweight-maven-plugin/pom.xml index 74d6a13033..7fe2d1eb72 100644 --- a/build/flyweight-maven-plugin/pom.xml +++ b/build/flyweight-maven-plugin/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla build - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/build/pom.xml b/build/pom.xml index 99a8e7a8fa..5e58384b6f 100644 --- a/build/pom.xml +++ b/build/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla zilla - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/cloud/docker-image/pom.xml b/cloud/docker-image/pom.xml index 0b37392348..60db60ec9b 100644 --- a/cloud/docker-image/pom.xml +++ b/cloud/docker-image/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla cloud - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/cloud/helm-chart/pom.xml b/cloud/helm-chart/pom.xml index 9a4887cb75..9851b6e26b 100644 --- a/cloud/helm-chart/pom.xml +++ b/cloud/helm-chart/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla cloud - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/cloud/pom.xml b/cloud/pom.xml index 68c51f05c8..1646b94827 100644 --- a/cloud/pom.xml +++ b/cloud/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla zilla - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/conf/pom.xml b/conf/pom.xml index 86622b6574..9b641dfcc4 100644 --- a/conf/pom.xml +++ b/conf/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla zilla - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/incubator/binding-amqp.spec/pom.xml b/incubator/binding-amqp.spec/pom.xml index 40846d1a35..263615131a 100644 --- a/incubator/binding-amqp.spec/pom.xml +++ b/incubator/binding-amqp.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla incubator - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/incubator/binding-amqp/pom.xml b/incubator/binding-amqp/pom.xml index 71c9c61d90..17e5b260b3 100644 --- a/incubator/binding-amqp/pom.xml +++ b/incubator/binding-amqp/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla incubator - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/incubator/catalog-filesystem.spec/pom.xml b/incubator/catalog-filesystem.spec/pom.xml index 204a195e91..7d1d3d3277 100644 --- a/incubator/catalog-filesystem.spec/pom.xml +++ b/incubator/catalog-filesystem.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla incubator - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/incubator/catalog-filesystem/pom.xml b/incubator/catalog-filesystem/pom.xml index c09fada8d9..68eb226529 100644 --- a/incubator/catalog-filesystem/pom.xml +++ b/incubator/catalog-filesystem/pom.xml @@ -6,7 +6,7 @@ io.aklivity.zilla incubator - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/incubator/command-dump/pom.xml b/incubator/command-dump/pom.xml index 129f9ed179..cfcbf7e651 100644 --- a/incubator/command-dump/pom.xml +++ b/incubator/command-dump/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla incubator - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/incubator/command-log/pom.xml b/incubator/command-log/pom.xml index 588aee3edc..efd7bcd6a6 100644 --- a/incubator/command-log/pom.xml +++ b/incubator/command-log/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla incubator - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/incubator/command-tune/pom.xml b/incubator/command-tune/pom.xml index 7374c89ed9..8bf9760c40 100644 --- a/incubator/command-tune/pom.xml +++ b/incubator/command-tune/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla incubator - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/incubator/pom.xml b/incubator/pom.xml index 3a394cb372..4646e235ca 100644 --- a/incubator/pom.xml +++ b/incubator/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla zilla - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/manager/pom.xml b/manager/pom.xml index 8d8a444944..c9e5682f37 100644 --- a/manager/pom.xml +++ b/manager/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla zilla - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/pom.xml b/pom.xml index e3c87486da..e111f46c9e 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ 4.0.0 io.aklivity.zilla zilla - develop-SNAPSHOT + 0.9.82 pom zilla https://github.com/aklivity/zilla diff --git a/runtime/binding-asyncapi/pom.xml b/runtime/binding-asyncapi/pom.xml index 0f8d8e23b1..811e313493 100644 --- a/runtime/binding-asyncapi/pom.xml +++ b/runtime/binding-asyncapi/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-echo/pom.xml b/runtime/binding-echo/pom.xml index c74bf3c8de..9058f89521 100644 --- a/runtime/binding-echo/pom.xml +++ b/runtime/binding-echo/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-fan/pom.xml b/runtime/binding-fan/pom.xml index 77b5e0aab5..8c7313de3a 100644 --- a/runtime/binding-fan/pom.xml +++ b/runtime/binding-fan/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-filesystem/pom.xml b/runtime/binding-filesystem/pom.xml index 46b6e08957..7bed1aad7b 100644 --- a/runtime/binding-filesystem/pom.xml +++ b/runtime/binding-filesystem/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-grpc-kafka/pom.xml b/runtime/binding-grpc-kafka/pom.xml index a9952b0a68..0a637e4565 100644 --- a/runtime/binding-grpc-kafka/pom.xml +++ b/runtime/binding-grpc-kafka/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-grpc/pom.xml b/runtime/binding-grpc/pom.xml index 8bf0b5f396..1c24bd59fb 100644 --- a/runtime/binding-grpc/pom.xml +++ b/runtime/binding-grpc/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-http-filesystem/pom.xml b/runtime/binding-http-filesystem/pom.xml index 19d46db62e..3729491236 100644 --- a/runtime/binding-http-filesystem/pom.xml +++ b/runtime/binding-http-filesystem/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-http-kafka/pom.xml b/runtime/binding-http-kafka/pom.xml index dabc54fb58..54fb2db179 100644 --- a/runtime/binding-http-kafka/pom.xml +++ b/runtime/binding-http-kafka/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-http/pom.xml b/runtime/binding-http/pom.xml index 15111be380..51ea98c09c 100644 --- a/runtime/binding-http/pom.xml +++ b/runtime/binding-http/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-kafka-grpc/pom.xml b/runtime/binding-kafka-grpc/pom.xml index 41b94c3bcb..f75bc1a062 100644 --- a/runtime/binding-kafka-grpc/pom.xml +++ b/runtime/binding-kafka-grpc/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-kafka/pom.xml b/runtime/binding-kafka/pom.xml index d643324d07..22b9fce678 100644 --- a/runtime/binding-kafka/pom.xml +++ b/runtime/binding-kafka/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-mqtt-kafka/pom.xml b/runtime/binding-mqtt-kafka/pom.xml index 78e8e17d68..242fab6cbf 100644 --- a/runtime/binding-mqtt-kafka/pom.xml +++ b/runtime/binding-mqtt-kafka/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-mqtt/pom.xml b/runtime/binding-mqtt/pom.xml index 92759b3de4..1976bee0b8 100644 --- a/runtime/binding-mqtt/pom.xml +++ b/runtime/binding-mqtt/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-openapi-asyncapi/pom.xml b/runtime/binding-openapi-asyncapi/pom.xml index 4b5e39cd7c..b46f0d0766 100644 --- a/runtime/binding-openapi-asyncapi/pom.xml +++ b/runtime/binding-openapi-asyncapi/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-openapi/pom.xml b/runtime/binding-openapi/pom.xml index f7b76f7381..c61f4f8175 100644 --- a/runtime/binding-openapi/pom.xml +++ b/runtime/binding-openapi/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-proxy/pom.xml b/runtime/binding-proxy/pom.xml index 38985b6961..36047c7385 100644 --- a/runtime/binding-proxy/pom.xml +++ b/runtime/binding-proxy/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-sse-kafka/pom.xml b/runtime/binding-sse-kafka/pom.xml index 73486efd4d..1445f74d7b 100644 --- a/runtime/binding-sse-kafka/pom.xml +++ b/runtime/binding-sse-kafka/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-sse/pom.xml b/runtime/binding-sse/pom.xml index 1d7aa891b8..97b8f3742c 100644 --- a/runtime/binding-sse/pom.xml +++ b/runtime/binding-sse/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-tcp/pom.xml b/runtime/binding-tcp/pom.xml index 513e5eae6b..10862b2bb1 100644 --- a/runtime/binding-tcp/pom.xml +++ b/runtime/binding-tcp/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-tls/pom.xml b/runtime/binding-tls/pom.xml index 71b1cd04a9..c23c9a7439 100644 --- a/runtime/binding-tls/pom.xml +++ b/runtime/binding-tls/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/binding-ws/pom.xml b/runtime/binding-ws/pom.xml index 6f846d1a83..09d2a56dc6 100644 --- a/runtime/binding-ws/pom.xml +++ b/runtime/binding-ws/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/catalog-apicurio/pom.xml b/runtime/catalog-apicurio/pom.xml index d352e584b8..ac7224aa51 100644 --- a/runtime/catalog-apicurio/pom.xml +++ b/runtime/catalog-apicurio/pom.xml @@ -6,7 +6,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/catalog-inline/pom.xml b/runtime/catalog-inline/pom.xml index 23d4127026..343c69df4d 100644 --- a/runtime/catalog-inline/pom.xml +++ b/runtime/catalog-inline/pom.xml @@ -6,7 +6,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/catalog-karapace/pom.xml b/runtime/catalog-karapace/pom.xml index c7bc7ddee3..045ed934bd 100644 --- a/runtime/catalog-karapace/pom.xml +++ b/runtime/catalog-karapace/pom.xml @@ -6,7 +6,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/command-metrics/pom.xml b/runtime/command-metrics/pom.xml index ad52828b76..20be048775 100644 --- a/runtime/command-metrics/pom.xml +++ b/runtime/command-metrics/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/command-start/pom.xml b/runtime/command-start/pom.xml index 8ab765d93d..af2a8c1e59 100644 --- a/runtime/command-start/pom.xml +++ b/runtime/command-start/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/command-stop/pom.xml b/runtime/command-stop/pom.xml index 9334bc51e1..6a9f537973 100644 --- a/runtime/command-stop/pom.xml +++ b/runtime/command-stop/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/command/pom.xml b/runtime/command/pom.xml index 29712783a0..357fab1d10 100644 --- a/runtime/command/pom.xml +++ b/runtime/command/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/common/pom.xml b/runtime/common/pom.xml index a96a6c1cab..53108913bb 100644 --- a/runtime/common/pom.xml +++ b/runtime/common/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/engine/pom.xml b/runtime/engine/pom.xml index 0c55bb67f6..e8a38a147d 100644 --- a/runtime/engine/pom.xml +++ b/runtime/engine/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/exporter-otlp/pom.xml b/runtime/exporter-otlp/pom.xml index c4e4f10565..30efac8e19 100644 --- a/runtime/exporter-otlp/pom.xml +++ b/runtime/exporter-otlp/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/exporter-prometheus/pom.xml b/runtime/exporter-prometheus/pom.xml index dad973cdb8..80423ac80b 100644 --- a/runtime/exporter-prometheus/pom.xml +++ b/runtime/exporter-prometheus/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/exporter-stdout/pom.xml b/runtime/exporter-stdout/pom.xml index f32e0c0d24..2020a94c5f 100644 --- a/runtime/exporter-stdout/pom.xml +++ b/runtime/exporter-stdout/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/guard-jwt/pom.xml b/runtime/guard-jwt/pom.xml index 00df1533ea..f43fc5b8bd 100644 --- a/runtime/guard-jwt/pom.xml +++ b/runtime/guard-jwt/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/metrics-grpc/pom.xml b/runtime/metrics-grpc/pom.xml index 5713263b66..9a67d77b31 100644 --- a/runtime/metrics-grpc/pom.xml +++ b/runtime/metrics-grpc/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/metrics-http/pom.xml b/runtime/metrics-http/pom.xml index f505adfd82..f9f9e3ebf0 100644 --- a/runtime/metrics-http/pom.xml +++ b/runtime/metrics-http/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/metrics-stream/pom.xml b/runtime/metrics-stream/pom.xml index 9ccb8e23dd..69d1036905 100644 --- a/runtime/metrics-stream/pom.xml +++ b/runtime/metrics-stream/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/model-avro/pom.xml b/runtime/model-avro/pom.xml index ca62af879a..3abb417c0d 100644 --- a/runtime/model-avro/pom.xml +++ b/runtime/model-avro/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/model-core/pom.xml b/runtime/model-core/pom.xml index 128a99d912..310d200c68 100644 --- a/runtime/model-core/pom.xml +++ b/runtime/model-core/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/model-json/pom.xml b/runtime/model-json/pom.xml index c1b1993207..97213c966b 100644 --- a/runtime/model-json/pom.xml +++ b/runtime/model-json/pom.xml @@ -6,7 +6,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/model-protobuf/pom.xml b/runtime/model-protobuf/pom.xml index 9728ca1d60..e81ff80984 100644 --- a/runtime/model-protobuf/pom.xml +++ b/runtime/model-protobuf/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/pom.xml b/runtime/pom.xml index 444f321674..1321f6c3ac 100644 --- a/runtime/pom.xml +++ b/runtime/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla zilla - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/resolver-env/pom.xml b/runtime/resolver-env/pom.xml index a7c5e5d4a5..f7ce61e05e 100644 --- a/runtime/resolver-env/pom.xml +++ b/runtime/resolver-env/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/runtime/vault-filesystem/pom.xml b/runtime/vault-filesystem/pom.xml index d5ab82cb01..c678ee63db 100644 --- a/runtime/vault-filesystem/pom.xml +++ b/runtime/vault-filesystem/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla runtime - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-asyncapi.spec/pom.xml b/specs/binding-asyncapi.spec/pom.xml index f34019e7b5..e55ef7a7ce 100644 --- a/specs/binding-asyncapi.spec/pom.xml +++ b/specs/binding-asyncapi.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-echo.spec/pom.xml b/specs/binding-echo.spec/pom.xml index d0bfed636e..9a0bd6567c 100644 --- a/specs/binding-echo.spec/pom.xml +++ b/specs/binding-echo.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-fan.spec/pom.xml b/specs/binding-fan.spec/pom.xml index 027d3210d7..ddb07be7c4 100644 --- a/specs/binding-fan.spec/pom.xml +++ b/specs/binding-fan.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-filesystem.spec/pom.xml b/specs/binding-filesystem.spec/pom.xml index cc20c4134b..e86490b6ac 100644 --- a/specs/binding-filesystem.spec/pom.xml +++ b/specs/binding-filesystem.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-grpc-kafka.spec/pom.xml b/specs/binding-grpc-kafka.spec/pom.xml index 546b34371a..ceaf9538ea 100644 --- a/specs/binding-grpc-kafka.spec/pom.xml +++ b/specs/binding-grpc-kafka.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-grpc.spec/pom.xml b/specs/binding-grpc.spec/pom.xml index c75c35e62e..2faf749c96 100644 --- a/specs/binding-grpc.spec/pom.xml +++ b/specs/binding-grpc.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-http-filesystem.spec/pom.xml b/specs/binding-http-filesystem.spec/pom.xml index b9499ae553..5773c02598 100644 --- a/specs/binding-http-filesystem.spec/pom.xml +++ b/specs/binding-http-filesystem.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-http-kafka.spec/pom.xml b/specs/binding-http-kafka.spec/pom.xml index 456a3106b5..fef29183a9 100644 --- a/specs/binding-http-kafka.spec/pom.xml +++ b/specs/binding-http-kafka.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-http.spec/pom.xml b/specs/binding-http.spec/pom.xml index 38a7898be9..492554c5cf 100644 --- a/specs/binding-http.spec/pom.xml +++ b/specs/binding-http.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-kafka-grpc.spec/pom.xml b/specs/binding-kafka-grpc.spec/pom.xml index 197265531f..9a96c623a5 100644 --- a/specs/binding-kafka-grpc.spec/pom.xml +++ b/specs/binding-kafka-grpc.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-kafka.spec/pom.xml b/specs/binding-kafka.spec/pom.xml index c4559fdb23..33c0ffcf47 100644 --- a/specs/binding-kafka.spec/pom.xml +++ b/specs/binding-kafka.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-mqtt-kafka.spec/pom.xml b/specs/binding-mqtt-kafka.spec/pom.xml index 517559d0d0..cce6120ee3 100644 --- a/specs/binding-mqtt-kafka.spec/pom.xml +++ b/specs/binding-mqtt-kafka.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-mqtt.spec/pom.xml b/specs/binding-mqtt.spec/pom.xml index f624bb85ae..e072cace5f 100644 --- a/specs/binding-mqtt.spec/pom.xml +++ b/specs/binding-mqtt.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-openapi-asyncapi.spec/pom.xml b/specs/binding-openapi-asyncapi.spec/pom.xml index a968c22788..0c4350528c 100644 --- a/specs/binding-openapi-asyncapi.spec/pom.xml +++ b/specs/binding-openapi-asyncapi.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-openapi.spec/pom.xml b/specs/binding-openapi.spec/pom.xml index a364289b7f..958dbfb6fa 100644 --- a/specs/binding-openapi.spec/pom.xml +++ b/specs/binding-openapi.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-proxy.spec/pom.xml b/specs/binding-proxy.spec/pom.xml index fcdb4b9723..d16011e6c8 100644 --- a/specs/binding-proxy.spec/pom.xml +++ b/specs/binding-proxy.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-sse-kafka.spec/pom.xml b/specs/binding-sse-kafka.spec/pom.xml index 20b099b140..18a0a5df34 100644 --- a/specs/binding-sse-kafka.spec/pom.xml +++ b/specs/binding-sse-kafka.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-sse.spec/pom.xml b/specs/binding-sse.spec/pom.xml index 5cf8e9018c..3414860f83 100644 --- a/specs/binding-sse.spec/pom.xml +++ b/specs/binding-sse.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-tcp.spec/pom.xml b/specs/binding-tcp.spec/pom.xml index 94ef266ac7..f5234e387a 100644 --- a/specs/binding-tcp.spec/pom.xml +++ b/specs/binding-tcp.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-tls.spec/pom.xml b/specs/binding-tls.spec/pom.xml index d38dbf81e7..8240ad63c8 100644 --- a/specs/binding-tls.spec/pom.xml +++ b/specs/binding-tls.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/binding-ws.spec/pom.xml b/specs/binding-ws.spec/pom.xml index d7f9373844..09a94841d9 100644 --- a/specs/binding-ws.spec/pom.xml +++ b/specs/binding-ws.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/catalog-apicurio.spec/pom.xml b/specs/catalog-apicurio.spec/pom.xml index 8b082e8aca..1d7ba4964e 100644 --- a/specs/catalog-apicurio.spec/pom.xml +++ b/specs/catalog-apicurio.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/catalog-inline.spec/pom.xml b/specs/catalog-inline.spec/pom.xml index 6971804b7d..2e6047ac66 100644 --- a/specs/catalog-inline.spec/pom.xml +++ b/specs/catalog-inline.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/catalog-karapace.spec/pom.xml b/specs/catalog-karapace.spec/pom.xml index 718b1efaab..6491baf7eb 100644 --- a/specs/catalog-karapace.spec/pom.xml +++ b/specs/catalog-karapace.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/engine.spec/pom.xml b/specs/engine.spec/pom.xml index 1d5bd3dafe..47ec94237e 100644 --- a/specs/engine.spec/pom.xml +++ b/specs/engine.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/exporter-otlp.spec/pom.xml b/specs/exporter-otlp.spec/pom.xml index ff8a3d5c12..5fb45e3a06 100644 --- a/specs/exporter-otlp.spec/pom.xml +++ b/specs/exporter-otlp.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/exporter-prometheus.spec/pom.xml b/specs/exporter-prometheus.spec/pom.xml index ab815f1981..e42cd94f30 100644 --- a/specs/exporter-prometheus.spec/pom.xml +++ b/specs/exporter-prometheus.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/exporter-stdout.spec/pom.xml b/specs/exporter-stdout.spec/pom.xml index fab6c4231a..93ae11bbbe 100644 --- a/specs/exporter-stdout.spec/pom.xml +++ b/specs/exporter-stdout.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/guard-jwt.spec/pom.xml b/specs/guard-jwt.spec/pom.xml index ab5d123aed..81c2f9a24c 100644 --- a/specs/guard-jwt.spec/pom.xml +++ b/specs/guard-jwt.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/metrics-grpc.spec/pom.xml b/specs/metrics-grpc.spec/pom.xml index 884c0b1751..6beb93b810 100644 --- a/specs/metrics-grpc.spec/pom.xml +++ b/specs/metrics-grpc.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/metrics-http.spec/pom.xml b/specs/metrics-http.spec/pom.xml index 7d6afa3aaa..961f6fdd63 100644 --- a/specs/metrics-http.spec/pom.xml +++ b/specs/metrics-http.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/metrics-stream.spec/pom.xml b/specs/metrics-stream.spec/pom.xml index 5d51e42f16..e00ec43149 100644 --- a/specs/metrics-stream.spec/pom.xml +++ b/specs/metrics-stream.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/model-avro.spec/pom.xml b/specs/model-avro.spec/pom.xml index efb3437304..2b929788eb 100644 --- a/specs/model-avro.spec/pom.xml +++ b/specs/model-avro.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/model-core.spec/pom.xml b/specs/model-core.spec/pom.xml index f80887668e..79fb07babd 100644 --- a/specs/model-core.spec/pom.xml +++ b/specs/model-core.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/model-json.spec/pom.xml b/specs/model-json.spec/pom.xml index ba45a835dc..212f994903 100644 --- a/specs/model-json.spec/pom.xml +++ b/specs/model-json.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/model-protobuf.spec/pom.xml b/specs/model-protobuf.spec/pom.xml index 146ea32d91..d8525e3455 100644 --- a/specs/model-protobuf.spec/pom.xml +++ b/specs/model-protobuf.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/pom.xml b/specs/pom.xml index 95053f440b..e541df0eb3 100644 --- a/specs/pom.xml +++ b/specs/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla zilla - develop-SNAPSHOT + 0.9.82 ../pom.xml diff --git a/specs/vault-filesystem.spec/pom.xml b/specs/vault-filesystem.spec/pom.xml index b248001b27..f31e388f15 100644 --- a/specs/vault-filesystem.spec/pom.xml +++ b/specs/vault-filesystem.spec/pom.xml @@ -8,7 +8,7 @@ io.aklivity.zilla specs - develop-SNAPSHOT + 0.9.82 ../pom.xml