-
Notifications
You must be signed in to change notification settings - Fork 70
Mqtt-kafka single group support cont #731
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 5 commits
0ca1a1e
ee72e10
f165cd1
52b778d
5238097
196b0ba
0808b40
66988b8
89f2857
276ca31
f9292af
13a19ff
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -93,9 +93,9 @@ public final class KafkaClientProduceFactory extends KafkaClientSaslHandshaker i | |
| private static final byte RECORD_BATCH_MAGIC = 2; | ||
| private static final short RECORD_BATCH_ATTRIBUTES_NONE = 0; | ||
| private static final short RECORD_BATCH_ATTRIBUTES_NO_TIMESTAMP = 0x08; | ||
| private static final int RECORD_BATCH_PRODUCER_ID_NONE = -1; | ||
| private static final long RECORD_BATCH_PRODUCER_ID_NONE = -1; | ||
| private static final short RECORD_BATCH_PRODUCER_EPOCH_NONE = -1; | ||
| private static final short RECORD_BATCH_SEQUENCE_NONE = -1; | ||
| private static final int RECORD_BATCH_SEQUENCE_NONE = -1; | ||
| private static final byte RECORD_ATTRIBUTES_NONE = 0; | ||
|
|
||
| private static final String TRANSACTION_ID_NONE = null; | ||
|
|
@@ -531,6 +531,9 @@ private int flushRecordInit( | |
| assert kafkaDataEx.kind() == KafkaDataExFW.KIND_PRODUCE; | ||
| final KafkaProduceDataExFW kafkaProduceDataEx = kafkaDataEx.produce(); | ||
| final long timestamp = kafkaProduceDataEx.timestamp(); | ||
| final long producerId = kafkaProduceDataEx.producerId(); | ||
| final short producerEpoch = kafkaProduceDataEx.producerEpoch(); | ||
| final int sequence = kafkaProduceDataEx.sequence(); | ||
| final KafkaAckMode ackMode = kafkaProduceDataEx.ackMode().get(); | ||
| final KafkaKeyFW key = kafkaProduceDataEx.key(); | ||
| final Array32FW<KafkaHeaderFW> headers = kafkaProduceDataEx.headers(); | ||
|
|
@@ -542,11 +545,21 @@ private int flushRecordInit( | |
| final int maxEncodeableBytes = client.encodeSlotLimit + client.valueCompleteSize + produceRecordFramingSize; | ||
|
|
||
| if (client.encodeSlot != NO_SLOT && | ||
| maxEncodeableBytes > encodePool.slotCapacity()) | ||
| (maxEncodeableBytes > encodePool.slotCapacity() || | ||
| client.producerId != producerId && client.producerEpoch != producerEpoch)) | ||
| { | ||
| client.doEncodeRequestIfNecessary(traceId, budgetId); | ||
| } | ||
|
|
||
| if (client.producerId == RECORD_BATCH_PRODUCER_ID_NONE || | ||
| client.producerId != producerId) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should this be Please remove extra whitespace around |
||
| { | ||
| client.sequence = sequence; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Suggest we rename to |
||
| } | ||
|
|
||
| client.producerId = producerId; | ||
| client.producerEpoch = producerEpoch; | ||
|
|
||
| client.doEncodeRecordInit(traceId, timestamp, ackMode, key, payload, headers); | ||
| if (client.encodeSlot != NO_SLOT) | ||
| { | ||
|
|
@@ -1237,6 +1250,10 @@ private final class KafkaProduceClient extends KafkaSaslClient | |
| private LongLongConsumer encoder; | ||
| private boolean flushable; | ||
|
|
||
| private long producerId = RECORD_BATCH_PRODUCER_ID_NONE; | ||
| private short producerEpoch = RECORD_BATCH_PRODUCER_EPOCH_NONE; | ||
| private int sequence = RECORD_BATCH_SEQUENCE_NONE; | ||
|
|
||
| KafkaProduceClient( | ||
| KafkaProduceStream stream, | ||
| long resolvedId, | ||
|
|
@@ -1878,6 +1895,9 @@ private void doEncodeProduceRequest( | |
| ? RECORD_BATCH_ATTRIBUTES_NO_TIMESTAMP | ||
| : RECORD_BATCH_ATTRIBUTES_NONE; | ||
|
|
||
| final int sequence = client.producerId == RECORD_BATCH_PRODUCER_ID_NONE ? RECORD_BATCH_SEQUENCE_NONE : | ||
| client.sequence; | ||
|
|
||
| final RecordBatchFW recordBatch = recordBatchRW.wrap(encodeBuffer, encodeProgress, encodeLimit) | ||
| .baseOffset(0) | ||
| .length(recordBatchLength) | ||
|
|
@@ -1888,9 +1908,9 @@ private void doEncodeProduceRequest( | |
| .lastOffsetDelta(encodeableRecordCount - 1) | ||
| .firstTimestamp(encodeableRecordBatchTimestamp) | ||
| .maxTimestamp(encodeableRecordBatchTimestampMax) | ||
| .producerId(RECORD_BATCH_PRODUCER_ID_NONE) | ||
| .producerEpoch(RECORD_BATCH_PRODUCER_EPOCH_NONE) | ||
| .baseSequence(RECORD_BATCH_SEQUENCE_NONE) | ||
| .producerId(client.producerId) | ||
| .producerEpoch(client.producerEpoch) | ||
| .baseSequence(sequence) | ||
| .recordCount(encodeableRecordCount) | ||
| .build(); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Need to reset |
||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| /* | ||
| * Copyright 2021-2023 Aklivity Inc. | ||
| * | ||
| * Aklivity licenses this file to you under the Apache License, | ||
| * version 2.0 (the "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at: | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | ||
| * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| * License for the specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
| package io.aklivity.zilla.runtime.binding.kafka.internal.stream; | ||
|
|
||
| import static io.aklivity.zilla.runtime.binding.kafka.internal.KafkaConfigurationTest.KAFKA_CLIENT_SASL_SCRAM_NONCE_NAME; | ||
| import static java.util.concurrent.TimeUnit.SECONDS; | ||
| import static org.junit.rules.RuleChain.outerRule; | ||
|
|
||
| import org.junit.Rule; | ||
| import org.junit.Test; | ||
| import org.junit.rules.DisableOnDebug; | ||
| import org.junit.rules.TestRule; | ||
| import org.junit.rules.Timeout; | ||
| import org.kaazing.k3po.junit.annotation.Specification; | ||
| import org.kaazing.k3po.junit.rules.K3poRule; | ||
|
|
||
| import io.aklivity.zilla.runtime.engine.test.EngineRule; | ||
| import io.aklivity.zilla.runtime.engine.test.annotation.Configuration; | ||
| import io.aklivity.zilla.runtime.engine.test.annotation.Configure; | ||
|
|
||
| public class ClientInitProducerIdSaslIT | ||
| { | ||
| private final K3poRule k3po = new K3poRule() | ||
| .addScriptRoot("net", "io/aklivity/zilla/specs/binding/kafka/streams/network/init.producer.id.v4.sasl.handshake.v1") | ||
| .addScriptRoot("app", "io/aklivity/zilla/specs/binding/kafka/streams/application/init.producer.id"); | ||
|
|
||
| private final TestRule timeout = new DisableOnDebug(new Timeout(15, SECONDS)); | ||
|
|
||
| private final EngineRule engine = new EngineRule() | ||
| .directory("target/zilla-itests") | ||
| .countersBufferCapacity(8192) | ||
| .configurationRoot("io/aklivity/zilla/specs/binding/kafka/config") | ||
| .external("net0") | ||
| .clean(); | ||
|
|
||
| @Rule | ||
| public final TestRule chain = outerRule(engine).around(k3po).around(timeout); | ||
|
|
||
|
|
||
| @Test | ||
| @Configuration("client.options.sasl.plain.yaml") | ||
| @Specification({ | ||
| "${app}/produce.new.id/client", | ||
| "${net}/produce.new.id.sasl.plain/server"}) | ||
| public void shouldGenerateNewProducerIdWithSaslPlain() throws Exception | ||
| { | ||
| k3po.finish(); | ||
| } | ||
|
|
||
| @Test | ||
| @Configuration("client.options.sasl.scram.yaml") | ||
| @Specification({ | ||
| "${app}/produce.new.id/client", | ||
| "${net}/produce.new.id.sasl.scram/server"}) | ||
| @Configure(name = KAFKA_CLIENT_SASL_SCRAM_NONCE_NAME, | ||
| value = "io.aklivity.zilla.runtime.binding.kafka.internal.stream.ClientInitProducerIdSaslIT::supplyNonce") | ||
| public void shouldGenerateNewProducerIdWithSaslScram() throws Exception | ||
| { | ||
| k3po.finish(); | ||
| } | ||
|
|
||
| public static String supplyNonce() | ||
| { | ||
| return "fyko+d2lbbFgONRv9qkxdawL"; | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -41,3 +41,4 @@ write zilla:begin.ext ${kafka:beginEx() | |
| .producerEpoch(2) | ||
| .build() | ||
| .build()} | ||
| write flush | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
only need one of these to differ to trigger request since both are expected to be consistent for entire batch, right?