Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Applied feedback from PR
  • Loading branch information
akrambek committed Jan 14, 2024
commit 196b0ba506a731fafbf560572786148be18a4980
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public final class KafkaClientProduceFactory extends KafkaClientSaslHandshaker i
private static final short RECORD_BATCH_ATTRIBUTES_NO_TIMESTAMP = 0x08;
private static final long RECORD_BATCH_PRODUCER_ID_NONE = -1;
private static final short RECORD_BATCH_PRODUCER_EPOCH_NONE = -1;
private static final int RECORD_BATCH_SEQUENCE_NONE = -1;
private static final int RECORD_BATCH_BASE_SEQUENCE_NONE = -1;
private static final byte RECORD_ATTRIBUTES_NONE = 0;

private static final String TRANSACTION_ID_NONE = null;
Expand Down Expand Up @@ -546,15 +546,14 @@ private int flushRecordInit(

if (client.encodeSlot != NO_SLOT &&
(maxEncodeableBytes > encodePool.slotCapacity() ||
client.producerId != producerId && client.producerEpoch != producerEpoch))
client.producerId != producerId || client.producerEpoch != producerEpoch))
{
client.doEncodeRequestIfNecessary(traceId, budgetId);
}

if (client.producerId == RECORD_BATCH_PRODUCER_ID_NONE ||
client.producerId != producerId)
if (client.producerId == RECORD_BATCH_PRODUCER_ID_NONE)
{
client.sequence = sequence;
client.baseSequence = sequence;
}

client.producerId = producerId;
Expand Down Expand Up @@ -1252,7 +1251,7 @@ private final class KafkaProduceClient extends KafkaSaslClient

private long producerId = RECORD_BATCH_PRODUCER_ID_NONE;
private short producerEpoch = RECORD_BATCH_PRODUCER_EPOCH_NONE;
private int sequence = RECORD_BATCH_SEQUENCE_NONE;
private int baseSequence = RECORD_BATCH_BASE_SEQUENCE_NONE;

KafkaProduceClient(
KafkaProduceStream stream,
Expand Down Expand Up @@ -1895,8 +1894,8 @@ private void doEncodeProduceRequest(
? RECORD_BATCH_ATTRIBUTES_NO_TIMESTAMP
: RECORD_BATCH_ATTRIBUTES_NONE;

final int sequence = client.producerId == RECORD_BATCH_PRODUCER_ID_NONE ? RECORD_BATCH_SEQUENCE_NONE :
client.sequence;
final int baseSequence = client.producerId == RECORD_BATCH_PRODUCER_ID_NONE ? RECORD_BATCH_BASE_SEQUENCE_NONE :
client.baseSequence;

final RecordBatchFW recordBatch = recordBatchRW.wrap(encodeBuffer, encodeProgress, encodeLimit)
.baseOffset(0)
Expand All @@ -1910,7 +1909,7 @@ private void doEncodeProduceRequest(
.maxTimestamp(encodeableRecordBatchTimestampMax)
.producerId(client.producerId)
.producerEpoch(client.producerEpoch)
.baseSequence(sequence)
.baseSequence(baseSequence)
.recordCount(encodeableRecordCount)
.build();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Need to reset producerId and producerEpoch after encoding request so that next request can batch as expected?


Expand Down Expand Up @@ -1942,6 +1941,9 @@ private void doEncodeProduceRequest(
encodeableRecordBatchTimestamp = TIMESTAMP_NONE;
encodedAckMode = encodeableAckMode;
encodeableAckMode = KafkaAckMode.NONE;
client.producerId = RECORD_BATCH_PRODUCER_ID_NONE;
client.producerEpoch = RECORD_BATCH_PRODUCER_EPOCH_NONE;
client.baseSequence = RECORD_BATCH_BASE_SEQUENCE_NONE;

assert encodeSlot != NO_SLOT;
final MutableDirectBuffer encodeSlotBuffer = encodePool.buffer(encodeSlot);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,16 @@ public void shouldProduceMergedMessageValuesByDefault() throws Exception
k3po.finish();
}

@Test
@Configuration("cache.yaml")
@Specification({
"${app}/merged.produce.message.values.producer.id/client",
"${app}/unmerged.produce.message.values.producer.id/server"})
public void shouldProduceMergedMessageValuesWithProducerId() throws Exception
{
k3po.finish();
}

@Test
@Configuration("cache.options.merged.yaml")
@Specification({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ write zilla:data.ext ${kafka:dataEx()
.merged()
.produce()
.timestamp(newTimestamp)
.producerId(1)
.producerEpoch(1)
.producerId(1)
.producerEpoch(1)
.partition(0, 1)
.build()
.build()}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,8 @@ write zilla:data.ext ${kafka:dataEx()
.typeId(zilla:id("kafka"))
.produce()
.timestamp(newTimestamp)
.producerId(1)
.producerEpoch(1)
.sequence(1)
.ackMode("LEADER_ONLY")
.build()
Expand All @@ -145,6 +147,8 @@ write zilla:data.ext ${kafka:dataEx()
.typeId(zilla:id("kafka"))
.produce()
.timestamp(newTimestamp)
.producerId(1)
.producerEpoch(1)
.sequence(2)
.ackMode("LEADER_ONLY")
.build()
Expand Down Expand Up @@ -180,6 +184,8 @@ write zilla:data.ext ${kafka:dataEx()
.typeId(zilla:id("kafka"))
.produce()
.timestamp(newTimestamp)
.producerId(1)
.producerEpoch(1)
.sequence(1)
.ackMode("LEADER_ONLY")
.build()
Expand All @@ -191,6 +197,8 @@ write zilla:data.ext ${kafka:dataEx()
.typeId(zilla:id("kafka"))
.produce()
.timestamp(newTimestamp)
.producerId(1)
.producerEpoch(1)
.sequence(2)
.ackMode("LEADER_ONLY")
.build()
Expand Down Expand Up @@ -226,6 +234,8 @@ write zilla:data.ext ${kafka:dataEx()
.typeId(zilla:id("kafka"))
.produce()
.timestamp(newTimestamp)
.producerId(1)
.producerEpoch(1)
.sequence(1)
.ackMode("LEADER_ONLY")
.build()
Expand All @@ -237,6 +247,8 @@ write zilla:data.ext ${kafka:dataEx()
.typeId(zilla:id("kafka"))
.produce()
.timestamp(newTimestamp)
.producerId(1)
.producerEpoch(1)
.sequence(2)
.ackMode("LEADER_ONLY")
.build()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,8 @@ read zilla:data.ext ${kafka:matchDataEx()
.typeId(zilla:id("kafka"))
.produce()
.timestamp(newTimestamp)
.producerId(1)
.producerEpoch(1)
.sequence(1)
.ackMode("LEADER_ONLY")
.build()
Expand All @@ -144,6 +146,8 @@ read zilla:data.ext ${kafka:matchDataEx()
.typeId(zilla:id("kafka"))
.produce()
.timestamp(newTimestamp)
.producerId(1)
.producerEpoch(1)
.sequence(2)
.ackMode("LEADER_ONLY")
.build()
Expand Down Expand Up @@ -175,6 +179,8 @@ read zilla:data.ext ${kafka:matchDataEx()
.typeId(zilla:id("kafka"))
.produce()
.timestamp(newTimestamp)
.producerId(1)
.producerEpoch(1)
.sequence(1)
.ackMode("LEADER_ONLY")
.build()
Expand All @@ -185,6 +191,8 @@ read zilla:data.ext ${kafka:matchDataEx()
.typeId(zilla:id("kafka"))
.produce()
.timestamp(newTimestamp)
.producerId(1)
.producerEpoch(1)
.sequence(2)
.ackMode("LEADER_ONLY")
.build()
Expand Down Expand Up @@ -216,6 +224,8 @@ read zilla:data.ext ${kafka:matchDataEx()
.typeId(zilla:id("kafka"))
.produce()
.timestamp(newTimestamp)
.producerId(1)
.producerEpoch(1)
.sequence(1)
.ackMode("LEADER_ONLY")
.build()
Expand All @@ -226,6 +236,8 @@ read zilla:data.ext ${kafka:matchDataEx()
.typeId(zilla:id("kafka"))
.produce()
.timestamp(newTimestamp)
.producerId(1)
.producerEpoch(1)
.sequence(2)
.ackMode("LEADER_ONLY")
.build()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
#
# Copyright 2021-2023 Aklivity Inc.
#
# Aklivity licenses this file to you under the Apache License,
# version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#

property deltaMillis 0L
property newTimestamp ${kafka:timestamp() + deltaMillis}

connect "zilla://streams/app0"
option zilla:window 8192
option zilla:transmission "half-duplex"

write zilla:begin.ext ${kafka:beginEx()
.typeId(zilla:id("kafka"))
.meta()
.topic("test")
.build()
.build()}

connected

read zilla:begin.ext ${kafka:beginEx()
.typeId(zilla:id("kafka"))
.meta()
.topic("test")
.build()
.build()}

read zilla:data.ext ${kafka:dataEx()
.typeId(zilla:id("kafka"))
.meta()
.partition(0, 177)
.build()
.build()}

read notify ROUTED_BROKER_CLIENT

connect await ROUTED_BROKER_CLIENT
"zilla://streams/app0"
option zilla:window 8192
option zilla:transmission "half-duplex"
option zilla:affinity 0xb1

write zilla:begin.ext ${kafka:beginEx()
.typeId(zilla:id("kafka"))
.produce()
.topic("test")
.partition(0)
.build()
.build()}

connected

read zilla:begin.ext ${kafka:beginEx()
.typeId(zilla:id("kafka"))
.produce()
.topic("test")
.partition(0)
.build()
.build()}

write zilla:data.ext ${kafka:dataEx()
.typeId(zilla:id("kafka"))
.produce()
.timestamp(newTimestamp)
.producerId(1)
.producerEpoch(1)
.sequence(0)
.build()
.build()}
write "Hello, world"
write flush

write zilla:data.ext ${kafka:dataEx()
.typeId(zilla:id("kafka"))
.produce()
.timestamp(newTimestamp)
.producerId(2)
.producerEpoch(2)
.sequence(0)
.build()
.build()}
write "Hello, again"
write flush
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
#
# Copyright 2021-2023 Aklivity Inc.
#
# Aklivity licenses this file to you under the Apache License,
# version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#

property serverAddress "zilla://streams/app0"

accept ${serverAddress}
option zilla:window 8192
option zilla:padding 512
option zilla:transmission "half-duplex"

accepted

read zilla:begin.ext ${kafka:beginEx()
.typeId(zilla:id("kafka"))
.meta()
.topic("test")
.build()
.build()}

connected

write zilla:begin.ext ${kafka:beginEx()
.typeId(zilla:id("kafka"))
.meta()
.topic("test")
.build()
.build()}
write flush

write zilla:data.ext ${kafka:dataEx()
.typeId(zilla:id("kafka"))
.meta()
.partition(0, 177)
.build()
.build()}
write flush

accepted

read zilla:begin.ext ${kafka:beginEx()
.typeId(zilla:id("kafka"))
.produce()
.topic("test")
.partition(0)
.build()
.build()}

connected

write zilla:begin.ext ${kafka:beginEx()
.typeId(zilla:id("kafka"))
.produce()
.topic("test")
.partition(0)
.build()
.build()}

read zilla:data.ext ${kafka:matchDataEx()
.typeId(zilla:id("kafka"))
.produce()
.producerId(1)
.producerEpoch(1)
.sequence(0)
.build()
.build()}

read "Hello, world"

read zilla:data.ext ${kafka:matchDataEx()
.typeId(zilla:id("kafka"))
.produce()
.producerId(2)
.producerEpoch(2)
.sequence(0)
.build()
.build()}

read "Hello, again"
Loading