Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
901f0b2
separating publish streams based on qos
bmaidics Jan 10, 2024
79ddafd
Fix test
bmaidics Jan 11, 2024
1265671
Checkpoint
bmaidics Jan 9, 2024
7847580
Checkpoint
bmaidics Jan 9, 2024
7895cd5
check
bmaidics Jan 9, 2024
f831ff8
start of idempotent work
bmaidics Jan 13, 2024
412019a
Merge remote-tracking branch 'upstream/feature/mqtt-kafka' into featu…
bmaidics Jan 15, 2024
167a33a
Optimize memory allocation for mqtt-kafka offset tracking (#694)
bmaidics Jan 15, 2024
cf08e0f
checkpoint
bmaidics Jan 15, 2024
80c0e09
checkpoint
bmaidics Jan 18, 2024
62db2cb
Merge remote-tracking branch 'upstream/feature/mqtt-kafka' into qos2_…
bmaidics Jan 18, 2024
49a7b35
Checkpoint with retained offsetCommit stream
bmaidics Jan 18, 2024
92b262a
checkpoint
bmaidics Jan 19, 2024
29d24f7
checkpoint
bmaidics Jan 22, 2024
f9fe624
mqtt-kafka checkpoint
bmaidics Jan 23, 2024
5195271
checkpoint
bmaidics Jan 24, 2024
99f1c1f
checkpoint
bmaidics Jan 24, 2024
8db664d
Fixes
bmaidics Jan 24, 2024
7038c04
Fix flaky test
bmaidics Jan 24, 2024
e00dd8a
Merge remote-tracking branch 'upstream/feature/mqtt-kafka' into qos2_…
bmaidics Jan 25, 2024
e3d92ee
fixes
bmaidics Jan 25, 2024
31e26ed
fix dump
attilakreiner Jan 25, 2024
f4b27c1
Fix init produce id request
akrambek Jan 25, 2024
9c5fd24
fix
bmaidics Jan 26, 2024
35c6976
Fix bug
bmaidics Jan 26, 2024
a3c7e09
Fix
bmaidics Jan 29, 2024
5cf9068
Don't flush early if the sequence number is not set
akrambek Jan 29, 2024
bc38ac7
Draft
bmaidics Jan 31, 2024
724bd70
checkpoint
bmaidics Feb 1, 2024
7df5123
Fixes
bmaidics Feb 1, 2024
4204bc1
Adrress review comments
bmaidics Feb 1, 2024
2a0204f
Merge remote-tracking branch 'upstream/feature/mqtt-kafka' into qos2_…
bmaidics Feb 1, 2024
ebd9cee
Merge fixes
bmaidics Feb 1, 2024
8988e6b
Include producerId and producerEpoch into cache entry
akrambek Feb 2, 2024
739f79d
fix dump
attilakreiner Feb 2, 2024
878ec78
Fix qos2 large message
bmaidics Feb 3, 2024
0563e13
Fix typo
akrambek Feb 5, 2024
7f2ca2a
Merge branch 'qos2_idempontent' of github.com:bmaidics/zilla into qos…
akrambek Feb 5, 2024
7d9afee
reviews
bmaidics Feb 5, 2024
c274636
more feedback
bmaidics Feb 5, 2024
215791e
checkpoint
bmaidics Feb 6, 2024
d3635b3
Refactor
bmaidics Feb 6, 2024
007e8d1
Adjust code coverage ratio
jfallows Feb 6, 2024
ccae27c
Ignore IT that fails only on GitHub Actions, see issue #786
jfallows Feb 6, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Adrress review comments
  • Loading branch information
bmaidics committed Feb 1, 2024
commit 4204bc11b025f78b010cb7cfd6b1bdfd6778265e
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ public class MqttKafkaConfiguration extends Configuration
public static final IntPropertyDef WILL_STREAM_RECONNECT_DELAY;
public static final BooleanPropertyDef BOOTSTRAP_AVAILABLE;
public static final IntPropertyDef BOOTSTRAP_STREAM_RECONNECT_DELAY;
public static final IntPropertyDef PUBLISH_MAX_QOS;

static
{
Expand All @@ -58,7 +57,6 @@ public class MqttKafkaConfiguration extends Configuration
WILL_STREAM_RECONNECT_DELAY = config.property("will.stream.reconnect", 2);
BOOTSTRAP_AVAILABLE = config.property("bootstrap.available", true);
BOOTSTRAP_STREAM_RECONNECT_DELAY = config.property("bootstrap.stream.reconnect", 2);
PUBLISH_MAX_QOS = config.property("publish.max.qos", 2);
MQTT_KAFKA_CONFIG = config;
}

Expand Down Expand Up @@ -117,10 +115,6 @@ public int bootstrapStreamReconnectDelay()
{
return BOOTSTRAP_STREAM_RECONNECT_DELAY.getAsInt(this);
}
public int publishMaxQos()
{
return PUBLISH_MAX_QOS.getAsInt(this);
}


private static StringSupplier decodeStringSupplier(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.InstanceId;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.MqttKafkaConfiguration;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.config.MqttKafkaBindingConfig;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.stream.MqttKafkaSessionFactory.PublishClientMetadata;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.OctetsFW;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.BeginFW;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.ExtensionFW;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,9 @@
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.config.MqttKafkaBindingConfig;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.config.MqttKafkaHeaderHelper;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.config.MqttKafkaRouteConfig;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.stream.MqttKafkaSessionFactory.KafkaGroup;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.stream.MqttKafkaSessionFactory.KafkaTopicPartition;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.stream.MqttKafkaSessionFactory.PublishClientMetadata;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.stream.MqttKafkaSessionFactory.PublishOffsetMetadata;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.stream.PublishClientMetadata.KafkaGroup;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.stream.PublishClientMetadata.KafkaTopicPartition;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.stream.PublishClientMetadata.PublishOffsetMetadata;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.Array32FW;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.Flyweight;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.KafkaAckMode;
Expand All @@ -69,7 +68,6 @@
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.KafkaResetExFW;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.MqttBeginExFW;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.MqttDataExFW;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.MqttFlushExFW;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.MqttPublishBeginExFW;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.MqttPublishDataExFW;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.stream.MqttPublishOffsetMetadataFW;
Expand Down Expand Up @@ -135,7 +133,6 @@ public class MqttKafkaPublishFactory implements MqttKafkaStreamFactory
private final KafkaBeginExFW.Builder kafkaBeginExRW = new KafkaBeginExFW.Builder();
private final KafkaFlushExFW.Builder kafkaFlushExRW = new KafkaFlushExFW.Builder();
private final KafkaDataExFW.Builder kafkaDataExRW = new KafkaDataExFW.Builder();
private final MqttFlushExFW.Builder mqttFlushExRW = new MqttFlushExFW.Builder();
private final MqttResetExFW.Builder mqttResetExRW = new MqttResetExFW.Builder();
private final MqttPublishOffsetMetadataFW.Builder mqttOffsetMetadataRW = new MqttPublishOffsetMetadataFW.Builder();
private final Array32FW.Builder<KafkaHeaderFW.Builder, KafkaHeaderFW> kafkaHeadersRW =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.config.MqttKafkaBindingConfig;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.config.MqttKafkaHeaderHelper;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.config.MqttKafkaRouteConfig;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.stream.PublishClientMetadata.KafkaGroup;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.stream.PublishClientMetadata.KafkaTopicPartition;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.stream.PublishClientMetadata.PublishOffsetMetadata;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.Array32FW;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.Flyweight;
import io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.types.KafkaAckMode;
Expand Down Expand Up @@ -262,7 +265,6 @@ public class MqttKafkaSessionFactory implements MqttKafkaStreamFactory
private final InstanceId instanceId;
private final boolean willAvailable;
private final int reconnectDelay;
private final int publishMaxQos;
private final Int2ObjectHashMap<String16FW> qosLevels;
private final Long2ObjectHashMap<PublishClientMetadata> clientMetadata;

Expand Down Expand Up @@ -310,7 +312,6 @@ public MqttKafkaSessionFactory(
this.sessionExpiryIds = new Object2LongHashMap<>(-1);
this.instanceId = instanceId;
this.reconnectDelay = config.willStreamReconnectDelay();
this.publishMaxQos = config.publishMaxQos();
this.qosLevels = new Int2ObjectHashMap<>();
this.qosLevels.put(0, new String16FW("0"));
this.qosLevels.put(1, new String16FW("1"));
Expand Down Expand Up @@ -436,8 +437,9 @@ public final class MqttSessionProxy
private int sessionPadding;
private String willId;
private int delay;
private int publishMaxQos;
private int unfetchedKafkaTopics;
private Long2ObjectHashMap<PublishOffsetMetadata> offsets;
private Long2ObjectHashMap<PublishClientMetadata.PublishOffsetMetadata> offsets;
private Int2ObjectHashMap<KafkaTopicPartition> partitions;
private Int2ObjectHashMap<KafkaTopicPartition> retainedPartitions;
private PublishClientMetadata metadata;
Expand Down Expand Up @@ -537,6 +539,7 @@ private void onMqttBegin(

sessionExpiryMillis = (int) SECONDS.toMillis(mqttSessionBeginEx.expiry());
sessionFlags = mqttSessionBeginEx.flags();
publishMaxQos = mqttSessionBeginEx.publishQosMax();
Comment thread
bmaidics marked this conversation as resolved.
Outdated

if (!isSetWillFlag(sessionFlags) || isSetCleanStart(sessionFlags))
{
Expand Down Expand Up @@ -2891,7 +2894,7 @@ private void onKafkaBegin(
.session(sessionBuilder -> sessionBuilder
.flags(delegate.sessionFlags)
.expiry((int) TimeUnit.MILLISECONDS.toSeconds(delegate.sessionExpiryMillis))
.qosMax(MQTT_KAFKA_MAX_QOS)
.subscribeQosMax(MQTT_KAFKA_MAX_QOS)
.capabilities(MQTT_KAFKA_CAPABILITIES)
.clientId(delegate.clientId))
.build();
Expand Down Expand Up @@ -3569,7 +3572,7 @@ private void onKafkaFlush(
{
delegate.session.doKafkaEnd(traceId, authorization);

if (publishMaxQos < 2)
if (delegate.publishMaxQos < 2)
{
final long routedId = delegate.session.routedId;
delegate.session = new KafkaSessionStateProxy(originId, routedId, delegate);
Expand Down Expand Up @@ -3635,7 +3638,7 @@ private void onKafkaBegin(
delegate.sessionExpiryMillis = sessionExpiryMillisInRange;
}

if (publishMaxQos == 2)
if (delegate.publishMaxQos == 2)
{
doKafkaWindow(traceId, authorization, 0, 0, 0);
}
Expand All @@ -3646,7 +3649,7 @@ private void onKafkaBegin(
.session(sessionBuilder -> sessionBuilder
.flags(delegate.sessionFlags)
.expiry((int) TimeUnit.MILLISECONDS.toSeconds(delegate.sessionExpiryMillis))
.qosMax(MQTT_KAFKA_MAX_QOS)
.subscribeQosMax(MQTT_KAFKA_MAX_QOS)
.capabilities(MQTT_KAFKA_CAPABILITIES)
.clientId(delegate.clientId))
.build();
Expand Down Expand Up @@ -4275,7 +4278,7 @@ else if (delegate.unfetchedKafkaTopics == 0)
sessionBuilder
.flags(delegate.sessionFlags)
.expiry((int) TimeUnit.MILLISECONDS.toSeconds(delegate.sessionExpiryMillis))
.qosMax(MQTT_KAFKA_MAX_QOS)
.subscribeQosMax(MQTT_KAFKA_MAX_QOS)
.capabilities(MQTT_KAFKA_CAPABILITIES)
.clientId(delegate.clientId);

Expand Down Expand Up @@ -4797,7 +4800,7 @@ else if (delegate.retainedPartitions.containsKey(packetId))
sessionBuilder
.flags(delegate.sessionFlags)
.expiry((int) TimeUnit.MILLISECONDS.toSeconds(delegate.sessionExpiryMillis))
.qosMax(MQTT_KAFKA_MAX_QOS)
.subscribeQosMax(MQTT_KAFKA_MAX_QOS)
.capabilities(MQTT_KAFKA_CAPABILITIES)
.clientId(delegate.clientId);

Expand Down Expand Up @@ -4929,27 +4932,6 @@ private String16FW offsetMetadataToString(
offsetMetadata.offset(), offsetMetadata.limit()));
}

public static final class PublishClientMetadata
{
final Long2ObjectHashMap<PublishOffsetMetadata> offsets;
final Int2ObjectHashMap<KafkaTopicPartition> partitions;
final Int2ObjectHashMap<KafkaTopicPartition> retainedPartitions;
final Long2LongHashMap leaderEpochs;

KafkaGroup group;

private PublishClientMetadata(
Long2ObjectHashMap<PublishOffsetMetadata> offsets,
Int2ObjectHashMap<KafkaTopicPartition> partitions,
Int2ObjectHashMap<KafkaTopicPartition> retainedPartitions,
Long2LongHashMap leaderEpochs)
{
this.offsets = offsets;
this.partitions = partitions;
this.retainedPartitions = retainedPartitions;
this.leaderEpochs = leaderEpochs;
}
}

private static final class KafkaPartition
{
Expand All @@ -4966,70 +4948,6 @@ private KafkaPartition(
}
}

public static final class KafkaGroup
{
public final String instanceId;
public final String groupId;
public final String memberId;
public final int generationId;

KafkaGroup(
String instanceId,
String groupId,
String memberId,
int generationId)
{
this.instanceId = instanceId;
this.groupId = groupId;
this.memberId = memberId;
this.generationId = generationId;
}
}

public static final class KafkaTopicPartition
{
private final String topic;
private final int partitionId;

KafkaTopicPartition(
String topic,
int partitionId)
{
this.topic = topic;
this.partitionId = partitionId;
}
}

public static final class PublishOffsetMetadata
{
public final long producerId;
public final short producerEpoch;
public final IntArrayList packetIds;

public long sequence;

PublishOffsetMetadata(
long producerId,
short producerEpoch)
{
this.sequence = 1;
this.producerId = producerId;
this.producerEpoch = producerEpoch;
this.packetIds = new IntArrayList();
}

PublishOffsetMetadata(
long producerId,
short producerEpoch,
IntArrayList packetIds)
{
this.sequence = 1;
this.producerId = producerId;
this.producerEpoch = producerEpoch;
this.packetIds = packetIds;
}
}

private void doBegin(
MessageConsumer receiver,
long originId,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
/*
* Copyright 2021-2023 Aklivity Inc
*
* Licensed under the Aklivity Community License (the "License"); you may not use
* this file except in compliance with the License. You may obtain a copy of the
* License at
*
* https://www.aklivity.io/aklivity-community-license/
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package io.aklivity.zilla.runtime.binding.mqtt.kafka.internal.stream;

import org.agrona.collections.Int2ObjectHashMap;
import org.agrona.collections.IntArrayList;
import org.agrona.collections.Long2LongHashMap;
import org.agrona.collections.Long2ObjectHashMap;

public class PublishClientMetadata
Comment thread
bmaidics marked this conversation as resolved.
Outdated
{
final Long2ObjectHashMap<PublishOffsetMetadata> offsets;
final Int2ObjectHashMap<KafkaTopicPartition> partitions;
final Int2ObjectHashMap<KafkaTopicPartition> retainedPartitions;
final Long2LongHashMap leaderEpochs;

KafkaGroup group;

public PublishClientMetadata(
Long2ObjectHashMap<PublishOffsetMetadata> offsets,
Int2ObjectHashMap<KafkaTopicPartition> partitions,
Int2ObjectHashMap<KafkaTopicPartition> retainedPartitions,
Long2LongHashMap leaderEpochs)
{
this.offsets = offsets;
this.partitions = partitions;
this.retainedPartitions = retainedPartitions;
this.leaderEpochs = leaderEpochs;
}

public static final class KafkaGroup
{
public final String instanceId;
public final String groupId;
public final String memberId;
public final int generationId;

KafkaGroup(
String instanceId,
String groupId,
String memberId,
int generationId)
{
this.instanceId = instanceId;
this.groupId = groupId;
this.memberId = memberId;
this.generationId = generationId;
}
}

public static final class KafkaTopicPartition
{
public final String topic;
public final int partitionId;

KafkaTopicPartition(
String topic,
int partitionId)
{
this.topic = topic;
this.partitionId = partitionId;
}
}

public static final class PublishOffsetMetadata
Comment thread
bmaidics marked this conversation as resolved.
Outdated
{
public final long producerId;
public final short producerEpoch;
public final IntArrayList packetIds;

public long sequence;

PublishOffsetMetadata(
long producerId,
short producerEpoch)
{
this.sequence = 1;
this.producerId = producerId;
this.producerEpoch = producerEpoch;
this.packetIds = new IntArrayList();
Comment thread
bmaidics marked this conversation as resolved.
Outdated
}

PublishOffsetMetadata(
long producerId,
short producerEpoch,
IntArrayList packetIds)
{
this.sequence = 1;
this.producerId = producerId;
this.producerEpoch = producerEpoch;
this.packetIds = packetIds;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -1567,7 +1567,7 @@ private int onDecodeConnack(
.session(sessionBuilder -> sessionBuilder
.flags(flags)
.expiry((int) TimeUnit.MILLISECONDS.toSeconds(sessionExpiry))
.qosMax(maximumQos)
.subscribeQosMax(maximumQos)
.packetSizeMax(maximumPacketSize)
.capabilities(capabilities)
.clientId(clientId))
Expand All @@ -1593,7 +1593,7 @@ private int onDecodeConnack(
.session(sessionBuilder -> sessionBuilder
.flags(flags)
.expiry(sessionExpiry)
.qosMax(maximumQos)
.subscribeQosMax(maximumQos)
.packetSizeMax(maximumPacketSize)
.capabilities(capabilities)
.clientId(clientId))
Expand Down
Loading