Finalize replay envelopes

buildSrc/src/main/java/Config.kt

@@ -195,6 +195,7 @@ object Config {
         val jsonUnit = "net.javacrumbs.json-unit:json-unit:2.32.0"
         val hsqldb = "org.hsqldb:hsqldb:2.6.1"
         val javaFaker = "com.github.javafaker:javafaker:1.0.2"
+        val msgpack = "org.msgpack:msgpack-core:0.9.8"
     }
 
     object QualityPlugins {

sentry/build.gradle.kts

@@ -32,6 +32,7 @@ dependencies {
     testImplementation(Config.TestLibs.mockitoInline)
     testImplementation(Config.TestLibs.awaitility)
     testImplementation(Config.TestLibs.javaFaker)
+    testImplementation(Config.TestLibs.msgpack)
     testImplementation(projects.sentryTestSupport)
 }
 

sentry/src/main/java/io/sentry/JsonObjectWriter.java

@@ -52,6 +52,12 @@ public JsonObjectWriter value(final @Nullable String value) throws IOException {
     return this;
   }
 
+  @Override
+  public ObjectWriter jsonValue(@Nullable String value) throws IOException {
+    jsonWriter.jsonValue(value);
+    return this;
+  }
+
   @Override
   public JsonObjectWriter nullValue() throws IOException {
     jsonWriter.nullValue();
@@ -103,6 +109,11 @@ public JsonObjectWriter value(final @NotNull ILogger logger, final @Nullable Obj
     return this;
   }
 
+  @Override
+  public void setLenient(final boolean lenient) {
+    jsonWriter.setLenient(lenient);
+  }
+
   public void setIndent(final @NotNull String indent) {
     jsonWriter.setIndent(indent);
   }

sentry/src/main/java/io/sentry/JsonSerializer.java

@@ -92,6 +92,7 @@ public JsonSerializer(@NotNull SentryOptions options) {
     deserializersByClass.put(
         ProfileMeasurementValue.class, new ProfileMeasurementValue.Deserializer());
     deserializersByClass.put(Request.class, new Request.Deserializer());
+    deserializersByClass.put(ReplayRecording.class, new ReplayRecording.Deserializer());
     deserializersByClass.put(RRWebEventType.class, new RRWebEventType.Deserializer());
     deserializersByClass.put(RRWebMetaEvent.class, new RRWebMetaEvent.Deserializer());
     deserializersByClass.put(RRWebVideoEvent.class, new RRWebVideoEvent.Deserializer());
@@ -107,6 +108,7 @@ public JsonSerializer(@NotNull SentryOptions options) {
     deserializersByClass.put(SentryLockReason.class, new SentryLockReason.Deserializer());
     deserializersByClass.put(SentryPackage.class, new SentryPackage.Deserializer());
     deserializersByClass.put(SentryRuntime.class, new SentryRuntime.Deserializer());
+    deserializersByClass.put(SentryReplayEvent.class, new SentryReplayEvent.Deserializer());
     deserializersByClass.put(SentrySpan.class, new SentrySpan.Deserializer());
     deserializersByClass.put(SentryStackFrame.class, new SentryStackFrame.Deserializer());
     deserializersByClass.put(SentryStackTrace.class, new SentryStackTrace.Deserializer());

sentry/src/main/java/io/sentry/ObjectWriter.java

@@ -17,6 +17,8 @@ public interface ObjectWriter {
 
   ObjectWriter value(final @Nullable String value) throws IOException;
 
+  ObjectWriter jsonValue(final @Nullable String value) throws IOException;
+
   ObjectWriter nullValue() throws IOException;
 
   ObjectWriter value(final boolean value) throws IOException;
@@ -31,4 +33,6 @@ public interface ObjectWriter {
 
   ObjectWriter value(final @NotNull ILogger logger, final @Nullable Object object)
       throws IOException;
+
+  void setLenient(boolean lenient);
 }

sentry/src/main/java/io/sentry/ReplayRecording.java

@@ -1,8 +1,15 @@
 package io.sentry;
 
 import io.sentry.rrweb.RRWebEvent;
+import io.sentry.rrweb.RRWebEventType;
+import io.sentry.rrweb.RRWebMetaEvent;
+import io.sentry.rrweb.RRWebVideoEvent;
+import io.sentry.util.MapObjectReader;
+import io.sentry.util.Objects;
 import io.sentry.vendor.gson.stream.JsonToken;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -37,6 +44,19 @@ public void setPayload(@Nullable List<? extends RRWebEvent> payload) {
     this.payload = payload;
   }
 
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+    ReplayRecording that = (ReplayRecording) o;
+    return Objects.equals(segmentId, that.segmentId) && Objects.equals(payload, that.payload);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(segmentId, payload);
+  }
+
   @Override
   public void serialize(final @NotNull ObjectWriter writer, final @NotNull ILogger logger)
       throws IOException {
@@ -52,6 +72,15 @@ public void serialize(final @NotNull ObjectWriter writer, final @NotNull ILogger
       }
     }
     writer.endObject();
+
+    // {"segment_id":0}\n{json-serialized-rrweb-protocol}
+
+    writer.setLenient(true);
+    writer.jsonValue("\n");
+    if (payload != null) {
+      writer.value(logger, payload);
+    }
+    writer.setLenient(false);
   }
 
   @Override
@@ -66,14 +95,16 @@ public void setUnknown(@Nullable Map<String, Object> unknown) {
 
   public static final class Deserializer implements JsonDeserializer<ReplayRecording> {
 
+    @SuppressWarnings("unchecked")
     @Override
     public @NotNull ReplayRecording deserialize(
-        @NotNull JsonObjectReader reader, @NotNull ILogger logger) throws Exception {
+        @NotNull ObjectReader reader, @NotNull ILogger logger) throws Exception {
 
       final ReplayRecording replay = new ReplayRecording();
 
       @Nullable Map<String, Object> unknown = null;
       @Nullable Integer segmentId = null;
+      @Nullable List<RRWebEvent> payload = null;
 
       reader.beginObject();
       while (reader.peek() == JsonToken.NAME) {
@@ -92,7 +123,58 @@ public static final class Deserializer implements JsonDeserializer<ReplayRecordi
       }
       reader.endObject();
 
+      // {"segment_id":0}\n{json-serialized-rrweb-protocol}
+
+      reader.setLenient(true);
+      List<Object> events = (List<Object>) reader.nextObjectOrNull();
+      reader.setLenient(false);
+
+      // since we lose the type of an rrweb event at runtime, we have to recover it from a map
+      if (events != null) {
+        payload = new ArrayList<>(events.size());
+        for (Object event : events) {
+          if (event instanceof Map) {
+            final Map<String, Object> eventMap = (Map<String, Object>) event;
+            final ObjectReader mapReader = new MapObjectReader(eventMap);
+            for (Map.Entry<String, Object> entry : eventMap.entrySet()) {
+              final String key = entry.getKey();
+              final Object value = entry.getValue();
+              if (key.equals("type")) {
+                RRWebEventType type = RRWebEventType.values()[(int) value];
+                switch (type) {
+                  case Meta:
+                    final RRWebEvent metaEvent =
+                        new RRWebMetaEvent.Deserializer().deserialize(mapReader, logger);
+                    payload.add(metaEvent);
+                    break;
+                  case Custom:
+                    final Map<String, Object> data =
+                        (Map<String, Object>) eventMap.getOrDefault("data", Collections.emptyMap());
+                    final String tag =
+                        (String) data.getOrDefault(RRWebEvent.JsonKeys.TAG, "default");
+                    switch (tag) {
+                      case RRWebVideoEvent.EVENT_TAG:
+                        final RRWebEvent videoEvent =
+                            new RRWebVideoEvent.Deserializer().deserialize(mapReader, logger);
+                        payload.add(videoEvent);
+                        break;
+                      default:
+                        logger.log(SentryLevel.DEBUG, "Unsupported rrweb event type %s", type);
+                        break;
+                    }
+                    break;
+                  default:
+                    logger.log(SentryLevel.DEBUG, "Unsupported rrweb event type %s", type);
+                    break;
+                }
+              }
+            }
+          }
+        }
+      }
+
       replay.setSegmentId(segmentId);
+      replay.setPayload(payload);
       replay.setUnknown(unknown);
       return replay;
     }

sentry/src/main/java/io/sentry/SentryEnvelopeItem.java

@@ -21,6 +21,7 @@
 import java.io.Reader;
 import java.io.Writer;
 import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.nio.charset.Charset;
 import java.util.HashMap;
 import java.util.Map;
@@ -365,22 +366,16 @@ public static SentryEnvelopeItem fromReplay(
                   replayPayload.put(SentryItemType.ReplayEvent.getItemType(), stream.toByteArray());
                   stream.reset();
 
-                  // next serialize replay recording in the following format:
-                  // {"segment_id":0}\n{json-serialized-rrweb-protocol}
+                  // next serialize replay recording
                   if (replayRecording != null) {
                     serializer.serialize(replayRecording, writer);
-                    writer.write("\n");
-                    writer.flush();
-                    if (replayRecording.getPayload() != null) {
-                      serializer.serialize(replayRecording.getPayload(), writer);
-                    }
                     replayPayload.put(
                         SentryItemType.ReplayRecording.getItemType(), stream.toByteArray());
                     stream.reset();
                   }
 
                   // next serialize replay video bytes from given file
-                  if (replayVideo.exists()) {
+                  if (replayVideo != null && replayVideo.exists()) {
                     final byte[] videoBytes =
                         readBytesFromFile(
                             replayVideo.getPath(), SentryReplayEvent.REPLAY_VIDEO_MAX_SIZE);
@@ -395,7 +390,9 @@ public static SentryEnvelopeItem fromReplay(
                 logger.log(SentryLevel.ERROR, "Could not serialize replay recording", t);
                 return null;
               } finally {
-                replayVideo.delete();
+                if (replayVideo != null) {
+                  replayVideo.delete();
+                }
               }
             });
 
@@ -428,7 +425,7 @@ public CachedItem(final @Nullable Callable<byte[]> dataFactory) {
     }
   }
 
-  @SuppressWarnings("CharsetObjectCanBeUsed")
+  @SuppressWarnings({"CharsetObjectCanBeUsed", "UnnecessaryParentheses"})
   private static byte[] serializeToMsgpack(Map<String, byte[]> map) throws IOException {
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
 
@@ -440,24 +437,17 @@ private static byte[] serializeToMsgpack(Map<String, byte[]> map) throws IOExcep
       // Pack the key as a string
       byte[] keyBytes = entry.getKey().getBytes(Charset.forName("UTF-8"));
       int keyLength = keyBytes.length;
-      if (keyLength <= 31) {
-        baos.write((byte) (0xA0 | keyLength));
-      } else {
-        baos.write((byte) (0xD9));
-        baos.write((byte) (keyLength));
-      }
+      // string up to 255 chars
+      baos.write((byte) (0xd9));
+      baos.write((byte) (keyLength));
       baos.write(keyBytes);
 
       // Pack the value as a binary string
       byte[] valueBytes = entry.getValue();
       int valueLength = valueBytes.length;
-      if (valueLength <= 255) {
-        baos.write((byte) (0xC4));
-        baos.write((byte) (valueLength));
-      } else {
-        baos.write((byte) (0xC5));
-        baos.write(ByteBuffer.allocate(4).putInt(valueLength).array());
-      }
+      // We will always use the 4 bytes data length for simplicity.
+      baos.write((byte) (0xc6));
+      baos.write(ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN).putInt(valueLength).array());
       baos.write(valueBytes);
     }
 

sentry/src/main/java/io/sentry/SentryReplayEvent.java

@@ -1,6 +1,7 @@
 package io.sentry;
 
 import io.sentry.protocol.SentryId;
+import io.sentry.util.Objects;
 import io.sentry.vendor.gson.stream.JsonToken;
 import java.io.File;
 import java.io.IOException;
@@ -29,8 +30,8 @@ public void serialize(@NotNull ObjectWriter writer, @NotNull ILogger logger)
 
     public static final class Deserializer implements JsonDeserializer<ReplayType> {
       @Override
-      public @NotNull ReplayType deserialize(
-          @NotNull JsonObjectReader reader, @NotNull ILogger logger) throws Exception {
+      public @NotNull ReplayType deserialize(@NotNull ObjectReader reader, @NotNull ILogger logger)
+          throws Exception {
         return ReplayType.valueOf(reader.nextString().toUpperCase(Locale.ROOT));
       }
     }
@@ -39,7 +40,7 @@ public static final class Deserializer implements JsonDeserializer<ReplayType> {
   public static final long REPLAY_VIDEO_MAX_SIZE = 10 * 1024 * 1024;
   public static final String REPLAY_EVENT_TYPE = "replay_event";
 
-  private @NotNull File videoFile;
+  private @Nullable File videoFile;
   private @NotNull String type;
   private @NotNull ReplayType replayType;
   private @Nullable SentryId replayId;
@@ -62,12 +63,12 @@ public SentryReplayEvent() {
     timestamp = DateUtils.getCurrentDateTime();
   }
 
-  @NotNull
+  @Nullable
   public File getVideoFile() {
     return videoFile;
   }
 
-  public void setVideoFile(final @NotNull File videoFile) {
+  public void setVideoFile(final @Nullable File videoFile) {
     this.videoFile = videoFile;
   }
 
@@ -151,6 +152,25 @@ public void setReplayType(final @NotNull ReplayType replayType) {
     this.replayType = replayType;
   }
 
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+    SentryReplayEvent that = (SentryReplayEvent) o;
+    return segmentId == that.segmentId
+        && Objects.equals(type, that.type)
+        && replayType == that.replayType
+        && Objects.equals(replayId, that.replayId)
+        && Objects.equals(urls, that.urls)
+        && Objects.equals(errorIds, that.errorIds)
+        && Objects.equals(traceIds, that.traceIds);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(type, replayType, replayId, segmentId, urls, errorIds, traceIds);
+  }
+
   // region json
   public static final class JsonKeys {
     public static final String TYPE = "type";
@@ -219,7 +239,7 @@ public static final class Deserializer implements JsonDeserializer<SentryReplayE
     @SuppressWarnings("unchecked")
     @Override
     public @NotNull SentryReplayEvent deserialize(
-        final @NotNull JsonObjectReader reader, final @NotNull ILogger logger) throws Exception {
+        final @NotNull ObjectReader reader, final @NotNull ILogger logger) throws Exception {
 
       SentryBaseEvent.Deserializer baseEventDeserializer = new SentryBaseEvent.Deserializer();