diff --git a/CHANGELOG.md b/CHANGELOG.md index d39cae9..1ccc1ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.0.6](https://github.com/cloudquery/plugin-sdk-java/compare/v0.0.5...v0.0.6) (2023-08-29) + + +### Bug Fixes + +* make the underlying timestamp consistent ([#105](https://github.com/cloudquery/plugin-sdk-java/issues/105)) ([d97dabd](https://github.com/cloudquery/plugin-sdk-java/commit/d97dabd95e0a91d43dd921d9c646290b1a3b13c9)) + ## [0.0.5](https://github.com/cloudquery/plugin-sdk-java/compare/v0.0.4...v0.0.5) (2023-08-29) diff --git a/lib/build.gradle b/lib/build.gradle index 0968af8..43aef70 100644 --- a/lib/build.gradle +++ b/lib/build.gradle @@ -11,7 +11,7 @@ ext { group 'io.cloudquery.plugin.sdk' // x-release-please-start-version -version = '0.0.5' +version = '0.0.6' // x-release-please-end repositories { diff --git a/lib/src/main/java/io/cloudquery/scalar/Timestamp.java b/lib/src/main/java/io/cloudquery/scalar/Timestamp.java index 37d00f0..8702814 100644 --- a/lib/src/main/java/io/cloudquery/scalar/Timestamp.java +++ b/lib/src/main/java/io/cloudquery/scalar/Timestamp.java @@ -1,6 +1,11 @@ package io.cloudquery.scalar; -import java.time.*; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -8,7 +13,8 @@ public class Timestamp extends Scalar { public static final ZoneId zoneID = ZoneOffset.UTC; // TODO: add more units support later - private static final ArrowType dt = new ArrowType.Timestamp(TimeUnit.SECOND, zoneID.toString()); + private static final ArrowType dt = + new ArrowType.Timestamp(TimeUnit.MILLISECOND, zoneID.toString()); public Timestamp() { super(); @@ -26,34 +32,36 @@ public ArrowType dataType() { @Override public void setValue(Object value) throws ValidationException { if (value instanceof ZonedDateTime timestamp) { - this.value = timestamp.withZoneSameInstant(zoneID).toEpochSecond(); + this.value = timestamp.withZoneSameInstant(zoneID).toEpochSecond() * 1000; return; } if (value instanceof LocalDate date) { - this.value = date.atStartOfDay(zoneID).toEpochSecond(); + this.value = date.atStartOfDay(zoneID).toEpochSecond() * 1000; return; } if (value instanceof LocalDateTime date) { - this.value = date.atZone(zoneID).toEpochSecond(); + this.value = date.atZone(zoneID).toEpochSecond() * 1000; return; } if (value instanceof Integer integer) { this.value = - ZonedDateTime.ofInstant(Instant.ofEpochMilli(integer), ZoneOffset.UTC).toEpochSecond(); + ZonedDateTime.ofInstant(Instant.ofEpochMilli(integer), ZoneOffset.UTC).toEpochSecond() + * 1000; return; } if (value instanceof Long longValue) { this.value = - ZonedDateTime.ofInstant(Instant.ofEpochMilli(longValue), ZoneOffset.UTC).toEpochSecond(); + ZonedDateTime.ofInstant(Instant.ofEpochMilli(longValue), ZoneOffset.UTC).toEpochSecond() + * 1000; return; } if (value instanceof CharSequence sequence) { - this.value = ZonedDateTime.parse(sequence).toEpochSecond(); + this.value = ZonedDateTime.parse(sequence).toInstant().toEpochMilli(); return; } @@ -64,7 +72,7 @@ public void setValue(Object value) throws ValidationException { @Override public java.lang.String toString() { if (this.value != null) { - return ZonedDateTime.ofInstant(Instant.ofEpochSecond((Long) this.value), zoneID).toString(); + return ZonedDateTime.ofInstant(Instant.ofEpochMilli((Long) this.value), zoneID).toString(); } return NULL_VALUE_STRING; diff --git a/lib/src/test/java/io/cloudquery/helper/ArrowHelperTest.java b/lib/src/test/java/io/cloudquery/helper/ArrowHelperTest.java index ff5dc46..ff52029 100644 --- a/lib/src/test/java/io/cloudquery/helper/ArrowHelperTest.java +++ b/lib/src/test/java/io/cloudquery/helper/ArrowHelperTest.java @@ -12,6 +12,7 @@ import com.google.protobuf.ByteString; import io.cloudquery.schema.Column; +import io.cloudquery.schema.Resource; import io.cloudquery.schema.Table; import java.io.IOException; import java.util.List; @@ -19,6 +20,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class ArrowHelperTest { @@ -32,20 +34,21 @@ public class ArrowHelperTest { .columns( List.of( Column.builder() - .name("column1") + .name("string_column1") .type(ArrowType.Utf8.INSTANCE) .unique(true) .incrementalKey(true) .primaryKey(true) .build(), - Column.builder().name("column2").type(ArrowType.Utf8.INSTANCE).build())) + Column.builder().name("string_column2").type(ArrowType.Utf8.INSTANCE).build(), + Column.builder().name("boolean_column").type(ArrowType.Bool.INSTANCE).build())) .build(); @Test public void testToArrowSchema() { Schema arrowSchema = ArrowHelper.toArrowSchema(TEST_TABLE); - assertEquals(arrowSchema.getFields().get(0).getName(), "column1"); + assertEquals(arrowSchema.getFields().get(0).getName(), "string_column1"); assertEquals( arrowSchema.getFields().get(0).getMetadata(), Map.of( @@ -55,7 +58,7 @@ public void testToArrowSchema() { "true", CQ_EXTENSION_PRIMARY_KEY, "true")); - assertEquals(arrowSchema.getFields().get(1).getName(), "column2"); + assertEquals(arrowSchema.getFields().get(1).getName(), "string_column2"); assertEquals( arrowSchema.getFields().get(1).getMetadata(), Map.of( @@ -80,8 +83,8 @@ public void testToArrowSchema() { public void testFromArrowSchema() { List fields = List.of( - Field.nullable("column1", ArrowType.Utf8.INSTANCE), - Field.nullable("column2", ArrowType.Utf8.INSTANCE)); + Field.nullable("string_column1", ArrowType.Utf8.INSTANCE), + Field.nullable("string_column2", ArrowType.Utf8.INSTANCE)); Schema schema = new Schema(fields, Map.of(CQ_TABLE_NAME, "table1")); @@ -97,7 +100,7 @@ public void testFromArrowSchema() { } @Test - public void testRoundTrip() throws IOException { + public void testRoundTripTableEncoding() throws IOException { ByteString byteString = ArrowHelper.encode(TEST_TABLE); Table table = ArrowHelper.decode(byteString); @@ -111,4 +114,18 @@ public void testRoundTrip() throws IOException { assertEquals(TEST_TABLE.getColumns().get(i).getType(), table.getColumns().get(i).getType()); } } + + @Test + public void testRoundTripResourceEncoding() throws Exception { + Resource resource = Resource.builder().table(TEST_TABLE).build(); + resource.set("string_column1", "test_data"); + resource.set("string_column2", "test_data2"); + resource.set("boolean_column", true); + + Assertions.assertDoesNotThrow( + () -> { + ByteString byteString = ArrowHelper.encode(resource); + ArrowHelper.decodeResource(byteString); + }); + } } diff --git a/lib/src/test/java/io/cloudquery/internal/servers/plugin/v3/PluginServerTest.java b/lib/src/test/java/io/cloudquery/internal/servers/plugin/v3/PluginServerTest.java index cd8a1e2..f75019f 100644 --- a/lib/src/test/java/io/cloudquery/internal/servers/plugin/v3/PluginServerTest.java +++ b/lib/src/test/java/io/cloudquery/internal/servers/plugin/v3/PluginServerTest.java @@ -100,10 +100,15 @@ private static Write.Request generateMigrateTableMessage() throws IOException { } private Write.Request generateInsertMessage() throws IOException, ValidationException { - Column column = Column.builder().name("test_column").type(ArrowType.Utf8.INSTANCE).build(); - Table table = Table.builder().name("test").columns(List.of(column)).build(); + Column stringColumn = + Column.builder().name("test_column").type(ArrowType.Utf8.INSTANCE).build(); + Column booleanColumn = + Column.builder().name("boolean_column").type(ArrowType.Bool.INSTANCE).build(); + Table table = + Table.builder().name("test").columns(List.of(stringColumn, booleanColumn)).build(); Resource resource = Resource.builder().table(table).build(); resource.set("test_column", "test_data"); + resource.set("boolean_column", true); ByteString byteString = ArrowHelper.encode(resource); MessageInsert messageInsert = MessageInsert.newBuilder().setRecord(byteString).build(); return Write.Request.newBuilder().setInsert(messageInsert).build(); diff --git a/lib/src/test/java/io/cloudquery/scalar/TimestampTest.java b/lib/src/test/java/io/cloudquery/scalar/TimestampTest.java index 8f74f1d..e10faff 100644 --- a/lib/src/test/java/io/cloudquery/scalar/TimestampTest.java +++ b/lib/src/test/java/io/cloudquery/scalar/TimestampTest.java @@ -1,6 +1,12 @@ package io.cloudquery.scalar; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.time.Instant; import java.time.ZoneOffset; @@ -66,7 +72,7 @@ public void testToString() { @Test public void testDataType() { Timestamp timestamp = new Timestamp(); - assertEquals(new ArrowType.Timestamp(TimeUnit.SECOND, "Z"), timestamp.dataType()); + assertEquals(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "Z"), timestamp.dataType()); } @Test @@ -118,7 +124,7 @@ public void testGet() { timestamp.set(ts); }); assertTrue(timestamp.isValid()); - assertEquals(ts.toEpochSecond(), timestamp.get()); + assertEquals(ts.toEpochSecond() * 1000, timestamp.get()); assertDoesNotThrow( () -> {