chore: upgrade dependencies

.github/workflows/package.yml

@@ -25,7 +25,7 @@ jobs:
         run: ./gradlew :dagger-common:publish
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Publish minimal and shadow packages of core
-        run: ./gradlew :dagger-core:minimalAndShadowPublish
+      - name: Publish minimal package of core
+        run: ./gradlew :dagger-core:minimalPublish
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

dagger-common/build.gradle

@@ -58,26 +58,26 @@ dependencies {
     compileOnly group: 'org.apache.flink', name: 'flink-table', version: flinkVersion
     compileOnly group: 'org.apache.flink', name: 'flink-table-api-java-bridge_2.11', version: flinkVersion
     compileOnly group: 'org.apache.flink', name: 'flink-connector-kafka_2.11', version: flinkVersion
-    compileOnly 'com.gotocompany:stencil:0.4.0'
+    compileOnly 'com.gotocompany:stencil:0.6.0'
 
-    dependenciesCommonJar ('org.apache.hadoop:hadoop-client:2.8.3') {
-        exclude module:"commons-cli"
-        exclude module:"commons-compress"
+    dependenciesCommonJar('org.apache.hadoop:hadoop-client:3.3.6') {
+        exclude module: "commons-cli"
+        exclude module: "commons-compress"
     }
-    dependenciesCommonJar 'com.google.cloud.bigdataoss:gcs-connector:1.9.0-hadoop2'
+    dependenciesCommonJar group: 'com.google.cloud.bigdataoss', name: 'gcs-connector', version: 'hadoop2-2.2.16'
     dependenciesCommonJar 'org.apache.flink:flink-metrics-dropwizard:' + flinkVersion
     dependenciesCommonJar 'org.apache.flink:flink-json:' + flinkVersion
     dependenciesCommonJar 'com.jayway.jsonpath:json-path:2.4.0'
     dependenciesCommonJar 'com.google.code.gson:gson:2.8.2'
     dependenciesCommonJar 'org.apache.parquet:parquet-column:1.12.2'
 
-    testImplementation 'junit:junit:4.13'
+    testImplementation 'junit:junit:4.13.1'
     testImplementation 'org.jmockit:jmockit:1.25'
     testImplementation 'org.mockito:mockito-core:2.25.1'
     testImplementation 'io.grpc:grpc-protobuf:1.18.0'
     testImplementation 'io.grpc:grpc-api:1.37.0'
     testImplementation 'io.grpc:grpc-stub:1.18.0'
-    testImplementation 'com.google.protobuf:protobuf-java:3.5.0'
+    testImplementation 'com.google.protobuf:protobuf-java:3.16.3'
     testImplementation 'com.google.protobuf:protobuf-java-util:3.5.0'
     testImplementation 'org.grpcmock:grpcmock-junit5:0.5.0'
 }

dagger-common/src/main/java/com/gotocompany/dagger/common/core/StencilClientOrchestrator.java

@@ -5,8 +5,8 @@
 import com.gotocompany.stencil.cache.SchemaRefreshStrategy;
 import com.gotocompany.stencil.client.StencilClient;
 import com.gotocompany.stencil.config.StencilConfig;
-import org.apache.http.Header;
-import org.apache.http.message.BasicHeader;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.message.BasicHeader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

dagger-common/src/main/java/com/gotocompany/dagger/common/serde/proto/serialization/ProtoSerializer.java

@@ -103,7 +103,7 @@ private DynamicMessage.Builder populateBuilder(DynamicMessage.Builder builder, D
         if (data != null) {
             try {
                 builder = typeHandler.transformToProtoBuilder(builder, data);
-            } catch (IllegalArgumentException e) {
+            } catch (RuntimeException e) {
                 String protoType = fieldDescriptor.getType().toString();
                 if (fieldDescriptor.isRepeated()) {
                     protoType = String.format("REPEATED %s", fieldDescriptor.getType());

dagger-common/src/test/java/com/gotocompany/dagger/common/serde/proto/serialization/ProtoSerializerTest.java

@@ -5,7 +5,6 @@
 import com.gotocompany.dagger.common.exceptions.DescriptorNotFoundException;
 import com.gotocompany.dagger.common.exceptions.serde.DaggerSerializationException;
 import com.gotocompany.dagger.common.exceptions.serde.InvalidColumnMappingException;
-import com.gotocompany.dagger.common.exceptions.serde.InvalidDataTypeException;
 import com.gotocompany.dagger.consumer.*;
 import com.gotocompany.stencil.StencilClientFactory;
 import com.gotocompany.stencil.client.StencilClient;
@@ -279,9 +278,9 @@ public void shouldThrowExceptionWhenPrimitiveTypeCanNotBeCasted() {
         Row element = new Row(1);
         element.setField(0, "invalid_number");
 
-        InvalidDataTypeException exception = assertThrows(InvalidDataTypeException.class,
+        InvalidColumnMappingException exception = assertThrows(InvalidColumnMappingException.class,
                 () -> serializer.serializeValue(element));
-        assertEquals("type mismatch of field: customer_price, expecting DOUBLE type, actual type class java.lang.String",
+        assertEquals("column invalid: type mismatch of column customer_price, expecting DOUBLE type. Actual type class java.lang.String",
                 exception.getMessage());
     }
 

dagger-core/build.gradle

@@ -7,17 +7,20 @@ buildscript {
         maven {
             url "https://plugins.gradle.org/m2/"
         }
+        maven {
+            url 'https://repo1.maven.org/maven2'
+        }
     }
     dependencies {
-        classpath 'com.github.jengelman.gradle.plugins:shadow:6.0.0'
+        classpath 'com.github.jengelman.gradle.plugins:shadow:6.1.0'
         classpath "org.jfrog.buildinfo:build-info-extractor-gradle:4.17.0"
     }
 }
 
 plugins {
     id 'maven-publish'
     id 'com.jfrog.artifactory' version '4.17.0'
-    id 'com.github.johnrengelman.shadow' version '6.0.0'
+    id 'com.github.johnrengelman.shadow' version '6.1.0'
 }
 
 
@@ -26,12 +29,15 @@ def flinkVersion = rootProject.flinkVersion
 version = rootProject.file('version.txt').text.trim()
 
 def minimalVersion = version
-def dependenciesVersion = "0.3.2"
+def dependenciesVersion = "0.4.0"
 
 description = """dagger to the heart!"""
 
-sourceCompatibility = 1.8
-targetCompatibility = 1.8
+
+java {
+    sourceCompatibility = JavaVersion.VERSION_1_8
+    targetCompatibility = JavaVersion.VERSION_1_8
+}
 
 tasks.withType(JavaCompile) {
     options.encoding = 'UTF-8'
@@ -61,47 +67,62 @@ configurations {
 dependencies {
     minimalJar project(path: ':dagger-common', configuration: 'minimalCommonJar')
     minimalJar project(path: ':dagger-functions', configuration: 'minimalFunctionsJar')
-    minimalJar('com.gotocompany:depot:0.4.0') {
-        exclude group: 'org.apache.httpcomponents'
-        exclude group: 'com.google.protobuf'
-        exclude group: 'com.datadoghq'
-        exclude group: 'com.google.cloud', module:'google-cloud-bigtable'
+    minimalJar('com.gotocompany:depot:0.6.0') {
+        exclude group: 'com.google.cloud', module: 'google-cloud-bigtable'
+        exclude group: 'com.google.cloud', module: 'google-cloud-bigquerystorage'
+        exclude group: 'com.google.cloud', module: 'google-cloud-bigquery'
+        exclude group: 'com.google.protobuf', module: 'protobuf-java'
+        exclude group: 'com.google.protobuf', module: 'protobuf-java-util'
+        exclude group: "io.grpc"
+    }
+    minimalJar('com.gotocompany:stencil:0.6.0') {
+        exclude group: 'com.google.protobuf', module: 'protobuf-java'
+        exclude group: 'com.google.protobuf', module: 'protobuf-java-util'
     }
     compileOnly 'org.projectlombok:lombok:1.18.8'
     annotationProcessor 'org.projectlombok:lombok:1.18.8'
 
-    implementation 'org.slf4j:slf4j-log4j12:1.7.7'
+    implementation group: 'org.slf4j', name: 'slf4j-reload4j', version: '2.0.7'
     implementation 'org.apache.flink:flink-streaming-java_2.11:' + flinkVersion
     implementation 'org.apache.flink:flink-clients_2.11:' + flinkVersion
     implementation 'org.apache.flink:flink-table:' + flinkVersion
     implementation 'org.apache.flink:flink-table-api-java-bridge_2.11:' + flinkVersion
     implementation 'org.apache.flink:flink-table-planner_2.11:' + flinkVersion
-    implementation 'com.gotocompany:stencil:0.4.0'
 
+    dependenciesJar 'io.grpc:grpc-all:1.55.1'
     dependenciesJar project(path: ':dagger-common', configuration: 'dependenciesCommonJar')
     dependenciesJar project(path: ':dagger-functions', configuration: 'dependenciesFunctionsJar')
-
+    dependenciesJar(group: 'com.google.cloud', name: 'google-cloud-bigquerystorage', version: '2.39.1') {
+        exclude group: "io.grpc"
+    }
+    dependenciesJar(group: 'com.google.cloud', name: 'google-cloud-bigtable', version: '2.24.1') {
+        exclude group: "io.grpc"
+    }
+    dependenciesJar(group: 'com.google.cloud', name: 'google-cloud-bigquery', version: '2.29.0') {
+        exclude group: "io.grpc"
+    }
     dependenciesJar 'org.apache.flink:flink-connector-kafka_2.11:' + flinkVersion
-    dependenciesJar 'com.google.protobuf:protobuf-java:3.1.0'
+    dependenciesJar 'com.google.protobuf:protobuf-java:3.23.2'
     dependenciesJar 'com.google.protobuf:protobuf-java-util:3.1.0'
     dependenciesJar 'org.influxdb:influxdb-java:2.8'
     dependenciesJar 'org.elasticsearch.client:elasticsearch-rest-client:6.6.1'
-    dependenciesJar 'com.google.cloud.bigtable:bigtable-hbase-2.x:1.11.0'
+    dependenciesJar 'com.google.cloud.bigtable:bigtable-hbase-2.x:2.10.0'
     dependenciesJar 'org.asynchttpclient:async-http-client:2.10.1'
     dependenciesJar 'io.vertx:vertx-pg-client:3.9.0'
     dependenciesJar 'org.apache.commons:commons-pool2:2.4.3'
     dependenciesJar 'org.apache.parquet:parquet-protobuf:1.12.2'
 
     testImplementation project(':dagger-common').sourceSets.test.output
-    testImplementation 'junit:junit:4.13'
+    testImplementation 'junit:junit:4.13.1'
     testImplementation 'org.apache.flink:flink-test-utils_2.11:' + flinkVersion
-    testImplementation 'org.apache.kafka:kafka-clients:2.5.0'
+    testImplementation 'org.apache.kafka:kafka-clients:2.6.3'
     testImplementation 'com.github.tomakehurst:wiremock-standalone:2.27.0'
     testImplementation 'org.jmockit:jmockit:1.25'
     testImplementation 'org.mockito:mockito-core:2.25.1'
+    testImplementation 'io.grpc:grpc-all:1.55.1'
     testImplementation 'org.powermock:powermock-module-junit4:2.0.0-beta.5'
     testImplementation 'org.powermock:powermock-api-mockito2:2.0.0-beta.5'
-    testImplementation 'com.google.guava:guava:27.0.1-jre'
+    testImplementation 'com.google.guava:guava:30.0-jre'
     testImplementation 'org.grpcmock:grpcmock-junit5:0.5.0'
     testImplementation 'com.github.stefanbirkner:system-rules:1.19.0'
 }
@@ -163,20 +184,7 @@ jar {
     }
 }
 
-shadowJar {
-    mergeServiceFiles()
-    minimize {
-        exclude(dependency('io.vertx:vertx-pg-client:.*'))
-        exclude(dependency('com.google.cloud.bigtable:bigtable-hbase-2.x:.*'))
-    }
-    zip64 true
-    configurations = [project.configurations.minimalJar, project.configurations.dependenciesJar]
-    relocate('com.google.protobuf', 'shaded.com.google.protobuf') {
-        exclude 'combine.self="override"'
-    }
-}
-
-task minimalJar(type: ShadowJar) {
+tasks.register('minimalJar', ShadowJar) {
     manifest.attributes 'Main-Class': mainClassName
     archiveClassifier = 'minimal'
     zip64 true
@@ -187,7 +195,7 @@ task minimalJar(type: ShadowJar) {
     exclude("core-site.xml")
 }
 
-task dependenciesJar(type: ShadowJar) {
+tasks.register('dependenciesJar', ShadowJar) {
     mergeServiceFiles()
     manifest.attributes 'Main-Class': mainClassName
     archiveClassifier = 'dependencies'
@@ -197,7 +205,7 @@ task dependenciesJar(type: ShadowJar) {
     archiveVersion = dependenciesVersion
 }
 
-task fatJar(type: ShadowJar) {
+tasks.register('fatJar', ShadowJar) {
     description = "Builds a executable jar"
     manifest.attributes 'Main-Class': mainClassName
     archiveClassifier = 'fat'
@@ -217,10 +225,6 @@ private Properties loadEnv() {
 
 publishing {
     publications {
-        shadow(MavenPublication) {
-            publication ->
-                project.shadow.component(publication)
-        }
         minimalArtifact(MavenPublication) {
             artifact file("$buildDir/libs/dagger-core-${minimalVersion}-minimal.jar")
             groupId project.group
@@ -264,9 +268,9 @@ artifactory {
     clientConfig.info.setBuildNumber(System.env.BUILD_NUMBER)
 }
 
-task minimalAndShadowPublish(dependsOn: 'minimalJar') {
-    dependsOn('publishMinimalArtifactPublicationToGitHubPackagesRepository', 'publishShadowPublicationToGitHubPackagesRepository')
-    description('Publishes minimal and shadow jar')
+task minimalPublish(dependsOn: 'minimalJar') {
+    dependsOn('publishMinimalArtifactPublicationToGitHubPackagesRepository')
+    description('Publishes minimal jar')
 }
 
 task dependenciesPublish(dependsOn: 'dependenciesJar') {

dagger-functions/build.gradle

@@ -57,13 +57,13 @@ dependencies {
     compileOnly group: 'org.apache.flink', name: 'flink-table-api-java-bridge_2.11', version: flinkVersion
     compileOnly group: 'org.apache.flink', name: 'flink-streaming-java_2.11', version: flinkVersion
     compileOnly group: 'org.apache.flink', name: 'flink-metrics-dropwizard', version: flinkVersion
-    compileOnly 'com.gotocompany:stencil:0.4.0'
+    compileOnly 'com.gotocompany:stencil:0.6.0'
 
     dependenciesFunctionsJar 'com.github.davidmoten:geo:0.7.6'
     dependenciesFunctionsJar 'org.apache.flink:flink-python_2.11:' + flinkVersion
     dependenciesFunctionsJar group: 'org.apache.commons', name: 'commons-jexl3', version: '3.1'
     dependenciesFunctionsJar group: 'org.isuper', name: 's2-geometry-library-java', version: '0.0.1'
-    dependenciesFunctionsJar group: 'com.google.cloud', name: 'google-cloud-storage', version: '1.67.0'
+    dependenciesFunctionsJar group: 'com.google.cloud', name: 'google-cloud-storage', version: '2.23.0'
 
     testImplementation project(':dagger-common').sourceSets.test.output
     testImplementation group: 'junit', name: 'junit', version: '4.12'

docs/docs/guides/deployment.md

@@ -15,7 +15,7 @@ To know more about Flink's cluster mode deployment follow [this](https://ci.apac
 ./gradlew clean build
 
 #creating a fat jar
-./gradlew :dagger-core:shadowJar
+./gradlew :dagger-core:fatJar
 
 ```
 

version.txt

@@ -1 +1 @@
-0.9.6
+0.10.0