Adding aspect index creation to init.sql & upgrades

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCli.java

@@ -1,6 +1,7 @@
 package com.linkedin.datahub.upgrade;
 
 import com.linkedin.datahub.upgrade.impl.DefaultUpgradeManager;
+import com.linkedin.datahub.upgrade.nocode.CreateAspectIndexUpgrade;
 import com.linkedin.datahub.upgrade.nocode.NoCodeUpgrade;
 import java.util.List;
 import com.linkedin.datahub.upgrade.nocodecleanup.NoCodeCleanupUpgrade;
@@ -34,10 +35,15 @@ private static final class Args {
   @Named("noCodeCleanup")
   private NoCodeCleanupUpgrade noCodeCleanup;
 
+  @Inject
+  @Named("createAspectIndexUpgrade")
+  private CreateAspectIndexUpgrade createAspectIndexUpgrade;
+
   @Override
   public void run(String... cmdLineArgs) {
     _upgradeManager.register(noCodeUpgrade);
     _upgradeManager.register(noCodeCleanup);
+    _upgradeManager.register(createAspectIndexUpgrade);
 
     final Args args = new Args();
     new CommandLine(args).setCaseInsensitiveEnumValuesAllowed(true).parseArgs(cmdLineArgs);

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/CreateAspectIndexUpgradeConfig.java

@@ -0,0 +1,32 @@
+package com.linkedin.datahub.upgrade.config;
+
+import com.linkedin.datahub.upgrade.nocode.CreateAspectIndexUpgrade;
+import io.ebean.EbeanServerFactory;
+import io.ebean.config.ServerConfig;
+import javax.annotation.Nonnull;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.DependsOn;
+
+import static com.linkedin.metadata.entity.ebean.EbeanAspectDao.*;
+
+
+@Configuration
+public class CreateAspectIndexUpgradeConfig {
+
+  @Autowired
+  ApplicationContext applicationContext;
+
+  @Bean(name = "createAspectIndexUpgrade")
+  @DependsOn({"gmsEbeanServiceConfig"})
+  @Nonnull
+  public CreateAspectIndexUpgrade createInstance() {
+    final ServerConfig serverConfig = applicationContext.getBean(ServerConfig.class);
+    if (!serverConfig.getPackages().contains(EBEAN_MODEL_PACKAGE)) {
+      serverConfig.getPackages().add(EBEAN_MODEL_PACKAGE);
+    }
+    return new CreateAspectIndexUpgrade(EbeanServerFactory.create(serverConfig));
+  }
+}

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/CreateAspectIndexStep.java

@@ -0,0 +1,45 @@
+package com.linkedin.datahub.upgrade.nocode;
+
+import com.linkedin.datahub.upgrade.UpgradeContext;
+import com.linkedin.datahub.upgrade.UpgradeStep;
+import com.linkedin.datahub.upgrade.UpgradeStepResult;
+import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult;
+import io.ebean.EbeanServer;
+import java.util.function.Function;
+
+
+public class CreateAspectIndexStep implements UpgradeStep {
+
+  private final EbeanServer _server;
+
+  public CreateAspectIndexStep(final EbeanServer server) {
+    _server = server;
+  }
+
+  @Override
+  public String id() {
+    return "CreateAspectIndexStep";
+  }
+
+  @Override
+  public int retryCount() {
+    return 1;
+  }
+
+  @Override
+  public Function<UpgradeContext, UpgradeStepResult> executable() {
+    return (context) -> {
+      try {
+        _server.execute(_server.createSqlUpdate(
+            "CREATE INDEX aspectName\n"
+                + "ON metadata_aspect_v2 (aspect)"));
+      } catch (Exception e) {
+        context.report().addLine(String.format("Failed to create aspect index for metadata_aspect_v2: %s", e.toString()));
+        return new DefaultUpgradeStepResult(
+            id(),
+            UpgradeStepResult.Result.FAILED);
+      }
+      return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED);
+    };
+  }
+}

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/CreateAspectIndexUpgrade.java

@@ -0,0 +1,41 @@
+package com.linkedin.datahub.upgrade.nocode;
+
+import com.linkedin.datahub.upgrade.Upgrade;
+import com.linkedin.datahub.upgrade.UpgradeCleanupStep;
+import com.linkedin.datahub.upgrade.UpgradeStep;
+import io.ebean.EbeanServer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+
+public class CreateAspectIndexUpgrade implements Upgrade {
+
+  private final List<UpgradeStep> _steps;
+
+  // Upgrade requires the EbeanServer.
+  public CreateAspectIndexUpgrade(final EbeanServer server) {
+    _steps = buildUpgradeSteps(server);
+  }
+
+  @Override
+  public String id() {
+    return "CreateAspectIndexUpgrade";
+  }
+
+  @Override
+  public List<UpgradeStep> steps() {
+    return _steps;
+  }
+
+  @Override
+  public List<UpgradeCleanupStep> cleanupSteps() {
+    return Collections.emptyList();
+  }
+
+  private List<UpgradeStep> buildUpgradeSteps(final EbeanServer server) {
+    final List<UpgradeStep> steps = new ArrayList<>();
+    steps.add(new CreateAspectTableStep(server));
+    return steps;
+  }
+}

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java

@@ -60,6 +60,7 @@ private List<UpgradeStep> buildUpgradeSteps(
     steps.add(new GMSQualificationStep());
     steps.add(new UpgradeQualificationStep(server));
     steps.add(new CreateAspectTableStep(server));
+    steps.add(new CreateAspectIndexStep(server));
     steps.add(new IngestDataPlatformsStep(entityService));
     steps.add(new DataMigrationStep(server, entityService, entityRegistry));
     steps.add(new GMSEnableWriteModeStep());

docker/mariadb/init.sql

@@ -11,6 +11,10 @@ create table metadata_aspect_v2 (
   constraint pk_metadata_aspect primary key (urn,aspect,version)
 );
 
+-- create index on the "aspect" column to support aspect-oriented queries.
+CREATE INDEX aspectName
+ON metadata_aspect_v2 (aspect)
+
 insert into metadata_aspect_v2 (urn, aspect, version, metadata, createdon, createdby) values(
   'urn:li:corpuser:datahub',
   'corpUserInfo',

docker/mysql-setup/init.sql

@@ -38,6 +38,10 @@ SELECT * FROM temp_metadata_aspect_v2
 WHERE NOT EXISTS (SELECT * from metadata_aspect_v2);
 DROP TABLE temp_metadata_aspect_v2;
 
+-- create index on the "aspect" column to support aspect-oriented queries.
+CREATE INDEX aspectName
+ON metadata_aspect_v2 (aspect)
+
 -- create metadata index table
 CREATE TABLE IF NOT EXISTS metadata_index (
  `id` BIGINT NOT NULL AUTO_INCREMENT,

docker/mysql/init.sql

@@ -11,6 +11,10 @@ CREATE TABLE metadata_aspect_v2 (
   CONSTRAINT pk_metadata_aspect PRIMARY KEY (urn,aspect,version)
 );
 
+-- create index on the "aspect" column to support aspect-oriented queries.
+CREATE INDEX aspectName
+ON metadata_aspect_v2 (aspect)
+
 INSERT INTO metadata_aspect_v2 (urn, aspect, version, metadata, createdon, createdby) VALUES(
   'urn:li:corpuser:datahub',
   'corpUserInfo',

docker/postgres/init.sql

@@ -11,6 +11,10 @@ create table metadata_aspect_v2 (
   constraint pk_metadata_aspect primary key (urn,aspect,version)
 );
 
+-- create index on the "aspect" column to support aspect-oriented queries.
+CREATE INDEX aspectName
+ON metadata_aspect_v2 (aspect)
+
 insert into metadata_aspect_v2 (urn, aspect, version, metadata, createdon, createdby) values(
   'urn:li:corpuser:datahub',
   'corpUserInfo',