Implementing createDeploymentPlan (#3081)

* refactoring release step of functions deploy to use typescript

* Adding logic to build regional deployments

* Implementing createDeploymentPlan

* First round of PR feedback, removing most usages of lodash

* moving function prompts into their own file

* seperating out a bunch of code from functionsDeployHelper

* round of pr fixes

* adresses more pr comments, and adds some todos

src/deploy/functions/checkIam.ts

@@ -3,7 +3,8 @@ import { bold } from "cli-color";
 
 import { debug } from "../../logger";
 import * as track from "../../track";
-import { getReleaseNames, getFunctionsInfo, getFilterGroups } from "../../functionsDeployHelper";
+import { getReleaseNames, getFilterGroups } from "../../functionsDeployHelper";
+import { CloudFunctionTrigger } from "./deploymentPlanner";
 import { FirebaseError } from "../../error";
 import { testIamPermissions, testResourceIamPermissions } from "../../gcp/iam";
 
@@ -51,13 +52,12 @@ export async function checkServiceAccountIam(projectId: string): Promise<void> {
  * @param payload The deploy payload.
  */
 export async function checkHttpIam(context: any, options: any, payload: any): Promise<void> {
-  const triggers = payload.functions.triggers;
-  const functionsInfo = getFunctionsInfo(triggers, context.projectId);
+  const functionsInfo = payload.functions.triggers;
   const filterGroups = getFilterGroups(options);
 
   const httpFunctionNames: string[] = functionsInfo
-    .filter((f) => has(f, "httpsTrigger"))
-    .map((f) => f.name);
+    .filter((f: CloudFunctionTrigger) => has(f, "httpsTrigger"))
+    .map((f: CloudFunctionTrigger) => f.name);
   const httpFunctionFullNames: string[] = getReleaseNames(httpFunctionNames, [], filterGroups);
   const existingFunctionFullNames: string[] = context.existingFunctions.map(
     (f: { name: string }) => f.name

src/deploy/functions/deploymentPlanner.ts

@@ -0,0 +1,179 @@
+import * as deploymentTool from "../../deploymentTool";
+import { functionMatchesAnyGroup, getTopicName } from "../../functionsDeployHelper";
+
+// TODO: Better name for this?
+// It's really a CloudFuntion, not just a trigger,
+// but CloudFunction is a different exported type from firebase-functions
+export interface CloudFunctionTrigger {
+  name: string;
+  sourceUploadUrl?: string;
+  sourceToken?: string;
+  labels: { [key: string]: string };
+  environmentVariables: { [key: string]: string };
+  entryPoint: string;
+  runtime?: string;
+  vpcConnector?: string;
+  vpcConnectorEgressSettings?: string;
+  ingressSettings?: string;
+  availableMemoryMb?: number;
+  timeout?: number;
+  maxInstances?: number;
+  serviceAccountEmail?: string;
+  httpsTrigger?: any;
+  eventTrigger?: any;
+  failurePolicy?: {};
+  schedule?: object;
+  timeZone?: string;
+  regions?: string[];
+}
+
+export interface RegionMap {
+  [region: string]: CloudFunctionTrigger[];
+}
+
+export interface RegionalDeployment {
+  region: string;
+  sourceToken?: string;
+  firstFunctionDeployment?: () => any;
+  functionsToCreate: CloudFunctionTrigger[];
+  functionsToUpdate: CloudFunctionTrigger[];
+  schedulesToUpsert: CloudFunctionTrigger[];
+}
+
+export interface DeploymentPlan {
+  regionalDeployments: RegionalDeployment[];
+  functionsToDelete: string[];
+  schedulesToDelete: string[];
+}
+
+/**
+ * Creates a map of regions to all the CloudFunctions being deployed
+ * to that region.
+ * @param projectId The project in use.
+ * @param localFunctions A list of all CloudFunctions in the deployment.
+ */
+export function functionsByRegion(
+  projectId: string,
+  localFunctions: CloudFunctionTrigger[]
+): RegionMap {
+  const regionMap: RegionMap = {};
+  for (const trigger of localFunctions) {
+    if (!trigger.regions) {
+      trigger.regions = ["us-central1"];
+    }
+    // Create a separate CloudFunction for
+    // each region we deploy a function to
+    for (const region of trigger.regions) {
+      const triggerDeepCopy = JSON.parse(JSON.stringify(trigger));
+      if (triggerDeepCopy.regions) {
+        delete triggerDeepCopy.regions;
+      }
+      triggerDeepCopy.name = [
+        "projects",
+        projectId,
+        "locations",
+        region,
+        "functions",
+        trigger.name,
+      ].join("/");
+      regionMap[region] = regionMap[region] || [];
+      regionMap[region].push(triggerDeepCopy);
+    }
+  }
+  return regionMap;
+}
+
+/**
+ * Helper method to turn a RegionMap into a flat list of all functions in a deployment.
+ * @param regionMap A RegionMap for the deployment.
+ */
+export function allFunctions(regionMap: RegionMap): CloudFunctionTrigger[] {
+  const triggers: CloudFunctionTrigger[] = [];
+  for (const [k, v] of Object.entries(regionMap)) {
+    triggers.push(...v);
+  }
+  return triggers;
+}
+
+/**
+ * Create a plan for deploying all functions in one region.
+ * @param region The region of this deployment
+ * @param loclFunctionsByRegion The functions present in the code currently being deployed.
+ * @param existingFunctionNames The names of all functions that already exist.
+ * @param existingScheduledFunctionNames The names of all schedules functions that already exist.
+ * @param filters The filters, passed in by the user via  `--only functions:`
+ */
+export function createDeploymentPlan(
+  localFunctionsByRegion: RegionMap,
+  existingFunctions: CloudFunctionTrigger[],
+  filters: string[][]
+): DeploymentPlan {
+  let existingFnsCopy: CloudFunctionTrigger[] = [...existingFunctions];
+  const deployment: DeploymentPlan = {
+    regionalDeployments: [],
+    functionsToDelete: [],
+    schedulesToDelete: [],
+  };
+  // eslint-disable-next-line guard-for-in
+  for (const region in localFunctionsByRegion) {
+    const regionalDeployment: RegionalDeployment = {
+      region,
+      functionsToCreate: [],
+      functionsToUpdate: [],
+      schedulesToUpsert: [],
+    };
+    const localFunctionsInRegion = localFunctionsByRegion[region];
+    for (const fn of localFunctionsInRegion) {
+      // Check if this function matches the --only filters
+      if (!functionMatchesAnyGroup(fn.name, filters)) {
+        continue;
+      }
+      // Check if this local function has the same name as an exisiting one.
+      const matchingExistingFunction = existingFnsCopy.find((exFn) => exFn.name === fn.name);
+      // Check if the matching exisitng function is scheduled
+      const isMatchingExisitingFnScheduled =
+        matchingExistingFunction?.labels?.["deployment-scheduled"] === "true";
+      // Check if the local function is a scheduled function
+      if (fn.schedule) {
+        // If the local function is scheduled, set its trigger to the correct pubsub topic
+        fn.eventTrigger.resource = getTopicName(fn.name);
+        // and create or update a schedule.
+        regionalDeployment.schedulesToUpsert.push(fn);
+      } else if (isMatchingExisitingFnScheduled) {
+        // If the local function isn't scheduled but the existing one is, delete the schedule.
+        deployment.schedulesToDelete.push(matchingExistingFunction!.name);
+      }
+
+      if (matchingExistingFunction) {
+        regionalDeployment.functionsToUpdate.push(fn);
+        existingFnsCopy = existingFnsCopy.filter((exFn: CloudFunctionTrigger) => {
+          return exFn.name !== fn.name;
+        });
+      } else {
+        regionalDeployment.functionsToCreate.push(fn);
+      }
+    }
+    deployment.regionalDeployments.push(regionalDeployment);
+  }
+
+  // Delete any remaining existing functions that:
+  // 1 - Have the deployment-tool: 'firebase-cli' label and
+  // 2 - Match the --only filters, if any are provided.
+  const functionsToDelete = existingFnsCopy
+    .filter((fn) => {
+      return deploymentTool.isFirebaseManaged(fn.labels);
+    })
+    .filter((fn) => {
+      return functionMatchesAnyGroup(fn.name, filters);
+    });
+  deployment.functionsToDelete = functionsToDelete.map((fn) => {
+    return fn.name;
+  });
+  // Also delete any schedules for functions that we are deleting.
+  for (const fn of functionsToDelete) {
+    if (fn.labels?.["deployment-scheduled"] === "true") {
+      deployment.schedulesToDelete.push(fn.name);
+    }
+  }
+  return deployment;
+}

src/deploy/functions/prepare.ts

@@ -6,8 +6,11 @@ import * as functionsConfig from "../../functionsConfig";
 import * as getProjectId from "../../getProjectId";
 import { logBullet } from "../../utils";
 import { getRuntimeChoice } from "../../parseRuntimeAndValidateSDK";
-import { getFunctionsInfo, promptForFailurePolicies } from "../../functionsDeployHelper";
+import { functionMatchesAnyGroup, getFilterGroups } from "../../functionsDeployHelper";
+import { CloudFunctionTrigger, functionsByRegion, allFunctions } from "./deploymentPlanner";
+import { promptForFailurePolicies } from "./prompts";
 import { prepareFunctionsUpload } from "../../prepareFunctionsUpload";
+
 import * as validate from "./validate";
 import { checkRuntimeDependencies } from "./checkRuntimeDependencies";
 
@@ -49,17 +52,26 @@ export async function prepare(context: any, options: any, payload: any): Promise
 
   // Get a list of CloudFunctionTriggers, with duplicates for each region.
   payload.functions = {};
-  payload.functions.triggers = getFunctionsInfo(
-    options.config.get("functions.triggers"),
-    projectId
+  // TODO: Make byRegion an implementation detail of deploymentPlanner
+  // and only store a flat array of Functions in payload.
+  payload.functions.byRegion = functionsByRegion(
+    projectId,
+    options.config.get("functions.triggers")
   );
+  payload.functions.triggers = allFunctions(payload.functions.byRegion);
 
   // Validate the function code that is being deployed.
   validate.functionsDirectoryExists(options, sourceDirName);
   // validate.functionNamesAreValid(payload.functionNames);
   // TODO: This doesn't do anything meaningful right now because payload.functions is not defined
   validate.packageJsonIsValid(sourceDirName, sourceDir, projectDir, !!runtimeFromConfig);
 
-  // Display a warning and prompt if any functions have failurePolicies.
-  await promptForFailurePolicies(options, payload.functions.triggers);
+  // Check what --only filters have been passed in.
+  context.filters = getFilterGroups(options);
+
+  // Display a warning and prompt if any functions in the release have failurePolicies.
+  const localFnsInRelease = payload.functions.triggers.filter((fn: CloudFunctionTrigger) => {
+    return functionMatchesAnyGroup(fn.name, context.filters);
+  });
+  await promptForFailurePolicies(options, localFnsInRelease);
 }

src/deploy/functions/prompts.ts

@@ -0,0 +1,55 @@
+import * as clc from "cli-color";
+
+import { getFunctionLabel } from "../../functionsDeployHelper";
+import { CloudFunctionTrigger } from "./deploymentPlanner";
+import { FirebaseError } from "../../error";
+import { promptOnce } from "../../prompt";
+import * as utils from "../../utils";
+
+/**
+ * Checks if a deployment will create any functions with a failure policy.
+ * If there are any, prompts the user to acknowledge the retry behavior.
+ * @param options
+ * @param functions A list of all functions in the deployment
+ */
+export async function promptForFailurePolicies(
+  options: any,
+  functions: CloudFunctionTrigger[]
+): Promise<void> {
+  // Collect all the functions that have a retry policy
+  const failurePolicyFunctions = functions.filter((fn: CloudFunctionTrigger) => {
+    return !!fn.failurePolicy;
+  });
+
+  if (failurePolicyFunctions.length === 0) {
+    return;
+  }
+  const failurePolicyFunctionLabels = failurePolicyFunctions.map((fn: CloudFunctionTrigger) => {
+    return getFunctionLabel(fn.name);
+  });
+  const retryMessage =
+    "The following functions will be retried in case of failure: " +
+    clc.bold(failurePolicyFunctionLabels.join(", ")) +
+    ". " +
+    "Retried executions are billed as any other execution, and functions are retried repeatedly until they either successfully execute or the maximum retry period has elapsed, which can be up to 7 days. " +
+    "For safety, you might want to ensure that your functions are idempotent; see https://firebase.google.com/docs/functions/retries to learn more.";
+
+  utils.logLabeledWarning("functions", retryMessage);
+
+  if (options.force) {
+    return;
+  } else if (options.nonInteractive) {
+    throw new FirebaseError("Pass the --force option to deploy functions with a failure policy", {
+      exit: 1,
+    });
+  }
+  const proceed = await promptOnce({
+    type: "confirm",
+    name: "confirm",
+    default: false,
+    message: "Would you like to proceed with deployment?",
+  });
+  if (!proceed) {
+    throw new FirebaseError("Deployment canceled.", { exit: 1 });
+  }
+}

src/deploy/functions/release.ts

@@ -10,6 +10,7 @@ import * as deploymentTool from "../../deploymentTool";
 import * as track from "../../track";
 import * as utils from "../../utils";
 import * as helper from "../../functionsDeployHelper";
+import { CloudFunctionTrigger } from "./deploymentPlanner";
 import { FirebaseError } from "../../error";
 import { getHumanFriendlyRuntimeName } from "../../parseRuntimeAndValidateSDK";
 import { getAppEngineLocation } from "../../functionsConfig";
@@ -145,7 +146,7 @@ export async function release(context: any, options: any, payload: any): Promise
 
   const appEngineLocation = getAppEngineLocation(context.firebaseConfig);
   let functionsInfo = payload.functions.triggers;
-  functionsInfo = functionsInfo.map((fn: helper.CloudFunctionTrigger) => {
+  functionsInfo = functionsInfo.map((fn: CloudFunctionTrigger) => {
     if (
       fn.eventTrigger &&
       fn.schedule &&
@@ -301,7 +302,7 @@ export async function release(context: any, options: any, payload: any): Promise
   // Delete functions
   const functionsToDelete = _.chain(existingFunctions)
     .filter((functionInfo) => {
-      return deploymentTool.check(functionInfo.labels);
+      return deploymentTool.isFirebaseManaged(functionInfo.labels);
     }) // only delete functions uploaded via firebase-tools
     .map((fn) => {
       return _.get(fn, "name");
@@ -427,7 +428,7 @@ export async function release(context: any, options: any, payload: any): Promise
     }
   }
   // filter out functions that are excluded via --only and --except flags
-  const functionsInDeploy = functionsInfo.filter((trigger: helper.CloudFunctionTrigger) => {
+  const functionsInDeploy = functionsInfo.filter((trigger: CloudFunctionTrigger) => {
     return functionFilterGroups.length > 0 ? _.includes(deleteReleaseNames, trigger.name) : true;
   });
   await createOrUpdateSchedulesAndTopics(

src/deploymentTool.ts

@@ -14,6 +14,6 @@ export function labels() {
   };
 }
 
-export function check(labels: { [key: string]: any }) {
+export function isFirebaseManaged(labels: { [key: string]: any }) {
   return labels?.["deployment-tool"]?.startsWith(BASE);
 }