Improving stability and speed of intergration tests (firebase#384)

* Adds delay to ensure functions are deployed, adds ability to test node6 and node8 on different projects

* Support for 1 or 2 projects, run tests simulatenously when 2 projects

* retry deploy up to 3 times, fix typo

* Switch to cleaner looping

Co-Authored-By: joehan <[email protected]>

* code review;

Author: joehan

integration_test/functions/src/index.ts

@@ -148,7 +148,7 @@ export const integrationTests: any = functions
       })
       .then(() => {
         console.log('All tests pass!');
-        resp.status(200).send('PASS');
+        resp.status(200).send('PASS \n');
       })
       .catch(err => {
         console.log(`Some tests failed: ${err}`);

integration_test/run_tests.sh

@@ -4,15 +4,25 @@
 set -e
 
 function usage {
-  echo "Usage: $0 <project_id>"
+  echo "Usage: $0 <project_id_node_6> <project_id_node_8>"
   exit 1
 }
 
+# This script takes 1 or 2 params, both of which are Firebase project ids.
+# If there is only one given, that project will be used for both node6 and node8
+# Otherwise, param1 will be used for node6
+# and param2 will be used for node8
 # The first parameter is required and is the Firebase project id.
 if [[ $1 == "" ]]; then
   usage
 fi
-PROJECT_ID=$1
+if [[ $2 == "" ]]; then
+  PROJECT_ID_NODE_6=$1
+  PROJECT_ID_NODE_8=$1
+else
+  PROJECT_ID_NODE_6=$1
+  PROJECT_ID_NODE_8=$2
+fi
 
 # Directory where this script lives.
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@@ -31,11 +41,13 @@ function build_sdk {
 
 function pick_node6 {
   cd $DIR
+  PROJECT_ID=$PROJECT_ID_NODE_6
   cp package.node6.json functions/package.json
 }
 
 function pick_node8 {
   cd $DIR
+  PROJECT_ID=$PROJECT_ID_NODE_8
   cp package.node8.json functions/package.json
 }
 
@@ -51,15 +63,47 @@ function delete_all_functions {
   cd $DIR
   # Try to delete, if there are errors it is because the project is already empty,
   # in that case do nothing. 
-  firebase functions:delete callableTests createUserTests databaseTests deleteUserTests firestoreTests integrationTests pubsubTests remoteConfigTests --project=$PROJECT_ID -f || :
+  firebase functions:delete callableTests createUserTests databaseTests deleteUserTests firestoreTests integrationTests pubsubTests remoteConfigTests --force --project=$PROJECT_ID_NODE_6 || : &
+  if ! [[ $PROJECT_ID_NODE_6 == $PROJECT_ID_NODE_8 ]]; then
+    firebase functions:delete callableTests createUserTests databaseTests deleteUserTests firestoreTests integrationTests pubsubTests remoteConfigTests --force --project=$PROJECT_ID_NODE_8 || : &
+  fi
+  wait
   announce "Project emptied."
 }
 
 function deploy {
   cd $DIR
   ./functions/node_modules/.bin/tsc -p functions/
-  # Deploy functions, and security rules for database and Firestore
-  firebase deploy --project=$PROJECT_ID --only functions,database,firestore
+  # Deploy functions, and security rules for database and Firestore. If the deploy fails, retry twice
+  for i in 1 2 3; do firebase deploy --project=$PROJECT_ID --only functions,database,firestore && break; done
+}
+
+# At the moment, functions take 30-40 seconds AFTER firebase deploy returns successfully to go live
+# This needs to be fixed separately
+# However, so that we have working integration tests in the interim, waitForPropagation is a workaround
+function waitForPropagation {
+  announce "Waiting 50 seconds for functions changes to propagate"
+  sleep 50
+}
+
+function run_all_tests {
+  announce "Running the integration tests..."
+
+  # Constructs the URLs for both test functions. This may change in the future,
+  # causing this script to start failing, but currently we don't have a very
+  # reliable way of determining the URL dynamically.
+  TEST_DOMAIN="cloudfunctions.net"
+  if [[ $FIREBASE_FUNCTIONS_URL == "https://preprod-cloudfunctions.sandbox.googleapis.com" ]]; then
+    TEST_DOMAIN="txcloud.net"
+  fi
+  TEST_URL_NODE_6="https://us-central1-$PROJECT_ID_NODE_6.$TEST_DOMAIN/integrationTests"
+  TEST_URL_NODE_8="https://us-central1-$PROJECT_ID_NODE_8.$TEST_DOMAIN/integrationTests"
+  echo $TEST_URL_NODE_6
+  echo $TEST_URL_NODE_8
+  curl --fail $TEST_URL_NODE_6 & NODE6PID=$!
+  curl --fail $TEST_URL_NODE_8 & NODE8PID=$!
+  wait $NODE6PID && echo 'node 6 passed' || (announce 'Node 6 tests failed'; cleanup; announce 'Tests failed'; exit 1)
+  wait $NODE8PID && echo 'node 8 passed' || (announce 'Node 8 tests failed'; cleanup; announce 'Tests failed'; exit 1)
 }
 
 function run_tests {
@@ -93,10 +137,18 @@ install_deps
 delete_all_functions
 announce "Deploying functions to Node 8 runtime ..."
 deploy
-run_tests
+if [[ $PROJECT_ID_NODE_6 == $PROJECT_ID_NODE_8 ]]; then
+  waitForPropagation
+  run_tests
+fi
 pick_node6
 announce "Re-deploying the same functions to Node 6 runtime ..."
 deploy
-run_tests
+waitForPropagation
+if [[ $PROJECT_ID_NODE_6 == $PROJECT_ID_NODE_8 ]]; then
+  run_tests
+else
+  run_all_tests
+fi
 cleanup
 announce "All tests pass!"