Support running the end-to-end test repeatedly. (#2053)

This will aid in reproducing and fixing flaky test failures.

In particular, this is being added to help test fixes for #2051.

Author: ojarjur

tools/cli/tests/end-to-end.py

@@ -18,6 +18,7 @@
 # of the bundled CLI tool. This requires a GCP project in which the
 # test will create, connect to, and delete Datalab instances.
 
+import argparse
 import random
 import socket
 import subprocess
@@ -232,4 +233,18 @@ def run_connection_test(self):
 
 
 if __name__ == '__main__':
-    unittest.main()
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--runs', type=int, default=1, choices=range(1, 100),
+                        metavar='COUNT', dest='runs',
+                        help='Number of times to run the test suite')
+    args = parser.parse_args()
+
+    failed_count, run_count = 0, 0
+    for _ in range(0, args.runs):
+        suite = unittest.TestLoader().loadTestsFromTestCase(TestEndToEnd)
+        result = unittest.TextTestRunner(buffer=True).run(suite)
+        run_count += 1
+        if not result.wasSuccessful():
+            failed_count += 1
+
+    print('Ran {} test runs with {} failing'.format(run_count, failed_count))