diff --git a/scripts/live_test/.gitignore b/scripts/live_test/.gitignore new file mode 100644 index 00000000000..64233a9e958 --- /dev/null +++ b/scripts/live_test/.gitignore @@ -0,0 +1 @@ +index.html \ No newline at end of file diff --git a/scripts/live_test/CLITest.yml b/scripts/live_test/CLITest.yml index 188969fa60f..39976d5af91 100644 --- a/scripts/live_test/CLITest.yml +++ b/scripts/live_test/CLITest.yml @@ -1,6 +1,6 @@ # Some content of this file is generated. -name: CLI TEST RUN $(USER_TARGET) $(USER_LIVE) $(Date:yyyyMMdd)$(Rev:.r) +name: CLI TEST RUN $(USER_TARGET) $(USER_LIVE) $(USER_USERNAME) $(Date:yyyyMMdd)$(Rev:.r) trigger: branches: @@ -272,6 +272,7 @@ jobs: commit_id=`git ls-remote https://github.com/Azure/azure-cli.git HEAD` pip install sendgrid pip install mysql-connector-python + pip install requests # pip install certifi # Send notification az -v diff --git a/scripts/live_test/HISTORY.rst b/scripts/live_test/HISTORY.rst index b6ad321af35..11873a004b3 100644 --- a/scripts/live_test/HISTORY.rst +++ b/scripts/live_test/HISTORY.rst @@ -3,10 +3,20 @@ Release History =============== +0.4.0 +++++++ + +* Update pipeline run title. +* Generate index.html of testing results. +* Design a unique representation of a pipeline run. +* Make storage account container public. + 0.3.0 ++++++ * Support upgrading API version in pipeline. +* Fix a tiny DB bug. +* Fix pipe not close problem. 0.2.0 ++++++ diff --git a/scripts/live_test/generate_index.py b/scripts/live_test/generate_index.py new file mode 100644 index 00000000000..6a22b88f058 --- /dev/null +++ b/scripts/live_test/generate_index.py @@ -0,0 +1,143 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +""" +Generate index.html of testing results HTML pages. +""" +import traceback +import os +import requests +import xml.etree.ElementTree as ET + + +def generate(container, container_url, testdata, USER_REPO, USER_BRANCH, COMMIT_ID, USER_LIVE): + """ + Generate index.html. Upload it to storage account + :param container: + :param container_url: + :return: + """ + data = [] + url = container_url + '?restype=container&comp=list' + content = requests.get(url).content + # print(content) + root = ET.fromstring(content) + for blobs in root: + for blob in blobs: + name = url = '' + for e in blob: + if e.tag == 'Name': + name = e.text + if e.tag == 'Url': + url = e.text + if name == '' or url == '': + print('[Warning] Blob\'s name or url is empty, name: {}, url: {}'.format(name, url)) + if name.endswith('.html'): + data.append({'name': name, 'url': url}) + break + print(data) + html = render(data, container_url, testdata, USER_REPO, USER_BRANCH, COMMIT_ID, USER_LIVE) + with open('index.html', 'w') as f: + f.write(html) + + # Upload to storage account + cmd = 'az storage blob upload -f index.html -c {} -n index.html --account-name clitestresultstac'.format(container) + print('Running: ' + cmd) + os.system(cmd) + + +def render(data, container_url, testdata, USER_REPO, USER_BRANCH, COMMIT_ID, USER_LIVE): + content = """ + + + + + +

Testing results of Azure CLI

+ """ + + live = 'True' if USER_LIVE == '--live' else 'False' + + content += """ +

+ Repository: {}
+ Branch: {}
+ Commit: {}
+ Live: {}
+

+ """.format(USER_REPO, USER_BRANCH, COMMIT_ID, live) + + content += """ +

+ User Manual of Live Test Pipeline +

+

+ Word +

+ """ + + table = """ +

Test results summary

+ + + + + + + + """ + + for module, passed, failed, rate in testdata.modules: + table += """ + + + + + + + """.format(module, passed, failed, rate) + + table += """ + + + + + + +
ModulePassedFailedPass rate
{}{}{}{}
Total{}{}{}
+ """.format(testdata.total[1], testdata.total[2], testdata.total[3]) + + content += table + + content += """ +

Reports

+ """ + + for item in data: + name = item['name'] + url = item['url'] + content += """ + {}
+ """.format(url, name) + + content += """ + + + """ + return content + + +if __name__ == '__main__': + url = 'https://clitestresultstac.blob.core.windows.net/20200919213646live' + try: + generate(url) + except: + traceback.print_exc() diff --git a/scripts/live_test/sendemail.py b/scripts/live_test/sendemail.py index d3e96b062bf..b7e4a0cb3cd 100644 --- a/scripts/live_test/sendemail.py +++ b/scripts/live_test/sendemail.py @@ -7,7 +7,10 @@ import os import traceback import datetime +import random +import string import test_data +import generate_index SENDGRID_KEY = sys.argv[1] @@ -58,6 +61,13 @@ def main(): except Exception: print(traceback.format_exc()) + # Generate index.html + try: + container_url = 'https://clitestresultstac.blob.core.windows.net/' + container + generate_index.generate(container, container_url, testdata, USER_REPO, USER_BRANCH, COMMIT_ID, USER_LIVE) + except Exception: + print(traceback.format_exc()) + # Write database try: write_db(container, testdata) @@ -69,22 +79,13 @@ def main(): def get_container_name(): """ - Generate container name in storage account + Generate container name in storage account. It is also an identifier of the pipeline run. :return: """ print('Enter get_container_name()') - - date = datetime.datetime.now().strftime('%Y%m%d%H%M%S') - name = date - if USER_LIVE == '--live': - mode = 'live' - elif USER_LIVE == '': - mode = 'replay' - else: - mode = '' - name += mode - # if USER_TARGET == '' and USER_REPO == 'https://github.com/Azure/azure-cli.git' and USER_BRANCH == 'dev' and USER_LIVE == '--live': - # name += '_archive' + time = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') + random_id = ''.join(random.choice(string.digits) for _ in range(6)) + name = time + '-' + random_id print('Exit get_container_name()') return name @@ -98,7 +99,7 @@ def upload_files(container): print('Enter upload_files()') # Create container - cmd = 'az storage container create -n {} --account-name clitestresultstac --account-key {}' + cmd = 'az storage container create -n {} --account-name clitestresultstac --account-key {} --public-access container' os.popen(cmd.format(container, ACCOUNT_KEY)) # Upload files @@ -109,7 +110,7 @@ def upload_files(container): cmd = 'az storage blob upload -f {} -c {} -n {} --account-name clitestresultstac' cmd = cmd.format(fullpath, container, name) print('Running: ' + cmd) - os.popen(cmd) + os.system(cmd) print('Exit upload_files()') @@ -119,22 +120,25 @@ def write_db(container, testdata): Insert data to database. Sql statements to create table: USE clidb; - CREATE TABLE t1 ( - repo VARCHAR(200) COMMENT 'Repo URL', - branch VARCHAR(200) COMMENT 'Branch name', - commit VARCHAR(50) COMMENT 'Commit ID', - target VARCHAR(2000) COMMENT 'Target modules to test. Splited by space, Empty string represents all modules', - live TINYINT(1) COMMENT 'Live run or not', - user VARCHAR(50) COMMENT 'User (email address) who triggers the run', - pass INT COMMENT 'Number of passed tests', - fail INT COMMENT 'Number of failed tests', - rate VARCHAR(50) COMMENT 'Pass rate', - detail VARCHAR(10000) COMMENT 'Detail', - container VARCHAR(200) COMMENT 'Container URL', - date VARCHAR(10) COMMENT 'Date. E.g. 20200801', - time VARCHAR(10) COMMENT 'Time. E.g. 183000' + CREATE TABLE `t1` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `repr` varchar(30) DEFAULT NULL COMMENT 'date_time_random6digits', + `repo` varchar(200) DEFAULT NULL COMMENT 'Repo URL', + `branch` varchar(200) DEFAULT NULL COMMENT 'Branch name', + `commit` varchar(50) DEFAULT NULL COMMENT 'Commit ID', + `target` varchar(2000) DEFAULT NULL COMMENT 'Target modules to test. Splited by space, Empty string represents all modules', + `live` tinyint(1) DEFAULT NULL COMMENT 'Live run or not', + `user` varchar(50) DEFAULT NULL COMMENT 'User (email address) who triggers the run', + `pass` int(11) DEFAULT NULL COMMENT 'Number of passed tests', + `fail` int(11) DEFAULT NULL COMMENT 'Number of failed tests', + `rate` varchar(50) DEFAULT NULL COMMENT 'Pass rate', + `detail` varchar(10000) DEFAULT NULL COMMENT 'Detail', + `container` varchar(200) DEFAULT NULL COMMENT 'Container URL', + `date` varchar(10) DEFAULT NULL COMMENT 'Date. E.g. 20200801', + `time` varchar(10) DEFAULT NULL COMMENT 'Time. E.g. 183000', + PRIMARY KEY (`id`), + UNIQUE KEY `repr` (`repr`) ); - """ print('Enter write_db()') @@ -146,22 +150,23 @@ def write_db(container, testdata): host='clisqldbserver.mysql.database.azure.com', database='clidb') cursor = cnx.cursor() - sql = 'INSERT INTO t1 (repo, branch, commit, target, live, user, pass, fail, rate, detail, container, date, time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' + sql = 'INSERT INTO t1 (repr, repo, branch, commit, target, live, user, pass, fail, rate, detail, container, date, time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);' + repr = container repo = USER_REPO branch = USER_BRANCH commit = COMMIT_ID target = USER_TARGET live = 1 if USER_LIVE == '--live' else 0 user = REQUESTED_FOR_EMAIL - pass0 = testdata.total[2] - fail = testdata.total[1] + pass0 = testdata.total[1] + fail = testdata.total[2] rate = testdata.total[3] detail = str(testdata.modules) - container = 'https://clitestresultstac.blob.core.windows.net/' + container + container = 'https://clitestresultstac.blob.core.windows.net/{}/index.html'.format(container) d = datetime.datetime.now() date = d.strftime('%Y%m%d') time = d.strftime('%H%M%S') - data = (repo, branch, commit, target, live, user, pass0, fail, rate, detail, container, date, time) + data = (repr, repo, branch, commit, target, live, user, pass0, fail, rate, detail, container, date, time) print(data) cursor.execute(sql, data) @@ -205,6 +210,7 @@ def send_email(container, testdata): data['personalizations'][0]['to'].append({'email': REQUESTED_FOR_EMAIL}) if USER_TARGET == '' and USER_REPO == 'https://github.com/Azure/azure-cli.git' and USER_BRANCH == 'dev' and USER_LIVE == '--live' and REQUESTED_FOR_EMAIL == '': data['personalizations'][0]['to'].append({'email': 'AzPyCLI@microsoft.com'}) + print(data) sendgrid_key = sys.argv[1] sg = SendGridAPIClient(sendgrid_key) diff --git a/scripts/live_test/test_data.py b/scripts/live_test/test_data.py index 829cccc794c..646fc9e008b 100644 --- a/scripts/live_test/test_data.py +++ b/scripts/live_test/test_data.py @@ -12,9 +12,9 @@ class TestData: Model for testing results """ def __init__(self, artifact_dir): - # Value of the list should be (module_name, failed, passed, pass_rate) + # Value of the list should be (module_name, passed, failed, pass_rate) self.modules = [] - # ('Total', failed, passed, pass_rate) + # ('Total', passed, failed, pass_rate) self.total = None self.artifact_dir = artifact_dir