diff --git a/src/azdev/commands.py b/src/azdev/commands.py index b28bc18fb..64fdb57ab 100644 --- a/src/azdev/commands.py +++ b/src/azdev/commands.py @@ -51,6 +51,10 @@ def operation_group(name): g.command('add', 'add_extension') g.command('remove', 'remove_extension') g.command('list', 'list_extensions') + g.command('update-index', 'update_extension_index') + + with CommandGroup(self, 'group', operation_group('resource')) as g: + g.command('delete', 'delete_groups') # TODO: implement # with CommandGroup(self, operation_group('help')) as g: diff --git a/src/azdev/help.py b/src/azdev/help.py index a52d85a55..51f6c2569 100644 --- a/src/azdev/help.py +++ b/src/azdev/help.py @@ -17,12 +17,9 @@ long-summary: Find or clones the relevant repositories and installs the necessary modules. """ -helps['configure'] = """ - short-summary: Configure azdev for use without installing anything. -""" helps['configure'] = """ - short-summary: Configure `azdev` for development. + short-summary: Configure azdev for use without installing anything. """ @@ -153,3 +150,15 @@ helps['extension list'] = """ short-summary: List what extensions are currently visible to your development environment. """ + +helps['extension update-index'] = """ + short-summary: Update the extensions index.json from a built WHL file. +""" + +helps['group delete'] = """ + short-summary: Delete several resource groups with filters. Useful for cleaning up test resources. + long-summary: > + Can filter either by key tags used by the CLI infrastructure, or by name prefix. If name prefix + is used, the tag filters will be ignored. This command doesn't guarantee the resource group will + be deleted. +""" diff --git a/src/azdev/operations/extensions.py b/src/azdev/operations/extensions/__init__.py similarity index 62% rename from src/azdev/operations/extensions.py rename to src/azdev/operations/extensions/__init__.py index 1a622a979..630a57d32 100644 --- a/src/azdev/operations/extensions.py +++ b/src/azdev/operations/extensions/__init__.py @@ -1,8 +1,7 @@ -# ----------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# ----------------------------------------------------------------------------- +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- from collections import OrderedDict from glob import glob @@ -85,3 +84,58 @@ def list_extensions(): if long_name not in installed: results.append({'name': long_name, 'path': folder}) return results + + +def _get_sha256sum(a_file): + import hashlib + sha256 = hashlib.sha256() + with open(a_file, 'rb') as f: + sha256.update(f.read()) + return sha256.hexdigest() + + +def update_extension_index(extension): + import json + import re + import tempfile + + from .util import get_ext_metadata, get_whl_from_url + + NAME_REGEX = r'.*/([^/]*)-\d+.\d+.\d+' + + ext_path = get_ext_repo_path() + + # Get extension WHL from URL + if not extension.endswith('.whl') or not extension.startswith('https:'): + raise ValueError('usage error: only URL to a WHL file currently supported.') + + # Extract the extension name + try: + extension_name = re.findall(NAME_REGEX, extension)[0] + extension_name = extension_name.replace('_', '-') + except IndexError: + raise ValueError('unable to parse extension name') + + extensions_dir = tempfile.mkdtemp() + ext_dir = tempfile.mkdtemp(dir=extensions_dir) + whl_cache_dir = tempfile.mkdtemp() + whl_cache = {} + ext_file = get_whl_from_url(whl_path, extension_name, whl_cache_dir, whl_cache) + + with open(os.join(ext_path, 'src', 'index.json'), 'r') as infile: + curr_index = json.loads(infile.read()) + + try: + entry = curr_index['extensions'][extension_name] + except IndexError: + raise ValueError('{} not found in index.json'.format(extension_name)) + + entry[0]['downloadUrl'] = whl_path + entry[0]['sha256Digest'] = _get_sha256sum(ext_file) + entry[0]['filename'] = whl_path.split('/')[-1] + entry[0]['metadata'] = get_ext_metadata(ext_dir, ext_file, extension_name) + + # update index and write back to file + curr_index['extensions'][extension_name] = entry + with open(os.join(ext_path, 'src', 'index.json'), 'w') as outfile: + outfile.write(json.dumps(curr_index, indent=4, sort_keys=True)) diff --git a/src/azdev/operations/extensions/util.py b/src/azdev/operations/extensions/util.py new file mode 100644 index 000000000..e91e53286 --- /dev/null +++ b/src/azdev/operations/extensions/util.py @@ -0,0 +1,71 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import json +import zipfile +from wheel.install import WHEEL_INFO_RE + +from azdev.utilities import EXTENSION_PREFIX + + +def _get_extension_modname(ext_dir): + # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L153 + pos_mods = [n for n in os.listdir(ext_dir) + if n.startswith(EXTENSION_PREFIX) and os.path.isdir(os.path.join(ext_dir, n))] + if len(pos_mods) != 1: + raise AssertionError("Expected 1 module to load starting with " + "'{}': got {}".format(EXTENSION_PREFIX, pos_mods)) + return pos_mods[0] + + +def _get_azext_metadata(ext_dir): + # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L109 + AZEXT_METADATA_FILENAME = 'azext_metadata.json' + azext_metadata = None + ext_modname = _get_extension_modname(ext_dir=ext_dir) + azext_metadata_filepath = os.path.join(ext_dir, ext_modname, AZEXT_METADATA_FILENAME) + if os.path.isfile(azext_metadata_filepath): + with open(azext_metadata_filepath) as f: + azext_metadata = json.load(f) + return azext_metadata + + +def get_ext_metadata(ext_dir, ext_file, ext_name): + # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L89 + WHL_METADATA_FILENAME = 'metadata.json' + zip_ref = zipfile.ZipFile(ext_file, 'r') + zip_ref.extractall(ext_dir) + zip_ref.close() + metadata = {} + dist_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.dist-info')] + azext_metadata = _get_azext_metadata(ext_dir) + if azext_metadata: + metadata.update(azext_metadata) + for dist_info_dirname in dist_info_dirs: + parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname) + if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == ext_name.replace('-', '_'): + whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME) + if os.path.isfile(whl_metadata_filepath): + with open(whl_metadata_filepath) as f: + metadata.update(json.load(f)) + return metadata + + +def get_whl_from_url(url, filename, tmp_dir, whl_cache=None): + if not whl_cache: + whl_cache = {} + if url in whl_cache: + return whl_cache[url] + import requests + r = requests.get(url, stream=True) + assert r.status_code == 200, "Request to {} failed with {}".format(url, r.status_code) + ext_file = os.path.join(tmp_dir, filename) + with open(ext_file, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # ignore keep-alive new chunks + f.write(chunk) + whl_cache[url] = ext_file + return ext_file diff --git a/src/azdev/operations/performance.py b/src/azdev/operations/performance.py index 88f75e9f6..fd9105a0d 100644 --- a/src/azdev/operations/performance.py +++ b/src/azdev/operations/performance.py @@ -19,7 +19,6 @@ logger = get_logger(__name__) TOTAL = 'ALL' -NUM_RUNS = 3 DEFAULT_THRESHOLD = 10 # TODO: Treat everything as bubble instead of specific modules @@ -33,14 +32,14 @@ } -def check_load_time(): +def check_load_time(runs=3): heading('Module Load Performance') regex = r"[^']*'([^']*)'[\D]*([\d\.]*)" results = {TOTAL: []} # Time the module loading X times - for i in range(0, NUM_RUNS + 1): + for i in range(0, runs + 1): lines = cmd('az -h --debug', show_stderr=True).result if i == 0: # Ignore the first run since it can be longer due to *.pyc file compilation diff --git a/src/azdev/operations/resource.py b/src/azdev/operations/resource.py new file mode 100644 index 000000000..1bd2e2cf0 --- /dev/null +++ b/src/azdev/operations/resource.py @@ -0,0 +1,81 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- + +import json + +from knack.log import get_logger +from knack.prompting import prompt_y_n +from knack.util import CLIError + +from azdev.utilities import ( + cmd as run_cmd, heading, subheading, display) + +logger = get_logger(__name__) + + +class Data(object): + def __init__(self, **kw): + self.__dict__.update(kw) + if 'properties' in self.__dict__: + self.__dict__.update(self.properties) + del self.properties + + +def delete_groups(cmd, prefixes=None, older_than=6, product='azurecli', cause='automation', yes=False): + from datetime import datetime, timedelta + + groups = json.loads(run_cmd('az group list -ojson').result) + groups_to_delete = [] + + def _filter_by_tags(): + for group in groups: + group = Data(**group) + + if not group.tags: + continue + + tags = Data(**group.tags) + try: + date_tag = datetime.strptime(tags.date, '%Y-%m-%dT%H:%M:%SZ') + curr_time = datetime.utcnow() + if tags.product == product and tags.cause == cause and (curr_time - date_tag <= timedelta(hours=older_than + 1)): + groups_to_delete.append(group.name) + except AttributeError: + continue + + def _filter_by_prefix(): + for group in groups: + group = Data(**group) + + for prefix in prefixes: + if group.name.startswith(prefix): + groups_to_delete.append(group.name) + + def _delete(): + for group in groups_to_delete: + run_cmd('az group delete -g {} -y --no-wait'.format(group), message=True) + + if prefixes: + logger.info('Filter by prefix') + _filter_by_prefix() + else: + logger.info('Filter by tags') + _filter_by_tags + + if not groups_to_delete: + raise CLIError('No groups meet the criteria to delete.') + + if yes: + _delete() + else: + subheading('Groups to Delete') + for group in groups_to_delete: + display('\t{}'.format(group)) + + if prompt_y_n('Delete {} resource groups?'.format(len(groups_to_delete)), 'y'): + _delete() + else: + raise CLIError('Command cancelled.') \ No newline at end of file diff --git a/src/azdev/operations/setup.py b/src/azdev/operations/setup.py index 10767260a..3c2231672 100644 --- a/src/azdev/operations/setup.py +++ b/src/azdev/operations/setup.py @@ -17,7 +17,7 @@ from azdev.params import Flag from azdev.utilities import ( display, heading, subheading, cmd, py_cmd, pip_cmd, find_file, IS_WINDOWS, get_path_table, - get_azdev_config_dir) + get_env_config_dir, get_env_config) logger = get_logger(__name__) @@ -144,7 +144,7 @@ def _copy_config_files(): config_mod = import_module('azdev.config') config_dir_path = config_mod.__dict__['__path__'][0] - dest_path = os.path.join(get_azdev_config_dir(), 'config_files') + dest_path = os.path.join(get_env_config_dir(), 'config_files') if os.path.exists(dest_path): rmtree(dest_path) copytree(config_dir_path, dest_path) @@ -213,14 +213,14 @@ def setup(cmd, venv='env', cli_path=None, ext_path=None, yes=None): _get_venv_activate_command(venv) ) ) + config = get_env_config() # save data to config files - config = cmd.cli_ctx.config if ext_path: from azdev.utilities import get_azure_config config.set_value('ext', 'repo_path', ext_path) az_config = get_azure_config() - az_config.set_value('extension', 'dir', os.path.join(ext_path, 'src')) + az_config.set_value('extension', 'dir', os.path.join(ext_path)) if cli_path: config.set_value('cli', 'repo_path', cli_path) @@ -262,12 +262,12 @@ def configure(cmd, cli_path=None, ext_path=None): display("Azure CLI extensions repo found at: {}".format(ext_path)) # save data to config files - config = cmd.cli_ctx.config + config = get_env_config() if ext_path: from azdev.utilities import get_azure_config config.set_value('ext', 'repo_path', ext_path) az_config = get_azure_config() - az_config.set_value('extension', 'dir', os.path.join(ext_path, 'src')) + az_config.set_value('extension', 'dir', os.path.join(ext_path)) if cli_path: config.set_value('cli', 'repo_path', cli_path) diff --git a/src/azdev/operations/style.py b/src/azdev/operations/style.py index e65ec00b6..1d8db967c 100644 --- a/src/azdev/operations/style.py +++ b/src/azdev/operations/style.py @@ -19,7 +19,7 @@ from azdev.utilities import ( display, heading, subheading, py_cmd, get_path_table, EXTENSION_PREFIX, - get_azdev_config_dir) + get_env_config_dir) def check_style(cmd, modules=None, pylint=False, pep8=False): @@ -112,7 +112,7 @@ def _run_pylint(cli_path, ext_path, modules): def run(paths, rcfile, desc): if not paths: return None - config_path = os.path.join(get_azdev_config_dir(), 'config_files', rcfile) + config_path = os.path.join(get_env_config_dir(), 'config_files', rcfile) logger.info('Using rcfile file: %s', config_path) logger.info('Running on %s: %s', desc, ' '.join(paths)) command = 'pylint {} --rcfile={} -j {}'.format(' '.join(paths), @@ -133,7 +133,7 @@ def _run_pep8(cli_path, ext_path, modules): def run(paths, config_file, desc): if not paths: return - config_path = os.path.join(get_azdev_config_dir(), 'config_files', config_file) + config_path = os.path.join(get_env_config_dir(), 'config_files', config_file) logger.info('Using config file: %s', config_path) logger.info('Running on %s: %s', desc, ' '.join(paths)) command = 'flake8 --statistics --append-config={} {}'.format( diff --git a/src/azdev/operations/tests/__init__.py b/src/azdev/operations/tests/__init__.py index 3f4ffa3b9..0fbcd82fd 100644 --- a/src/azdev/operations/tests/__init__.py +++ b/src/azdev/operations/tests/__init__.py @@ -20,14 +20,14 @@ cmd, py_cmd, pip_cmd, find_file, IS_WINDOWS, ENV_VAR_TEST_MODULES, ENV_VAR_TEST_LIVE, COMMAND_MODULE_PREFIX, EXTENSION_PREFIX, - make_dirs, get_azdev_config_dir, + make_dirs, get_env_config_dir, get_path_table) logger = get_logger(__name__) DEFAULT_RESULT_FILE = 'test_results.xml' -DEFAULT_RESULT_PATH = os.path.join(get_azdev_config_dir(), DEFAULT_RESULT_FILE) +DEFAULT_RESULT_PATH = os.path.join(get_env_config_dir(), DEFAULT_RESULT_FILE) def run_tests(cmd, tests, xml_path=None, ci_mode=False, discover=False, in_series=False, @@ -266,7 +266,8 @@ def add_to_index(key, path): def _get_test_index(cmd, profile, discover): - test_index_dir = os.path.join(cmd.cli_ctx.config.config_dir, 'test_index') + config_dir = get_env_config_dir() + test_index_dir = os.path.join(config_dir, 'test_index') make_dirs(test_index_dir) test_index_path = os.path.join(test_index_dir, '{}.json'.format(profile)) test_index = {} diff --git a/src/azdev/params.py b/src/azdev/params.py index e177241d4..ab32feb8a 100644 --- a/src/azdev/params.py +++ b/src/azdev/params.py @@ -64,6 +64,19 @@ def load_arguments(self, command): c.argument('rules', options_list=['--rules', '-r'], nargs='+', help='Space-separated list of rules to run. Omit to run all rules.') c.argument('rule_types', options_list=['--rule-types', '-t'], nargs='+', choices=['params', 'commands', 'command_groups', 'help_entries'], help='Space-separated list of rule types to run. Omit to run all.') + with ArgumentsContext(self, 'perf') as c: + c.argument('runs', type=int, help='Number of runs to average performance over.') + for scope in ['extension add', 'extension remove']: with ArgumentsContext(self, scope) as c: c.positional('extensions', metavar='NAME', nargs='+', help='Space-separated list of extension names.') + + with ArgumentsContext(self, 'extension update-index') as c: + c.positional('extension', metavar='URL', help='URL to an extension WHL file.') + + with ArgumentsContext(self, 'group delete') as c: + c.argument('product', help='Value for tag `product` to mark for deletion.', arg_group='Tag') + c.argument('older_than', type=int, help='Minimum age (in hours) for tag `date` to mark for deletion.', arg_group='Tag') + c.argument('cause', help='Value for tag `cause` to mark for deletion.', arg_group='Tag') + c.argument('yes', options_list=['--yes', '-y'], help='Do not prompt.') + c.argument('prefixes', options_list=['--prefixes', '-p'], nargs='+', help='Space-separated list of prefixes to filter by.') diff --git a/src/azdev/utilities/__init__.py b/src/azdev/utilities/__init__.py index 244d96f12..515ffb015 100644 --- a/src/azdev/utilities/__init__.py +++ b/src/azdev/utilities/__init__.py @@ -7,7 +7,9 @@ get_azure_config, get_azure_config_dir, get_azdev_config, - get_azdev_config_dir + get_azdev_config_dir, + get_env_config, + get_env_config_dir ) from .command import ( call, @@ -53,6 +55,8 @@ 'get_azure_config', 'get_azdev_config_dir', 'get_azdev_config', + 'get_env_config', + 'get_env_config_dir', 'ENV_VAR_TEST_MODULES', 'ENV_VAR_TEST_LIVE', 'IS_WINDOWS', diff --git a/src/azdev/utilities/config.py b/src/azdev/utilities/config.py index a7596654c..c6b77e964 100644 --- a/src/azdev/utilities/config.py +++ b/src/azdev/utilities/config.py @@ -6,6 +6,8 @@ import os from knack.config import CLIConfig +from knack.util import CLIError + def get_azdev_config(): return CLIConfig(config_dir=get_azdev_config_dir(), config_env_var_prefix='AZDEV') @@ -15,6 +17,10 @@ def get_azure_config(): return CLIConfig(config_dir=get_azure_config_dir(), config_env_var_prefix='AZURE') +def get_env_config(): + return CLIConfig(config_dir=get_env_config_dir(), config_env_var_prefix='AZDEV') + + def get_azdev_config_dir(): """ Returns the user's .azdev directory. """ return os.getenv('AZDEV_CONFIG_DIR', None) or os.path.expanduser(os.path.join('~', '.azdev')) @@ -23,3 +29,14 @@ def get_azdev_config_dir(): def get_azure_config_dir(): """ Returns the user's Azure directory. """ return os.getenv('AZURE_CONFIG_DIR', None) or os.path.expanduser(os.path.join('~', '.azure')) + + +def get_env_config_dir(): + from azdev.utilities.const import ENV_VAR_VIRTUAL_ENV + + _, env_name = os.path.splitdrive(os.getenv(ENV_VAR_VIRTUAL_ENV)) + if not env_name: + raise CLIError('An active Python virtual environment is required.') + azdev_config_dir = get_azdev_config_dir() + config_dir = os.path.join(azdev_config_dir, 'env_config') + env_name + return config_dir diff --git a/src/azdev/utilities/path.py b/src/azdev/utilities/path.py index 1b6fdfb52..fbaa9f5ac 100644 --- a/src/azdev/utilities/path.py +++ b/src/azdev/utilities/path.py @@ -30,9 +30,9 @@ def get_cli_repo_path(): :returns: Path (str) to Azure CLI repo. """ - from .config import get_azdev_config + from .config import get_env_config try: - return get_azdev_config().get('cli', 'repo_path', None) + return get_env_config().get('cli', 'repo_path', None) except Exception: CLIError('Unable to retrieve CLI repo path from config. Please run `azdev setup`.') @@ -42,9 +42,9 @@ def get_ext_repo_path(): :returns: Path (str) to Azure CLI extensions repo. """ - from .config import get_azdev_config + from .config import get_env_config try: - return get_azdev_config().get('ext', 'repo_path') + return get_env_config().get('ext', 'repo_path') except Exception: CLIError('Unable to retrieve extensions repo path from config. Please run `azdev setup`.') @@ -72,7 +72,7 @@ def make_dirs(path): raise -def get_path_table(filter=None): +def get_path_table(include_only=None): """ Gets a table which contains the long and short names of different modules and extensions and the path to them. The structure looks like: { @@ -92,9 +92,9 @@ def get_path_table(filter=None): """ # determine whether the call will filter or return all - if isinstance(filter, str): - filter = [filter] - get_all = not filter + if isinstance(include_only, str): + include_only = [include_only] + get_all = not include_only table = {} cli_path = get_cli_repo_path() @@ -124,17 +124,17 @@ def _update_table(pattern, key): if get_all: table[key][long_name] = folder continue - elif not filter: + elif not include_only: # nothing left to filter return else: # check and update filter - if short_name in filter: - filter.remove(short_name) + if short_name in include_only: + include_only.remove(short_name) table[key][short_name] = folder - if long_name in filter: + if long_name in include_only: # long name takes precedence to ensure path doesn't appear twice - filter.remove(long_name) + include_only.remove(long_name) table[key].pop(short_name, None) table[key][long_name] = folder @@ -142,29 +142,7 @@ def _update_table(pattern, key): _update_table(core_pattern, 'core') _update_table(ext_pattern, 'ext') - if filter: - raise CLIError('unrecognized names: {}'.format(' '.join(filter))) + if include_only: + raise CLIError('unrecognized names: {}'.format(' '.join(include_only))) return table - - -def filter_module_paths(paths, filter): - """ Filter command module and extension paths. - - :param paths: [(str, str)] List of (name, path) tuples. - :param filter: [str] List of command module or extension names to return, or None to return all. - """ - if not filter: - return paths - - if isinstance(filter, str): - filter = [filter] - - filtered = [] - for name, path in paths: - if name in filter: - filtered.append((name, path)) - filter.remove(name) - if filter: - raise CLIError('unrecognized names: {}'.format(' '.join(filter))) - return filtered diff --git a/src/setup.py b/src/setup.py index 91e64d653..ac297cdc8 100644 --- a/src/setup.py +++ b/src/setup.py @@ -45,6 +45,7 @@ def read(fname): 'azdev.operations', 'azdev.operations.linter', 'azdev.operations.tests', + 'azdev.operations.extensions', 'azdev.utilities', ], install_requires=[