diff --git a/.azure-pipelines/templates/azdev_setup.yml b/.azure-pipelines/templates/azdev_setup.yml index 5dc98590d37..a97d5bd61d8 100644 --- a/.azure-pipelines/templates/azdev_setup.yml +++ b/.azure-pipelines/templates/azdev_setup.yml @@ -21,6 +21,8 @@ steps: else azdev setup -c $CLI_REPO_PATH -r $CLI_EXT_REPO_PATH --debug fi + # This helps detect issues in CI if a used SDK API version is deleted by the below script. + python $CLI_REPO_PATH/build_scripts/windows/scripts/remove_unused_api_versions.py displayName: 'azdev setup' env: CLI_REPO_PATH: ${{ parameters.CLIRepoPath }} diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5de97bfd00d..83afdb9bf9c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -227,15 +227,20 @@ jobs: $InstallArgs += $reinstall_option } Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow - Get-Content .\install_logs.txt + $install_time=Measure-Command {Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow} | select -expand TotalSeconds $installed_version=az version --query '\"azure-cli\"' -o tsv if ($installed_version -ne $to_be_installed_version){ echo "The MSI failed to install." Exit 1 } + echo 'Install time(seconds):' $install_time az --version + # Test bundled pip with extension installation + az extension add -n rdbms-connect az self-test + Get-Content .\install_logs.txt + - job: BuildDockerImage displayName: Build Docker Image diff --git a/build_scripts/windows/scripts/build.cmd b/build_scripts/windows/scripts/build.cmd index c5451e62aa4..ed2fe99d40f 100644 --- a/build_scripts/windows/scripts/build.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -9,10 +9,10 @@ if "%CLI_VERSION%"=="" ( echo Please set the CLI_VERSION environment variable, e.g. 2.0.13 goto ERROR ) -set PYTHON_VERSION=3.6.8 +set PYTHON_VERSION=3.8.9 set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip" -set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python368-32.zip" +set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python389-32.zip" set PROPAGATE_ENV_CHANGE_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/propagate_env_change.zip" :: Set up the output directory and temp. directories @@ -26,7 +26,7 @@ mkdir %ARTIFACTS_DIR% set TEMP_SCRATCH_FOLDER=%ARTIFACTS_DIR%\cli_scratch set BUILDING_DIR=%ARTIFACTS_DIR%\cli set WIX_DIR=%ARTIFACTS_DIR%\wix -set PYTHON_DIR=%ARTIFACTS_DIR%\Python368-32 +set PYTHON_DIR=%ARTIFACTS_DIR%\Python389-32 set PROPAGATE_ENV_CHANGE_DIR=%~dp0..\propagate_env_change set REPO_ROOT=%~dp0..\..\.. @@ -75,15 +75,14 @@ if not exist %PYTHON_DIR% ( mkdir %PYTHON_DIR% pushd %PYTHON_DIR% echo Downloading Python. - curl -o Python368-32.zip %PYTHON_DOWNLOAD_URL% -k - unzip -q Python368-32.zip + curl -o Python389-32.zip %PYTHON_DOWNLOAD_URL% -k + unzip -q Python389-32.zip if %errorlevel% neq 0 goto ERROR - del Python368-32.zip + del Python389-32.zip echo Python downloaded and extracted successfully. popd ) set PYTHON_EXE=%PYTHON_DIR%\python.exe -%PYTHON_EXE% -m pip install --upgrade pip==21.0.1 setuptools==52.0.0 robocopy %PYTHON_DIR% %BUILDING_DIR% /s /NFL /NDL @@ -100,6 +99,7 @@ if %errorlevel% neq 0 goto ERROR pushd %BUILDING_DIR% %BUILDING_DIR%\python.exe %~dp0\patch_models_v2.py +%BUILDING_DIR%\python.exe %~dp0\remove_unused_api_versions.py popd echo Creating the wbin (Windows binaries) folder that will be added to the path... @@ -123,11 +123,6 @@ for /f %%a in ('dir /b /s *_py3.*.pyc') do ( ) popd -:: Remove unused Network SDK API versions -pushd %BUILDING_DIR%\Lib\site-packages\azure\mgmt\network -rmdir /s /q v2016_09_01 v2016_12_01 v2017_03_01 v2017_06_01 v2017_08_01 v2017_09_01 v2017_11_01 v2018_02_01 v2018_04_01 v2018_06_01 v2018_10_01 v2018_12_01 v2019_04_01 v2019_08_01 v2019_09_01 v2019_11_01 v2019_12_01 v2020_03_01 -popd - :: Remove .py and only deploy .pyc files pushd %BUILDING_DIR%\Lib\site-packages for /f %%f in ('dir /b /s *.pyc') do ( diff --git a/build_scripts/windows/scripts/patch_models_v2.py b/build_scripts/windows/scripts/patch_models_v2.py index ce09f604385..bc4b0a562b6 100644 --- a/build_scripts/windows/scripts/patch_models_v2.py +++ b/build_scripts/windows/scripts/patch_models_v2.py @@ -245,7 +245,7 @@ def find_models_to_change(module_name): return [ importlib.import_module('.'+label+'.models', main_module.__name__) for (_, label, ispkg) in pkgutil.iter_modules(main_module.__path__) - if ispkg + if ispkg and label != 'aio' ] diff --git a/build_scripts/windows/scripts/remove_unused_api_versions.py b/build_scripts/windows/scripts/remove_unused_api_versions.py new file mode 100644 index 00000000000..0d07effea80 --- /dev/null +++ b/build_scripts/windows/scripts/remove_unused_api_versions.py @@ -0,0 +1,59 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import logging +import os +import re +import shutil +import azure.mgmt.network + +from azure.cli.core.profiles import AD_HOC_API_VERSIONS, AZURE_API_PROFILES, ResourceType + + +_LOGGER = logging.getLogger(__name__) + + +def remove_unused_network_api_versions(): + # Hard-coded API versions + used_network_api_versions = set(AD_HOC_API_VERSIONS[ResourceType.MGMT_NETWORK].values()) + + # API versions in profile + for _, profile in AZURE_API_PROFILES.items(): + if ResourceType.MGMT_NETWORK in profile: + used_network_api_versions.add(profile[ResourceType.MGMT_NETWORK]) + + # Normalize API version: 2019-02-01 -> v2019_02_01 + used_network_api_vers = {f"v{api.replace('-','_')}" for api in used_network_api_versions} + + # Network SDK has a set of versions imported in models.py. + # Let's keep them before we figure out how to remove a version in all related SDK files. + path = azure.mgmt.network.__path__[0] + model_file = os.path.join(path, 'models.py') + with open(model_file, 'r', encoding='utf-8') as f: + content = f.read() + for m in re.finditer(r'from \.(v[_\d]*)\.models import \*', content): + used_network_api_vers.add(m.group(1)) + + _LOGGER.info('Used network API versions:') + _LOGGER.info(sorted(used_network_api_vers)) + + all_api_vers = {d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d)) and d.startswith('v')} + _LOGGER.info('All network API versions:') + _LOGGER.info(sorted(all_api_vers)) + + remove_api_vers = sorted(all_api_vers - used_network_api_vers) + _LOGGER.info('Network API versions that will be removed:') + _LOGGER.info(remove_api_vers) + + for ver in remove_api_vers: + shutil.rmtree(os.path.join(path, ver)) + + +def main(): + remove_unused_network_api_versions() + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() diff --git a/build_scripts/windows/scripts/setup_msi_test.ps1 b/build_scripts/windows/scripts/setup_msi_test.ps1 index 3f63c137701..5ad7b16f277 100644 --- a/build_scripts/windows/scripts/setup_msi_test.ps1 +++ b/build_scripts/windows/scripts/setup_msi_test.ps1 @@ -16,6 +16,6 @@ $testsdk = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_ & 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $PSScriptRoot\..\..\..\artifacts\build\$($testsdk.Name) $fulltest = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_fulltest*.whl | Select-Object Name -& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $PSScriptRoot\..\..\..\artifacts\build\$($fulltest.Name) +& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install --no-deps $PSScriptRoot\..\..\..\artifacts\build\$($fulltest.Name) & 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' $PSScriptRoot\test_msi_package.py \ No newline at end of file diff --git a/src/azure-cli-core/azure/cli/core/profiles/__init__.py b/src/azure-cli-core/azure/cli/core/profiles/__init__.py index 4652a7a0416..55d94e50820 100644 --- a/src/azure-cli-core/azure/cli/core/profiles/__init__.py +++ b/src/azure-cli-core/azure/cli/core/profiles/__init__.py @@ -5,7 +5,7 @@ # pylint: disable=unused-import from azure.cli.core.profiles._shared import AZURE_API_PROFILES, ResourceType, CustomResourceType, PROFILE_TYPE,\ - SDKProfile + SDKProfile, AD_HOC_API_VERSIONS def get_api_version(cli_ctx, resource_type, as_sdk_profile=False): diff --git a/src/azure-cli-core/azure/cli/core/profiles/_shared.py b/src/azure-cli-core/azure/cli/core/profiles/_shared.py index 0df02c794d9..e1791c2ddd2 100644 --- a/src/azure-cli-core/azure/cli/core/profiles/_shared.py +++ b/src/azure-cli-core/azure/cli/core/profiles/_shared.py @@ -336,6 +336,19 @@ def default_api_version(self): } +# We should avoid using ad hoc API versions, +# use the version in a profile as much as possible. +AD_HOC_API_VERSIONS = { + ResourceType.MGMT_NETWORK: { + 'vm_default_target_network': '2018-01-01', + 'nw_connection_monitor': '2019-06-01', + 'container_network': '2018-08-01', + 'appservice_network': '2020-04-01', + 'appservice_ensure_subnet': '2019-02-01' + } +} + + class _ApiVersions: # pylint: disable=too-few-public-methods def __init__(self, client_type, sdk_profile, post_process): self._client_type = client_type diff --git a/src/azure-cli/azure/cli/command_modules/appservice/access_restrictions.py b/src/azure-cli/azure/cli/command_modules/appservice/access_restrictions.py index ec4b6f2aabe..24e24e393ae 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/access_restrictions.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/access_restrictions.py @@ -20,7 +20,6 @@ logger = get_logger(__name__) -NETWORK_API_VERSION = '2019-02-01' ALLOWED_HTTP_HEADER_NAMES = ['x-forwarded-host', 'x-forwarded-for', 'x-azure-fdid', 'x-fd-healthprobe'] @@ -164,6 +163,7 @@ def _validate_subnet(cli_ctx, subnet, vnet_name, resource_group_name): def _ensure_subnet_service_endpoint(cli_ctx, subnet_id): + from azure.cli.core.profiles import AD_HOC_API_VERSIONS, ResourceType subnet_id_parts = parse_resource_id(subnet_id) subnet_subscription_id = subnet_id_parts['subscription'] subnet_resource_group = subnet_id_parts['resource_group'] @@ -175,7 +175,8 @@ def _ensure_subnet_service_endpoint(cli_ctx, subnet_id): ' Use --ignore-missing-endpoint or -i to' ' skip validation and manually verify service endpoint.') - vnet_client = network_client_factory(cli_ctx, api_version=NETWORK_API_VERSION) + vnet_client = network_client_factory(cli_ctx, api_version=AD_HOC_API_VERSIONS[ResourceType.MGMT_NETWORK] + ['appservice_ensure_subnet']) subnet_obj = vnet_client.subnets.get(subnet_resource_group, subnet_vnet_name, subnet_name) subnet_obj.service_endpoints = subnet_obj.service_endpoints or [] service_endpoint_exists = False diff --git a/src/azure-cli/azure/cli/command_modules/appservice/appservice_environment.py b/src/azure-cli/azure/cli/command_modules/appservice/appservice_environment.py index b1ccb3d2435..effc28e1291 100644 --- a/src/azure-cli/azure/cli/command_modules/appservice/appservice_environment.py +++ b/src/azure-cli/azure/cli/command_modules/appservice/appservice_environment.py @@ -27,7 +27,6 @@ VERSION_2019_08_01 = "2019-08-01" VERSION_2019_10_01 = "2019-10-01" -VERSION_2020_04_01 = "2020-04-01" logger = get_logger(__name__) @@ -190,11 +189,12 @@ def _get_resource_client_factory(cli_ctx, api_version=None): def _get_network_client_factory(cli_ctx, api_version=None): + from azure.cli.core.profiles import AD_HOC_API_VERSIONS, ResourceType client = get_mgmt_service_client(cli_ctx, NetworkManagementClient) if api_version: client.api_version = api_version else: - client.api_version = VERSION_2020_04_01 + client.api_version = AD_HOC_API_VERSIONS[ResourceType.MGMT_NETWORK]['appservice_network'] return client diff --git a/src/azure-cli/azure/cli/command_modules/container/_client_factory.py b/src/azure-cli/azure/cli/command_modules/container/_client_factory.py index 698d007895f..7151b5d378a 100644 --- a/src/azure-cli/azure/cli/command_modules/container/_client_factory.py +++ b/src/azure-cli/azure/cli/command_modules/container/_client_factory.py @@ -54,4 +54,7 @@ def get_auth_management_client(cli_ctx, scope=None, **_): def cf_network(cli_ctx): from azure.mgmt.network import NetworkManagementClient from azure.cli.core.commands.client_factory import get_mgmt_service_client - return get_mgmt_service_client(cli_ctx, NetworkManagementClient, api_version="2018-08-01") + from azure.cli.core.profiles import AD_HOC_API_VERSIONS, ResourceType + return get_mgmt_service_client(cli_ctx, NetworkManagementClient, + api_version=AD_HOC_API_VERSIONS[ResourceType.MGMT_NETWORK] + ['container_network']) diff --git a/src/azure-cli/azure/cli/command_modules/network/custom.py b/src/azure-cli/azure/cli/command_modules/network/custom.py index 0e8526a841f..c491b6dd518 100644 --- a/src/azure-cli/azure/cli/command_modules/network/custom.py +++ b/src/azure-cli/azure/cli/command_modules/network/custom.py @@ -4677,9 +4677,12 @@ def create_nw_connection_monitor(cmd, tags, do_not_start, monitoring_interval) - client = get_mgmt_service_client(cmd.cli_ctx, - ResourceType.MGMT_NETWORK, - api_version='2019-06-01').connection_monitors + from azure.cli.core.profiles._shared import AD_HOC_API_VERSIONS + client = get_mgmt_service_client( + cmd.cli_ctx, + ResourceType.MGMT_NETWORK, + api_version=AD_HOC_API_VERSIONS[ResourceType.MGMT_NETWORK]['nw_connection_monitor'] + ).connection_monitors elif any(v2_required_parameter_set): # V2 creation connection_monitor = _create_nw_connection_monitor_v2(cmd, location, diff --git a/src/azure-cli/azure/cli/command_modules/vm/_vm_utils.py b/src/azure-cli/azure/cli/command_modules/vm/_vm_utils.py index 2444a9dd3a7..2950eb78abf 100644 --- a/src/azure-cli/azure/cli/command_modules/vm/_vm_utils.py +++ b/src/azure-cli/azure/cli/command_modules/vm/_vm_utils.py @@ -28,10 +28,10 @@ def get_target_network_api(cli_ctx): necessarily latest, network API version is order to avoid having to re-record every test that uses VM create (which there are a lot) whenever NRP bumps their API version (which is often)! """ - from azure.cli.core.profiles import get_api_version, ResourceType + from azure.cli.core.profiles import get_api_version, ResourceType, AD_HOC_API_VERSIONS version = get_api_version(cli_ctx, ResourceType.MGMT_NETWORK) if cli_ctx.cloud.profile == 'latest': - version = '2018-01-01' + version = AD_HOC_API_VERSIONS[ResourceType.MGMT_NETWORK]['vm_default_target_network'] return version