Unpin wheel version, update CI check

.github/CODEOWNERS

@@ -1,5 +1,7 @@
 # See for instructions on this file https://help.github.com/articles/about-codeowners/
 
+/linter_exclusions.yml @MyronFanQiu @haroldrandom @fengzhou-msft
+
 /src/index.json @haroldrandom @fengzhou-msft
 
 /src/image-copy/ @tamirkamara
@@ -70,6 +72,10 @@
 
 /src/ip-group/ @haroldrandom
 
+/src/connectedk8s/ @akashkeshari
+
+/src/storagesync/ @jsntcy
+
 /src/maintenance/ @gautamd-ms
 
 /src/ai-examples/ @mirdaki
@@ -91,3 +97,41 @@
 /src/powerbidedicated/ @Juliehzl
 
 /src/blueprint/ @fengzhou-msft
+
+/src/storage-or-preview/ @Juliehzl
+
+/src/logic/ @bquantump
+
+/src/hardware-security-modules/ @bquantump
+
+/src/databox/ @jsntcy
+
+/src/hpc-cache/ @qianwens
+
+/src/timeseriesinsights/ @jiasli
+
+/src/portal/ @YalinLi0312
+
+/src/import-export/ @arrownj
+
+/src/account/ @zikalino
+
+/src/datashare/ @fengzhou-msft
+
+/src/k8sconfiguration/ @NarayanThiru
+
+/src/log-analytics-solution/ @zhoxing-ms
+
+/src/kusto/ @ilayr @orhasban @astauben
+
+/src/ai-did-you-mean-this/ @christopher-o-toole
+
+/src/custom-providers/ @jsntcy
+
+/src/costmanagement/ @haroldrandom
+
+/src/blockchain/ @MyronFanQiu
+
+/src/codespaces/ @derekbekoe
+
+/src/desktopvirtualization/ @changlong-liu

README.md

@@ -33,6 +33,18 @@ Add your extension to the index to make it available in these CLI commands:
 - Your extension artifact (i.e. `.whl`) will not live in this repository. You can publish your extension to PyPI or somewhere else such as Azure Storage.
 - If you want your extension to appear in the index.json, modify the index.
 
+## About extension publishing
+
+There is a pipeline to automatically build, upload and publish extension wheels.
+
+Once your PR is merged into master branch, a new PR will be created to update `src/index.json` automatically.
+
+The precondition is to put your code inside this repo and upgrade the version in the PR but not to modify `src/index.json`.
+
+If you want to host the source code in your dedicated repo, you have to upload the WHL file and update the `src/index.json` manually.
+
+For detail, please visit [Publish section in Azure CLI Extension Authoring](https://github.com/Azure/azure-cli/blob/dev/doc/extensions/authoring.md#publish).
+
 ## FAQ
 
 ### How to generate sha256digest for an index.json entry?
@@ -55,16 +67,19 @@ Note: Hash should be in lowercase in index.json otherwise CI will fail.
 
 ### How to fill in the metadata for an index.json entry?
 
-The `azdev extension update-index` command can be used to simplify the process of updating the index file. Run `azdev extension update-index <URL>` where URL is the fully-qualified URL to your published extension WHL. This will gather the appropriate metadata and add an entry for your extension to the index. For more information visit https://github.com/Azure/azure-cli-dev-tools.
+The `azdev extension update-index` command can be used to simplify the process of updating the index file.  
 
-As a fallback:
+Run `azdev extension update-index <URL>` where URL is the fully-qualified URL to your published extension WHL.  
+This will gather the appropriate metadata and add an entry for your extension to the index. For more information visit https://github.com/Azure/azure-cli-dev-tools.
 
-The metadata needed to be filled is a combination of the contents present in:
-- `metadata.json` located in your unzipped extension artifact (`.whl` file) in the `<package>-<version>.dist-info` directory. This metadata is garnered from the `setup.py` folder.
-- `azext_metadata.json` (if it exists) under your extension.
+Metadata here consist of two parts:
+- The raw metadata of a Python Package which is autogenerated from build tool `wheel` and comes from setup.py.
+- The additional metadata for an extensions which is listed at `azext_metadata.json` of your extension.
 
 Note that CI will fail if this metadata does not match the contents of your published extension.
 
+It's not recommended to modify index.json manually.
+
 # Contributing
 
 This project welcomes contributions and suggestions.  Most contributions require you to agree to a

azure-pipelines.yml

@@ -18,20 +18,20 @@ jobs:
   pool:
     vmImage: "windows-2019"
   steps:
-    - task: ms-codeanalysis.vss-microsoft-security-code-analysis.build-task-credscan.CredScan@2
-      displayName: 'CredScan'
-      inputs:
-        toolVersion: 'Latest'
-        suppressionsFile: './scripts/ci/credscan/CredScanSuppressions.json'
-    - task: ms-codeanalysis.vss-microsoft-security-code-analysis.build-task-postanalysis.PostAnalysis@1
-      displayName: 'Post Analysis'
-      inputs:
-        AllTools: false
-        BinSkim: false
-        CredScan: true
-        RoslynAnalyzers: false
-        TSLint: false
-        ToolLogsNotFoundAction: 'Standard'
+  - task: ms-codeanalysis.vss-microsoft-security-code-analysis-devops.build-task-credscan.CredScan@2
+    displayName: 'Run Credential Scanner'
+    inputs:
+      toolMajorVersion: V2
+      suppressionsFile: './scripts/ci/credscan/CredScanSuppressions.json'
+  - task: ms-codeanalysis.vss-microsoft-security-code-analysis-devops.build-task-postanalysis.PostAnalysis@1
+    displayName: 'Post Analysis'
+    inputs:
+      AllTools: false
+      BinSkim: false
+      CredScan: true
+      RoslynAnalyzers: false
+      TSLint: false
+      ToolLogsNotFoundAction: 'Standard'
 
 - job: CheckLicenseHeader
   displayName: "Check License"
@@ -72,7 +72,7 @@ jobs:
       displayName: 'Use Python 3.6'
       inputs:
         versionSpec: 3.6
-    - bash: pip install wheel==0.30.0 pylint==1.9.5 flake8==3.5.0 requests
+    - bash: pip install wheel pylint==1.9.5 flake8==3.5.0 requests pkginfo
       displayName: 'Install wheel, pylint, flake8, requests'
     - bash: python scripts/ci/source_code_static_analysis.py
       displayName: "Static Analysis"
@@ -89,7 +89,7 @@ jobs:
   - bash: |
       #!/usr/bin/env bash
       set -ev
-      pip install wheel==0.30.0 requests
+      pip install wheel requests pkginfo
       export CI="ADO"
       python ./scripts/ci/test_index.py -v
     displayName: "Verify Extensions Index"
@@ -109,8 +109,8 @@ jobs:
       displayName: 'Use Python $(python.version)'
       inputs:
         versionSpec: '$(python.version)'
-    - bash: pip install wheel==0.30.0
-      displayName: 'Install wheel==0.30.0'
+    - bash: pip install wheel pkginfo
+      displayName: 'Install wheel'
     - bash: ./scripts/ci/test_source.sh
       displayName: 'Run integration test and build test'
       env:
@@ -144,9 +144,10 @@ jobs:
 
         azdev setup -c ../azure-cli -r ./
 
-        az --version
+        # overwrite the default AZURE_EXTENSION_DIR set by ADO
+        AZURE_EXTENSION_DIR=~/.azure/cliextensions az --version
 
-        python scripts/ci/verify_linter.py
+        AZURE_EXTENSION_DIR=~/.azure/cliextensions python scripts/ci/verify_linter.py
       displayName: "CLI Linter on Modified Extension"
       env:
         ADO_PULL_REQUEST_LATEST_COMMIT: $(System.PullRequest.SourceCommitId)
@@ -161,6 +162,8 @@ jobs:
     displayName: 'Use Python 3.7'
     inputs:
       versionSpec: 3.7
+  - bash: pip install wheel==0.30.0
+    displayName: 'Install wheel==0.30.0'
   - task: Bash@3
     displayName: "Verify Extension Ref Docs"
     inputs:

docs/README.md

@@ -12,7 +12,7 @@ Doc Sections
 
 - [Command Guidelines](https://github.com/Azure/azure-cli/blob/dev/doc/command_guidelines.md) - Command Guidelines
 
-- [Publishing](https://github.com/Azure/azure-cli/blob/dev/doc/extensions/publishing.md) - How to publish an extension
+- [Publishing](https://github.com/Azure/azure-cli/blob/dev/doc/extensions/authoring.md#publish) - How to publish an extension
 
 - [Summary Guidelines](https://github.com/Azure/azure-cli/blob/dev/doc/extensions/extension_summary_guidelines.md) - Summary guidelines for extensions to be published
 

linter_exclusions.yml

@@ -0,0 +1,131 @@
+aks create:
+  parameters:
+    node_resource_group:
+      rule_exclusions:
+      - parameter_should_not_end_in_resource_group
+blueprint assignment update:
+  parameters:
+    identity_type:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+databricks workspace create:
+  parameters:
+    managed_resource_group:
+      rule_exclusions:
+      - parameter_should_not_end_in_resource_group
+devops security permission update:
+  parameters:
+    merge:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+devops wiki page update:
+  parameters:
+    comment:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+    encoding:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+dt endpoint create servicebus:
+  parameters:
+    servicebus_resource_group:
+      rule_exclusions:
+      - parameter_should_not_end_in_resource_group
+dt endpoint create eventgrid:
+  parameters:
+    eventgrid_resource_group:
+      rule_exclusions:
+      - parameter_should_not_end_in_resource_group
+dt endpoint create eventhub:
+  parameters:
+    eventhub_resource_group:
+      rule_exclusions:
+      - parameter_should_not_end_in_resource_group
+eventgrid event-subscription update:
+  parameters:
+    endpoint_type:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+eventgrid partner namespace event-channel create:
+  parameters:
+    destination_resource_group:
+      rule_exclusions:
+      - parameter_should_not_end_in_resource_group
+eventgrid partner topic event-subscription update:
+  parameters:
+    endpoint_type:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+eventgrid system-topic event-subscription update:
+  parameters:
+    endpoint_type:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+image copy:
+  parameters:
+    source_resource_group_name:
+      rule_exclusions:
+      - parameter_should_not_end_in_resource_group
+    target_resource_group_name:
+      rule_exclusions:
+      - parameter_should_not_end_in_resource_group
+iot pnp capability-model update:
+  parameters:
+    repo_endpoint:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+iot pnp interface update:
+  parameters:
+    repo_endpoint:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+maintenance configuration update:
+  parameters:
+    maintenanceScope:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+ml endpoint realtime update:
+  parameters:
+    context:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+ml model update:
+  parameters:
+    context:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+ml service update:
+  parameters:
+    context:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+monitor action-rule list:
+  parameters:
+    target_resource_group:
+      rule_exclusions:
+      - parameter_should_not_end_in_resource_group
+network cross-connection peering update:
+  parameters:
+    ip_version:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+network firewall management-ip-config update:
+  parameters:
+    subnet:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+notification-hub credential apns update:
+  parameters:
+    endpoint:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+storage account update:
+  parameters:
+    encryption_key_source:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+storage azcopy run-command:
+  parameters:
+    command_args:
+      rule_exclusions:
+      - no_positional_parameters

pylintrc

@@ -28,7 +28,7 @@ disable=missing-docstring,
 
 [TYPECHECK]
 # For Azure CLI extensions, we ignore some import errors as they'll be available in the environment of the CLI
-ignored-modules=azure,azure.cli,azure.cli.core,azure.cli.core.commands,knack,msrestazure,argcomplete,azure_devtools
+ignored-modules=azure,azure.cli,azure.cli.core,azure.cli.core.commands,knack,msrestazure,argcomplete,azure_devtools,isodate,OpenSSL
 
 [FORMAT]
 max-line-length=120

scripts/automation/build_package.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+
+#-------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+#--------------------------------------------------------------------------
+
+import argparse
+import os
+import glob
+from subprocess import check_call
+
+DEFAULT_DEST_FOLDER = "./dist"
+
+def create_package(name, dest_folder=DEFAULT_DEST_FOLDER):
+    # a package will exist in either one, or the other folder. this is why we can resolve both at the same time.
+    absdirs = [os.path.dirname(package) for package in (glob.glob('{}/setup.py'.format(name)) + glob.glob('sdk/*/{}/setup.py'.format(name)))]
+    absdirpath = os.path.abspath(absdirs[0])
+    check_call(['python', 'setup.py', 'bdist_wheel', '-d', dest_folder], cwd=absdirpath)
+    check_call(['python', 'setup.py', "sdist", "--format", "zip", '-d', dest_folder], cwd=absdirpath)
+
+if __name__ == '__main__':
+    """
+    This file is used for Swagger CLI extension automation to build the wheel file and zip file
+    """
+    parser = argparse.ArgumentParser(description='Build Azure package.')
+    parser.add_argument('name', help='The package name')
+    parser.add_argument('--dest', '-d', default=DEFAULT_DEST_FOLDER,
+                        help='Destination folder. Relative to the package dir. [default: %(default)s]')
+    args = parser.parse_args()
+    create_package(args.name, args.dest)
+
+

scripts/ci/avail-ext-doc/update_extension_list.py

@@ -36,11 +36,18 @@ def get_extensions():
     for _, exts in index_extensions.items():
         # Get latest version
         exts = sorted(exts, key=lambda c: parse_version(c['metadata']['version']), reverse=True)
+
+        try:
+            # compatible with older wheel to build extension
+            home_page = exts[0]['metadata']['extensions']['python.details']['project_urls']['Home']
+        except KeyError:
+            home_page = exts[0]['metadata']['home_page']
+
         extensions.append({
             'name': exts[0]['metadata']['name'],
             'desc': exts[0]['metadata']['summary'],
             'version': exts[0]['metadata']['version'],
-            'project_url': exts[0]['metadata']['extensions']['python.details']['project_urls']['Home'],
+            'project_url': home_page,
             'preview': 'Yes' if exts[0]['metadata'].get('azext.isPreview') else ''
         })
     return extensions