Unpin wheel version, update CI check

README.md

@@ -55,16 +55,19 @@ Note: Hash should be in lowercase in index.json otherwise CI will fail.
 
 ### How to fill in the metadata for an index.json entry?
 
-The `azdev extension update-index` command can be used to simplify the process of updating the index file. Run `azdev extension update-index <URL>` where URL is the fully-qualified URL to your published extension WHL. This will gather the appropriate metadata and add an entry for your extension to the index. For more information visit https://github.com/Azure/azure-cli-dev-tools.
+The `azdev extension update-index` command can be used to simplify the process of updating the index file.  
 
-As a fallback:
+Run `azdev extension update-index <URL>` where URL is the fully-qualified URL to your published extension WHL.  
+This will gather the appropriate metadata and add an entry for your extension to the index. For more information visit https://github.com/Azure/azure-cli-dev-tools.
 
-The metadata needed to be filled is a combination of the contents present in:
-- `metadata.json` located in your unzipped extension artifact (`.whl` file) in the `<package>-<version>.dist-info` directory. This metadata is garnered from the `setup.py` folder.
-- `azext_metadata.json` (if it exists) under your extension.
+Metadata here consist of two parts:
+- The raw metadata of a Python Package which is autogenerated from build tool `wheel` and comes from setup.py.
+- The additional metadata for an extensions which is listed at `azext_metadata.json` of your extension.
 
 Note that CI will fail if this metadata does not match the contents of your published extension.
 
+It's not recommended to modify index.json manually.
+
 # Contributing
 
 This project welcomes contributions and suggestions.  Most contributions require you to agree to a

azure-pipelines.yml

@@ -72,7 +72,7 @@ jobs:
       displayName: 'Use Python 3.6'
       inputs:
         versionSpec: 3.6
-    - bash: pip install wheel==0.30.0 pylint==1.9.5 flake8==3.5.0 requests
+    - bash: pip install wheel pylint==1.9.5 flake8==3.5.0 requests pkginfo
       displayName: 'Install wheel, pylint, flake8, requests'
     - bash: python scripts/ci/source_code_static_analysis.py
       displayName: "Static Analysis"
@@ -89,7 +89,7 @@ jobs:
   - bash: |
       #!/usr/bin/env bash
       set -ev
-      pip install wheel==0.30.0 requests
+      pip install wheel requests pkginfo
       export CI="ADO"
       python ./scripts/ci/test_index.py -v
     displayName: "Verify Extensions Index"
@@ -109,8 +109,8 @@ jobs:
       displayName: 'Use Python $(python.version)'
       inputs:
         versionSpec: '$(python.version)'
-    - bash: pip install wheel==0.30.0
-      displayName: 'Install wheel==0.30.0'
+    - bash: pip install wheel pkginfo
+      displayName: 'Install wheel'
     - bash: ./scripts/ci/test_source.sh
       displayName: 'Run integration test and build test'
       env:

scripts/ci/avail-ext-doc/update_extension_list.py

@@ -36,11 +36,18 @@ def get_extensions():
     for _, exts in index_extensions.items():
         # Get latest version
         exts = sorted(exts, key=lambda c: parse_version(c['metadata']['version']), reverse=True)
+
+        try:
+            # compatible with older wheel to build extension
+            home_page = exts[0]['metadata']['extensions']['python.details']['project_urls']['Home']
+        except KeyError:
+            home_page = exts[0]['metadata']['home_page']
+
         extensions.append({
             'name': exts[0]['metadata']['name'],
             'desc': exts[0]['metadata']['summary'],
             'version': exts[0]['metadata']['version'],
-            'project_url': exts[0]['metadata']['extensions']['python.details']['project_urls']['Home'],
+            'project_url': home_page,
             'preview': 'Yes' if exts[0]['metadata'].get('azext.isPreview') else ''
         })
     return extensions

scripts/ci/test_index.py

@@ -10,15 +10,21 @@
 from __future__ import print_function
 
 import os
-import json
 import tempfile
 import unittest
 import hashlib
 import shutil
-from wheel.install import WHEEL_INFO_RE
+import re
+
 from util import get_ext_metadata, get_whl_from_url, get_index_data, verify_dependency
 
 
+WHEEL_INFO_RE = re.compile(
+    r"""^(?P<namever>(?P<name>.+?)-(?P<ver>\d.*?))(-(?P<build>\d.*?))?
+     -(?P<pyver>[a-z].+?)-(?P<abi>.+?)-(?P<plat>.+?)(\.whl|\.dist-info)$""",
+    re.VERBOSE).match
+
+
 def get_sha256sum(a_file):
     sha256 = hashlib.sha256()
     with open(a_file, 'rb') as f:
@@ -119,29 +125,40 @@ def test_metadata(self):
         for ext_name, exts in self.index['extensions'].items():
             for item in exts:
                 ext_dir = tempfile.mkdtemp(dir=extensions_dir)
-                ext_file = get_whl_from_url(item['downloadUrl'], item['filename'],
-                                            self.whl_cache_dir, self.whl_cache)
-                metadata = get_ext_metadata(ext_dir, ext_file, ext_name)
-                # Due to https://github.com/pypa/wheel/issues/195 we prevent whls built with 0.31.0 or greater.
-                # 0.29.0, 0.30.0 are the two previous versions before that release.
-                supported_generators = ['bdist_wheel (0.29.0)', 'bdist_wheel (0.30.0)']
-                self.assertIn(metadata.get('generator'), supported_generators,
-                              "{}: 'generator' should be one of {}. "
-                              "Build the extension with a different version of the 'wheel' package "
-                              "(e.g. `pip install wheel==0.30.0`). "
-                              "This is due to https://github.com/pypa/wheel/issues/195".format(ext_name,
-                                                                                               supported_generators))
-                self.assertDictEqual(metadata, item['metadata'],
-                                     "Metadata for {} in index doesn't match the expected of: \n"
-                                     "{}".format(item['filename'], json.dumps(metadata, indent=2, sort_keys=True,
-                                                                              separators=(',', ': '))))
-                run_requires = metadata.get('run_requires')
-                if run_requires:
-                    deps = run_requires[0]['requires']
-                    self.assertTrue(
-                        all(verify_dependency(dep) for dep in deps),
-                        "Dependencies of {} use disallowed extension dependencies. "
-                        "Remove these dependencies: {}".format(item['filename'], deps))
+                ext_file = get_whl_from_url(item['downloadUrl'], item['filename'], self.whl_cache_dir, self.whl_cache)
+
+                wheel_metadata = get_ext_metadata(ext_dir, ext_file)
+                index_metadata = item['metadata']
+
+                print('-' * 40, ext_name, '-' * 40)
+
+                # compatible way compare metadata between wheel file and index.json
+                self.assertEqual(index_metadata['name'], wheel_metadata['name'])
+                self.assertEqual(index_metadata['version'], wheel_metadata['version'])
+                self.assertEqual(index_metadata['license'], wheel_metadata['license'])
+                self.assertEqual(index_metadata['summary'], wheel_metadata['summary'])
+
+                if index_metadata.get('classifiers') and wheel_metadata.get('classifiers'):
+                    self.assertEqual(index_metadata['classifiers'], wheel_metadata['classifiers'])
+
+                if index_metadata.get('run_requires') and wheel_metadata.get('requires_dist'):
+                    self.assertEqual(sorted(index_metadata['run_requires'][0]['requires']),
+                                     sorted(wheel_metadata['requires_dist']))
+                if index_metadata.get('requires_dist') and wheel_metadata.get('requires_dist'):
+                    self.assertEqual(sorted(index_metadata['requires_dist']),
+                                     sorted(wheel_metadata['requires_dist']))
+
+                if index_metadata.get('extensions') is not None:
+                    python_details = index_metadata['extensions']['python.details']
+                    self.assertEqual(python_details['project_urls']['Home'], wheel_metadata['home_page'])
+                    self.assertEqual(python_details['contacts'][0]['name'], wheel_metadata['author'])
+                    self.assertEqual(python_details['contacts'][0]['email'], wheel_metadata['author_email'])
+                else:
+                    self.assertEqual(index_metadata.get('author'), wheel_metadata.get('author'))
+                    self.assertEqual(index_metadata.get('home_page'), wheel_metadata.get('home_page'))
+                    self.assertEqual(index_metadata.get('platforms'), wheel_metadata.get('platforms'))
+                    self.assertEqual(index_metadata.get('author_email'), wheel_metadata.get('author_email'))
+
         shutil.rmtree(extensions_dir)
 
 

scripts/ci/test_source.py

@@ -16,7 +16,6 @@
 from subprocess import check_output, check_call, CalledProcessError
 
 import mock
-from wheel.install import WHEEL_INFO_RE
 from six import with_metaclass
 
 from util import get_ext_metadata, verify_dependency, SRC_PATH
@@ -106,14 +105,14 @@ def test_source_wheels(self):
         for filename in os.listdir(built_whl_dir):
             ext_file = os.path.join(built_whl_dir, filename)
             ext_dir = tempfile.mkdtemp(dir=built_whl_dir)
-            ext_name = WHEEL_INFO_RE(filename).groupdict().get('name')
-            metadata = get_ext_metadata(ext_dir, ext_file, ext_name)
-            run_requires = metadata.get('run_requires')
-            if run_requires:
-                deps = run_requires[0]['requires']
-                self.assertTrue(all(verify_dependency(dep) for dep in deps),
+            metadata = get_ext_metadata(ext_dir, ext_file)
+
+            requires_dist = metadata.get('require_dist')
+            if requires_dist:
+                self.assertTrue(all(verify_dependency(dep) for dep in requires_dist),
                                 "Dependencies of {} use disallowed extension dependencies. "
-                                "Remove these dependencies: {}".format(filename, deps))
+                                "Remove these dependencies: {}".format(filename, requires_dist))
+
         shutil.rmtree(built_whl_dir)
 
 

scripts/ci/update_index.py

@@ -56,7 +56,7 @@ def main():
     entry[0]['downloadUrl'] = whl_path
     entry[0]['sha256Digest'] = get_sha256sum(ext_file)
     entry[0]['filename'] = whl_path.split('/')[-1]
-    entry[0]['metadata'] = get_ext_metadata(ext_dir, ext_file, extension_name)
+    entry[0]['metadata'] = get_ext_metadata(ext_dir, ext_file)
 
     # update index and write back to file
     curr_index['extensions'][extension_name] = entry

scripts/ci/util.py

@@ -6,7 +6,6 @@
 import os
 import json
 import zipfile
-from wheel.install import WHEEL_INFO_RE
 
 # Dependencies that will not be checked.
 # This is for packages starting with 'azure-' but do not use the 'azure' namespace.
@@ -43,24 +42,28 @@ def _get_azext_metadata(ext_dir):
     return azext_metadata
 
 
-def get_ext_metadata(ext_dir, ext_file, ext_name):
-    # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L89
-    WHL_METADATA_FILENAME = 'metadata.json'
+def get_ext_metadata(ext_dir, ext_file):
+    from pkginfo import Wheel
+
     zip_ref = zipfile.ZipFile(ext_file, 'r')
     zip_ref.extractall(ext_dir)
     zip_ref.close()
     metadata = {}
-    dist_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.dist-info')]
     azext_metadata = _get_azext_metadata(ext_dir)
     if azext_metadata:
         metadata.update(azext_metadata)
-    for dist_info_dirname in dist_info_dirs:
-        parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname)
-        if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == ext_name.replace('-', '_'):
-            whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME)
-            if os.path.isfile(whl_metadata_filepath):
-                with open(whl_metadata_filepath) as f:
-                    metadata.update(json.load(f))
+
+    try:
+        ext_wheel = Wheel(ext_file)
+
+        t = vars(ext_wheel)
+        del t['filename']
+        del t['description']    # del as description is trivial
+
+        metadata.update(t)
+    except ValueError:
+        raise '{} is not a valid wheel'.format(ext_file)
+
     return metadata
 
 

src/index.json

@@ -941,7 +941,7 @@
                     },
                     "extras": [],
                     "generator": "bdist_wheel (0.30.0)",
-                    "license": "Proprietary https://aka.ms/azureml-preview-sdk-license ",
+                    "license": "Proprietary https://aka.ms/azureml-preview-sdk-license",
                     "metadata_version": "2.0",
                     "name": "azure-cli-ml",
                     "requires_python": ">=3.5,<4",