Scenario Test Framework additions & Add test for 'az vm list-ip-addresses'

doc/recording_vcr_tests.md

@@ -65,7 +65,34 @@ This method runs a given command and records its output to the display for manua
 
 ####test(command_string, checks)
 
-This method runs a given command and automatically validates the output. The results are saved to `expected_results.res` if valid, but nothing is display on the screen. Valid checks include: `bool`, `str` and `dict`. A check with a `dict` can be used to check for multiple matching parameters (and logic). Child `dict` objects can be used as values to verify properties within nested objects.
+This method runs a given command and automatically validates the output. The results are saved to `expected_results.res` if valid, but nothing is display on the screen. Valid checks include: `bool`, `str`, `dict` and JMESPath checks (see below). A check with a `dict` can be used to check for multiple matching parameters (and logic). Child `dict` objects can be used as values to verify properties within nested objects.
+
+#####JMESPath Comparator
+
+You can use the JMESPathComparator object to validate the result from a command.
+This is useful for checking that the JSON result has fields you were expecting, arrays of certain lengths etc.
+
+######Usage
+```
+JMESPathComparator(query, expected_result)
+```
+- `query` - JMESPath query as a string.
+- `expected_result` - The expected result from the JMESPath query (see [jmespath.search()](https://github.com/jmespath/jmespath.py#api))
+
+######Example
+
+The example below shows how you can use a JMESPath query to validate the values from a command.
+When calling `test(command_string, checks)` you can pass in just one JMESPathComparator or a list of JMESPathComparators.
+
+```
+from azure.cli.utils.command_test_script import JMESPathComparator
+
+self.test('vm list-ip-addresses --resource-group myResourceGroup',
+    [
+        JMESPathComparator('length(@)', 1),
+        JMESPathComparator('[0].virtualMachine.name', 'myVMName')])
+```
+
 
 ####set_env(variable_name, value)
 

src/azure/cli/_logging.py

@@ -136,6 +136,9 @@ def configure_logging(argv):
     az_logger.setLevel(logging.DEBUG)
     az_logger.propagate = False
 
+    if len(root_logger.handlers) and len(az_logger.handlers):
+        # loggers already configured
+        return
     _init_console_handlers(root_logger, az_logger, log_level_config)
     _init_logfile_handlers(root_logger, az_logger)
 

src/azure/cli/commands/__init__.py

@@ -71,12 +71,15 @@ def __init__(self, start_msg='', finish_msg='', poll_interval_ms=1000.0):
         self.finish_msg = finish_msg
         self.poll_interval_ms = poll_interval_ms
 
+    def _delay(self):
+        time.sleep(self.poll_interval_ms / 1000.0)
+
     def __call__(self, poller):
         logger.warning(self.start_msg)
         logger.info("Starting long running operation '%s' with polling interval %s ms",
                     self.start_msg, self.poll_interval_ms)
         while not poller.done():
-            time.sleep(self.poll_interval_ms / 1000.0)
+            self._delay()
             logger.info("Long running operation '%s' polling now", self.start_msg)
         try:
             result = poller.result()

src/azure/cli/utils/command_test_script.py

@@ -3,12 +3,37 @@
 import json
 import os
 import traceback
-
+import collections
+import jmespath
 from six import StringIO
 
 from azure.cli.main import main as cli
 from azure.cli.parser import IncorrectUsageError
 
+class JMESPathComparatorAssertionError(AssertionError):
+
+    def __init__(self, comparator, actual_result, json_data):
+        message = "Actual value '{}' != Expected value '{}'. ".format(
+            actual_result,
+            comparator.expected_result)
+        message += "Query '{}' used on json data '{}'".format(comparator.query, json_data)
+        super(JMESPathComparatorAssertionError, self).__init__(message)
+
+class JMESPathComparator(object): #pylint: disable=too-few-public-methods
+
+    def __init__(self, query, expected_result):
+        self.query = query
+        self.expected_result = expected_result
+
+    def compare(self, json_data):
+        json_val = json.loads(json_data)
+        actual_result = jmespath.search(
+            self.query,
+            json_val,
+            jmespath.Options(collections.OrderedDict))
+        if not actual_result == self.expected_result:
+            raise JMESPathComparatorAssertionError(self, actual_result, json_data)
+
 class CommandTestScript(object): #pylint: disable=too-many-instance-attributes
 
     def __init__(self, set_up, test_body, tear_down):
@@ -48,7 +73,8 @@ def rec(self, command):
         turns off the flag that signals the test is fully automatic. '''
         self.auto = False
         output = StringIO()
-        cli(command.split(), file=output)
+        command_list = command if isinstance(command, list) else command.split()
+        cli(command_list, file=output)
         result = output.getvalue().strip()
         self._display.write('\n\n== {} ==\n\n{}'.format(command, result))
         self._raw.write(result)
@@ -59,7 +85,8 @@ def run(self, command): #pylint: disable=no-self-use
         ''' Run a command without recording the output as part of expected results. Useful if you
         need to run a command for branching logic or just to reset back to a known condition. '''
         output = StringIO()
-        cli(command.split(), file=output)
+        command_list = command if isinstance(command, list) else command.split()
+        cli(command_list, file=output)
         result = output.getvalue().strip()
         output.close()
         return result
@@ -76,11 +103,18 @@ def _check_json(source, checks):
                     assert source[check] == checks[check]
 
         output = StringIO()
-        command += ' -o json'
-        cli(command.split(), file=output)
+        command_list = command if isinstance(command, list) else command.split()
+        command_list += ['-o', 'json']
+        cli(command_list, file=output)
         result = output.getvalue().strip()
         self._raw.write(result)
-        if isinstance(checks, bool):
+        if isinstance(checks, list) and all(
+                isinstance(comparator, JMESPathComparator) for comparator in checks):
+            for comparator in checks:
+                comparator.compare(result)
+        elif isinstance(checks, JMESPathComparator):
+            checks.compare(result)
+        elif isinstance(checks, bool):
             result_val = str(result).lower().replace('"', '')
             bool_val = result_val in ('yes', 'true', 't', '1')
             assert bool_val == checks

src/azure/cli/utils/command_test_util.py

@@ -66,6 +66,10 @@ def load_subscriptions_mock(self): #pylint: disable=unused-argument
             def get_user_access_token_mock(_, _1, _2): #pylint: disable=unused-argument
                 return 'top-secret-token-for-you'
 
+            def operation_delay_mock(_):
+                # don't run time.sleep()
+                return
+
             def _get_expected_results_from_file(recording_dir):
                 expected_results_path = os.path.join(recording_dir, 'expected_results.res')
                 try:
@@ -126,6 +130,8 @@ def _test_impl(self, test_name, expected, recording_dir):
                         _remove_expected_result(test_name, recording_dir)
 
                 io.close()
+                if fail:
+                    self.fail(display_result)
                 self.assertEqual(actual_result, expected)
 
             expected_results = _get_expected_results_from_file(recording_dir)
@@ -149,6 +155,10 @@ def _test_impl(self, test_name, expected, recording_dir):
                             load_subscriptions_mock)
                 @mock.patch('azure.cli._profile.CredsCache.retrieve_token_for_user',
                             get_user_access_token_mock)
+                @mock.patch('msrestazure.azure_operation.AzureOperationPoller._delay',
+                            operation_delay_mock)
+                @mock.patch('azure.cli.commands.LongRunningOperation._delay',
+                            operation_delay_mock)
                 @self.my_vcr.use_cassette(cassette_path,
                                           filter_headers=CommandTestGenerator.FILTER_HEADERS)
                 def test(self):

src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/tests/test_commands.py

@@ -9,44 +9,10 @@ class TestCommands(unittest.TestCase):
 
 recording_dir = os.path.join(os.path.dirname(__file__), 'recordings')
 
-def _truncate_long_running_operation(data, lro_item):
-    interactions = data['interactions']
-    lro_index = interactions.index(lro_item)
-    for item in interactions[(lro_index+1):]:
-        method = item['request'].get('method')
-        code = item['response']['status'].get('code')
-        if method == 'GET' and code == 202:
-            interactions.remove(item)
-        elif method == 'GET' and code != 202:
-            lro_item['response'] = item['response']
-            interactions.remove(item)
-            return
-
-def _shorten_long_running_operations(test_name):
-    ''' Speeds up playback of tests that originally required HTTP polling by replacing the initial
-    request with the eventual response. '''
-    yaml_path = os.path.join(recording_dir, '{}.yaml'.format(test_name))
-    if not os.path.isfile(yaml_path):
-        return
-    with open(yaml_path, 'r+b') as f:
-        data = yaml.load(f)
-        for item in data['interactions']:
-            method = item['request'].get('method')
-            code = item['response']['status'].get('code')
-            # breaking a lease produces this pattern but should NOT be modified
-            lease_action = item['request']['headers'].get('x-ms-lease-action')
-            lease_action = lease_action[0] if lease_action else None
-            if method == 'PUT' and code == 202 and lease_action != 'break':
-                _truncate_long_running_operation(data, item)
-        f.seek(0)
-        f.write(yaml.dump(data).encode('utf-8'))
-        f.truncate()
-
 generator = CommandTestGenerator(recording_dir, TEST_DEF, ENV_VAR)
 tests = generator.generate_tests()
 
 for test_name in tests:
-    _shorten_long_running_operations(test_name)
     setattr(TestCommands, test_name, tests[test_name])
 
 if __name__ == '__main__':

src/command_modules/azure-cli-vm/azure/cli/command_modules/vm/custom.py

@@ -1,10 +1,7 @@
 import json
 import re
 
-try:
-    from urllib.request import urlopen
-except ImportError:
-    from urllib import urlopen # pylint: disable=no-name-in-module
+from six.moves.urllib.request import urlopen #pylint: disable=import-error
 
 from azure.mgmt.compute.models import DataDisk
 from azure.mgmt.compute.models.compute_management_client_enums import DiskCreateOptionTypes
@@ -241,10 +238,10 @@ def list_vm_images(self,
 
 
     def list_ip_addresses(self,
-                          optional_resource_group_name=None,
+                          resource_group_name=None,
                           vm_name=None):
         ''' Get IP addresses from one or more Virtual Machines
-        :param str optional_resource_group_name:Name of resource group.
+        :param str resource_group_name:Name of resource group.
         :param str vm_name:Name of virtual machine.
         '''
         from azure.mgmt.network import NetworkManagementClient, NetworkManagementClientConfiguration
@@ -268,7 +265,7 @@ def list_ip_addresses(self,
 
             # If provided, make sure that resource group name and vm name match the NIC we are
             # looking at before adding it to the result...
-            if (optional_resource_group_name in (None, nic_resource_group)
+            if (resource_group_name in (None, nic_resource_group)
                     and vm_name in (None, nic_vm_name)):
 
                 network_info = {

src/command_modules/azure-cli-vm/azure/cli/command_modules/vm/tests/command_specs.py

@@ -1,5 +1,5 @@
 # AZURE CLI VM TEST DEFINITIONS
-from azure.cli.utils.command_test_script import CommandTestScript
+from azure.cli.utils.command_test_script import CommandTestScript, JMESPathComparator
 
 #pylint: disable=method-hidden
 class VMImageListByAliasesScenarioTest(CommandTestScript):
@@ -22,6 +22,52 @@ def test_body(self):
     def __init__(self):
         super(VMImageListThruServiceScenarioTest, self).__init__(None, self.test_body, None)
 
+class VMListIPAddressesScenarioTest(CommandTestScript):
+
+    def __init__(self):
+        self.deployment_name = 'azurecli-test-deployment-vm-list-ips'
+        self.resource_group = 'cliTestRg_VmListIpAddresses'
+        self.location = 'westus'
+        self.vm_name = 'vm-with-public-ip'
+        self.ip_allocation_method = 'Dynamic'
+        super(VMListIPAddressesScenarioTest, self).__init__(
+            self.set_up,
+            self.test_body,
+            self.tear_down)
+
+    def set_up(self):
+        self.run('resource group create --location {} --name {}'.format(
+            self.location,
+            self.resource_group))
+
+    def test_body(self):
+        # Expecting no results at the beginning
+        self.test('vm list-ip-addresses --resource-group {}'.format(self.resource_group), None)
+        self.run(['vm', 'create', '-g', self.resource_group, '-l', self.location,
+                  '-n', self.vm_name, '--admin-username', 'ubuntu',
+                  '--image', 'Canonical:UbuntuServer:14.04.4-LTS:latest',
+                  '--admin-password', 'testPassword0', '--deployment-name', self.deployment_name,
+                  '--public-ip-address-allocation', self.ip_allocation_method,
+                  '--public-ip-address-type', 'new'])
+        # Expecting the one we just added
+        self.test('vm list-ip-addresses --resource-group {}'.format(self.resource_group),
+                  [
+                      JMESPathComparator('length(@)', 1),
+                      JMESPathComparator('[0].virtualMachine.name', self.vm_name),
+                      JMESPathComparator('[0].virtualMachine.resourceGroup', self.resource_group),
+                      JMESPathComparator(
+                          'length([0].virtualMachine.network.publicIpAddresses)',
+                          1),
+                      JMESPathComparator(
+                          '[0].virtualMachine.network.publicIpAddresses[0].ipAllocationMethod',
+                          self.ip_allocation_method),
+                      JMESPathComparator(
+                          'type([0].virtualMachine.network.publicIpAddresses[0].ipAddress)',
+                          'string')]
+                 )
+
+    def tear_down(self):
+        self.run('resource group delete --name {}'.format(self.resource_group))
 
 ENV_VAR = {}
 
@@ -34,6 +80,10 @@ def __init__(self):
         'test_name': 'vm_list_from_group',
         'command': 'vm list --resource-group XPLATTESTGEXTENSION9085',
     },
+    {
+        'test_name': 'vm_list_ip_addresses',
+        'command': VMListIPAddressesScenarioTest()
+    },
     {
         'test_name': 'vm_images_list_by_aliases',
         'command': VMImageListByAliasesScenarioTest()

src/command_modules/azure-cli-vm/azure/cli/command_modules/vm/tests/recordings/expected_results.res

@@ -2,5 +2,6 @@
     "test_vm_images_list_by_aliases": "",
     "test_vm_images_list_thru_services": "",
     "test_vm_list_from_group": "Availability Set    : None\nId                  : /subscriptions/0b1f6471-1bf0-4dda-aec3-cb9272f09590/resourceGroups/XPLATTESTGEXTENSION9085/providers/Microsoft.Compute/virtualMachines/xplatvmExt1314\nInstance View       : None\nLicense Type        : None\nLocation            : southeastasia\nName                : xplatvmExt1314\nPlan                : None\nProvisioning State  : Succeeded\nResource Group      : XPLATTESTGEXTENSION9085\nResources           : None\nType                : Microsoft.Compute/virtualMachines\nDiagnostics Profile :\n   Boot Diagnostics :\n      Enabled     : True\n      Storage Uri : https://xplatstoragext4633.blob.core.windows.net/\nHardware Profile    :\n   Vm Size : Standard_A1\nNetwork Profile     :\n   Network Interfaces :\n      Id             : /subscriptions/0b1f6471-1bf0-4dda-aec3-cb9272f09590/resourceGroups/xplatTestGExtension9085/providers/Microsoft.Network/networkInterfaces/xplatnicExt4843\n      Primary        : None\n      Resource Group : xplatTestGExtension9085\nOs Profile          :\n   Admin Password        : None\n   Admin Username        : azureuser\n   Computer Name         : xplatvmExt1314\n   Custom Data           : None\n   Linux Configuration   : None\n   Secrets               :\n      None\n   Windows Configuration :\n      Additional Unattend Content : None\n      Enable Automatic Updates    : True\n      Provision Vm Agent          : True\n      Time Zone                   : None\n      Win Rm                      : None\nStorage Profile     :\n   Data Disks      :\n      None\n   Image Reference :\n      Offer     : WindowsServerEssentials\n      Publisher : MicrosoftWindowsServerEssentials\n      Sku       : WindowsServerEssentials\n      Version   : 1.0.20131018\n   Os Disk         :\n      Caching             : ReadWrite\n      Create Option       : fromImage\n      Disk Size Gb        : None\n      Encryption Settings : None\n      Image               : None\n      Name                : cli1eaed78b36def353-os-1453419539945\n      Os Type             : Windows\n      Vhd                 :\n         Uri : https://xplatstoragext4633.blob.core.windows.net/xplatstoragecntext1789/cli1eaed78b36def353-os-1453419539945.vhd\nTags                :\n   None\n\n\n",
+    "test_vm_list_ip_addresses": "[\n  {\n    \"virtualMachine\": {\n      \"name\": \"vm-with-public-ip\",\n      \"network\": {\n        \"privateIpAddresses\": [\n          \"10.0.0.4\"\n        ],\n        \"publicIpAddresses\": [\n          {\n            \"id\": \"/subscriptions/0b1f6471-1bf0-4dda-aec3-cb9272f09590/resourceGroups/cliTestRg_VmListIpAddresses/providers/Microsoft.Network/publicIPAddresses/PublicIPvm-with-public-ip\",\n            \"ipAddress\": \"13.93.155.206\",\n            \"ipAllocationMethod\": \"Dynamic\",\n            \"name\": \"PublicIPvm-with-public-ip\",\n            \"resourceGroup\": \"cliTestRg_VmListIpAddresses\"\n          }\n        ]\n      },\n      \"resourceGroup\": \"cliTestRg_VmListIpAddresses\"\n    }\n  }\n]",
     "test_vm_usage_list_westus": "[\n  {\n    \"currentValue\": 0,\n    \"limit\": 2000,\n    \"name\": {\n      \"localizedValue\": \"Availability Sets\",\n      \"value\": \"availabilitySets\"\n    },\n    \"unit\": \"Count\"\n  },\n  {\n    \"currentValue\": 7,\n    \"limit\": 100,\n    \"name\": {\n      \"localizedValue\": \"Total Regional Cores\",\n      \"value\": \"cores\"\n    },\n    \"unit\": \"Count\"\n  },\n  {\n    \"currentValue\": 5,\n    \"limit\": 10000,\n    \"name\": {\n      \"localizedValue\": \"Virtual Machines\",\n      \"value\": \"virtualMachines\"\n    },\n    \"unit\": \"Count\"\n  },\n  {\n    \"currentValue\": 0,\n    \"limit\": 50,\n    \"name\": {\n      \"localizedValue\": \"Virtual Machine Scale Sets\",\n      \"value\": \"virtualMachineScaleSets\"\n    },\n    \"unit\": \"Count\"\n  },\n  {\n    \"currentValue\": 1,\n    \"limit\": 100,\n    \"name\": {\n      \"localizedValue\": \"Standard D Family Cores\",\n      \"value\": \"standardDFamily\"\n    },\n    \"unit\": \"Count\"\n  },\n  {\n    \"currentValue\": 6,\n    \"limit\": 100,\n    \"name\": {\n      \"localizedValue\": \"Standard A0-A7 Family Cores\",\n      \"value\": \"standardA0_A7Family\"\n    },\n    \"unit\": \"Count\"\n  }\n]\n"
 }

src/command_modules/azure-cli-vm/azure/cli/command_modules/vm/tests/recordings/test_vm_images_list_by_aliases.yaml

@@ -52,20 +52,20 @@ interactions:
       Content-Length: ['2297']
       Content-Security-Policy: [default-src 'none'; style-src 'unsafe-inline']
       Content-Type: [text/plain; charset=utf-8]
-      Date: ['Tue, 03 May 2016 03:40:49 GMT']
+      Date: ['Fri, 06 May 2016 23:13:25 GMT']
       ETag: ['"db78eb36618a060181b32ac2de91b1733f382e01"']
-      Expires: ['Tue, 03 May 2016 03:45:49 GMT']
+      Expires: ['Fri, 06 May 2016 23:18:25 GMT']
       Source-Age: ['0']
       Strict-Transport-Security: [max-age=31536000]
       Vary: ['Authorization,Accept-Encoding']
       Via: [1.1 varnish]
       X-Cache: [MISS]
       X-Cache-Hits: ['0']
       X-Content-Type-Options: [nosniff]
-      X-Fastly-Request-ID: [13192701239be1866c66d0f0c6155cadff9d7f9d]
+      X-Fastly-Request-ID: [b253eff940ef2df7215e167a5d80acb38ecfb287]
       X-Frame-Options: [deny]
-      X-GitHub-Request-Id: ['17EB2F14:6091:7F1D34B:57281DC1']
-      X-Served-By: [cache-sjc3122-SJC]
+      X-GitHub-Request-Id: ['17EB2F14:3F24:391C37F:572D2515']
+      X-Served-By: [cache-sjc3124-SJC]
       X-XSS-Protection: [1; mode=block]
     status: {code: 200, message: OK}
 version: 1