[rdbms] az postgres flexible-server migration: Add customer facing feature to migrate postgres db servers from Sterling to Meru platform (#18161)

* Support for "az postgres flexible-server migration" commands

* Fix a linter error caused by copy-paste.

* Fix more linter errors

* Use custom_show_command instead of custom_command for 'show' to fix a linter finding.

* 1. Remove the --subscription-id parameter. Use the CLI command param --subscription instead

* Rename body to properties, which makes more sense in this context.

* Fix isses reported by "azdev style rdbms"

* Fix linter issues

* Fix issues reported by azdev style check

* Mark the migration command group as experimental

* Cannot have is_preview and is_experimental at the same time. So removed is_preview.

* 1. Added a confirmation for migration delete, 2. Deleted unused code, 3. Return instead of just the migration_id from update

* Replace parameters db1, db2, ...db8 with an db-names parameter which takes a space-separated list of values

Author: Rajesh1Ganesan

src/azure-cli/azure/cli/command_modules/rdbms/_helptext_pg.py

@@ -176,6 +176,59 @@
     text: az postgres flexible-server list --resource-group testGroup
 """
 
+helps['postgres flexible-server migration'] = """
+type: group
+short-summary: Manage migration workflows for PostgreSQL Flexible Servers.
+"""
+
+helps['postgres flexible-server migration create'] = """
+type: command
+short-summary: Create a new migration workflow for a flexible server.
+examples:
+  - name: Start a migration workflow on the target server identified by the parameters. The configurations of the migration should be specified in the migrationConfig.json file.
+    text: az postgres flexible-server migration create --subscription xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --properties @"migrationConfig.json"
+"""
+
+helps['postgres flexible-server migration list'] = """
+type: command
+short-summary: List the migrations of a flexible server.
+examples:
+  - name: List the currently active migrations of a target flexible server.
+    text: az postgres flexible-server migration list --subscription xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --filter Active
+  - name: List all (Active/Completed) migrations of a target flexible server.
+    text: az postgres flexible-server migration list --subscription xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --filter All
+"""
+
+helps['postgres flexible-server migration show'] = """
+type: command
+short-summary: Get the details of a specific migration.
+examples:
+  - name: Get the details of a specific migration of a target flexible server.
+    text: az postgres flexible-server migration show --subscription xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --migration-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+"""
+
+helps['postgres flexible-server migration update'] = """
+type: command
+short-summary: Update a specific migration.
+examples:
+  - name: Allow the migration workflow to setup logical replication on the source. Note that this command will restart the source server.
+    text: az postgres flexible-server migration update --subscription xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --migration-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --setup-replication
+  - name: Space-separated list of DBs to migrate. A minimum of 1 and a maximum of 8 DBs can be specified. You can migrate more DBs concurrently using additional migrations. Note that each additional DB affects the performance of the source server.
+    text: az postgres flexible-server migration update --subscription xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --migration-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --db-names db1 db2
+  - name: Allow the migration workflow to overwrite the DB on the target.
+    text: az postgres flexible-server migration update --subscription xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --migration-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --overwrite-dbs
+  - name: Cutover the data migration. After this is complete, subsequent updates to the source DB will not be migrated to the target.
+    text: az postgres flexible-server migration update --subscription xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --migration-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --cutover
+"""
+
+helps['postgres flexible-server migration delete'] = """
+type: command
+short-summary: Delete a specific migration.
+examples:
+  - name: Cancel/delete the migration workflow. The migration workflows can be canceled/deleted at any point.
+    text: az postgres flexible-server migration delete --subscription xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --migration-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+"""
+
 helps['postgres flexible-server parameter'] = """
 type: group
 short-summary: Commands for managing server parameter values for flexible server.

src/azure-cli/azure/cli/command_modules/rdbms/_params.py

@@ -228,6 +228,10 @@ def _flexible_server_params(command_group):
                                                help="Name of the server. The name can contain only lowercase letters, numbers, and the hyphen (-) character. Minimum 3 characters and maximum 63 characters.",
                                                local_context_attribute=LocalContextAttribute(name='server_name', actions=[LocalContextAction.SET, LocalContextAction.GET], scopes=['{} flexible-server'.format(command_group)]))
 
+        migration_id_arg_type = CLIArgumentType(metavar='NAME',
+                                                help="ID of the migration.",
+                                                local_context_attribute=LocalContextAttribute(name='migration_id', actions=[LocalContextAction.SET, LocalContextAction.GET], scopes=['{} flexible-server'.format(command_group)]))
+
         administrator_login_setter_arg_type = CLIArgumentType(metavar='NAME',
                                                               local_context_attribute=LocalContextAttribute(name='administrator_login', actions=[LocalContextAction.SET], scopes=['{} flexible-server'.format(command_group)]))
 
@@ -400,6 +404,8 @@ def _flexible_server_params(command_group):
                 else:
                     c.argument('server_name', id_part='name', options_list=['--name', '-n'], arg_type=server_name_arg_type)
 
+        handle_migration_parameters(command_group, server_name_arg_type, migration_id_arg_type)
+
         for scope in ['create', 'delete', 'show', 'update']:
             argument_context_string = '{} flexible-server firewall-rule {}'.format(command_group, scope)
             with self.argument_context(argument_context_string) as c:
@@ -482,5 +488,44 @@ def _flexible_server_params(command_group):
             c.argument('action_name', options_list=['--action-name'], help='The name of the github action')
             c.argument('branch', options_list=['--branch'], help='The name of the branch you want upload github action file. The default will be your current branch.')
 
+    def handle_migration_parameters(command_group, server_name_arg_type, migration_id_arg_type):
+        for scope in ['create', 'show', 'list', 'update', 'delete']:
+            argument_context_string = '{} flexible-server migration {}'.format(command_group, scope)
+            with self.argument_context(argument_context_string) as c:
+                c.argument('resource_group_name', arg_type=resource_group_name_type,
+                           help='Resource Group Name of the migration target server.')
+                c.argument('server_name', id_part='name', options_list=['--name', '-n'], arg_type=server_name_arg_type,
+                           help='Migration target server name.')
+                if scope == "create":
+                    c.argument('properties', options_list=['--properties', '-b'],
+                               help='Request properties. Use @{file} to load from a file. For quoting issues in different terminals, '
+                               'see https://github.com/Azure/azure-cli/blob/dev/doc/use_cli_effectively.md#quoting-issues')
+                    c.argument('migration_id', arg_type=migration_id_arg_type, options_list=['--migration-id'],
+                               help='Name or ID of the migration.')
+                elif scope == "show":
+                    c.argument('migration_id', arg_type=migration_id_arg_type, options_list=['--migration-id'],
+                               help='Name or ID of the migration.')
+                    c.argument('level', options_list=['--level'], required=False,
+                               help='Specify the level of migration details requested. Valid values are Active and All. Active is the default.')
+                elif scope == "list":
+                    c.argument('migration_filter', options_list=['--filter'], required=False,
+                               help='Indicate whether all the migrations or just the Active migrations are returned. Active is the default. Valid values are: Active, All.')
+                elif scope == "update":
+                    c.argument('migration_id', arg_type=migration_id_arg_type, options_list=['--migration-id'],
+                               help='Name or ID of the migration.')
+                    c.argument('setup_logical_replication', options_list=['--setup-replication'], action='store_true', required=False,
+                               help='Allow the migration workflow to setup logical replication on the source. Note that this command will restart the source server.')
+                    c.argument('db_names', nargs='+', options_list=['--db-names', '--dbs'], required=False,
+                               help='Space-separated list of DBs to migrate. A minimum of 1 and a maximum of 8 DBs can be specified. You can migrate more DBs concurrently using additional migrations. Note that each additional DB affects the performance of the source server.')
+                    c.argument('overwrite_dbs', options_list=['--overwrite-dbs'], action='store_true', required=False,
+                               help='Allow the migration workflow to overwrite the DB on the target.')
+                    c.argument('cutover', options_list=['--cutover'], action='store_true', required=False,
+                               help='Cut-over the data migration. After this is complete, subsequent updates to the source DB will not be migrated to the target.')
+                elif scope == "delete":
+                    c.argument('migration_id', arg_type=migration_id_arg_type, options_list=['--migration-id'],
+                               help='Name or ID of the migration.')
+                    c.argument('yes', options_list=['--yes', '-y'], action='store_true',
+                               help='Do not prompt for confirmation.')
+
     _flexible_server_params('postgres')
     _flexible_server_params('mysql')

src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_commands.py

@@ -128,6 +128,16 @@ def load_flexibleserver_command_table(self, _):
                                  custom_func_name='flexible_firewall_rule_update_custom_func',
                                  custom_func_type=flexible_server_custom_common)
 
+    with self.command_group('postgres flexible-server migration', postgres_flexible_firewall_rule_sdk,
+                            custom_command_type=flexible_servers_custom_postgres,
+                            client_factory=cf_postgres_flexible_firewall_rules,
+                            is_experimental=True) as g:
+        g.custom_command('create', 'migration_create_func', custom_command_type=flexible_server_custom_common)
+        g.custom_show_command('show', 'migration_show_func', custom_command_type=flexible_server_custom_common)
+        g.custom_command('list', 'migration_list_func', custom_command_type=flexible_server_custom_common)
+        g.custom_command('update', 'migration_update_func', custom_command_type=flexible_server_custom_common)
+        g.custom_command('delete', 'migration_delete_func', custom_command_type=flexible_server_custom_common)
+
     with self.command_group('postgres flexible-server parameter', postgres_flexible_config_sdk,
                             custom_command_type=flexible_servers_custom_postgres,
                             client_factory=cf_postgres_flexible_config,

src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_common.py

@@ -4,9 +4,14 @@
 # --------------------------------------------------------------------------------------------
 
 # pylint: disable=unused-argument, line-too-long
+
+import uuid
 from datetime import datetime
 from knack.log import get_logger
 from knack.util import CLIError
+from azure.cli.core.azclierror import MutuallyExclusiveArgumentError
+from azure.cli.core.commands.client_factory import get_subscription_id
+from azure.cli.core.util import send_raw_request
 from azure.cli.core.util import user_confirmation
 from azure.cli.core.azclierror import ClientRequestError, RequiredArgumentMissingError
 from azure.mgmt.rdbms.mysql_flexibleservers.operations._servers_operations import ServersOperations as MySqlServersOperations
@@ -75,6 +80,89 @@ def firewall_rule_create_func(client, resource_group_name, server_name, firewall
         parameters)
 
 
+def migration_create_func(cmd, client, resource_group_name, server_name, properties, migration_id=None):
+
+    subscription_id = get_subscription_id(cmd.cli_ctx)
+
+    if migration_id is None:
+        # Convert a UUID to a string of hex digits in standard form
+        migration_id = str(uuid.uuid4())
+
+    r = send_raw_request(cmd.cli_ctx, "put", "https://management.azure.com/subscriptions/{}/resourceGroups/{}/providers/Microsoft.DBforPostgreSQL/flexibleServers/{}/migrations/{}?api-version=2020-02-14-privatepreview".format(subscription_id, resource_group_name, server_name, migration_id), None, None, properties)
+
+    return r.json()
+
+
+def migration_show_func(cmd, client, resource_group_name, server_name, migration_id, level="Default"):
+
+    subscription_id = get_subscription_id(cmd.cli_ctx)
+
+    r = send_raw_request(cmd.cli_ctx, "get", "https://management.azure.com/subscriptions/{}/resourceGroups/{}/providers/Microsoft.DBforPostgreSQL/flexibleServers/{}/migrations/{}?level={}&api-version=2020-02-14-privatepreview".format(subscription_id, resource_group_name, server_name, migration_id, level))
+
+    return r.json()
+
+
+def migration_list_func(cmd, client, resource_group_name, server_name, migration_filter="Active"):
+
+    subscription_id = get_subscription_id(cmd.cli_ctx)
+
+    r = send_raw_request(cmd.cli_ctx, "get", "https://management.azure.com/subscriptions/{}/resourceGroups/{}/providers/Microsoft.DBforPostgreSQL/flexibleServers/{}/migrations?migrationListFilter={}&api-version=2020-02-14-privatepreview".format(subscription_id, resource_group_name, server_name, migration_filter))
+
+    return r.json()
+
+
+def migration_update_func(cmd, client, resource_group_name, server_name, migration_id, setup_logical_replication=None, db_names=None, overwrite_dbs=None, cutover=None):
+
+    subscription_id = get_subscription_id(cmd.cli_ctx)
+
+    operationSpecified = False
+    if setup_logical_replication is True:
+        operationSpecified = True
+        properties = "{\"properties\": {\"setupLogicalReplicationOnSourceDBIfNeeded\": \"true\"} }"
+
+    if db_names is not None:
+        if operationSpecified is True:
+            raise MutuallyExclusiveArgumentError("Incorrect Usage: Can only specify one update operation.")
+        operationSpecified = True
+        prefix = "{ \"properties\": { \"dBsToMigrate\": ["
+        db_names_str = "\"" + "\", \"".join(db_names) + "\""
+        suffix = "] } }"
+        properties = prefix + db_names_str + suffix
+
+    if overwrite_dbs is True:
+        if operationSpecified is True:
+            raise MutuallyExclusiveArgumentError("Incorrect Usage: Can only specify one update operation.")
+        operationSpecified = True
+        properties = "{\"properties\": {\"overwriteDBsInTarget\": \"true\"} }"
+
+    if cutover is True:
+        if operationSpecified is True:
+            raise MutuallyExclusiveArgumentError("Incorrect Usage: Can only specify one update operation.")
+        operationSpecified = True
+        properties = "{\"properties\": {\"triggerCutover\": \"true\"} }"
+
+    if operationSpecified is False:
+        raise RequiredArgumentMissingError("Incorrect Usage: Atleast one update operation needs to be specified.")
+
+    r = send_raw_request(cmd.cli_ctx, "patch", "https://management.azure.com/subscriptions/{}/resourceGroups/{}/providers/Microsoft.DBforPostgreSQL/flexibleServers/{}/migrations/{}?api-version=2020-02-14-privatepreview".format(subscription_id, resource_group_name, server_name, migration_id), None, None, properties)
+
+    return r.json()
+
+
+def migration_delete_func(cmd, client, resource_group_name, server_name, migration_id, yes=None):
+
+    subscription_id = get_subscription_id(cmd.cli_ctx)
+
+    if not yes:
+        user_confirmation(
+            "Are you sure you want to delete the migration '{0}' on target server '{1}', resource group '{2}'".format(
+                migration_id, server_name, resource_group_name))
+
+    r = send_raw_request(cmd.cli_ctx, "delete", "https://management.azure.com/subscriptions/{}/resourceGroups/{}/providers/Microsoft.DBforPostgreSQL/flexibleServers/{}/migrations/{}?api-version=2020-02-14-privatepreview".format(subscription_id, resource_group_name, server_name, migration_id))
+
+    return r.json()
+
+
 def firewall_rule_delete_func(client, resource_group_name, server_name, firewall_rule_name, yes=None):
     result = None
     if not yes:

src/azure-cli/azure/cli/command_modules/rdbms/tests/latest/migrationPublic.json

@@ -0,0 +1,30 @@
+{
+    "properties": {
+        "SourceDBServerResourceId": "subscriptions/6a37df99-a9de-48c4-91e5-7e6ab00b2362/resourceGroups/raganesa-s-s-pg-1/providers/Microsoft.DBforPostgreSQL/servers/raganesa-s-s-pg-1",
+        "SecretParameters":
+        {
+            "AdminCredentials":
+            {
+                "SourceServerPassword": "xxxx",
+                "TargetServerPassword": "xxxx"
+            },
+            "AADApp":
+            {
+                "ClientId": "xxxxxx",
+                "TenantId": "72f988bf-86f1-41af-91ab-2d7cd011db47",
+                "AadSecret": "xxxxx"
+            }
+        },
+        "DBsToMigrate": [
+            "dvdrental"
+        ],
+        "MigrationResourceGroup":
+        {
+            "ResourceId": "subscriptions/6a37df99-a9de-48c4-91e5-7e6ab00b2362/resourceGroups/raganesa-DMSBuddy-Demo",
+            "Location": "West US 2"
+        },
+        "SetupLogicalReplicationOnSourceDBIfNeeded": "true",
+        "OverwriteDBsinTarget": "true",
+        "TriggerCutover": "true"
+    }
+}

src/azure-cli/azure/cli/command_modules/rdbms/tests/latest/migrationVNet.json

@@ -0,0 +1,31 @@
+{
+    "properties": {
+        "SourceDBServerResourceId": "subscriptions/6a37df99-a9de-48c4-91e5-7e6ab00b2362/resourceGroups/raganesa-s-s-pg-1/providers/Microsoft.DBforPostgreSQL/servers/raganesa-s-s-pg-1-vnet-1",
+        "SecretParameters":
+        {
+            "AdminCredentials":
+            {
+                "SourceServerPassword": "xxxx",
+                "TargetServerPassword": "xxxx"
+            },
+            "AADApp":
+            {
+                "ClientId": "xxxxxx",
+                "TenantId": "72f988bf-86f1-41af-91ab-2d7cd011db47",
+                "AadSecret": "xxxxx"
+            }
+        },
+        "DBsToMigrate": [
+            "dvdrental"
+        ],
+        "MigrationResourceGroup":
+        {
+            "ResourceId": "subscriptions/6a37df99-a9de-48c4-91e5-7e6ab00b2362/resourceGroups/raganesa-DMSBuddy-Demo",
+            "Location": "West US 2"
+        },
+        "TargetDBServerSubnetResourceId": "subscriptions/6a37df99-a9de-48c4-91e5-7e6ab00b2362/resourceGroups/raganesa-s-s-pg-1/providers/Microsoft.Network/virtualNetworks/raganesa-s-s-pg-1-vnet/subnets/raganesa-s-s-pg-1-vnet-s-s-subnet",
+        "SetupLogicalReplicationOnSourceDBIfNeeded": "true",
+        "OverwriteDBsInTarget": "true",
+        "TriggerCutover": "true"
+    }
+}