diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 94d19fb..946fd2f 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -12,6 +12,14 @@ body: description: What is the problem? A clear and concise description of the bug. validations: required: true + - type: checkboxes + id: regression + attributes: + label: Regression Issue + description: What is a regression? If it worked in a previous version but doesn't in the latest version, it's considered a regression. In this case, please provide specific version number in the report. + options: + - label: Select this option if this issue appears to be a regression. + required: false - type: textarea id: expected attributes: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd09c7e..8776272 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,22 +8,22 @@ on: env: RUN: ${{ github.run_id }}-${{ github.run_number }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + CI_SDK_V1_ROLE: arn:aws:iam::180635532705:role/CI_SDK_V1_ROLE PACKAGE_NAME: aws-iot-device-sdk-python AWS_EC2_METADATA_DISABLED: true jobs: unit-tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: fail-fast: false steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: '3.6.7' + python-version: '3.8' - name: Unit tests run: | python3 setup.py install @@ -33,16 +33,23 @@ jobs: integration-tests: runs-on: ubuntu-latest + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout strategy: fail-fast: false matrix: test-type: [ MutualAuth, Websocket, ALPN ] - python-version: [ '3.7', '3.12' ] + python-version: [ '3.8', '3.13' ] steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.CI_SDK_V1_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Integration tests run: | pip install pytest diff --git a/.github/workflows/handle-stale-discussions.yml b/.github/workflows/handle-stale-discussions.yml index e92e660..4fbcd70 100644 --- a/.github/workflows/handle-stale-discussions.yml +++ b/.github/workflows/handle-stale-discussions.yml @@ -1,18 +1,19 @@ name: HandleStaleDiscussions on: -schedule: -- cron: '0 */4 * * *' -discussion_comment: -types: [created] + schedule: + - cron: '0 */4 * * *' + discussion_comment: + types: [created] jobs: -handle-stale-discussions: -name: Handle stale discussions -runs-on: ubuntu-latest -permissions: - discussions: write -steps: - - name: Stale discussions action - uses: aws-github-ops/handle-stale-discussions@v1 - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} \ No newline at end of file + handle-stale-discussions: + name: Handle stale discussions + runs-on: ubuntu-latest + permissions: + discussions: write + steps: + - name: Stale discussions action + uses: aws-github-ops/handle-stale-discussions@v1 + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + \ No newline at end of file diff --git a/.github/workflows/issue-regression-labeler.yml b/.github/workflows/issue-regression-labeler.yml new file mode 100644 index 0000000..bd00071 --- /dev/null +++ b/.github/workflows/issue-regression-labeler.yml @@ -0,0 +1,32 @@ +# Apply potential regression label on issues +name: issue-regression-label +on: + issues: + types: [opened, edited] +jobs: + add-regression-label: + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Fetch template body + id: check_regression + uses: actions/github-script@v7 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TEMPLATE_BODY: ${{ github.event.issue.body }} + with: + script: | + const regressionPattern = /\[x\] Select this option if this issue appears to be a regression\./i; + const template = `${process.env.TEMPLATE_BODY}` + const match = regressionPattern.test(template); + core.setOutput('is_regression', match); + - name: Manage regression label + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ "${{ steps.check_regression.outputs.is_regression }}" == "true" ]; then + gh issue edit ${{ github.event.issue.number }} --add-label "potential-regression" -R ${{ github.repository }} + else + gh issue edit ${{ github.event.issue.number }} --remove-label "potential-regression" -R ${{ github.repository }} + fi diff --git a/AWSIoTPythonSDK/core/protocol/mqtt_core.py b/AWSIoTPythonSDK/core/protocol/mqtt_core.py index fbdd6bf..bff6456 100644 --- a/AWSIoTPythonSDK/core/protocol/mqtt_core.py +++ b/AWSIoTPythonSDK/core/protocol/mqtt_core.py @@ -28,7 +28,7 @@ from AWSIoTPythonSDK.core.protocol.internal.defaults import METRICS_PREFIX from AWSIoTPythonSDK.core.protocol.internal.defaults import ALPN_PROTCOLS from AWSIoTPythonSDK.core.protocol.internal.events import FixedEventMids -from AWSIoTPythonSDK.core.protocol.paho.client import MQTT_ERR_SUCCESS +from AWSIoTPythonSDK.core.protocol.paho.client import MQTT_ERR_SUCCESS, SUBACK_ERROR from AWSIoTPythonSDK.exception.AWSIoTExceptions import connectError from AWSIoTPythonSDK.exception.AWSIoTExceptions import connectTimeoutException from AWSIoTPythonSDK.exception.AWSIoTExceptions import disconnectError @@ -41,7 +41,7 @@ from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeQueueDisabledException from AWSIoTPythonSDK.exception.AWSIoTExceptions import unsubscribeQueueFullException from AWSIoTPythonSDK.exception.AWSIoTExceptions import unsubscribeQueueDisabledException -from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeError +from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeError, subackError from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeTimeoutException from AWSIoTPythonSDK.exception.AWSIoTExceptions import unsubscribeError from AWSIoTPythonSDK.exception.AWSIoTExceptions import unsubscribeTimeoutException @@ -58,6 +58,12 @@ from queue import Queue +class SubackPacket(object): + def __init__(self): + self.event = Event() + self.data = None + + class MqttCore(object): _logger = logging.getLogger(__name__) @@ -298,12 +304,15 @@ def subscribe(self, topic, qos, message_callback=None): if ClientStatus.STABLE != self._client_status.get_status(): self._handle_offline_request(RequestTypes.SUBSCRIBE, (topic, qos, message_callback, None)) else: - event = Event() - rc, mid = self._subscribe_async(topic, qos, self._create_blocking_ack_callback(event), message_callback) - if not event.wait(self._operation_timeout_sec): + suback = SubackPacket() + rc, mid = self._subscribe_async(topic, qos, self._create_blocking_suback_callback(suback), message_callback) + if not suback.event.wait(self._operation_timeout_sec): self._internal_async_client.remove_event_callback(mid) self._logger.error("Subscribe timed out") raise subscribeTimeoutException() + if suback.data and suback.data[0] == SUBACK_ERROR: + self._logger.error(f"Suback error return code: {suback.data[0]}") + raise subackError(suback=suback.data) ret = True return ret @@ -361,6 +370,12 @@ def ack_callback(mid, data=None): event.set() return ack_callback + def _create_blocking_suback_callback(self, ack: SubackPacket): + def ack_callback(mid, data=None): + ack.data = data + ack.event.set() + return ack_callback + def _handle_offline_request(self, type, data): self._logger.info("Offline request detected!") offline_request = QueueableRequest(type, data) diff --git a/AWSIoTPythonSDK/core/protocol/paho/client.py b/AWSIoTPythonSDK/core/protocol/paho/client.py index 0b637c5..1c60c81 100755 --- a/AWSIoTPythonSDK/core/protocol/paho/client.py +++ b/AWSIoTPythonSDK/core/protocol/paho/client.py @@ -97,6 +97,9 @@ CONNACK_REFUSED_BAD_USERNAME_PASSWORD = 4 CONNACK_REFUSED_NOT_AUTHORIZED = 5 +# SUBACK codes +SUBACK_ERROR = 0x80 + # Connection state mqtt_cs_new = 0 mqtt_cs_connected = 1 @@ -137,6 +140,7 @@ MSG_QUEUEING_DROP_OLDEST = 0 MSG_QUEUEING_DROP_NEWEST = 1 + if sys.version_info[0] < 3: sockpair_data = "0" else: diff --git a/AWSIoTPythonSDK/exception/AWSIoTExceptions.py b/AWSIoTPythonSDK/exception/AWSIoTExceptions.py index 0de5401..d5b6d63 100755 --- a/AWSIoTPythonSDK/exception/AWSIoTExceptions.py +++ b/AWSIoTPythonSDK/exception/AWSIoTExceptions.py @@ -79,6 +79,11 @@ class subscribeError(operationError.operationError): def __init__(self, errorCode): self.message = "Subscribe Error: " + str(errorCode) +class subackError(operationError.operationError): + def __init__(self, suback=None): + self.message = "Received Error suback. Subscription failed." + self.suback = suback + class subscribeQueueFullException(operationError.operationError): def __init__(self): diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 15ad4b5..765c557 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,10 +2,16 @@ CHANGELOG ========= +1.5.5 +===== +* chore: Update minimum Python version based on current supportable levels + 1.5.4 +===== * chore: CD pipeline flushing try #1 1.5.3 +===== * improvement: Support Python3.12+ by conditionally removing deprecated API usage 1.4.9 diff --git a/README.rst b/README.rst index 991007f..ba88218 100755 --- a/README.rst +++ b/README.rst @@ -70,22 +70,10 @@ also allows the use of the same connection for shadow operations and non-shadow, Installation ~~~~~~~~~~~~ -Minimum Requirements +Requirements ____________________ -- Python 2.7+ or Python 3.3+ for X.509 certificate-based mutual authentication via port 8883 - and MQTT over WebSocket protocol with AWS Signature Version 4 authentication -- Python 2.7.10+ or Python 3.5+ for X.509 certificate-based mutual authentication via port 443 -- OpenSSL version 1.0.1+ (TLS version 1.2) compiled with the Python executable for - X.509 certificate-based mutual authentication - - To check your version of OpenSSL, use the following command in a Python interpreter: - - .. code-block:: python - - >>> import ssl - >>> ssl.OPENSSL_VERSION - +- Python3.8+. The SDK has worked for older Python versions in the past, but they are no longer formally supported. Over time, expect the minimum Python version to loosely track the minimum non-end-of-life version. Install from pip ________________ diff --git a/setup.py b/setup.py index d0c2b8d..0ca4cfa 100644 --- a/setup.py +++ b/setup.py @@ -20,15 +20,11 @@ download_url = 'https://s3.amazonaws.com/aws-iot-device-sdk-python/aws-iot-device-sdk-python-latest.zip', keywords = ['aws', 'iot', 'mqtt'], classifiers = [ - "Development Status :: 5 - Production/Stable", + "Development Status :: 6 - Mature", "Intended Audience :: Developers", "Natural Language :: English", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5" + "Programming Language :: Python :: 3" ] ) diff --git a/setup_test.py b/setup_test.py index 820d715..2a4e78e 100644 --- a/setup_test.py +++ b/setup_test.py @@ -24,15 +24,11 @@ download_url = 'https://s3.amazonaws.com/aws-iot-device-sdk-python/aws-iot-device-sdk-python-latest.zip', keywords = ['aws', 'iot', 'mqtt'], classifiers = [ - "Development Status :: 5 - Production/Stable", + "Development Status :: 6 - Mature", "Intended Audience :: Developers", "Natural Language :: English", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5" ] ) diff --git a/test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py b/test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py index b69fdcd..577c5fa 100644 --- a/test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py +++ b/test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py @@ -32,7 +32,6 @@ TOPIC = "topic/test/async_cb/" MESSAGE_PREFIX = "MagicMessage-" NUMBER_OF_PUBLISHES = 3 -HOST = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" ROOT_CA = "./test-integration/Credentials/rootCA.crt" CERT = "./test-integration/Credentials/certificate.pem.crt" KEY = "./test-integration/Credentials/privateKey.pem.key" @@ -102,9 +101,10 @@ def get_random_string(length): ############################################################################ # Main # # Check inputs -my_check_in_manager = checkInManager(1) +my_check_in_manager = checkInManager(2) my_check_in_manager.verify(sys.argv) mode = my_check_in_manager.mode +host = my_check_in_manager.host skip_when_match(ModeIsALPN(mode).And( Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) @@ -115,7 +115,7 @@ def get_random_string(length): print("Connecting...") callback_manager = CallbackManager() sdk_mqtt_client = MQTTClientManager()\ - .create_nonconnected_mqtt_client(mode, CLIENT_ID, HOST, (ROOT_CA, CERT, KEY), callback_manager) + .create_nonconnected_mqtt_client(mode, CLIENT_ID, host, (ROOT_CA, CERT, KEY), callback_manager) sdk_mqtt_client.connectAsync(keepAliveIntervalSecond=1, ackCallback=callback_manager.connack) # Add callback print("Wait some time to make sure we are connected...") time.sleep(10) # 10 sec diff --git a/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py b/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py index 83b66c2..e6c1bee 100644 --- a/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py +++ b/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py @@ -135,14 +135,14 @@ def threadBRuntime(self, pyCoreClient, callback): ############################################################################ # Main # # Check inputs -myCheckInManager = checkInManager.checkInManager(1) +myCheckInManager = checkInManager.checkInManager(2) myCheckInManager.verify(sys.argv) -host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" rootCA = "./test-integration/Credentials/rootCA.crt" certificate = "./test-integration/Credentials/certificate.pem.crt" privateKey = "./test-integration/Credentials/privateKey.pem.key" mode = myCheckInManager.mode +host = myCheckInManager.host skip_when_match(ModeIsALPN(mode).And( Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) diff --git a/test-integration/IntegrationTests/IntegrationTestClientReusability.py b/test-integration/IntegrationTests/IntegrationTestClientReusability.py index f747e19..56e77b8 100644 --- a/test-integration/IntegrationTests/IntegrationTestClientReusability.py +++ b/test-integration/IntegrationTests/IntegrationTestClientReusability.py @@ -40,7 +40,6 @@ NUMBER_OF_MESSAGES_PER_LOOP = 3 NUMBER_OF_LOOPS = 3 SUB_WAIT_TIME_OUT_SEC = 20 -HOST = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" ROOT_CA = "./test-integration/Credentials/rootCA.crt" CERT = "./test-integration/Credentials/certificate.pem.crt" KEY = "./test-integration/Credentials/privateKey.pem.key" @@ -94,9 +93,10 @@ def verify(self): ############################################################################ # Main # -my_check_in_manager = checkInManager(1) +my_check_in_manager = checkInManager(2) my_check_in_manager.verify(sys.argv) mode = my_check_in_manager.mode +host = my_check_in_manager.host skip_when_match(ModeIsALPN(mode).And( Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) @@ -104,9 +104,9 @@ def verify(self): simple_thread_manager = simpleThreadManager() -client_pub = MQTTClientManager().create_nonconnected_mqtt_client(mode, CLIENT_ID_PUB, HOST, (ROOT_CA, CERT, KEY)) +client_pub = MQTTClientManager().create_nonconnected_mqtt_client(mode, CLIENT_ID_PUB, host, (ROOT_CA, CERT, KEY)) print("Client publisher initialized.") -client_sub = MQTTClientManager().create_nonconnected_mqtt_client(mode, CLIENT_ID_SUB, HOST, (ROOT_CA, CERT, KEY)) +client_sub = MQTTClientManager().create_nonconnected_mqtt_client(mode, CLIENT_ID_SUB, host, (ROOT_CA, CERT, KEY)) print("Client subscriber initialized.") client_twins = ClientTwins(client_pub, client_sub) print("Client twins initialized.") diff --git a/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py b/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py index d6bfdc5..0d78f4f 100644 --- a/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py +++ b/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py @@ -274,10 +274,10 @@ def performConfigurableOfflinePublishQueueTest(clientPub, clientSub): # Check inputs -myCheckInManager = checkInManager.checkInManager(1) +myCheckInManager = checkInManager.checkInManager(2) myCheckInManager.verify(sys.argv) -host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +host = myCheckInManager.host rootCA = "./test-integration/Credentials/rootCA.crt" certificate = "./test-integration/Credentials/certificate.pem.crt" privateKey = "./test-integration/Credentials/privateKey.pem.key" diff --git a/test-integration/IntegrationTests/IntegrationTestDiscovery.py b/test-integration/IntegrationTests/IntegrationTestDiscovery.py index 8f23aa9..2fac25b 100644 --- a/test-integration/IntegrationTests/IntegrationTestDiscovery.py +++ b/test-integration/IntegrationTests/IntegrationTestDiscovery.py @@ -8,13 +8,13 @@ from TestToolLibrary.skip import ModeIsWebSocket -HOST = "arc9d2oott9lj-ats.iot.us-east-1.amazonaws.com" # 003261610643 PORT = 8443 CA = "./test-integration/Credentials/rootCA.crt" CERT = "./test-integration/Credentials/certificate_drs.pem.crt" KEY = "./test-integration/Credentials/privateKey_drs.pem.key" TIME_OUT_SEC = 30 # This is a pre-generated test data from DRS integration tests +# The test resources point to account # 003261610643 ID_PREFIX = "Id-" GGC_ARN = "arn:aws:iot:us-east-1:003261610643:thing/DRS_GGC_0kegiNGA_0" GGC_PORT_NUMBER_BASE = 8080 @@ -108,10 +108,14 @@ } ''' +my_check_in_manager = checkInManager(2) +my_check_in_manager.verify(sys.argv) +mode = my_check_in_manager.mode +host = my_check_in_manager.host def create_discovery_info_provider(): discovery_info_provider = DiscoveryInfoProvider() - discovery_info_provider.configureEndpoint(HOST, PORT) + discovery_info_provider.configureEndpoint(host, PORT) discovery_info_provider.configureCredentials(CA, CERT, KEY) discovery_info_provider.configureTimeout(TIME_OUT_SEC) return discovery_info_provider @@ -196,9 +200,6 @@ def verify_group_object(discovery_info): ############################################################################ # Main # -my_check_in_manager = checkInManager(1) -my_check_in_manager.verify(sys.argv) -mode = my_check_in_manager.mode skip_when_match(ModeIsWebSocket(mode), "This test is not applicable for mode: %s. Skipping..." % mode) diff --git a/test-integration/IntegrationTests/IntegrationTestJobsClient.py b/test-integration/IntegrationTests/IntegrationTestJobsClient.py index 18d8aa5..3653725 100644 --- a/test-integration/IntegrationTests/IntegrationTestJobsClient.py +++ b/test-integration/IntegrationTests/IntegrationTestJobsClient.py @@ -154,10 +154,10 @@ def _test_send_response_confirm(self, sendResult): ############################################################################ # Main # # Check inputs -myCheckInManager = checkInManager.checkInManager(1) +myCheckInManager = checkInManager.checkInManager(2) myCheckInManager.verify(sys.argv) -host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +host = myCheckInManager.host rootCA = "./test-integration/Credentials/rootCA.crt" certificate = "./test-integration/Credentials/certificate.pem.crt" privateKey = "./test-integration/Credentials/privateKey.pem.key" diff --git a/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py b/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py index 252770f..9adc38c 100644 --- a/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py +++ b/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py @@ -84,10 +84,10 @@ def _performPublish(self, pyCoreClient, topic, qos, payload): ############################################################################ # Main # # Check inputs -myCheckInManager = checkInManager.checkInManager(2) +myCheckInManager = checkInManager.checkInManager(3) myCheckInManager.verify(sys.argv) -host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +host = myCheckInManager.host rootCA = "./test-integration/Credentials/rootCA.crt" certificate = "./test-integration/Credentials/certificate.pem.crt" privateKey = "./test-integration/Credentials/privateKey.pem.key" diff --git a/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py b/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py index c06847d..37c1862 100644 --- a/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py +++ b/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py @@ -47,7 +47,6 @@ def get_random_string(length): TOPIC_B = "topic/test/offline_sub_unsub/b" + get_random_string(4) MESSAGE_PREFIX = "MagicMessage-" NUMBER_OF_PUBLISHES = 3 -HOST = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" ROOT_CA = "./test-integration/Credentials/rootCA.crt" CERT = "./test-integration/Credentials/certificate.pem.crt" KEY = "./test-integration/Credentials/privateKey.pem.key" @@ -74,7 +73,7 @@ def __init__(self, mode): time.sleep(2) # Make sure the subscription is valid def _create_connected_client(self, id_prefix): - return MQTTClientManager().create_connected_mqtt_client(self.__mode, id_prefix, HOST, (ROOT_CA, CERT, KEY)) + return MQTTClientManager().create_connected_mqtt_client(self.__mode, id_prefix, host, (ROOT_CA, CERT, KEY)) def start(self): thread_client_sub_unsub = Thread(target=self._thread_client_sub_unsub_runtime) @@ -192,9 +191,10 @@ def verify(self): ############################################################################ # Main # # Check inputs -my_check_in_manager = checkInManager(1) +my_check_in_manager = checkInManager(2) my_check_in_manager.verify(sys.argv) mode = my_check_in_manager.mode +host = my_check_in_manager.host skip_when_match(ModeIsALPN(mode).And( Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) diff --git a/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py b/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py index cd7b7ec..fc937ef 100644 --- a/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py +++ b/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py @@ -220,11 +220,11 @@ def verifyBackoffTime(answerList, resultList): ############################################################################ # Main # # Check inputs -myCheckInManager = checkInManager.checkInManager(2) +myCheckInManager = checkInManager.checkInManager(3) myCheckInManager.verify(sys.argv) #host via describe-endpoint on this OdinMS: com.amazonaws.iot.device.sdk.credentials.testing.websocket -host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +host = myCheckInManager.host rootCA = "./test-integration/Credentials/rootCA.crt" certificate = "./test-integration/Credentials/certificate.pem.crt" privateKey = "./test-integration/Credentials/privateKey.pem.key" diff --git a/test-integration/IntegrationTests/IntegrationTestShadow.py b/test-integration/IntegrationTests/IntegrationTestShadow.py index 9e2c2a5..9b2d85a 100644 --- a/test-integration/IntegrationTests/IntegrationTestShadow.py +++ b/test-integration/IntegrationTests/IntegrationTestShadow.py @@ -150,10 +150,10 @@ def randomString(lengthOfString): ############################################################################ # Main # # Check inputs -myCheckInManager = checkInManager.checkInManager(2) +myCheckInManager = checkInManager.checkInManager(3) myCheckInManager.verify(sys.argv) -host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +host = myCheckInManager.host rootCA = "./test-integration/Credentials/rootCA.crt" certificate = "./test-integration/Credentials/certificate.pem.crt" privateKey = "./test-integration/Credentials/privateKey.pem.key" diff --git a/test-integration/IntegrationTests/TestToolLibrary/checkInManager.py b/test-integration/IntegrationTests/TestToolLibrary/checkInManager.py index 2faaa02..aeeedd9 100644 --- a/test-integration/IntegrationTests/TestToolLibrary/checkInManager.py +++ b/test-integration/IntegrationTests/TestToolLibrary/checkInManager.py @@ -7,6 +7,7 @@ class checkInManager: def __init__(self, numberOfInputParameters): self._numberOfInputParameters = numberOfInputParameters self.mode = None + self.host = None self.customParameter = None def verify(self, args): @@ -14,5 +15,6 @@ def verify(self, args): if len(args) != self._numberOfInputParameters + 1: exit(4) self.mode = str(args[1]) - if self._numberOfInputParameters + 1 > 2: - self.customParameter = int(args[2]) + self.host = str(args[2]) + if self._numberOfInputParameters + 1 > 3: + self.customParameter = int(args[3]) diff --git a/test-integration/run/run.sh b/test-integration/run/run.sh index 0eb933b..8e23c91 100755 --- a/test-integration/run/run.sh +++ b/test-integration/run/run.sh @@ -33,14 +33,14 @@ # Define const USAGE="usage: run.sh " -AWSMutualAuth_TodWorker_private_key="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestPrivateKey-vNUQU8" -AWSMutualAuth_TodWorker_certificate="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestCertificate-vTRwjE" +UnitTestHostArn="arn:aws:secretsmanager:us-east-1:180635532705:secret:unit-test/endpoint-HSpeEu" +GreenGrassHostArn="arn:aws:secretsmanager:us-east-1:180635532705:secret:ci/greengrassv1/endpoint-DgM00X" -AWSGGDiscovery_TodWorker_private_key="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestGGDiscoveryPrivateKey-YHQI1F" -AWSGGDiscovery_TodWorker_certificate="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestGGDiscoveryCertificate-TwlAcS" +AWSMutualAuth_TodWorker_private_key="arn:aws:secretsmanager:us-east-1:180635532705:secret:ci/mqtt5/us/Mqtt5Prod/key-kqgyvf" +AWSMutualAuth_TodWorker_certificate="arn:aws:secretsmanager:us-east-1:180635532705:secret:ci/mqtt5/us/Mqtt5Prod/cert-VDI1Gd" -AWSSecretForWebsocket_TodWorker_KeyId="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestWebsocketAccessKeyId-1YdB9z" -AWSSecretForWebsocket_TodWorker_SecretKey="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestWebsocketSecretAccessKey-MKTSaV" +AWSGGDiscovery_TodWorker_private_key="arn:aws:secretsmanager:us-east-1:180635532705:secret:V1IotSdkIntegrationTestGGDiscoveryPrivateKey-BsLvNP" +AWSGGDiscovery_TodWorker_certificate="arn:aws:secretsmanager:us-east-1:180635532705:secret:V1IotSdkIntegrationTestGGDiscoveryCertificate-DSwdhA" SDKLocation="./AWSIoTPythonSDK" @@ -49,6 +49,8 @@ CREDENTIAL_DIR="./test-integration/Credentials/" TEST_DIR="./test-integration/IntegrationTests/" CA_CERT_URL="https://www.amazontrust.com/repository/AmazonRootCA1.pem" CA_CERT_PATH=${CREDENTIAL_DIR}rootCA.crt +TestHost=$(python ${RetrieveAWSKeys} ${UnitTestHostArn}) +GreengrassHost=$(python ${RetrieveAWSKeys} ${GreenGrassHostArn}) @@ -82,11 +84,7 @@ else python ${RetrieveAWSKeys} ${AWSDRSName_certificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt python ${RetrieveAWSKeys} ${AWSDRSName_privatekey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key elif [ "$1"x == "Websocket"x ]; then - ACCESS_KEY_ID_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_TodWorker_KeyId}) - ACCESS_SECRET_KEY_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_TodWorker_SecretKey}) TestMode="Websocket" - export AWS_ACCESS_KEY_ID=${ACCESS_KEY_ID_ARN} - export AWS_SECRET_ACCESS_KEY=${ACCESS_SECRET_KEY_ARN} curl -s "${CA_CERT_URL}" > ${CA_CERT_PATH} echo -e "URL retrieved certificate data\n" elif [ "$1"x == "ALPN"x ]; then @@ -115,11 +113,11 @@ else echo "***************************************************" for file in `ls ${TEST_DIR}` do - # if [ ${file}x == "IntegrationTestMQTTConnection.py"x ]; then if [ ${file##*.}x == "py"x ]; then echo "[SUB] Running test: ${file}..." - + Scale=10 + Host=TestHost case "$file" in "IntegrationTestMQTTConnection.py") Scale=$2 ;; @@ -131,7 +129,8 @@ else ;; "IntegrationTestConfigurablePublishMessageQueueing.py") Scale="" ;; - "IntegrationTestDiscovery.py") Scale="" + "IntegrationTestDiscovery.py") Scale="" + Host=${GreengrassHost} ;; "IntegrationTestAsyncAPIGeneralNotificationCallbacks.py") Scale="" ;; @@ -142,7 +141,7 @@ else "IntegrationTestJobsClient.py") Scale="" esac - python ${TEST_DIR}${file} ${TestMode} ${Scale} + python ${TEST_DIR}${file} ${TestMode} ${TestHost} ${Scale} currentTestStatus=$? echo "[SUB] Test: ${file} completed. Exiting with status: ${currentTestStatus}" if [ ${currentTestStatus} -ne 0 ]; then diff --git a/test/core/protocol/test_mqtt_core.py b/test/core/protocol/test_mqtt_core.py index 8469ea6..3cb0cb0 100644 --- a/test/core/protocol/test_mqtt_core.py +++ b/test/core/protocol/test_mqtt_core.py @@ -1,5 +1,6 @@ import AWSIoTPythonSDK from AWSIoTPythonSDK.core.protocol.mqtt_core import MqttCore +from AWSIoTPythonSDK.core.protocol.mqtt_core import SubackPacket from AWSIoTPythonSDK.core.protocol.internal.clients import InternalAsyncMqttClient from AWSIoTPythonSDK.core.protocol.internal.clients import ClientStatusContainer from AWSIoTPythonSDK.core.protocol.internal.clients import ClientStatus @@ -20,6 +21,7 @@ from AWSIoTPythonSDK.exception.AWSIoTExceptions import publishQueueFullException from AWSIoTPythonSDK.exception.AWSIoTExceptions import publishQueueDisabledException from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeError +from AWSIoTPythonSDK.exception.AWSIoTExceptions import subackError from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeTimeoutException from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeQueueFullException from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeQueueDisabledException @@ -29,6 +31,7 @@ from AWSIoTPythonSDK.exception.AWSIoTExceptions import unsubscribeQueueDisabledException from AWSIoTPythonSDK.core.protocol.paho.client import MQTT_ERR_SUCCESS from AWSIoTPythonSDK.core.protocol.paho.client import MQTT_ERR_ERRNO +from AWSIoTPythonSDK.core.protocol.paho.client import SUBACK_ERROR from AWSIoTPythonSDK.core.protocol.paho.client import MQTTv311 from AWSIoTPythonSDK.core.protocol.internal.defaults import ALPN_PROTCOLS try: @@ -61,6 +64,7 @@ KEY_EXPECTED_QUEUE_APPEND_RESULT = "ExpectedQueueAppendResult" KEY_EXPECTED_REQUEST_MID_OVERRIDE = "ExpectedRequestMidOverride" KEY_EXPECTED_REQUEST_TIMEOUT = "ExpectedRequestTimeout" +KEY_EXPECTED_ACK_RESULT = "ExpectedAckPacketResult" SUCCESS_RC_EXPECTED_VALUES = { KEY_EXPECTED_REQUEST_RC : DUMMY_SUCCESS_RC } @@ -73,6 +77,10 @@ NO_TIMEOUT_EXPECTED_VALUES = { KEY_EXPECTED_REQUEST_TIMEOUT : False } +ERROR_SUBACK_EXPECTED_VALUES = { + KEY_EXPECTED_ACK_RESULT : (SUBACK_ERROR, None) +} + QUEUED_EXPECTED_VALUES = { KEY_EXPECTED_QUEUE_APPEND_RESULT : AppendResults.APPEND_SUCCESS } @@ -121,6 +129,9 @@ def setup_class(cls): RequestTypes.SUBSCRIBE: subscribeError, RequestTypes.UNSUBSCRIBE: unsubscribeError } + cls.ack_error = { + RequestTypes.SUBSCRIBE : subackError, + } cls.request_queue_full = { RequestTypes.PUBLISH : publishQueueFullException, RequestTypes.SUBSCRIBE: subscribeQueueFullException, @@ -518,6 +529,9 @@ def test_subscribe_success(self): def test_subscribe_timeout(self): self._internal_test_sync_api_with(RequestTypes.SUBSCRIBE, TIMEOUT_EXPECTED_VALUES) + + def test_subscribe_error_suback(self): + self._internal_test_sync_api_with(RequestTypes.SUBSCRIBE, ERROR_SUBACK_EXPECTED_VALUES) def test_subscribe_queued(self): self._internal_test_sync_api_with(RequestTypes.SUBSCRIBE, QUEUED_EXPECTED_VALUES) @@ -547,6 +561,7 @@ def _internal_test_sync_api_with(self, request_type, expected_values): expected_request_mid = expected_values.get(KEY_EXPECTED_REQUEST_MID_OVERRIDE) expected_timeout = expected_values.get(KEY_EXPECTED_REQUEST_TIMEOUT) expected_append_result = expected_values.get(KEY_EXPECTED_QUEUE_APPEND_RESULT) + expected_suback_result = expected_values.get(KEY_EXPECTED_ACK_RESULT) if expected_request_mid is None: expected_request_mid = DUMMY_REQUEST_MID @@ -562,7 +577,16 @@ def _internal_test_sync_api_with(self, request_type, expected_values): self.invoke_mqtt_core_sync_api[request_type](self, message_callback) else: self.python_event_mock.wait.return_value = True - assert self.invoke_mqtt_core_sync_api[request_type](self, message_callback) is True + if expected_suback_result is not None: + self._use_mock_python_suback() + # mock the suback with expected suback result + self.python_suback_mock.data = expected_suback_result + if expected_suback_result[0] == SUBACK_ERROR: + with pytest.raises(self.ack_error[request_type]): + self.invoke_mqtt_core_sync_api[request_type](self, message_callback) + self.python_suback_patcher.stop() + else: + assert self.invoke_mqtt_core_sync_api[request_type](self, message_callback) is True if expected_append_result is not None: self.client_status_mock.get_status.return_value = ClientStatus.ABNORMAL_DISCONNECT @@ -583,3 +607,10 @@ def _use_mock_python_event(self): self.python_event_constructor = self.python_event_patcher.start() self.python_event_mock = MagicMock() self.python_event_constructor.return_value = self.python_event_mock + + # Create a SubackPacket mock, which would mock the data in SubackPacket + def _use_mock_python_suback(self): + self.python_suback_patcher = patch(PATCH_MODULE_LOCATION + "SubackPacket", spec=SubackPacket) + self.python_suback_constructor = self.python_suback_patcher.start() + self.python_suback_mock = MagicMock() + self.python_suback_constructor.return_value = self.python_suback_mock