From 4cc7a91f4f5dfe22ecd094e4d563ec92a6c1c892 Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Thu, 2 Oct 2025 14:00:02 -0400 Subject: [PATCH 01/14] Add new preferred transfer client mode to enable CRT in all environments --- .changes/next-release/feature-s3-53747.json | 5 +++++ boto3/s3/constants.py | 1 + boto3/s3/transfer.py | 9 +++++---- tests/functional/test_crt.py | 9 +++++++++ 4 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 .changes/next-release/feature-s3-53747.json diff --git a/.changes/next-release/feature-s3-53747.json b/.changes/next-release/feature-s3-53747.json new file mode 100644 index 0000000000..d3c2694830 --- /dev/null +++ b/.changes/next-release/feature-s3-53747.json @@ -0,0 +1,5 @@ +{ + "type": "feature", + "category": "``s3``", + "description": "Added ``crt`` mode to ``preferred_transfer_client`` parameter in ``TransferConfig`` to enable CRT transfer client in all environments." +} diff --git a/boto3/s3/constants.py b/boto3/s3/constants.py index c7f691fc21..bd387780aa 100644 --- a/boto3/s3/constants.py +++ b/boto3/s3/constants.py @@ -14,4 +14,5 @@ # TransferConfig preferred_transfer_client settings CLASSIC_TRANSFER_CLIENT = "classic" +CRT_TRANSFER_CLIENT = "crt" AUTO_RESOLVE_TRANSFER_CLIENT = "auto" diff --git a/boto3/s3/transfer.py b/boto3/s3/transfer.py index 6032b624a5..fc061d15e4 100644 --- a/boto3/s3/transfer.py +++ b/boto3/s3/transfer.py @@ -184,15 +184,15 @@ def create_transfer_manager(client, config, osutil=None): def _should_use_crt(config): # This feature requires awscrt>=0.19.18 - if HAS_CRT and has_minimum_crt_version((0, 19, 18)): - is_optimized_instance = awscrt.s3.is_optimized_for_system() - else: - is_optimized_instance = False + has_min_crt = HAS_CRT and has_minimum_crt_version((0, 19, 18)) + is_optimized_instance = has_min_crt and awscrt.s3.is_optimized_for_system() pref_transfer_client = config.preferred_transfer_client.lower() if ( is_optimized_instance and pref_transfer_client == constants.AUTO_RESOLVE_TRANSFER_CLIENT + ) or ( + has_min_crt and pref_transfer_client == constants.CRT_TRANSFER_CLIENT ): logger.debug( "Attempting to use CRTTransferManager. Config settings may be ignored." @@ -296,6 +296,7 @@ def __init__( are made with supported environment and settings. * classic - Only use the origin S3TransferManager with requests. Disables possible CRT upgrade on requests. + * crt - Only use the CRTTransferManager with requests. """ super().__init__( multipart_threshold=multipart_threshold, diff --git a/tests/functional/test_crt.py b/tests/functional/test_crt.py index cc13168e3b..48ac950c39 100644 --- a/tests/functional/test_crt.py +++ b/tests/functional/test_crt.py @@ -68,6 +68,15 @@ def test_create_transfer_manager_on_optimized_instance(self): transfer_manager = create_transfer_manager(client, config) assert isinstance(transfer_manager, CRTTransferManager) + @requires_crt() + def test_create_transfer_manager_with_crt_preferred(self): + client = create_mock_client() + config = TransferConfig( + preferred_transfer_client='crt', + ) + transfer_manager = create_transfer_manager(client, config) + assert isinstance(transfer_manager, CRTTransferManager) + @requires_crt() def test_minimum_crt_version(self): assert has_minimum_crt_version((0, 16, 12)) is True From f05191eaee9ec7f5f6ce054d6eaead19a93c6710 Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Thu, 2 Oct 2025 14:54:17 -0400 Subject: [PATCH 02/14] Raise if no min CRT version --- boto3/s3/transfer.py | 16 +++++++++++++--- tests/functional/test_crt.py | 22 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/boto3/s3/transfer.py b/boto3/s3/transfer.py index fc061d15e4..086a06f3d4 100644 --- a/boto3/s3/transfer.py +++ b/boto3/s3/transfer.py @@ -188,12 +188,22 @@ def _should_use_crt(config): is_optimized_instance = has_min_crt and awscrt.s3.is_optimized_for_system() pref_transfer_client = config.preferred_transfer_client.lower() + if ( + pref_transfer_client == constants.CRT_TRANSFER_CLIENT + and not has_min_crt + ): + msg = ( + "CRT transfer client is configured but is missing minimum CRT " + f"version. CRT installed: {HAS_CRT}" + ) + if HAS_CRT: + msg += f", with version: {awscrt.__version__}" + raise Exception(msg) + if ( is_optimized_instance and pref_transfer_client == constants.AUTO_RESOLVE_TRANSFER_CLIENT - ) or ( - has_min_crt and pref_transfer_client == constants.CRT_TRANSFER_CLIENT - ): + ) or pref_transfer_client == constants.CRT_TRANSFER_CLIENT: logger.debug( "Attempting to use CRTTransferManager. Config settings may be ignored." ) diff --git a/tests/functional/test_crt.py b/tests/functional/test_crt.py index 48ac950c39..f9fd55fa33 100644 --- a/tests/functional/test_crt.py +++ b/tests/functional/test_crt.py @@ -77,6 +77,28 @@ def test_create_transfer_manager_with_crt_preferred(self): transfer_manager = create_transfer_manager(client, config) assert isinstance(transfer_manager, CRTTransferManager) + @mock.patch("boto3.s3.transfer.HAS_CRT", False) + def test_create_transfer_manager_with_crt_preferred_no_crt(self): + client = create_mock_client() + config = TransferConfig( + preferred_transfer_client='crt', + ) + with pytest.raises(Exception) as exc: + create_transfer_manager(client, config) + assert "missing minimum CRT" in str(exc.value) + + @requires_crt() + @mock.patch("awscrt.__version__", "0.19.0") + def test_create_transfer_manager_with_crt_preferred_bad_version(self): + client = create_mock_client() + config = TransferConfig( + preferred_transfer_client='crt', + ) + with pytest.raises(Exception) as exc: + create_transfer_manager(client, config) + assert "missing minimum CRT" in str(exc.value) + assert "with version: 0.19.0" in str(exc.value) + @requires_crt() def test_minimum_crt_version(self): assert has_minimum_crt_version((0, 16, 12)) is True From 893c6a824036a095f79cc1812befac2cf4466e55 Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Thu, 2 Oct 2025 14:58:16 -0400 Subject: [PATCH 03/14] Update raised exception --- boto3/exceptions.py | 4 ++++ boto3/s3/transfer.py | 8 ++++++-- tests/functional/test_crt.py | 5 +++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/boto3/exceptions.py b/boto3/exceptions.py index 7d9ceaf18d..70643b5958 100644 --- a/boto3/exceptions.py +++ b/boto3/exceptions.py @@ -124,3 +124,7 @@ class PythonDeprecationWarning(Warning): """ pass + + +class MissingMinimumCrtVersionError(Boto3Error): + pass diff --git a/boto3/s3/transfer.py b/boto3/s3/transfer.py index 086a06f3d4..5107e2d10c 100644 --- a/boto3/s3/transfer.py +++ b/boto3/s3/transfer.py @@ -139,7 +139,11 @@ def __call__(self, bytes_amount): from s3transfer.utils import OSUtils import boto3.s3.constants as constants -from boto3.exceptions import RetriesExceededError, S3UploadFailedError +from boto3.exceptions import ( + MissingMinimumCrtVersionError, + RetriesExceededError, + S3UploadFailedError, +) if HAS_CRT: import awscrt.s3 @@ -198,7 +202,7 @@ def _should_use_crt(config): ) if HAS_CRT: msg += f", with version: {awscrt.__version__}" - raise Exception(msg) + raise MissingMinimumCrtVersionError(msg) if ( is_optimized_instance diff --git a/tests/functional/test_crt.py b/tests/functional/test_crt.py index f9fd55fa33..b3acf870ca 100644 --- a/tests/functional/test_crt.py +++ b/tests/functional/test_crt.py @@ -17,6 +17,7 @@ from botocore.compat import HAS_CRT from botocore.credentials import Credentials +from boto3.exceptions import MissingMinimumCrtVersionError from boto3.s3.transfer import ( TransferConfig, create_transfer_manager, @@ -83,7 +84,7 @@ def test_create_transfer_manager_with_crt_preferred_no_crt(self): config = TransferConfig( preferred_transfer_client='crt', ) - with pytest.raises(Exception) as exc: + with pytest.raises(MissingMinimumCrtVersionError) as exc: create_transfer_manager(client, config) assert "missing minimum CRT" in str(exc.value) @@ -94,7 +95,7 @@ def test_create_transfer_manager_with_crt_preferred_bad_version(self): config = TransferConfig( preferred_transfer_client='crt', ) - with pytest.raises(Exception) as exc: + with pytest.raises(MissingMinimumCrtVersionError) as exc: create_transfer_manager(client, config) assert "missing minimum CRT" in str(exc.value) assert "with version: 0.19.0" in str(exc.value) From fb365d4dff90b494edfa91e03f2a396a31f2cbf4 Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Fri, 7 Nov 2025 12:22:24 -0500 Subject: [PATCH 04/14] Expose CRT config options --- boto3/crt.py | 38 +++++++++++++++++++++++++++++++++++- boto3/s3/transfer.py | 46 ++++++++++++++++++++++++++++++++++---------- 2 files changed, 73 insertions(+), 11 deletions(-) diff --git a/boto3/crt.py b/boto3/crt.py index 4b8df3140e..79368e09df 100644 --- a/boto3/crt.py +++ b/boto3/crt.py @@ -31,6 +31,8 @@ create_s3_crt_client, ) +from boto3.s3.constants import CRT_TRANSFER_CLIENT + # Singletons for CRT-backed transfers CRT_S3_CLIENT = None BOTOCORE_CRT_SERIALIZER = None @@ -39,6 +41,15 @@ PROCESS_LOCK_NAME = 'boto3' +ALLOWED_CRT_TRANSFER_CONFIG_OPTIONS = { + 'multipart_threshold', + 'max_concurrency', + 'max_request_concurrency', + 'multipart_chunksize', + 'preferred_transfer_client', +} + + def _create_crt_client(session, config, region_name, cred_provider): """Create a CRT S3 Client for file transfer. @@ -157,11 +168,36 @@ def compare_identity(boto3_creds, crt_s3_creds): return is_matching_identity +def _validate_crt_transfer_config(config): + # CRT client can also be configured via `AUTO_RESOLVE_TRANSFER_CLIENT` + # but it predates this validation. We only validate against CRT client + # configured via `CRT_TRANSFER_CLIENT` to preserve compatibility. + if config.preferred_transfer_client != CRT_TRANSFER_CLIENT: + return + invalid_crt_args = [] + for param in config.DEFAULTS.keys(): + val = config.get_deep_attr(param) + if ( + param not in ALLOWED_CRT_TRANSFER_CONFIG_OPTIONS + and val is not config.UNSET_DEFAULT + ): + invalid_crt_args.append(param) + if len(invalid_crt_args) > 0: + raise ValueError( + "The following transfer config options are invalid " + "when `preferred_transfer_client` is set to `crt`: `" + f"{'`, `'.join(invalid_crt_args)}`" + ) + + def create_crt_transfer_manager(client, config): """Create a CRTTransferManager for optimized data transfer.""" crt_s3_client = get_crt_s3_client(client, config) if is_crt_compatible_request(client, crt_s3_client): + _validate_crt_transfer_config(config) return CRTTransferManager( - crt_s3_client.crt_client, BOTOCORE_CRT_SERIALIZER + crt_s3_client.crt_client, + BOTOCORE_CRT_SERIALIZER, + config=config, ) return None diff --git a/boto3/s3/transfer.py b/boto3/s3/transfer.py index 5107e2d10c..77971ae399 100644 --- a/boto3/s3/transfer.py +++ b/boto3/s3/transfer.py @@ -249,18 +249,31 @@ class TransferConfig(S3TransferConfig): 'max_concurrency': 'max_request_concurrency', 'max_io_queue': 'max_io_queue_size', } + DEFAULTS = { + 'multipart_threshold': 8 * MB, + 'max_concurrency': 10, + 'max_request_concurrency': 10, + 'multipart_chunksize': 8 * MB, + 'num_download_attempts': 5, + 'max_io_queue': 100, + 'max_io_queue_size': 100, + 'io_chunksize': 256 * KB, + 'use_threads': True, + 'max_bandwidth': None, + 'preferred_transfer_client': constants.AUTO_RESOLVE_TRANSFER_CLIENT, + } def __init__( self, - multipart_threshold=8 * MB, - max_concurrency=10, - multipart_chunksize=8 * MB, - num_download_attempts=5, - max_io_queue=100, - io_chunksize=256 * KB, - use_threads=True, - max_bandwidth=None, - preferred_transfer_client=constants.AUTO_RESOLVE_TRANSFER_CLIENT, + multipart_threshold=S3TransferConfig.UNSET_DEFAULT, + max_concurrency=S3TransferConfig.UNSET_DEFAULT, + multipart_chunksize=S3TransferConfig.UNSET_DEFAULT, + num_download_attempts=S3TransferConfig.UNSET_DEFAULT, + max_io_queue=S3TransferConfig.UNSET_DEFAULT, + io_chunksize=S3TransferConfig.UNSET_DEFAULT, + use_threads=S3TransferConfig.UNSET_DEFAULT, + max_bandwidth=S3TransferConfig.UNSET_DEFAULT, + preferred_transfer_client=S3TransferConfig.UNSET_DEFAULT, ): """Configuration object for managed S3 transfers @@ -325,7 +338,11 @@ def __init__( # S3TransferConfig so we add aliases so you can still access the # old version of the names. for alias in self.ALIAS: - setattr(self, alias, getattr(self, self.ALIAS[alias])) + setattr( + self, + alias, + object.__getattribute__(self, self.ALIAS[alias]), + ) self.use_threads = use_threads self.preferred_transfer_client = preferred_transfer_client @@ -337,6 +354,15 @@ def __setattr__(self, name, value): # Always set the value of the actual name provided. super().__setattr__(name, value) + def __getattribute__(self, item): + value = object.__getattribute__(self, item) + defaults = object.__getattribute__(self, 'DEFAULTS') + if item not in defaults: + return value + if value is self.UNSET_DEFAULT: + return self.DEFAULTS[item] + return value + class S3Transfer: ALLOWED_DOWNLOAD_ARGS = TransferManager.ALLOWED_DOWNLOAD_ARGS From 5834f128413bfb336ce919cc10332b86e10eba22 Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Mon, 10 Nov 2025 11:29:53 -0500 Subject: [PATCH 05/14] Add tests for config and validation --- boto3/crt.py | 9 ++++-- boto3/exceptions.py | 4 +++ tests/functional/test_crt.py | 32 +++++++++++++++++++- tests/unit/s3/test_transfer.py | 54 ++++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 4 deletions(-) diff --git a/boto3/crt.py b/boto3/crt.py index 79368e09df..8855c0999e 100644 --- a/boto3/crt.py +++ b/boto3/crt.py @@ -31,6 +31,7 @@ create_s3_crt_client, ) +from boto3.exceptions import InvalidCrtTransferConfigError from boto3.s3.constants import CRT_TRANSFER_CLIENT # Singletons for CRT-backed transfers @@ -41,7 +42,7 @@ PROCESS_LOCK_NAME = 'boto3' -ALLOWED_CRT_TRANSFER_CONFIG_OPTIONS = { +_ALLOWED_CRT_TRANSFER_CONFIG_OPTIONS = { 'multipart_threshold', 'max_concurrency', 'max_request_concurrency', @@ -169,6 +170,8 @@ def compare_identity(boto3_creds, crt_s3_creds): def _validate_crt_transfer_config(config): + if config is None: + return # CRT client can also be configured via `AUTO_RESOLVE_TRANSFER_CLIENT` # but it predates this validation. We only validate against CRT client # configured via `CRT_TRANSFER_CLIENT` to preserve compatibility. @@ -178,12 +181,12 @@ def _validate_crt_transfer_config(config): for param in config.DEFAULTS.keys(): val = config.get_deep_attr(param) if ( - param not in ALLOWED_CRT_TRANSFER_CONFIG_OPTIONS + param not in _ALLOWED_CRT_TRANSFER_CONFIG_OPTIONS and val is not config.UNSET_DEFAULT ): invalid_crt_args.append(param) if len(invalid_crt_args) > 0: - raise ValueError( + raise InvalidCrtTransferConfigError( "The following transfer config options are invalid " "when `preferred_transfer_client` is set to `crt`: `" f"{'`, `'.join(invalid_crt_args)}`" diff --git a/boto3/exceptions.py b/boto3/exceptions.py index 70643b5958..7bc78475c6 100644 --- a/boto3/exceptions.py +++ b/boto3/exceptions.py @@ -128,3 +128,7 @@ class PythonDeprecationWarning(Warning): class MissingMinimumCrtVersionError(Boto3Error): pass + + +class InvalidCrtTransferConfigError(Boto3Error): + pass diff --git a/tests/functional/test_crt.py b/tests/functional/test_crt.py index b3acf870ca..f1d3072bfe 100644 --- a/tests/functional/test_crt.py +++ b/tests/functional/test_crt.py @@ -17,7 +17,10 @@ from botocore.compat import HAS_CRT from botocore.credentials import Credentials -from boto3.exceptions import MissingMinimumCrtVersionError +from boto3.exceptions import ( + InvalidCrtTransferConfigError, + MissingMinimumCrtVersionError, +) from boto3.s3.transfer import ( TransferConfig, create_transfer_manager, @@ -119,3 +122,30 @@ def test_minimum_crt_version_bad_crt_version(self, bad_version): vers.return_value = bad_version assert has_minimum_crt_version((0, 16, 12)) is False + + @requires_crt() + def test_crt_transfer_manager_raises_with_invalid_crt_config(self): + client = create_mock_client() + config = TransferConfig( + preferred_transfer_client='crt', + # `max_bandwidth` is not an allowed CRT config option. + max_bandwidth=1024, + ) + with pytest.raises(InvalidCrtTransferConfigError) as exc: + create_transfer_manager(client, config) + assert "transfer config options are invalid" in str(exc.value) + assert "max_bandwidth" in str(exc.value) + + @requires_crt() + @MockOptimizedInstance() + def test_auto_transfer_manager_succeeds_with_invalid_crt_config(self): + client = create_mock_client() + config = TransferConfig( + preferred_transfer_client='auto', + # `max_bandwidth` is not an allowed CRT config option. + # But config should only be validated when + # `preferred_transfer_client` == `crt`. + max_bandwidth=1024, + ) + transfer_manager = create_transfer_manager(client, config) + assert isinstance(transfer_manager, CRTTransferManager) diff --git a/tests/unit/s3/test_transfer.py b/tests/unit/s3/test_transfer.py index dfc9d38159..b627ae2852 100644 --- a/tests/unit/s3/test_transfer.py +++ b/tests/unit/s3/test_transfer.py @@ -76,6 +76,17 @@ def test_create_transfer_manager_with_default_config(self): client, config, None, None ) + def test_classic_transfer_manager_succeeds_with_invalid_crt_config(self): + client = create_mock_client() + config = TransferConfig( + preferred_transfer_client="classic", + # `max_bandwidth` is not an allowed CRT config option. + max_bandwidth=MB, + ) + with mock.patch('boto3.s3.transfer.TransferManager') as manager: + create_transfer_manager(client, config) + assert manager.call_args == mock.call(client, config, None, None) + class TestTransferConfig(unittest.TestCase): def assert_value_of_actual_and_alias( @@ -174,6 +185,49 @@ def test_transferconfig_copy(self): == copied_config.preferred_transfer_client ) + def test_unset_default(self): + config = TransferConfig() + defaults = TransferConfig.DEFAULTS + + # Assert top-level params and aliases return defaults + # when accessed via public property. + assert config.multipart_threshold == defaults['multipart_threshold'] + assert config.max_concurrency == defaults['max_concurrency'] + assert ( + config.max_request_concurrency + == defaults['max_request_concurrency'] + ) + assert config.multipart_chunksize == defaults['multipart_chunksize'] + assert ( + config.num_download_attempts == defaults['num_download_attempts'] + ) + assert config.max_io_queue == defaults['max_io_queue'] + assert config.max_io_queue_size == defaults['max_io_queue_size'] + assert config.io_chunksize == defaults['io_chunksize'] + assert config.use_threads == defaults['use_threads'] + assert config.max_bandwidth == defaults['max_bandwidth'] + assert ( + config.preferred_transfer_client + == defaults['preferred_transfer_client'] + ) + # Assert top-level params and aliases return UNSET_DEFAULT + # when directly accessed. + for param in defaults: + assert config.get_deep_attr(param) == TransferConfig.UNSET_DEFAULT + + def test_explicit_config_values(self): + config = TransferConfig( + multipart_threshold=4 * MB, + ) + config.max_concurrency = 1 + + assert config.multipart_threshold == 4 * MB + assert config.max_concurrency == 1 + assert config.max_request_concurrency == 1 + assert config.get_deep_attr('multipart_threshold') == 4 * MB + assert config.get_deep_attr('max_concurrency') == 1 + assert config.get_deep_attr('max_request_concurrency') == 1 + class TestProgressCallbackInvoker(unittest.TestCase): def test_on_progress(self): From 52f1641444be8eb5a4450248dad0930ceef5eb9f Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Mon, 10 Nov 2025 16:25:21 -0500 Subject: [PATCH 06/14] Add changelog entry --- .changes/next-release/enhancement-s3-75585.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changes/next-release/enhancement-s3-75585.json diff --git a/.changes/next-release/enhancement-s3-75585.json b/.changes/next-release/enhancement-s3-75585.json new file mode 100644 index 0000000000..56d1e5b5db --- /dev/null +++ b/.changes/next-release/enhancement-s3-75585.json @@ -0,0 +1,5 @@ +{ + "type": "enhancement", + "category": "``s3``", + "description": "Adds partial ``TransferConfig`` support for CRT transfer managers." +} From d03c59268d10e71f273053604e7918904778fcb5 Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Tue, 11 Nov 2025 11:37:22 -0500 Subject: [PATCH 07/14] small change --- boto3/s3/transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boto3/s3/transfer.py b/boto3/s3/transfer.py index 77971ae399..2611818b79 100644 --- a/boto3/s3/transfer.py +++ b/boto3/s3/transfer.py @@ -360,7 +360,7 @@ def __getattribute__(self, item): if item not in defaults: return value if value is self.UNSET_DEFAULT: - return self.DEFAULTS[item] + return defaults[item] return value From 1332c33be16fd62bd3e28798349f7aa639996d6a Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Fri, 21 Nov 2025 10:41:55 -0500 Subject: [PATCH 08/14] Use botocore MissingDependencyException --- boto3/exceptions.py | 4 ---- boto3/s3/transfer.py | 5 ++--- tests/functional/test_crt.py | 6 +++--- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/boto3/exceptions.py b/boto3/exceptions.py index 7bc78475c6..1dbe37dfe0 100644 --- a/boto3/exceptions.py +++ b/boto3/exceptions.py @@ -126,9 +126,5 @@ class PythonDeprecationWarning(Warning): pass -class MissingMinimumCrtVersionError(Boto3Error): - pass - - class InvalidCrtTransferConfigError(Boto3Error): pass diff --git a/boto3/s3/transfer.py b/boto3/s3/transfer.py index 2611818b79..9a1391f211 100644 --- a/boto3/s3/transfer.py +++ b/boto3/s3/transfer.py @@ -128,7 +128,7 @@ def __call__(self, bytes_amount): from os import PathLike, fspath, getpid from botocore.compat import HAS_CRT -from botocore.exceptions import ClientError +from botocore.exceptions import ClientError, MissingDependencyException from s3transfer.exceptions import ( RetriesExceededError as S3TransferRetriesExceededError, ) @@ -140,7 +140,6 @@ def __call__(self, bytes_amount): import boto3.s3.constants as constants from boto3.exceptions import ( - MissingMinimumCrtVersionError, RetriesExceededError, S3UploadFailedError, ) @@ -202,7 +201,7 @@ def _should_use_crt(config): ) if HAS_CRT: msg += f", with version: {awscrt.__version__}" - raise MissingMinimumCrtVersionError(msg) + raise MissingDependencyException(msg=msg) if ( is_optimized_instance diff --git a/tests/functional/test_crt.py b/tests/functional/test_crt.py index f1d3072bfe..aa0718797b 100644 --- a/tests/functional/test_crt.py +++ b/tests/functional/test_crt.py @@ -16,10 +16,10 @@ import pytest from botocore.compat import HAS_CRT from botocore.credentials import Credentials +from botocore.exceptions import MissingDependencyException from boto3.exceptions import ( InvalidCrtTransferConfigError, - MissingMinimumCrtVersionError, ) from boto3.s3.transfer import ( TransferConfig, @@ -87,7 +87,7 @@ def test_create_transfer_manager_with_crt_preferred_no_crt(self): config = TransferConfig( preferred_transfer_client='crt', ) - with pytest.raises(MissingMinimumCrtVersionError) as exc: + with pytest.raises(MissingDependencyException) as exc: create_transfer_manager(client, config) assert "missing minimum CRT" in str(exc.value) @@ -98,7 +98,7 @@ def test_create_transfer_manager_with_crt_preferred_bad_version(self): config = TransferConfig( preferred_transfer_client='crt', ) - with pytest.raises(MissingMinimumCrtVersionError) as exc: + with pytest.raises(MissingDependencyException) as exc: create_transfer_manager(client, config) assert "missing minimum CRT" in str(exc.value) assert "with version: 0.19.0" in str(exc.value) From f360749414a177379e8e87e133d3aeadbda7ebf6 Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Fri, 21 Nov 2025 14:12:09 -0500 Subject: [PATCH 09/14] Specify unsupported CRT config options --- boto3/s3/transfer.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/boto3/s3/transfer.py b/boto3/s3/transfer.py index 9a1391f211..8a47ba1da1 100644 --- a/boto3/s3/transfer.py +++ b/boto3/s3/transfer.py @@ -296,23 +296,30 @@ def __init__( Other retryable exceptions such as throttling errors and 5xx errors are already retried by botocore (this default is 5). This does not take into account the number of exceptions retried by - botocore. + botocore. Note: This value is ignored when resolved transfer + manager type is CRTTransferManager. :param max_io_queue: The maximum amount of read parts that can be queued in memory to be written for a download. The size of each of these read parts is at most the size of ``io_chunksize``. + Note: This value is ignored when resolved transfer manager type + is CRTTransferManager. :param io_chunksize: The max size of each chunk in the io queue. Currently, this is size used when ``read`` is called on the - downloaded stream as well. + downloaded stream as well. Note: This value is ignored when + resolved transfer manager type is CRTTransferManager. :param use_threads: If True, threads will be used when performing S3 transfers. If False, no threads will be used in performing transfers; all logic will be run in the current thread. + Note: This value is ignored when resolved transfer manager type is + CRTTransferManager. :param max_bandwidth: The maximum bandwidth that will be consumed in uploading and downloading file content. The value is an integer - in terms of bytes per second. + in terms of bytes per second. Note: This value is ignored when + resolved transfer manager type is CRTTransferManager. :param preferred_transfer_client: String specifying preferred transfer client for transfer operations. From fcde56835beed1f19483a2874ce54c943915e446 Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Mon, 24 Nov 2025 10:38:31 -0500 Subject: [PATCH 10/14] Bump minimum CRT version to 0.29.0 --- boto3/s3/transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boto3/s3/transfer.py b/boto3/s3/transfer.py index 8a47ba1da1..6327a45c69 100644 --- a/boto3/s3/transfer.py +++ b/boto3/s3/transfer.py @@ -187,7 +187,7 @@ def create_transfer_manager(client, config, osutil=None): def _should_use_crt(config): # This feature requires awscrt>=0.19.18 - has_min_crt = HAS_CRT and has_minimum_crt_version((0, 19, 18)) + has_min_crt = HAS_CRT and has_minimum_crt_version((0, 29, 0)) is_optimized_instance = has_min_crt and awscrt.s3.is_optimized_for_system() pref_transfer_client = config.preferred_transfer_client.lower() From 319306ce7e07fb697eb519c69c1f5c961dd8615f Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Mon, 24 Nov 2025 15:59:21 -0500 Subject: [PATCH 11/14] Reset min CRT validation --- boto3/s3/transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boto3/s3/transfer.py b/boto3/s3/transfer.py index 6327a45c69..8a47ba1da1 100644 --- a/boto3/s3/transfer.py +++ b/boto3/s3/transfer.py @@ -187,7 +187,7 @@ def create_transfer_manager(client, config, osutil=None): def _should_use_crt(config): # This feature requires awscrt>=0.19.18 - has_min_crt = HAS_CRT and has_minimum_crt_version((0, 29, 0)) + has_min_crt = HAS_CRT and has_minimum_crt_version((0, 19, 18)) is_optimized_instance = has_min_crt and awscrt.s3.is_optimized_for_system() pref_transfer_client = config.preferred_transfer_client.lower() From 370042e690a7cfa6ae5f06d401e984e0fd879769 Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Tue, 25 Nov 2025 16:09:27 -0500 Subject: [PATCH 12/14] Provide compat with older s3transfer versions --- boto3/compat.py | 5 ++++ boto3/crt.py | 15 ++++++----- boto3/s3/transfer.py | 62 +++++++++++++++++++++++++++++++------------- 3 files changed, 58 insertions(+), 24 deletions(-) diff --git a/boto3/compat.py b/boto3/compat.py index e5f09911f1..44a6431caf 100644 --- a/boto3/compat.py +++ b/boto3/compat.py @@ -18,6 +18,8 @@ from boto3.exceptions import PythonDeprecationWarning +from s3transfer.manager import TransferConfig + # In python3, socket.error is OSError, which is too general # for what we want (i.e FileNotFoundError is a subclass of OSError). # In py3 all the socket related errors are in a newly created @@ -29,6 +31,9 @@ import collections.abc as collections_abc +TRANSFER_CONFIG_SUPPORTS_CRT = hasattr(TransferConfig, 'UNSET_DEFAULT') + + if sys.platform.startswith('win'): def rename_file(current_filename, new_filename): try: diff --git a/boto3/crt.py b/boto3/crt.py index 8855c0999e..cadd3fe7c4 100644 --- a/boto3/crt.py +++ b/boto3/crt.py @@ -31,6 +31,7 @@ create_s3_crt_client, ) +from boto3.compat import TRANSFER_CONFIG_SUPPORTS_CRT from boto3.exceptions import InvalidCrtTransferConfigError from boto3.s3.constants import CRT_TRANSFER_CLIENT @@ -197,10 +198,12 @@ def create_crt_transfer_manager(client, config): """Create a CRTTransferManager for optimized data transfer.""" crt_s3_client = get_crt_s3_client(client, config) if is_crt_compatible_request(client, crt_s3_client): - _validate_crt_transfer_config(config) - return CRTTransferManager( - crt_s3_client.crt_client, - BOTOCORE_CRT_SERIALIZER, - config=config, - ) + crt_transfer_manager_kwargs = { + 'crt_s3_client': crt_s3_client.crt_client, + 'crt_request_serializer': BOTOCORE_CRT_SERIALIZER, + } + if TRANSFER_CONFIG_SUPPORTS_CRT: + _validate_crt_transfer_config(config) + crt_transfer_manager_kwargs['config'] = config + return CRTTransferManager(**crt_transfer_manager_kwargs) return None diff --git a/boto3/s3/transfer.py b/boto3/s3/transfer.py index 8a47ba1da1..c42ae4b178 100644 --- a/boto3/s3/transfer.py +++ b/boto3/s3/transfer.py @@ -139,6 +139,7 @@ def __call__(self, bytes_amount): from s3transfer.utils import OSUtils import boto3.s3.constants as constants +from boto3.compat import TRANSFER_CONFIG_SUPPORTS_CRT from boto3.exceptions import ( RetriesExceededError, S3UploadFailedError, @@ -264,15 +265,15 @@ class TransferConfig(S3TransferConfig): def __init__( self, - multipart_threshold=S3TransferConfig.UNSET_DEFAULT, - max_concurrency=S3TransferConfig.UNSET_DEFAULT, - multipart_chunksize=S3TransferConfig.UNSET_DEFAULT, - num_download_attempts=S3TransferConfig.UNSET_DEFAULT, - max_io_queue=S3TransferConfig.UNSET_DEFAULT, - io_chunksize=S3TransferConfig.UNSET_DEFAULT, - use_threads=S3TransferConfig.UNSET_DEFAULT, - max_bandwidth=S3TransferConfig.UNSET_DEFAULT, - preferred_transfer_client=S3TransferConfig.UNSET_DEFAULT, + multipart_threshold=None, + max_concurrency=None, + multipart_chunksize=None, + num_download_attempts=None, + max_io_queue=None, + io_chunksize=None, + use_threads=None, + max_bandwidth=None, + preferred_transfer_client=None, ): """Configuration object for managed S3 transfers @@ -331,14 +332,26 @@ def __init__( requests. Disables possible CRT upgrade on requests. * crt - Only use the CRTTransferManager with requests. """ + init_args = { + 'multipart_threshold': multipart_threshold, + 'max_concurrency': max_concurrency, + 'multipart_chunksize': multipart_chunksize, + 'num_download_attempts': num_download_attempts, + 'max_io_queue': max_io_queue, + 'io_chunksize': io_chunksize, + 'use_threads': use_threads, + 'max_bandwidth': max_bandwidth, + 'preferred_transfer_client': preferred_transfer_client, + } + resolved = self._resolve_init_args(init_args) super().__init__( - multipart_threshold=multipart_threshold, - max_request_concurrency=max_concurrency, - multipart_chunksize=multipart_chunksize, - num_download_attempts=num_download_attempts, - max_io_queue_size=max_io_queue, - io_chunksize=io_chunksize, - max_bandwidth=max_bandwidth, + multipart_threshold=resolved['multipart_threshold'], + max_request_concurrency=resolved['max_concurrency'], + multipart_chunksize=resolved['multipart_chunksize'], + num_download_attempts=resolved['num_download_attempts'], + max_io_queue_size=resolved['max_io_queue'], + io_chunksize=resolved['io_chunksize'], + max_bandwidth=resolved['max_bandwidth'], ) # Some of the argument names are not the same as the inherited # S3TransferConfig so we add aliases so you can still access the @@ -349,8 +362,8 @@ def __init__( alias, object.__getattribute__(self, self.ALIAS[alias]), ) - self.use_threads = use_threads - self.preferred_transfer_client = preferred_transfer_client + self.use_threads = resolved['use_threads'] + self.preferred_transfer_client = resolved['preferred_transfer_client'] def __setattr__(self, name, value): # If the alias name is used, make sure we set the name that it points @@ -362,6 +375,8 @@ def __setattr__(self, name, value): def __getattribute__(self, item): value = object.__getattribute__(self, item) + if not TRANSFER_CONFIG_SUPPORTS_CRT: + return value defaults = object.__getattribute__(self, 'DEFAULTS') if item not in defaults: return value @@ -369,6 +384,17 @@ def __getattribute__(self, item): return defaults[item] return value + def _resolve_init_args(self, init_args): + resolved = {} + for init_arg, val in init_args.items(): + if val is not None: + resolved[init_arg] = val + elif TRANSFER_CONFIG_SUPPORTS_CRT: + resolved[init_arg] = self.UNSET_DEFAULT + else: + resolved[init_arg] = self.DEFAULTS[init_arg] + return resolved + class S3Transfer: ALLOWED_DOWNLOAD_ARGS = TransferManager.ALLOWED_DOWNLOAD_ARGS From 03ea9ea162ca0da83c4f471e23ce68695ba58626 Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Wed, 26 Nov 2025 10:34:20 -0500 Subject: [PATCH 13/14] Add warning --- boto3/crt.py | 8 ++++++++ tests/unit/test_crt.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/boto3/crt.py b/boto3/crt.py index cadd3fe7c4..093ed3f089 100644 --- a/boto3/crt.py +++ b/boto3/crt.py @@ -19,6 +19,7 @@ contained within are subject to abrupt breaking changes. """ +import logging import threading import botocore.exceptions @@ -35,6 +36,8 @@ from boto3.exceptions import InvalidCrtTransferConfigError from boto3.s3.constants import CRT_TRANSFER_CLIENT +logger = logging.getLogger(__name__) + # Singletons for CRT-backed transfers CRT_S3_CLIENT = None BOTOCORE_CRT_SERIALIZER = None @@ -205,5 +208,10 @@ def create_crt_transfer_manager(client, config): if TRANSFER_CONFIG_SUPPORTS_CRT: _validate_crt_transfer_config(config) crt_transfer_manager_kwargs['config'] = config + if not TRANSFER_CONFIG_SUPPORTS_CRT and config: + logger.warning( + 'Using TransferConfig with CRT client requires ' + 's3transfer >= 0.16.0, configured values will be ignored.' + ) return CRTTransferManager(**crt_transfer_manager_kwargs) return None diff --git a/tests/unit/test_crt.py b/tests/unit/test_crt.py index c88eb36fc2..b0d60c7952 100644 --- a/tests/unit/test_crt.py +++ b/tests/unit/test_crt.py @@ -213,3 +213,21 @@ def test_get_crt_s3_client_w_wrong_region( ) assert use1_crt_s3_client is crt_s3_client assert use1_crt_s3_client.region == "us-west-2" + + @requires_crt() + @mock.patch('boto3.crt.TRANSFER_CONFIG_SUPPORTS_CRT', False) + def test_config_without_crt_support_emits_warning( + self, + mock_crt_process_lock, + mock_crt_client_singleton, + mock_serializer_singleton, + caplog, + ): + config = TransferConfig() + boto3.crt.create_crt_transfer_manager(USW2_S3_CLIENT, config) + assert any( + [ + 'requires s3transfer >= 0.16.0' in r.message + for r in caplog.records + ] + ) From 5c7a036ea590e66c3c87c61930dca793d6aaa5a4 Mon Sep 17 00:00:00 2001 From: Steve Yoo Date: Wed, 26 Nov 2025 12:35:53 -0500 Subject: [PATCH 14/14] Remove backticks from error msg --- boto3/crt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/boto3/crt.py b/boto3/crt.py index 093ed3f089..fb30401d5b 100644 --- a/boto3/crt.py +++ b/boto3/crt.py @@ -192,8 +192,8 @@ def _validate_crt_transfer_config(config): if len(invalid_crt_args) > 0: raise InvalidCrtTransferConfigError( "The following transfer config options are invalid " - "when `preferred_transfer_client` is set to `crt`: `" - f"{'`, `'.join(invalid_crt_args)}`" + "when preferred_transfer_client is set to crt: " + f"{', '.join(invalid_crt_args)}`" )