Expose CRT config options

Author: hssyoo

boto3/crt.py

@@ -31,6 +31,8 @@
     create_s3_crt_client,
 )
 
+from boto3.s3.constants import CRT_TRANSFER_CLIENT
+
 # Singletons for CRT-backed transfers
 CRT_S3_CLIENT = None
 BOTOCORE_CRT_SERIALIZER = None
@@ -39,6 +41,15 @@
 PROCESS_LOCK_NAME = 'boto3'
 
 
+ALLOWED_CRT_TRANSFER_CONFIG_OPTIONS = {
+    'multipart_threshold',
+    'max_concurrency',
+    'max_request_concurrency',
+    'multipart_chunksize',
+    'preferred_transfer_client',
+}
+
+
 def _create_crt_client(session, config, region_name, cred_provider):
     """Create a CRT S3 Client for file transfer.
 
@@ -157,11 +168,36 @@ def compare_identity(boto3_creds, crt_s3_creds):
     return is_matching_identity
 
 
+def _validate_crt_transfer_config(config):
+    # CRT client can also be configured via `AUTO_RESOLVE_TRANSFER_CLIENT`
+    # but it predates this validation. We only validate against CRT client
+    # configured via `CRT_TRANSFER_CLIENT` to preserve compatibility.
+    if config.preferred_transfer_client != CRT_TRANSFER_CLIENT:
+        return
+    invalid_crt_args = []
+    for param in config.DEFAULTS.keys():
+        val = config.get_deep_attr(param)
+        if (
+            param not in ALLOWED_CRT_TRANSFER_CONFIG_OPTIONS
+            and val is not config.UNSET_DEFAULT
+        ):
+            invalid_crt_args.append(param)
+    if len(invalid_crt_args) > 0:
+        raise ValueError(
+            "The following transfer config options are invalid "
+            "when `preferred_transfer_client` is set to `crt`: `"
+            f"{'`, `'.join(invalid_crt_args)}`"
+        )
+
+
 def create_crt_transfer_manager(client, config):
     """Create a CRTTransferManager for optimized data transfer."""
     crt_s3_client = get_crt_s3_client(client, config)
     if is_crt_compatible_request(client, crt_s3_client):
+        _validate_crt_transfer_config(config)
         return CRTTransferManager(
-            crt_s3_client.crt_client, BOTOCORE_CRT_SERIALIZER
+            crt_s3_client.crt_client,
+            BOTOCORE_CRT_SERIALIZER,
+            config=config,
         )
     return None

boto3/s3/transfer.py

@@ -249,18 +249,31 @@ class TransferConfig(S3TransferConfig):
         'max_concurrency': 'max_request_concurrency',
         'max_io_queue': 'max_io_queue_size',
     }
+    DEFAULTS = {
+        'multipart_threshold': 8 * MB,
+        'max_concurrency': 10,
+        'max_request_concurrency': 10,
+        'multipart_chunksize': 8 * MB,
+        'num_download_attempts': 5,
+        'max_io_queue': 100,
+        'max_io_queue_size': 100,
+        'io_chunksize': 256 * KB,
+        'use_threads': True,
+        'max_bandwidth': None,
+        'preferred_transfer_client': constants.AUTO_RESOLVE_TRANSFER_CLIENT,
+    }
 
     def __init__(
         self,
-        multipart_threshold=8 * MB,
-        max_concurrency=10,
-        multipart_chunksize=8 * MB,
-        num_download_attempts=5,
-        max_io_queue=100,
-        io_chunksize=256 * KB,
-        use_threads=True,
-        max_bandwidth=None,
-        preferred_transfer_client=constants.AUTO_RESOLVE_TRANSFER_CLIENT,
+        multipart_threshold=S3TransferConfig.UNSET_DEFAULT,
+        max_concurrency=S3TransferConfig.UNSET_DEFAULT,
+        multipart_chunksize=S3TransferConfig.UNSET_DEFAULT,
+        num_download_attempts=S3TransferConfig.UNSET_DEFAULT,
+        max_io_queue=S3TransferConfig.UNSET_DEFAULT,
+        io_chunksize=S3TransferConfig.UNSET_DEFAULT,
+        use_threads=S3TransferConfig.UNSET_DEFAULT,
+        max_bandwidth=S3TransferConfig.UNSET_DEFAULT,
+        preferred_transfer_client=S3TransferConfig.UNSET_DEFAULT,
     ):
         """Configuration object for managed S3 transfers
 
@@ -325,7 +338,11 @@ def __init__(
         # S3TransferConfig so we add aliases so you can still access the
         # old version of the names.
         for alias in self.ALIAS:
-            setattr(self, alias, getattr(self, self.ALIAS[alias]))
+            setattr(
+                self,
+                alias,
+                object.__getattribute__(self, self.ALIAS[alias]),
+            )
         self.use_threads = use_threads
         self.preferred_transfer_client = preferred_transfer_client
 
@@ -337,6 +354,15 @@ def __setattr__(self, name, value):
         # Always set the value of the actual name provided.
         super().__setattr__(name, value)
 
+    def __getattribute__(self, item):
+        value = object.__getattribute__(self, item)
+        defaults = object.__getattribute__(self, 'DEFAULTS')
+        if item not in defaults:
+            return value
+        if value is self.UNSET_DEFAULT:
+            return self.DEFAULTS[item]
+        return value
+
 
 class S3Transfer:
     ALLOWED_DOWNLOAD_ARGS = TransferManager.ALLOWED_DOWNLOAD_ARGS