Various code improvements

With this commit we reflect on the review process and improve code
and testing process.

.github/workflows/test.yml

@@ -34,26 +34,29 @@ jobs:
         env:
           cache-name: cache-certgen
         with:
-          path: ~/cache
+          path: ~/.minio/certs
           key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.cache-revision }}
-      - name: Download minio and certgen on cache miss
-        if: steps.cache-minio.outputs.cache-hit != 'true' || steps.cache-certgen.outputs.cache-hit != 'true'
+      - name: Download minio on cache miss
+        if: steps.cache-minio.outputs.cache-hit != 'true'
         run: |
           mkdir -p ~/cache
           test ! -e ~/cache/minio && wget -O ~/cache/minio https://dl.minio.io/server/minio/release/linux-amd64/minio || echo "Minio already in cache"
-          test ! -e ~/cache/certgen && wget -O ~/cache/certgen https://github.com/minio/certgen/releases/download/v0.0.2/certgen-linux-amd64 || echo "Certgen already in cache"
+      - name: Download certgen on cache miss
+        if: steps.cache-certgen.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p ~/.minio/certs
+          test ! -e ~/.minio/certs/certgen && wget -O ~/.minio/certs/certgen https://github.com/minio/certgen/releases/download/v0.0.2/certgen-linux-amd64 || echo "Certgen already in cache"
+          chmod +x ~/.minio/certs/certgen
+          cd ~/.minio/certs
+          ~/.minio/certs/certgen -ca -host "localhost"
       - name: Start a local instance of minio
         run: |
           export AWS_ACCESS_KEY_ID=Q3AM3UQ867SPQQA43P2F
           export AWS_SECRET_ACCESS_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
           export MINIO_ROOT_USER=Q3AM3UQ867SPQQA43P2F
           export MINIO_ROOT_PASSWORD=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
           chmod +x ~/cache/minio
-          chmod +x ~/cache/certgen
           mkdir -p ~/minio_tmp
-          mkdir -p ~/.minio/certs
-          cd ~/.minio/certs
-          ~/cache/certgen -ca -host "localhost,127.0.0.1,172.17.0.1"
           ~/cache/minio server ~/minio_tmp &
           sleep 4 # give minio some time to start
       - name: Run tests

S3/S3.py

@@ -705,14 +705,12 @@ def object_put(self, filename, uri, extra_headers = None, extra_label = ""):
             headers['x-amz-server-side-encryption-aws-kms-key-id'] = self.config.kms_key
 
         if self.config.sse_customer_key:
-            md5s = md5()
-            sse_customer_key = self.config.sse_customer_key.encode()
-            md5s.update(sse_customer_key)
-            md5_encoded = b64encode(md5s.digest())
+            sse_customer_key = encode_to_s3(self.config.sse_customer_key)
             key_encoded = b64encode(sse_customer_key)
+            md5_encoded = b64encode(md5(sse_customer_key).digest())
             headers["x-amz-server-side-encryption-customer-algorithm"] = "AES256"
-            headers["x-amz-server-side-encryption-customer-key"] = key_encoded.decode()
-            headers["x-amz-server-side-encryption-customer-key-md5"] = md5_encoded.decode()
+            headers["x-amz-server-side-encryption-customer-key"] = decode_from_s3(key_encoded)
+            headers["x-amz-server-side-encryption-customer-key-md5"] = decode_from_s3(md5_encoded)
 
         ## MIME-type handling
         headers["content-type"] = self.content_type(filename=filename)
@@ -784,14 +782,12 @@ def object_get(self, uri, stream, dest_name, extra_headers, start_position = 0,
             headers['x-amz-server-side-encryption-aws-kms-key-id'] = self.config.kms_key
 
         if self.config.sse_customer_key:
-            md5s = md5()
-            sse_customer_key = self.config.sse_customer_key.encode()
-            md5s.update(sse_customer_key)
-            md5_encoded = b64encode(md5s.digest())
+            sse_customer_key = encode_to_s3(self.config.sse_customer_key)
             key_encoded = b64encode(sse_customer_key)
+            md5_encoded = b64encode(md5(sse_customer_key).digest())
             headers["x-amz-server-side-encryption-customer-algorithm"] = "AES256"
-            headers["x-amz-server-side-encryption-customer-key"] = key_encoded.decode()
-            headers["x-amz-server-side-encryption-customer-key-md5"] = md5_encoded.decode()
+            headers["x-amz-server-side-encryption-customer-key"] = decode_from_s3(key_encoded)
+            headers["x-amz-server-side-encryption-customer-key-md5"] = decode_from_s3(md5_encoded)
 
         request = self.create_request("OBJECT_GET", uri = uri, headers=headers)
         labels = { 'source' : uri.uri(), 'destination' : dest_name, 'extra' : extra_label }
@@ -990,14 +986,12 @@ def object_copy(self, src_uri, dst_uri, extra_headers=None,
                 self.config.kms_key
 
         if self.config.sse_copy_source_customer_key:
-            md5s = md5()
-            sse_copy_source_customer_key = self.config.sse_copy_source_customer_key.encode()
-            md5s.update(sse_copy_source_customer_key)
-            md5_encoded = b64encode(md5s.digest())
+            sse_copy_source_customer_key = encode_to_s3(self.config.sse_copy_source_customer_key)
             key_encoded = b64encode(sse_copy_source_customer_key)
+            md5_encoded = b64encode(md5(sse_copy_source_customer_key).digest())
             headers["x-amz-copy-source-server-side-encryption-customer-algorithm"] = "AES256"
-            headers["x-amz-copy-source-server-side-encryption-customer-key"] = key_encoded.decode()
-            headers["x-amz-copy-source-server-side-encryption-customer-key-md5"] = md5_encoded.decode()
+            headers["x-amz-copy-source-server-side-encryption-customer-key"] = decode_from_s3(key_encoded)
+            headers["x-amz-copy-source-server-side-encryption-customer-key-md5"] = decode_from_s3(md5_encoded)
 
         # Following meta data are not updated in simple COPY by aws.
         if extra_headers:

s3cmd

@@ -1929,7 +1929,7 @@ def cmd_sync_local2remote(args):
             cfg.sync_checks.remove("md5")
         except Exception:
             pass
-        if cfg.sync_checks.count("mtime") == 0:
+        if "mtime" not in cfg.sync_checks:
             cfg.sync_checks.append("mtime")
 
     for arg in args[:-1]: