fix(core): S3 'consistency' Issues (aws-samples#564)

* Adding enable versioning Custom Resource

* Adding replication to S3 bucket

* Prettier

* Adding lifecycle rule while enabling s3 version

* Fixing tests

Author: naveenkoppula

src/deployments/cdk/package.json

@@ -99,6 +99,7 @@
     "@aws-accelerator/custom-resource-ssm-create-document": "workspace:^0.0.1",
     "@aws-accelerator/custom-resource-disassociate-hosted-zones": "workspace:^0.0.1",
     "@aws-accelerator/custom-resource-ec2-modify-transit-gateway-vpc-attachment": "workspace:^0.0.1",
+    "@aws-accelerator/custom-resource-s3-put-bucket-versioning": "workspace:^0.0.1",
     "@aws-cdk/aws-accessanalyzer": "1.75.0",
     "@aws-cdk/aws-autoscaling": "1.75.0",
     "@aws-cdk/aws-budgets": "1.75.0",

src/deployments/cdk/src/apps/phase--1.ts

@@ -17,6 +17,7 @@ import * as globalRoles from '../deployments/iam';
  *   - Creating required roles for createLogsMetricFilter custom resource
  *   - Creating required roles for SnsSubscriberLambda custom resource
  *   - Creating required role for SsmIncreaseThroughput custom resource
+ *   - Creating required role for S3PutBucketReplication custom resource
  */
 export async function deploy({ acceleratorConfig, accountStacks, accounts }: PhaseInput) {
   // creates roles for macie custom resources
@@ -111,4 +112,10 @@ export async function deploy({ acceleratorConfig, accountStacks, accounts }: Pha
     accountStacks,
     accounts,
   });
+
+  // Creates required role for S3PutBucketReplication custom resource
+  await globalRoles.createS3PutReplicationRole({
+    accountStacks,
+    accounts,
+  });
 }

src/deployments/cdk/src/apps/phase-5.ts

@@ -1,4 +1,3 @@
-import * as ssm from '@aws-cdk/aws-ssm';
 import { getAccountId } from '../utils/accounts';
 import { VpcOutputFinder } from '@aws-accelerator/common-outputs/src/vpc';
 import { getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output';
@@ -15,6 +14,7 @@ import { ArtifactOutputFinder } from '../deployments/artifacts/outputs';
 import { ImageIdOutputFinder } from '@aws-accelerator/common-outputs/src/ami-output';
 import * as cloudWatchDeployment from '../deployments/cloud-watch';
 import * as ssmDeployment from '../deployments/ssm';
+import * as defaults from '../deployments/defaults';
 
 /**
  * This is the main entry point to deploy phase 5
@@ -29,6 +29,21 @@ import * as ssmDeployment from '../deployments/ssm';
  */
 
 export async function deploy({ acceleratorConfig, accountStacks, accounts, context, outputs }: PhaseInput) {
+  // Find the account buckets in the outputs
+  const accountBuckets = defaults.AccountBucketOutput.getAccountBuckets({
+    accounts,
+    accountStacks,
+    config: acceleratorConfig,
+    outputs,
+  });
+
+  // Find the central bucket in the outputs
+  const centralBucket = defaults.CentralBucketOutput.getBucket({
+    accountStacks,
+    config: acceleratorConfig,
+    outputs,
+  });
+
   const accountNames = acceleratorConfig
     .getMandatoryAccountConfigs()
     .map(([_, accountConfig]) => accountConfig['account-name']);
@@ -205,4 +220,13 @@ export async function deploy({ acceleratorConfig, accountStacks, accounts, conte
     config: acceleratorConfig,
     outputs,
   });
+
+  await defaults.step3({
+    accountBuckets,
+    accountStacks,
+    accounts,
+    config: acceleratorConfig,
+    centralBucket,
+    outputs,
+  });
 }

src/deployments/cdk/src/deployments/defaults/index.ts

@@ -1,3 +1,4 @@
 export * from './outputs';
 export * from './step-1';
 export * from './step-2';
+export * from './step-3';

src/deployments/cdk/src/deployments/defaults/shared.ts

@@ -40,14 +40,16 @@ export function createDefaultS3Bucket(props: {
   accountStack: AccountStack;
   encryptionKey: kms.Key;
   logRetention: number;
+  versioned?: boolean;
 }): Bucket {
-  const { accountStack, encryptionKey, logRetention } = props;
+  const { accountStack, encryptionKey, logRetention, versioned } = props;
 
   // Generate fixed bucket name so we can do initialize cross-account bucket replication
   const bucket = new Bucket(accountStack, 'DefaultBucket', {
     encryptionKey,
     expirationInDays: logRetention,
     removalPolicy: cdk.RemovalPolicy.RETAIN,
+    versioned,
   });
 
   // Let the bucket name be generated by CloudFormation

src/deployments/cdk/src/deployments/defaults/step-1.ts

@@ -93,7 +93,6 @@ function createCentralBucketCopy(props: DefaultsStep1Props) {
 
   const bucket = new s3.Bucket(masterAccountStack, 'CentralBucketCopy', {
     encryptionKey,
-    versioned: true,
     blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
     removalPolicy: cdk.RemovalPolicy.RETAIN,
   });
@@ -177,6 +176,7 @@ function createCentralLogBucket(props: DefaultsStep1Props) {
     accountStack: logAccountStack,
     encryptionKey: logKey.encryptionKey,
     logRetention: defaultLogRetention!,
+    versioned: true,
   });
 
   // Allow replication from all Accelerator accounts

src/deployments/cdk/src/deployments/defaults/step-2.ts

@@ -82,12 +82,6 @@ function createDefaultS3Buckets(props: DefaultsStep2Props) {
       }),
     );
 
-    bucket.replicateTo({
-      destinationBucket: centralLogBucket,
-      destinationAccountId: logAccountId,
-      // Only replicate files under ACCOUNT_ID/
-      prefix: `${cdk.Aws.ACCOUNT_ID}/`,
-    });
     buckets[accountKey] = bucket;
 
     new CfnAccountBucketOutput(accountStack, 'DefaultBucketOutput', {

src/deployments/cdk/src/deployments/defaults/step-3.ts

@@ -0,0 +1,111 @@
+import * as cdk from '@aws-cdk/core';
+
+import { AccountStacks } from '../../common/account-stacks';
+import { Account, getAccountId } from '../../utils/accounts';
+import { AccountBuckets, RegionalBucket } from './outputs';
+import { AcceleratorConfig } from '@aws-accelerator/common-config/src';
+import { S3PutBucketVersioning } from '@aws-accelerator/custom-resource-s3-put-bucket-versioning';
+import { BucketReplication } from '@aws-accelerator/cdk-constructs/src/s3';
+import { IamRoleOutputFinder } from '@aws-accelerator/common-outputs/src/iam-role';
+import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output';
+
+export interface DefaultsStep3Props {
+  accountStacks: AccountStacks;
+  accounts: Account[];
+  config: AcceleratorConfig;
+  accountBuckets: AccountBuckets;
+  centralBucket: RegionalBucket;
+  outputs: StackOutput[];
+}
+
+export async function step3(props: DefaultsStep3Props) {
+  const { config, accountBuckets, accountStacks, centralBucket, accounts, outputs } = props;
+  const logAccountKey = config['global-options']['central-log-services'].account;
+  const masterAccountKey = config['global-options']['aws-org-master'].account;
+  const centralLogBucket = accountBuckets[logAccountKey];
+  const defaultLogRetention = config['global-options']['default-s3-retention'];
+  for (const [accountKey, accountBucket] of Object.entries(accountBuckets)) {
+    if (accountKey === logAccountKey) {
+      continue;
+    }
+    console.log(`Enabling Versioning for accountBucket "${accountKey}"`);
+    const accountStack = accountStacks.tryGetOrCreateAccountStack(accountKey);
+    if (!accountStack) {
+      console.warn(`Cannot find account stack ${accountKey}`);
+      continue;
+    }
+
+    const accountConfig = config.getAccountByKey(accountKey);
+    const logRetention = accountConfig?.['s3-retention'] ?? defaultLogRetention;
+
+    const s3PutReplicationRole = IamRoleOutputFinder.tryFindOneByName({
+      outputs,
+      accountKey,
+      roleKey: 'S3PutReplicationRole',
+    });
+
+    if (!s3PutReplicationRole) {
+      console.warn(`S3PutBucketReplication role is not created for account "${accountKey}"`);
+      continue;
+    }
+
+    const versioning = new S3PutBucketVersioning(accountStack, `AccountBucket-PutBucketVersioning`, {
+      bucketName: accountBucket.bucketName,
+      logRetention,
+      roleArn: s3PutReplicationRole.roleArn,
+    });
+
+    // Get IBucket object for enabling replication on it
+    const bucket = new BucketReplication(accountStack, 'DefaultBucketReplication', {
+      bucket: accountBucket,
+      s3PutReplicationRole: s3PutReplicationRole.roleArn,
+    });
+    bucket.node.addDependency(versioning);
+
+    bucket.replicateTo({
+      destinationBucket: centralLogBucket,
+      destinationAccountId: getAccountId(accounts, logAccountKey)!,
+      id: 'LogArchiveAccountBucket',
+      // Only replicate files under ACCOUNT_ID/
+      prefix: `${cdk.Aws.ACCOUNT_ID}/`,
+    });
+  }
+
+  console.log(`Enabling Versioning for centralBucket "${masterAccountKey}"`);
+  const accountStack = accountStacks.tryGetOrCreateAccountStack(masterAccountKey);
+  if (!accountStack) {
+    console.warn(`Cannot find account stack ${masterAccountKey}`);
+    return;
+  }
+
+  const s3PutReplicationRoleMaster = IamRoleOutputFinder.tryFindOneByName({
+    outputs,
+    accountKey: masterAccountKey,
+    roleKey: 'S3PutReplicationRole',
+  });
+
+  if (!s3PutReplicationRoleMaster) {
+    console.warn(`S3PutBucketReplication role is not created for account "${masterAccountKey}"`);
+    return;
+  }
+
+  const centralBucketVersioning = new S3PutBucketVersioning(accountStack, `CentralBucket-PutBucketVersioning`, {
+    bucketName: centralBucket.bucketName,
+    roleArn: s3PutReplicationRoleMaster.roleArn,
+  });
+
+  // Get IBucket object for enabling replication on it
+  const centralBucketObj = new BucketReplication(accountStack, 'CentralBucketReplication', {
+    bucket: centralBucket,
+    s3PutReplicationRole: s3PutReplicationRoleMaster.roleArn,
+  });
+  centralBucketObj.node.addDependency(centralBucketVersioning);
+
+  centralBucketObj.replicateTo({
+    destinationBucket: centralLogBucket,
+    destinationAccountId: getAccountId(accounts, logAccountKey)!,
+    id: 'LogArchiveAccountBucket',
+    // Only replicate files under ACCOUNT_ID/
+    prefix: `${cdk.Aws.ACCOUNT_ID}/`,
+  });
+}

src/deployments/cdk/src/deployments/iam/index.ts

@@ -17,3 +17,4 @@ export * from './cleanup-role';
 export * from './central-endpoints-deployment-roles';
 export * from './ssm-throughput-roles';
 export * from './ec2';
+export * from './s3-put-replication-role';

src/deployments/cdk/src/deployments/iam/s3-put-replication-role.ts

@@ -0,0 +1,41 @@
+import * as iam from '@aws-cdk/aws-iam';
+import { AccountStacks, AccountStack } from '../../common/account-stacks';
+import { createIamRoleOutput } from './outputs';
+import { Account } from '@aws-accelerator/common-outputs/src/accounts';
+
+export interface S3PutReplicationRoleProps {
+  accountStacks: AccountStacks;
+  accounts: Account[];
+}
+
+export async function createS3PutReplicationRole(props: S3PutReplicationRoleProps): Promise<void> {
+  const { accountStacks, accounts } = props;
+
+  for (const account of accounts) {
+    const accountStack = accountStacks.getOrCreateAccountStack(account.key);
+    const iamRole = await createRole(accountStack);
+    createIamRoleOutput(accountStack, iamRole, 'S3PutReplicationRole');
+  }
+}
+
+async function createRole(stack: AccountStack) {
+  const role = new iam.Role(stack, 'Custom::S3PutReplicationRole', {
+    assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'),
+  });
+
+  role.addToPrincipalPolicy(
+    new iam.PolicyStatement({
+      actions: [
+        'iam:PassRole',
+        'logs:CreateLogStream',
+        'logs:CreateLogGroup',
+        'logs:PutLogEvents',
+        's3:PutLifecycleConfiguration',
+        's3:PutReplicationConfiguration',
+        's3:PutBucketVersioning',
+      ],
+      resources: ['*'],
+    }),
+  );
+  return role;
+}