Merge pull request nccgroup#508 from nccgroup/develop

x4v13r64 · web-flow · commit 76c59ad5183b · 2019-09-23T13:12:10.000+02:00
release/5.4.0
diff --git a/ScoutSuite/__init__.py b/ScoutSuite/__init__.py
@@ -1,5 +1,5 @@
 __author__ = 'NCC Group'
-__version__ = '5.3.3'
+__version__ = '5.4.0'
 
 ERRORS_LIST = []
 
diff --git a/ScoutSuite/core/cli_parser.py b/ScoutSuite/core/cli_parser.py
@@ -38,7 +38,7 @@ def _init_aws_parser(self):
         aws_parser = parser.add_argument_group('Authentication modes')
         aws_auth_params = parser.add_argument_group('Authentication parameters')
 
-        aws_auth_modes = aws_parser.add_mutually_exclusive_group(required=True)
+        aws_auth_modes = aws_parser.add_mutually_exclusive_group(required=False)
 
         aws_auth_modes.add_argument('-p',
                                     '--profile',
diff --git a/ScoutSuite/core/conditions.py b/ScoutSuite/core/conditions.py
@@ -4,7 +4,7 @@
 import netaddr
 import re
 
-from iampoliciesgonewild import get_actions_from_statement, _expand_wildcard_action
+from policyuniverse.expander_minimizer import get_actions_from_statement, _expand_wildcard_action
 
 from ScoutSuite.core.console import print_error, print_exception
 
diff --git a/ScoutSuite/core/rule.py b/ScoutSuite/core/rule.py
@@ -8,15 +8,15 @@
 
 ip_ranges_from_args = 'ip-ranges-from-args'
 
-re_aws_account_id = re.compile(r'_AWS_ACCOUNT_ID_')
+re_account_id = re.compile(r'_ACCOUNT_ID_')
 re_ip_ranges_from_file = re.compile(r'_IP_RANGES_FROM_FILE_\((.*?)(,.*?)\)')
 re_ip_ranges_from_local_file = re.compile(r'_IP_RANGES_FROM_LOCAL_FILE_\((.*?)(,.*?)\)')
 re_strip_dots = re.compile(r'(_STRIPDOTS_\((.*?)\))')
 
 testcases = [
     {
-        'name': 'aws_account_id',
-        'regex': re_aws_account_id
+        'name': 'account_id',
+        'regex': re_account_id
     },
     {
         'name': 'ip_ranges_from_file',
diff --git a/ScoutSuite/output/data/html/partials/aws/services.awslambda.regions.id.functions.html b/ScoutSuite/output/data/html/partials/aws/services.awslambda.regions.id.functions.html
@@ -1,15 +1,32 @@
+<!-- Lambda function partial -->
+<script id="services.awslambda.regions.id.functions.partial" type="text/x-handlebars-template">
+    <div id="resource-name" class="list-group-item active">
+        <h4 class="list-group-item-heading">{{name}}</h4>
+    </div>
+    <div class="list-group-item">
+        <h4 class="list-group-item-heading">Information</h4>
+        <div class="list-group-item-text item-margin">Description: <span id="awslambda.regions.{{region}}.functions.{{@key}}.description"><samp>{{value_or_none description}}</samp></span></div>
+        <div class="list-group-item-text item-margin">Role: <span id="awslambda.regions.{{region}}.functions.{{@key}}.role"><samp>{{value_or_none role}}</samp></span></div>
+        <div class="list-group-item-text item-margin">Last Modified: <span id="awslambda.regions.{{region}}.functions.{{@key}}.last_modified"><samp>{{format_date last_modified}}</samp></span></div>
+        <div class="list-group-item-text item-margin">Runtime: <span id="awslambda.regions.{{region}}.functions.{{@key}}.runtime"><samp>{{value_or_none runtime}}</samp></span></div>
+        <div class="list-group-item-text item-margin">Version: <span id="awslambda.regions.{{region}}.functions.{{@key}}.version"><samp>{{value_or_none version}}</samp></span></div>
+        <div class="list-group-item-text item-margin">Revision ID: <span id="awslambda.regions.{{region}}.functions.{{@key}}.revision_id"><samp>{{value_or_none revision_id}}</samp></span></div>
+        <div class="list-group-item-text item-margin">Code Sha256: <span id="awslambda.regions.{{region}}.functions.{{@key}}.code_sha256"><samp>{{value_or_none code_sha256}}</samp></span></div>
+        <div class="list-group-item-text item-margin">Handler: <span id="awslambda.regions.{{region}}.functions.{{@key}}.handler"><samp>{{value_or_none handler}}</samp></span></div>
+        <div class="list-group-item-text item-margin">Code Size: <span id="awslambda.regions.{{region}}.functions.{{@key}}.code_size"><samp>{{value_or_none code_size}}</samp></span></div>
+        <div class="list-group-item-text item-margin">Memory Size: <span id="awslambda.regions.{{region}}.functions.{{@key}}.memory_size"><samp>{{value_or_none memory_size}}</samp></span></div>
+        <div class="list-group-item-text item-margin">Timeout: <span id="awslambda.regions.{{region}}.functions.{{@key}}.timeout"><samp>{{value_or_none timeout}}</samp></span></div>
+    </div>
+</script>
 
-    <!-- Lambda function partial -->
-    <script id="services.awslambda.regions.id.functions.partial" type="text/x-handlebars-template">
-        <div class="list-group-item active">
-            <h4 class="list-group-item-heading">{{name}}</h4>
-        </div>
-        <div class="list-group-item">
-            <h4 class="list-group-item-heading">Attributes</h4>
-            {{> generic_object resource}}
-        </div>
-    </script>
-    <script>
-      Handlebars.registerPartial("services.awslambda.regions.id.functions", $("#services\\.awslambda\\.regions\\.id\\.functions\\.partial").html());
-    </script>
+<script>
+    Handlebars.registerPartial("services.awslambda.regions.id.functions", $("#services\\.awslambda\\.regions\\.id\\.functions\\.partial").html());
+</script>
 
+<!-- Single awslambda function template -->
+<script id="single_awslambda_function-template" type="text/x-handlebars-template">
+    {{> modal-template template='services.awslambda.regions.id.functions'}}
+</script>
+<script>
+    var single_awslambda_function_template = Handlebars.compile($("#single_awslambda_function-template").html());
+</script>
diff --git a/ScoutSuite/output/data/html/partials/aws/services.s3.buckets.html b/ScoutSuite/output/data/html/partials/aws/services.s3.buckets.html
@@ -15,12 +15,14 @@ <h4 class="list-group-item-heading">Information</h4>
           <div class="list-group-item-text item-margin">Secure transport: <span id="s3.buckets.{{@key}}.secure_transport_enabled">{{convert_bool_to_enabled secure_transport_enabled}}</span></div>
           <div class="list-group-item-text item-margin">Static website hosting: <span id="s3.buckets.{{@key}}.web_hosting_enabled">{{convert_bool_to_enabled web_hosting_enabled}}</span></div>
         </div>
+        <div class="list-group-item">
+            {{#if policy}}
+                {{> accordion_policy name = 'Bucket Policy' document = policy policy_path = (concat 's3.buckets' @key 'policy')}}
+            {{else}}
+                <h4 class="list-group-item-heading accordion-heading text-secondary">Bucket Policy</h4>
+            {{/if}}
+        </div>
         {{> services.s3.acls resource_type = 'bucket' resource_path = (concat 's3.buckets' @key)}}
-        {{#if policy}}
-          <div class="list-group-item">
-            {{> accordion_policy name = 'Bucket policy' document = policy policy_path = (concat 's3.buckets' @key 'policy')}}
-          </div>
-        {{/if}}
         {{> services.s3.bucket_iam_policies resource_type = 'groups' resource_count = groups_count}}
         {{> services.s3.bucket_iam_policies resource_type = 'roles' resource_count = roles_count}}
         {{> services.s3.bucket_iam_policies resource_type = 'users' resource_count = users_count}}
diff --git a/ScoutSuite/output/data/html/report.html b/ScoutSuite/output/data/html/report.html
@@ -20,6 +20,8 @@
     
     <!-- Fontawesome CSS -->
     <link href="inc-fontawesome/css/all.min.css" rel="stylesheet">
+    <!-- Fallback fonts to solve CORS issue-->
+    <link href="https://use.fontawesome.com/releases/v5.6.3/css/all.css" rel="stylesheet">
     <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
     <!--[if lt IE 9]>
     <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
diff --git a/ScoutSuite/providers/__init__.py b/ScoutSuite/providers/__init__.py
@@ -1,18 +1,17 @@
-import sys
-
-from ScoutSuite.providers.aws.provider import AWSProvider
-from ScoutSuite.providers.azure.provider import AzureProvider
-from ScoutSuite.providers.gcp.provider import GCPProvider
-from ScoutSuite.providers.aliyun.provider import AliyunProvider
-from ScoutSuite.providers.oci.provider import OracleProvider
-
 providers_dict = {'aws': 'AWSProvider',
                   'gcp': 'GCPProvider',
                   'azure': 'AzureProvider',
                   'aliyun': 'AliyunProvider',
                   'oci': 'OracleProvider'}
 
 
+def get_provider_object(provider):
+    provider_class = providers_dict.get(provider)
+    provider_module = __import__('ScoutSuite.providers.{}.provider'.format(provider), fromlist=[provider_class])
+    provider_object = getattr(provider_module, provider_class)
+    return provider_object
+
+
 def get_provider(provider,
                  profile=None,
                  project_id=None, folder_id=None, organization_id=None,
@@ -34,8 +33,7 @@ def get_provider(provider,
     services = [] if services is None else services
     skipped_services = [] if skipped_services is None else skipped_services
 
-    provider_class = providers_dict.get(provider)
-    provider_object = getattr(sys.modules[__name__], provider_class)
+    provider_object = get_provider_object(provider)
     provider_instance = provider_object(profile=profile,
                                         project_id=project_id,
                                         folder_id=folder_id,
diff --git a/ScoutSuite/providers/aws/authentication_strategy.py b/ScoutSuite/providers/aws/authentication_strategy.py
@@ -1,7 +1,8 @@
 import boto3
+import logging
 
-from ScoutSuite.providers.base.authentication_strategy import AuthenticationStrategy, AuthenticationException
 from ScoutSuite.providers.aws.utils import get_caller_identity
+from ScoutSuite.providers.base.authentication_strategy import AuthenticationStrategy, AuthenticationException
 
 
 class AWSCredentials:
@@ -22,6 +23,11 @@ def authenticate(self,
 
         try:
 
+            # Set logging level to error for libraries as otherwise generates a lot of warnings
+            logging.getLogger('botocore').setLevel(logging.ERROR)
+            logging.getLogger('botocore.auth').setLevel(logging.ERROR)
+            logging.getLogger('urllib3').setLevel(logging.ERROR)
+
             if profile:
                 session = boto3.Session(profile_name=profile)
             elif aws_access_key_id and aws_secret_access_key:
@@ -37,7 +43,7 @@ def authenticate(self,
                         aws_secret_access_key=aws_secret_access_key,
                     )
             else:
-                raise AuthenticationException('Insufficient credentials provided')
+                session = boto3.Session()
 
             # Test querying for current user
             identity = get_caller_identity(session)
diff --git a/ScoutSuite/providers/aws/facade/base.py b/ScoutSuite/providers/aws/facade/base.py
@@ -1,6 +1,4 @@
-from collections import Counter
-
-from botocore.session import Session
+from boto3.session import Session
 
 from ScoutSuite.providers.aws.facade.awslambda import LambdaFacade
 from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade
@@ -45,20 +43,33 @@ def __init__(self, credentials=None):
         self._instantiate_facades()
 
     async def build_region_list(self, service: str, chosen_regions=None, excluded_regions=None, partition_name='aws'):
-        service = 'ec2containerservice' if service == 'ecs' else service
-        available_services = await run_concurrently(lambda: Session().get_available_services())
 
+        service = 'ec2containerservice' if service == 'ecs' else service
+        available_services = await run_concurrently(lambda: Session(region_name='eu-west-1').get_available_services())
         if service not in available_services:
             raise Exception('Service ' + service + ' is not available.')
 
-        regions = await run_concurrently(lambda: Session().get_available_regions(service, partition_name))
+        regions = await run_concurrently(lambda: Session(region_name='eu-west-1').get_available_regions(service,
+                                                                                                        partition_name))
+
+        # identify regions that are not opted-in
+        ec2_not_opted_in_regions = self.session.client('ec2', 'eu-west-1')\
+            .describe_regions(AllRegions=True, Filters=[{'Name': 'opt-in-status', 'Values': ['not-opted-in']}])
+
+        not_opted_in_regions = []
+        if ec2_not_opted_in_regions['Regions']:
+            for r in ec2_not_opted_in_regions['Regions']:
+                not_opted_in_regions.append(r['RegionName'])
 
         # include specific regions
         if chosen_regions:
             regions = [r for r in regions if r in chosen_regions]
         # exclude specific regions
         if excluded_regions:
             regions = [r for r in regions if r not in excluded_regions]
+        # exclude not opted in regions
+        if not_opted_in_regions:
+            regions = [r for r in regions if r not in not_opted_in_regions]
 
         return regions
 
diff --git a/ScoutSuite/providers/aws/facade/ec2.py b/ScoutSuite/providers/aws/facade/ec2.py
@@ -1,6 +1,8 @@
 import asyncio
 import base64
 import boto3
+import zlib
+import codecs
 
 from ScoutSuite.core.console import print_exception
 from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade
@@ -30,7 +32,12 @@ async def get_instance_user_data(self, region: str, instance_id: str):
         else:
             if 'Value' not in user_data_response['UserData'].keys():
                 return None
-            return base64.b64decode(user_data_response['UserData']['Value']).decode('utf-8')
+            else:
+                value = base64.b64decode(user_data_response['UserData']['Value'])
+                if value[0:2] == b'\x1f\x8b':  # GZIP magic number
+                    return zlib.decompress(value, zlib.MAX_WBITS | 32).decode('utf-8')
+                else:
+                    return value.decode('utf-8')
 
     async def get_instances(self, region: str, vpc: str):
         filters = [{'Name': 'vpc-id', 'Values': [vpc]}]
diff --git a/ScoutSuite/providers/aws/metadata.json b/ScoutSuite/providers/aws/metadata.json
@@ -226,7 +226,7 @@
         },
         "elbv2": {
             "resources": {
-                "lbs": {
+                "elbs": {
                     "cols": 2,
                     "path": "services.elbv2.regions.id.vpcs.id.lbs",
                     "callbacks": [
@@ -247,7 +247,7 @@
                 "functions": {
                     "path": "services.awslambda.regions.id.functions",
                     "callbacks": [
-                        [ "match_security_groups_and_resources_callback", {"status_path": ["Runtime"], "sg_list_attribute_name": [ "VpcConfig", "SecurityGroupIds" ]} ]
+                        [ "match_security_groups_and_resources_callback", {"status_path": ["runtime"], "sg_list_attribute_name": [ "VpcConfig", "SecurityGroupIds" ]} ]
                     ]
                 }
             }
diff --git a/ScoutSuite/providers/aws/resources/awslambda/functions.py b/ScoutSuite/providers/aws/resources/awslambda/functions.py
@@ -14,5 +14,21 @@ async def fetch_all(self):
             self[name] = resource
 
     def _parse_function(self, raw_function):
-        raw_function['name'] = raw_function.pop('FunctionName')
-        return raw_function['name'], raw_function
+
+        function_dict = {}
+        function_dict['name'] = raw_function.get('FunctionName')
+        function_dict['arn'] = raw_function.get('FunctionArn')
+        function_dict['runtime'] = raw_function.get('Runtime')
+        function_dict['role'] = raw_function.get('Role')
+        function_dict['handler'] = raw_function.get('Handler')
+        function_dict['code_size'] = raw_function.get('CodeSize')
+        function_dict['description'] = raw_function.get('Description')
+        function_dict['timeout'] = raw_function.get('Timeout')
+        function_dict['memory_size'] = raw_function.get('MemorySize')
+        function_dict['last_modified'] = raw_function.get('LastModified')
+        function_dict['code_sha256'] = raw_function.get('CodeSha256')
+        function_dict['version'] = raw_function.get('Version')
+        function_dict['tracing_config'] = raw_function.get('TracingConfig')
+        function_dict['revision_id'] = raw_function.get('RevisionId')
+        return function_dict['name'], function_dict
+
diff --git a/ScoutSuite/providers/aws/resources/cloudformation/stacks.py b/ScoutSuite/providers/aws/resources/cloudformation/stacks.py
@@ -37,7 +37,7 @@ def _parse_stack(self, raw_stack):
     def has_deletion_policy(template):
         """
         Return region to be used for global calls such as list bucket and get bucket location
-        :param template:                    The api response containing the stack's template
+        :param template: The api response containing the stack's template
         :return:
         """
         has_dp = True
diff --git a/ScoutSuite/providers/aws/rules/findings/cloudtrail-no-log-file-validation.json b/ScoutSuite/providers/aws/rules/findings/cloudtrail-no-log-file-validation.json
@@ -3,6 +3,7 @@
 	"rationale": "<b>Description:</b><br><br>The lack of log file validation prevents one from verifying the integrity of the log files.",
     "path": "cloudtrail.regions.id.trails.id",
     "dashboard_name": "Trails",
+    "display_path": "cloudtrail.regions.id.trails.id",
     "conditions": [ "and",
         [ "cloudtrail.regions.id.trails.id.", "withKey", "LogFileValidationEnabled" ],
         [ "cloudtrail.regions.id.trails.id.LogFileValidationEnabled", "false", "" ]
diff --git a/ScoutSuite/providers/azure/authentication_strategy.py b/ScoutSuite/providers/azure/authentication_strategy.py
@@ -1,4 +1,5 @@
 import json
+import logging
 from getpass import getpass
 
 from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials, get_azure_cli_credentials
@@ -33,6 +34,12 @@ def authenticate(self,
         Implements authentication for the Azure provider
         """
         try:
+
+            # Set logging level to error for libraries as otherwise generates a lot of warnings
+            logging.getLogger('adal-python').setLevel(logging.ERROR)
+            logging.getLogger('msrest').setLevel(logging.ERROR)
+            logging.getLogger('urllib3').setLevel(logging.ERROR)
+
             if cli:
                 credentials, subscription_id, tenant_id = get_azure_cli_credentials(with_tenant=True)
                 graphrbac_credentials, placeholder_1, placeholder_2 = get_azure_cli_credentials(with_tenant=True,
diff --git a/ScoutSuite/providers/base/authentication_strategy_factory.py b/ScoutSuite/providers/base/authentication_strategy_factory.py
@@ -1,21 +1,23 @@
-from ScoutSuite.providers.aws.authentication_strategy import AWSAuthenticationStrategy
-from ScoutSuite.providers.gcp.authentication_strategy import GCPAuthenticationStrategy
-from ScoutSuite.providers.azure.authentication_strategy import AzureAuthenticationStrategy
-from ScoutSuite.providers.aliyun.authentication_strategy import AliyunAuthenticationStrategy
-from ScoutSuite.providers.oci.authentication_strategy import OracleAuthenticationStrategy
-
 _strategies = {
-    'aws': AWSAuthenticationStrategy,
-    'gcp': GCPAuthenticationStrategy,
-    'azure': AzureAuthenticationStrategy,
-    'aliyun': AliyunAuthenticationStrategy,
-    'oci': OracleAuthenticationStrategy
+    'aws': 'AWSAuthenticationStrategy',
+    'gcp': 'GCPAuthenticationStrategy',
+    'azure': 'AzureAuthenticationStrategy',
+    'aliyun': 'AliyunAuthenticationStrategy',
+    'oci': 'OracleAuthenticationStrategy'
 }
 
 
+def import_authentication_strategy(provider):
+    strategy_class = _strategies[provider]
+    module = __import__('ScoutSuite.providers.{}.authentication_strategy'.format(provider), fromlist=[strategy_class])
+    authentication_strategy = getattr(module, strategy_class)
+    return authentication_strategy
+
+
 def get_authentication_strategy(provider: str):
     """
         Returns an authentication strategy implementation for a provider.
         :param provider: The authentication strategy 
     """
-    return _strategies[provider]()
+    authentication_strategy = import_authentication_strategy(provider)
+    return authentication_strategy()
diff --git a/ScoutSuite/providers/gcp/authentication_strategy.py b/ScoutSuite/providers/gcp/authentication_strategy.py
@@ -1,7 +1,8 @@
+import logging
 import os
 import warnings
 
-import google
+from google import auth
 
 from ScoutSuite.providers.base.authentication_strategy import AuthenticationStrategy, AuthenticationException
 
@@ -16,6 +17,12 @@ def authenticate(self, user_account=None, service_account=None, **kargs):
 
         try:
 
+            # Set logging level to error for libraries as otherwise generates a lot of warnings
+            logging.getLogger('googleapiclient').setLevel(logging.ERROR)
+            logging.getLogger('google.auth').setLevel(logging.ERROR)
+            logging.getLogger('google_auth_httplib2').setLevel(logging.ERROR)
+            logging.getLogger('urllib3').setLevel(logging.ERROR)
+
             if user_account:
                 # disable GCP warning about using User Accounts
                 warnings.filterwarnings("ignore", "Your application has authenticated using end user credentials")
@@ -26,7 +33,7 @@ def authenticate(self, user_account=None, service_account=None, **kargs):
             else:
                 raise AuthenticationException('Failed to authenticate to GCP - no supported account type')
 
-            credentials, default_project_id = google.auth.default()
+            credentials, default_project_id = auth.default()
 
             if not credentials:
                 raise AuthenticationException('No credentials')
diff --git a/ScoutSuite/providers/gcp/facade/base.py b/ScoutSuite/providers/gcp/facade/base.py
diff --git a/ScoutSuite/providers/gcp/provider.py b/ScoutSuite/providers/gcp/provider.py
diff --git a/requirements.txt b/requirements.txt
