From 3e43937d8fb2e11c5ffe9a611cebfb3fd424e176 Mon Sep 17 00:00:00 2001 From: Dan Kilman Date: Thu, 10 May 2018 15:20:53 +0300 Subject: [PATCH 01/22] Add parallel command execution to SSH command (#1) * Add multihost support * Add parallel execution support * improve parallel output * allow selecting a subset of matches * improve selection message * pretty line! * improve output * code cleanup * clean some more * Add formatter that shows both private and public ips --- .gitignore | 43 ++++++++++ ec2grep/__init__.py | 192 +++++++++++++++++++++++++++----------------- 2 files changed, 163 insertions(+), 72 deletions(-) create mode 100644 .gitignore mode change 100755 => 100644 ec2grep/__init__.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..921154a --- /dev/null +++ b/.gitignore @@ -0,0 +1,43 @@ +.idea +*.iml +out +outfds +**/.bundle/ +*COMMIT_MSG + +*.py[cod] + +# C extensions +*.so +*.iml + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject diff --git a/ec2grep/__init__.py b/ec2grep/__init__.py old mode 100755 new mode 100644 index f5dca80..9aa88c5 --- a/ec2grep/__init__.py +++ b/ec2grep/__init__.py @@ -1,69 +1,40 @@ -#!/usr/bin/env python -from concurrent.futures import wait, ThreadPoolExecutor as Executor -from functools import partial -from operator import itemgetter +#! /usr/bin/env python -import os +import concurrent.futures +import functools +import itertools +import operator +import pprint +import subprocess import sys -import click +import threading + import boto3 -import itertools +import click import six -executor = Executor(4) -name = (lambda i: {tag['Key']: tag['Value'] for tag in i.get('Tags', [])}.get('Name', '')) -public_ip = itemgetter('PublicIpAddress') -private_ip = itemgetter('PrivateIpAddress') -extended_public = (lambda i: "{} ({})".format(name(i), public_ip(i))) -extended_private = (lambda i: "{} ({})".format(name(i), private_ip(i))) -DEFAULT_ATTRIBUTES = [ + +DEFAULT_ATTRIBUTES = ( 'tag:Name', 'network-interface.addresses.association.public-ip', 'network-interface.addresses.private-ip-address', - 'network-interface.private-dns-name' -] - - -def _get_instances(ec2, filter_): - response = ec2.describe_instances(Filters=[filter_]) - reservations = response['Reservations'] - return list(itertools.chain.from_iterable(r['Instances'] for r in reservations)) - - -def match_instances(region_name, query, attributes=DEFAULT_ATTRIBUTES): - ec2 = boto3.client('ec2', region_name=region_name) - get_instances = partial(_get_instances, ec2) - instance_lists = executor.map(get_instances, [ - {'Name': attr, 'Values': ['*{}*'.format(query)]} for attr in attributes - ]) - chained = (i for i in itertools.chain.from_iterable(instance_lists) if 'PublicIpAddress' in i) - return sorted(chained, key=name) - - -def die(*args): - click.echo(*args, err=True) - sys.exit(1) - - -def read_number(min_value, max_value): - while True: - try: - choice = six.moves.input() - choice = int(choice) - if not (min_value <= choice <= max_value): - raise ValueError("Invalid input") - return choice - except ValueError as e: - click.echo("{}".format(e), err=True) - continue + 'network-interface.private-dns-name', +) +name = (lambda i: {tag['Key']: tag['Value'] for tag in i.get('Tags', [])}.get('Name', '')) +public_ip = operator.itemgetter('PublicIpAddress') +private_ip = operator.itemgetter('PrivateIpAddress') +extended_public = (lambda i: '{} ({})'.format(name(i), public_ip(i))) +extended_private = (lambda i: '{} ({})'.format(name(i), private_ip(i))) +extended = (lambda i: '{} (public: {}, private: {})'.format(name(i), public_ip(i), private_ip(i))) formatters = { + 'extended': extended, 'extended_public': extended_public, 'extended_private': extended_private, 'public_ip': public_ip, 'private_ip': private_ip, - 'name': name + 'name': name, } @@ -77,50 +48,127 @@ def cli(ctx, region): @cli.command() @click.option('--key', '-i') @click.option('--prefer-public-ip', '-p', is_flag=True, default=False) +@click.option('--parallel', is_flag=True, default=False) @click.argument('query') @click.argument('ssh_args', nargs=-1, type=click.UNPROCESSED) @click.pass_context -def ssh(ctx, key, prefer_public_ip, query, ssh_args): +def ssh(ctx, key, prefer_public_ip, parallel, query, ssh_args): get_ip = public_ip if prefer_public_ip else private_ip fmt_match = extended_public if prefer_public_ip else extended_private - extra_args = [] - matches = match_instances(ctx.obj['region'], query) if not matches: - die("No matches found") - + die('No matches found') if len(matches) > 1: + click.echo('[0] All') for i, inst in enumerate(matches): - click.echo("[{}] {}".format(i+1, fmt_match(inst))) - click.echo("pick an option [1-{}] ".format(len(matches)), nl=False) - index = read_number(1, len(matches)) - 1 - choice = matches[index] - click.echo("") + click.echo('[{}] {}'.format(i+1, fmt_match(inst))) + click.echo('select servers [0-{}] (comma separated) '.format(len(matches)), nl=False) + indices = read_numbers(0, len(matches)) + if len(indices) == 1 and indices[0] == 0: + choices = matches + else: + choices = [matches[index - 1] for index in indices] + click.echo() else: - choice = matches[0] - - click.echo("sshing {}".format(fmt_match(choice))) - + choices = [matches[0]] + command = ['ssh', '-oStrictHostKeyChecking=no'] if key: - extra_args.extend(['-i', key]) - os.execvp('ssh', ['ssh', '-oStrictHostKeyChecking=no'] + extra_args + [get_ip(choice)] + list(ssh_args)) + command.extend(['-i', key]) + if parallel and len(choices) > 1: + message = 'sshing:\n{}'.format(pprint.pformat([fmt_match(c) for c in choices])) + click.echo(message) + click.echo() + processes = [] + for choice in choices: + ip = get_ip(choice) + p = subprocess.Popen(command + [ip] + list(ssh_args), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout_consumer = OutputConsumer(input=p.stdout, output=sys.stdout, ip=ip) + stderr_consumer = OutputConsumer(input=p.stderr, output=sys.stderr, ip=ip) + processes.append({'out': stdout_consumer, 'err': stderr_consumer, 'p': p}) + for p in processes: + p['p'].wait() + p['out'].join() + p['err'].join() + else: + for choice in choices: + message = 'sshing {}'.format(fmt_match(choice)) + click.echo() + click.echo(message) + click.echo(u'\u2500' * len(message)) + subprocess.call(command + [get_ip(choice)] + list(ssh_args)) @cli.command() -@click.argument('query') @click.option('--delim', '-d', default='\n') @click.option('--formatter', '-f', default='extended_private') @click.option('--custom-format', '-c', is_flag=True, default=False) +@click.argument('query') @click.pass_context -def ls(ctx, query, formatter, delim, custom_format): +def ls(ctx, formatter, delim, custom_format, query): matches = match_instances(ctx.obj['region'], query) + if not matches: + die('No matches found') if not custom_format: formatter = formatter.join('{}') - if not matches: - die("No matches found") - click.echo(delim.join(formatter.format(**{k: f(m) for k, f in formatters.iteritems()}) for m in matches)) +def _get_instances(ec2, filter_): + response = ec2.describe_instances(Filters=[filter_]) + reservations = response['Reservations'] + return list(itertools.chain.from_iterable(r['Instances'] for r in reservations)) + + +def match_instances(region_name, query, attributes=DEFAULT_ATTRIBUTES): + ec2 = boto3.client('ec2', region_name=region_name) + get_instances = functools.partial(_get_instances, ec2) + with concurrent.futures.ThreadPoolExecutor(len(attributes)) as executor: + instance_lists = executor.map(get_instances, [ + {'Name': attr, 'Values': ['*{}*'.format(query)]} for attr in attributes + ]) + chained = (i for i in itertools.chain.from_iterable(instance_lists) if 'PublicIpAddress' in i) + return sorted(chained, key=name) + + +def die(*args): + click.echo(*args, err=True) + sys.exit(1) + + +def read_numbers(min_value, max_value): + while True: + try: + choices = six.moves.input() + choices = [int(c.strip()) for c in choices.split(',')] + if not (all(min_value <= c <= max_value for c in choices)): + raise ValueError('Invalid input') + if len(choices) != 1 and 0 in choices: + raise ValueError('Invalid input') + return choices + except ValueError as e: + click.echo(str(e), err=True) + continue + + +class OutputConsumer(object): + + def __init__(self, input, output, ip): + self.ip = ip + self.input = input + self.output = output + self.consumer = threading.Thread(target=self.consume_output) + self.consumer.daemon = True + self.consumer.start() + + def consume_output(self): + for line in iter(self.input.readline, b''): + self.output.write('[{:<15}] {}'.format(self.ip, line)) + + def join(self): + self.consumer.join() + + if __name__ == '__main__': cli() From 3915a7c118e48c14869359a8dfc42d8207d58906 Mon Sep 17 00:00:00 2001 From: Ran Ziv Date: Fri, 11 May 2018 20:03:57 +0300 Subject: [PATCH 02/22] * Add SCP support, including multihost and parallel capabilities * Add SSH-config file support for SSH and SCP commands * Add explicit user flag for SSH command --- ec2grep/__init__.py | 121 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 96 insertions(+), 25 deletions(-) diff --git a/ec2grep/__init__.py b/ec2grep/__init__.py index 9aa88c5..e6f0588 100644 --- a/ec2grep/__init__.py +++ b/ec2grep/__init__.py @@ -4,6 +4,7 @@ import functools import itertools import operator +import os import pprint import subprocess import sys @@ -46,42 +47,85 @@ def cli(ctx, region): @cli.command() -@click.option('--key', '-i') +@click.option('--user', '-u') +@click.option('--key', '-i', help='SSH key') +@click.option('--ssh-config', '-F', help='SSH config file') @click.option('--prefer-public-ip', '-p', is_flag=True, default=False) @click.option('--parallel', is_flag=True, default=False) @click.argument('query') @click.argument('ssh_args', nargs=-1, type=click.UNPROCESSED) @click.pass_context -def ssh(ctx, key, prefer_public_ip, parallel, query, ssh_args): +def ssh(ctx, user, key, ssh_config, prefer_public_ip, parallel, query, ssh_args): + validate_identity_parameters(user, key, ssh_config) + + def op(host, host_count): + command = ['ssh', '-oStrictHostKeyChecking=no'] + command.extend(['-i', key]) if key else command.extend(['-F', ssh_config]) + if user: + command.extend(['-l', user]) + command.extend([host] + list(ssh_args)) + return command + + run_op_for_hosts(ctx, prefer_public_ip, parallel, query, 'ssh', op) + + +@cli.command() +@click.option('--user', '-u') +@click.option('--key', '-i', help='SSH key') +@click.option('--ssh-config', '-F', help='SSH config file') +@click.option('--prefer-public-ip', '-p', is_flag=True, default=False) +@click.option('--parallel', is_flag=True, default=False) +@click.option('--download', '-d', is_flag=True, default=False) +@click.argument('query') +@click.argument('source', type=str, required=True) +@click.argument('target', type=str, required=False) +@click.argument('scp_args', nargs=-1, type=click.UNPROCESSED) +@click.pass_context +def scp(ctx, user, key, ssh_config, prefer_public_ip, parallel, query, download, source, target, scp_args): + validate_identity_parameters(user, key, ssh_config) + + if download and not os.path.isdir(target): + die('Download target must be an existing directory') + + def format_scp_remote_host_path(user, host, path): + return '{}@{}:{}'.format(user, host, path) if user else '{}:{}'.format(host, path) + + def op(host, host_count): + command = ['scp', '-oStrictHostKeyChecking=no'] + command.extend(['-i', key]) if key else command.extend(['-F', ssh_config]) + command.extend(list(scp_args)) + if download: + if host_count > 1: + # when downloading from multiple hosts, use a specific directory per each host + host_specific_dir_name = host.replace('.', '-') + full_target = os.path.join(target, host_specific_dir_name) + os.mkdir(full_target) + else: + full_target = target + command.extend( + [format_scp_remote_host_path(user, host, source), full_target]) + else: + command.extend( + [source, format_scp_remote_host_path(user, host, target)]) + return command + + run_op_for_hosts(ctx, prefer_public_ip, parallel, query, 'scp', op) + + +def run_op_for_hosts(ctx, prefer_public_ip, parallel, query, op_name, op): get_ip = public_ip if prefer_public_ip else private_ip fmt_match = extended_public if prefer_public_ip else extended_private - matches = match_instances(ctx.obj['region'], query) - if not matches: - die('No matches found') - if len(matches) > 1: - click.echo('[0] All') - for i, inst in enumerate(matches): - click.echo('[{}] {}'.format(i+1, fmt_match(inst))) - click.echo('select servers [0-{}] (comma separated) '.format(len(matches)), nl=False) - indices = read_numbers(0, len(matches)) - if len(indices) == 1 and indices[0] == 0: - choices = matches - else: - choices = [matches[index - 1] for index in indices] - click.echo() - else: - choices = [matches[0]] - command = ['ssh', '-oStrictHostKeyChecking=no'] - if key: - command.extend(['-i', key]) + + choices = get_hosts_choice(ctx, fmt_match, query) + if parallel and len(choices) > 1: - message = 'sshing:\n{}'.format(pprint.pformat([fmt_match(c) for c in choices])) + message = 'running {}:\n{}'.format(op_name, pprint.pformat([fmt_match(c) for c in choices])) click.echo(message) click.echo() processes = [] for choice in choices: ip = get_ip(choice) - p = subprocess.Popen(command + [ip] + list(ssh_args), + p = subprocess.Popen(op(ip, len(choices)), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_consumer = OutputConsumer(input=p.stdout, output=sys.stdout, ip=ip) @@ -93,11 +137,31 @@ def ssh(ctx, key, prefer_public_ip, parallel, query, ssh_args): p['err'].join() else: for choice in choices: - message = 'sshing {}'.format(fmt_match(choice)) + message = 'running {} {}'.format(op_name, fmt_match(choice)) click.echo() click.echo(message) click.echo(u'\u2500' * len(message)) - subprocess.call(command + [get_ip(choice)] + list(ssh_args)) + subprocess.call(op(get_ip(choice), len(choices))) + + +def get_hosts_choice(ctx, fmt_match, query): + matches = match_instances(ctx.obj['region'], query) + if not matches: + die('No matches found') + if len(matches) > 1: + click.echo('[0] All') + for i, inst in enumerate(matches): + click.echo('[{}] {}'.format(i + 1, fmt_match(inst))) + click.echo('select servers [0-{}] (comma separated) '.format(len(matches)), nl=False) + indices = read_numbers(0, len(matches)) + if len(indices) == 1 and indices[0] == 0: + choices = matches + else: + choices = [matches[index - 1] for index in indices] + click.echo() + else: + choices = [matches[0]] + return choices @cli.command() @@ -152,6 +216,13 @@ def read_numbers(min_value, max_value): continue +def validate_identity_parameters(user, key, ssh_config): + if not key and not ssh_config: + die('Must supply either SSH key or SSH config file') + if ssh_config and (user or key): + die("The ssh-config option is mutually exclusive with the user and key options") + + class OutputConsumer(object): def __init__(self, input, output, ip): From ab478e1b8acf758537fb6918995f4a1cecdcb417 Mon Sep 17 00:00:00 2001 From: Re'em Bensimhon Date: Tue, 9 Oct 2018 11:56:46 +0300 Subject: [PATCH 03/22] Make ec2grep a standard forter package --- Dockerfile | 1 + Jenkinsfile | 13 +++++++++++++ Makefile | 8 ++++++++ VERSION | 1 + requirements.txt | 3 +++ setup.py | 6 +----- system-dependencies.sh | 0 7 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 Dockerfile create mode 100644 Jenkinsfile create mode 100644 Makefile create mode 100644 VERSION create mode 100644 requirements.txt create mode 100755 system-dependencies.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a89920f --- /dev/null +++ b/Dockerfile @@ -0,0 +1 @@ +FROM 174522763890.dkr.ecr.us-east-1.amazonaws.com/ubuntu-python2-onbuild \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..e9af16c --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,13 @@ +// vim: filetype=groovy +def base = evaluate(new File('/var/jenkins_home/workspace/Infra/build-scripts/build/Jenkinsfile')) +base.execute([ + customStages: [ + 'post-dist': { + def pkg = base.pypi_build() + base.pypi_dist_package(pkg) + } + ], + distImages: [], + allocateNode: true, + nodeLabel: 'master' +]) \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bc5404a --- /dev/null +++ b/Makefile @@ -0,0 +1,8 @@ +build: + docker build --pull -t forter/ec2grep . + +ci-test: + @echo "No tests" + +dist: + @echo "No images" diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..49d5957 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.1 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..98197bc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +boto3 +futures +click \ No newline at end of file diff --git a/setup.py b/setup.py index fc529d1..b16d0eb 100644 --- a/setup.py +++ b/setup.py @@ -11,11 +11,7 @@ packages=['ec2grep'], verion='0.1', keywords=['ec2', 'cli', 'aws', 'ssh'], - install_requires=[ - 'boto3', - 'futures', - 'click', - ], + install_requires=[l.strip() for l in open('requirements.txt').readlines()], entry_points={ 'console_scripts': [ 'ec2 = ec2grep:cli' diff --git a/system-dependencies.sh b/system-dependencies.sh new file mode 100755 index 0000000..e69de29 From 9b91a25767ef6a4537ec1a99f7e60772abbde532 Mon Sep 17 00:00:00 2001 From: Re'em Bensimhon Date: Tue, 9 Oct 2018 12:16:09 +0300 Subject: [PATCH 04/22] Make ec2grep a standard forter package --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index b16d0eb..8a92592 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,7 @@ author_email='roey.berman@gmail.com', packages=['ec2grep'], verion='0.1', + include_package_data=True, keywords=['ec2', 'cli', 'aws', 'ssh'], install_requires=[l.strip() for l in open('requirements.txt').readlines()], entry_points={ From 8ce17f82840bf019bc490e36004ddc55cc1422c5 Mon Sep 17 00:00:00 2001 From: Re'em Bensimhon Date: Tue, 9 Oct 2018 14:48:15 +0300 Subject: [PATCH 05/22] Make ec2grep a standard forter package --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8a92592..725bad9 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ author='Roey Berman', author_email='roey.berman@gmail.com', packages=['ec2grep'], - verion='0.1', + version='0.1', include_package_data=True, keywords=['ec2', 'cli', 'aws', 'ssh'], install_requires=[l.strip() for l in open('requirements.txt').readlines()], From b3ed19744836a1555f8e52b7b3e9702b9332f05e Mon Sep 17 00:00:00 2001 From: Re'em Bensimhon Date: Tue, 9 Oct 2018 14:52:14 +0300 Subject: [PATCH 06/22] Make ec2grep a standard forter package --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 725bad9..6ebb29f 100644 --- a/setup.py +++ b/setup.py @@ -3,13 +3,18 @@ except ImportError: from distutils.core import setup + +def get_version(): + with open('VERSION') as v: + return v.read().strip() + setup( name='ec2grep', description='EC2 cli tool', author='Roey Berman', author_email='roey.berman@gmail.com', packages=['ec2grep'], - version='0.1', + version=get_version(), include_package_data=True, keywords=['ec2', 'cli', 'aws', 'ssh'], install_requires=[l.strip() for l in open('requirements.txt').readlines()], From 0413cf98d035c0f34c4b4b85a604a29d6a11c3b3 Mon Sep 17 00:00:00 2001 From: Re'em Bensimhon Date: Tue, 9 Oct 2018 14:58:18 +0300 Subject: [PATCH 07/22] Make ec2grep a standard forter package --- MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..a777ab2 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include VERSION +include requirements.txt \ No newline at end of file From 73d93d20c5e6936fd4a9b2cb69b19b478fc41c53 Mon Sep 17 00:00:00 2001 From: Dan Kilman Date: Thu, 25 Oct 2018 11:04:28 +0300 Subject: [PATCH 08/22] Support ls for instances with no public ip (#9) --- ec2grep/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ec2grep/__init__.py b/ec2grep/__init__.py index e6f0588..13f49f9 100644 --- a/ec2grep/__init__.py +++ b/ec2grep/__init__.py @@ -24,8 +24,8 @@ name = (lambda i: {tag['Key']: tag['Value'] for tag in i.get('Tags', [])}.get('Name', '')) -public_ip = operator.itemgetter('PublicIpAddress') private_ip = operator.itemgetter('PrivateIpAddress') +public_ip = (lambda i: getattr(i, 'PublicIpAddress', None)) extended_public = (lambda i: '{} ({})'.format(name(i), public_ip(i))) extended_private = (lambda i: '{} ({})'.format(name(i), private_ip(i))) extended = (lambda i: '{} (public: {}, private: {})'.format(name(i), public_ip(i), private_ip(i))) @@ -176,7 +176,7 @@ def ls(ctx, formatter, delim, custom_format, query): die('No matches found') if not custom_format: formatter = formatter.join('{}') - click.echo(delim.join(formatter.format(**{k: f(m) for k, f in formatters.iteritems()}) for m in matches)) + click.echo(delim.join(formatter.format(**{k: f(m) for k, f in formatters.items()}) for m in matches)) def _get_instances(ec2, filter_): @@ -192,7 +192,7 @@ def match_instances(region_name, query, attributes=DEFAULT_ATTRIBUTES): instance_lists = executor.map(get_instances, [ {'Name': attr, 'Values': ['*{}*'.format(query)]} for attr in attributes ]) - chained = (i for i in itertools.chain.from_iterable(instance_lists) if 'PublicIpAddress' in i) + chained = (i for i in itertools.chain.from_iterable(instance_lists)) return sorted(chained, key=name) From 231fceb90b89d33848c536be7614e7c1fb5fa9ed Mon Sep 17 00:00:00 2001 From: Dan Kilman Date: Thu, 25 Oct 2018 15:17:04 +0300 Subject: [PATCH 09/22] fix regression --- ec2grep/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ec2grep/__init__.py b/ec2grep/__init__.py index 13f49f9..a4b4f46 100644 --- a/ec2grep/__init__.py +++ b/ec2grep/__init__.py @@ -24,8 +24,8 @@ name = (lambda i: {tag['Key']: tag['Value'] for tag in i.get('Tags', [])}.get('Name', '')) -private_ip = operator.itemgetter('PrivateIpAddress') -public_ip = (lambda i: getattr(i, 'PublicIpAddress', None)) +private_ip = (lambda i: i.get('PrivateIpAddress', None)) +public_ip = (lambda i: i.get('PublicIpAddress', None)) extended_public = (lambda i: '{} ({})'.format(name(i), public_ip(i))) extended_private = (lambda i: '{} ({})'.format(name(i), private_ip(i))) extended = (lambda i: '{} (public: {}, private: {})'.format(name(i), public_ip(i), private_ip(i))) @@ -192,7 +192,7 @@ def match_instances(region_name, query, attributes=DEFAULT_ATTRIBUTES): instance_lists = executor.map(get_instances, [ {'Name': attr, 'Values': ['*{}*'.format(query)]} for attr in attributes ]) - chained = (i for i in itertools.chain.from_iterable(instance_lists)) + chained = (i for i in itertools.chain.from_iterable(instance_lists) if 'PublicIpAddress' in i or 'PrivateIpAddress' in i) return sorted(chained, key=name) From ec571f0c916a5a90436a25abc24e8ebdf796ec2e Mon Sep 17 00:00:00 2001 From: Re'em Bensimhon Date: Sun, 4 Nov 2018 15:36:01 +0200 Subject: [PATCH 10/22] Add constraints.txt file to build --- constraints.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 constraints.txt diff --git a/constraints.txt b/constraints.txt new file mode 100644 index 0000000..e69de29 From 334a2b41e9f97eb152c6841751fc5bb297a54112 Mon Sep 17 00:00:00 2001 From: Re'em Bensimhon Date: Wed, 16 Jan 2019 20:48:39 +0200 Subject: [PATCH 11/22] User not mutually exclusive with config --- ec2grep/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ec2grep/__init__.py b/ec2grep/__init__.py index a4b4f46..cc38e11 100644 --- a/ec2grep/__init__.py +++ b/ec2grep/__init__.py @@ -219,8 +219,8 @@ def read_numbers(min_value, max_value): def validate_identity_parameters(user, key, ssh_config): if not key and not ssh_config: die('Must supply either SSH key or SSH config file') - if ssh_config and (user or key): - die("The ssh-config option is mutually exclusive with the user and key options") + if ssh_config and key: + die("The ssh-config option is mutually exclusive with the key option") class OutputConsumer(object): From 981be314aa5e33aa780ec62f86e1a26020dbea82 Mon Sep 17 00:00:00 2001 From: Dan Kilman Date: Mon, 1 Apr 2019 00:35:21 +0300 Subject: [PATCH 12/22] require tty in ssh commands --- ec2grep/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ec2grep/__init__.py b/ec2grep/__init__.py index cc38e11..504794f 100644 --- a/ec2grep/__init__.py +++ b/ec2grep/__init__.py @@ -59,7 +59,7 @@ def ssh(ctx, user, key, ssh_config, prefer_public_ip, parallel, query, ssh_args) validate_identity_parameters(user, key, ssh_config) def op(host, host_count): - command = ['ssh', '-oStrictHostKeyChecking=no'] + command = ['ssh', '-oStrictHostKeyChecking=no', '-t'] command.extend(['-i', key]) if key else command.extend(['-F', ssh_config]) if user: command.extend(['-l', user]) From fcad7ea73bb868db881cd00ecf6c8dc61d634d47 Mon Sep 17 00:00:00 2001 From: Shalom Yerushalmy Date: Wed, 20 Nov 2019 08:49:01 +0200 Subject: [PATCH 13/22] add who to ec2 (#14) --- ec2grep/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ec2grep/__init__.py b/ec2grep/__init__.py index 504794f..e4f64cf 100644 --- a/ec2grep/__init__.py +++ b/ec2grep/__init__.py @@ -24,11 +24,13 @@ name = (lambda i: {tag['Key']: tag['Value'] for tag in i.get('Tags', [])}.get('Name', '')) +initiator = (lambda i: {tag['Key']: tag['Value'] for tag in i.get('Tags', [])}.get('Initiator', '')) private_ip = (lambda i: i.get('PrivateIpAddress', None)) public_ip = (lambda i: i.get('PublicIpAddress', None)) extended_public = (lambda i: '{} ({})'.format(name(i), public_ip(i))) extended_private = (lambda i: '{} ({})'.format(name(i), private_ip(i))) extended = (lambda i: '{} (public: {}, private: {})'.format(name(i), public_ip(i), private_ip(i))) +extended_initiator = (lambda i: '{} (name: {})'.format(initiator(i), name(i))) formatters = { 'extended': extended, 'extended_public': extended_public, @@ -36,6 +38,8 @@ 'public_ip': public_ip, 'private_ip': private_ip, 'name': name, + 'initiator': initiator, + 'extended_initiator': extended_initiator, } From ea595d12cd773129a2480134fd4ceba9c3943b05 Mon Sep 17 00:00:00 2001 From: Pavel Brodsky Date: Sun, 5 Apr 2020 18:27:42 +0300 Subject: [PATCH 14/22] #1169628070917550: Updatge ec2grep dependencies (#15) * Transfer to pyproject.toml * futures is only relevant for python 2 --- Dockerfile | 3 +- Jenkinsfile | 18 ++--- MANIFEST.in | 2 - Makefile | 33 +++++++-- constraints.txt | 0 poetry.lock | 157 +++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 34 +++++++++ requirements.txt | 3 - setup.py | 26 ------- system-dependencies.sh | 0 10 files changed, 228 insertions(+), 48 deletions(-) delete mode 100644 MANIFEST.in delete mode 100644 constraints.txt create mode 100644 poetry.lock create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.py delete mode 100755 system-dependencies.sh diff --git a/Dockerfile b/Dockerfile index a89920f..4f22d14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1 +1,2 @@ -FROM 174522763890.dkr.ecr.us-east-1.amazonaws.com/ubuntu-python2-onbuild \ No newline at end of file +ARG PYTHON_VERSION=2.7.17 +FROM 174522763890.dkr.ecr.us-east-1.amazonaws.com/ubuntu-python-pyproject:${PYTHON_VERSION} \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index e9af16c..ce85bb5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,13 +1,7 @@ // vim: filetype=groovy -def base = evaluate(new File('/var/jenkins_home/workspace/Infra/build-scripts/build/Jenkinsfile')) -base.execute([ - customStages: [ - 'post-dist': { - def pkg = base.pypi_build() - base.pypi_dist_package(pkg) - } - ], - distImages: [], - allocateNode: true, - nodeLabel: 'master' -]) \ No newline at end of file +node('master') { + def base = load('/var/jenkins_home/workspace/Infra/build-scripts/build/Jenkinsfile') + base.execute([ + customStages: base.get_pypi_stages(true) + ]) +} \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index a777ab2..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include VERSION -include requirements.txt \ No newline at end of file diff --git a/Makefile b/Makefile index bc5404a..031c828 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,33 @@ -build: - docker build --pull -t forter/ec2grep . +ECR_URL ?= 174522763890.dkr.ecr.us-east-1.amazonaws.com + +.PHONY: build +build: build-py2 build-py3 + +build-py2: PYTHON_VERSION = 2.7.17 +build-py3: PYTHON_VERSION = 3.8.1 + + +build-py3 build-py2: + @echo "======= Building Python $(PYTHON_VERSION) ========" + docker build --pull --build-arg "PYTHON_VERSION=$(PYTHON_VERSION)" . ci-test: - @echo "No tests" + echo "No tests" dist: - @echo "No images" + echo "Actual dist is done in the post-dist stage" + +ci-lint: + @echo "======= Linting ========" + docker run \ + -v $(PWD)/:/app/ \ + -v $(PWD)/test-reports:/test-reports/ \ + ${ECR_URL}/python-black + +black: + docker run \ + -v $(PWD)/:/app/ \ + --entrypoint black \ + ${ECR_URL}/python-black \ + /app/ + diff --git a/constraints.txt b/constraints.txt deleted file mode 100644 index e69de29..0000000 diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..49782f6 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,157 @@ +[[package]] +category = "main" +description = "The AWS SDK for Python" +name = "boto3" +optional = false +python-versions = "*" +version = "1.12.36" + +[package.dependencies] +botocore = ">=1.15.36,<1.16.0" +jmespath = ">=0.7.1,<1.0.0" +s3transfer = ">=0.3.0,<0.4.0" + +[[package]] +category = "main" +description = "Low-level, data-driven core of boto 3." +name = "botocore" +optional = false +python-versions = "*" +version = "1.15.36" + +[package.dependencies] +docutils = ">=0.10,<0.16" +jmespath = ">=0.7.1,<1.0.0" +python-dateutil = ">=2.1,<3.0.0" + +[package.dependencies.urllib3] +python = "<3.4.0 || >=3.5.0" +version = ">=1.20,<1.26" + +[[package]] +category = "main" +description = "Composable command line interface toolkit" +name = "click" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "7.0" + +[[package]] +category = "main" +description = "Docutils -- Python Documentation Utilities" +name = "docutils" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "0.15.2" + +[[package]] +category = "main" +description = "Backport of the concurrent.futures package from Python 3" +marker = "python_version >= \"2.7\" and python_version < \"2.8\" or python_version == \"2.7\"" +name = "futures" +optional = false +python-versions = ">=2.6, <3" +version = "3.3.0" + +[[package]] +category = "main" +description = "JSON Matching Expressions" +name = "jmespath" +optional = false +python-versions = "*" +version = "0.9.5" + +[[package]] +category = "main" +description = "Extensions to the standard Python datetime module" +name = "python-dateutil" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +version = "2.8.1" + +[package.dependencies] +six = ">=1.5" + +[[package]] +category = "main" +description = "An Amazon S3 Transfer Manager" +name = "s3transfer" +optional = false +python-versions = "*" +version = "0.3.3" + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.dependencies.futures] +python = ">=2.7,<2.8" +version = ">=2.2.0,<4.0.0" + +[[package]] +category = "main" +description = "Python 2 and 3 compatibility utilities" +name = "six" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +version = "1.14.0" + +[[package]] +category = "main" +description = "HTTP library with thread-safe connection pooling, file post, and more." +marker = "python_version != \"3.4\"" +name = "urllib3" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +version = "1.25.8" + +[package.extras] +brotli = ["brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] + +[metadata] +content-hash = "8dbdfa76c724dff6269e96df4475a2318b3795304c54c6a43cb26cadeaee645c" +python-versions = "~2.7 || ^3.6" + +[metadata.files] +boto3 = [ + {file = "boto3-1.12.36-py2.py3-none-any.whl", hash = "sha256:57397f9ad3e9afc17e6100a38c6e631b6545aabc7f8c38b86ff2c6f5931d0ebf"}, + {file = "boto3-1.12.36.tar.gz", hash = "sha256:911994ef46595e8ab9f08eee6b666caea050937b96d54394292e958330cd7ad5"}, +] +botocore = [ + {file = "botocore-1.15.36-py2.py3-none-any.whl", hash = "sha256:d2233e19b6a60792185b15fe4cd8f92c5579298e5079daf17f66f6e639585e3a"}, + {file = "botocore-1.15.36.tar.gz", hash = "sha256:5fc5e8629b5375d591a47d05baaff6ccdf5c3b617aad1e14286be458092c9e53"}, +] +click = [ + {file = "Click-7.0-py2.py3-none-any.whl", hash = "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13"}, + {file = "Click-7.0.tar.gz", hash = "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"}, +] +docutils = [ + {file = "docutils-0.15.2-py2-none-any.whl", hash = "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827"}, + {file = "docutils-0.15.2-py3-none-any.whl", hash = "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0"}, + {file = "docutils-0.15.2.tar.gz", hash = "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99"}, +] +futures = [ + {file = "futures-3.3.0-py2-none-any.whl", hash = "sha256:49b3f5b064b6e3afc3316421a3f25f66c137ae88f068abbf72830170033c5e16"}, + {file = "futures-3.3.0.tar.gz", hash = "sha256:7e033af76a5e35f58e56da7a91e687706faf4e7bdfb2cbc3f2cca6b9bcda9794"}, +] +jmespath = [ + {file = "jmespath-0.9.5-py2.py3-none-any.whl", hash = "sha256:695cb76fa78a10663425d5b73ddc5714eb711157e52704d69be03b1a02ba4fec"}, + {file = "jmespath-0.9.5.tar.gz", hash = "sha256:cca55c8d153173e21baa59983015ad0daf603f9cb799904ff057bfb8ff8dc2d9"}, +] +python-dateutil = [ + {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, + {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"}, +] +s3transfer = [ + {file = "s3transfer-0.3.3-py2.py3-none-any.whl", hash = "sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13"}, + {file = "s3transfer-0.3.3.tar.gz", hash = "sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db"}, +] +six = [ + {file = "six-1.14.0-py2.py3-none-any.whl", hash = "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"}, + {file = "six-1.14.0.tar.gz", hash = "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a"}, +] +urllib3 = [ + {file = "urllib3-1.25.8-py2.py3-none-any.whl", hash = "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc"}, + {file = "urllib3-1.25.8.tar.gz", hash = "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e8dbff5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,34 @@ +[tool.poetry] +name = "ec2grep" +version = "0.1.0" +description = "EC2 cli tool" +authors = ["Roey Berman "] +keywords = ["ec2", "cli", "aws", "ssh"] + +packages = [ + { include = "ec2grep" } +] + +[tool.poetry.scripts] +ec2 = "ec2grep:cli" + +[[tool.poetry.source]] +name = "forter" +url = "https://artifactory.frdstr.com/artifactory/api/pypi/pypi/simple" +secondary = true + +[tool.poetry.dependencies] +python = "~2.7 || ^3.6" + +boto3 = "^1.9.86" +click = "~7.0" + +# Python 2 specific packages: +futures = { version = "3.3.0", python = "~2.7"} + +[tool.black] +line-length = 120 + +[build-system] +requires = ["poetry>=1.0.0"] +build-backend = "poetry.masonry.api" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 98197bc..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -boto3 -futures -click \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 6ebb29f..0000000 --- a/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -try: - from setuptools import setup -except ImportError: - from distutils.core import setup - - -def get_version(): - with open('VERSION') as v: - return v.read().strip() - -setup( - name='ec2grep', - description='EC2 cli tool', - author='Roey Berman', - author_email='roey.berman@gmail.com', - packages=['ec2grep'], - version=get_version(), - include_package_data=True, - keywords=['ec2', 'cli', 'aws', 'ssh'], - install_requires=[l.strip() for l in open('requirements.txt').readlines()], - entry_points={ - 'console_scripts': [ - 'ec2 = ec2grep:cli' - ] - } -) diff --git a/system-dependencies.sh b/system-dependencies.sh deleted file mode 100755 index e69de29..0000000 From 3b095d5da38fbf7efff5c12473cdb447156aa9f1 Mon Sep 17 00:00:00 2001 From: Pavel Brodsky Date: Sun, 5 Apr 2020 18:31:18 +0300 Subject: [PATCH 15/22] Actually blackify (#16) --- ec2grep/__init__.py | 177 ++++++++++++++++++++++---------------------- 1 file changed, 87 insertions(+), 90 deletions(-) diff --git a/ec2grep/__init__.py b/ec2grep/__init__.py index e4f64cf..f6c2909 100644 --- a/ec2grep/__init__.py +++ b/ec2grep/__init__.py @@ -16,104 +16,102 @@ DEFAULT_ATTRIBUTES = ( - 'tag:Name', - 'network-interface.addresses.association.public-ip', - 'network-interface.addresses.private-ip-address', - 'network-interface.private-dns-name', + "tag:Name", + "network-interface.addresses.association.public-ip", + "network-interface.addresses.private-ip-address", + "network-interface.private-dns-name", ) -name = (lambda i: {tag['Key']: tag['Value'] for tag in i.get('Tags', [])}.get('Name', '')) -initiator = (lambda i: {tag['Key']: tag['Value'] for tag in i.get('Tags', [])}.get('Initiator', '')) -private_ip = (lambda i: i.get('PrivateIpAddress', None)) -public_ip = (lambda i: i.get('PublicIpAddress', None)) -extended_public = (lambda i: '{} ({})'.format(name(i), public_ip(i))) -extended_private = (lambda i: '{} ({})'.format(name(i), private_ip(i))) -extended = (lambda i: '{} (public: {}, private: {})'.format(name(i), public_ip(i), private_ip(i))) -extended_initiator = (lambda i: '{} (name: {})'.format(initiator(i), name(i))) +name = lambda i: {tag["Key"]: tag["Value"] for tag in i.get("Tags", [])}.get("Name", "") +initiator = lambda i: {tag["Key"]: tag["Value"] for tag in i.get("Tags", [])}.get("Initiator", "") +private_ip = lambda i: i.get("PrivateIpAddress", None) +public_ip = lambda i: i.get("PublicIpAddress", None) +extended_public = lambda i: "{} ({})".format(name(i), public_ip(i)) +extended_private = lambda i: "{} ({})".format(name(i), private_ip(i)) +extended = lambda i: "{} (public: {}, private: {})".format(name(i), public_ip(i), private_ip(i)) +extended_initiator = lambda i: "{} (name: {})".format(initiator(i), name(i)) formatters = { - 'extended': extended, - 'extended_public': extended_public, - 'extended_private': extended_private, - 'public_ip': public_ip, - 'private_ip': private_ip, - 'name': name, - 'initiator': initiator, - 'extended_initiator': extended_initiator, + "extended": extended, + "extended_public": extended_public, + "extended_private": extended_private, + "public_ip": public_ip, + "private_ip": private_ip, + "name": name, + "initiator": initiator, + "extended_initiator": extended_initiator, } @click.group() -@click.option('--region', '-r', default='us-east-1') +@click.option("--region", "-r", default="us-east-1") @click.pass_context def cli(ctx, region): - ctx.obj = {'region': region} + ctx.obj = {"region": region} @cli.command() -@click.option('--user', '-u') -@click.option('--key', '-i', help='SSH key') -@click.option('--ssh-config', '-F', help='SSH config file') -@click.option('--prefer-public-ip', '-p', is_flag=True, default=False) -@click.option('--parallel', is_flag=True, default=False) -@click.argument('query') -@click.argument('ssh_args', nargs=-1, type=click.UNPROCESSED) +@click.option("--user", "-u") +@click.option("--key", "-i", help="SSH key") +@click.option("--ssh-config", "-F", help="SSH config file") +@click.option("--prefer-public-ip", "-p", is_flag=True, default=False) +@click.option("--parallel", is_flag=True, default=False) +@click.argument("query") +@click.argument("ssh_args", nargs=-1, type=click.UNPROCESSED) @click.pass_context def ssh(ctx, user, key, ssh_config, prefer_public_ip, parallel, query, ssh_args): validate_identity_parameters(user, key, ssh_config) def op(host, host_count): - command = ['ssh', '-oStrictHostKeyChecking=no', '-t'] - command.extend(['-i', key]) if key else command.extend(['-F', ssh_config]) + command = ["ssh", "-oStrictHostKeyChecking=no", "-t"] + command.extend(["-i", key]) if key else command.extend(["-F", ssh_config]) if user: - command.extend(['-l', user]) + command.extend(["-l", user]) command.extend([host] + list(ssh_args)) return command - run_op_for_hosts(ctx, prefer_public_ip, parallel, query, 'ssh', op) + run_op_for_hosts(ctx, prefer_public_ip, parallel, query, "ssh", op) @cli.command() -@click.option('--user', '-u') -@click.option('--key', '-i', help='SSH key') -@click.option('--ssh-config', '-F', help='SSH config file') -@click.option('--prefer-public-ip', '-p', is_flag=True, default=False) -@click.option('--parallel', is_flag=True, default=False) -@click.option('--download', '-d', is_flag=True, default=False) -@click.argument('query') -@click.argument('source', type=str, required=True) -@click.argument('target', type=str, required=False) -@click.argument('scp_args', nargs=-1, type=click.UNPROCESSED) +@click.option("--user", "-u") +@click.option("--key", "-i", help="SSH key") +@click.option("--ssh-config", "-F", help="SSH config file") +@click.option("--prefer-public-ip", "-p", is_flag=True, default=False) +@click.option("--parallel", is_flag=True, default=False) +@click.option("--download", "-d", is_flag=True, default=False) +@click.argument("query") +@click.argument("source", type=str, required=True) +@click.argument("target", type=str, required=False) +@click.argument("scp_args", nargs=-1, type=click.UNPROCESSED) @click.pass_context def scp(ctx, user, key, ssh_config, prefer_public_ip, parallel, query, download, source, target, scp_args): validate_identity_parameters(user, key, ssh_config) if download and not os.path.isdir(target): - die('Download target must be an existing directory') + die("Download target must be an existing directory") def format_scp_remote_host_path(user, host, path): - return '{}@{}:{}'.format(user, host, path) if user else '{}:{}'.format(host, path) + return "{}@{}:{}".format(user, host, path) if user else "{}:{}".format(host, path) def op(host, host_count): - command = ['scp', '-oStrictHostKeyChecking=no'] - command.extend(['-i', key]) if key else command.extend(['-F', ssh_config]) + command = ["scp", "-oStrictHostKeyChecking=no"] + command.extend(["-i", key]) if key else command.extend(["-F", ssh_config]) command.extend(list(scp_args)) if download: if host_count > 1: # when downloading from multiple hosts, use a specific directory per each host - host_specific_dir_name = host.replace('.', '-') + host_specific_dir_name = host.replace(".", "-") full_target = os.path.join(target, host_specific_dir_name) os.mkdir(full_target) else: full_target = target - command.extend( - [format_scp_remote_host_path(user, host, source), full_target]) + command.extend([format_scp_remote_host_path(user, host, source), full_target]) else: - command.extend( - [source, format_scp_remote_host_path(user, host, target)]) + command.extend([source, format_scp_remote_host_path(user, host, target)]) return command - run_op_for_hosts(ctx, prefer_public_ip, parallel, query, 'scp', op) + run_op_for_hosts(ctx, prefer_public_ip, parallel, query, "scp", op) def run_op_for_hosts(ctx, prefer_public_ip, parallel, query, op_name, op): @@ -123,40 +121,38 @@ def run_op_for_hosts(ctx, prefer_public_ip, parallel, query, op_name, op): choices = get_hosts_choice(ctx, fmt_match, query) if parallel and len(choices) > 1: - message = 'running {}:\n{}'.format(op_name, pprint.pformat([fmt_match(c) for c in choices])) + message = "running {}:\n{}".format(op_name, pprint.pformat([fmt_match(c) for c in choices])) click.echo(message) click.echo() processes = [] for choice in choices: ip = get_ip(choice) - p = subprocess.Popen(op(ip, len(choices)), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = subprocess.Popen(op(ip, len(choices)), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_consumer = OutputConsumer(input=p.stdout, output=sys.stdout, ip=ip) stderr_consumer = OutputConsumer(input=p.stderr, output=sys.stderr, ip=ip) - processes.append({'out': stdout_consumer, 'err': stderr_consumer, 'p': p}) + processes.append({"out": stdout_consumer, "err": stderr_consumer, "p": p}) for p in processes: - p['p'].wait() - p['out'].join() - p['err'].join() + p["p"].wait() + p["out"].join() + p["err"].join() else: for choice in choices: - message = 'running {} {}'.format(op_name, fmt_match(choice)) + message = "running {} {}".format(op_name, fmt_match(choice)) click.echo() click.echo(message) - click.echo(u'\u2500' * len(message)) + click.echo(u"\u2500" * len(message)) subprocess.call(op(get_ip(choice), len(choices))) def get_hosts_choice(ctx, fmt_match, query): - matches = match_instances(ctx.obj['region'], query) + matches = match_instances(ctx.obj["region"], query) if not matches: - die('No matches found') + die("No matches found") if len(matches) > 1: - click.echo('[0] All') + click.echo("[0] All") for i, inst in enumerate(matches): - click.echo('[{}] {}'.format(i + 1, fmt_match(inst))) - click.echo('select servers [0-{}] (comma separated) '.format(len(matches)), nl=False) + click.echo("[{}] {}".format(i + 1, fmt_match(inst))) + click.echo("select servers [0-{}] (comma separated) ".format(len(matches)), nl=False) indices = read_numbers(0, len(matches)) if len(indices) == 1 and indices[0] == 0: choices = matches @@ -169,34 +165,36 @@ def get_hosts_choice(ctx, fmt_match, query): @cli.command() -@click.option('--delim', '-d', default='\n') -@click.option('--formatter', '-f', default='extended_private') -@click.option('--custom-format', '-c', is_flag=True, default=False) -@click.argument('query') +@click.option("--delim", "-d", default="\n") +@click.option("--formatter", "-f", default="extended_private") +@click.option("--custom-format", "-c", is_flag=True, default=False) +@click.argument("query") @click.pass_context def ls(ctx, formatter, delim, custom_format, query): - matches = match_instances(ctx.obj['region'], query) + matches = match_instances(ctx.obj["region"], query) if not matches: - die('No matches found') + die("No matches found") if not custom_format: - formatter = formatter.join('{}') + formatter = formatter.join("{}") click.echo(delim.join(formatter.format(**{k: f(m) for k, f in formatters.items()}) for m in matches)) def _get_instances(ec2, filter_): response = ec2.describe_instances(Filters=[filter_]) - reservations = response['Reservations'] - return list(itertools.chain.from_iterable(r['Instances'] for r in reservations)) + reservations = response["Reservations"] + return list(itertools.chain.from_iterable(r["Instances"] for r in reservations)) def match_instances(region_name, query, attributes=DEFAULT_ATTRIBUTES): - ec2 = boto3.client('ec2', region_name=region_name) + ec2 = boto3.client("ec2", region_name=region_name) get_instances = functools.partial(_get_instances, ec2) with concurrent.futures.ThreadPoolExecutor(len(attributes)) as executor: - instance_lists = executor.map(get_instances, [ - {'Name': attr, 'Values': ['*{}*'.format(query)]} for attr in attributes - ]) - chained = (i for i in itertools.chain.from_iterable(instance_lists) if 'PublicIpAddress' in i or 'PrivateIpAddress' in i) + instance_lists = executor.map( + get_instances, [{"Name": attr, "Values": ["*{}*".format(query)]} for attr in attributes] + ) + chained = ( + i for i in itertools.chain.from_iterable(instance_lists) if "PublicIpAddress" in i or "PrivateIpAddress" in i + ) return sorted(chained, key=name) @@ -209,11 +207,11 @@ def read_numbers(min_value, max_value): while True: try: choices = six.moves.input() - choices = [int(c.strip()) for c in choices.split(',')] + choices = [int(c.strip()) for c in choices.split(",")] if not (all(min_value <= c <= max_value for c in choices)): - raise ValueError('Invalid input') + raise ValueError("Invalid input") if len(choices) != 1 and 0 in choices: - raise ValueError('Invalid input') + raise ValueError("Invalid input") return choices except ValueError as e: click.echo(str(e), err=True) @@ -222,13 +220,12 @@ def read_numbers(min_value, max_value): def validate_identity_parameters(user, key, ssh_config): if not key and not ssh_config: - die('Must supply either SSH key or SSH config file') + die("Must supply either SSH key or SSH config file") if ssh_config and key: die("The ssh-config option is mutually exclusive with the key option") class OutputConsumer(object): - def __init__(self, input, output, ip): self.ip = ip self.input = input @@ -238,12 +235,12 @@ def __init__(self, input, output, ip): self.consumer.start() def consume_output(self): - for line in iter(self.input.readline, b''): - self.output.write('[{:<15}] {}'.format(self.ip, line)) + for line in iter(self.input.readline, b""): + self.output.write("[{:<15}] {}".format(self.ip, line)) def join(self): self.consumer.join() -if __name__ == '__main__': +if __name__ == "__main__": cli() From d3436b252d472f34bbbae67862786be8fec6ddaf Mon Sep 17 00:00:00 2001 From: Udi Luxenburg Date: Thu, 9 Jul 2020 15:44:59 +0300 Subject: [PATCH 16/22] update README install instructions - point to correct location --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 438daff..2abc2fb 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Preferally for the default system interpreter instead of a virtualenv. ```bash -pip install git+https://github.com/bergundy/ec2grep +pip install git+https://github.com/forter/ec2grep ``` ### Usage From d50b0d7a2f07d73855f6307449121bdf177ce8da Mon Sep 17 00:00:00 2001 From: Gilad Peleg Date: Tue, 15 Sep 2020 14:25:56 +0300 Subject: [PATCH 17/22] Update ec2grep click version contraints (#18) --- Makefile | 6 ++---- ec2grep/__init__.py | 2 -- poetry.lock | 46 ++++++++++++++++++++++----------------------- pyproject.toml | 9 ++------- 4 files changed, 27 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index 031c828..a2f218a 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,10 @@ -ECR_URL ?= 174522763890.dkr.ecr.us-east-1.amazonaws.com +ECR_URL ?= 174522763890.dkr.ecr.us-east-1.amazonaws.com .PHONY: build build: build-py2 build-py3 build-py2: PYTHON_VERSION = 2.7.17 -build-py3: PYTHON_VERSION = 3.8.1 - +build-py3: PYTHON_VERSION = 3.8.5 build-py3 build-py2: @echo "======= Building Python $(PYTHON_VERSION) ========" @@ -30,4 +29,3 @@ black: --entrypoint black \ ${ECR_URL}/python-black \ /app/ - diff --git a/ec2grep/__init__.py b/ec2grep/__init__.py index f6c2909..d87abdd 100644 --- a/ec2grep/__init__.py +++ b/ec2grep/__init__.py @@ -3,7 +3,6 @@ import concurrent.futures import functools import itertools -import operator import os import pprint import subprocess @@ -14,7 +13,6 @@ import click import six - DEFAULT_ATTRIBUTES = ( "tag:Name", "network-interface.addresses.association.public-ip", diff --git a/poetry.lock b/poetry.lock index 49782f6..048e1e7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4,10 +4,10 @@ description = "The AWS SDK for Python" name = "boto3" optional = false python-versions = "*" -version = "1.12.36" +version = "1.14.61" [package.dependencies] -botocore = ">=1.15.36,<1.16.0" +botocore = ">=1.17.61,<1.18.0" jmespath = ">=0.7.1,<1.0.0" s3transfer = ">=0.3.0,<0.4.0" @@ -17,7 +17,7 @@ description = "Low-level, data-driven core of boto 3." name = "botocore" optional = false python-versions = "*" -version = "1.15.36" +version = "1.17.61" [package.dependencies] docutils = ">=0.10,<0.16" @@ -33,8 +33,8 @@ category = "main" description = "Composable command line interface toolkit" name = "click" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "7.0" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "7.1.2" [[package]] category = "main" @@ -58,8 +58,8 @@ category = "main" description = "JSON Matching Expressions" name = "jmespath" optional = false -python-versions = "*" -version = "0.9.5" +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "0.10.0" [[package]] category = "main" @@ -93,7 +93,7 @@ description = "Python 2 and 3 compatibility utilities" name = "six" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -version = "1.14.0" +version = "1.15.0" [[package]] category = "main" @@ -102,29 +102,29 @@ marker = "python_version != \"3.4\"" name = "urllib3" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" -version = "1.25.8" +version = "1.25.10" [package.extras] brotli = ["brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0.14)", "ipaddress"] socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] [metadata] -content-hash = "8dbdfa76c724dff6269e96df4475a2318b3795304c54c6a43cb26cadeaee645c" +content-hash = "1ad7ed3ff7853839b9c46ddaac5c5a6d1d331e35619bf928675122c82110f5b2" +lock-version = "1.0" python-versions = "~2.7 || ^3.6" [metadata.files] boto3 = [ - {file = "boto3-1.12.36-py2.py3-none-any.whl", hash = "sha256:57397f9ad3e9afc17e6100a38c6e631b6545aabc7f8c38b86ff2c6f5931d0ebf"}, - {file = "boto3-1.12.36.tar.gz", hash = "sha256:911994ef46595e8ab9f08eee6b666caea050937b96d54394292e958330cd7ad5"}, + {file = "boto3-1.14.61.tar.gz", hash = "sha256:a1c738ff178fc6ed8951559053c1142c71d166d17642361bbe63584be2f50a00"}, ] botocore = [ - {file = "botocore-1.15.36-py2.py3-none-any.whl", hash = "sha256:d2233e19b6a60792185b15fe4cd8f92c5579298e5079daf17f66f6e639585e3a"}, - {file = "botocore-1.15.36.tar.gz", hash = "sha256:5fc5e8629b5375d591a47d05baaff6ccdf5c3b617aad1e14286be458092c9e53"}, + {file = "botocore-1.17.61-py2.py3-none-any.whl", hash = "sha256:4a2931b6cdbb1caf797eaf81f1f42f9cf243f94f06b3144d36a2f016a58ad125"}, + {file = "botocore-1.17.61.tar.gz", hash = "sha256:71160b06c52f21cc477bc4b5eb35fc78f1f89e363b86d93fcbebfe366cc4d8f0"}, ] click = [ - {file = "Click-7.0-py2.py3-none-any.whl", hash = "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13"}, - {file = "Click-7.0.tar.gz", hash = "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"}, + {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, + {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, ] docutils = [ {file = "docutils-0.15.2-py2-none-any.whl", hash = "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827"}, @@ -136,8 +136,8 @@ futures = [ {file = "futures-3.3.0.tar.gz", hash = "sha256:7e033af76a5e35f58e56da7a91e687706faf4e7bdfb2cbc3f2cca6b9bcda9794"}, ] jmespath = [ - {file = "jmespath-0.9.5-py2.py3-none-any.whl", hash = "sha256:695cb76fa78a10663425d5b73ddc5714eb711157e52704d69be03b1a02ba4fec"}, - {file = "jmespath-0.9.5.tar.gz", hash = "sha256:cca55c8d153173e21baa59983015ad0daf603f9cb799904ff057bfb8ff8dc2d9"}, + {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, + {file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"}, ] python-dateutil = [ {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, @@ -148,10 +148,10 @@ s3transfer = [ {file = "s3transfer-0.3.3.tar.gz", hash = "sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db"}, ] six = [ - {file = "six-1.14.0-py2.py3-none-any.whl", hash = "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"}, - {file = "six-1.14.0.tar.gz", hash = "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a"}, + {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, + {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] urllib3 = [ - {file = "urllib3-1.25.8-py2.py3-none-any.whl", hash = "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc"}, - {file = "urllib3-1.25.8.tar.gz", hash = "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"}, + {file = "urllib3-1.25.10-py2.py3-none-any.whl", hash = "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"}, + {file = "urllib3-1.25.10.tar.gz", hash = "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a"}, ] diff --git a/pyproject.toml b/pyproject.toml index e8dbff5..d5898d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,19 +12,14 @@ packages = [ [tool.poetry.scripts] ec2 = "ec2grep:cli" -[[tool.poetry.source]] -name = "forter" -url = "https://artifactory.frdstr.com/artifactory/api/pypi/pypi/simple" -secondary = true - [tool.poetry.dependencies] python = "~2.7 || ^3.6" boto3 = "^1.9.86" -click = "~7.0" +click = "^7.0" # Python 2 specific packages: -futures = { version = "3.3.0", python = "~2.7"} +futures = { version = "3.3.0", python = "~2.7" } [tool.black] line-length = 120 From 5f84ff1ffc0152c4956cf2b6901d36599d3fc52e Mon Sep 17 00:00:00 2001 From: Gilad Peleg Date: Tue, 15 Sep 2020 15:12:37 +0300 Subject: [PATCH 18/22] Update readme with pipx install instructions (#19) --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2abc2fb..3e29edd 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,30 @@ ## ec2grep - EC2 cli tool ### Install -Preferally for the default system interpreter instead of a virtualenv. + +Make sure you have `pipx` available in your system (`brew install pipx` if not). ```bash -pip install git+https://github.com/forter/ec2grep +pipx install ec2grep --index-url https://artifactory.frdstr.com/artifactory/api/pypi/pypi/simple ``` ### Usage ##### ls Basic usage, find by name tag, external / internal IP, DNS + ```bash ec2 ls my-hostname ``` -Custom formatter (name, ip, extended) +Using a custom formatter (name, ip, extended) + ```bash ec2 ls --format=name my-hostname ``` ##### ssh + Open an SSH session ```bash ec2 ssh my-hostname @@ -32,6 +36,7 @@ ec2 ssh my-hostname -- w ``` ##### custom region + ```bash ec2 --region us-west-2 ls my-hostname ``` From 288cd562f8755ac7bed510148b53a5369649b80c Mon Sep 17 00:00:00 2001 From: Pavel Brodsky Date: Mon, 19 Oct 2020 11:28:54 +0300 Subject: [PATCH 19/22] Upgrade poetry version (#20) --- poetry.lock | 91 +++++++++++++++++++++-------------------------------- 1 file changed, 35 insertions(+), 56 deletions(-) diff --git a/poetry.lock b/poetry.lock index 048e1e7..c48661b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,108 +1,91 @@ [[package]] -category = "main" -description = "The AWS SDK for Python" name = "boto3" +version = "1.15.18" +description = "The AWS SDK for Python" +category = "main" optional = false python-versions = "*" -version = "1.14.61" [package.dependencies] -botocore = ">=1.17.61,<1.18.0" +botocore = ">=1.18.18,<1.19.0" jmespath = ">=0.7.1,<1.0.0" s3transfer = ">=0.3.0,<0.4.0" [[package]] -category = "main" -description = "Low-level, data-driven core of boto 3." name = "botocore" +version = "1.18.18" +description = "Low-level, data-driven core of boto 3." +category = "main" optional = false python-versions = "*" -version = "1.17.61" [package.dependencies] -docutils = ">=0.10,<0.16" jmespath = ">=0.7.1,<1.0.0" python-dateutil = ">=2.1,<3.0.0" - -[package.dependencies.urllib3] -python = "<3.4.0 || >=3.5.0" -version = ">=1.20,<1.26" +urllib3 = {version = ">=1.20,<1.26", markers = "python_version != \"3.4\""} [[package]] -category = "main" -description = "Composable command line interface toolkit" name = "click" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" version = "7.1.2" - -[[package]] +description = "Composable command line interface toolkit" category = "main" -description = "Docutils -- Python Documentation Utilities" -name = "docutils" optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -version = "0.15.2" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] -category = "main" -description = "Backport of the concurrent.futures package from Python 3" -marker = "python_version >= \"2.7\" and python_version < \"2.8\" or python_version == \"2.7\"" name = "futures" +version = "3.3.0" +description = "Backport of the concurrent.futures package from Python 3" +category = "main" optional = false python-versions = ">=2.6, <3" -version = "3.3.0" [[package]] -category = "main" -description = "JSON Matching Expressions" name = "jmespath" +version = "0.10.0" +description = "JSON Matching Expressions" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -version = "0.10.0" [[package]] -category = "main" -description = "Extensions to the standard Python datetime module" name = "python-dateutil" +version = "2.8.1" +description = "Extensions to the standard Python datetime module" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -version = "2.8.1" [package.dependencies] six = ">=1.5" [[package]] -category = "main" -description = "An Amazon S3 Transfer Manager" name = "s3transfer" +version = "0.3.3" +description = "An Amazon S3 Transfer Manager" +category = "main" optional = false python-versions = "*" -version = "0.3.3" [package.dependencies] botocore = ">=1.12.36,<2.0a.0" - -[package.dependencies.futures] -python = ">=2.7,<2.8" -version = ">=2.2.0,<4.0.0" +futures = {version = ">=2.2.0,<4.0.0", markers = "python_version == \"2.7\""} [[package]] -category = "main" -description = "Python 2 and 3 compatibility utilities" name = "six" +version = "1.15.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -version = "1.15.0" [[package]] -category = "main" -description = "HTTP library with thread-safe connection pooling, file post, and more." -marker = "python_version != \"3.4\"" name = "urllib3" +version = "1.25.10" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" -version = "1.25.10" [package.extras] brotli = ["brotlipy (>=0.6.0)"] @@ -110,27 +93,23 @@ secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0 socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] [metadata] -content-hash = "1ad7ed3ff7853839b9c46ddaac5c5a6d1d331e35619bf928675122c82110f5b2" -lock-version = "1.0" +lock-version = "1.1" python-versions = "~2.7 || ^3.6" +content-hash = "1ad7ed3ff7853839b9c46ddaac5c5a6d1d331e35619bf928675122c82110f5b2" [metadata.files] boto3 = [ - {file = "boto3-1.14.61.tar.gz", hash = "sha256:a1c738ff178fc6ed8951559053c1142c71d166d17642361bbe63584be2f50a00"}, + {file = "boto3-1.15.18-py2.py3-none-any.whl", hash = "sha256:9ab957090f7893172768bb8b8d2c5cce0afd36a9d36d73a9fb14168f72d75a8b"}, + {file = "boto3-1.15.18.tar.gz", hash = "sha256:f56148e2c6b9a2d704218da42f07d72f00270bfddb13bc1bdea20d3327daa51e"}, ] botocore = [ - {file = "botocore-1.17.61-py2.py3-none-any.whl", hash = "sha256:4a2931b6cdbb1caf797eaf81f1f42f9cf243f94f06b3144d36a2f016a58ad125"}, - {file = "botocore-1.17.61.tar.gz", hash = "sha256:71160b06c52f21cc477bc4b5eb35fc78f1f89e363b86d93fcbebfe366cc4d8f0"}, + {file = "botocore-1.18.18-py2.py3-none-any.whl", hash = "sha256:de5f9fc0c7e88ee7ba831fa27475be258ae09ece99143ed623d3618a3c84ee2c"}, + {file = "botocore-1.18.18.tar.gz", hash = "sha256:e224754230e7e015836ba20037cac6321e8e2ce9b8627c14d579fcb37249decd"}, ] click = [ {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, ] -docutils = [ - {file = "docutils-0.15.2-py2-none-any.whl", hash = "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827"}, - {file = "docutils-0.15.2-py3-none-any.whl", hash = "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0"}, - {file = "docutils-0.15.2.tar.gz", hash = "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99"}, -] futures = [ {file = "futures-3.3.0-py2-none-any.whl", hash = "sha256:49b3f5b064b6e3afc3316421a3f25f66c137ae88f068abbf72830170033c5e16"}, {file = "futures-3.3.0.tar.gz", hash = "sha256:7e033af76a5e35f58e56da7a91e687706faf4e7bdfb2cbc3f2cca6b9bcda9794"}, From 6a8fbed08f29ee7bb8f112f77f8b519ee3d34bb7 Mon Sep 17 00:00:00 2001 From: Gilad Peleg Date: Mon, 19 Oct 2020 11:42:28 +0300 Subject: [PATCH 20/22] Update ec2grep deps & lockfile for poetry 1.1.3 (#21) --- poetry.lock | 4 ++-- pyproject.toml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index c48661b..c98acdc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -94,8 +94,8 @@ socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] [metadata] lock-version = "1.1" -python-versions = "~2.7 || ^3.6" -content-hash = "1ad7ed3ff7853839b9c46ddaac5c5a6d1d331e35619bf928675122c82110f5b2" +python-versions = "^2.7 || ^3.8" +content-hash = "dc3843bfd4d2891b4fd3974dcf4577c2a8d9675af9bcc185091aa02fcc2ea13c" [metadata.files] boto3 = [ diff --git a/pyproject.toml b/pyproject.toml index d5898d1..5df4005 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,10 +13,10 @@ packages = [ ec2 = "ec2grep:cli" [tool.poetry.dependencies] -python = "~2.7 || ^3.6" +python = "^2.7 || ^3.8" -boto3 = "^1.9.86" -click = "^7.0" +boto3 = "^1.15.18" +click = "^7.1.2" # Python 2 specific packages: futures = { version = "3.3.0", python = "~2.7" } @@ -25,5 +25,5 @@ futures = { version = "3.3.0", python = "~2.7" } line-length = 120 [build-system] -requires = ["poetry>=1.0.0"] +requires = ["poetry>=1.1.3"] build-backend = "poetry.masonry.api" From efc195a8ade42ab6dcf01a5b86ef0a9380f1c6d4 Mon Sep 17 00:00:00 2001 From: Pavel Brodsky Date: Sun, 15 May 2022 12:15:13 +0300 Subject: [PATCH 21/22] Update ec2grep artifact (#22) PR-Creator: mcouthon PR-Reviewer: omerpeleg22 PR-URL: https://github.com/forter/ec2grep/pull/22 PR-Branch: pavel-brodsky-update-ec2grep-artifact --- Dockerfile | 4 ++-- Jenkinsfile | 10 ++++++++-- Makefile | 11 +++-------- poetry.lock | 19 +++---------------- pyproject.toml | 5 +---- 5 files changed, 17 insertions(+), 32 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4f22d14..2c060bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,2 +1,2 @@ -ARG PYTHON_VERSION=2.7.17 -FROM 174522763890.dkr.ecr.us-east-1.amazonaws.com/ubuntu-python-pyproject:${PYTHON_VERSION} \ No newline at end of file +ARG PYTHON_VERSION=3.8 +FROM 174522763890.dkr.ecr.us-east-1.amazonaws.com/ubuntu-python-pyproject:${PYTHON_VERSION} diff --git a/Jenkinsfile b/Jenkinsfile index ce85bb5..c96edf8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,13 @@ // vim: filetype=groovy -node('master') { +node('general') { def base = load('/var/jenkins_home/workspace/Infra/build-scripts/build/Jenkinsfile') base.execute([ - customStages: base.get_pypi_stages(true) + customStages: base.get_pypi_stages( + /* is_poetry */true, + /* python_version */'3.8', + /* should_bump_version */true, + /* additional_package_roots */null, + /* is_wheel */true, + ), ]) } \ No newline at end of file diff --git a/Makefile b/Makefile index a2f218a..e3e8674 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,9 @@ ECR_URL ?= 174522763890.dkr.ecr.us-east-1.amazonaws.com .PHONY: build -build: build-py2 build-py3 - -build-py2: PYTHON_VERSION = 2.7.17 -build-py3: PYTHON_VERSION = 3.8.5 - -build-py3 build-py2: - @echo "======= Building Python $(PYTHON_VERSION) ========" - docker build --pull --build-arg "PYTHON_VERSION=$(PYTHON_VERSION)" . +build: + @echo "======= Building Python ========" + docker build --pull . ci-test: echo "No tests" diff --git a/poetry.lock b/poetry.lock index c98acdc..e1715e4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -32,14 +32,6 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -[[package]] -name = "futures" -version = "3.3.0" -description = "Backport of the concurrent.futures package from Python 3" -category = "main" -optional = false -python-versions = ">=2.6, <3" - [[package]] name = "jmespath" version = "0.10.0" @@ -69,7 +61,6 @@ python-versions = "*" [package.dependencies] botocore = ">=1.12.36,<2.0a.0" -futures = {version = ">=2.2.0,<4.0.0", markers = "python_version == \"2.7\""} [[package]] name = "six" @@ -90,12 +81,12 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" [package.extras] brotli = ["brotlipy (>=0.6.0)"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0.14)", "ipaddress"] -socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "1.1" -python-versions = "^2.7 || ^3.8" -content-hash = "dc3843bfd4d2891b4fd3974dcf4577c2a8d9675af9bcc185091aa02fcc2ea13c" +python-versions = "^3.8" +content-hash = "7e497a0e4a0bf428707924a7cff19e1c9ea5a6f67452102bcc7f75ee323f5051" [metadata.files] boto3 = [ @@ -110,10 +101,6 @@ click = [ {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, ] -futures = [ - {file = "futures-3.3.0-py2-none-any.whl", hash = "sha256:49b3f5b064b6e3afc3316421a3f25f66c137ae88f068abbf72830170033c5e16"}, - {file = "futures-3.3.0.tar.gz", hash = "sha256:7e033af76a5e35f58e56da7a91e687706faf4e7bdfb2cbc3f2cca6b9bcda9794"}, -] jmespath = [ {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, {file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"}, diff --git a/pyproject.toml b/pyproject.toml index 5df4005..239abc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,14 +13,11 @@ packages = [ ec2 = "ec2grep:cli" [tool.poetry.dependencies] -python = "^2.7 || ^3.8" +python = "^3.8" boto3 = "^1.15.18" click = "^7.1.2" -# Python 2 specific packages: -futures = { version = "3.3.0", python = "~2.7" } - [tool.black] line-length = 120 From ae2901d94e0cd7a00615ed8533d574c2278a29a8 Mon Sep 17 00:00:00 2001 From: Nathan Meisels Date: Mon, 30 May 2022 13:23:46 +0300 Subject: [PATCH 22/22] Add Parallel to docs --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 3e29edd..33b22f1 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,11 @@ With arguments ec2 ssh my-hostname -- w ``` +Parallel command +```bash +ec2 ssh --parallel -F ~/.ssh/gozer my-hostname -- "" +``` + ##### custom region ```bash