From 9ee8de47797cd4a4bd7316c81663067880df198d Mon Sep 17 00:00:00 2001 From: Arlind Qirjazi Date: Tue, 1 Apr 2025 12:22:41 +0200 Subject: [PATCH 1/6] Update setup.py --- setup.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/setup.py b/setup.py index c2e135b..75cd2c6 100644 --- a/setup.py +++ b/setup.py @@ -1,23 +1,25 @@ #!/usr/bin/env python -# coding: utf-8 +# -*- coding: utf-8 -*- import os +from setuptools import setup, find_packages + +# Get project directory and name thisdir = os.path.abspath(os.path.dirname(__file__)) projName = 'swarmy' -from setuptools import setup, find_packages - -version = open(os.path.join(thisdir, 'version.txt'), 'rb').read().strip() +# Read version (Python 3 text mode) +with open(os.path.join(thisdir, 'version.txt'), 'r') as f: + version = f.read().strip() -#Write out the version to the _version.py file -with open(os.path.join(thisdir, projName, '_version.py'), 'wt') as out: - out.write('version="%s"' % version) +# Write version to _version.py (Python 3 text mode) +with open(os.path.join(thisdir, projName, '_version.py'), 'w') as out: + out.write(f'version = "{version}"\n') # f-string (Python 3.6+) def get_long_desc(): - root_dir = os.path.dirname(__file__) - if not root_dir: - root_dir = '.' - return open(os.path.join(root_dir, 'README.md')).read() + root_dir = os.path.dirname(__file__) or '.' + with open(os.path.join(root_dir, 'README.md'), 'r', encoding='utf-8') as f: + return f.read() testing_extras = [] @@ -26,14 +28,14 @@ def get_long_desc(): version=version, description="AWS Autoscaling and Metadata utilities", long_description=get_long_desc(), + long_description_content_type='text/markdown', # Required for PyPI markdown url='https://github.com/Crunch-io/swarmy', download_url='https://github.com/Crunch-io/crunch-lib/archive/master.zip', - classifiers=[ - "Programming Language :: Python", + "Programming Language :: Python :: 3", "Topic :: Software Development :: Libraries :: Python Modules", ], - author=u'Crunch.io', + author='Crunch.io', author_email='dev@crunch.io', license='MIT', install_requires=[ @@ -42,9 +44,7 @@ def get_long_desc(): 'docopt', ], packages=find_packages(), - #namespace_packages=['swarmy.extras'], include_package_data=True, - #tests_require=testing_extras, package_data={ 'swarmy': ['*.json', '*.csv'] }, @@ -58,4 +58,3 @@ def get_long_desc(): ] }, ) - From df804eb837b644da9327f8b939b28c094e3195dd Mon Sep 17 00:00:00 2001 From: Arlind Qirjazi Date: Tue, 1 Apr 2025 15:25:23 +0200 Subject: [PATCH 2/6] Update lib.py --- swarmy/lib.py | 284 +++++++++++++++++++------------------------------- 1 file changed, 110 insertions(+), 174 deletions(-) diff --git a/swarmy/lib.py b/swarmy/lib.py index 1f1a027..203ad60 100644 --- a/swarmy/lib.py +++ b/swarmy/lib.py @@ -5,6 +5,11 @@ import sys from functools import wraps +# Global cache for metadata and tags +_metadata_cache = None +_tags_cache = None +_region_cache = None + def error(errno, message): print(message, file=sys.stderr) return errno @@ -20,188 +25,119 @@ def get_instance_metadata(): response = requests.get(metadata_url, headers=metadata_headers) return response.json() -class EC2Metadata: +def metadata_required(func): """ - A class to handle AWS EC2 instance metadata and tags operations. - - This class provides methods to interact with EC2 instance metadata, manage tags, - and perform Route53 DNS operations. It implements caching for metadata and tags - to minimize API calls. - - Attributes: - _metadata (dict): Cache for instance metadata - _tags (dict): Cache for instance tags - _region (str): Cache for AWS region + A decorator that ensures instance metadata is loaded before function execution. + """ + @wraps(func) + def wrapper(*args, **kwargs): + global _metadata_cache + if _metadata_cache is None: + _metadata_cache = get_instance_metadata() + return func(*args, **kwargs) + return wrapper + +@metadata_required +def get_region(): + global _region_cache + if not _region_cache: + _region_cache = _metadata_cache['region'] + return _region_cache + +@metadata_required +def get_instance_tags(): """ - def __init__(self): - self._metadata = None - self._tags = None - self._region = None - - def metadata_required(func): - """ - A decorator that ensures instance metadata is loaded before function execution. - - Args: - func: The function to be wrapped - - Returns: - wrapper: The wrapped function that checks for metadata - """ - @wraps(func) - def wrapper(self, *args, **kwargs): - if self._metadata is None: - self._metadata = get_instance_metadata() - return func(self, *args, **kwargs) - return wrapper - - @metadata_required - def get_region(self): - if not self._region: - self._region = self._metadata['region'] - return self._region - - @metadata_required - def get_instance_tags(self): - """ - Retrieve all tags associated with the current EC2 instance. - - This method caches the tags to minimize API calls to AWS. - If tags are already cached, returns the cached version. - - Returns: - dict: A dictionary of instance tags where keys are tag names - and values are tag values. - - Example: - >>> ec2_meta = EC2Metadata() - >>> tags = ec2_meta.get_instance_tags() - >>> print(tags) - {'Name': 'my-instance', 'Environment': 'production'} - """ - if self._tags is None: - instance_id = self._metadata['instanceId'] - ec2 = boto3.resource('ec2', region_name=self.get_region()) - instance = ec2.Instance(instance_id) - self._tags = {tag['Key']: tag['Value'] for tag in instance.tags or []} - return self._tags - - @metadata_required - def set_instance_tag(self, key, value): - """ - Set a tag on the current EC2 instance. - - Args: - key (str): The tag key to set - value (str): The value to set for the tag - - Returns: - dict: Updated dictionary of all instance tags - - Raises: - boto3.exceptions.Boto3Error: If the tag update fails - - Example: - >>> ec2_meta = EC2Metadata() - >>> ec2_meta.set_instance_tag('Environment', 'production') - """ - instance_id = self._metadata['instanceId'] - ec2 = boto3.resource('ec2', region_name=self.get_region()) + Retrieve all tags associated with the current EC2 instance. + Returns a dictionary of instance tags. + """ + global _tags_cache + if _tags_cache is None: + instance_id = _metadata_cache['instanceId'] + ec2 = boto3.resource('ec2', region_name=get_region()) instance = ec2.Instance(instance_id) - instance.create_tags(Tags=[{'Key': key, 'Value': value}]) - self._tags = None # Reset cache - return self.get_instance_tags() - - def set_instance_name(self, fqdn): - """Set the Name tag of the instance""" - return self.set_instance_tag('Name', fqdn) - - @metadata_required - def route53_upsert_a_record(self, domain, fqdn, ip, ttl, wait=0): - """ - Update or create an A record in Route53 DNS. - - This method will either create a new A record or update an existing one - in the specified Route53 hosted zone. - - Args: - domain (str): The domain name of the hosted zone (e.g., 'example.com') - fqdn (str): The fully qualified domain name for the A record - ip (str): The IP address for the A record - ttl (int): Time-to-live in seconds for the DNS record - wait (int): Time in seconds to wait for the change to propagate. - Use -1 to wait indefinitely, 0 to return immediately. - - Returns: - dict: The Route53 change response object - - Raises: - ValueError: If the hosted zone for the domain is not found - boto3.exceptions.Boto3Error: If the Route53 API call fails - - Example: - >>> ec2_meta = EC2Metadata() - >>> ec2_meta.route53_upsert_a_record( - ... domain='example.com', - ... fqdn='server1.example.com', - ... ip='10.0.0.1', - ... ttl=300, - ... wait=60 - ... ) - """ - route53 = boto3.client('route53', region_name=self.get_region()) - - # Get the hosted zone ID - zones = route53.list_hosted_zones_by_name(DNSName=domain) - zone_id = None - for zone in zones['HostedZones']: - if zone['Name'].rstrip('.') == domain: - zone_id = zone['Id'] - break - - if not zone_id: - raise ValueError(f"Hosted zone for domain {domain} not found") - - # Prepare the change batch - change_batch = { - 'Changes': [{ - 'Action': 'UPSERT', - 'ResourceRecordSet': { - 'Name': fqdn, - 'Type': 'A', - 'TTL': ttl, - 'ResourceRecords': [{'Value': ip}] - } - }] - } - - # Make the change - response = route53.change_resource_record_sets( - HostedZoneId=zone_id, - ChangeBatch=change_batch - ) + _tags_cache = {tag['Key']: tag['Value'] for tag in instance.tags or []} + return _tags_cache - if wait: - waiter = route53.get_waiter('resource_record_sets_changed') - waiter.wait( - Id=response['ChangeInfo']['Id'], - WaiterConfig={'Delay': 10, 'MaxAttempts': 60 if wait == -1 else int(wait/10)} - ) +def instance_tags_required(func): + """ + A decorator that ensures instance tags is loaded before function execution. + """ + @wraps(func) + def wrapper(*args, **kwargs): + global _tags_cache + if _metadata_cache is None: + _metadata_cache = get_instance_tags() + return func(*args, **kwargs) + return wrapper + +@metadata_required +def set_instance_tag(key, value): + """ + Set a tag on the current EC2 instance. + Returns updated dictionary of all instance tags. + """ + global _tags_cache + instance_id = _metadata_cache['instanceId'] + ec2 = boto3.resource('ec2', region_name=get_region()) + instance = ec2.Instance(instance_id) + instance.create_tags(Tags=[{'Key': key, 'Value': value}]) + _tags_cache = None # Reset cache + return get_instance_tags() + +def set_instance_name(fqdn): + """Set the Name tag of the instance""" + return set_instance_tag('Name', fqdn) + +@metadata_required +def route53_upsert_a_record(domain, fqdn, ip, ttl, wait=0): + """ + Update or create an A record in Route53 DNS. + Returns the Route53 change response object. + """ + route53 = boto3.client('route53', region_name=get_region()) + + # Get the hosted zone ID + zones = route53.list_hosted_zones_by_name(DNSName=domain) + zone_id = None + for zone in zones['HostedZones']: + if zone['Name'].rstrip('.') == domain: + zone_id = zone['Id'] + break + + if not zone_id: + raise ValueError(f"Hosted zone for domain {domain} not found") + + # Prepare the change batch + change_batch = { + 'Changes': [{ + 'Action': 'UPSERT', + 'ResourceRecordSet': { + 'Name': fqdn, + 'Type': 'A', + 'TTL': ttl, + 'ResourceRecords': [{'Value': ip}] + } + }] + } + + # Make the change + response = route53.change_resource_record_sets( + HostedZoneId=zone_id, + ChangeBatch=change_batch + ) + + if wait: + waiter = route53.get_waiter('resource_record_sets_changed') + waiter.wait( + Id=response['ChangeInfo']['Id'], + WaiterConfig={'Delay': 10, 'MaxAttempts': 60 if wait == -1 else int(wait/10)} + ) - return response + return response def set_hostname(fqdn): """ Set the system hostname to the specified FQDN. - - Args: - fqdn (str): The fully qualified domain name to set as hostname - - Returns: - int: Return code from the hostname command (0 for success) - - Example: - >>> set_hostname('server1.example.com') - 0 + Returns return code from the hostname command (0 for success). """ return os.system(f"hostname {fqdn}") From 1488e927969a291869f02798fa75ad9a2254276b Mon Sep 17 00:00:00 2001 From: Arlind Qirjazi Date: Tue, 1 Apr 2025 15:39:38 +0200 Subject: [PATCH 3/6] Update dynamic_hostname.py --- swarmy/dynamic_hostname.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/swarmy/dynamic_hostname.py b/swarmy/dynamic_hostname.py index 46f1cc1..e1f8fca 100644 --- a/swarmy/dynamic_hostname.py +++ b/swarmy/dynamic_hostname.py @@ -30,7 +30,7 @@ --domain-tag= Instance tag name to use to retrieve name. Overrides --domain --hosted-zone= - By default, we try to add the hostname to the zone name found using --domain or --domain-tag, but if this is different for you, specify this parameter. Ignored if --no-dns is specified + By default, we try to add the hostname to the zone name found using --domain or --domain-tag, but if this isdifferent for you, specify this parameter. Ignored if --no-dns is specified --prefix= The hostname prefix to use. Will be used verbatim (no '-' will be appended). E.g., 'ip' will become 'ip10-2-3-4' and 'ip-' will become 'ip-10-2-3-4' [default: ip-] --prefix-tag= @@ -44,7 +44,7 @@ from typing import Optional, Dict, Any, Union from dataclasses import dataclass -import lib +from swarmy import lib, _version from docopt import docopt import sys @@ -60,7 +60,7 @@ class HostnameConfig: sep: str = '' @lib.metadata_required -def get_ip(public: bool = False) -> str: +def get_ip(public: bool = False) -> Optional[str]: """ Get the current ip address (private by default) @@ -71,8 +71,8 @@ def get_ip(public: bool = False) -> str: str: IP address """ if public: - return lib._metadata.get('public-ipv4', '169.254.0.1') - return lib._metadata.get('local-ipv4') + return lib._metadata_cache['public-ipv4', '169.254.0.1'] + return lib._metadata_cache['privateIp'] @lib.instance_tags_required def get_domain_from_tags(tagName: str) -> Optional[str]: @@ -85,7 +85,7 @@ def get_domain_from_tags(tagName: str) -> Optional[str]: Returns: Optional[str]: Domain name if found, None otherwise """ - return lib._tags.get(tagName) + return lib._tags_cache(tagName) @lib.instance_tags_required def get_prefix_from_tags(tagName: str) -> Optional[str]: @@ -98,28 +98,31 @@ def get_prefix_from_tags(tagName: str) -> Optional[str]: Returns: Optional[str]: Prefix if found, None otherwise """ - return lib._tags.get(tagName) + return lib._tags_cache(tagName) def gen_fqdn(config: HostnameConfig) -> str: """ - Generate FQDN from configuration + Generate FQDN from configuration with proper IP part handling Args: - config: HostnameConfig instance containing all necessary parameters + config: HostnameConfig with ip, domain, numparts, prefix, and sep Returns: str: Generated FQDN + + Raises: + ValueError: If IP is invalid or numparts is out of range """ if config.numparts >= 1: host_ip_part = '-'.join(config.ip.split('.')[0-config.numparts:]) else: host_ip_part = "" - return f"{config.prefix}{config.sep}{host_ip_part}.{config.domain}" + return f"{config.prefix}{host_ip_part}.{config.domain}" def parse_arguments() -> Dict[str, Any]: """Parse and validate command line arguments""" - return docopt(helpstr, version=f"dynamic_hostname {lib.version}") + return docopt(helpstr, version=f"dynamic_hostname {_version.version}") def get_numparts(arguments: Dict[str, Any]) -> int: """Determine number of IP parts to use""" From a7df1de013c2cc9b78d33571e9502260daf68b74 Mon Sep 17 00:00:00 2001 From: Arlind Qirjazi Date: Tue, 1 Apr 2025 15:40:20 +0200 Subject: [PATCH 4/6] Update dynamic_hostname.py --- swarmy/dynamic_hostname.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/swarmy/dynamic_hostname.py b/swarmy/dynamic_hostname.py index e1f8fca..b6d10aa 100644 --- a/swarmy/dynamic_hostname.py +++ b/swarmy/dynamic_hostname.py @@ -109,9 +109,6 @@ def gen_fqdn(config: HostnameConfig) -> str: Returns: str: Generated FQDN - - Raises: - ValueError: If IP is invalid or numparts is out of range """ if config.numparts >= 1: host_ip_part = '-'.join(config.ip.split('.')[0-config.numparts:]) From 21979e8ed36059498edabdc072d77394c05493c0 Mon Sep 17 00:00:00 2001 From: Arlind Qirjazi Date: Tue, 1 Apr 2025 18:14:38 +0200 Subject: [PATCH 5/6] Update stage2.sh --- scripts/stage2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/stage2.sh b/scripts/stage2.sh index eed12f4..3c189ff 100755 --- a/scripts/stage2.sh +++ b/scripts/stage2.sh @@ -5,7 +5,7 @@ if [ -n "$DEBUG" ]; then fi # First, let's deal with the hostname: -dynamic_hostname $HOSTNAME_ARGS +dynamic_hostname $HOSTNAME_ARGS || echo "this needs to succeed regardless" ### TODO: do this, or write a util.py #if [ -n "$JENKINS_BASE" ]; then From 55593c34c9be0650f31af17d40383e892b2a445a Mon Sep 17 00:00:00 2001 From: Arlind Qirjazi Date: Tue, 14 Oct 2025 13:36:50 +0200 Subject: [PATCH 6/6] Modify nvme labeling --- scripts/prepephemeral.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepephemeral.sh b/scripts/prepephemeral.sh index aacf909..4dad426 100755 --- a/scripts/prepephemeral.sh +++ b/scripts/prepephemeral.sh @@ -67,7 +67,7 @@ function get_ephemeral_disks { DEVICES+=('nvme1n1') ;; i3en.6xlarge) - DEVICES+=('nvme1n1' 'nvme2n1') + DEVICES+=('nvme0n1' 'nvme2n1') ;; #ENA enabled/SR-IOV where EBS volumes are listed as nvme drives m5d.24xlarge)