Adding install_efa_exporter

1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/observability/Makefile

@@ -5,4 +5,5 @@ install-on-head:
 
 install-on-workers:
 	srun -N 2 sudo bash ./install_dcgm_exporter.sh
+	srun -N 2 sudo bash ./install_efa_exporter.sh
 

1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/observability/Notes.md

@@ -13,4 +13,9 @@
 
     > An error occurred (AccessDeniedException) when calling the GetAuthorizationToken operation: User: arn:aws:sts::662012767933:assumed-role/sagemaker-slurm-observability-1-c77364c3ExecRole/SageMaker is not authorized to perform: ecr:GetAuthorizationToken on resource: * because no identity-based policy allows the ecr:GetAuthorizationToken action
 
-    > Error response from daemon: pull access denied for 602401143452.dkr.ecr.us-west-2.amazonaws.com/hyperpod/dcgm_exporter, repository does not exist or may require 'docker login': denied: User: arn:aws:sts::662012767933:assumed-role/sagemaker-slurm-observability-1-c77364c3ExecRole/SageMaker is not authorized to perform: ecr:BatchGetImage on resource: arn:aws:ecr:us-west-2:602401143452:repository/hyperpod/dcgm_exporter because no identity-based policy allows the ecr:BatchGetImage action
+    > Error response from daemon: pull access denied for 602401143452.dkr.ecr.us-west-2.amazonaws.com/hyperpod/dcgm_exporter, repository does not exist or may require 'docker login': denied: User: arn:aws:sts::662012767933:assumed-role/sagemaker-slurm-observability-1-c77364c3ExecRole/SageMaker is not authorized to perform: ecr:BatchGetImage on resource: arn:aws:ecr:us-west-2:602401143452:repository/hyperpod/dcgm_exporter because no identity-based policy allows the ecr:BatchGetImage action
+
+
+- Adding "aws ecr get-login-password" in multiple places but this can be a single place.
+
+

1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/observability/install_efa_exporter.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# FIXME: should auto detect
+REGION=us-west-2
+
+# Define the container name
+CONTAINER_NAME="efa-node-exporter"
+
+ECR_ACCOUNT_ID=602401143452
+VERSION=1.0.0
+IMAGE="$ECR_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/hyperpod/efa_exporter:${VERSION}"
+
+# Maximum number of retries
+MAX_RETRIES=5
+RETRY_DELAY=5  # Initial delay in seconds
+
+# Check if the container exists and is running
+if docker ps --filter "name=$CONTAINER_NAME" --filter "status=running" | grep -q "$CONTAINER_NAME"; then
+    echo "Container $CONTAINER_NAME is already running."
+    exit 0
+else
+    echo "Container $CONTAINER_NAME is not running or does not exist..."
+    echo "Checking if $CONTAINER_NAME container exists but is not running. If yes, removing it..."
+    docker rm -f $CONTAINER_NAME && echo "Container $CONTAINER_NAME has been removed."
+    echo "Proceeding with script..."
+fi
+
+# Retry logic for pulling the image
+attempt=0
+while [ $attempt -lt $MAX_RETRIES ]; do
+    echo "Attempting to pull image ($attempt/$MAX_RETRIES)..."
+
+    aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin $ECR_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com
+
+    if sudo docker pull "$IMAGE"; then
+        echo "Successfully pulled image."
+        break
+    else
+        attempt=$((attempt + 1))
+        if [ $attempt -lt $MAX_RETRIES ]; then
+            echo "Pull failed. Retrying in $RETRY_DELAY seconds..."
+            sleep $RETRY_DELAY
+            RETRY_DELAY=$((RETRY_DELAY * 2))  # Exponential backoff
+        else
+            echo "Failed to pull Docker image after $MAX_RETRIES attempts. Exiting..."
+            exit 1
+        fi
+    fi
+done
+
+# Run the Docker container with appropriate configurations
+if sudo docker run -d --restart always \
+    --name=$CONTAINER_NAME \
+    --net="host" \
+    --pid="host" \
+    -v "/:/host:ro,rslave" \
+    $IMAGE \
+    --path.rootfs=/host; then
+    echo "Successfully started EFA Node Exporter on node"
+    exit 0
+else
+    echo "Failed to run Docker container"
+    exit 1
+fi