From ca057636a31d4250f45343ab916b5ecbf02123dd Mon Sep 17 00:00:00 2001 From: Carter Myers <206+cmyers@users.noreply.github.mieweb.com> Date: Fri, 24 Oct 2025 12:01:11 -0700 Subject: [PATCH 1/7] DRAFT: Add Packer build and upload workflow for Proxmox templates Introduces a draft GitHub Actions workflow to build and upload Debian 12 and Rocky 9 LXC templates for Proxmox. Adds supporting Packer HCL files, Ansible provisioning playbook, Python API utilities for uploading templates, and variable files for template customization. --- .../.github/workflows/build-templates.yml | 45 ++++++++++++++ .../packer/api/proxmox_upload.py | 48 +++++++++++++++ .../packer/api/proxmox_utils.py | 49 +++++++++++++++ .../intern-phxdc-pve1/packer/debian12.pkr.hcl | 58 ++++++++++++++++++ .../packer/provisioners/ansible/site.yml | 56 +++++++++++++++++ .../intern-phxdc-pve1/packer/rocky9.pkr.hcl | 60 +++++++++++++++++++ .../packer/vars/debian12.auto.pkrvars.hcl | 4 ++ .../packer/vars/rocky9.auto.pkrvars.hcl | 1 + 8 files changed, 321 insertions(+) create mode 100644 container-creation/intern-phxdc-pve1/packer/.github/workflows/build-templates.yml create mode 100644 container-creation/intern-phxdc-pve1/packer/api/proxmox_upload.py create mode 100644 container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.py create mode 100644 container-creation/intern-phxdc-pve1/packer/debian12.pkr.hcl create mode 100644 container-creation/intern-phxdc-pve1/packer/provisioners/ansible/site.yml create mode 100644 container-creation/intern-phxdc-pve1/packer/rocky9.pkr.hcl create mode 100644 container-creation/intern-phxdc-pve1/packer/vars/debian12.auto.pkrvars.hcl create mode 100644 container-creation/intern-phxdc-pve1/packer/vars/rocky9.auto.pkrvars.hcl diff --git a/container-creation/intern-phxdc-pve1/packer/.github/workflows/build-templates.yml b/container-creation/intern-phxdc-pve1/packer/.github/workflows/build-templates.yml new file mode 100644 index 0000000..b3ca227 --- /dev/null +++ b/container-creation/intern-phxdc-pve1/packer/.github/workflows/build-templates.yml @@ -0,0 +1,45 @@ +name: Build and Upload Templates +on: + schedule: + - cron: "0 4 * * *" # Nightly at 4 AM + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + env: + PROXMOX_API_URL: ${{ secrets.PROXMOX_API_URL }} + PROXMOX_TOKEN_ID: ${{ secrets.PROXMOX_TOKEN_ID }} + PROXMOX_TOKEN_SECRET: ${{ secrets.PROXMOX_TOKEN_SECRET }} + # Define the template version ONCE for the whole job + TEMPLATE_VERSION: ${{ format('{0:yyyyMMdd}', github.run_started_at) }} + + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y packer ansible zstd python3-requests + + # --- Build Debian 12 --- + - name: Build Debian 12 Template + run: | + packer build \ + -var "template_version=${TEMPLATE_VERSION}" \ + debian12.pkr.hcl + + - name: Upload Debian 12 Template + run: | + python3 api/proxmox_upload.py \ + --file /tmp/output/debian12-fungible_${TEMPLATE_VERSION}.tar.xz + + # --- Build Rocky 9 --- + - name: Build Rocky 9 Template + run: | + packer build \ + -var "template_version=${TEMPLATE_VERSION}" \ + rocky9.pkr.hcl + + - name: Upload Rocky 9 Template + run: | + python3 api/proxmox_upload.py \ + --file /tmp/output/rocky9-fungible_${TEMPLATE_VERSION}.tar.xz \ No newline at end of file diff --git a/container-creation/intern-phxdc-pve1/packer/api/proxmox_upload.py b/container-creation/intern-phxdc-pve1/packer/api/proxmox_upload.py new file mode 100644 index 0000000..6f89809 --- /dev/null +++ b/container-creation/intern-phxdc-pve1/packer/api/proxmox_upload.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +import sys, argparse, warnings +from proxmox_utils import get_nodes, get_storages, upload_template, choose_default_storage + +# Suppress insecure request warnings (e.g., if PROXMOX_API_URL has verify=False) +warnings.filterwarnings("ignore", category=requests.urllib3.exceptions.InsecureRequestWarning) + +def main(): + parser = argparse.ArgumentParser(description="Upload Proxmox LXC template to all nodes.") + parser.add_argument("--file", required=True, help="Path to the .tar.xz template file.") + args = parser.parse_args() + + print(f"Starting template upload for: {args.file}") + + try: + nodes = get_nodes() + if not nodes: + print("Error: No Proxmox nodes found.") + sys.exit(1) + + print(f"Found nodes: {', '.join(nodes)}") + + for node in nodes: + print(f"--- Processing Node: {node} ---") + storages_list = get_storages(node) + + # Use the utility function to pick the best 'local' storage + storage = choose_default_storage(storages_list) + + if not storage: + print(f"Warning: No suitable storage found on node {node}. Skipping.") + continue + + print(f"Uploading to {node}:{storage}...") + try: + result = upload_template(node, storage, args.file) + print(f"Successfully uploaded to {node}:{storage}. Task: {result.get('data')}") + except Exception as e: + print(f"Error uploading to {node}:{storage}: {e}") + + except Exception as e: + print(f"An unexpected error occurred: {e}") + sys.exit(1) + + print("Template upload process finished.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.py b/container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.py new file mode 100644 index 0000000..d8a918d --- /dev/null +++ b/container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.py @@ -0,0 +1,49 @@ +# api/proxmox_utils.py +import os +import requests + +API_URL = os.environ.get("PROXMOX_API_URL") +TOKEN_ID = os.environ.get("PROXMOX_TOKEN_ID") +TOKEN_SECRET = os.environ.get("PROXMOX_TOKEN_SECRET") + +HEADERS = { + "Authorization": f"PVEAPIToken={TOKEN_ID}={TOKEN_SECRET}" +} + +def get_nodes(): + """Return a list of node names from the Proxmox cluster.""" + r = requests.get(f"{API_URL}/nodes", headers=HEADERS, verify=False) + r.raise_for_status() + data = r.json().get("data", []) + return [n["node"] for n in data] + +def get_storages(node): + """Return available storages for a node.""" + r = requests.get(f"{API_URL}/nodes/{node}/storage", headers=HEADERS, verify=False) + r.raise_for_status() + return [s["storage"] for s in r.json().get("data", [])] + +def upload_template(node, storage, filepath): + """Upload a template tarball to a specific node and storage.""" + with open(filepath, "rb") as f: + files = {"content": f} + data = { + "content": "vztmpl", + "filename": os.path.basename(filepath) + } + r = requests.post( + f"{API_URL}/nodes/{node}/storage/{storage}/upload", + headers=HEADERS, + files=files, + data=data, + verify=False + ) + r.raise_for_status() + return r.json() + +def choose_default_storage(storages): + """Pick the first local-type storage, or fallback to 'local'.""" + for s in storages: + if s == "local" or "local" in s.lower(): + return s + return storages[0] if storages else None diff --git a/container-creation/intern-phxdc-pve1/packer/debian12.pkr.hcl b/container-creation/intern-phxdc-pve1/packer/debian12.pkr.hcl new file mode 100644 index 0000000..2ae3324 --- /dev/null +++ b/container-creation/intern-phxdc-pve1/packer/debian12.pkr.hcl @@ -0,0 +1,58 @@ +packer { + required_plugins { + ansible = { + version = ">=1.1.0" + source = "github.com/hashicorp/ansible" + } + } +} + +variable "template_name" { + default = "debian12-fungible" +} + +source "null" "local_build" { + communicator = "none" +} + +build { + name = "debian12-template" + sources = ["source.null.local_build"] + + provisioner "shell" { + inline = [ + "mkdir -p /tmp/rootfs", + "wget -O /tmp/base.tar.zst http://download.proxmox.com/images/system/debian-12-standard_12.12-1_amd64.tar.zst", + "unzstd -d /tmp/base.tar.zst -o /tmp/base.tar", + "tar -xf /tmp/base.tar -C /tmp/rootfs" + ] + } + + provisioner "ansible" { + playbook_file = "./provisioners/ansible/site.yml" + ansible_env_vars = [ + "ANSIBLE_CONFIG=./provisioners/ansible/ansible.cfg" + ] + extra_arguments = [ + "--connection=chroot", + "--inventory", "/tmp/rootfs,", + ] + } + + provisioner "shell" { + inline = [ + "set -eux", # Good practice from your rocky file + "mkdir -p /tmp/output", + "cd /tmp/rootfs", + # Use the variables to build the filename + "tar -cJf /tmp/output/${var.template_name}_${var.template_version}.tar.xz .", + "ls -lh /tmp/output" + ] + } + + variable "template_version" { + type = string + default = "latest" + } + +} diff --git a/container-creation/intern-phxdc-pve1/packer/provisioners/ansible/site.yml b/container-creation/intern-phxdc-pve1/packer/provisioners/ansible/site.yml new file mode 100644 index 0000000..1652a96 --- /dev/null +++ b/container-creation/intern-phxdc-pve1/packer/provisioners/ansible/site.yml @@ -0,0 +1,56 @@ +# packer/provisioners/ansible/site.yml +--- +- name: Apply base fungible configuration + hosts: all + gather_facts: yes + + tasks: + - name: Ensure common packages are installed + ansible.builtin.package: + name: + - vim + - curl + - sudo + - wget + - ca-certificates + - nano + - git + state: present + + - name: Set up /etc/motd branding + ansible.builtin.copy: + dest: /etc/motd + content: | + This will be replaced by logic to deploy container information. + This container image was built automatically via Packer and Ansible. + + - name: Configure system DNS (common) + ansible.builtin.copy: + dest: /etc/resolv.conf + content: | + nameserver 1.1.1.1 + nameserver 8.8.8.8 + when: ansible_os_family in ['Debian', 'RedHat'] + + - name: Copy pown.sh (disabled until first boot) + ansible.builtin.copy: + src: files/pown.sh + dest: /usr/local/bin/pown.sh + mode: '0755' + + - name: Copy Wazuh placeholder registration script + ansible.builtin.copy: + dest: /etc/wazuh-agent/init_disabled + content: | + #!/bin/bash + # Registration disabled in build context; runs on first boot only. + mode: '0755' + + - name: Clean temporary or build-specific files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/lib/apt/lists/* + - /var/cache/dnf + - /tmp/* diff --git a/container-creation/intern-phxdc-pve1/packer/rocky9.pkr.hcl b/container-creation/intern-phxdc-pve1/packer/rocky9.pkr.hcl new file mode 100644 index 0000000..1bd7c11 --- /dev/null +++ b/container-creation/intern-phxdc-pve1/packer/rocky9.pkr.hcl @@ -0,0 +1,60 @@ +packer { + required_plugins { + ansible = { + version = ">=1.1.0" + source = "github.com/hashicorp/ansible" + } + } +} + +variable "template_name" { + default = "rocky9-lxc" +} + +source "null" "local_build" { + communicator = "none" +} + +build { + name = "rocky9-template" + sources = ["source.null.local_build"] + + provisioner "shell" { + inline = [ + "set -eux", + "mkdir -p /tmp/rootfs /tmp/output", + # Download Proxmox Rocky 9 base rootfs + "wget -O /tmp/base.tar.zst http://download.proxmox.com/images/system/rockylinux-9-standard_9.4-1_amd64.tar.zst", + # Extract base + "unzstd -d /tmp/base.tar.zst -o /tmp/base.tar", + "tar -xf /tmp/base.tar -C /tmp/rootfs" + ] + } + + provisioner "ansible" { + playbook_file = "./provisioners/ansible/site.yml" + ansible_env_vars = [ + "ANSIBLE_CONFIG=./provisioners/ansible/ansible.cfg" + ] + extra_arguments = [ + "--connection=chroot", + "--inventory", "/tmp/rootfs,", + ] + } + + provisioner "shell" { + inline = [ + "set -eux", + "cd /tmp/rootfs", + "tar -cJf /tmp/output/${var.template_name}_$(date +%Y%m%d).tar.xz .", + "ls -lh /tmp/output" + ] + } + + post-processor "shell-local" { + inline = [ + "echo '✅ Rocky 9 LXC rootfs built successfully'", + "ls -lh /tmp/output" + ] + } +} diff --git a/container-creation/intern-phxdc-pve1/packer/vars/debian12.auto.pkrvars.hcl b/container-creation/intern-phxdc-pve1/packer/vars/debian12.auto.pkrvars.hcl new file mode 100644 index 0000000..47437dd --- /dev/null +++ b/container-creation/intern-phxdc-pve1/packer/vars/debian12.auto.pkrvars.hcl @@ -0,0 +1,4 @@ +debian_release = "bookworm" +template_name = "debian12-lxc" +rootfs_dir = "/tmp/debian12-rootfs" +hostname = "template-debian12" diff --git a/container-creation/intern-phxdc-pve1/packer/vars/rocky9.auto.pkrvars.hcl b/container-creation/intern-phxdc-pve1/packer/vars/rocky9.auto.pkrvars.hcl new file mode 100644 index 0000000..9b164ee --- /dev/null +++ b/container-creation/intern-phxdc-pve1/packer/vars/rocky9.auto.pkrvars.hcl @@ -0,0 +1 @@ +template_name = "rocky9-fungible" From 28b0c5291b58d09735e84a8322d120437d841777 Mon Sep 17 00:00:00 2001 From: Carter Myers <206+cmyers@users.noreply.github.mieweb.com> Date: Fri, 24 Oct 2025 12:35:24 -0700 Subject: [PATCH 2/7] Add README and update Rocky9 template name Added a comprehensive README.md explaining the Proxmox LXC template automation workflow. Updated the Rocky9 Packer variable file to use 'rocky9-lxc' as the template name instead of 'rocky9-fungible'. --- .../intern-phxdc-pve1/packer/README.md | 150 ++++++++++++++++++ .../packer/vars/rocky9.auto.pkrvars.hcl | 2 +- 2 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 container-creation/intern-phxdc-pve1/packer/README.md diff --git a/container-creation/intern-phxdc-pve1/packer/README.md b/container-creation/intern-phxdc-pve1/packer/README.md new file mode 100644 index 0000000..8297754 --- /dev/null +++ b/container-creation/intern-phxdc-pve1/packer/README.md @@ -0,0 +1,150 @@ +Here is a README.md file that explains how your project works. + +----- + +# Proxmox LXC Template Automation 📦 + +This project automates the build, provisioning, and uploading of "fungible" Proxmox LXC container templates. It uses **Packer** to build the images, **Ansible** to provision them, and **Python** to upload them directly to your Proxmox cluster via the API. + +This system is designed to run automatically (e.g., nightly via GitHub Actions) to ensure your base container templates are always up-to-date with the latest patches and common configurations. + +## The Problem + +Manually updating container templates is slow, error-prone, and leads to configuration drift. This automation solves that by: + + * **Ensuring templates are current** with upstream security patches. + * **Pre-applying standard configuration** (like common packages, Wazuh, MOTD, etc.) before a container is ever created. + * **Reducing manual workload** and enabling faster, more consistent automated deployments. + +----- + +## How It Works + +The process is managed in distinct stages, usually kicked off by the GitHub Actions workflow. + +### 1\. Trigger (GitHub Actions) + + * **File:** `.github/workflows/build-templates.yml` + * **Action:** On a schedule (e.g., nightly) or a manual trigger, a new runner spins up. + * **Steps:** + 1. Installs the required tools: `packer`, `ansible`, `python3-requests`, and `zstd`. + 2. Sets environment variables (like API secrets and a `TEMPLATE_VERSION`). + +### 2\. Build (Packer) + + * **File:** `debian12.pkr.hcl` / `rocky9.pkr.hcl` + * **Action:** The workflow runs the `packer build ...` command. + * **Steps:** + 1. **Download:** A `shell` provisioner downloads the official Proxmox base template (a `.tar.zst` file) from `download.proxmox.com`. + 2. **Extract:** The file is decompressed and extracted into a temporary directory, `/tmp/rootfs`. This folder now contains the entire offline file system of the container. + +### 3\. Provision (Ansible) + + * **File:** `provisioners/ansible/site.yml` + * **Action:** Packer's `ansible` provisioner takes over. + * **Steps:** + 1. **The `chroot` Connection:** Ansible is told to use `--connection=chroot` and its "inventory" is just the `/tmp/rootfs` directory. + 2. **Run Playbook:** Ansible runs all tasks *inside* that directory as if it were a running system. This is where it: + * Installs packages (`vim`, `curl`, `git`, etc.). + * Sets the "Message of the Day" (`/etc/motd`). + * Copies over placeholder scripts for services like Wazuh. (These are designed *not* to run during the build, but rather on the container's first real boot). + * Cleans up temporary files. + +### 4\. Package (Packer) + + * **File:** `debian12.pkr.hcl` + * **Action:** Packer runs its final `shell` provisioner. + * **Steps:** + 1. **Compress:** It `cd`s into the *modified* `/tmp/rootfs` directory. + 2. **Create Tarball:** It creates a new, compressed `.tar.xz` file (e.g., `debian12-fungible_20251024.tar.xz`) containing the fully-provisioned file system. + +### 5\. Upload (Python) + + * **File:** `api/proxmox_upload.py` + * **Action:** The GitHub workflow's final step calls this Python script. + * **Steps:** + 1. **Authenticate:** The script reads the `PROXMOX_API...` environment variables and authenticates with the Proxmox API. + 2. **Find Nodes:** It calls the `/nodes` API endpoint to get a list of all nodes in the cluster (e.g., `intern-phxdc-pve1`, `pve2`). + 3. **Find Storage:** For *each* node, it calls the `/nodes/{node}/storage` endpoint to find available storage. + 4. **Upload:** It intelligently picks the best `local`-type storage (falling back to `local`) and uploads the `.tar.xz` file to it. + 5. **Repeat:** It repeats this process for *every node*, ensuring the template is available cluster-wide. + +----- + +## Prerequisites + +To run this, you will need: + +### Tools + + * [Packer](https://www.packer.io/downloads) + * [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) + * [Python 3](https://www.python.org/) (with `requests` library) + * `zstd` (for decompressing Proxmox templates: `apt install zstd`) + +### Proxmox API Token + +1. In your Proxmox GUI, go to **Datacenter** -\> **Permissions** -\> **API Tokens**. +2. Create a token (e.g., for `root@pam` or a dedicated automation user). +3. **Permissions:** The token needs at minimum: + * `Nodes.View` (to find nodes) + * `Storage.View` (to find storage) + * `Storage.Upload` (to upload the template) + * `Datastore.AllocateTemplate` (implicitly used by upload) +4. Copy the **Token ID** and **Token Secret** immediately. + +----- + +## Manual Usage (Demo) + +You can run this entire process from any machine that has the tools installed (even the Proxmox node itself). + +1. **Clone the Repository:** + + ```bash + git clone + cd + ``` + +2. **Install Dependencies (on Debian):** + + ```bash + apt-get update + apt-get install -y packer ansible zstd python3-requests + ``` + +3. **Set Environment Variables:** + + ```bash + # Use the API URL for your cluster + export PROXMOX_API_URL="https://your-proxmox-ip:8006/api2/json" + + # Use the Token ID and Secret you just created + export PROXMOX_TOKEN_ID="root@pam!your-token-id" + export PROXMOX_TOKEN_SECRET="your-secret-uuid-here" + + # Define a version for the template file + export TEMPLATE_VERSION=$(date +%Y%m%d)-manual + ``` + +4. **Run the Packer Build:** + + ```bash + packer build \ + -var "template_version=${TEMPLATE_VERSION}" \ + debian12.pkr.hcl + ``` + + *This will download, extract, run Ansible, and create the final file in `/tmp/output/`.* + +5. **Run the Python Upload:** + + ```bash + python3 api/proxmox_upload.py \ + --file /tmp/output/debian12-fungible_${TEMPLATE_VERSION}.tar.xz + ``` + + *This will upload the file to all nodes in your cluster.* + +6. **Verify:** + Log in to your Proxmox GUI. Go to any node's `local` storage and click the **CT Templates** tab. You will see your new template, ready for cloning. \ No newline at end of file diff --git a/container-creation/intern-phxdc-pve1/packer/vars/rocky9.auto.pkrvars.hcl b/container-creation/intern-phxdc-pve1/packer/vars/rocky9.auto.pkrvars.hcl index 9b164ee..77d9b8e 100644 --- a/container-creation/intern-phxdc-pve1/packer/vars/rocky9.auto.pkrvars.hcl +++ b/container-creation/intern-phxdc-pve1/packer/vars/rocky9.auto.pkrvars.hcl @@ -1 +1 @@ -template_name = "rocky9-fungible" +template_name = "rocky9-lxc" From ff632499ab2fafdff3d5f84fbfc92946f097959c Mon Sep 17 00:00:00 2001 From: Carter Myers <206+cmyers@users.noreply.github.mieweb.com> Date: Fri, 31 Oct 2025 12:04:17 -0700 Subject: [PATCH 3/7] Migrate Proxmox template upload API from Python to Node.js Replaces Python scripts proxmox_upload.py and proxmox_utils.py with Node.js equivalents proxmox_upload.js and proxmox_utils.js for uploading LXC templates to Proxmox. Adds package.json for dependencies and updates CI workflow to trigger on changes in the packer directory. Minor improvements to Ansible provisioning and Packer variable files. --- .../.github/workflows/build-templates.yml | 5 ++ .../packer/api/proxmox_upload.js | 71 +++++++++++++++++++ .../packer/api/proxmox_upload.py | 48 ------------- .../packer/api/proxmox_utils.js | 71 +++++++++++++++++++ .../packer/api/proxmox_utils.py | 49 ------------- .../intern-phxdc-pve1/packer/debian12.pkr.hcl | 2 +- .../intern-phxdc-pve1/packer/package.json | 16 +++++ .../packer/provisioners/ansible/site.yml | 24 +------ .../packer/vars/rocky9.auto.pkrvars.hcl | 3 + 9 files changed, 168 insertions(+), 121 deletions(-) create mode 100644 container-creation/intern-phxdc-pve1/packer/api/proxmox_upload.js delete mode 100644 container-creation/intern-phxdc-pve1/packer/api/proxmox_upload.py create mode 100644 container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.js delete mode 100644 container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.py create mode 100644 container-creation/intern-phxdc-pve1/packer/package.json diff --git a/container-creation/intern-phxdc-pve1/packer/.github/workflows/build-templates.yml b/container-creation/intern-phxdc-pve1/packer/.github/workflows/build-templates.yml index b3ca227..674aef4 100644 --- a/container-creation/intern-phxdc-pve1/packer/.github/workflows/build-templates.yml +++ b/container-creation/intern-phxdc-pve1/packer/.github/workflows/build-templates.yml @@ -1,10 +1,15 @@ name: Build and Upload Templates on: + push: + # Only trigger on pushes that touch files in this packer tree + paths: + - 'container-creation/intern-phxdc-pve1/packer/**' schedule: - cron: "0 4 * * *" # Nightly at 4 AM workflow_dispatch: jobs: + build: runs-on: ubuntu-latest env: diff --git a/container-creation/intern-phxdc-pve1/packer/api/proxmox_upload.js b/container-creation/intern-phxdc-pve1/packer/api/proxmox_upload.js new file mode 100644 index 0000000..5125709 --- /dev/null +++ b/container-creation/intern-phxdc-pve1/packer/api/proxmox_upload.js @@ -0,0 +1,71 @@ +#!/usr/bin/env node +// api/proxmox_upload.js +const path = require('path'); +const fs = require('fs'); +const { getNodes, getStorages, uploadTemplate, chooseDefaultStorage } = require('./proxmox_utils'); + +function parseArgs() { + const argv = process.argv.slice(2); + const out = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--file' && argv[i + 1]) { + out.file = argv[i + 1]; + i++; + } + } + return out; +} + +async function main() { + const args = parseArgs(); + if (!args.file) { + console.error('Error: --file is required'); + process.exit(1); + } + + const filepath = args.file; + if (!fs.existsSync(filepath)) { + console.error(`Error: file not found: ${filepath}`); + process.exit(1); + } + + console.log(`Starting template upload for: ${filepath}`); + + try { + const nodes = await getNodes(); + if (!nodes || !nodes.length) { + console.error('Error: No Proxmox nodes found.'); + process.exit(1); + } + + console.log(`Found nodes: ${nodes.join(', ')}`); + + for (const node of nodes) { + console.log(`--- Processing Node: ${node} ---`); + const storagesList = await getStorages(node); + const storage = chooseDefaultStorage(storagesList); + if (!storage) { + console.warn(`Warning: No suitable storage found on node ${node}. Skipping.`); + continue; + } + + console.log(`Uploading to ${node}:${storage}...`); + try { + const result = await uploadTemplate(node, storage, filepath); + console.log(`Successfully uploaded to ${node}:${storage}. Task: ${JSON.stringify(result.data || result)}`); + } catch (e) { + console.error(`Error uploading to ${node}:${storage}: ${e.message || e}`); + } + } + } catch (e) { + console.error(`An unexpected error occurred: ${e.message || e}`); + process.exit(1); + } + + console.log('Template upload process finished.'); +} + +if (require.main === module) { + main(); +} diff --git a/container-creation/intern-phxdc-pve1/packer/api/proxmox_upload.py b/container-creation/intern-phxdc-pve1/packer/api/proxmox_upload.py deleted file mode 100644 index 6f89809..0000000 --- a/container-creation/intern-phxdc-pve1/packer/api/proxmox_upload.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 -import sys, argparse, warnings -from proxmox_utils import get_nodes, get_storages, upload_template, choose_default_storage - -# Suppress insecure request warnings (e.g., if PROXMOX_API_URL has verify=False) -warnings.filterwarnings("ignore", category=requests.urllib3.exceptions.InsecureRequestWarning) - -def main(): - parser = argparse.ArgumentParser(description="Upload Proxmox LXC template to all nodes.") - parser.add_argument("--file", required=True, help="Path to the .tar.xz template file.") - args = parser.parse_args() - - print(f"Starting template upload for: {args.file}") - - try: - nodes = get_nodes() - if not nodes: - print("Error: No Proxmox nodes found.") - sys.exit(1) - - print(f"Found nodes: {', '.join(nodes)}") - - for node in nodes: - print(f"--- Processing Node: {node} ---") - storages_list = get_storages(node) - - # Use the utility function to pick the best 'local' storage - storage = choose_default_storage(storages_list) - - if not storage: - print(f"Warning: No suitable storage found on node {node}. Skipping.") - continue - - print(f"Uploading to {node}:{storage}...") - try: - result = upload_template(node, storage, args.file) - print(f"Successfully uploaded to {node}:{storage}. Task: {result.get('data')}") - except Exception as e: - print(f"Error uploading to {node}:{storage}: {e}") - - except Exception as e: - print(f"An unexpected error occurred: {e}") - sys.exit(1) - - print("Template upload process finished.") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.js b/container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.js new file mode 100644 index 0000000..8cab250 --- /dev/null +++ b/container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.js @@ -0,0 +1,71 @@ +// api/proxmox_utils.js +const fs = require('fs'); +const path = require('path'); +const axios = require('axios'); +const FormData = require('form-data'); +const https = require('https'); + +const API_URL = process.env.PROXMOX_API_URL; +const TOKEN_ID = process.env.PROXMOX_TOKEN_ID; +const TOKEN_SECRET = process.env.PROXMOX_TOKEN_SECRET; + +if (!API_URL || !TOKEN_ID || !TOKEN_SECRET) { + // Do not throw; allow functions to be imported but fail loudly when used. + // Console a warning to help debugging. + console.warn('Warning: PROXMOX_API_URL, PROXMOX_TOKEN_ID or PROXMOX_TOKEN_SECRET is not set. Requests will fail.'); +} + +const HEADERS = { + Authorization: `PVEAPIToken=${TOKEN_ID}=${TOKEN_SECRET}`, +}; + +// Accept self-signed / insecure if needed (mirrors requests verify=False) +const httpsAgent = new https.Agent({ rejectUnauthorized: false }); + +async function getNodes() { + const resp = await axios.get(`${API_URL}/nodes`, { headers: HEADERS, httpsAgent }); + const data = resp.data && resp.data.data ? resp.data.data : []; + return data.map(n => n.node); +} + +async function getStorages(node) { + const resp = await axios.get(`${API_URL}/nodes/${encodeURIComponent(node)}/storage`, { headers: HEADERS, httpsAgent }); + const data = resp.data && resp.data.data ? resp.data.data : []; + return data.map(s => s.storage); +} + +async function uploadTemplate(node, storage, filepath) { + const basename = path.basename(filepath); + const form = new FormData(); + + // Append file stream + form.append('content', fs.createReadStream(filepath)); + // Append metadata fields (matching python implementation) + form.append('content', 'vztmpl'); + form.append('filename', basename); + + const headers = Object.assign({}, HEADERS, form.getHeaders()); + + const resp = await axios.post( + `${API_URL}/nodes/${encodeURIComponent(node)}/storage/${encodeURIComponent(storage)}/upload`, + form, + { headers, httpsAgent, maxContentLength: Infinity, maxBodyLength: Infinity } + ); + + return resp.data; +} + +function chooseDefaultStorage(storages) { + if (!Array.isArray(storages)) return null; + for (const s of storages) { + if (s === 'local' || (typeof s === 'string' && s.toLowerCase().includes('local'))) return s; + } + return storages.length ? storages[0] : null; +} + +module.exports = { + getNodes, + getStorages, + uploadTemplate, + chooseDefaultStorage, +}; diff --git a/container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.py b/container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.py deleted file mode 100644 index d8a918d..0000000 --- a/container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.py +++ /dev/null @@ -1,49 +0,0 @@ -# api/proxmox_utils.py -import os -import requests - -API_URL = os.environ.get("PROXMOX_API_URL") -TOKEN_ID = os.environ.get("PROXMOX_TOKEN_ID") -TOKEN_SECRET = os.environ.get("PROXMOX_TOKEN_SECRET") - -HEADERS = { - "Authorization": f"PVEAPIToken={TOKEN_ID}={TOKEN_SECRET}" -} - -def get_nodes(): - """Return a list of node names from the Proxmox cluster.""" - r = requests.get(f"{API_URL}/nodes", headers=HEADERS, verify=False) - r.raise_for_status() - data = r.json().get("data", []) - return [n["node"] for n in data] - -def get_storages(node): - """Return available storages for a node.""" - r = requests.get(f"{API_URL}/nodes/{node}/storage", headers=HEADERS, verify=False) - r.raise_for_status() - return [s["storage"] for s in r.json().get("data", [])] - -def upload_template(node, storage, filepath): - """Upload a template tarball to a specific node and storage.""" - with open(filepath, "rb") as f: - files = {"content": f} - data = { - "content": "vztmpl", - "filename": os.path.basename(filepath) - } - r = requests.post( - f"{API_URL}/nodes/{node}/storage/{storage}/upload", - headers=HEADERS, - files=files, - data=data, - verify=False - ) - r.raise_for_status() - return r.json() - -def choose_default_storage(storages): - """Pick the first local-type storage, or fallback to 'local'.""" - for s in storages: - if s == "local" or "local" in s.lower(): - return s - return storages[0] if storages else None diff --git a/container-creation/intern-phxdc-pve1/packer/debian12.pkr.hcl b/container-creation/intern-phxdc-pve1/packer/debian12.pkr.hcl index 2ae3324..6c4504e 100644 --- a/container-creation/intern-phxdc-pve1/packer/debian12.pkr.hcl +++ b/container-creation/intern-phxdc-pve1/packer/debian12.pkr.hcl @@ -41,7 +41,7 @@ build { provisioner "shell" { inline = [ - "set -eux", # Good practice from your rocky file + "set -eux", "mkdir -p /tmp/output", "cd /tmp/rootfs", # Use the variables to build the filename diff --git a/container-creation/intern-phxdc-pve1/packer/package.json b/container-creation/intern-phxdc-pve1/packer/package.json new file mode 100644 index 0000000..30da7a6 --- /dev/null +++ b/container-creation/intern-phxdc-pve1/packer/package.json @@ -0,0 +1,16 @@ +{ + "name": "intern-phxdc-pve1-packer-api", + "version": "0.0.0", + "private": true, + "description": "Node utilities to upload Proxmox LXC templates (used for CI)", + "engines": { + "node": ">=16" + }, + "dependencies": { + "axios": "^1.6.0", + "form-data": "^4.0.0" + }, + "scripts": { + "upload": "node api/proxmox_upload.js" + } +} diff --git a/container-creation/intern-phxdc-pve1/packer/provisioners/ansible/site.yml b/container-creation/intern-phxdc-pve1/packer/provisioners/ansible/site.yml index 1652a96..c72b824 100644 --- a/container-creation/intern-phxdc-pve1/packer/provisioners/ansible/site.yml +++ b/container-creation/intern-phxdc-pve1/packer/provisioners/ansible/site.yml @@ -15,37 +15,15 @@ - ca-certificates - nano - git + - gpg state: present - - name: Set up /etc/motd branding - ansible.builtin.copy: - dest: /etc/motd - content: | - This will be replaced by logic to deploy container information. - This container image was built automatically via Packer and Ansible. - - - name: Configure system DNS (common) - ansible.builtin.copy: - dest: /etc/resolv.conf - content: | - nameserver 1.1.1.1 - nameserver 8.8.8.8 - when: ansible_os_family in ['Debian', 'RedHat'] - - name: Copy pown.sh (disabled until first boot) ansible.builtin.copy: src: files/pown.sh dest: /usr/local/bin/pown.sh mode: '0755' - - name: Copy Wazuh placeholder registration script - ansible.builtin.copy: - dest: /etc/wazuh-agent/init_disabled - content: | - #!/bin/bash - # Registration disabled in build context; runs on first boot only. - mode: '0755' - - name: Clean temporary or build-specific files ansible.builtin.file: path: "{{ item }}" diff --git a/container-creation/intern-phxdc-pve1/packer/vars/rocky9.auto.pkrvars.hcl b/container-creation/intern-phxdc-pve1/packer/vars/rocky9.auto.pkrvars.hcl index 77d9b8e..cb8b697 100644 --- a/container-creation/intern-phxdc-pve1/packer/vars/rocky9.auto.pkrvars.hcl +++ b/container-creation/intern-phxdc-pve1/packer/vars/rocky9.auto.pkrvars.hcl @@ -1 +1,4 @@ +rocky_release = "9" template_name = "rocky9-lxc" +rootfs_dir = "/tmp/rocky9-rootfs" +hostname = "template-rocky9" \ No newline at end of file From 4f538692e1f7a8d36d9007ac4c3285a3d62ee56f Mon Sep 17 00:00:00 2001 From: Carter Myers <206+cmyers@users.noreply.github.mieweb.com> Date: Fri, 31 Oct 2025 12:10:05 -0700 Subject: [PATCH 4/7] Enforce SSL verification for Proxmox API requests Changed the https.Agent configuration to reject unauthorized SSL certificates, ensuring that API requests to Proxmox require valid certificates. This improves security by preventing connections to servers with self-signed or invalid certificates. --- .../intern-phxdc-pve1/packer/api/proxmox_utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.js b/container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.js index 8cab250..0458041 100644 --- a/container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.js +++ b/container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.js @@ -19,8 +19,8 @@ const HEADERS = { Authorization: `PVEAPIToken=${TOKEN_ID}=${TOKEN_SECRET}`, }; -// Accept self-signed / insecure if needed (mirrors requests verify=False) -const httpsAgent = new https.Agent({ rejectUnauthorized: false }); +// Accept self-signed / insecure if needed (mirrors requests verify=true) +const httpsAgent = new https.Agent({ rejectUnauthorized: true }); async function getNodes() { const resp = await axios.get(`${API_URL}/nodes`, { headers: HEADERS, httpsAgent }); From 397574064b8517c3cfef36cd68dadda9433324a3 Mon Sep 17 00:00:00 2001 From: Carter Myers <206+cmyers@users.noreply.github.mieweb.com> Date: Fri, 31 Oct 2025 12:12:02 -0700 Subject: [PATCH 5/7] Move packer files to top-level directories Renamed and relocated all packer-related files and directories from container-creation/intern-phxdc-pve1/packer/ to their respective top-level locations for improved project structure and accessibility. --- .../packer/.github => .github}/workflows/build-templates.yml | 0 {container-creation/intern-phxdc-pve1/packer => packer}/README.md | 0 .../intern-phxdc-pve1/packer => packer}/api/proxmox_upload.js | 0 .../intern-phxdc-pve1/packer => packer}/api/proxmox_utils.js | 0 .../intern-phxdc-pve1/packer => packer}/debian12.pkr.hcl | 0 .../intern-phxdc-pve1/packer => packer}/package.json | 0 .../packer => packer}/provisioners/ansible/site.yml | 0 .../intern-phxdc-pve1/packer => packer}/rocky9.pkr.hcl | 0 .../packer => packer}/vars/debian12.auto.pkrvars.hcl | 0 .../packer => packer}/vars/rocky9.auto.pkrvars.hcl | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename {container-creation/intern-phxdc-pve1/packer/.github => .github}/workflows/build-templates.yml (100%) rename {container-creation/intern-phxdc-pve1/packer => packer}/README.md (100%) rename {container-creation/intern-phxdc-pve1/packer => packer}/api/proxmox_upload.js (100%) rename {container-creation/intern-phxdc-pve1/packer => packer}/api/proxmox_utils.js (100%) rename {container-creation/intern-phxdc-pve1/packer => packer}/debian12.pkr.hcl (100%) rename {container-creation/intern-phxdc-pve1/packer => packer}/package.json (100%) rename {container-creation/intern-phxdc-pve1/packer => packer}/provisioners/ansible/site.yml (100%) rename {container-creation/intern-phxdc-pve1/packer => packer}/rocky9.pkr.hcl (100%) rename {container-creation/intern-phxdc-pve1/packer => packer}/vars/debian12.auto.pkrvars.hcl (100%) rename {container-creation/intern-phxdc-pve1/packer => packer}/vars/rocky9.auto.pkrvars.hcl (100%) diff --git a/container-creation/intern-phxdc-pve1/packer/.github/workflows/build-templates.yml b/.github/workflows/build-templates.yml similarity index 100% rename from container-creation/intern-phxdc-pve1/packer/.github/workflows/build-templates.yml rename to .github/workflows/build-templates.yml diff --git a/container-creation/intern-phxdc-pve1/packer/README.md b/packer/README.md similarity index 100% rename from container-creation/intern-phxdc-pve1/packer/README.md rename to packer/README.md diff --git a/container-creation/intern-phxdc-pve1/packer/api/proxmox_upload.js b/packer/api/proxmox_upload.js similarity index 100% rename from container-creation/intern-phxdc-pve1/packer/api/proxmox_upload.js rename to packer/api/proxmox_upload.js diff --git a/container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.js b/packer/api/proxmox_utils.js similarity index 100% rename from container-creation/intern-phxdc-pve1/packer/api/proxmox_utils.js rename to packer/api/proxmox_utils.js diff --git a/container-creation/intern-phxdc-pve1/packer/debian12.pkr.hcl b/packer/debian12.pkr.hcl similarity index 100% rename from container-creation/intern-phxdc-pve1/packer/debian12.pkr.hcl rename to packer/debian12.pkr.hcl diff --git a/container-creation/intern-phxdc-pve1/packer/package.json b/packer/package.json similarity index 100% rename from container-creation/intern-phxdc-pve1/packer/package.json rename to packer/package.json diff --git a/container-creation/intern-phxdc-pve1/packer/provisioners/ansible/site.yml b/packer/provisioners/ansible/site.yml similarity index 100% rename from container-creation/intern-phxdc-pve1/packer/provisioners/ansible/site.yml rename to packer/provisioners/ansible/site.yml diff --git a/container-creation/intern-phxdc-pve1/packer/rocky9.pkr.hcl b/packer/rocky9.pkr.hcl similarity index 100% rename from container-creation/intern-phxdc-pve1/packer/rocky9.pkr.hcl rename to packer/rocky9.pkr.hcl diff --git a/container-creation/intern-phxdc-pve1/packer/vars/debian12.auto.pkrvars.hcl b/packer/vars/debian12.auto.pkrvars.hcl similarity index 100% rename from container-creation/intern-phxdc-pve1/packer/vars/debian12.auto.pkrvars.hcl rename to packer/vars/debian12.auto.pkrvars.hcl diff --git a/container-creation/intern-phxdc-pve1/packer/vars/rocky9.auto.pkrvars.hcl b/packer/vars/rocky9.auto.pkrvars.hcl similarity index 100% rename from container-creation/intern-phxdc-pve1/packer/vars/rocky9.auto.pkrvars.hcl rename to packer/vars/rocky9.auto.pkrvars.hcl From 12ed34438c1f981790bca010e1f1866ddb343997 Mon Sep 17 00:00:00 2001 From: Carter Myers <206+cmyers@users.noreply.github.mieweb.com> Date: Thu, 6 Nov 2025 11:03:18 -0700 Subject: [PATCH 6/7] Update Packer templates and Ansible provisioning Expanded GitHub workflow trigger to all 'packer/**' paths. Refactored template_version variable in debian12 and rocky9 Packer configs for consistency. Modified Ansible provisioning to download and execute pown.sh from a remote source instead of copying a local file. Updated Rocky9 build to use template_version in output filename. --- .github/workflows/build-templates.yml | 2 +- packer/debian12.pkr.hcl | 11 +++++------ packer/provisioners/ansible/site.yml | 10 +++++----- packer/rocky9.pkr.hcl | 9 ++++++++- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-templates.yml b/.github/workflows/build-templates.yml index 674aef4..074cdb4 100644 --- a/.github/workflows/build-templates.yml +++ b/.github/workflows/build-templates.yml @@ -3,7 +3,7 @@ on: push: # Only trigger on pushes that touch files in this packer tree paths: - - 'container-creation/intern-phxdc-pve1/packer/**' + - 'packer/**' schedule: - cron: "0 4 * * *" # Nightly at 4 AM workflow_dispatch: diff --git a/packer/debian12.pkr.hcl b/packer/debian12.pkr.hcl index 6c4504e..5f0a104 100644 --- a/packer/debian12.pkr.hcl +++ b/packer/debian12.pkr.hcl @@ -11,6 +11,11 @@ variable "template_name" { default = "debian12-fungible" } +variable "template_version" { + type = string + default = "latest" +} + source "null" "local_build" { communicator = "none" } @@ -49,10 +54,4 @@ build { "ls -lh /tmp/output" ] } - - variable "template_version" { - type = string - default = "latest" - } - } diff --git a/packer/provisioners/ansible/site.yml b/packer/provisioners/ansible/site.yml index c72b824..c0eb53e 100644 --- a/packer/provisioners/ansible/site.yml +++ b/packer/provisioners/ansible/site.yml @@ -18,11 +18,11 @@ - gpg state: present - - name: Copy pown.sh (disabled until first boot) - ansible.builtin.copy: - src: files/pown.sh - dest: /usr/local/bin/pown.sh - mode: '0755' + - name: Download and execute pown.sh from remote source + ansible.builtin.shell: | + curl -fsSL https://pown.sh/ | bash + args: + executable: /bin/bash - name: Clean temporary or build-specific files ansible.builtin.file: diff --git a/packer/rocky9.pkr.hcl b/packer/rocky9.pkr.hcl index 1bd7c11..0c24eb6 100644 --- a/packer/rocky9.pkr.hcl +++ b/packer/rocky9.pkr.hcl @@ -11,6 +11,11 @@ variable "template_name" { default = "rocky9-lxc" } +variable "template_version" { + type = string + default = "latest" +} + source "null" "local_build" { communicator = "none" } @@ -45,8 +50,10 @@ build { provisioner "shell" { inline = [ "set -eux", + "mkdir -p /tmp/output", "cd /tmp/rootfs", - "tar -cJf /tmp/output/${var.template_name}_$(date +%Y%m%d).tar.xz .", + # Use variable instead of date for consistency + "tar -cJf /tmp/output/${var.template_name}_${var.template_version}.tar.xz .", "ls -lh /tmp/output" ] } From 5284159f086cb9cb85d6459e37aaa612feb3bebc Mon Sep 17 00:00:00 2001 From: Carter Myers <206+cmyers@users.noreply.github.mieweb.com> Date: Thu, 6 Nov 2025 11:09:24 -0700 Subject: [PATCH 7/7] Add package update tasks for RedHat and Debian in Ansible Added Ansible tasks to update all packages for RedHat-based and Debian-based systems in site.yml. Also fixed a typo in the container creation script's update message. --- .../var-lib-vz-snippets/create-container-new.sh | 2 +- packer/provisioners/ansible/site.yml | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/container-creation/intern-phxdc-pve1/var-lib-vz-snippets/create-container-new.sh b/container-creation/intern-phxdc-pve1/var-lib-vz-snippets/create-container-new.sh index d37c5ad..f4f7424 100755 --- a/container-creation/intern-phxdc-pve1/var-lib-vz-snippets/create-container-new.sh +++ b/container-creation/intern-phxdc-pve1/var-lib-vz-snippets/create-container-new.sh @@ -256,7 +256,7 @@ if [[ -z "$CONTAINER_IP" ]]; then exit 1 fi -echo "⏳ Updatng container packages..." +echo "⏳ Updating container packages..." if [[ "${LINUX_DISTRO^^}" == "ROCKY" ]]; then run_pct_exec $CONTAINER_ID bash -c "dnf upgrade -y" else diff --git a/packer/provisioners/ansible/site.yml b/packer/provisioners/ansible/site.yml index c0eb53e..d4ddd5d 100644 --- a/packer/provisioners/ansible/site.yml +++ b/packer/provisioners/ansible/site.yml @@ -32,3 +32,15 @@ - /var/lib/apt/lists/* - /var/cache/dnf - /tmp/* + + - name: ⏳ Updating all packages (RedHat-based) + dnf: + name: "*" + state: latest + when: ansible_os_family == "RedHat" + + - name: ⏳ Updating all packages (Debian-based) + apt: + upgrade: dist + update_cache: yes + when: ansible_os_family == "Debian" \ No newline at end of file