ai-dynamo · dagil-nvidia · Oct 10, 2025 · Oct 10, 2025 · Oct 10, 2025 · Oct 10, 2025
@@ -0,0 +1,48 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: 'Dependency Extraction Setup'
+description: 'Set up Python environment and install dependencies for dependency extraction'
+inputs:
+  python-version:
+    description: 'Python version to use'
+    required: false
+    default: '3.12'
+  fetch-depth:
+    description: 'Depth for git checkout (0 for full history, 1 for shallow)'
+    required: false
+    default: '0'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: ${{ inputs.fetch-depth }}
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Install dependencies
+      shell: bash
+      run: pip install pyyaml
+
+    - name: Create reports directory
+      shell: bash
+      run: mkdir -p .github/reports
+
@@ -0,0 +1,196 @@
+# Dependency Extraction Configuration
+# This file defines where to find dependency information for each component
+
+github:
+  repo: "ai-dynamo/dynamo"
+  branch: "main"
+
+baseline:
+  # Fallback baseline count for warnings (if no previous CSV exists)
+  # The script automatically uses the previous extraction's count as the baseline
+  dependency_count: 251
+
+known_version_discrepancies:
+  # Document intentional version discrepancies to reduce noise
+  # These will still be reported but marked as "known" with the provided reason
+  - dependency: "PyTorch"
+    reason: "TensorRT-LLM uses NVIDIA container (2.8.0), vLLM uses 2.7.1+cu128 (ARM64 wheel compatibility)"
+  - dependency: "torchvision"
+    reason: "Matches corresponding PyTorch versions across components"
+
+critical_dependencies:
+  # List of critical dependencies (case-insensitive matching)
+  # Supports exact names or partial matches (e.g., "CUDA" matches "NVIDIA CUDA")
+
+  # Core Runtime & Languages
+  - name: "Python"
+    reason: "Primary runtime language"
+  - name: "Rust"
+    reason: "Systems programming language"
+  - name: "CUDA"
+    reason: "GPU compute platform"
+
+  # Infrastructure & Orchestration (Docker Compose and Helm Chart dependencies)
+  - name: "etcd"
+    reason: "Distributed configuration store"
+  - name: "bitnami"
+    reason: "ETCD container image"
+  - name: "nats"
+    reason: "Messaging system"
+  - name: "prometheus-nats-exporter"
+    reason: "NATS monitoring"
+  - name: "Kubernetes"
+    reason: "Container orchestration"
+  - name: "dynamo-operator"
+    reason: "Dynamo Kubernetes operator"
+
+  # ML Frameworks & Base Images
+  - name: "PyTorch"
+    reason: "Deep learning framework"
+  - name: "NVIDIA PyTorch"
+    reason: "TensorRT-LLM base container"
+  - name: "CUDA-dl-base"
+    reason: "vLLM/SGLang base container"
+  - name: "TensorRT-LLM"
+    reason: "NVIDIA TensorRT-LLM inference framework"
+  - name: "tensorrt-llm"
+    reason: "TensorRT-LLM Python package"
+  - name: "vLLM"
+    reason: "vLLM inference framework"
+  - name: "vllm"
+    reason: "vLLM Python package"
+  - name: "SGLang"
+    reason: "SGLang inference framework"
+  - name: "sglang"
+    reason: "SGLang Python package"
+
+  # Network & Communication (ARG names are formatted: NIXL_REF -> Nixl Ref)
+  - name: "Nixl"
+    reason: "Network interconnect library"
+  - name: "Nixl Ref"
+    reason: "Network interconnect library version"
+  - name: "Nixl Ucx Ref"
+    reason: "NIXL with UCX version"
+  - name: "Ucx"
+    reason: "Unified communication framework"
+  - name: "ucx-py"
+    reason: "UCX Python bindings"
+  - name: "Nvshmem"
+    reason: "NVIDIA SHMEM communication"
+
+  # ML Inference & Optimization (ARG names are formatted: FLASH_ATTN_VER -> Flash Attn Ver)
+  - name: "Flash Attn"
+    reason: "Flash attention implementation"
+  - name: "Flashinf"
+    reason: "Flash attention (FlashInfer)"
+  - name: "flashinfer"
+    reason: "Flash attention in dependencies"
+  - name: "vllm [flashinfer]"
+    reason: "vLLM with FlashInfer support"
+  - name: "Deepgemm"
+    reason: "Optimized GEMM operations"
+  - name: "pplx"
+    reason: "Inference kernels"
+
+  # Build & Package Tools
+  - name: "CMake"
+    reason: "Build system"
+  - name: "CMAKE_VERSION"
+    reason: "CMake version ARG"
+  - name: "Uvicorn"
+    reason: "ASGI server"
+
+  # Performance & Benchmarking
+  - name: "Genai Perf"
+    reason: "GenAI performance testing"
+  - name: "genai-perf"
+    reason: "GenAI performance testing (pip package)"
+  - name: "aiperf"
+    reason: "AI performance benchmarking"
+
+  # Custom Components
+  - name: "ModelExpress"
+    reason: "Model serving infrastructure"
+  - name: "modelexpress"
+    reason: "Model serving infrastructure (Rust crate)"
+  - name: "grove"
+    reason: "Scheduling component"
+  - name: "Kai"
+    reason: "Scheduler"
+
+components:
+  trtllm:
+    dockerfiles:
+      - "container/Dockerfile.trtllm"
+      - "containers/Dockerfile.trtllm"  # fallback if moved
+    scripts:
+      - "container/deps/trtllm/install_nixl.sh"
+      - "container/dependencies/trtllm/install_nixl.sh"  # fallback
+    required: true
+
+  vllm:
+    dockerfiles:
+      - "container/Dockerfile.vllm"
+      - "containers/Dockerfile.vllm"  # fallback
+    scripts:
+      - "container/deps/vllm/install_vllm.sh"
+      - "container/dependencies/vllm/install_vllm.sh"  # fallback
+    required: true
+
+  sglang:
+    dockerfiles:
+      - "container/Dockerfile.sglang"
+      - "container/Dockerfile.sglang-wideep"  # Has CMAKE_VERSION
+      - "containers/Dockerfile.sglang"  # fallback
+    scripts: []
+    required: true
+
+  operator:
+    dockerfiles:
+      - "deploy/cloud/operator/Dockerfile"
+      - "deployment/cloud/operator/Dockerfile"  # fallback
+    go_modules:
+      - "deploy/cloud/operator/go.mod"
+      - "deployment/cloud/operator/go.mod"  # fallback
+    scripts: []
+    required: true
+
+  shared:
+    dockerfiles:
+      - "container/Dockerfile"
+      - "containers/Dockerfile"  # fallback
+    requirements:
+      # Use glob patterns for multiple files
+      - pattern: "container/deps/requirements*.txt"
+        exclude: ["**/requirements.in"]
+      - pattern: "containers/deps/requirements*.txt"  # fallback
+        exclude: []
+    pyproject:
+      - "pyproject.toml"
+      - "benchmarks/pyproject.toml"
+    docker_compose:
+      - "deploy/docker-compose.yml"
+    helm_charts:
+      - "deploy/cloud/helm/platform/Chart.yaml"
+      - "deploy/helm/chart/Chart.yaml"
+    rust_toolchain:
+      - "rust-toolchain.toml"
+    cargo_toml:
+      - "Cargo.toml"
+    k8s_recipes:
+      - pattern: "recipes/**/perf.yaml"
+        exclude: []
+    scripts: []
+    required: true
+
+# Optional: Define custom extraction patterns
+extraction:
+  dockerfile:
+    base_image_keywords: ["FROM"]
+    version_arg_keywords: ["VERSION", "REF", "TAG", "_VER"]
+
+  requirements:
+    version_operators: ["==", ">=", "<=", ">", "<", "~=", "!="]
+
+  go_mod:
+    skip_indirect: false  # Set to true to skip indirect dependencies
@@ -0,0 +1,132 @@
+# Dependency Reports
+
+This directory contains the latest dependency extraction reports for the Dynamo repository.
+
+## Files
+
+### `dependency_versions_latest.csv`
+The most recent dependency extraction results. Updated nightly by the automated CI workflow.
+
+### `unversioned_dependencies_latest.csv`
+List of dependencies without explicit version constraints. These should be reviewed and pinned for reproducible builds.
+
+### `releases/dependency_versions_vX.X.X.csv`
+Permanent snapshots of dependencies for each release version. Created automatically when release branches are cut.
+
+**Examples:**
+- `releases/dependency_versions_v1.2.3.csv` - Release 1.2.3 snapshot
+- `releases/dependency_versions_v2.0.0.csv` - Release 2.0.0 snapshot
+
+**CSV Columns:**
+- **Component** - Component category (trtllm, vllm, sglang, operator, shared)
+- **Category** - Dependency type (Base Image, Framework, Go Module, Python Package, Docker Compose Service, Helm Chart, etc.)
+- **Dependency Name** - Human-readable name
+- **Version** - Version number or constraint
+- **Source File** - Relative path to file defining the dependency
+- **GitHub URL** - Clickable link to source line
+- **Package Source URL** - Direct link to package documentation:
+  - PyPI for Python packages
+  - Docker Hub or NGC Catalog for containers
+  - Artifact Hub for Helm charts
+  - pkg.go.dev for Go modules
+  - Official download pages for languages/tools
+- **Status** - Legacy status field (New, Changed, Unchanged)
+- **Diff from Latest** - Comparison to latest nightly:
+  - `New` - New dependency not in latest nightly
+  - `Unchanged` - Same version as latest nightly
+  - `X → Y` - Version changed from X to Y
+  - `N/A` - No latest nightly to compare against
+- **Diff from Release** - Comparison to latest release:
+  - `New` - New dependency not in latest release
+  - `Unchanged` - Same version as latest release
+  - `X → Y` - Version changed from X to Y
+  - `N/A` - No release snapshot to compare against
+- **Critical** - Yes/No flag for critical dependencies
+- **NVIDIA Product** - Yes/No flag indicating if dependency is an NVIDIA product
+- **Notes** - Additional context
+
+**CSV Sorting:**
+The CSV is sorted to make critical dependencies easy to identify:
+1. By Component (trtllm → vllm → sglang → operator → shared)
+2. By Critical status (Yes before No) within each component
+3. Alphabetically by dependency name
+
+**Extraction Sources:**
+The script extracts dependencies from multiple sources:
+- **Dockerfiles** - Base images and ARG/ENV versions
+- **requirements.txt** - Python packages (main, test, docs, standard)
+- **pyproject.toml** - Project metadata and dependencies
+- **go.mod** - Go module dependencies
+- **shell scripts** - Version variables from install scripts
+- **docker-compose.yml** - Service container versions
+- **Chart.yaml** - Helm chart and dependency versions
+- **rust-toolchain.toml** - Rust compiler version
+- **Cargo.toml** - Rust Git dependencies
+- **K8s recipe YAML** - Git-based pip installs from recipe files
+
+### Critical Dependencies
+
+Critical dependencies are flagged in the CSV to highlight components that require special attention for:
+- Security updates
+- Version compatibility
+- Production stability
+- Compliance requirements
+
+The list of critical dependencies is **explicitly maintained** in `../dependency-extraction/config.yaml` under the `critical_dependencies` section. Only dependencies listed in this configuration file are marked as critical. Examples include:
+- CUDA (compute platform)
+- PyTorch (ML framework)
+- TensorRT-LLM (inference framework)
+- vLLM (inference framework)
+- SGLang (inference framework)
+- Python (runtime)
+- Kubernetes (orchestration)
+- NATS (message broker)
+- etcd (key-value store)
+
+## Timestamped Versions
+
+Timestamped CSV files (e.g., `dependency_versions_20251009_1924.csv`) are:
+- **Generated** by the nightly workflow
+- **Stored** in GitHub Artifacts (90-day retention)
+- **Not committed** to the repo to avoid clutter
+- **Available** for download from the workflow run page
+
+## Workflows
+
+### Nightly Tracking (`.github/workflows/dependency-extraction-nightly.yml`)
+- **Schedule:** Daily at 2 AM UTC
+- **Trigger:** Can be manually triggered via Actions UI
+- **Output:** Updates `*_latest.csv` files, creates PR when changes detected
+- **Artifacts:** Uploads timestamped CSVs for 90-day retention
+
+### Release Snapshots (`.github/workflows/dependency-extraction-release.yml`)
+- **Trigger:** Automatically when `release/*.*.*` branches are pushed
+- **Output:** Creates permanent `releases/dependency_versions_vX.X.X.csv`
+- **Purpose:** Permanent record of dependencies for each release
+- **Artifacts:** Stored for 365 days (1 year)
+
+## Manual Extraction
+
+To run manually from repository root:
+
+```bash
+# Basic extraction
+python3 .github/workflows/extract_dependency_versions.py
+
+# With options
+python3 .github/workflows/extract_dependency_versions.py \
+  --output .github/reports/dependency_versions_latest.csv \
+  --report-unversioned
+
+# Validate configuration
+python3 .github/workflows/extract_dependency_versions.py --validate
+
+# See all options
+python3 .github/workflows/extract_dependency_versions.py --help
+```
+
+## Links
+
+- 🤖 [Extraction Script](../workflows/extract_dependency_versions.py)
+- ⚙️ [Configuration](../dependency-extraction/config.yaml)
+- 🔄 [Unified Workflow](../workflows/dependency-extraction.yml) (handles both nightly and release)