From 140041dd68f80d8bbe9879de1584eb586f9b069e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 13:41:23 -0400 Subject: [PATCH 01/19] Bump the nuget-dependencies group with 7 updates (#23) Bumps DemaConsulting.TestResults from 1.5.0 to 1.6.0 Bumps dotnet-sonarscanner from 11.1.0 to 11.2.0 Bumps Microsoft.CodeAnalysis.NetAnalyzers from 10.0.103 to 10.0.201 Bumps Microsoft.Extensions.FileSystemGlobbing from 10.0.3 to 10.0.5 Bumps Microsoft.SourceLink.GitHub from 10.0.103 to 10.0.201 Bumps Polyfill from 9.18.0 to 9.22.0 Bumps SonarAnalyzer.CSharp from 10.20.0.135146 to 10.21.0.135717 --- updated-dependencies: - dependency-name: DemaConsulting.TestResults dependency-version: 1.6.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: nuget-dependencies - dependency-name: dotnet-sonarscanner dependency-version: 11.2.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: nuget-dependencies - dependency-name: Microsoft.CodeAnalysis.NetAnalyzers dependency-version: 10.0.201 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: nuget-dependencies - dependency-name: Microsoft.CodeAnalysis.NetAnalyzers dependency-version: 10.0.201 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: nuget-dependencies - dependency-name: Microsoft.Extensions.FileSystemGlobbing dependency-version: 10.0.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: nuget-dependencies - dependency-name: Microsoft.SourceLink.GitHub dependency-version: 10.0.201 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: nuget-dependencies - dependency-name: Polyfill dependency-version: 9.22.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: nuget-dependencies - dependency-name: SonarAnalyzer.CSharp dependency-version: 10.21.0.135717 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: nuget-dependencies - dependency-name: SonarAnalyzer.CSharp dependency-version: 10.21.0.135717 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: nuget-dependencies ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .config/dotnet-tools.json | 2 +- .../DemaConsulting.ReviewMark.csproj | 12 ++++++------ .../DemaConsulting.ReviewMark.Tests.csproj | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index c650c2a..88d4aa0 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "dotnet-sonarscanner": { - "version": "11.1.0", + "version": "11.2.0", "commands": [ "dotnet-sonarscanner" ] diff --git a/src/DemaConsulting.ReviewMark/DemaConsulting.ReviewMark.csproj b/src/DemaConsulting.ReviewMark/DemaConsulting.ReviewMark.csproj index 3deb4ca..bf8e20b 100644 --- a/src/DemaConsulting.ReviewMark/DemaConsulting.ReviewMark.csproj +++ b/src/DemaConsulting.ReviewMark/DemaConsulting.ReviewMark.csproj @@ -48,17 +48,17 @@ - + - + - - + + @@ -68,11 +68,11 @@ in packages that consume this tool. - IncludeAssets lists all asset types (including 'analyzers' and 'buildtransitive') to ensure Roslyn analyzers and MSBuild targets are fully activated during the build. --> - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/test/DemaConsulting.ReviewMark.Tests/DemaConsulting.ReviewMark.Tests.csproj b/test/DemaConsulting.ReviewMark.Tests/DemaConsulting.ReviewMark.Tests.csproj index c8ea4a5..32fa8d5 100644 --- a/test/DemaConsulting.ReviewMark.Tests/DemaConsulting.ReviewMark.Tests.csproj +++ b/test/DemaConsulting.ReviewMark.Tests/DemaConsulting.ReviewMark.Tests.csproj @@ -46,11 +46,11 @@ in any project that references this test project. - IncludeAssets lists all asset types (including 'analyzers' and 'buildtransitive') to ensure Roslyn analyzers and MSBuild targets are fully activated during the build. --> - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive From a84765c22cc0ec9c05d080371dc474c9a5152667 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 21:02:51 -0400 Subject: [PATCH 02/19] Bump the nuget-dependencies group with 1 update (#24) Bumps coverlet.collector from 8.0.0 to 8.0.1 --- updated-dependencies: - dependency-name: coverlet.collector dependency-version: 8.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: nuget-dependencies ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../DemaConsulting.ReviewMark.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/DemaConsulting.ReviewMark.Tests/DemaConsulting.ReviewMark.Tests.csproj b/test/DemaConsulting.ReviewMark.Tests/DemaConsulting.ReviewMark.Tests.csproj index 32fa8d5..e961571 100644 --- a/test/DemaConsulting.ReviewMark.Tests/DemaConsulting.ReviewMark.Tests.csproj +++ b/test/DemaConsulting.ReviewMark.Tests/DemaConsulting.ReviewMark.Tests.csproj @@ -29,7 +29,7 @@ - PrivateAssets="all" keeps this test-coverage tool out of any consuming project's dependencies. - IncludeAssets lists all asset types (including 'build' and 'buildtransitive') to ensure the data collector MSBuild targets are activated so coverage is collected during test runs. --> - + all runtime; build; native; contentfiles; analyzers; buildtransitive From c34b46a99f68b15728233b9106351a8ef6ee2eca Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 17:30:07 -0400 Subject: [PATCH 03/19] Bring in latest repo-consistency agent and apply template improvements from TemplateDotNetTool (#25) * Initial plan * Bring in latest repo-consistency agent and template improvements from TemplateDotNetTool Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/1ac7f038-065a-41da-8e89-1a36e85c95fa * Apply remaining missed template improvements: file renames, yaml format conversions, lint script rewrite Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/42beae80-b9d8-4be1-ad38-c556f57170b4 * Fix linting issues: wrap long lines in CONTRIBUTING.md, update workflow, add spell check policy to AGENTS.md Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/3150f0bf-4a5f-4a48-a324-6ed80d75870a * Update .cspell.json references to .cspell.yaml in contributing guide and technical-writer agent Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/81812042-1ec3-4213-a9b5-de4ea286f9c1 --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --- .config/dotnet-tools.json | 2 +- .cspell.json | 91 -------- .cspell.yaml | 101 ++++++++ .gitattributes | 7 + .github/agents/code-quality-agent.md | 85 ------- .github/agents/code-quality.agent.md | 216 ++++++++++++++++++ .github/agents/code-review.agent.md | 46 ++++ .github/agents/repo-consistency-agent.md | 150 ------------ .github/agents/repo-consistency.agent.md | 39 ++++ ...rements-agent.md => requirements.agent.md} | 6 +- ...veloper.md => software-developer.agent.md} | 6 +- ...al-writer.md => technical-writer.agent.md} | 8 +- ...t-developer.md => test-developer.agent.md} | 6 +- .github/workflows/build.yaml | 28 ++- .gitignore | 1 + .markdownlint-cli2.jsonc | 15 -- .markdownlint-cli2.yaml | 47 ++++ .yamllint.yaml | 9 + AGENTS.md | 14 +- CONTRIBUTING.md | 20 +- lint.bat | 46 ++-- lint.sh | 41 +++- package.json | 2 + pip-requirements.txt | 1 + 24 files changed, 587 insertions(+), 400 deletions(-) delete mode 100644 .cspell.json create mode 100644 .cspell.yaml create mode 100644 .gitattributes delete mode 100644 .github/agents/code-quality-agent.md create mode 100644 .github/agents/code-quality.agent.md create mode 100644 .github/agents/code-review.agent.md delete mode 100644 .github/agents/repo-consistency-agent.md create mode 100644 .github/agents/repo-consistency.agent.md rename .github/agents/{requirements-agent.md => requirements.agent.md} (96%) rename .github/agents/{software-developer.md => software-developer.agent.md} (94%) rename .github/agents/{technical-writer.md => technical-writer.agent.md} (90%) rename .github/agents/{test-developer.md => test-developer.agent.md} (97%) delete mode 100644 .markdownlint-cli2.jsonc create mode 100644 .markdownlint-cli2.yaml create mode 100644 pip-requirements.txt diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 88d4aa0..e1f510b 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -39,7 +39,7 @@ ] }, "demaconsulting.buildmark": { - "version": "0.4.0", + "version": "0.4.1", "commands": [ "buildmark" ] diff --git a/.cspell.json b/.cspell.json deleted file mode 100644 index 6955147..0000000 --- a/.cspell.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "version": "0.2", - "language": "en", - "words": [ - "Anson", - "Blockquotes", - "buildmark", - "BuildMark", - "buildnotes", - "camelcase", - "Checkmarx", - "CodeQL", - "copilot", - "cspell", - "csproj", - "dbproj", - "dcterms", - "Dema", - "demaconsulting", - "DEMACONSULTINGNUGETKEY", - "Dependabot", - "dependabot", - "doctitle", - "dotnet", - "editorconfig", - "filepart", - "fsproj", - "Gidget", - "gitattributes", - "ibiqlik", - "LINQ", - "maintainer", - "markdownlint", - "mermaid", - "mstest", - "myterm", - "ncipollo", - "nuget", - "nupkg", - "opencover", - "pagetitle", - "pandoc", - "Pylint", - "Qube", - "reqstream", - "ReqStream", - "Sarif", - "SarifMark", - "SBOM", - "Semgrep", - "semver", - "slnx", - "snupkg", - "sonarmark", - "SonarMark", - "SonarQube", - "spdx", - "streetsidesoftware", - "empira", - "fileshare", - "Pdfs", - "PdfSharp", - "reindex", - "reviewmark", - "ReviewMark", - "testname", - "tracematrix", - "triaging", - "Trivy", - "trx", - "vbproj", - "vcxproj", - "Weasyprint", - "yamllint" - ], - "ignorePaths": [ - "node_modules", - ".git", - "bin", - "obj", - "*.nupkg", - "*.snupkg", - "*.dll", - "*.exe", - "*.trx", - "*.spdx.json", - "package-lock.json", - "yarn.lock", - "AGENT_REPORT_*.md" - ] -} diff --git a/.cspell.yaml b/.cspell.yaml new file mode 100644 index 0000000..f941b4f --- /dev/null +++ b/.cspell.yaml @@ -0,0 +1,101 @@ +--- +# Spell-Checking +# +# PURPOSE: +# - Maintain professional documentation and code quality +# - Catch spelling errors before publication +# - Support consistent technical terminology usage +# - Misspelled words should be fixed in the source +# - NEVER add a misspelled word to the 'words' list +# - PROPOSE only genuine technical terms/names as needed + +version: "0.2" +language: en + +# Project-specific technical terms and tool names +words: + - Anson + - Blockquotes + - buildmark + - BuildMark + - buildnotes + - camelcase + - Checkmarx + - CodeQL + - copilot + - cspell + - csproj + - dbproj + - dcterms + - Dema + - demaconsulting + - DEMACONSULTINGNUGETKEY + - Dependabot + - dependabot + - doctitle + - dotnet + - editorconfig + - empira + - filepart + - fileshare + - fsproj + - Gidget + - gitattributes + - ibiqlik + - LINQ + - maintainer + - markdownlint + - mermaid + - mstest + - myterm + - ncipollo + - nuget + - nupkg + - opencover + - pagetitle + - pandoc + - Pdfs + - PdfSharp + - Propagatable + - Pylint + - Qube + - reindex + - reqstream + - ReqStream + - reviewmark + - ReviewMark + - Sarif + - SarifMark + - SBOM + - Semgrep + - semver + - slnx + - snupkg + - sonarmark + - SonarMark + - SonarQube + - spdx + - streetsidesoftware + - testname + - tracematrix + - triaging + - Trivy + - trx + - vbproj + - vcxproj + - versionmark + - Weasyprint + - yamllint + +# Exclude common build artifacts, dependencies, and vendored third-party code +ignorePaths: + - "**/.git/**" + - "**/node_modules/**" + - "**/.venv/**" + - "**/thirdparty/**" + - "**/third-party/**" + - "**/3rd-party/**" + - "**/AGENT_REPORT_*.md" + - "**/bin/**" + - "**/obj/**" + - package-lock.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2f09872 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# Set default behavior: normalize line endings to LF on checkout for all text files. +# This ensures consistent SHA256 fingerprints for reviewmark across all platforms. +* text=auto eol=lf + +# Windows batch files require CRLF line endings to function correctly. +*.bat text eol=crlf +*.cmd text eol=crlf diff --git a/.github/agents/code-quality-agent.md b/.github/agents/code-quality-agent.md deleted file mode 100644 index 6f71c74..0000000 --- a/.github/agents/code-quality-agent.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -name: Code Quality Agent -description: Ensures code quality through linting and static analysis - responsible for security, maintainability, and correctness ---- - -# Code Quality Agent - Template DotNet Tool - -Enforce quality standards through linting, static analysis, and security scanning. - -## When to Invoke This Agent - -Invoke the code-quality-agent for: - -- Running and fixing linting issues (markdown, YAML, spell check, code formatting) -- Ensuring static analysis passes with zero warnings -- Verifying code security -- Enforcing quality gates before merging -- Validating the project does what it claims to do - -## Responsibilities - -### Primary Responsibility - -Ensure the project is: - -- **Secure**: No security vulnerabilities -- **Maintainable**: Clean, well-formatted, documented code -- **Correct**: Does what it claims to do (requirements met) - -### Quality Gates (ALL Must Pass) - -1. **Build**: Zero warnings (TreatWarningsAsErrors=true) -2. **Linting**: - - markdownlint (`.markdownlint-cli2.jsonc`) - - cspell (`.cspell.json`) - - yamllint (`.yamllint.yaml`) - - dotnet format (`.editorconfig`) -3. **Static Analysis**: - - Microsoft.CodeAnalysis.NetAnalyzers - - SonarAnalyzer.CSharp -4. **Requirements Traceability**: - - `dotnet reqstream --requirements requirements.yaml --tests "test-results/**/*.trx" --enforce` -5. **Tests**: All validation tests passing - -### Template DotNet Tool-Specific - -- **XML Docs**: Enforce on ALL members (public/internal/private) -- **Code Style**: Verify `.editorconfig` compliance -- **Test Naming**: Check `TemplateTool_*` pattern for self-validation tests - -### Commands to Run - -```bash -# Code formatting -dotnet format --verify-no-changes - -# Build with zero warnings -dotnet build --configuration Release - -# Run self-validation tests -dotnet run --project src/DemaConsulting.TemplateDotNetTool \ - --configuration Release --framework net10.0 --no-build -- --validate - -# Requirements enforcement -dotnet reqstream --requirements requirements.yaml \ - --tests "test-results/**/*.trx" --enforce - -# Run all linters -./lint.sh # Linux/macOS -lint.bat # Windows -``` - -## Defer To - -- **Requirements Agent**: For requirements quality and test linkage strategy -- **Technical Writer Agent**: For fixing documentation content -- **Software Developer Agent**: For fixing production code issues -- **Test Developer Agent**: For fixing test code issues - -## Don't - -- Disable quality checks to make builds pass -- Ignore security warnings -- Skip enforcement of requirements traceability -- Change functional code without consulting appropriate developer agent diff --git a/.github/agents/code-quality.agent.md b/.github/agents/code-quality.agent.md new file mode 100644 index 0000000..4c15c87 --- /dev/null +++ b/.github/agents/code-quality.agent.md @@ -0,0 +1,216 @@ +--- +name: code-quality +description: Ensures code quality through comprehensive linting and static analysis. +tools: [read, search, edit, execute, github, agent] +user-invocable: true +--- + +# Code Quality Agent + +Enforce comprehensive quality standards through linting, static analysis, +security scanning, and Continuous Compliance gate verification. + +## Reporting + +If detailed documentation of code quality analysis is needed, create a report using the +filename pattern `AGENT_REPORT_quality_analysis.md` to document quality metrics, +identified patterns, and improvement recommendations. + +## When to Invoke This Agent + +Use the Code Quality Agent for: + +- Enforcing all quality gates before merge/release +- Running and resolving linting issues across all file types +- Ensuring static analysis passes with zero blockers +- Verifying security scanning results and addressing vulnerabilities +- Validating Continuous Compliance requirements +- Maintaining lint scripts and linting tool infrastructure +- Troubleshooting quality gate failures in CI/CD + +## Primary Responsibilities + +**Quality Enforcement Context**: Code quality is enforced through CI pipelines +and automated workflows. Your role is to analyze, validate, and ensure quality +standards are met using existing tools and infrastructure, not to create new +enforcement mechanisms or helper scripts. + +### Comprehensive Quality Gate Enforcement + +The project MUST be: + +- **Secure**: Zero security vulnerabilities (CodeQL, SonarQube) +- **Maintainable**: Clean, formatted, documented code with zero warnings +- **Compliant**: Requirements traceability enforced, file reviews current +- **Correct**: Does what requirements specify with passing tests + +### Universal Quality Gates (ALL Must Pass) + +#### 1. Linting Standards (Zero Tolerance) + +**Primary Interface**: Use the comprehensive linting scripts for all routine checks: + +```bash +# Run comprehensive linting suite +./lint.sh # Unix/Linux/macOS +# or +lint.bat # Windows +``` + +**Note**: The @code-quality agent is responsible for maintaining the `lint.sh`/`lint.bat` scripts. + +#### 2. Build Quality (Zero Warnings) + +All builds must be configured to treat warnings as errors. +This ensures that compiler warnings are addressed immediately rather than accumulating as technical debt. + +#### 3. Static Analysis (Zero Blockers) + +- **SonarQube/SonarCloud**: Code quality and security analysis +- **CodeQL**: Security vulnerability scanning (SARIF output) +- **Language Analyzers**: Microsoft.CodeAnalysis.NetAnalyzers, SonarAnalyzer.CSharp +- **Custom Rules**: Project-specific quality rules + +#### 4. Continuous Compliance Verification + +```bash +# Requirements traceability enforcement +dotnet reqstream \ + --requirements requirements.yaml \ + --tests "test-results/**/*.trx" \ + --enforce + +# File review status enforcement (uses .reviewmark.yaml) +dotnet reviewmark --enforce +``` + +#### 5. Test Quality & Coverage + +- All tests must pass (zero failures) +- Requirements coverage enforced (no uncovered requirements) +- Test result artifacts properly generated (TRX, JUnit XML) + +## Comprehensive Tool Configuration + +**The @code-quality agent is responsible for maintaining the repository's linting +infrastructure, specifically the `lint.sh`/`lint.bat` scripts.** + +### Lint Script Maintenance + +When updating tool versions or maintaining linting infrastructure, +modify the lint scripts: + +- **`lint.sh`** - Unix/Linux/macOS comprehensive linting script +- **`lint.bat`** - Windows comprehensive linting script + +**IMPORTANT**: Modifications should be limited to tool version updates, +path corrections, or infrastructure improvements. Do not modify enforcement +standards, rule configurations, or quality thresholds as these define +compliance requirements. + +These scripts automatically handle: + +- Node.js tool installation (markdownlint-cli2, cspell) +- Python virtual environment setup and yamllint installation +- Tool execution with proper error handling and reporting + +### Static Analysis Integration + +#### SonarQube Quality Profile + +- **Reliability**: A rating (zero bugs) +- **Security**: A rating (zero vulnerabilities) +- **Maintainability**: A rating (zero code smells for new code) +- **Coverage**: Minimum threshold (typically 80%+ for new code) +- **Duplication**: Maximum threshold (typically <3% for new code) + +#### CodeQL Security Scanning + +- **Schedule**: On every push and pull request +- **Language Coverage**: All supported languages in repository +- **SARIF Output**: Integration with GitHub Security tab +- **Blocking**: Pipeline fails on HIGH/CRITICAL findings + +## Quality Gate Execution Workflow + +### 1. Pre-Merge Quality Gates + +```bash +# Run comprehensive linting suite +./lint.sh # Unix/Linux/macOS +# or +lint.bat # Windows + +# Build with warnings as errors +dotnet build --configuration Release --no-restore /p:TreatWarningsAsErrors=true + +# Run static analysis +dotnet sonarscanner begin /k:"project-key" +dotnet build +dotnet test --collect:"XPlat Code Coverage" +dotnet sonarscanner end + +# Verify requirements compliance +dotnet reqstream --requirements requirements.yaml --tests "**/*.trx" --enforce +``` + +### 2. Security Gate Validation + +```bash +# CodeQL analysis (automated in GitHub Actions) +codeql database create --language=csharp +codeql database analyze --format=sarif-latest --output=results.sarif + +# Dependency vulnerability scanning +dotnet list package --vulnerable --include-transitive +npm audit --audit-level=moderate # if Node.js dependencies +``` + +### 3. Documentation & Compliance Gates + +```bash +# File review status validation +dotnet reviewmark --definition .reviewmark.yaml --enforce + +# Generate compliance documentation +dotnet buildmark --tools tools.yaml --output docs/build_notes.md +dotnet reqstream --report docs/requirements_doc/requirements.md --justifications docs/requirements_doc/justifications.md +``` + +## Cross-Agent Coordination + +### Hand-off to Other Agents + +- If code quality issues need to be fixed, then call the @software-developer agent with the **request** to fix code + quality, security, or linting issues with **context** of specific quality gate failures and + **additional instructions** to maintain coding standards. +- If test coverage needs improvement or tests are failing, then call the @test-developer agent with the **request** + to improve test coverage or fix failing tests with **context** of current coverage metrics and failing test details. +- If documentation linting fails or documentation is missing, then call the @technical-writer agent with the + **request** to fix documentation linting or generate missing docs with **context** of specific linting failures and + documentation gaps. +- If requirements traceability fails, then call the @requirements agent with the **request** to address requirements + traceability failures with **context** of enforcement errors and missing test linkages. + +## Compliance Verification Checklist + +### Before Approving Any Changes + +1. **Linting**: All linting tools pass (markdownlint, cspell, yamllint, language linters) +2. **Build**: Zero warnings, zero errors in all configurations +3. **Static Analysis**: SonarQube quality gate GREEN, CodeQL no HIGH/CRITICAL findings +4. **Requirements**: ReqStream enforcement passes, all requirements covered +5. **Tests**: All tests pass, adequate coverage maintained +6. **Documentation**: All generated docs current, spell-check passes +7. **Security**: No vulnerability findings in dependencies or code +8. **File Reviews**: All reviewable files have current reviews (if applicable) + +## Don't Do These Things + +- **Never disable quality checks** to make builds pass (fix the underlying issue) +- **Never ignore security warnings** without documented risk acceptance +- **Never skip requirements enforcement** for "quick fixes" +- **Never modify functional code** without appropriate developer agent involvement +- **Never lower quality thresholds** without compliance team approval +- **Never commit with linting failures** (CI should block this) +- **Never bypass static analysis** findings without documented justification diff --git a/.github/agents/code-review.agent.md b/.github/agents/code-review.agent.md new file mode 100644 index 0000000..fb01a20 --- /dev/null +++ b/.github/agents/code-review.agent.md @@ -0,0 +1,46 @@ +--- +name: code-review +description: Assists in performing formal file reviews. +tools: [read, search, edit, execute, github, web, agent] +user-invocable: true +--- + +# Code Review Agent + +Execute comprehensive code reviews with emphasis on structured compliance verification and file review status +requirements. + +## Reporting + +Create a report using the filename pattern `AGENT_REPORT_code_review_[review-set].md` +(e.g., `AGENT_REPORT_code_review_auth-module.md`) to document review criteria, identified issues, and recommendations +for the specific review-set. + +## Review Steps + +1. Download the + + to get the checklist to fill in +2. Use `dotnet reviewmark --elaborate [review-set]` to get the files to review +3. Review the files all together +4. Populate the checklist with the findings to make the report + +## Hand-off to Other Agents + +Only attempt to apply review fixes if requested. + +- If code quality, logic, or structural issues need fixing, call the @software-developer agent +- If test coverage gaps or quality issues are identified, call the @test-developer agent +- If documentation accuracy or completeness issues are found, call the @technical-writer agent +- If quality gate verification is needed after fixes, call the @code-quality agent +- If requirements traceability issues are discovered, call the @requirements agent + +## Don't Do These Things + +- **Never modify code during review** (document findings only, delegate fixes) +- **Never skip applicable checklist items** (comprehensive review required) +- **Never approve reviews with unresolved critical findings** +- **Never bypass review status requirements** for compliance +- **Never conduct reviews without proper documentation** +- **Never ignore security or compliance findings** +- **Never approve without verifying all quality gates** diff --git a/.github/agents/repo-consistency-agent.md b/.github/agents/repo-consistency-agent.md deleted file mode 100644 index 3f289af..0000000 --- a/.github/agents/repo-consistency-agent.md +++ /dev/null @@ -1,150 +0,0 @@ ---- -name: Repo Consistency Agent -description: Ensures downstream repositories remain consistent with the TemplateDotNetTool template patterns and best practices ---- - -# Repo Consistency Agent - Template DotNet Tool - -Maintain consistency between downstream projects and the TemplateDotNetTool template at . - -## When to Invoke This Agent - -Invoke the repo-consistency-agent for: - -- Periodic reviews of downstream repositories based on this template -- Checking if downstream projects follow the latest template patterns -- Identifying drift from template standards -- Recommending updates to bring projects back in sync with template - -**Note**: This agent should NOT be invoked for the TemplateDotNetTool repository itself (), -as that would try to ensure the repository is consistent with itself (implicitly a no-op). - -## Responsibilities - -### Consistency Checks - -The agent reviews the following areas for consistency with the template: - -#### GitHub Configuration - -- **Issue Templates**: `.github/ISSUE_TEMPLATE/` files (bug_report.yml, feature_request.yml, config.yml) -- **Pull Request Template**: `.github/pull_request_template.md` -- **Workflow Patterns**: General structure of `.github/workflows/` (build.yaml, build_on_push.yaml, release.yaml) - - Note: Some projects may need workflow deviations for specific requirements - -#### Agent Configuration - -- **Agent Definitions**: `.github/agents/` directory structure -- **Agent Documentation**: `AGENTS.md` file listing available agents - -#### Code Structure and Patterns - -- **Context Parsing**: `Context.cs` pattern for command-line argument handling -- **Self-Validation**: `Validation.cs` pattern for built-in tests -- **Program Entry**: `Program.cs` pattern with version/help/validation routing -- **Standard Arguments**: Support for `-v`, `--version`, `-?`, `-h`, `--help`, `--silent`, `--validate`, `--results`, `--log` - -#### Documentation - -- **README Structure**: Follows template README.md pattern (badges, features, installation, - usage, structure, CI/CD, documentation, license) -- **Standard Files**: Presence and structure of: - - `CONTRIBUTING.md` - - `CODE_OF_CONDUCT.md` - - `SECURITY.md` - - `LICENSE` - -#### Quality Configuration - -- **Linting Rules**: `.cspell.json`, `.markdownlint-cli2.jsonc`, `.yamllint.yaml` - - Note: Spelling exceptions will be repository-specific -- **Editor Config**: `.editorconfig` settings (file-scoped namespaces, 4-space indent, UTF-8+BOM, LF endings) -- **Code Style**: C# code style rules and analyzer configuration - -#### Project Configuration - -- **csproj Sections**: Key sections in .csproj files: - - NuGet Tool Package Configuration - - Symbol Package Configuration - - Code Quality Configuration (TreatWarningsAsErrors, GenerateDocumentationFile, etc.) - - SBOM Configuration - - Common package references (DemaConsulting.TestResults, Microsoft.SourceLink.GitHub, analyzers) - -#### Documentation Generation - -- **Document Structure**: `docs/` directory with: - - `guide/` (user guide) - - `requirements/` (auto-generated) - - `justifications/` (auto-generated) - - `tracematrix/` (auto-generated) - - `buildnotes/` (auto-generated) - - `quality/` (auto-generated) -- **Definition Files**: `definition.yaml` files for document generation - -### Tracking Template Evolution - -To ensure downstream projects benefit from recent template improvements, review recent pull requests -merged into the template repository: - -1. **List Recent PRs**: Retrieve recently merged PRs from `demaconsulting/TemplateDotNetTool` - - Review the last 10-20 PRs to identify template improvements - -2. **Identify Propagatable Changes**: For each PR, determine if changes should apply to downstream - projects: - - Focus on structural changes (workflows, agents, configurations) over content-specific changes - - Note changes to `.github/`, linting configurations, project patterns, and documentation - structure - -3. **Check Downstream Application**: Verify if identified changes exist in the downstream project: - - Check if similar files/patterns exist in downstream - - Compare file contents between template and downstream project - - Look for similar PR titles or commit messages in downstream repository history - -4. **Recommend Missing Updates**: For changes not yet applied, include them in the consistency - review with: - - Description of the template change (reference PR number) - - Explanation of benefits for the downstream project - - Specific files or patterns that need updating - -This technique ensures downstream projects don't miss important template improvements and helps -maintain long-term consistency. - -### Review Process - -1. **Identify Differences**: Compare downstream repository structure with template -2. **Assess Impact**: Determine if differences are intentional variations or drift -3. **Recommend Updates**: Suggest specific files or patterns that should be updated -4. **Respect Customizations**: Recognize valid project-specific customizations - -### What NOT to Flag - -- Project-specific naming (tool names, package IDs, repository URLs) -- Project-specific spell check exceptions in `.cspell.json` -- Workflow variations for specific project needs -- Additional requirements or features beyond the template -- Project-specific dependencies - -## Defer To - -- **Software Developer Agent**: For implementing code changes recommended by consistency check -- **Technical Writer Agent**: For updating documentation to match template -- **Requirements Agent**: For updating requirements.yaml -- **Test Developer Agent**: For updating test patterns -- **Code Quality Agent**: For applying linting and code style changes - -## Usage Pattern - -This agent is typically invoked on downstream repositories (not on TemplateDotNetTool itself): - -1. Clone or access the downstream repository -2. Invoke repo-consistency-agent to review consistency with the TemplateDotNetTool template () -3. Review agent recommendations -4. Apply relevant changes using appropriate specialized agents -5. Test changes to ensure they don't break existing functionality - -## Key Principles - -- **Template Evolution**: As the template evolves, this agent helps downstream projects stay current -- **Respect Customization**: Not all differences are problems - some are valid customizations -- **Incremental Adoption**: Downstream projects can adopt template changes incrementally -- **Documentation**: When recommending changes, explain why they align with best practices diff --git a/.github/agents/repo-consistency.agent.md b/.github/agents/repo-consistency.agent.md new file mode 100644 index 0000000..8591e2f --- /dev/null +++ b/.github/agents/repo-consistency.agent.md @@ -0,0 +1,39 @@ +--- +name: repo-consistency +description: Ensures downstream repositories remain consistent with the TemplateDotNetTool template patterns and best practices. +tools: [read, search, edit, execute, github, agent] +user-invocable: true +--- + +# Repo Consistency Agent + +Maintain consistency between downstream projects and the TemplateDotNetTool template, ensuring repositories +benefit from template evolution while respecting project-specific customizations. + +## Reporting + +If detailed documentation of consistency analysis is needed, create a report using the filename pattern +`AGENT_REPORT_consistency_[repo_name].md` (e.g., `AGENT_REPORT_consistency_MyTool.md`) to document +consistency gaps, template evolution updates, and recommended changes for the specific repository. + +## Consistency Steps + +1. Fetch the 20 most recently merged PRs (`is:pr is:merged sort:updated-desc`) from +2. Determine the intent of the template pull requests (what changes were performed to which files) +3. Apply missing changes to this repository's files (if appropriate and with translation) + +## Don't Do These Things + +- **Never recommend changes without understanding project context** (some differences are intentional) +- **Never flag valid project-specific customizations** as consistency problems +- **Never apply template changes blindly** without assessing downstream project impact +- **Never ignore template evolution benefits** when they clearly improve downstream projects +- **Never recommend breaking changes** without migration guidance and impact assessment +- **Never skip validation** of preserved functionality after template alignment +- **Never assume all template patterns apply universally** (assess project-specific needs) + +## Key Principles + +- **Evolutionary Consistency**: Template improvements should enhance downstream projects systematically +- **Intelligent Customization Respect**: Distinguished valid customizations from unintentional drift +- **Incremental Template Adoption**: Support phased adoption of template improvements based on project capacity diff --git a/.github/agents/requirements-agent.md b/.github/agents/requirements.agent.md similarity index 96% rename from .github/agents/requirements-agent.md rename to .github/agents/requirements.agent.md index 4f56242..bfb9294 100644 --- a/.github/agents/requirements-agent.md +++ b/.github/agents/requirements.agent.md @@ -1,6 +1,8 @@ --- -name: Requirements Agent -description: Develops requirements and ensures appropriate test coverage - knows which requirements need unit/integration/self-validation tests +name: requirements +description: Develops requirements and ensures appropriate test coverage. +tools: [read, search, edit, execute, github, web, agent] +user-invocable: true --- # Requirements Agent - Template DotNet Tool diff --git a/.github/agents/software-developer.md b/.github/agents/software-developer.agent.md similarity index 94% rename from .github/agents/software-developer.md rename to .github/agents/software-developer.agent.md index 91aa379..efa5758 100644 --- a/.github/agents/software-developer.md +++ b/.github/agents/software-developer.agent.md @@ -1,6 +1,8 @@ --- -name: Software Developer -description: Writes production code and self-validation tests - targets design-for-testability and literate programming style +name: software-developer +description: Writes production code and self-validation tests. +tools: [read, search, edit, execute, github, agent] +user-invocable: true --- # Software Developer - Template DotNet Tool diff --git a/.github/agents/technical-writer.md b/.github/agents/technical-writer.agent.md similarity index 90% rename from .github/agents/technical-writer.md rename to .github/agents/technical-writer.agent.md index 1f62e0c..b300631 100644 --- a/.github/agents/technical-writer.md +++ b/.github/agents/technical-writer.agent.md @@ -1,6 +1,8 @@ --- -name: Technical Writer -description: Ensures documentation is accurate and complete - knowledgeable about regulatory documentation and special document types +name: technical-writer +description: Ensures documentation is accurate and complete. +tools: [read, search, edit, execute, github, agent] +user-invocable: true --- # Technical Writer - Template DotNet Tool @@ -41,7 +43,7 @@ Invoke the technical-writer for: #### Linting Requirements - **markdownlint**: Style and structure compliance -- **cspell**: Spelling (add technical terms to `.cspell.json`) +- **cspell**: Spelling (add technical terms to `.cspell.yaml`) - **yamllint**: YAML file validation ### Regulatory Documentation diff --git a/.github/agents/test-developer.md b/.github/agents/test-developer.agent.md similarity index 97% rename from .github/agents/test-developer.md rename to .github/agents/test-developer.agent.md index ae33179..2ce95d9 100644 --- a/.github/agents/test-developer.md +++ b/.github/agents/test-developer.agent.md @@ -1,6 +1,8 @@ --- -name: Test Developer -description: Writes unit and integration tests following AAA pattern - clear documentation of what's tested and proved +name: test-developer +description: Writes unit and integration tests. +tools: [read, search, edit, execute, github, agent] +user-invocable: true --- # Test Developer - Template DotNet Tool diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4f26964..d7893db 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -34,6 +34,16 @@ jobs: run: > dotnet tool restore + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: 24.x + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.14' + - name: Capture tool versions shell: bash run: | @@ -55,21 +65,9 @@ jobs: # This section runs all quality checks for the project. # Downstream projects: Add any additional quality checks here. - - name: Run markdown linter - uses: DavidAnson/markdownlint-cli2-action@v22 - with: - globs: '**/*.md' - - - name: Run spell checker - uses: streetsidesoftware/cspell-action@v8 - with: - files: '**/*.{md,cs}' - incremental_files_only: false - - - name: Run YAML linter - uses: ibiqlik/action-yamllint@v3 - with: - config_file: .yamllint.yaml + - name: Run linters + shell: bash + run: bash ./lint.sh - name: Upload quality artifacts uses: actions/upload-artifact@v7 diff --git a/.gitignore b/.gitignore index 467dfd7..ec91165 100644 --- a/.gitignore +++ b/.gitignore @@ -85,6 +85,7 @@ npm-debug.log __pycache__/ *.py[cod] *$py.class +.venv/ # Generated documentation docs/**/*.html diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc deleted file mode 100644 index a46ee1a..0000000 --- a/.markdownlint-cli2.jsonc +++ /dev/null @@ -1,15 +0,0 @@ -{ - "config": { - "default": true, - "MD003": { "style": "atx" }, - "MD007": { "indent": 2 }, - "MD013": { "line_length": 120 }, - "MD025": false, - "MD033": false, - "MD041": false - }, - "ignores": [ - "node_modules", - "**/AGENT_REPORT_*.md" - ] -} diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml new file mode 100644 index 0000000..04f1f80 --- /dev/null +++ b/.markdownlint-cli2.yaml @@ -0,0 +1,47 @@ +--- +# Markdown Linting Standards +# +# PURPOSE: +# - Maintain professional technical documentation standards +# - Ensure consistent formatting for readability and maintenance +# - Support automated documentation generation and publishing +# +# DO NOT MODIFY: These rules represent coding standards +# - If files fail linting, fix the files to meet these standards +# - Do not relax rules to accommodate existing non-compliant files +# - Consistency across repositories is critical for documentation quality + +config: + # Enable all default rules + default: true + + # Require ATX-style headers (# Header) instead of Setext-style + MD003: + style: atx + + # Set consistent indentation for nested lists + MD007: + indent: 2 + + # Allow longer lines for URLs and technical content + MD013: + line_length: 120 + + # Allow multiple top-level headers per document + MD025: false + + # Allow inline HTML for enhanced documentation + MD033: false + + # Allow documents without top-level header (for fragments) + MD041: false + +# Exclude common build artifacts, dependencies, and vendored third-party code +ignores: + - "**/.git/**" + - "**/node_modules/**" + - "**/.venv/**" + - "**/thirdparty/**" + - "**/third-party/**" + - "**/3rd-party/**" + - "**/AGENT_REPORT_*.md" diff --git a/.yamllint.yaml b/.yamllint.yaml index e269fb0..947ca60 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -4,6 +4,15 @@ extends: default +# Exclude common build artifacts, dependencies, and vendored third-party code +ignore: | + .git/ + node_modules/ + .venv/ + thirdparty/ + third-party/ + 3rd-party/ + rules: # Allow 'on:' in GitHub Actions workflows (not a boolean value) truthy: diff --git a/AGENTS.md b/AGENTS.md index 219fd21..7e4fcbe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,7 @@ file-review evidence management in regulated environments. - **Test Developer** - Creates unit and integration tests following AAA pattern - **Code Quality Agent** - Enforces linting, static analysis, and security standards - **Repo Consistency Agent** - Ensures downstream repositories remain consistent with template patterns +- **Code Review Agent** - Assists in performing formal file reviews ## Agent Selection Guide @@ -23,6 +24,7 @@ file-review evidence management in regulated environments. - Ensure test coverage linkage in `requirements.yaml` → **Requirements Agent** - Run security scanning or address CodeQL alerts → **Code Quality Agent** - Propagate template changes → **Repo Consistency Agent** +- Perform file reviews → **Code Review Agent** ## Tech Stack @@ -32,7 +34,17 @@ file-review evidence management in regulated environments. - **`requirements.yaml`** - All requirements with test linkage (enforced via `dotnet reqstream --enforce`) - **`.editorconfig`** - Code style (file-scoped namespaces, 4-space indent, UTF-8, LF endings) -- **`.cspell.json`, `.markdownlint-cli2.jsonc`, `.yamllint.yaml`** - Linting configs +- **`.cspell.yaml`, `.markdownlint-cli2.yaml`, `.yamllint.yaml`** - Linting configs + +### Spell check word list policy + +**Never** add a word to the `.cspell.yaml` word list in order to silence a spell-checking failure. +Doing so defeats the purpose of spell-checking and reduces the quality of the repository. + +- If cspell flags a word that is **misspelled**, fix the spelling in the source file. +- If cspell flags a word that is a **genuine technical term** (tool name, project identifier, etc.) and is + spelled correctly, raise a **proposal** (e.g. comment in a pull request) explaining why the word + should be added. The proposal must be reviewed and approved before the word is added to the list. ## Requirements diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e2fc908..da570c0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -187,14 +187,18 @@ All markdown files must follow these rules (enforced by markdownlint): ### Spell Checking -All files are spell-checked using cspell. Add project-specific terms to `.cspell.json`: - -```json -{ - "words": [ - "myterm" - ] -} +All files are spell-checked using cspell. **Never** add a word to the `.cspell.yaml` word list in order to silence a +spell-checking failure. Doing so defeats the purpose of spell-checking and reduces the quality of the repository. + +- If cspell flags a word that is **misspelled**, fix the spelling in the source file. +- If cspell flags a word that is a **genuine technical term** (tool name, project identifier, etc.) and is spelled + correctly, raise a **proposal** (e.g. comment in a pull request) explaining why the word should be added. The + proposal must be reviewed and approved before the word is added to the list. + +```yaml +# .cspell.yaml +words: + - myterm ``` ## Quality Checks diff --git a/lint.bat b/lint.bat index ee86ba8..f94b53d 100644 --- a/lint.bat +++ b/lint.bat @@ -1,20 +1,40 @@ @echo off -REM Run all linters for ReviewMark (Windows) +setlocal -echo Checking markdown... -call npx markdownlint-cli2 "**/*.md" -if %errorlevel% neq 0 exit /b %errorlevel% +REM Comprehensive Linting Script +REM +REM PURPOSE: +REM - Run ALL lint checks when executed (no options or modes) +REM - Output lint failures directly for agent parsing +REM - NO command-line arguments, pretty printing, or colorization +REM - Agents execute this script to identify files needing fixes + +set "LINT_ERROR=0" + +REM Install npm dependencies +call npm install -echo Checking spelling... -call npx cspell "**/*.{cs,md,json,yaml,yml}" --no-progress -if %errorlevel% neq 0 exit /b %errorlevel% +REM Create Python virtual environment (for yamllint) if missing +if not exist ".venv\Scripts\activate.bat" ( + python -m venv .venv +) +call .venv\Scripts\activate.bat +pip install -r pip-requirements.txt + +REM Run spell check +call npx cspell --no-progress --no-color "**/*.{md,yaml,yml,json,cs,cpp,hpp,h,txt}" +if errorlevel 1 set "LINT_ERROR=1" + +REM Run markdownlint check +call npx markdownlint-cli2 "**/*.md" +if errorlevel 1 set "LINT_ERROR=1" -echo Checking YAML... -call yamllint -c .yamllint.yaml . -if %errorlevel% neq 0 exit /b %errorlevel% +REM Run yamllint check +yamllint . +if errorlevel 1 set "LINT_ERROR=1" -echo Checking code formatting... +REM Run .NET formatting check (verifies no changes are needed) dotnet format --verify-no-changes -if %errorlevel% neq 0 exit /b %errorlevel% +if errorlevel 1 set "LINT_ERROR=1" -echo All linting passed! +exit /b %LINT_ERROR% diff --git a/lint.sh b/lint.sh index efe7bd4..7d8116b 100755 --- a/lint.sh +++ b/lint.sh @@ -1,18 +1,35 @@ -#!/usr/bin/env bash -# Run all linters for ReviewMark +#!/bin/bash -set -e # Exit on error +# Comprehensive Linting Script +# +# PURPOSE: +# - Run ALL lint checks when executed (no options or modes) +# - Output lint failures directly for agent parsing +# - NO command-line arguments, pretty printing, or colorization +# - Agents execute this script to identify files needing fixes -echo "📝 Checking markdown..." -npx markdownlint-cli2 "**/*.md" +lint_error=0 -echo "🔤 Checking spelling..." -npx cspell "**/*.{cs,md,json,yaml,yml}" --no-progress +# Install npm dependencies +npm install -echo "📋 Checking YAML..." -yamllint -c .yamllint.yaml . +# Create Python virtual environment (for yamllint) +if [ ! -d ".venv" ]; then + python -m venv .venv +fi +source .venv/bin/activate +pip install -r pip-requirements.txt -echo "🎨 Checking code formatting..." -dotnet format --verify-no-changes +# Run spell check +npx cspell --no-progress --no-color "**/*.{md,yaml,yml,json,cs,cpp,hpp,h,txt}" || lint_error=1 -echo "✨ All linting passed!" +# Run markdownlint check +npx markdownlint-cli2 "**/*.md" || lint_error=1 + +# Run yamllint check +yamllint . || lint_error=1 + +# Run .NET formatting check (verifies no changes are needed) +dotnet format --verify-no-changes || lint_error=1 + +exit $lint_error diff --git a/package.json b/package.json index 9487fa5..a57dc1f 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,8 @@ "private": true, "devDependencies": { "@mermaid-js/mermaid-cli": "11.12.0", + "cspell": "9.7.0", + "markdownlint-cli2": "0.21.0", "mermaid-filter": "1.4.7" } } diff --git a/pip-requirements.txt b/pip-requirements.txt new file mode 100644 index 0000000..7ce0eab --- /dev/null +++ b/pip-requirements.txt @@ -0,0 +1 @@ +yamllint==1.38.0 From 4beb01d3043d0ccd3a89ce425f75a4d08feffc0c Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 13:07:39 -0400 Subject: [PATCH 04/19] feat: Add --lint support for validating definition files (#27) * Initial plan * feat: Add --lint support for validating definition files Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/a75bab3c-44b6-4dcf-8da7-eb85c41f6ec3 * refactor: address review feedback - TestDirectory helper, error message assertions, corrupted YAML tests Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/245f177b-039d-4058-a8eb-3fce0818cd47 * feat: improve lint error quality - filename/line numbers, includes support, and stronger test assertions Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/7318deb4-3700-47ba-b4b8-b7c3e37c4b4e * revert: remove includes feature - reviewmark uses a single definition file Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/25f4d0e6-fdf3-493b-803c-d19cfb0967fd * refactor: accumulate all lint errors in one pass via ReviewMarkConfiguration.Lint Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/0745bda8-94ca-4460-a7b2-c361d77474c0 * fix: clarify case-sensitivity intent and tighten Lint unit test count assertion Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/0745bda8-94ca-4460-a7b2-c361d77474c0 * fix: dispose Context before reading log file, consistent error messages, TestDirectory safety, shared type validator Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/206afe15-d999-4902-a3d6-381cc23061cb * docs: add --lint to README and user guide Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/a5fb2a08-3a4d-4a2e-908c-d95bbe84dae0 * fix: correct British spelling 'recognised' to American 'recognized' Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/2712fd02-b8b6-44bc-99bf-2a973437ffbb * feat: change lint output to [location]: [severity]: [issue] format Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/3576c6ae-5203-472e-b193-49f8d6e46feb --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --- README.md | 14 +- docs/guide/guide.md | 84 +++- requirements.yaml | 28 ++ src/DemaConsulting.ReviewMark/Context.cs | 15 + src/DemaConsulting.ReviewMark/Program.cs | 34 +- .../ReviewMarkConfiguration.cs | 348 ++++++++++--- src/DemaConsulting.ReviewMark/Validation.cs | 33 ++ .../ContextTests.cs | 29 ++ .../ProgramTests.cs | 457 ++++++++++++++---- .../ReviewMarkConfigurationTests.cs | 76 +++ .../TestDirectory.cs | 65 +++ 11 files changed, 1018 insertions(+), 165 deletions(-) create mode 100644 test/DemaConsulting.ReviewMark.Tests/TestDirectory.cs diff --git a/README.md b/README.md index bae4b91..75504e9 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ DEMA Consulting tool for automated file-review evidence management in regulated - 📋 **Coverage Reporting** - Review plan shows which files are covered and flags uncovered files - 📊 **Status Reporting** - Review report shows whether each review-set is Current, Stale, Missing, or Failed - 🔍 **Review Elaboration** - `--elaborate` prints the ID, fingerprint, and file list for a review set +- 🔎 **Configuration Linting** - `--lint` validates the definition file and reports all structural and semantic issues - 🚦 **Enforcement** - `--enforce` exits non-zero if any review-set is stale or missing, or any file is uncovered - 🔄 **Re-indexing** - `--index` scans PDF evidence files and writes an up-to-date `index.json` - ✅ **Self-Validation** - Built-in validation tests with TRX and JUnit output @@ -100,6 +101,12 @@ reviewmark --validate # Save validation results reviewmark --validate --results results.trx +# Validate definition file +reviewmark --lint + +# Validate a specific definition file +reviewmark --lint --definition path/to/definition.yaml + # Silent mode with logging reviewmark --silent --log output.log ``` @@ -112,6 +119,7 @@ reviewmark --silent --log output.log | `-?`, `-h`, `--help` | Display help message | | `--silent` | Suppress console output | | `--validate` | Run self-validation | +| `--lint` | Validate the definition file and report issues | | `--results ` | Write validation results to file (TRX or JUnit format) | | `--log ` | Write output to log file | | `--definition ` | Specify the definition YAML file (default: .reviewmark.yaml) | @@ -147,9 +155,10 @@ Running self-validation produces a report containing the following information: ✓ ReviewMark_WorkingDirectoryOverride - Passed ✓ ReviewMark_Enforce - Passed ✓ ReviewMark_Elaborate - Passed +✓ ReviewMark_Lint - Passed -Total Tests: 8 -Passed: 8 +Total Tests: 9 +Passed: 9 Failed: 0 ``` @@ -163,6 +172,7 @@ Each test in the report proves: - **`ReviewMark_WorkingDirectoryOverride`** - `--dir` overrides the working directory for file operations. - **`ReviewMark_Enforce`** - `--enforce` exits with non-zero code when reviews have issues. - **`ReviewMark_Elaborate`** - `--elaborate` prints a Markdown elaboration of a review set. +- **`ReviewMark_Lint`** - `--lint` validates a definition file and reports issues. See the [User Guide][link-guide] for more details on the self-validation tests. diff --git a/docs/guide/guide.md b/docs/guide/guide.md index 161db14..63dfd2c 100644 --- a/docs/guide/guide.md +++ b/docs/guide/guide.md @@ -103,9 +103,10 @@ Example validation report: ✓ ReviewMark_WorkingDirectoryOverride - Passed ✓ ReviewMark_Enforce - Passed ✓ ReviewMark_Elaborate - Passed +✓ ReviewMark_Lint - Passed -Total Tests: 8 -Passed: 8 +Total Tests: 9 +Passed: 9 Failed: 0 ``` @@ -121,6 +122,59 @@ Each test proves specific functionality works correctly: - **`ReviewMark_WorkingDirectoryOverride`** - `--dir` overrides the working directory for file operations. - **`ReviewMark_Enforce`** - `--enforce` exits with non-zero code when reviews have issues. - **`ReviewMark_Elaborate`** - `--elaborate` prints a Markdown elaboration of a review set. +- **`ReviewMark_Lint`** - `--lint` validates a definition file and reports issues. + +## Lint Definition File + +The `--lint` command validates the definition file (`.reviewmark.yaml`) and reports all +structural and semantic issues in a single pass. Unlike running the full tool, `--lint` never +queries the evidence store — it only checks the definition file itself. + +A successful lint exits with code 0; any issue causes a non-zero exit code. + +### Running Lint + +Lint the default definition file (`.reviewmark.yaml` in the working directory): + +```bash +reviewmark --lint +``` + +Lint a specific definition file: + +```bash +reviewmark --lint --definition path/to/definition.yaml +``` + +### What Lint Checks + +Lint checks the following: + +- **File readability** — the definition file exists and can be read. +- **YAML syntax** — the file is valid YAML; syntax errors include the filename and line number. +- **`evidence-source` block** — the block is present, has a `type` field (`url` or `fileshare`), + and has a `location` field. +- **Review sets** — each set has an `id`, a `title`, and at least one `paths` entry. +- **Duplicate IDs** — no two review sets share the same `id`. + +All detected issues are reported together so you can fix multiple problems in one pass. + +### Lint Error Messages + +Lint errors follow the standard `[location]: [severity]: [issue]` format. For YAML syntax +errors the location includes the line and column number: + +```text +definition.yaml:3:5: error: (yaml parse details) +definition.yaml: error: Configuration is missing required 'evidence-source' block. +definition.yaml: error: reviews[1] has duplicate ID 'core-module' (first defined at reviews[0]). +``` + +When no issues are found: + +```text +definition.yaml: No issues found +``` ## Silent Mode @@ -163,6 +217,7 @@ The following command-line options are supported: | `-?`, `-h`, `--help` | Display help message | | `--silent` | Suppress console output | | `--validate` | Run self-validation | +| `--lint` | Validate the definition file and report issues | | `--results ` | Write validation results to file (TRX or JUnit format) | | `--log ` | Write output to log file | | `--definition ` | Specify the definition YAML file (default: .reviewmark.yaml) | @@ -561,13 +616,34 @@ reviews: - "!src/Data/Generated/**" # exclude auto-generated entity classes ``` -## Example 4: Self-Validation with Results +## Example 4: Lint a Definition File + +Lint the default definition file (`.reviewmark.yaml`) to catch all configuration errors before +running the full tool: + +```bash +reviewmark --lint +``` + +Lint a specific definition file: + +```bash +reviewmark --lint --definition path/to/.reviewmark.yaml +``` + +With silent mode and logging (useful in CI pipelines): + +```bash +reviewmark --silent --log lint.log --lint +``` + +## Example 5: Self-Validation with Results ```bash reviewmark --validate --results validation-results.trx ``` -## Example 5: Silent Mode with Logging +## Example 6: Silent Mode with Logging ```bash reviewmark --silent --log tool-output.log diff --git a/requirements.yaml b/requirements.yaml index 2536894..06e361a 100644 --- a/requirements.yaml +++ b/requirements.yaml @@ -244,6 +244,28 @@ sections: - Program_Run_WithElaborateFlag_UnknownId_ReportsError - ReviewMark_Elaborate + - id: ReviewMark-Cmd-Lint + title: The tool shall support --lint flag to validate the definition file and report issues. + justification: | + Users need a way to verify that the .reviewmark.yaml configuration file is valid + before running the main tool, providing clear error messages about the cause and + location of any issues. + tests: + - Context_Create_LintFlag_SetsLintTrue + - Context_Create_NoArguments_LintIsFalse + - Program_Run_WithHelpFlag_IncludesLintOption + - Program_Run_WithLintFlag_ValidConfig_ReportsSuccess + - Program_Run_WithLintFlag_MissingConfig_ReportsError + - Program_Run_WithLintFlag_DuplicateIds_ReportsError + - Program_Run_WithLintFlag_UnknownSourceType_ReportsError + - Program_Run_WithLintFlag_CorruptedYaml_ReportsError + - Program_Run_WithLintFlag_MissingEvidenceSource_ReportsError + - Program_Run_WithLintFlag_MultipleErrors_ReportsAll + - ReviewMarkConfiguration_Load_InvalidYaml_ErrorIncludesFilenameAndLine + - ReviewMarkConfiguration_Load_MissingEvidenceSource_ErrorIncludesFilename + - ReviewMarkConfiguration_Lint_MultipleErrors_ReturnsAll + - ReviewMark_Lint + - title: Configuration Reading requirements: - id: ReviewMark-Config-Reading @@ -321,6 +343,7 @@ sections: - "windows@ReviewMark_Enforce" - "windows@ReviewMark_WorkingDirectoryOverride" - "windows@ReviewMark_Elaborate" + - "windows@ReviewMark_Lint" - id: ReviewMark-Platform-Linux title: The tool shall build and run on Linux platforms. @@ -336,6 +359,7 @@ sections: - "ubuntu@ReviewMark_Enforce" - "ubuntu@ReviewMark_WorkingDirectoryOverride" - "ubuntu@ReviewMark_Elaborate" + - "ubuntu@ReviewMark_Lint" - id: ReviewMark-Platform-MacOS title: The tool shall build and run on macOS platforms. @@ -351,6 +375,7 @@ sections: - "macos@ReviewMark_Enforce" - "macos@ReviewMark_WorkingDirectoryOverride" - "macos@ReviewMark_Elaborate" + - "macos@ReviewMark_Lint" - id: ReviewMark-Platform-Net8 title: The tool shall support .NET 8 runtime. @@ -365,6 +390,7 @@ sections: - "dotnet8.x@ReviewMark_Enforce" - "dotnet8.x@ReviewMark_WorkingDirectoryOverride" - "dotnet8.x@ReviewMark_Elaborate" + - "dotnet8.x@ReviewMark_Lint" - id: ReviewMark-Platform-Net9 title: The tool shall support .NET 9 runtime. @@ -379,6 +405,7 @@ sections: - "dotnet9.x@ReviewMark_Enforce" - "dotnet9.x@ReviewMark_WorkingDirectoryOverride" - "dotnet9.x@ReviewMark_Elaborate" + - "dotnet9.x@ReviewMark_Lint" - id: ReviewMark-Platform-Net10 title: The tool shall support .NET 10 runtime. @@ -393,6 +420,7 @@ sections: - "dotnet10.x@ReviewMark_Enforce" - "dotnet10.x@ReviewMark_WorkingDirectoryOverride" - "dotnet10.x@ReviewMark_Elaborate" + - "dotnet10.x@ReviewMark_Lint" - title: OTS Software requirements: diff --git a/src/DemaConsulting.ReviewMark/Context.cs b/src/DemaConsulting.ReviewMark/Context.cs index bf7fa11..e2315e7 100644 --- a/src/DemaConsulting.ReviewMark/Context.cs +++ b/src/DemaConsulting.ReviewMark/Context.cs @@ -60,6 +60,11 @@ internal sealed class Context : IDisposable /// public bool Validate { get; private init; } + /// + /// Gets a value indicating whether the lint flag was specified. + /// + public bool Lint { get; private init; } + /// /// Gets the validation results file path. /// @@ -159,6 +164,7 @@ public static Context Create(string[] args) Help = parser.Help, Silent = parser.Silent, Validate = parser.Validate, + Lint = parser.Lint, ResultsFile = parser.ResultsFile, DefinitionFile = parser.DefinitionFile, PlanFile = parser.PlanFile, @@ -226,6 +232,11 @@ private sealed class ArgumentParser /// public bool Validate { get; private set; } + /// + /// Gets a value indicating whether the lint flag was specified. + /// + public bool Lint { get; private set; } + /// /// Gets the log file path. /// @@ -328,6 +339,10 @@ private int ParseArgument(string arg, string[] args, int index) Validate = true; return index; + case "--lint": + Lint = true; + return index; + case "--log": LogFile = GetRequiredStringArgument(arg, args, index, FilenameArgument); return index + 1; diff --git a/src/DemaConsulting.ReviewMark/Program.cs b/src/DemaConsulting.ReviewMark/Program.cs index a87a942..061adf5 100644 --- a/src/DemaConsulting.ReviewMark/Program.cs +++ b/src/DemaConsulting.ReviewMark/Program.cs @@ -112,7 +112,14 @@ public static void Run(Context context) return; } - // Priority 4: Main tool functionality + // Priority 4: Lint + if (context.Lint) + { + RunLintLogic(context); + return; + } + + // Priority 5: Main tool functionality RunToolLogic(context); } @@ -140,6 +147,7 @@ private static void PrintHelp(Context context) context.WriteLine(" -?, -h, --help Display this help message"); context.WriteLine(" --silent Suppress console output"); context.WriteLine(" --validate Run self-validation"); + context.WriteLine(" --lint Lint the definition file and report issues"); context.WriteLine(" --results Write validation results to file (.trx or .xml)"); context.WriteLine(" --log Write output to log file"); context.WriteLine(" --definition Specify the definition YAML file (default: .reviewmark.yaml)"); @@ -154,6 +162,30 @@ private static void PrintHelp(Context context) context.WriteLine(" --elaborate Print a Markdown elaboration of the specified review set"); } + /// + /// Runs the lint logic to validate the definition file. + /// + /// The context containing command line arguments and program state. + private static void RunLintLogic(Context context) + { + // Determine the definition file path (explicit or default) + var directory = context.WorkingDirectory ?? Directory.GetCurrentDirectory(); + var definitionFile = context.DefinitionFile ?? PathHelpers.SafePathCombine(directory, ".reviewmark.yaml"); + + // Lint the file, collecting all detectable errors in one pass. + var errors = ReviewMarkConfiguration.Lint(definitionFile); + foreach (var error in errors) + { + context.WriteError(error); + } + + // Report overall result + if (errors.Count == 0) + { + context.WriteLine($"{definitionFile}: No issues found"); + } + } + /// /// Runs the main tool logic. /// diff --git a/src/DemaConsulting.ReviewMark/ReviewMarkConfiguration.cs b/src/DemaConsulting.ReviewMark/ReviewMarkConfiguration.cs index 6800b5a..2a6eb6b 100644 --- a/src/DemaConsulting.ReviewMark/ReviewMarkConfiguration.cs +++ b/src/DemaConsulting.ReviewMark/ReviewMarkConfiguration.cs @@ -123,6 +123,152 @@ file sealed class ReviewSetYaml public List? Paths { get; set; } } +// --------------------------------------------------------------------------- +// File-local helpers — use file-local YAML types +// --------------------------------------------------------------------------- + +/// +/// File-local static helper that encapsulates YAML deserialization and model validation +/// on behalf of . Because both this class and +/// are file-local, C# allows them to appear in the +/// method signatures here. +/// +file static class ReviewMarkConfigurationHelpers +{ + /// + /// Returns true when is a recognized evidence-source + /// type (url or fileshare, case-insensitive). + /// + /// The type string to test. + /// true if the type is supported; false otherwise. + public static bool IsSupportedEvidenceSourceType(string type) => + string.Equals(type, "url", StringComparison.OrdinalIgnoreCase) || + string.Equals(type, "fileshare", StringComparison.OrdinalIgnoreCase); + + /// + /// Deserializes a YAML string into the raw model. + /// + /// YAML content to parse. + /// + /// Optional file path used to produce actionable error messages. When null, + /// YAML errors are thrown as (preserving the + /// contract). When non-null, + /// they are thrown as and include the + /// file name, line, and column. + /// + /// The deserialized . + /// + /// Thrown when is null and the YAML is invalid. + /// + /// + /// Thrown when is set and the YAML is invalid. + /// + public static ReviewMarkYaml DeserializeRaw(string yaml, string? filePath) + { + var deserializer = new DeserializerBuilder() + .WithNamingConvention(NullNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + try + { + if (filePath != null) + { + return deserializer.Deserialize(yaml) + ?? throw new InvalidOperationException( + $"Configuration file '{filePath}' is empty or null."); + } + + return deserializer.Deserialize(yaml) + ?? throw new ArgumentException("YAML content is empty or invalid.", nameof(yaml)); + } + catch (YamlException ex) + { + if (filePath != null) + { + throw new InvalidOperationException( + $"Failed to parse '{filePath}' at line {ex.Start.Line}, column {ex.Start.Column}: {ex.Message}", + ex); + } + + throw new ArgumentException($"Invalid YAML content: {ex.Message}", nameof(yaml), ex); + } + } + + /// + /// Validates a raw model and builds a + /// from it. + /// + /// The deserialized raw model to validate. + /// A validated . + /// + /// Thrown when required fields are absent or malformed. + /// + public static ReviewMarkConfiguration BuildConfiguration(ReviewMarkYaml raw) + { + // Map needs-review patterns (default to empty list if absent) + var needsReviewPatterns = (IReadOnlyList)(raw.NeedsReview ?? []); + + // Map evidence-source (required: evidence-source block, type, and location) + if (raw.EvidenceSource is not { } es) + { + throw new ArgumentException("Configuration is missing required 'evidence-source' block."); + } + + if (string.IsNullOrWhiteSpace(es.Type)) + { + throw new ArgumentException("Configuration 'evidence-source' is missing a required 'type' field."); + } + + if (!IsSupportedEvidenceSourceType(es.Type)) + { + throw new ArgumentException( + $"Configuration 'evidence-source' type '{es.Type}' is not supported (must be 'url' or 'fileshare')."); + } + + if (string.IsNullOrWhiteSpace(es.Location)) + { + throw new ArgumentException("Configuration 'evidence-source' is missing a required 'location' field."); + } + + var evidenceSource = new EvidenceSource( + Type: es.Type, + Location: es.Location, + UsernameEnv: es.Credentials?.UsernameEnv, + PasswordEnv: es.Credentials?.PasswordEnv); + + // Map review sets, requiring id, title, and paths for each entry + var reviews = (raw.Reviews ?? []) + .Select((r, i) => + { + // Each review set must have an id + if (string.IsNullOrWhiteSpace(r.Id)) + { + throw new ArgumentException($"Review set at index {i} is missing a required 'id' field."); + } + + // Each review set must have a title + if (string.IsNullOrWhiteSpace(r.Title)) + { + throw new ArgumentException($"Review set '{r.Id}' is missing a required 'title' field."); + } + + // Each review set must have at least one non-empty path pattern + var paths = r.Paths; + if (paths is null || !paths.Any(p => !string.IsNullOrWhiteSpace(p))) + { + throw new ArgumentException( + $"Review set '{r.Id}' at index {i} is missing required 'paths' entries."); + } + + return new ReviewSet(r.Id, r.Title, paths); + }) + .ToList(); + + return new ReviewMarkConfiguration(needsReviewPatterns, evidenceSource, reviews); + } +} + // --------------------------------------------------------------------------- // Public API — internal to the assembly // --------------------------------------------------------------------------- @@ -281,7 +427,7 @@ internal sealed class ReviewMarkConfiguration /// Glob patterns for files requiring review. /// Evidence-source configuration. /// Review set definitions. - private ReviewMarkConfiguration( + internal ReviewMarkConfiguration( IReadOnlyList needsReviewPatterns, EvidenceSource evidenceSource, IReadOnlyList reviews) @@ -297,7 +443,11 @@ private ReviewMarkConfiguration( /// Absolute or relative path to the configuration file. /// A populated instance. /// Thrown when is null or empty. - /// Thrown when the file cannot be read. + /// + /// Thrown when the file cannot be read, the YAML is invalid, or required configuration fields are + /// missing. The exception message always identifies the problematic file and, for YAML syntax + /// errors, the line and column number. + /// internal static ReviewMarkConfiguration Load(string filePath) { // Validate the file path argument @@ -321,8 +471,23 @@ internal static ReviewMarkConfiguration Load(string filePath) throw new InvalidOperationException($"Failed to read configuration file '{filePath}': {ex.Message}", ex); } - // Delegate to Parse for deserialization and apply path resolution - var config = Parse(yaml); + // Deserialize the raw YAML model, embedding the file path and line number in any parse error. + var raw = ReviewMarkConfigurationHelpers.DeserializeRaw(yaml, filePath); + + // Determine the base directory for resolving relative fileshare locations. + var baseDirectory = Path.GetDirectoryName(Path.GetFullPath(filePath)) + ?? throw new InvalidOperationException($"Cannot determine base directory for configuration file '{filePath}'."); + + // Validate the raw model, embedding the file path in any semantic error. + ReviewMarkConfiguration config; + try + { + config = ReviewMarkConfigurationHelpers.BuildConfiguration(raw); + } + catch (ArgumentException ex) + { + throw new InvalidOperationException($"Invalid configuration in '{filePath}': {ex.Message}", ex); + } // Resolve relative fileshare locations against the config file's directory so that // a relative location (e.g., "index.json") works correctly regardless of the process @@ -330,8 +495,6 @@ internal static ReviewMarkConfiguration Load(string filePath) if (string.Equals(config.EvidenceSource.Type, "fileshare", StringComparison.OrdinalIgnoreCase) && !Path.IsPathRooted(config.EvidenceSource.Location)) { - var baseDirectory = Path.GetDirectoryName(Path.GetFullPath(filePath)) - ?? throw new InvalidOperationException($"Cannot determine base directory for configuration file '{filePath}'."); var absoluteLocation = Path.GetFullPath(config.EvidenceSource.Location, baseDirectory); return new ReviewMarkConfiguration( config.NeedsReviewPatterns, @@ -343,88 +506,143 @@ internal static ReviewMarkConfiguration Load(string filePath) } /// - /// Parses a YAML string into a . + /// Lints a .reviewmark.yaml file and returns all detected issues. + /// Unlike , this method does not stop at the first error; + /// it accumulates every detectable problem and returns them all so the caller + /// can report a complete list in a single pass. /// - /// The YAML content to parse. - /// A populated instance. - /// Thrown when is null. - /// Thrown when the YAML is invalid or missing required fields. - internal static ReviewMarkConfiguration Parse(string yaml) + /// Absolute or relative path to the configuration file. + /// + /// A read-only list of error messages. The list is empty when the file is + /// structurally and semantically valid. + /// + /// Thrown when is null or empty. + internal static IReadOnlyList Lint(string filePath) { - // Validate the yaml input - ArgumentNullException.ThrowIfNull(yaml); + // Validate the file path argument + if (string.IsNullOrWhiteSpace(filePath)) + { + throw new ArgumentException("File path must not be null or empty.", nameof(filePath)); + } - // Build a YamlDotNet deserializer that ignores unmatched fields - var deserializer = new DeserializerBuilder() - .WithNamingConvention(NullNamingConvention.Instance) - .IgnoreUnmatchedProperties() - .Build(); + var errors = new List(); - // Deserialize the raw YAML into the internal model - ReviewMarkYaml raw; + // Try to read the file; if this fails we cannot continue. + string yaml; try { - raw = deserializer.Deserialize(yaml) - ?? throw new ArgumentException("YAML content is empty or invalid.", nameof(yaml)); + yaml = File.ReadAllText(filePath); } - catch (YamlException ex) + catch (Exception ex) when (ex is not InvalidOperationException) { - throw new ArgumentException($"Invalid YAML content: {ex.Message}", nameof(yaml), ex); + errors.Add($"{filePath}: error: {ex.Message}"); + return errors; } - // Map needs-review patterns (default to empty list if absent) - var needsReviewPatterns = (IReadOnlyList)(raw.NeedsReview ?? []); - - // Map evidence-source (required: evidence-source block, type, and location) - if (raw.EvidenceSource is not { } es) + // Try to parse the raw YAML model; if this fails we cannot do semantic checks. + // When the inner exception is a YamlException, format the location as "file:line:col" + // to match the standard linting output convention. + ReviewMarkYaml raw; + try { - throw new ArgumentException("Configuration is missing required 'evidence-source' block.", nameof(yaml)); + raw = ReviewMarkConfigurationHelpers.DeserializeRaw(yaml, filePath); } - - if (string.IsNullOrWhiteSpace(es.Type)) + catch (InvalidOperationException ex) when (ex.InnerException is YamlException yamlEx) + { + errors.Add($"{filePath}:{yamlEx.Start.Line}:{yamlEx.Start.Column}: error: {yamlEx.Message}"); + return errors; + } + catch (InvalidOperationException ex) { - throw new ArgumentException("Configuration 'evidence-source' is missing a required 'type' field.", nameof(yaml)); + errors.Add($"{filePath}: error: {ex.Message}"); + return errors; } - if (string.IsNullOrWhiteSpace(es.Location)) + // Validate the evidence-source block, collecting all field-level errors. + var es = raw.EvidenceSource; + if (es == null) { - throw new ArgumentException("Configuration 'evidence-source' is missing a required 'location' field.", nameof(yaml)); + errors.Add( + $"{filePath}: error: Configuration is missing required 'evidence-source' block."); } + else + { + if (string.IsNullOrWhiteSpace(es.Type)) + { + errors.Add( + $"{filePath}: error: 'evidence-source' is missing a required 'type' field."); + } + else if (!ReviewMarkConfigurationHelpers.IsSupportedEvidenceSourceType(es.Type)) + { + errors.Add( + $"{filePath}: error: 'evidence-source' type '{es.Type}' is not supported (must be 'url' or 'fileshare')."); + } - var evidenceSource = new EvidenceSource( - Type: es.Type, - Location: es.Location, - UsernameEnv: es.Credentials?.UsernameEnv, - PasswordEnv: es.Credentials?.PasswordEnv); - // Map review sets, requiring id, title, and paths for each entry - var reviews = (raw.Reviews ?? []) - .Select((r, i) => + if (string.IsNullOrWhiteSpace(es.Location)) { - // Each review set must have an id - if (string.IsNullOrWhiteSpace(r.Id)) - { - throw new ArgumentException($"Review set at index {i} is missing a required 'id' field."); - } + errors.Add( + $"{filePath}: error: 'evidence-source' is missing a required 'location' field."); + } + } - // Each review set must have a title - if (string.IsNullOrWhiteSpace(r.Title)) - { - throw new ArgumentException($"Review set '{r.Id}' is missing a required 'title' field."); - } + // Validate each review set, accumulating all structural and uniqueness errors. + // Review IDs are treated as case-sensitive identifiers (Ordinal), which is intentional: + // "Core-Logic" and "core-logic" are distinct IDs. Evidence-source type uses OrdinalIgnoreCase + // because YAML convention allows any casing for keyword values like "url" or "fileshare". + var seenIds = new Dictionary(StringComparer.Ordinal); + var reviews = raw.Reviews ?? []; + for (var i = 0; i < reviews.Count; i++) + { + var r = reviews[i]; - // Each review set must have at least one non-empty path pattern - var paths = r.Paths; - if (paths is null || !paths.Any(p => !string.IsNullOrWhiteSpace(p))) - { - throw new ArgumentException( - $"Review set '{r.Id}' at index {i} is missing required 'paths' entries."); - } + if (string.IsNullOrWhiteSpace(r.Id)) + { + errors.Add( + $"{filePath}: error: Review set at index {i} is missing a required 'id' field."); + } + else if (seenIds.TryGetValue(r.Id, out var firstIndex)) + { + errors.Add( + $"{filePath}: error: reviews[{i}] has duplicate ID '{r.Id}' (first defined at reviews[{firstIndex}])."); + } + else + { + seenIds[r.Id] = i; + } - return new ReviewSet(r.Id, r.Title, paths); - }) - .ToList(); + if (string.IsNullOrWhiteSpace(r.Title)) + { + errors.Add( + $"{filePath}: error: Review set at index {i} is missing a required 'title' field."); + } - return new ReviewMarkConfiguration(needsReviewPatterns, evidenceSource, reviews); + if (r.Paths == null || !r.Paths.Any(p => !string.IsNullOrWhiteSpace(p))) + { + errors.Add( + $"{filePath}: error: Review set at index {i} is missing required 'paths' entries."); + } + } + + return errors; + } + + /// + /// Parses a YAML string into a . + /// + /// The YAML content to parse. + /// A populated instance. + /// Thrown when is null. + /// Thrown when the YAML is invalid or missing required fields. + internal static ReviewMarkConfiguration Parse(string yaml) + { + // Validate the yaml input + ArgumentNullException.ThrowIfNull(yaml); + + // Deserialize without a file path so YAML errors are wrapped as ArgumentException (not + // InvalidOperationException) which is what callers of Parse (unit tests) expect. + var raw = ReviewMarkConfigurationHelpers.DeserializeRaw(yaml, filePath: null); + + return ReviewMarkConfigurationHelpers.BuildConfiguration(raw); } /// diff --git a/src/DemaConsulting.ReviewMark/Validation.cs b/src/DemaConsulting.ReviewMark/Validation.cs index ce93b16..d7e0568 100644 --- a/src/DemaConsulting.ReviewMark/Validation.cs +++ b/src/DemaConsulting.ReviewMark/Validation.cs @@ -56,6 +56,7 @@ public static void Run(Context context) RunDirTest(context, testResults); RunEnforceTest(context, testResults); RunElaborateTest(context, testResults); + RunLintTest(context, testResults); // Calculate totals var totalTests = testResults.Results.Count; @@ -378,6 +379,38 @@ private static void RunElaborateTest(Context context, DemaConsulting.TestResults }); } + /// + /// Runs a test for lint functionality. + /// + /// The context for output. + /// The test results collection. + private static void RunLintTest(Context context, DemaConsulting.TestResults.TestResults testResults) + { + RunValidationTest(context, testResults, "ReviewMark_Lint", () => + { + using var tempDir = new TemporaryDirectory(); + var (definitionFile, _) = CreateTestDefinitionFixtures(tempDir.DirectoryPath); + var logFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "lint-test.log"); + + // Run the program to lint the definition file + int exitCode; + using (var testContext = Context.Create(["--silent", "--log", logFile, "--lint", "--definition", definitionFile])) + { + Program.Run(testContext); + exitCode = testContext.ExitCode; + } + + if (exitCode != 0) + { + return $"Program exited with code {exitCode}"; + } + + // Verify the log contains a success message + var logContent = File.ReadAllText(logFile); + return logContent.Contains("No issues found") ? null : "Lint output does not contain 'No issues found'"; + }); + } + /// /// Runs a single validation test, recording the outcome in the test results collection. /// diff --git a/test/DemaConsulting.ReviewMark.Tests/ContextTests.cs b/test/DemaConsulting.ReviewMark.Tests/ContextTests.cs index 9df417d..e360756 100644 --- a/test/DemaConsulting.ReviewMark.Tests/ContextTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/ContextTests.cs @@ -712,5 +712,34 @@ public void Context_Create_ElaborateFlag_WithoutValue_ThrowsArgumentException() // Act & Assert - --elaborate without an ID argument should throw Assert.Throws(() => Context.Create(["--elaborate"])); } + + /// + /// Test that --lint flag sets Lint to true. + /// + [TestMethod] + public void Context_Create_LintFlag_SetsLintTrue() + { + // Act + using var context = Context.Create(["--lint"]); + + // Assert — Lint is true, other flags remain false, and exit code is zero + Assert.IsTrue(context.Lint); + Assert.IsFalse(context.Version); + Assert.IsFalse(context.Help); + Assert.AreEqual(0, context.ExitCode); + } + + /// + /// Test that Lint is false when --lint is not specified. + /// + [TestMethod] + public void Context_Create_NoArguments_LintIsFalse() + { + // Act + using var context = Context.Create([]); + + // Assert — Lint is false when --lint is not specified + Assert.IsFalse(context.Lint); + } } diff --git a/test/DemaConsulting.ReviewMark.Tests/ProgramTests.cs b/test/DemaConsulting.ReviewMark.Tests/ProgramTests.cs index 18ab989..ee250bc 100644 --- a/test/DemaConsulting.ReviewMark.Tests/ProgramTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/ProgramTests.cs @@ -26,6 +26,10 @@ namespace DemaConsulting.ReviewMark.Tests; [TestClass] public class ProgramTests { + /// + /// Log file name used across lint tests. + /// + private const string LintLogFile = "lint.log"; /// /// Test that Run with version flag displays version only. /// @@ -187,63 +191,51 @@ public void Program_Run_WithHelpFlag_IncludesElaborateOption() public void Program_Run_WithElaborateFlag_OutputsElaboration() { // Arrange — create temp directory with a definition file and source file - var testDirectory = PathHelpers.SafePathCombine( - Path.GetTempPath(), $"ProgramTests_Elaborate_{Guid.NewGuid()}"); + using var tempDir = new TestDirectory(); + var srcDir = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "src"); + Directory.CreateDirectory(srcDir); + File.WriteAllText(PathHelpers.SafePathCombine(srcDir, "A.cs"), "class A {}"); + + var indexFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "index.json"); + File.WriteAllText(indexFile, """{"reviews":[]}"""); + + var definitionFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "definition.yaml"); + File.WriteAllText(definitionFile, $""" + needs-review: + - "src/**/*.cs" + evidence-source: + type: fileshare + location: {indexFile} + reviews: + - id: Core-Logic + title: Review of core business logic + paths: + - "src/**/*.cs" + """); + + var originalOut = Console.Out; try { - Directory.CreateDirectory(testDirectory); - var srcDir = PathHelpers.SafePathCombine(testDirectory, "src"); - Directory.CreateDirectory(srcDir); - File.WriteAllText(PathHelpers.SafePathCombine(srcDir, "A.cs"), "class A {}"); + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create([ + "--definition", definitionFile, + "--dir", tempDir.DirectoryPath, + "--elaborate", "Core-Logic"]); - var indexFile = PathHelpers.SafePathCombine(testDirectory, "index.json"); - File.WriteAllText(indexFile, """{"reviews":[]}"""); + // Act + Program.Run(context); - var definitionFile = PathHelpers.SafePathCombine(testDirectory, "definition.yaml"); - File.WriteAllText(definitionFile, $""" - needs-review: - - "src/**/*.cs" - evidence-source: - type: fileshare - location: {indexFile} - reviews: - - id: Core-Logic - title: Review of core business logic - paths: - - "src/**/*.cs" - """); - - var originalOut = Console.Out; - try - { - using var outWriter = new StringWriter(); - Console.SetOut(outWriter); - using var context = Context.Create([ - "--definition", definitionFile, - "--dir", testDirectory, - "--elaborate", "Core-Logic"]); - - // Act - Program.Run(context); - - // Assert — output contains the review set ID and fingerprint heading - var output = outWriter.ToString(); - Assert.Contains("Core-Logic", output); - Assert.Contains("Fingerprint", output); - Assert.Contains("Files", output); - Assert.AreEqual(0, context.ExitCode); - } - finally - { - Console.SetOut(originalOut); - } + // Assert — output contains the review set ID and fingerprint heading + var output = outWriter.ToString(); + Assert.Contains("Core-Logic", output); + Assert.Contains("Fingerprint", output); + Assert.Contains("Files", output); + Assert.AreEqual(0, context.ExitCode); } finally { - if (Directory.Exists(testDirectory)) - { - Directory.Delete(testDirectory, recursive: true); - } + Console.SetOut(originalOut); } } @@ -254,56 +246,335 @@ public void Program_Run_WithElaborateFlag_OutputsElaboration() public void Program_Run_WithElaborateFlag_UnknownId_ReportsError() { // Arrange — create temp directory with a definition file - var testDirectory = PathHelpers.SafePathCombine( - Path.GetTempPath(), $"ProgramTests_ElaborateUnknown_{Guid.NewGuid()}"); + using var tempDir = new TestDirectory(); + + var indexFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "index.json"); + File.WriteAllText(indexFile, """{"reviews":[]}"""); + + var definitionFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "definition.yaml"); + File.WriteAllText(definitionFile, $""" + needs-review: + - "src/**/*.cs" + evidence-source: + type: fileshare + location: {indexFile} + reviews: + - id: Core-Logic + title: Review of core business logic + paths: + - "src/**/*.cs" + """); + + var originalError = Console.Error; try { - Directory.CreateDirectory(testDirectory); + using var errWriter = new StringWriter(); + Console.SetError(errWriter); + using var context = Context.Create([ + "--silent", + "--definition", definitionFile, + "--elaborate", "Unknown-Id"]); - var indexFile = PathHelpers.SafePathCombine(testDirectory, "index.json"); - File.WriteAllText(indexFile, """{"reviews":[]}"""); + // Act + Program.Run(context); - var definitionFile = PathHelpers.SafePathCombine(testDirectory, "definition.yaml"); - File.WriteAllText(definitionFile, $""" - needs-review: - - "src/**/*.cs" - evidence-source: - type: fileshare - location: {indexFile} - reviews: - - id: Core-Logic - title: Review of core business logic - paths: - - "src/**/*.cs" - """); - - var originalError = Console.Error; - try - { - using var errWriter = new StringWriter(); - Console.SetError(errWriter); - using var context = Context.Create([ - "--silent", - "--definition", definitionFile, - "--elaborate", "Unknown-Id"]); - - // Act - Program.Run(context); - - // Assert — non-zero exit code when the review-set ID is not found - Assert.AreEqual(1, context.ExitCode); - } - finally - { - Console.SetError(originalError); - } + // Assert — non-zero exit code when the review-set ID is not found + Assert.AreEqual(1, context.ExitCode); + } + finally + { + Console.SetError(originalError); + } + } + + /// + /// Test that Run with --help flag includes --lint in the usage information. + /// + [TestMethod] + public void Program_Run_WithHelpFlag_IncludesLintOption() + { + // Arrange + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--help"]); + + // Act + Program.Run(context); + + // Assert — help text includes the --lint option + var output = outWriter.ToString(); + Assert.Contains("--lint", output); } finally { - if (Directory.Exists(testDirectory)) - { - Directory.Delete(testDirectory, recursive: true); - } + Console.SetOut(originalOut); + } + } + + /// + /// Test that Run with --lint flag on a valid definition file reports success. + /// + [TestMethod] + public void Program_Run_WithLintFlag_ValidConfig_ReportsSuccess() + { + // Arrange — create temp directory with a valid definition file + using var tempDir = new TestDirectory(); + var indexFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "index.json"); + File.WriteAllText(indexFile, """{"reviews":[]}"""); + + var definitionFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "definition.yaml"); + File.WriteAllText(definitionFile, $""" + needs-review: + - "src/**/*.cs" + evidence-source: + type: fileshare + location: {indexFile} + reviews: + - id: Core-Logic + title: Review of core business logic + paths: + - "src/**/*.cs" + """); + + var logFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, LintLogFile); + + // Act — dispose the context before reading the log to release the file handle on Windows + int exitCode; + using (var context = Context.Create(["--silent", "--log", logFile, "--lint", "--definition", definitionFile])) + { + Program.Run(context); + exitCode = context.ExitCode; + } + + // Assert — exit code is zero and log contains success message + var logContent = File.ReadAllText(logFile); + Assert.AreEqual(0, exitCode); + Assert.Contains("No issues found", logContent); + } + + /// + /// Test that Run with --lint flag on a missing definition file reports an error. + /// + [TestMethod] + public void Program_Run_WithLintFlag_MissingConfig_ReportsError() + { + // Arrange — use a non-existent definition file + using var tempDir = new TestDirectory(); + var nonExistentFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "nonexistent.yaml"); + var logFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, LintLogFile); + + // Act — dispose the context before reading the log to release the file handle on Windows + int exitCode; + using (var context = Context.Create(["--silent", "--log", logFile, "--lint", "--definition", nonExistentFile])) + { + Program.Run(context); + exitCode = context.ExitCode; + } + + // Assert — non-zero exit code and log contains an error mentioning the missing file + var logContent = File.ReadAllText(logFile); + Assert.AreEqual(1, exitCode); + Assert.Contains("error:", logContent); + Assert.Contains("nonexistent.yaml", logContent); + } + + /// + /// Test that Run with --lint flag detects duplicate review set IDs and reports an error. + /// + [TestMethod] + public void Program_Run_WithLintFlag_DuplicateIds_ReportsError() + { + // Arrange — create temp directory with a definition file containing duplicate IDs + using var tempDir = new TestDirectory(); + var indexFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "index.json"); + File.WriteAllText(indexFile, """{"reviews":[]}"""); + + var definitionFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "definition.yaml"); + File.WriteAllText(definitionFile, $""" + needs-review: + - "src/**/*.cs" + evidence-source: + type: fileshare + location: {indexFile} + reviews: + - id: Core-Logic + title: Review of core business logic + paths: + - "src/**/*.cs" + - id: Core-Logic + title: Duplicate review set + paths: + - "src/**/*.cs" + """); + + var logFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, LintLogFile); + + // Act — dispose the context before reading the log to release the file handle on Windows + int exitCode; + using (var context = Context.Create(["--silent", "--log", logFile, "--lint", "--definition", definitionFile])) + { + Program.Run(context); + exitCode = context.ExitCode; + } + + // Assert — non-zero exit code and log contains a clear duplicate-ID error message + var logContent = File.ReadAllText(logFile); + Assert.AreEqual(1, exitCode); + Assert.Contains("error:", logContent); + Assert.Contains("duplicate ID", logContent); + Assert.Contains("Core-Logic", logContent); + } + + /// + /// Test that Run with --lint flag detects unknown evidence-source type and reports an error. + /// + [TestMethod] + public void Program_Run_WithLintFlag_UnknownSourceType_ReportsError() + { + // Arrange — create temp directory with a definition file having an unknown source type + using var tempDir = new TestDirectory(); + var definitionFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "definition.yaml"); + File.WriteAllText(definitionFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: ftp + location: ftp://example.com/index.json + reviews: + - id: Core-Logic + title: Review of core business logic + paths: + - "src/**/*.cs" + """); + + var logFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, LintLogFile); + + // Act — dispose the context before reading the log to release the file handle on Windows + int exitCode; + using (var context = Context.Create(["--silent", "--log", logFile, "--lint", "--definition", definitionFile])) + { + Program.Run(context); + exitCode = context.ExitCode; + } + + // Assert — non-zero exit code and log contains a clear unsupported-type error message + var logContent = File.ReadAllText(logFile); + Assert.AreEqual(1, exitCode); + Assert.Contains("error:", logContent); + Assert.Contains("ftp", logContent); + Assert.Contains("not supported", logContent); + } + + /// + /// Test that Run with --lint flag reports a clear error for corrupted (invalid) YAML. + /// + [TestMethod] + public void Program_Run_WithLintFlag_CorruptedYaml_ReportsError() + { + // Arrange — create a definition file with invalid YAML syntax + using var tempDir = new TestDirectory(); + var definitionFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "definition.yaml"); + File.WriteAllText(definitionFile, """ + {{{this is not valid yaml + """); + + var logFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, LintLogFile); + + // Act — dispose the context before reading the log to release the file handle on Windows + int exitCode; + using (var context = Context.Create(["--silent", "--log", logFile, "--lint", "--definition", definitionFile])) + { + Program.Run(context); + exitCode = context.ExitCode; + } + + // Assert — non-zero exit code and log contains an error naming the definition file and a line number + var logContent = File.ReadAllText(logFile); + Assert.AreEqual(1, exitCode); + Assert.Contains("error:", logContent); + Assert.Contains("definition.yaml:", logContent); + } + + /// + /// Test that Run with --lint flag reports a clear error when required fields are missing. + /// + [TestMethod] + public void Program_Run_WithLintFlag_MissingEvidenceSource_ReportsError() + { + // Arrange — create a definition file with no evidence-source block + using var tempDir = new TestDirectory(); + var definitionFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "definition.yaml"); + File.WriteAllText(definitionFile, """ + needs-review: + - "src/**/*.cs" + reviews: + - id: Core-Logic + title: Review of core business logic + paths: + - "src/**/*.cs" + """); + + var logFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, LintLogFile); + + // Act — dispose the context before reading the log to release the file handle on Windows + int exitCode; + using (var context = Context.Create(["--silent", "--log", logFile, "--lint", "--definition", definitionFile])) + { + Program.Run(context); + exitCode = context.ExitCode; } + + // Assert — non-zero exit code and log names the file and the missing field + var logContent = File.ReadAllText(logFile); + Assert.AreEqual(1, exitCode); + Assert.Contains("error:", logContent); + Assert.Contains("definition.yaml", logContent); + Assert.Contains("evidence-source", logContent); + } + + /// + /// Test that Run with --lint flag reports ALL errors in one pass when the file has + /// multiple detectable issues (missing evidence-source AND duplicate review IDs). + /// + [TestMethod] + public void Program_Run_WithLintFlag_MultipleErrors_ReportsAll() + { + // Arrange — create a definition file that is missing evidence-source AND has duplicate IDs + using var tempDir = new TestDirectory(); + var definitionFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "definition.yaml"); + File.WriteAllText(definitionFile, """ + needs-review: + - "src/**/*.cs" + reviews: + - id: Core-Logic + title: Review of core business logic + paths: + - "src/**/*.cs" + - id: Core-Logic + title: Duplicate review set + paths: + - "src/**/*.cs" + """); + + var logFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, LintLogFile); + + // Act — dispose the context before reading the log to release the file handle on Windows + int exitCode; + using (var context = Context.Create(["--silent", "--log", logFile, "--lint", "--definition", definitionFile])) + { + Program.Run(context); + exitCode = context.ExitCode; + } + + // Assert — non-zero exit code and log contains BOTH the missing evidence-source error + // AND the duplicate ID error, proving all errors are accumulated in one pass. + var logContent = File.ReadAllText(logFile); + Assert.AreEqual(1, exitCode); + Assert.Contains("evidence-source", logContent); + Assert.Contains("duplicate ID", logContent); + Assert.Contains("Core-Logic", logContent); } } diff --git a/test/DemaConsulting.ReviewMark.Tests/ReviewMarkConfigurationTests.cs b/test/DemaConsulting.ReviewMark.Tests/ReviewMarkConfigurationTests.cs index 1836039..26ab57a 100644 --- a/test/DemaConsulting.ReviewMark.Tests/ReviewMarkConfigurationTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/ReviewMarkConfigurationTests.cs @@ -301,6 +301,82 @@ public void ReviewMarkConfiguration_Load_NonExistentFile_ThrowsException() ReviewMarkConfiguration.Load(nonExistentPath)); } + /// + /// Test that Load includes the file name in the error message when YAML is invalid. + /// + [TestMethod] + public void ReviewMarkConfiguration_Load_InvalidYaml_ErrorIncludesFilenameAndLine() + { + // Arrange — write a configuration file with invalid YAML syntax + var configPath = PathHelpers.SafePathCombine(_testDirectory, ".reviewmark.yaml"); + File.WriteAllText(configPath, "{{{invalid yaml"); + + // Act & Assert + var ex = Assert.Throws(() => + ReviewMarkConfiguration.Load(configPath)); + Assert.Contains(".reviewmark.yaml", ex.Message); + Assert.Contains("at line", ex.Message); + } + + /// + /// Test that Load includes the file name in the error message when required fields are missing. + /// + [TestMethod] + public void ReviewMarkConfiguration_Load_MissingEvidenceSource_ErrorIncludesFilename() + { + // Arrange — write a valid YAML file that is missing the required evidence-source block + var configPath = PathHelpers.SafePathCombine(_testDirectory, ".reviewmark.yaml"); + File.WriteAllText(configPath, """ + needs-review: + - "src/**/*.cs" + reviews: + - id: Core-Logic + title: Review of core business logic + paths: + - "src/**/*.cs" + """); + + // Act & Assert + var ex = Assert.Throws(() => + ReviewMarkConfiguration.Load(configPath)); + Assert.Contains(".reviewmark.yaml", ex.Message); + Assert.Contains("evidence-source", ex.Message); + } + + /// + /// Test that Lint returns all errors from a file with multiple detectable issues + /// (missing evidence-source AND duplicate review IDs) without stopping at the first. + /// + [TestMethod] + public void ReviewMarkConfiguration_Lint_MultipleErrors_ReturnsAll() + { + // Arrange — write a YAML file missing evidence-source and containing duplicate IDs + var configPath = PathHelpers.SafePathCombine(_testDirectory, ".reviewmark.yaml"); + File.WriteAllText(configPath, """ + needs-review: + - "src/**/*.cs" + reviews: + - id: Core-Logic + title: Review of core business logic + paths: + - "src/**/*.cs" + - id: Core-Logic + title: Duplicate review set + paths: + - "src/**/*.cs" + """); + + // Act + var errors = ReviewMarkConfiguration.Lint(configPath); + + // Assert — both the missing evidence-source error and the duplicate ID error are returned + Assert.AreEqual(2, errors.Count); + Assert.IsTrue(errors.Any(e => e.Contains("evidence-source")), + "Expected an error about missing evidence-source."); + Assert.IsTrue(errors.Any(e => e.Contains("duplicate ID") && e.Contains("Core-Logic")), + "Expected an error about duplicate ID 'Core-Logic'."); + } + /// /// Test that Load resolves a relative fileshare location against the config file's directory. /// diff --git a/test/DemaConsulting.ReviewMark.Tests/TestDirectory.cs b/test/DemaConsulting.ReviewMark.Tests/TestDirectory.cs new file mode 100644 index 0000000..b94ab9e --- /dev/null +++ b/test/DemaConsulting.ReviewMark.Tests/TestDirectory.cs @@ -0,0 +1,65 @@ +// Copyright (c) DEMA Consulting +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +namespace DemaConsulting.ReviewMark.Tests; + +/// +/// Represents a temporary directory that is automatically deleted when disposed. +/// +internal sealed class TestDirectory : IDisposable +{ + /// + /// Gets the path to the temporary directory. + /// + public string DirectoryPath { get; } + + /// + /// Initializes a new instance of the class. + /// + public TestDirectory() + { + DirectoryPath = PathHelpers.SafePathCombine(Path.GetTempPath(), $"reviewmark_test_{Guid.NewGuid()}"); + Directory.CreateDirectory(DirectoryPath); + } + + /// + /// Deletes the temporary directory and all its contents. + /// + public void Dispose() + { + if (!Directory.Exists(DirectoryPath)) + { + return; + } + + try + { + Directory.Delete(DirectoryPath, recursive: true); + } + catch (IOException) + { + // Ignore cleanup failures in tests (e.g., transient file locks on Windows). + } + catch (UnauthorizedAccessException) + { + // Ignore cleanup failures in tests (e.g., transient access issues on Windows). + } + } +} From dada8832724e6ed7068cb6c406b1c5a1b4414fc3 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 21:21:38 -0400 Subject: [PATCH 05/19] Add 'none' evidence source to project setup (#29) * Initial plan * Add 'none' evidence source type that returns an empty index Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/75a0f826-6b34-4a3a-b44e-45546b78f006 * Complete 'none' evidence source: tests, docs, requirements Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/a8d3447c-9cdf-471f-9314-e21a1eb99b7d * Update src/DemaConsulting.ReviewMark/ReviewMarkConfiguration.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Address review feedback: update guide.md intro and README comment for 'none' source Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/729531fc-fd47-41a4-ae10-68d50821e6f1 * Update src/DemaConsulting.ReviewMark/ReviewMarkConfiguration.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Co-authored-by: Malcolm Nixon Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- README.md | 2 +- THEORY-OF-OPERATIONS.md | 5 +- docs/guide/guide.md | 24 +++++-- requirements.yaml | 21 +++++- src/DemaConsulting.ReviewMark/Index.cs | 8 +++ .../ReviewMarkConfiguration.cs | 20 +++--- .../IndexTests.cs | 46 ++++++++++++ .../ReviewMarkConfigurationTests.cs | 70 +++++++++++++++++++ 8 files changed, 177 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 75504e9..9613b2a 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ needs-review: - "!src/Generated/**" # exclude auto-generated files evidence-source: - type: url # 'url' or 'fileshare' + type: url # 'none', 'url', or 'fileshare' location: https://reviews.example.com/evidence/index.json reviews: diff --git a/THEORY-OF-OPERATIONS.md b/THEORY-OF-OPERATIONS.md index 9a59dad..516076a 100644 --- a/THEORY-OF-OPERATIONS.md +++ b/THEORY-OF-OPERATIONS.md @@ -27,7 +27,7 @@ needs-review: - "!src/Generated/**" # exclude auto-generated files evidence-source: - type: url # 'url' or 'fileshare' + type: url # 'none', 'url', or 'fileshare' location: https://reviews.example.com/evidence/index.json reviews: @@ -66,10 +66,11 @@ expiry due to refactoring or directory restructuring. ### Evidence Source -ReviewMark queries the configured evidence source for review PDFs. Two source types are supported: +ReviewMark queries the configured evidence source for review PDFs. Three source types are supported: | Type | Description | | :--- | :---------- | +| `none` | No evidence source; always returns an empty index (useful during initial project setup) | | `url` | Full HTTP/HTTPS URL to `index.json`; credentials supplied via environment variables | | `fileshare` | Full UNC or local file-system path to `index.json`; access uses OS/share permissions | diff --git a/docs/guide/guide.md b/docs/guide/guide.md index 63dfd2c..de6ca85 100644 --- a/docs/guide/guide.md +++ b/docs/guide/guide.md @@ -357,15 +357,27 @@ reviews: ## Evidence Source -The `evidence-source` block tells ReviewMark where to find `index.json` — the catalogue of -completed review PDFs. +The `evidence-source` block configures how ReviewMark obtains review evidence. For `url` and +`fileshare` sources it points to `index.json` — the catalogue of completed review PDFs. The +`none` source skips loading any index and always returns empty evidence (useful during initial +project setup before an evidence store is provisioned). ### Source Types -| Type | Description | -| :----------- | :------------------------------------------------------ | -| `fileshare` | Full UNC or local file-system path to `index.json` | -| `url` | Full HTTP or HTTPS URL to `index.json` | +| Type | Description | +| :----------- | :----------------------------------------------------------------------------------- | +| `none` | No evidence source; always returns an empty index (for initial project setup) | +| `fileshare` | Full UNC or local file-system path to `index.json` | +| `url` | Full HTTP or HTTPS URL to `index.json` | + +#### None + +Use `none` when an evidence source has not yet been provisioned: + +```yaml +evidence-source: + type: none +``` #### File Share diff --git a/requirements.yaml b/requirements.yaml index 06e361a..7e3984c 100644 --- a/requirements.yaml +++ b/requirements.yaml @@ -289,10 +289,11 @@ sections: - ReviewMarkConfiguration_Load_FileshareRelativeLocation_ResolvesToAbsolutePath - id: ReviewMark-Index-EvidenceSource - title: The tool shall load a ReviewIndex from an EvidenceSource supporting fileshare and url types. + title: The tool shall load a ReviewIndex from an EvidenceSource supporting none, fileshare, and url types. justification: | The tool must be able to load review evidence index data from the EvidenceSource - specified in its configuration. Two source types are supported: `fileshare` loads + specified in its configuration. Three source types are supported: `none` returns an + empty index immediately (useful during initial project setup), `fileshare` loads the index JSON from a local or network file path, and `url` downloads it over HTTP(S) with optional Basic-auth credentials read from environment variables. An internal overload accepting an HttpClient enables unit testing via a fake @@ -300,6 +301,8 @@ sections: tests: - ReviewIndex_Load_EvidenceSource_NullSource_ThrowsArgumentNullException - ReviewIndex_Load_EvidenceSource_UnknownType_ThrowsInvalidOperationException + - ReviewIndex_Load_EvidenceSource_None_ReturnsEmptyIndex + - ReviewIndex_Load_EvidenceSource_None_HttpClientOverload_ReturnsEmptyIndex - ReviewIndex_Load_EvidenceSource_Fileshare_LoadsFromFile - ReviewIndex_Load_EvidenceSource_Fileshare_NonExistentFile_ThrowsInvalidOperationException - ReviewIndex_Load_EvidenceSource_Fileshare_InvalidJson_ThrowsInvalidOperationException @@ -311,6 +314,20 @@ sections: - ReviewIndex_Load_EvidenceSource_Url_InvalidJson_ThrowsInvalidOperationException - ReviewIndex_Load_EvidenceSource_NullHttpClient_ThrowsArgumentNullException + - id: ReviewMark-EvidenceSource-None + title: The tool shall support a 'none' evidence source type that provides no review evidence. + justification: | + When a project is first starting out, it should be able to set the evidence-source + to 'none' until an evidence store is provisioned. The 'none' type requires no + location field and always returns an empty index, allowing the tool to run without + error during initial repository setup. + tests: + - ReviewIndex_Load_EvidenceSource_None_ReturnsEmptyIndex + - ReviewIndex_Load_EvidenceSource_None_HttpClientOverload_ReturnsEmptyIndex + - ReviewMarkConfiguration_Parse_NoneEvidenceSource_ParsedCorrectly + - ReviewMarkConfiguration_Parse_NoneEvidenceSource_NoLocationRequired + - ReviewMarkConfiguration_Lint_NoneEvidenceSource_NoErrors + - id: ReviewMark-Index-PdfParsing title: The tool shall parse PDF metadata from the Keywords field when indexing evidence files. justification: | diff --git a/src/DemaConsulting.ReviewMark/Index.cs b/src/DemaConsulting.ReviewMark/Index.cs index fb90ab6..347b8f6 100644 --- a/src/DemaConsulting.ReviewMark/Index.cs +++ b/src/DemaConsulting.ReviewMark/Index.cs @@ -155,6 +155,7 @@ private ReviewIndex() /// /// Loads a from an . + /// For none sources an empty index is returned immediately. /// For fileshare sources the is treated as the /// path to the index.json file. For url sources the location is the HTTP(S) URL /// of the index.json file; an with optional pre-emptive @@ -174,6 +175,12 @@ internal static ReviewIndex Load(EvidenceSource evidenceSource) { ArgumentNullException.ThrowIfNull(evidenceSource); + // Short-circuit for none sources — return an empty index + if (evidenceSource.Type.Equals("none", StringComparison.OrdinalIgnoreCase)) + { + return Empty(); + } + // Short-circuit for fileshare sources — no HttpClient needed if (evidenceSource.Type.Equals("fileshare", StringComparison.OrdinalIgnoreCase)) { @@ -208,6 +215,7 @@ internal static ReviewIndex Load(EvidenceSource evidenceSource, HttpClient httpC // Dispatch to the appropriate loader based on the evidence-source type return evidenceSource.Type.ToLowerInvariant() switch { + "none" => Empty(), "fileshare" => LoadFromFile(evidenceSource.Location), "url" => LoadFromUrl(evidenceSource.Location, httpClient), _ => throw new InvalidOperationException( diff --git a/src/DemaConsulting.ReviewMark/ReviewMarkConfiguration.cs b/src/DemaConsulting.ReviewMark/ReviewMarkConfiguration.cs index 2a6eb6b..726a1e5 100644 --- a/src/DemaConsulting.ReviewMark/ReviewMarkConfiguration.cs +++ b/src/DemaConsulting.ReviewMark/ReviewMarkConfiguration.cs @@ -137,11 +137,12 @@ file static class ReviewMarkConfigurationHelpers { /// /// Returns true when is a recognized evidence-source - /// type (url or fileshare, case-insensitive). + /// type (none, url, or fileshare, case-insensitive). /// /// The type string to test. /// true if the type is supported; false otherwise. public static bool IsSupportedEvidenceSourceType(string type) => + string.Equals(type, "none", StringComparison.OrdinalIgnoreCase) || string.Equals(type, "url", StringComparison.OrdinalIgnoreCase) || string.Equals(type, "fileshare", StringComparison.OrdinalIgnoreCase); @@ -223,17 +224,17 @@ public static ReviewMarkConfiguration BuildConfiguration(ReviewMarkYaml raw) if (!IsSupportedEvidenceSourceType(es.Type)) { throw new ArgumentException( - $"Configuration 'evidence-source' type '{es.Type}' is not supported (must be 'url' or 'fileshare')."); + $"Configuration 'evidence-source' type '{es.Type}' is not supported (must be 'none', 'url', or 'fileshare')."); } - if (string.IsNullOrWhiteSpace(es.Location)) + if (string.IsNullOrWhiteSpace(es.Location) && !string.Equals(es.Type, "none", StringComparison.OrdinalIgnoreCase)) { throw new ArgumentException("Configuration 'evidence-source' is missing a required 'location' field."); } var evidenceSource = new EvidenceSource( Type: es.Type, - Location: es.Location, + Location: es.Location ?? string.Empty, UsernameEnv: es.Credentials?.UsernameEnv, PasswordEnv: es.Credentials?.PasswordEnv); @@ -276,8 +277,11 @@ public static ReviewMarkConfiguration BuildConfiguration(ReviewMarkYaml raw) /// /// Represents the evidence-source configuration from .reviewmark.yaml. /// -/// The source type, e.g. url or fileshare. -/// The URL or path for the evidence source. +/// The source type, e.g. none, url, or fileshare. +/// +/// The URL or path for the evidence source; required for url and fileshare types, +/// and optional/ignored when is none. +/// /// Optional environment-variable name that holds the username credential. /// Optional environment-variable name that holds the password credential. internal sealed record EvidenceSource( @@ -575,10 +579,10 @@ internal static IReadOnlyList Lint(string filePath) else if (!ReviewMarkConfigurationHelpers.IsSupportedEvidenceSourceType(es.Type)) { errors.Add( - $"{filePath}: error: 'evidence-source' type '{es.Type}' is not supported (must be 'url' or 'fileshare')."); + $"{filePath}: error: 'evidence-source' type '{es.Type}' is not supported (must be 'none', 'url', or 'fileshare')."); } - if (string.IsNullOrWhiteSpace(es.Location)) + if (string.IsNullOrWhiteSpace(es.Location) && !string.Equals(es.Type, "none", StringComparison.OrdinalIgnoreCase)) { errors.Add( $"{filePath}: error: 'evidence-source' is missing a required 'location' field."); diff --git a/test/DemaConsulting.ReviewMark.Tests/IndexTests.cs b/test/DemaConsulting.ReviewMark.Tests/IndexTests.cs index 7d397a9..e287413 100644 --- a/test/DemaConsulting.ReviewMark.Tests/IndexTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/IndexTests.cs @@ -502,6 +502,52 @@ public void ReviewIndex_Load_EvidenceSource_NullHttpClient_ThrowsArgumentNullExc #pragma warning restore CS8604 } + /// + /// Test that with a none + /// source returns an empty without accessing any file + /// or network resource. + /// + [TestMethod] + public void ReviewIndex_Load_EvidenceSource_None_ReturnsEmptyIndex() + { + // Arrange + var source = new EvidenceSource( + Type: "none", + Location: string.Empty, + UsernameEnv: null, + PasswordEnv: null); + + // Act + var index = ReviewIndex.Load(source); + + // Assert — a none source always returns an empty index + Assert.IsNull(index.GetEvidence("any-id", "any-fingerprint")); + } + + /// + /// Test that with a none + /// source returns an empty without making any HTTP request. + /// + [TestMethod] + public void ReviewIndex_Load_EvidenceSource_None_HttpClientOverload_ReturnsEmptyIndex() + { + // Arrange — use a fake handler that fails if actually called + using var handler = new FakeHttpMessageHandler(new HttpResponseMessage(System.Net.HttpStatusCode.InternalServerError)); + using var httpClient = new HttpClient(handler); + + var source = new EvidenceSource( + Type: "none", + Location: string.Empty, + UsernameEnv: null, + PasswordEnv: null); + + // Act + var index = ReviewIndex.Load(source, httpClient); + + // Assert — a none source always returns an empty index without touching the handler + Assert.IsNull(index.GetEvidence("any-id", "any-fingerprint")); + } + // ------------------------------------------------------------------------- // Save tests // ------------------------------------------------------------------------- diff --git a/test/DemaConsulting.ReviewMark.Tests/ReviewMarkConfigurationTests.cs b/test/DemaConsulting.ReviewMark.Tests/ReviewMarkConfigurationTests.cs index 26ab57a..1e2095b 100644 --- a/test/DemaConsulting.ReviewMark.Tests/ReviewMarkConfigurationTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/ReviewMarkConfigurationTests.cs @@ -406,6 +406,76 @@ public void ReviewMarkConfiguration_Load_FileshareRelativeLocation_ResolvesToAbs Assert.AreEqual(PathHelpers.SafePathCombine(_testDirectory, "index.json"), config.EvidenceSource.Location); } + /// + /// Test that an evidence-source with type none is parsed correctly + /// and produces an empty . + /// + [TestMethod] + public void ReviewMarkConfiguration_Parse_NoneEvidenceSource_ParsedCorrectly() + { + // Arrange + var yaml = """ + evidence-source: + type: none + reviews: + - id: Core-Logic + title: Review of core business logic + paths: + - "src/**/*.cs" + """; + + // Act + var config = ReviewMarkConfiguration.Parse(yaml); + + // Assert — type is 'none' and location is empty + Assert.AreEqual("none", config.EvidenceSource.Type); + Assert.AreEqual(string.Empty, config.EvidenceSource.Location); + } + + /// + /// Test that an evidence-source with type none does not require a + /// location field. + /// + [TestMethod] + public void ReviewMarkConfiguration_Parse_NoneEvidenceSource_NoLocationRequired() + { + // Arrange — YAML with a none source and no location field + var yaml = """ + evidence-source: + type: none + """; + + // Act & Assert — parsing must succeed without throwing + var config = ReviewMarkConfiguration.Parse(yaml); + Assert.AreEqual("none", config.EvidenceSource.Type); + } + + /// + /// Test that Lint does not report an error when the evidence-source type is none + /// and no location field is present. + /// + [TestMethod] + public void ReviewMarkConfiguration_Lint_NoneEvidenceSource_NoErrors() + { + // Arrange — write a valid config with a none evidence source + var configPath = PathHelpers.SafePathCombine(_testDirectory, ".reviewmark.yaml"); + File.WriteAllText(configPath, """ + evidence-source: + type: none + reviews: + - id: Core-Logic + title: Review of core business logic + paths: + - "src/**/*.cs" + """); + + // Act + var errors = ReviewMarkConfiguration.Lint(configPath); + + // Assert — no errors for a valid none source + Assert.HasCount(0, errors); + } + // ------------------------------------------------------------------------- // PublishReviewPlan tests // ------------------------------------------------------------------------- From 61697eee7df86bbf18a4c8059b6f210ab4c92875 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Mar 2026 16:13:17 -0400 Subject: [PATCH 06/19] Sync with TemplateDotNetTool template PRs (#58, #65, #66, #67, #70, #75) (#30) * Initial plan * Apply template repo changes from TemplateDotNetTool PRs Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/4f05d485-5375-4cca-952d-ddacf934b681 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Improve .reviewmark.yaml review-sets with requirements and design context Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/45abfc5f-35fa-4006-b4f7-b799bd49c895 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Split requirements.yaml into per-software-item files under docs/reqstream/ and update .reviewmark.yaml Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/05c02ae5-9340-45f7-bc6a-bccbaa4d5dc3 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Update .github/agents/technical-writer.agent.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update docs/code_review_report/introduction.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Fix build-docs ReviewMark step to use global 'reviewmark' tool instead of dotnet run Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/5a482424-635c-4f04-afff-9a32e7b57c52 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Co-authored-by: Malcolm Nixon Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .cspell.yaml | 9 +- .github/agents/requirements.agent.md | 414 ++++++++++++-- .github/agents/software-developer.agent.md | 272 +++++++-- .github/agents/technical-writer.agent.md | 272 +++++++-- .github/agents/test-developer.agent.md | 348 ++++++++---- .github/workflows/build.yaml | 106 ++-- .github/workflows/release.yaml | 7 +- .gitignore | 16 +- .reviewmark.yaml | 80 +++ AGENTS.md | 65 ++- docs/build_notes/definition.yaml | 12 + .../introduction.md | 0 docs/{buildnotes => build_notes}/title.txt | 0 docs/buildnotes/definition.yaml | 12 - docs/code_quality/definition.yaml | 12 + .../{quality => code_quality}/introduction.md | 0 docs/{quality => code_quality}/title.txt | 0 docs/code_review_plan/definition.yaml | 11 + docs/code_review_plan/introduction.md | 33 ++ docs/code_review_plan/title.txt | 13 + docs/code_review_report/definition.yaml | 11 + docs/code_review_report/introduction.md | 32 ++ docs/code_review_report/title.txt | 13 + docs/justifications/definition.yaml | 11 - docs/justifications/introduction.md | 29 - docs/justifications/title.txt | 13 - docs/quality/definition.yaml | 12 - docs/reqstream/cli-requirements.yaml | 243 ++++++++ .../reqstream/configuration-requirements.yaml | 30 + docs/reqstream/index-requirements.yaml | 66 +++ docs/reqstream/ots-requirements.yaml | 102 ++++ docs/reqstream/platform-requirements.yaml | 104 ++++ docs/requirements/definition.yaml | 11 - docs/requirements_doc/definition.yaml | 12 + .../introduction.md | 0 .../title.txt | 0 docs/requirements_report/definition.yaml | 11 + .../introduction.md | 0 .../title.txt | 0 docs/tracematrix/definition.yaml | 11 - requirements.yaml | 535 +----------------- 41 files changed, 2001 insertions(+), 927 deletions(-) create mode 100644 .reviewmark.yaml create mode 100644 docs/build_notes/definition.yaml rename docs/{buildnotes => build_notes}/introduction.md (100%) rename docs/{buildnotes => build_notes}/title.txt (100%) delete mode 100644 docs/buildnotes/definition.yaml create mode 100644 docs/code_quality/definition.yaml rename docs/{quality => code_quality}/introduction.md (100%) rename docs/{quality => code_quality}/title.txt (100%) create mode 100644 docs/code_review_plan/definition.yaml create mode 100644 docs/code_review_plan/introduction.md create mode 100644 docs/code_review_plan/title.txt create mode 100644 docs/code_review_report/definition.yaml create mode 100644 docs/code_review_report/introduction.md create mode 100644 docs/code_review_report/title.txt delete mode 100644 docs/justifications/definition.yaml delete mode 100644 docs/justifications/introduction.md delete mode 100644 docs/justifications/title.txt delete mode 100644 docs/quality/definition.yaml create mode 100644 docs/reqstream/cli-requirements.yaml create mode 100644 docs/reqstream/configuration-requirements.yaml create mode 100644 docs/reqstream/index-requirements.yaml create mode 100644 docs/reqstream/ots-requirements.yaml create mode 100644 docs/reqstream/platform-requirements.yaml delete mode 100644 docs/requirements/definition.yaml create mode 100644 docs/requirements_doc/definition.yaml rename docs/{requirements => requirements_doc}/introduction.md (100%) rename docs/{requirements => requirements_doc}/title.txt (100%) create mode 100644 docs/requirements_report/definition.yaml rename docs/{tracematrix => requirements_report}/introduction.md (100%) rename docs/{tracematrix => requirements_report}/title.txt (100%) delete mode 100644 docs/tracematrix/definition.yaml diff --git a/.cspell.yaml b/.cspell.yaml index f941b4f..e40779a 100644 --- a/.cspell.yaml +++ b/.cspell.yaml @@ -16,11 +16,14 @@ language: en words: - Anson - Blockquotes + - build_notes - buildmark - BuildMark - - buildnotes - camelcase - Checkmarx + - code_quality + - code_review_plan + - code_review_report - CodeQL - copilot - cspell @@ -62,6 +65,8 @@ words: - reindex - reqstream - ReqStream + - requirements_doc + - requirements_report - reviewmark - ReviewMark - Sarif @@ -77,7 +82,7 @@ words: - spdx - streetsidesoftware - testname - - tracematrix + - trace_matrix - triaging - Trivy - trx diff --git a/.github/agents/requirements.agent.md b/.github/agents/requirements.agent.md index bfb9294..bfd0a30 100644 --- a/.github/agents/requirements.agent.md +++ b/.github/agents/requirements.agent.md @@ -5,79 +5,383 @@ tools: [read, search, edit, execute, github, web, agent] user-invocable: true --- -# Requirements Agent - Template DotNet Tool +# Requirements Agent -Develop and maintain high-quality requirements with proper test coverage linkage. +Develop and maintain high-quality requirements with comprehensive test coverage linkage following Continuous +Compliance methodology for automated evidence generation and audit compliance. + +## Reporting + +If detailed documentation of requirements analysis is needed, create a report using the filename pattern +`AGENT_REPORT_requirements.md` to document requirement mappings, gap analysis, and traceability results. ## When to Invoke This Agent -Invoke the requirements-agent for: +Use the Requirements Agent for: + +- Creating new requirements in organized `docs/reqstream/` structure +- Establishing subsystem and software unit requirement files for independent review +- Reviewing and improving existing requirements quality and organization +- Ensuring proper requirements-to-test traceability +- Validating requirements enforcement in CI/CD pipelines +- Differentiating requirements from design/implementation details + +## Continuous Compliance Methodology + +### Core Principles + +The @requirements agent implements the Continuous Compliance methodology +, which provides automated compliance evidence +generation through structured requirements management: + +- **📚 Complete Methodology Documentation:** +- **📋 Detailed Requirements Guidelines:** + +- **🔧 ReqStream Tool Documentation:** + +#### Automated Evidence Generation + +- **Requirements Traceability**: Automated linking between requirements and test evidence +- **Compliance Reports**: Generated documentation for audit and regulatory compliance +- **Quality Gate Enforcement**: Pipeline failures prevent non-compliant code from merging +- **Platform-Specific Evidence**: Source filters ensure correct testing environment validation + +#### Continuous Compliance Benefits + +- **Audit Trail**: Complete requirements-to-implementation traceability +- **Regulatory Support**: Meets medical device, aerospace, automotive compliance standards +- **Quality Assurance**: Automated verification prevents compliance gaps +- **Documentation**: Generated reports reduce manual documentation overhead + +## Primary Responsibilities + +### Requirements Engineering Excellence + +- Focus on **observable behavior and characteristics**, not implementation details +- Write clear, testable requirements with measurable acceptance criteria +- Ensure semantic requirement IDs (`Project-Section-ShortDesc` format preferred over `REQ-042`) +- Include comprehensive justification explaining business/regulatory rationale +- Maintain hierarchical requirement structure with proper parent-child relationships + +### Requirements Organization for Review-Sets + +Organize requirements into separate files under `docs/reqstream/` to enable independent review processes: + +#### Subsystem-Level Requirements + +- **File Pattern**: `{subsystem}-subsystem.yaml` (e.g., `auth-subsystem.yaml`) +- **Content Focus**: High-level subsystem behavior, interfaces, and integration requirements +- **Review Scope**: Architectural and subsystem design reviews +- **Team Assignment**: Can be reviewed independently by subsystem teams + +#### Software Unit Requirements + +- **File Pattern**: `{subsystem}-{class}-class.yaml` (e.g., `auth-passwordvalidator-class.yaml`) +- **Content Focus**: Individual class behavior, method contracts, and invariants +- **Review Scope**: Code-level implementation reviews +- **Team Assignment**: Enable focused class-level review processes + +#### OTS Software Requirements + +- **File Pattern**: `ots-{component}.yaml` (e.g., `ots-systemtextjson.yaml`) +- **Content Focus**: Required functionality from third-party components, libraries, and frameworks +- **Review Scope**: Dependency validation and integration testing reviews +- **Team Assignment**: Can be reviewed by teams responsible for external dependency management +- **Section Structure**: Must use "OTS Software Requirements" as top-level section with component subsections: + +```yaml +sections: + - title: OTS Software Requirements + sections: + - title: System.Text.Json + requirements: + - id: Project-SystemTextJson-ReadJson + title: System.Text.Json shall be able to read JSON files. + # ... requirements for this OTS component + - title: NUnit + requirements: + - id: Project-NUnit-ParameterizedTests + title: NUnit shall support parameterized test methods. + # ... requirements for this OTS component +``` + +#### Benefits for Continuous Compliance + +- **Parallel Review Workflows**: Multiple teams can review different subsystems, classes, and OTS components simultaneously +- **Granular Status Tracking**: Review status maintained at subsystem, class, and OTS dependency level +- **Scalable Organization**: Supports large projects without requirement file conflicts +- **Independent Evidence**: Each file provides focused compliance evidence +- **Dependency Management**: OTS requirements enable systematic third-party component validation + +### Continuous Compliance Enforcement + +Following the Continuous Compliance methodology , +requirements management operates on these enforcement principles: + +#### Traceability Requirements (ENFORCED) + +- **Mandatory Coverage**: ALL requirements MUST link to passing tests - CI pipeline fails otherwise +- **Automated Verification**: `dotnet reqstream --enforce` validates complete traceability +- **Evidence Chain**: Requirements → Tests → Results → Documentation must be unbroken +- **Platform Compliance**: Source filters ensure correct testing environment evidence + +#### Quality Gate Integration + +- **Pipeline Enforcement**: CI/CD fails on any requirements without test coverage +- **Documentation Generation**: Automated requirements reports for audit compliance +- **Regulatory Support**: Meets FDA, DO-178C, ISO 26262, and other regulatory standards +- **Continuous Monitoring**: Every build verifies requirements compliance status + +#### Compliance Documentation + +Per Continuous Compliance requirements documentation +: + +- **Requirements Reports**: Generated documentation showing all requirements and their status +- **Justifications**: Business and regulatory rationale for each requirement +- **Trace Matrix**: Complete mapping of requirements to test evidence +- **Audit Trails**: Historical compliance evidence for regulatory reviews + +### Test Coverage Strategy & Linking + +#### Coverage Rules + +- **Requirements coverage**: Mandatory for all stated requirements +- **Test flexibility**: Not all tests need requirement links (corner cases, design validation, failure scenarios allowed) +- **Platform evidence**: Use source filters for platform/framework-specific requirements + +#### Source Filter Patterns (CRITICAL - DO NOT REMOVE) + +```yaml +tests: + - "windows@TestMethodName" # Windows platform evidence only + - "ubuntu@TestMethodName" # Linux (Ubuntu) platform evidence only + - "net8.0@TestMethodName" # .NET 8 runtime evidence only + - "net9.0@TestMethodName" # .NET 9 runtime evidence only + - "net10.0@TestMethodName" # .NET 10 runtime evidence only + - "TestMethodName" # Any platform evidence acceptable +``` + +**WARNING**: Removing source filters invalidates platform-specific compliance evidence and may cause audit failures. + +### Quality Gate Verification + +Before completing any requirements work, verify: + +#### 1. Requirements Quality + +- [ ] Semantic IDs follow `Project-Section-ShortDesc` pattern +- [ ] Clear, testable acceptance criteria defined +- [ ] Comprehensive justification provided +- [ ] Observable behavior specified (not implementation details) + +#### 2. Traceability Compliance + +- [ ] All requirements linked to appropriate tests +- [ ] Source filters applied for platform-specific requirements +- [ ] ReqStream enforcement passes: `dotnet reqstream --enforce` +- [ ] Generated reports current (requirements, justifications, trace matrix) + +#### 3. CI/CD Integration + +- [ ] Requirements files pass yamllint validation +- [ ] Test result formats compatible with ReqStream (TRX, JUnit XML) +- [ ] Pipeline configured with `--enforce` flag +- [ ] Build fails appropriately on coverage gaps + +## ReqStream Tool Integration + +### ReqStream Overview + +ReqStream is the core tool for implementing Continuous Compliance requirements management: + +**🔧 ReqStream Repository:** + +#### Key Capabilities + +- **Traceability Enforcement**: `dotnet reqstream --enforce` validates all requirements have test coverage +- **Multi-Format Support**: Handles TRX, JUnit XML, and other test result formats +- **Report Generation**: Creates requirements reports, justifications, and trace matrices +- **Source Filtering**: Validates platform-specific testing requirements +- **CI/CD Integration**: Provides exit codes for pipeline quality gates + +#### Essential ReqStream Commands + +```bash +# Validate requirements traceability (use in CI/CD) +dotnet reqstream --requirements requirements.yaml --tests "test-results/**/*.trx" --enforce + +# Generate requirements documentation (for publication) +dotnet reqstream --requirements requirements.yaml --report docs/requirements_doc/requirements.md + +# Generate justifications report (for publication) +dotnet reqstream --requirements requirements.yaml --justifications docs/requirements_doc/justifications.md + +# Generate trace matrix +dotnet reqstream --requirements requirements.yaml --tests "test-results/**/*.trx" --matrix docs/requirements_report/trace_matrix.md +``` + +### Required Tools & Configuration + +- **ReqStream**: Core requirements traceability and enforcement (`dotnet tool install DemaConsulting.ReqStream`) +- **yamllint**: YAML structure validation for requirements files +- **cspell**: Spell-checking for requirement text and justifications + +### Standard File Structure for Review-Set Organization + +```text +requirements.yaml # Root requirements file with includes only +docs/ + reqstream/ # Organized requirements files for independent review + # System-level requirements + system-requirements.yaml + + # Subsystem requirements (enable subsystem review-sets) + auth-subsystem.yaml # Authentication subsystem requirements + data-subsystem.yaml # Data management subsystem requirements + ui-subsystem.yaml # User interface subsystem requirements + + # Software unit requirements (enable class-level review-sets) + auth-passwordvalidator-class.yaml # PasswordValidator class requirements + data-repository-class.yaml # Repository pattern class requirements + ui-controller-class.yaml # UI Controller class requirements + + # OTS Software requirements (enable dependency review-sets) + ots-systemtextjson.yaml # System.Text.Json OTS requirements + ots-nunit.yaml # NUnit framework OTS requirements + ots-entityframework.yaml # Entity Framework OTS requirements + + requirements_doc/ # Pandoc document folder for requirements publication + definition.yaml # Document content definition + title.txt # Document metadata + requirements.md # Auto-generated requirements report + justifications.md # Auto-generated justifications + + requirements_report/ # Pandoc document folder for requirements testing publication + definition.yaml # Document content definition + title.txt # Document metadata + trace_matrix.md # Auto-generated trace matrix +``` + +#### Review-Set Benefits + +This file organization enables independent review workflows: + +- **Subsystem Reviews**: Each subsystem file can be reviewed independently by different teams +- **Software Unit Reviews**: Class-level requirements enable focused code reviews +- **OTS Dependency Reviews**: Third-party component requirements enable systematic dependency validation +- **Parallel Development**: Teams can work on requirements without conflicts +- **Granular Tracking**: Review status tracking per subsystem, software unit, and OTS dependency +- **Scalable Organization**: Supports large projects with multiple development teams + +#### Root Requirements File Structure + +```yaml +# requirements.yaml - Root configuration with includes only +includes: + # System and subsystem requirements + - docs/reqstream/system-requirements.yaml + - docs/reqstream/auth-subsystem.yaml + - docs/reqstream/data-subsystem.yaml + - docs/reqstream/ui-subsystem.yaml + # Software unit requirements (classes) + - docs/reqstream/auth-passwordvalidator-class.yaml + - docs/reqstream/data-repository-class.yaml + - docs/reqstream/ui-controller-class.yaml + # OTS Software requirements (third-party components) + - docs/reqstream/ots-systemtextjson.yaml + - docs/reqstream/ots-nunit.yaml + - docs/reqstream/ots-entityframework.yaml +``` + +## Continuous Compliance Best Practices + +### Requirements Quality Standards + +Following Continuous Compliance requirements guidelines +: + +#### 1. **Observable Behavior Focus** + +- Requirements specify WHAT the system shall do, not HOW it should be implemented +- Focus on externally observable characteristics and behavior +- Avoid implementation details, design constraints, or technology choices + +#### 2. **Testable Acceptance Criteria** + +- Each requirement must have clear, measurable acceptance criteria +- Requirements must be verifiable through automated or manual testing +- Ambiguous or untestable requirements cause compliance failures -- Creating new requirements in `requirements.yaml` -- Reviewing and improving existing requirements -- Ensuring requirements have appropriate test coverage -- Determining which type of test (unit, integration, or self-validation) is appropriate -- Differentiating requirements from design details +#### 3. **Comprehensive Justification** -## Responsibilities +- Business rationale explaining why the requirement exists +- Regulatory or standard references where applicable +- Risk mitigation or quality improvement justification -### Writing Good Requirements +#### 4. **Semantic Requirement IDs** -- Focus on **what** the system must do, not **how** it does it -- Requirements describe observable behavior or characteristics -- Design details (implementation choices) are NOT requirements -- Use clear, testable language with measurable acceptance criteria -- Each requirement should be traceable to test evidence +- Use meaningful IDs: `TestProject-CommandLine-DisplayHelp` instead of `REQ-042` +- Follow `Project-Section-ShortDesc` pattern for clarity +- Enable better requirement organization and traceability -### Test Coverage Strategy +### Platform-Specific Requirements -- **All requirements MUST be linked to tests** - this is enforced in CI -- **Not all tests need to be linked to requirements** - tests may exist for: - - Exploring corner cases - - Testing design decisions - - Failure-testing scenarios - - Implementation validation beyond requirement scope -- **Self-validation tests** (`TemplateTool_*`): Preferred for command-line behavior, features - that ship with the product -- **Unit tests**: For internal component behavior, isolated logic -- **Integration tests**: For cross-component interactions, end-to-end scenarios +Critical for regulatory compliance in multi-platform environments: -### Requirements Format +#### Source Filter Implementation -Follow the `requirements.yaml` structure: +```yaml +requirements: + - id: Platform-Windows-Compatibility + title: Windows Platform Support + description: The software shall operate on Windows 10 and later versions + tests: + - windows@PlatformTests.TestWindowsCompatibility # MUST run on Windows + + - id: Target-IAR-Build + title: IAR Compiler Compatibility + description: The firmware shall compile successfully with IAR C compiler + tests: + - iar@CompilerTests.TestIarBuild # MUST use IAR toolchain +``` -- Clear ID and description -- Justification explaining why the requirement is needed -- Linked to appropriate test(s) -- Enforced via: `dotnet reqstream --requirements requirements.yaml --tests "test-results/**/*.trx" --enforce` +**WARNING**: Source filters are REQUIRED for platform-specific compliance evidence. +Removing them invalidates regulatory audit trails. -### Test Source Filters +## Cross-Agent Coordination -Test links in `requirements.yaml` can include a source filter prefix to restrict which test results count as -evidence. This is critical for platform and framework requirements - **never remove these filters**. +### Hand-off to Other Agents -- `windows@TestName` - proves the test passed on a Windows platform -- `ubuntu@TestName` - proves the test passed on a Linux (Ubuntu) platform -- `net8.0@TestName` - proves the test passed under the .NET 8 target framework -- `net9.0@TestName` - proves the test passed under the .NET 9 target framework -- `net10.0@TestName` - proves the test passed under the .NET 10 target framework -- `dotnet8.x@TestName` - proves the self-validation test ran on a machine with .NET 8.x runtime -- `dotnet9.x@TestName` - proves the self-validation test ran on a machine with .NET 9.x runtime -- `dotnet10.x@TestName` - proves the self-validation test ran on a machine with .NET 10.x runtime +- If features need to be implemented to satisfy requirements, then call the @software-developer agent with the + **request** to implement features that satisfy requirements with **context** of specific requirement details + and **goal** of requirement compliance. +- If tests need to be created to validate requirements, then call the @test-developer agent with the **request** + to create tests that validate requirements with **context** of requirement specifications and + **additional instructions** for traceability setup. +- If requirements traceability needs to be enforced in CI/CD, then call the @code-quality agent with the **request** + to enforce requirements traceability in CI/CD with **context** of current enforcement status and **goal** of + automated compliance verification. +- If requirements documentation needs generation or maintenance, then call the @technical-writer agent with the + **request** to generate and maintain requirements documentation with **context** of current requirements and + **goal** of regulatory compliance documentation. -Without the source filter, a test result from any platform/framework satisfies the requirement. Removing a -filter invalidates the evidence for platform/framework requirements. +## Compliance Verification Checklist -## Defer To +### Before Completing Work -- **Software Developer Agent**: For implementing self-validation tests -- **Test Developer Agent**: For implementing unit and integration tests -- **Technical Writer Agent**: For documentation of requirements and processes -- **Code Quality Agent**: For verifying test quality and enforcement +1. **Requirement Quality**: Clear, testable, with proper justification +2. **Test Linkage**: All requirements have appropriate test coverage +3. **Source Filters**: Platform requirements have correct source filters +4. **Tool Validation**: yamllint, ReqStream enforcement passing +5. **Documentation**: Generated reports current and accessible +6. **CI Integration**: Pipeline properly configured for enforcement -## Don't +## Don't Do These Things -- Mix requirements with implementation details -- Create requirements without test linkage -- Expect all tests to be linked to requirements (some tests exist for other purposes) -- Change code directly (delegate to developer agents) +- Create requirements without test linkage (CI will fail) +- Remove source filters from platform-specific requirements (breaks compliance) +- Mix implementation details with requirements (separate concerns) +- Skip justification text (required for compliance audits) +- Change test code directly (delegate to @test-developer agent) +- Modify CI/CD enforcement thresholds without compliance review diff --git a/.github/agents/software-developer.agent.md b/.github/agents/software-developer.agent.md index efa5758..891f281 100644 --- a/.github/agents/software-developer.agent.md +++ b/.github/agents/software-developer.agent.md @@ -5,79 +5,249 @@ tools: [read, search, edit, execute, github, agent] user-invocable: true --- -# Software Developer - Template DotNet Tool +# Software Developer Agent -Develop production code and self-validation tests with emphasis on testability and clarity. +Develop production code with emphasis on testability, clarity, and compliance integration. + +## Reporting + +If detailed documentation of development work is needed, create a report using the filename pattern +`AGENT_REPORT_development.md` to document code changes, design decisions, and implementation details. ## When to Invoke This Agent -Invoke the software-developer for: +Use the Software Developer Agent for: + +- Implementing production code features and APIs +- Refactoring existing code for testability and maintainability +- Creating self-validation and demonstration code +- Implementing requirement-driven functionality +- Code architecture and design decisions +- Integration with Continuous Compliance tooling -- Implementing production code features -- Creating and maintaining self-validation tests (`TemplateTool_*`) -- Code refactoring for testability and maintainability -- Implementing command-line argument parsing and program logic +## Primary Responsibilities -## Responsibilities +### Literate Programming Style (MANDATORY) -### Code Style - Literate Programming +Write all code in **literate style** for maximum clarity and maintainability. -Write code in a **literate style**: +#### Literate Style Rules -- Every paragraph of code starts with a comment explaining what it's trying to do -- Blank lines separate logical paragraphs -- Comments describe intent, not mechanics -- Code should read like a well-structured document -- Reading just the literate comments should explain how the code works -- The code can be reviewed against the literate comments to check the implementation +- **Intent Comments:** - Every paragraph starts with a comment explaining intent (not mechanics) +- **Logical Separation:** - Blank lines separate logical code paragraphs +- **Purpose Over Process:** - Comments describe why, code shows how +- **Standalone Clarity:** - Reading comments alone should explain the algorithm/approach +- **Verification Support:** - Code can be verified against the literate comments for correctness -Example: +#### Examples + +**C# Example:** ```csharp -// Parse the command line arguments -var options = ParseArguments(args); +// Validate input parameters to prevent downstream errors +if (string.IsNullOrEmpty(input)) +{ + throw new ArgumentException("Input cannot be null or empty", nameof(input)); +} + +// Transform input data using the configured processing pipeline +var processedData = ProcessingPipeline.Transform(input); -// Validate the input file exists -if (!File.Exists(options.InputFile)) - throw new InvalidOperationException($"Input file not found: {options.InputFile}"); +// Apply business rules and validation logic +var validatedResults = BusinessRuleEngine.ValidateAndProcess(processedData); -// Process the file contents -var results = ProcessFile(options.InputFile); +// Return formatted results matching the expected output contract +return OutputFormatter.Format(validatedResults); ``` -### Design for Testability +**C++ Example:** + +```cpp +// Acquire exclusive hardware access using RAII pattern +std::lock_guard hardwareLock(m_hardwareMutex); + +// Validate sensor data integrity before processing +if (!sensorData.IsValid() || sensorData.GetTimestamp() < m_lastValidTimestamp) +{ + throw std::invalid_argument("Sensor data failed integrity validation"); +} + +// Apply hardware-specific calibration coefficients +auto calibratedReading = ApplyCalibration(sensorData.GetRawValue(), + m_calibrationCoefficients); + +// Filter noise using moving average with bounds checking +const auto filteredValue = m_noiseFilter.ApplyFilter(calibratedReading); +if (filteredValue < kMinOperationalThreshold || filteredValue > kMaxOperationalThreshold) +{ + LogWarning("Filtered sensor value outside operational range"); +} + +// Package result with quality metadata for downstream consumers +return SensorResult{filteredValue, CalculateQualityMetric(sensorData), + std::chrono::steady_clock::now()}; +``` + +### Design for Testability & Compliance + +#### Code Architecture Principles + +- **Single Responsibility**: Functions with focused, testable purposes +- **Dependency Injection**: External dependencies injected for testing +- **Pure Functions**: Minimize side effects and hidden state +- **Clear Interfaces**: Well-defined API contracts +- **Separation of Concerns**: Business logic separate from infrastructure + +#### Compliance-Ready Code Structure + +- **Documentation Standards**: Language-specific documentation required on ALL members for compliance +- **Error Handling**: Comprehensive error cases with appropriate logging +- **Configuration**: Externalize settings for different compliance environments +- **Traceability**: Code comments linking back to requirements where applicable + +### Quality Gate Verification + +Before completing any code changes, verify: + +#### 1. Code Quality Standards -- Small, focused functions with single responsibilities -- Dependency injection for external dependencies -- Avoid hidden state and side effects -- Clear separation of concerns +- [ ] Zero compiler warnings (`TreatWarningsAsErrors=true`) +- [ ] Follows `.editorconfig` and `.clang-format` formatting rules +- [ ] All code follows literate programming style +- [ ] Language-specific documentation complete on all members (XML for C#, Doxygen for C++) +- [ ] Passes static analysis (SonarQube, CodeQL, language analyzers) -### Template DotNet Tool-Specific Rules +#### 2. Testability & Design -- **XML Docs**: On ALL members (public/internal/private) with spaces after `///` - - Follow standard XML indentation rules with four-space indentation -- **Errors**: `ArgumentException` for parsing, `InvalidOperationException` for runtime issues -- **Namespace**: File-scoped namespaces only -- **Using Statements**: Top of file only -- **String Formatting**: Use interpolated strings ($"") for clarity +- [ ] Functions have single, clear responsibilities +- [ ] External dependencies are injectable/mockable +- [ ] Code is structured for unit testing +- [ ] Error handling covers expected failure scenarios +- [ ] Configuration externalized from business logic -### Self-Validation Tests +#### 3. Compliance Integration + +- [ ] Code supports requirements traceability +- [ ] Logging/telemetry appropriate for audit trails +- [ ] Security considerations addressed (input validation, authorization) +- [ ] Platform compatibility maintained for multi-platform requirements + +## Tool Integration Requirements + +### Required Development Tools + +- **Language Formatters**: Applied via `.editorconfig`, `.clang-format` +- **Static Analyzers**: Microsoft.CodeAnalysis.NetAnalyzers, SonarAnalyzer.CSharp +- **Security Scanning**: CodeQL integration for vulnerability detection +- **Documentation**: XML docs generation for API documentation + +### Code Quality Tools Integration + +- **SonarQube/SonarCloud**: Continuous code quality monitoring +- **Build Integration**: Warnings as errors enforcement +- **IDE Integration**: Real-time feedback on code quality issues +- **CI/CD Integration**: Automated quality gate enforcement + +## Cross-Agent Coordination + +### Hand-off to Other Agents + +- If comprehensive tests need to be created for implemented functionality, then call the @test-developer agent with the + **request** to create comprehensive tests for implemented functionality with **context** of new code changes and + **goal** of achieving adequate test coverage. +- If quality gates and linting requirements need verification, then call the @code-quality agent with the **request** + to verify all quality gates and linting requirements with **context** of completed implementation and **goal** of + compliance verification. +- If documentation needs updating to reflect code changes, then call the @technical-writer agent with the **request** + to update documentation reflecting code changes with **context** of specific implementation changes and + **additional instructions** for maintaining documentation currency. +- If implementation validation against requirements is needed, then call the @requirements agent with the **request** + to validate implementation satisfies requirements with **context** of completed functionality and **goal** of + requirements compliance verification. + +## Implementation Standards by Language + +### C# Development + +#### C# Documentation Standards + +- **XML Documentation**: Required on ALL members (public/internal/private) with spaces after `///` +- **Standard XML Tags**: Use ``, ``, ``, `` +- **Compliance**: XML docs support automated compliance documentation generation + +**Example:** + +```csharp +/// +/// Processes user input data according to business rules +/// +/// User input data to process +/// Processed result with validation status +/// Thrown when input is invalid +public ProcessingResult ProcessUserData(UserData userData) +{ + // Validate input parameters meet business rule constraints + if (!InputValidator.IsValid(userData)) + { + throw new ArgumentException("User data does not meet validation requirements"); + } + + // Apply business transformation logic + var transformedData = BusinessEngine.Transform(userData); + + // Return structured result with success indicators + return new ProcessingResult(transformedData, ProcessingStatus.Success); +} +``` + +### C++ Development + +#### C++ Documentation Standards + +- **Doxygen Documentation**: Required on ALL members (public/protected/private) +- **Standard Doxygen Tags**: Use `@brief`, `@param`, `@return`, `@throws` +- **Compliance**: Doxygen comments support automated API documentation and compliance reports + +**Example:** + +```cpp +/// @brief Processes sensor data and validates against specifications +/// @param sensorReading Raw sensor data from hardware interface +/// @return Processed measurement with validation status +/// @throws std::invalid_argument if sensor reading is out of range +ProcessedMeasurement ProcessSensorData(const SensorReading& sensorReading) +{ + // Validate sensor reading falls within expected operational range + if (!IsValidSensorReading(sensorReading)) + { + throw std::invalid_argument("Sensor reading outside valid operational range"); + } + + // Apply calibration and filtering algorithms + auto calibratedValue = CalibrationEngine::Apply(sensorReading); + + // Return measurement with quality indicators + return ProcessedMeasurement{calibratedValue, MeasurementQuality::Valid}; +} +``` -- Naming: `TemplateTool_FeatureBeingValidated` -- These tests ship with the product and run via `--validate` flag -- Must support TRX/JUnit output format -- Link to requirements in `requirements.yaml` +## Compliance Verification Checklist -## Defer To +### Before Completing Implementation -- **Requirements Agent**: For new requirement creation and test strategy -- **Test Developer Agent**: For unit and integration tests -- **Technical Writer Agent**: For documentation updates -- **Code Quality Agent**: For linting, formatting, and static analysis +1. **Code Quality**: Zero warnings, passes all static analysis +2. **Documentation**: Comprehensive XML documentation (C#) or Doxygen comments (C++) on ALL members +3. **Testability**: Code structured for comprehensive testing +4. **Security**: Input validation, error handling, authorization checks +5. **Traceability**: Implementation traceable to requirements +6. **Standards**: Follows all coding standards and formatting rules -## Don't +## Don't Do These Things -- Write code without explanatory comments -- Create large monolithic functions -- Skip XML documentation -- Ignore the literate programming style +- Skip literate programming comments (mandatory for all code) +- Disable compiler warnings to make builds pass +- Create untestable code with hidden dependencies +- Skip XML documentation (C#) or Doxygen comments (C++) on any members +- Implement functionality without requirement traceability +- Ignore static analysis or security scanning results +- Write monolithic functions with multiple responsibilities diff --git a/.github/agents/technical-writer.agent.md b/.github/agents/technical-writer.agent.md index b300631..0e1832e 100644 --- a/.github/agents/technical-writer.agent.md +++ b/.github/agents/technical-writer.agent.md @@ -5,64 +5,254 @@ tools: [read, search, edit, execute, github, agent] user-invocable: true --- -# Technical Writer - Template DotNet Tool +# Technical Writer Agent -Create and maintain clear, accurate, and complete documentation following best practices. +Create and maintain clear, accurate, and +compliance-ready documentation following regulatory best practices and Continuous Compliance standards. + +## Reporting + +If detailed documentation of writing and editing activities is needed, +create a report using the filename pattern `AGENT_REPORT_documentation.md` to document content changes, +style decisions, and editorial processes. ## When to Invoke This Agent -Invoke the technical-writer for: +Use the Technical Writer Agent for: + +- Creating and updating project documentation (README, guides, specifications) +- Ensuring documentation accuracy, completeness, and compliance +- Implementing regulatory documentation best practices +- Managing auto-generated compliance documentation +- Applying markdown linting and style standards + +## Primary Responsibilities + +### Continuous Compliance Documentation Standards + +#### Auto-Generated Documentation (CRITICAL - Do Not Edit Manually) + +```yaml +docs/ + requirements_doc/ + requirements.md # Generated by ReqStream + justifications.md # Generated by ReqStream + requirements_report/ + trace_matrix.md # Generated by ReqStream + build_notes.md # Generated by BuildMark + build_notes/ + versions.md # Generated by VersionMark + code_quality/ + sonar-quality.md # Generated by SonarMark + codeql-quality.md # Generated by SarifMark + code_review_plan/ + plan.md # Generated by ReviewMark + code_review_report/ + report.md # Generated by ReviewMark +``` + +**WARNING**: These files are regenerated on every CI/CD run. Manual edits will be lost. + +#### Project Documentation + +- **README.md**: Project overview, installation, usage +- **docs/*.md**: Architecture, design, user guides + +#### Code Documentation Coordination + +- **XML Documentation (C#)** and **Doxygen Comments (C++)**: Can be read and reviewed by @technical-writer agent for + accuracy and completeness +- **Code Comment Updates**: Must be performed by @software-developer agent, which maintains the proper formatting + rules and language-specific standards +- **Documentation Review**: @technical-writer agent verifies that code documentation aligns with overall project + documentation standard + +### Documentation Quality Standards + +#### Regulatory Documentation Excellence + +- **Purpose Statements**: Clear problem definition and document scope +- **Scope Boundaries**: Explicit inclusion/exclusion criteria +- **Traceability**: Links to requirements, tests, and implementation +- **Version Control**: Proper change tracking and approval workflows +- **Audience Targeting**: Appropriate detail level for intended readers + +#### Compliance-Ready Structure + +```markdown +# Document Title + +## Purpose + +[Why this document exists, what problem it solves] + +## Scope + +[What is covered, what is explicitly out of scope] + +## References + +[Links to related requirements, specifications, standards] + +# [Content sections organized logically] +``` + +#### Content Longevity Principles + +**Avoid Transitory Information**: Long-term documentation should not include information that becomes stale quickly: + +- **❌ Avoid**: Tool version numbers, specific counts (requirements, tests, files), current dates, "latest" references +- **❌ Examples**: "Currently using Node.js 18.2.1", "The system has 47 requirements", "As of March 2024" +- **✅ Instead**: Reference auto-generated reports, use relative descriptions, focus on stable concepts +- **✅ Examples**: "See docs/build_notes.md for current tool versions", "The requirements are organized by subsystem", + "The architecture follows..." + +**Exception**: Include transitory information only when documenting specific releases, version history, or +when the temporal context is the document's purpose. + +## Comprehensive Markdown & Documentation Standards + +### Link Style Rules by File Type + +#### Published Documents (README.md & Pandoc Document Structure) + +```markdown + +For more information, see [Continuous Compliance](https://github.com/demaconsulting/ContinuousCompliance). +Visit our website at https://docs.example.com/project-name +``` + +**CRITICAL**: Published documents (README.md and +any document in a Pandoc Document Structure) must use absolute URLs for all external links. +Relative links will break when documents are published, distributed as packages, or converted to PDF/other formats. + +**Published Document Types:** + +- README.md (shipped in packages and releases) +- Documents processed by Pandoc (typically in `docs/` with YAML frontmatter) +- Any document intended for standalone distribution + +#### AI Agent Files (`.github/agents/*.md`) + +```markdown + +For more information, see [Continuous Compliance](https://github.com/demaconsulting/ContinuousCompliance). +``` + +#### All Other Markdown Files + +```markdown + +For details, see the [Requirements Documentation][req-docs] and [Quality Standards][quality]. + +[req-docs]: https://github.com/demaconsulting/ContinuousCompliance/raw/refs/heads/main/docs/requirements.md +[quality]: https://github.com/demaconsulting/ContinuousCompliance/raw/refs/heads/main/docs/quality.md +``` + +### Documentation Linting Requirements + +Documentation formatting and spelling issues are automatically detected and reported by the project's lint scripts. +Run the repository's linting infrastructure to identify and resolve any documentation quality issues. + +### Pandoc Document Generation + +#### Pandoc Document Structure + +```yaml +docs/ + doc_folder/ + definition.yaml # Pandoc content definition + title.txt # Document metadata + introduction.md # Document introduction + sections/ # Individual content sections + sub-section.md # Sub-section document +``` + +#### Integration with CI/CD Pipeline + +```yaml +# Typical pipeline integration +- name: Generate Documentation + run: | + pandoc --metadata-file=docs/title.txt \ + --defaults=docs/definition.yaml \ + --output=docs/complete-document.pdf +``` + +### Diagram Integration Standards + +#### Mermaid Diagrams for Markdown + +Use **Mermaid diagrams** for all embedded diagrams in Markdown documents: -- Creating or updating project documentation (README, guides, CONTRIBUTING, etc.) -- Ensuring documentation accuracy and completeness -- Applying regulatory documentation best practices (purpose, scope statements) -- Special document types (architecture, design, user guides) -- Markdown and spell checking compliance +```mermaid +graph TD + A[User Request] --> B[Auth Service] + B --> C[Business Logic] + C --> D[Data Layer] + D --> E[Database] +``` -## Responsibilities +### Benefits of Mermaid Integration -### Documentation Best Practices +- **Version Control**: Diagrams stored as text, enabling proper diff tracking +- **Maintainability**: Easy to update diagrams alongside code changes +- **Consistency**: Standardized diagram styling across all documentation +- **Tooling Support**: Rendered automatically in GitHub, documentation sites, and modern editors +- **Accessibility**: Text-based format supports screen readers and accessibility tools -- **Purpose statements**: Why the document exists, what problem it solves -- **Scope statements**: What is covered and what is explicitly out of scope -- **Architecture docs**: System structure, component relationships, key design decisions -- **Design docs**: Implementation approach, algorithms, data structures -- **User guides**: Task-oriented, clear examples, troubleshooting +## Quality Gate Verification -### Template DotNet Tool-Specific Rules +### Documentation Linting Checklist -#### Markdown Style +- [ ] markdownlint-cli2 passes with zero errors +- [ ] cspell passes with zero spelling errors +- [ ] yamllint passes for any YAML content +- [ ] Links are functional and use correct style +- [ ] Generated documents compile without errors -- **All markdown files**: Use reference-style links `[text][ref]` with `[ref]: url` at document end -- **Exceptions**: - - **README.md**: Use absolute URLs in the links (shipped in NuGet package) - - **AI agent markdown files** (`.github/agents/*.md`): Use inline links `[text](url)` so URLs are visible in agent context -- Max 120 characters per line -- Lists require blank lines (MD032) +### Content Quality Standards -#### Linting Requirements +- [ ] Purpose and scope clearly defined +- [ ] Audience-appropriate detail level +- [ ] Traceability to requirements maintained +- [ ] Examples and code snippets tested +- [ ] Cross-references accurate and current -- **markdownlint**: Style and structure compliance -- **cspell**: Spelling (add technical terms to `.cspell.yaml`) -- **yamllint**: YAML file validation +## Cross-Agent Coordination -### Regulatory Documentation +### Hand-off to Other Agents -For documents requiring regulatory compliance: +- If code examples, API documentation, or code comments need updating, then call the @software-developer agent with + the **request** to update code examples, API documentation, and code comments (XML/Doxygen) with **context** of + documentation requirements and **additional instructions** for maintaining code-documentation consistency. +- If documentation linting and quality checks need to be run, then call the @code-quality agent with the **request** + to run documentation linting and quality checks with **context** of updated documentation and **goal** of compliance + verification. +- If test procedures and coverage need documentation, then call the @test-developer agent with the **request** to + document test procedures and coverage with **context** of current test suite and **goal** of comprehensive test + documentation. -- Clear purpose and scope sections -- Appropriate detail level for audience -- Traceability to requirements where applicable +## Compliance Verification Checklist -## Defer To +### Before Completing Documentation Work -- **Requirements Agent**: For requirements.yaml content and test linkage -- **Software Developer Agent**: For code examples and self-validation behavior -- **Test Developer Agent**: For test documentation -- **Code Quality Agent**: For running linters and fixing lint issues +1. **Linting**: All documentation passes markdownlint-cli2, cspell +2. **Structure**: Purpose and scope clearly defined +3. **Traceability**: Links to requirements, tests, code maintained +4. **Accuracy**: Content reflects current implementation +5. **Completeness**: All sections required for compliance included +6. **Generation**: Auto-generated docs compile successfully +7. **Links**: All references functional and use correct style +8. **Spelling**: Technical terms added to .cspell.yaml dictionary -## Don't +## Don't Do These Things -- Change code to match documentation (code is source of truth) -- Document non-existent features -- Skip linting before committing changes +- **Never edit auto-generated documentation** manually (will be overwritten) +- **Never edit code comments directly** (XML/Doxygen comments should be updated by @software-developer agent) +- **Never skip purpose and scope sections** in regulatory documents +- **Never ignore spelling errors** (add terms to .cspell.yaml instead) +- **Never use incorrect link styles** for file types (breaks tooling) +- **Never commit documentation** without linting verification +- **Never skip traceability links** in compliance-critical documents +- **Never document non-existent features** (code is source of truth) diff --git a/.github/agents/test-developer.agent.md b/.github/agents/test-developer.agent.md index 2ce95d9..0c7f94b 100644 --- a/.github/agents/test-developer.agent.md +++ b/.github/agents/test-developer.agent.md @@ -5,145 +5,295 @@ tools: [read, search, edit, execute, github, agent] user-invocable: true --- -# Test Developer - Template DotNet Tool +# Test Developer Agent -Develop comprehensive unit and integration tests following best practices. +Develop comprehensive unit and integration tests with emphasis on requirements coverage and +Continuous Compliance verification. + +## Reporting + +If detailed documentation of testing activities is needed, +create a report using the filename pattern `AGENT_REPORT_testing.md` to document test strategies, coverage analysis, +and validation results. ## When to Invoke This Agent -Invoke the test-developer for: +Use the Test Developer Agent for: + +- Creating unit tests for new functionality +- Writing integration tests for component interactions +- Improving test coverage for compliance requirements +- Implementing AAA (Arrange-Act-Assert) pattern tests +- Generating platform-specific test evidence +- Upgrading legacy test suites to modern standards + +## Primary Responsibilities + +### Comprehensive Test Coverage Strategy -- Creating unit tests for individual components -- Creating integration tests for cross-component behavior -- Improving test coverage -- Refactoring existing tests for clarity +#### Requirements Coverage (MANDATORY) -## Responsibilities +- **All requirements MUST have linked tests** - Enforced by ReqStream +- **Platform-specific tests** must generate evidence with source filters +- **Test result formats** must be compatible (TRX, JUnit XML) +- **Coverage tracking** for audit and compliance purposes -### AAA Pattern (Arrange-Act-Assert) +#### Test Type Strategy -All tests must follow the AAA pattern with clear sections: +- **Unit Tests**: Individual component/function behavior +- **Integration Tests**: Component interaction and data flow +- **Platform Tests**: Platform-specific functionality validation +- **Validation Tests**: Self-validation and compliance verification + +### AAA Pattern Implementation (MANDATORY) + +All tests MUST follow Arrange-Act-Assert pattern for clarity and maintainability: ```csharp [TestMethod] -public void ClassName_MethodUnderTest_Scenario_ExpectedBehavior() +public void UserService_CreateUser_ValidInput_ReturnsSuccessResult() { - // Arrange - Set up test conditions - var input = "test data"; - var expected = "expected result"; - var component = new Component(); + // Arrange - Set up test data and dependencies + var mockRepository = Substitute.For(); + var mockValidator = Substitute.For(); + var userService = new UserService(mockRepository, mockValidator); + var validUserData = new UserData + { + Name = "John Doe", + Email = "john@example.com" + }; + + // Act - Execute the system under test + var result = userService.CreateUser(validUserData); + + // Assert - Verify expected outcomes + Assert.IsTrue(result.IsSuccess); + Assert.AreEqual("John Doe", result.CreatedUser.Name); + mockRepository.Received(1).Save(Arg.Any()); +} +``` - // Act - Execute the behavior being tested - var actual = component.Method(input); +### Test Naming Standards - // Assert - Verify the results - Assert.AreEqual(expected, actual); -} +#### C# Test Naming + +```csharp +// Pattern: ClassName_MethodUnderTest_Scenario_ExpectedBehavior +UserService_CreateUser_ValidInput_ReturnsSuccessResult() +UserService_CreateUser_InvalidEmail_ThrowsArgumentException() +UserService_CreateUser_DuplicateUser_ReturnsFailureResult() ``` -### Test Documentation +#### C++ Test Naming -- Test name clearly states what is being tested and the scenario -- Comments document: - - What is being tested (the behavior/requirement) - - What the assertions prove (the expected outcome) - - Any non-obvious setup or conditions +```cpp +// Pattern: test_object_scenario_expected +test_user_service_valid_input_returns_success() +test_user_service_invalid_email_throws_exception() +test_user_service_duplicate_user_returns_failure() +``` -### Test Quality +## Quality Gate Verification -- Tests should be independent and isolated -- Each test verifies one behavior/scenario -- Use meaningful test data (avoid magic values) -- Clear failure messages for assertions -- Consider edge cases and error conditions +### Test Quality Standards -### Tests and Requirements +- [ ] All tests follow AAA pattern consistently +- [ ] Test names clearly describe scenario and expected outcome +- [ ] Each test validates single, specific behavior +- [ ] Both happy path and edge cases covered +- [ ] Platform-specific tests generate appropriate evidence +- [ ] Test results in standard formats (TRX, JUnit XML) -- **All requirements MUST have linked tests** - this is enforced in CI -- **Not all tests need requirements** - tests may be created for: - - Exploring corner cases not explicitly stated in requirements - - Testing design decisions and implementation details - - Failure-testing and error handling scenarios - - Verifying internal behavior beyond requirement scope +### Requirements Traceability -### Test Source Filters +- [ ] Tests linked to specific requirements in requirements.yaml +- [ ] Source filters applied for platform-specific requirements +- [ ] Test coverage adequate for all stated requirements +- [ ] ReqStream validation passes with linked tests -Test links in `requirements.yaml` can include a source filter prefix to restrict which test results count as -evidence. These filters are critical for platform and framework requirements - **do not remove them**. +### Test Framework Standards -- `windows@TestName` - proves the test passed on a Windows platform -- `ubuntu@TestName` - proves the test passed on a Linux (Ubuntu) platform -- `net8.0@TestName` - proves the test passed under the .NET 8 target framework -- `net9.0@TestName` - proves the test passed under the .NET 9 target framework -- `net10.0@TestName` - proves the test passed under the .NET 10 target framework -- `dotnet8.x@TestName` - proves the self-validation test ran on a machine with .NET 8.x runtime -- `dotnet9.x@TestName` - proves the self-validation test ran on a machine with .NET 9.x runtime -- `dotnet10.x@TestName` - proves the self-validation test ran on a machine with .NET 10.x runtime +#### C# Testing (MSTest V4) -Removing a source filter means a test result from any environment can satisfy the requirement, which invalidates -the evidence-based proof that the tool works on a specific platform or framework. +```csharp +[TestClass] +public class UserServiceTests +{ + private IUserRepository mockRepository; + private IValidator mockValidator; + + [TestInitialize] + public void Setup() + { + mockRepository = Substitute.For(); + mockValidator = Substitute.For(); + } + + [TestMethod] + public void UserService_ValidateUser_ValidData_ReturnsTrue() + { + // AAA implementation + } + + [TestCleanup] + public void Cleanup() + { + // Test cleanup if needed + } +} +``` -### Template DotNet Tool-Specific +#### C++ Testing (MSTest C++ / IAR Port) -- **NOT self-validation tests** - those are handled by Software Developer Agent -- Unit tests live in `test/` directory -- Use MSTest V4 testing framework -- Follow existing naming conventions in the test suite +```cpp +TEST_CLASS(UserServiceTests) +{ + TEST_METHOD(test_user_service_validate_user_valid_data_returns_true) + { + // Arrange - setup test data + UserService service; + UserData validData{"John Doe", "john@example.com"}; + + // Act - execute test + bool result = service.ValidateUser(validData); + + // Assert - verify results + Assert::IsTrue(result); + } +}; +``` -### MSTest V4 Best Practices +## Cross-Agent Coordination -Common anti-patterns to avoid (not exhaustive): +### Hand-off to Other Agents -1. **Avoid Assertions in Catch Blocks (MSTEST0058)** - Instead of wrapping code in try/catch and asserting in the - catch block, use `Assert.ThrowsExactly()`: +- If test quality gates and coverage metrics need verification, then call the @code-quality agent with the **request** + to verify test quality gates and coverage metrics with **context** of current test results and **goal** of meeting + coverage requirements. +- If test linkage needs to satisfy requirements traceability, then call the @requirements agent with the **request** + to ensure test linkage satisfies requirements traceability with **context** of test coverage and + **additional instructions** for maintaining traceability compliance. +- If testable code structure improvements are needed, then call the @software-developer agent with the **request** to + improve testable code structure with **context** of testing challenges and **goal** of enhanced testability. - ```csharp - var ex = Assert.ThrowsExactly(() => SomeWork()); - Assert.Contains("Some message", ex.Message); - ``` +## Testing Infrastructure Requirements -2. **Avoid using Assert.IsTrue / Assert.IsFalse for equality checks** - Use `Assert.AreEqual` / - `Assert.AreNotEqual` instead, as it provides better failure messages: +### Required Testing Tools - ```csharp - // ❌ Bad: Assert.IsTrue(result == expected); - // ✅ Good: Assert.AreEqual(expected, result); - ``` +```xml + + + + + + +``` -3. **Avoid non-public test classes and methods** - Test classes and `[TestMethod]` methods must be `public` or - they will be silently ignored: +### Test Result Generation - ```csharp - // ❌ Bad: internal class MyTests - // ✅ Good: public class MyTests - ``` +```bash +# Generate test results with coverage +dotnet test --collect:"XPlat Code Coverage" --logger trx --results-directory TestResults -4. **Avoid Assert.IsTrue(collection.Count == N)** - Use `Assert.HasCount` for count assertions: +# Platform-specific test execution +dotnet test --configuration Release --framework net8.0-windows --logger "trx;LogFileName=windows-tests.trx" +``` + +### CI/CD Integration + +```yaml +# Typical CI pipeline test stage +- name: Run Tests + run: | + dotnet test --configuration Release \ + --collect:"XPlat Code Coverage" \ + --logger trx \ + --results-directory TestResults \ + --verbosity normal + +- name: Upload Test Results + uses: actions/upload-artifact@v7 + with: + name: test-results + path: TestResults/**/*.trx +``` - ```csharp - // ❌ Bad: Assert.IsTrue(collection.Count == 3); - // ✅ Good: Assert.HasCount(3, collection); - ``` +## Test Development Patterns -5. **Avoid Assert.IsTrue for string prefix checks** - Use `Assert.StartsWith` instead of wrapping - `string.StartsWith` in `Assert.IsTrue`, as it produces clearer failure messages that show the expected prefix - and actual value: +### Comprehensive Test Coverage + +```csharp +[TestClass] +public class CalculatorTests +{ + [TestMethod] + public void Calculator_Add_PositiveNumbers_ReturnsSum() + { + // Happy path test + } + + [TestMethod] + public void Calculator_Add_NegativeNumbers_ReturnsSum() + { + // Edge case test + } + + [TestMethod] + public void Calculator_Divide_ByZero_ThrowsException() + { + // Error condition test + } + + [TestMethod] + public void Calculator_Divide_MaxValues_HandlesOverflow() + { + // Boundary condition test + } +} +``` + +### Mock and Dependency Testing + +```csharp +[TestMethod] +public void OrderService_ProcessOrder_ValidOrder_CallsPaymentService() +{ + // Arrange - Setup mocks and dependencies + var mockPaymentService = Substitute.For(); + var mockInventoryService = Substitute.For(); + var orderService = new OrderService(mockPaymentService, mockInventoryService); + + var testOrder = new Order { ProductId = 1, Quantity = 2, CustomerId = 123 }; + + // Act - Execute the system under test + var result = orderService.ProcessOrder(testOrder); + + // Assert - Verify interactions and outcomes + Assert.IsTrue(result.Success); + mockPaymentService.Received(1).ProcessPayment(Arg.Any()); + mockInventoryService.Received(1).ReserveItems(1, 2); +} +``` - ```csharp - // ❌ Bad: Assert.IsTrue(value.StartsWith("prefix")); - // ✅ Good: Assert.StartsWith("prefix", value); - ``` +## Compliance Verification Checklist -## Defer To +### Before Completing Test Work -- **Requirements Agent**: For test strategy and coverage requirements -- **Software Developer Agent**: For self-validation tests and production code issues -- **Technical Writer Agent**: For test documentation in markdown -- **Code Quality Agent**: For test linting and static analysis +1. **AAA Pattern**: All tests follow Arrange-Act-Assert structure consistently +2. **Naming**: Test names clearly describe scenario and expected behavior +3. **Coverage**: Requirements coverage adequate, platform tests have source filters +4. **Quality**: Tests pass consistently, no flaky or unreliable tests +5. **Documentation**: Test intent and coverage clearly documented +6. **Integration**: Test results compatible with ReqStream and CI/CD pipeline +7. **Standards**: Follows framework-specific testing patterns and conventions -## Don't +## Don't Do These Things -- Write tests that test multiple behaviors in one test -- Skip test documentation -- Create brittle tests with tight coupling to implementation details -- Write self-validation tests (delegate to Software Developer Agent) +- **Never skip AAA pattern** in test structure (mandatory for consistency) +- **Never create tests without clear names** (must describe scenario/expectation) +- **Never write flaky tests** that pass/fail inconsistently +- **Never test implementation details** (test behavior, not internal mechanics) +- **Never skip edge cases** and error conditions +- **Never create tests without requirements linkage** (for compliance requirements) +- **Never ignore platform-specific test evidence** requirements +- **Never commit failing tests** (all tests must pass before merge) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d7893db..8d64b2d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -491,16 +491,16 @@ jobs: dotnet reqstream --requirements requirements.yaml --tests "artifacts/**/*.trx" - --report docs/requirements/requirements.md - --justifications docs/justifications/justifications.md - --matrix docs/tracematrix/tracematrix.md + --report docs/requirements_doc/requirements.md + --justifications docs/requirements_doc/justifications.md + --matrix docs/requirements_report/trace_matrix.md --enforce - name: Generate CodeQL Quality Report with SarifMark run: > dotnet sarifmark --sarif artifacts/csharp.sarif - --report docs/quality/codeql-quality.md + --report docs/code_quality/codeql-quality.md --heading "ReviewMark CodeQL Analysis" --report-depth 1 @@ -508,7 +508,7 @@ jobs: shell: bash run: | echo "=== CodeQL Quality Report ===" - cat docs/quality/codeql-quality.md + cat docs/code_quality/codeql-quality.md - name: Generate SonarCloud Quality Report shell: bash @@ -520,14 +520,37 @@ jobs: --project-key demaconsulting_ReviewMark --branch ${{ github.ref_name }} --token "$SONAR_TOKEN" - --report docs/quality/sonar-quality.md + --report docs/code_quality/sonar-quality.md --report-depth 1 - name: Display SonarCloud Quality Report shell: bash run: | echo "=== SonarCloud Quality Report ===" - cat docs/quality/sonar-quality.md + cat docs/code_quality/sonar-quality.md + + - name: Generate Review Plan and Review Report with ReviewMark + shell: bash + # TODO: Add --enforce once reviews branch is populated with review evidence PDFs and index.json + run: > + reviewmark + --definition .reviewmark.yaml + --plan docs/code_review_plan/plan.md + --plan-depth 1 + --report docs/code_review_report/report.md + --report-depth 1 + + - name: Display Review Plan + shell: bash + run: | + echo "=== Review Plan ===" + cat docs/code_review_plan/plan.md + + - name: Display Review Report + shell: bash + run: | + echo "=== Review Report ===" + cat docs/code_review_report/report.md - name: Generate Build Notes with BuildMark shell: bash @@ -536,20 +559,20 @@ jobs: run: > dotnet buildmark --build-version ${{ inputs.version }} - --report docs/buildnotes.md + --report docs/build_notes.md --report-depth 1 - name: Display Build Notes Report shell: bash run: | echo "=== Build Notes Report ===" - cat docs/buildnotes.md + cat docs/build_notes.md - name: Publish Tool Versions shell: bash run: | echo "Publishing tool versions..." - dotnet versionmark --publish --report docs/buildnotes/versions.md --report-depth 1 \ + dotnet versionmark --publish --report docs/build_notes/versions.md --report-depth 1 \ -- "artifacts/**/versionmark-*.json" echo "✓ Tool versions published" @@ -557,7 +580,7 @@ jobs: shell: bash run: | echo "=== Tool Versions Report ===" - cat docs/buildnotes/versions.md + cat docs/build_notes/versions.md # === GENERATE HTML DOCUMENTS WITH PANDOC === # This section converts markdown documents to HTML using Pandoc. @@ -567,11 +590,11 @@ jobs: shell: bash run: > dotnet pandoc - --defaults docs/buildnotes/definition.yaml + --defaults docs/build_notes/definition.yaml --filter node_modules/.bin/mermaid-filter.cmd --metadata version="${{ inputs.version }}" --metadata date="$(date +'%Y-%m-%d')" - --output docs/buildnotes/buildnotes.html + --output docs/build_notes/buildnotes.html - name: Generate Guide HTML with Pandoc shell: bash @@ -587,41 +610,51 @@ jobs: shell: bash run: > dotnet pandoc - --defaults docs/quality/definition.yaml + --defaults docs/code_quality/definition.yaml --filter node_modules/.bin/mermaid-filter.cmd --metadata version="${{ inputs.version }}" --metadata date="$(date +'%Y-%m-%d')" - --output docs/quality/quality.html + --output docs/code_quality/quality.html - name: Generate Requirements HTML with Pandoc shell: bash run: > dotnet pandoc - --defaults docs/requirements/definition.yaml + --defaults docs/requirements_doc/definition.yaml --filter node_modules/.bin/mermaid-filter.cmd --metadata version="${{ inputs.version }}" --metadata date="$(date +'%Y-%m-%d')" - --output docs/requirements/requirements.html + --output docs/requirements_doc/requirements.html - - name: Generate Requirements Justifications HTML with Pandoc + - name: Generate Trace Matrix HTML with Pandoc shell: bash run: > dotnet pandoc - --defaults docs/justifications/definition.yaml + --defaults docs/requirements_report/definition.yaml --filter node_modules/.bin/mermaid-filter.cmd --metadata version="${{ inputs.version }}" --metadata date="$(date +'%Y-%m-%d')" - --output docs/justifications/justifications.html + --output docs/requirements_report/trace_matrix.html - - name: Generate Trace Matrix HTML with Pandoc + - name: Generate Review Plan HTML with Pandoc shell: bash run: > dotnet pandoc - --defaults docs/tracematrix/definition.yaml + --defaults docs/code_review_plan/definition.yaml --filter node_modules/.bin/mermaid-filter.cmd --metadata version="${{ inputs.version }}" --metadata date="$(date +'%Y-%m-%d')" - --output docs/tracematrix/tracematrix.html + --output docs/code_review_plan/plan.html + + - name: Generate Review Report HTML with Pandoc + shell: bash + run: > + dotnet pandoc + --defaults docs/code_review_report/definition.yaml + --filter node_modules/.bin/mermaid-filter.cmd + --metadata version="${{ inputs.version }}" + --metadata date="$(date +'%Y-%m-%d')" + --output docs/code_review_report/report.html # === GENERATE PDF DOCUMENTS WITH WEASYPRINT === # This section converts HTML documents to PDF using Weasyprint. @@ -631,7 +664,7 @@ jobs: run: > dotnet weasyprint --pdf-variant pdf/a-3u - docs/buildnotes/buildnotes.html + docs/build_notes/buildnotes.html "docs/ReviewMark Build Notes.pdf" - name: Generate Guide PDF with Weasyprint @@ -645,29 +678,36 @@ jobs: run: > dotnet weasyprint --pdf-variant pdf/a-3u - docs/quality/quality.html + docs/code_quality/quality.html "docs/ReviewMark Code Quality.pdf" - name: Generate Requirements PDF with Weasyprint run: > dotnet weasyprint --pdf-variant pdf/a-3u - docs/requirements/requirements.html + docs/requirements_doc/requirements.html "docs/ReviewMark Requirements.pdf" - - name: Generate Requirements Justifications PDF with Weasyprint + - name: Generate Trace Matrix PDF with Weasyprint run: > dotnet weasyprint --pdf-variant pdf/a-3u - docs/justifications/justifications.html - "docs/ReviewMark Requirements Justifications.pdf" + docs/requirements_report/trace_matrix.html + "docs/ReviewMark Trace Matrix.pdf" - - name: Generate Trace Matrix PDF with Weasyprint + - name: Generate Review Plan PDF with Weasyprint run: > dotnet weasyprint --pdf-variant pdf/a-3u - docs/tracematrix/tracematrix.html - "docs/ReviewMark Trace Matrix.pdf" + docs/code_review_plan/plan.html + "docs/ReviewMark Review Plan.pdf" + + - name: Generate Review Report PDF with Weasyprint + run: > + dotnet weasyprint + --pdf-variant pdf/a-3u + docs/code_review_report/report.html + "docs/ReviewMark Review Report.pdf" # === UPLOAD ARTIFACTS === # This section uploads all generated documentation artifacts. @@ -679,4 +719,4 @@ jobs: name: documents path: | docs/*.pdf - docs/buildnotes.md + docs/build_notes.md diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9a81642..842250d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -63,18 +63,13 @@ jobs: name: documents path: artifacts - - name: Move buildnotes.md to root - run: | - set -e - mv artifacts/buildnotes.md buildnotes.md - - name: Create GitHub Release if: inputs.publish == 'release' || inputs.publish == 'publish' uses: ncipollo/release-action@v1 with: tag: ${{ inputs.version }} artifacts: artifacts/* - bodyFile: buildnotes.md + bodyFile: artifacts/build_notes.md generateReleaseNotes: false - name: Publish to NuGet.org diff --git a/.gitignore b/.gitignore index ec91165..48dc886 100644 --- a/.gitignore +++ b/.gitignore @@ -91,13 +91,15 @@ __pycache__/ docs/**/*.html docs/**/*.pdf !docs/template/** -docs/requirements/requirements.md -docs/justifications/justifications.md -docs/tracematrix/tracematrix.md -docs/quality/codeql-quality.md -docs/quality/sonar-quality.md -docs/buildnotes.md -docs/buildnotes/versions.md +docs/requirements_doc/requirements.md +docs/requirements_doc/justifications.md +docs/requirements_report/trace_matrix.md +docs/code_quality/codeql-quality.md +docs/code_quality/sonar-quality.md +docs/code_review_plan/plan.md +docs/code_review_report/report.md +docs/build_notes.md +docs/build_notes/versions.md # Test results TestResults/ diff --git a/.reviewmark.yaml b/.reviewmark.yaml new file mode 100644 index 0000000..d964e3c --- /dev/null +++ b/.reviewmark.yaml @@ -0,0 +1,80 @@ +--- +# ReviewMark Configuration File +# This file defines which files require review, where the evidence store is located, +# and how files are grouped into named review-sets following software unit boundaries. + +# Patterns identifying all files that require review. +# Processed in order; prefix a pattern with '!' to exclude. +needs-review: + - "**/*.cs" # All C# source and test files + - "docs/reqstream/*.yaml" # Per-software-item requirements files + - "!**/obj/**" # Exclude build output + - "!**/bin/**" # Exclude build output + +# Evidence source: review data and index.json are located in the 'reviews' branch +# of this repository, accessed through the GitHub public HTTPS raw content access. +# Note: The 'reviews' branch must be created and populated with review evidence PDFs +# and an index.json before enforcement (--enforce flag) can be enabled in the pipeline. +evidence-source: + type: url + location: https://raw.githubusercontent.com/demaconsulting/ReviewMark/reviews/index.json + +# Review sets grouping files by software unit. +# Each review-set groups requirements, source, and tests for a coherent software unit +# so that an AI-assisted review can verify consistency across the full evidence chain: +# - requirements: what the code must do and why +# - source: what the code actually does +# - tests: which behaviors are verified and how +reviews: + # Software unit reviews - one per class + - id: ReviewMark-Context + title: Review of Context software unit (command-line argument handling) + paths: + - "docs/reqstream/cli-requirements.yaml" # requirements + - "src/**/Context.cs" # implementation + - "test/**/ContextTests.cs" # tests + + - id: ReviewMark-GlobMatcher + title: Review of GlobMatcher software unit (file pattern matching) + paths: + - "src/**/GlobMatcher.cs" # implementation + - "test/**/GlobMatcherTests.cs" # tests + + - id: ReviewMark-Index + title: Review of Index software unit (review evidence indexing) + paths: + - "docs/reqstream/index-requirements.yaml" # requirements + - "src/**/Index.cs" # implementation + - "test/**/IndexTests.cs" # tests + + - id: ReviewMark-PathHelpers + title: Review of PathHelpers software unit (file path utilities) + paths: + - "src/**/PathHelpers.cs" # implementation + - "test/**/PathHelpersTests.cs" # tests + + - id: ReviewMark-Program + title: Review of Program software unit (main entry point and tool orchestration) + paths: + - "docs/reqstream/cli-requirements.yaml" # requirements + - "docs/reqstream/platform-requirements.yaml" # platform requirements + - "docs/guide/guide.md" # user guide + - "src/**/Program.cs" # implementation + - "test/**/ProgramTests.cs" # unit tests + - "test/**/IntegrationTests.cs" # integration tests + - "test/**/Runner.cs" # test infrastructure + - "test/**/TestDirectory.cs" # test infrastructure + - "test/**/AssemblyInfo.cs" # test infrastructure + + - id: ReviewMark-Configuration + title: Review of ReviewMarkConfiguration software unit (configuration parsing and processing) + paths: + - "docs/reqstream/configuration-requirements.yaml" # requirements + - "src/**/ReviewMarkConfiguration.cs" # implementation + - "test/**/ReviewMarkConfigurationTests.cs" # tests + + - id: ReviewMark-Validation + title: Review of Validation software unit (self-validation test execution) + paths: + - "docs/reqstream/ots-requirements.yaml" # OTS requirements verified by self-validation + - "src/**/Validation.cs" # implementation diff --git a/AGENTS.md b/AGENTS.md index 7e4fcbe..c0d6359 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,30 +1,53 @@ # Agent Quick Reference -Project-specific guidance for agents working on ReviewMark - a tool for automated +Comprehensive guidance for AI agents working on ReviewMark - a tool for automated file-review evidence management in regulated environments. ## Available Specialized Agents -- **Requirements Agent** - Develops requirements and ensures test coverage linkage -- **Technical Writer** - Creates accurate documentation following regulatory best practices -- **Software Developer** - Writes production code and self-validation tests in literate style -- **Test Developer** - Creates unit and integration tests following AAA pattern -- **Code Quality Agent** - Enforces linting, static analysis, and security standards -- **Repo Consistency Agent** - Ensures downstream repositories remain consistent with template patterns -- **Code Review Agent** - Assists in performing formal file reviews - -## Agent Selection Guide - -- Fix a bug → **Software Developer** -- Add a new feature → **Requirements Agent** → **Software Developer** → **Test Developer** -- Write a test → **Test Developer** -- Fix linting or static analysis issues → **Code Quality Agent** -- Update documentation → **Technical Writer** -- Add or update requirements → **Requirements Agent** -- Ensure test coverage linkage in `requirements.yaml` → **Requirements Agent** -- Run security scanning or address CodeQL alerts → **Code Quality Agent** -- Propagate template changes → **Repo Consistency Agent** -- Perform file reviews → **Code Review Agent** +- **requirements** - Develops requirements and ensures test coverage linkage +- **technical-writer** - Creates accurate documentation following regulatory best practices +- **software-developer** - Writes production code and self-validation tests in literate style +- **test-developer** - Creates unit and integration tests following AAA pattern +- **code-quality** - Enforces linting, static analysis, and security standards +- **code-review** - Assists in performing formal file reviews +- **repo-consistency** - Ensures downstream repositories remain consistent with template patterns + +## Agent Selection + +- To fix a bug, call the @software-developer agent with the **context** of the bug details and **goal** of resolving + the issue while maintaining code quality. +- To add a new feature, call the @requirements agent with the **request** to define feature requirements and **context** + of business needs and **goal** of comprehensive requirement specification. +- To write or fix tests, call the @test-developer agent with the **context** of the functionality to be tested and + **goal** of achieving comprehensive test coverage. +- To update documentation, call the @technical-writer agent with the **context** of changes requiring documentation and + **goal** of maintaining current and accurate documentation. +- To manage requirements and traceability, call the @requirements agent with the **context** of requirement changes and + **goal** of maintaining compliance traceability. +- To resolve quality or linting issues, call the @code-quality agent with the **context** of quality gate failures and + **goal** of achieving compliance standards. +- To update linting tools or scripts, call the @code-quality agent with the **context** of tool requirements and + **goal** of maintaining quality infrastructure. +- To address security alerts or scanning issues, call the @code-quality agent with the **context** of security findings + and **goal** of resolving vulnerabilities. +- To perform file reviews, call the @code-review agent with the **context** of files requiring review and **goal** of + compliance verification. +- To ensure template consistency, call the @repo-consistency agent with the **context** of downstream repository + and **goal** of maintaining template alignment. + +## Quality Gate Enforcement (ALL Agents Must Verify) + +Configuration files and scripts are self-documenting with their design intent and +modification policies in header comments. + +1. **Linting Standards**: `./lint.sh` (Unix) or `lint.bat` (Windows) - comprehensive linting suite +2. **Build Quality**: Zero warnings (`TreatWarningsAsErrors=true`) +3. **Static Analysis**: SonarQube/CodeQL passing with no blockers +4. **Requirements Traceability**: `dotnet reqstream --enforce` passing +5. **Test Coverage**: All requirements linked to passing tests +6. **Documentation Currency**: All docs current and generated +7. **File Review Status**: All reviewable files have current reviews ## Tech Stack diff --git a/docs/build_notes/definition.yaml b/docs/build_notes/definition.yaml new file mode 100644 index 0000000..207a375 --- /dev/null +++ b/docs/build_notes/definition.yaml @@ -0,0 +1,12 @@ +--- +resource-path: + - docs/build_notes + - docs/template +input-files: + - docs/build_notes/title.txt + - docs/build_notes/introduction.md + - docs/build_notes.md + - docs/build_notes/versions.md +template: template.html +table-of-contents: true +number-sections: true diff --git a/docs/buildnotes/introduction.md b/docs/build_notes/introduction.md similarity index 100% rename from docs/buildnotes/introduction.md rename to docs/build_notes/introduction.md diff --git a/docs/buildnotes/title.txt b/docs/build_notes/title.txt similarity index 100% rename from docs/buildnotes/title.txt rename to docs/build_notes/title.txt diff --git a/docs/buildnotes/definition.yaml b/docs/buildnotes/definition.yaml deleted file mode 100644 index 62699f2..0000000 --- a/docs/buildnotes/definition.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -resource-path: - - docs/buildnotes - - docs/template -input-files: - - docs/buildnotes/title.txt - - docs/buildnotes/introduction.md - - docs/buildnotes.md - - docs/buildnotes/versions.md -template: template.html -table-of-contents: true -number-sections: true diff --git a/docs/code_quality/definition.yaml b/docs/code_quality/definition.yaml new file mode 100644 index 0000000..68c58f2 --- /dev/null +++ b/docs/code_quality/definition.yaml @@ -0,0 +1,12 @@ +--- +resource-path: + - docs/code_quality + - docs/template +input-files: + - docs/code_quality/title.txt + - docs/code_quality/introduction.md + - docs/code_quality/codeql-quality.md + - docs/code_quality/sonar-quality.md +template: template.html +table-of-contents: true +number-sections: true diff --git a/docs/quality/introduction.md b/docs/code_quality/introduction.md similarity index 100% rename from docs/quality/introduction.md rename to docs/code_quality/introduction.md diff --git a/docs/quality/title.txt b/docs/code_quality/title.txt similarity index 100% rename from docs/quality/title.txt rename to docs/code_quality/title.txt diff --git a/docs/code_review_plan/definition.yaml b/docs/code_review_plan/definition.yaml new file mode 100644 index 0000000..3a24f0b --- /dev/null +++ b/docs/code_review_plan/definition.yaml @@ -0,0 +1,11 @@ +--- +resource-path: + - docs/code_review_plan + - docs/template +input-files: + - docs/code_review_plan/title.txt + - docs/code_review_plan/introduction.md + - docs/code_review_plan/plan.md +template: template.html +table-of-contents: true +number-sections: true diff --git a/docs/code_review_plan/introduction.md b/docs/code_review_plan/introduction.md new file mode 100644 index 0000000..807b8c4 --- /dev/null +++ b/docs/code_review_plan/introduction.md @@ -0,0 +1,33 @@ +# Introduction + +This document contains the review plan for the ReviewMark project. + +## Purpose + +This review plan provides a comprehensive overview of all files requiring formal review +in the ReviewMark project. It identifies which review-sets cover which +files and serves as evidence that every file requiring review is covered by at least +one named review-set. + +## Scope + +This review plan covers: + +- C# source code files requiring formal review +- YAML configuration and requirements files requiring formal review +- Mapping of reviewed files to named review-sets + +## Generation Source + +This plan is automatically generated by the ReviewMark tool, analyzing the +`.reviewmark.yaml` configuration and the review evidence store. It serves as evidence +that every file requiring review is covered by a current, valid review. + +## Audience + +This document is intended for: + +- Software developers working on ReviewMark +- Quality assurance teams validating review coverage +- Project stakeholders reviewing compliance status +- Auditors verifying that all required files have been reviewed diff --git a/docs/code_review_plan/title.txt b/docs/code_review_plan/title.txt new file mode 100644 index 0000000..bfe74cd --- /dev/null +++ b/docs/code_review_plan/title.txt @@ -0,0 +1,13 @@ +--- +title: ReviewMark Review Plan +subtitle: File Review Plan for ReviewMark +author: DEMA Consulting +description: File Review Plan for ReviewMark +lang: en-US +keywords: + - ReviewMark + - Review Plan + - File Reviews + - .NET + - Tool +--- diff --git a/docs/code_review_report/definition.yaml b/docs/code_review_report/definition.yaml new file mode 100644 index 0000000..6498e6c --- /dev/null +++ b/docs/code_review_report/definition.yaml @@ -0,0 +1,11 @@ +--- +resource-path: + - docs/code_review_report + - docs/template +input-files: + - docs/code_review_report/title.txt + - docs/code_review_report/introduction.md + - docs/code_review_report/report.md +template: template.html +table-of-contents: true +number-sections: true diff --git a/docs/code_review_report/introduction.md b/docs/code_review_report/introduction.md new file mode 100644 index 0000000..a669629 --- /dev/null +++ b/docs/code_review_report/introduction.md @@ -0,0 +1,32 @@ +# Introduction + +This document contains the review report for the ReviewMark project. + +## Purpose + +This review report provides evidence that each review-set is current — the review +evidence matches the current file fingerprints. It confirms that all formal reviews +conducted for ReviewMark remain valid for the current state of the reviewed files. + +## Scope + +This review report covers: + +- Current review-set status (current, stale, failed, or missing) +- File fingerprints and review evidence matching +- Review coverage verification + +## Generation Source + +This report is automatically generated by the ReviewMark tool, comparing the current +file fingerprints against the review evidence store. It serves as evidence that all +review-sets are current and no reviewed file has changed since its review was conducted. + +## Audience + +This document is intended for: + +- Software developers working on ReviewMark +- Quality assurance teams validating review currency +- Project stakeholders reviewing compliance status +- Auditors verifying that all reviews remain valid for the current release diff --git a/docs/code_review_report/title.txt b/docs/code_review_report/title.txt new file mode 100644 index 0000000..3a8f95b --- /dev/null +++ b/docs/code_review_report/title.txt @@ -0,0 +1,13 @@ +--- +title: ReviewMark Review Report +subtitle: File Review Report for ReviewMark +author: DEMA Consulting +description: File Review Report for ReviewMark +lang: en-US +keywords: + - ReviewMark + - Review Report + - File Reviews + - .NET + - Tool +--- diff --git a/docs/justifications/definition.yaml b/docs/justifications/definition.yaml deleted file mode 100644 index d0bbbee..0000000 --- a/docs/justifications/definition.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -resource-path: - - docs/justifications - - docs/template -input-files: - - docs/justifications/title.txt - - docs/justifications/introduction.md - - docs/justifications/justifications.md -template: template.html -table-of-contents: true -number-sections: true diff --git a/docs/justifications/introduction.md b/docs/justifications/introduction.md deleted file mode 100644 index 33593b1..0000000 --- a/docs/justifications/introduction.md +++ /dev/null @@ -1,29 +0,0 @@ -# Introduction - -This document contains the justifications for the requirements of the ReviewMark project. - -## Purpose - -This justifications document provides the rationale behind each requirement in the ReviewMark -project. Each requirement justification explains why the requirement exists, what problem it -solves, and how it contributes to the overall value of the tool. - -## Scope - -This document covers justifications for: - -- Command-line interface requirements -- Self-validation framework requirements -- Test result output requirements -- Logging requirements -- Platform support requirements -- Documentation generation requirements - -## Audience - -This document is intended for: - -- Software developers understanding design decisions -- Quality assurance teams reviewing requirement rationale -- Project stakeholders evaluating project scope -- Compliance and audit teams reviewing requirements traceability diff --git a/docs/justifications/title.txt b/docs/justifications/title.txt deleted file mode 100644 index 6cd0b1d..0000000 --- a/docs/justifications/title.txt +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: ReviewMark Requirements Justifications -subtitle: Requirements Justifications for the ReviewMark -author: DEMA Consulting -description: Requirements Justifications for the ReviewMark -lang: en-US -keywords: - - ReviewMark - - Requirements - - Justifications - - .NET - - Documentation ---- diff --git a/docs/quality/definition.yaml b/docs/quality/definition.yaml deleted file mode 100644 index 1b63510..0000000 --- a/docs/quality/definition.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -resource-path: - - docs/quality - - docs/template -input-files: - - docs/quality/title.txt - - docs/quality/introduction.md - - docs/quality/codeql-quality.md - - docs/quality/sonar-quality.md -template: template.html -table-of-contents: true -number-sections: true diff --git a/docs/reqstream/cli-requirements.yaml b/docs/reqstream/cli-requirements.yaml new file mode 100644 index 0000000..71e9bdc --- /dev/null +++ b/docs/reqstream/cli-requirements.yaml @@ -0,0 +1,243 @@ +--- +# Command-Line Interface Subsystem Requirements +# +# PURPOSE: +# - Define requirements for the ReviewMark command-line interface subsystem +# - The CLI subsystem spans Context.cs (argument parsing) and Program.cs (orchestration) +# - Subsystem requirements describe the externally visible CLI behavior + +sections: + - title: Command-Line Interface Subsystem Requirements + requirements: + - id: ReviewMark-Cmd-Context + title: The tool shall implement a Context class for command-line argument handling. + justification: | + Provides a standardized approach to command-line argument parsing and output + handling across all DEMA Consulting DotNet Tools. + tests: + - Context_Create_NoArguments_ReturnsDefaultContext + - Context_Create_VersionFlag_SetsVersionTrue + - Context_Create_HelpFlag_SetsHelpTrue + - Context_Create_SilentFlag_SetsSilentTrue + - Context_Create_ValidateFlag_SetsValidateTrue + - Context_Create_ResultsFlag_SetsResultsFile + - Context_Create_LogFlag_OpensLogFile + + - id: ReviewMark-Cmd-Version + title: The tool shall support -v and --version flags to display version information. + justification: | + Users need to quickly identify the version of the tool they are using for + troubleshooting and compatibility verification. + tests: + - Context_Create_VersionFlag_SetsVersionTrue + - Context_Create_ShortVersionFlag_SetsVersionTrue + - Program_Run_WithVersionFlag_DisplaysVersionOnly + - Program_Version_ReturnsNonEmptyString + - IntegrationTest_VersionFlag_OutputsVersion + + - id: ReviewMark-Cmd-Help + title: The tool shall support -?, -h, and --help flags to display usage information. + justification: | + Users need access to command-line usage documentation without requiring + external resources. + tests: + - Context_Create_HelpFlag_SetsHelpTrue + - Context_Create_ShortHelpFlag_H_SetsHelpTrue + - Context_Create_ShortHelpFlag_Question_SetsHelpTrue + - Program_Run_WithHelpFlag_DisplaysUsageInformation + - IntegrationTest_HelpFlag_OutputsUsageInformation + + - id: ReviewMark-Cmd-Silent + title: The tool shall support --silent flag to suppress console output. + justification: | + Enables automated scripts and CI/CD pipelines to run the tool without + cluttering output logs. + tests: + - Context_Create_SilentFlag_SetsSilentTrue + - Context_WriteLine_Silent_DoesNotWriteToConsole + - IntegrationTest_SilentFlag_SuppressesOutput + + - id: ReviewMark-Cmd-Validate + title: The tool shall support --validate flag to run self-validation tests. + justification: | + Provides a built-in mechanism to verify the tool is functioning correctly + in the deployment environment. + tests: + - Context_Create_ValidateFlag_SetsValidateTrue + - Program_Run_WithValidateFlag_RunsValidation + - IntegrationTest_ValidateFlag_RunsValidation + + - id: ReviewMark-Cmd-Results + title: The tool shall support --results flag to write validation results in TRX or JUnit format. + justification: | + Enables integration with CI/CD systems that expect standard test result formats. + tests: + - Context_Create_ResultsFlag_SetsResultsFile + - IntegrationTest_ValidateWithResults_GeneratesTrxFile + - IntegrationTest_ValidateWithResults_GeneratesJUnitFile + + - id: ReviewMark-Cmd-Log + title: The tool shall support --log flag to write output to a log file. + justification: | + Provides persistent logging for debugging and audit trails. + tests: + - Context_Create_LogFlag_OpensLogFile + - IntegrationTest_LogFlag_WritesOutputToFile + + - id: ReviewMark-Cmd-ErrorOutput + title: The tool shall write error messages to stderr. + justification: | + Error messages must be written to stderr so they remain visible to the user + without polluting stdout, which consumers may pipe or redirect for data capture. + tests: + - Context_WriteError_NotSilent_WritesToConsole + - IntegrationTest_UnknownArgument_ReturnsError + + - id: ReviewMark-Cmd-InvalidArgs + title: The tool shall reject unknown or malformed command-line arguments with a descriptive error. + justification: | + Providing clear feedback for invalid arguments helps users quickly correct + mistakes and prevents silent misconfiguration. + tests: + - Context_Create_UnknownArgument_ThrowsArgumentException + - Context_Create_LogFlag_WithoutValue_ThrowsArgumentException + - Context_Create_ResultsFlag_WithoutValue_ThrowsArgumentException + - IntegrationTest_UnknownArgument_ReturnsError + + - id: ReviewMark-Cmd-ExitCode + title: The tool shall return a non-zero exit code on failure. + justification: | + Callers (scripts, CI/CD pipelines) must be able to detect failure conditions + programmatically via the process exit code. + tests: + - Context_WriteError_SetsErrorExitCode + - IntegrationTest_UnknownArgument_ReturnsError + + - id: ReviewMark-Cmd-Definition + title: The tool shall support --definition flag to specify the definition YAML file. + justification: | + Users must be able to specify the path to the .reviewmark.yaml definition file, + which configures needs-review patterns, evidence source, and review set definitions. + tests: + - Context_Create_DefinitionFlag_SetsDefinitionFile + - Context_Create_DefinitionFlag_WithoutValue_ThrowsArgumentException + - ReviewMark_ReviewPlanGeneration + - ReviewMark_ReviewReportGeneration + + - id: ReviewMark-Cmd-Plan + title: The tool shall support --plan flag to write the review plan to a Markdown file. + justification: | + Enables automated generation of a review plan document that lists all review sets + and coverage status, suitable for inclusion in release documentation. + tests: + - Context_Create_PlanFlag_SetsPlanFile + - ReviewMark_ReviewPlanGeneration + + - id: ReviewMark-Cmd-PlanDepth + title: The tool shall support --plan-depth flag to set the Markdown heading depth for the review plan. + justification: | + Allows the review plan to be embedded at any heading level within a larger + Markdown document, with a default depth of 1 when not specified. + tests: + - Context_Create_PlanDepthFlag_SetsPlanDepth + - Context_Create_PlanDepthFlag_WithInvalidValue_ThrowsArgumentException + - Context_Create_PlanDepthFlag_WithZeroValue_ThrowsArgumentException + - Context_Create_NoArguments_PlanDepthDefaultsToOne + + - id: ReviewMark-Cmd-Report + title: The tool shall support --report flag to write the review report to a Markdown file. + justification: | + Enables automated generation of a review report document showing the current + status of each review set against the evidence index, suitable for release documentation. + tests: + - Context_Create_ReportFlag_SetsReportFile + - ReviewMark_ReviewReportGeneration + + - id: ReviewMark-Cmd-ReportDepth + title: The tool shall support --report-depth flag to set the Markdown heading depth for the review report. + justification: | + Allows the review report to be embedded at any heading level within a larger + Markdown document, with a default depth of 1 when not specified. + tests: + - Context_Create_ReportDepthFlag_SetsReportDepth + - Context_Create_NoArguments_ReportDepthDefaultsToOne + + - id: ReviewMark-Cmd-Index + title: The tool shall support --index flag to scan PDF evidence files matching a glob path and write + index.json. + justification: | + Provides a mechanism to regenerate the review evidence index from scanned PDF + files, reading embedded metadata from each PDF's Keywords field to populate + the index with review IDs, fingerprints, dates, results, and file names. + tests: + - Context_Create_IndexFlag_AddsIndexPath + - Context_Create_IndexFlag_MultipleTimes_AddsAllPaths + - Context_Create_NoArguments_IndexPathsEmpty + - ReviewMark_IndexScan + + - id: ReviewMark-Cmd-Enforce + title: The tool shall support --enforce flag to exit with a non-zero code when there are review issues. + justification: | + Enables CI/CD pipelines to block downstream stages when review sets are failed, + stale, or missing, or when files requiring review are not covered by any review-set. + Without --enforce the tool generates the plan and report but exits with code 0. + tests: + - Context_Create_EnforceFlag_SetsEnforceTrue + - Context_Create_NoArguments_EnforceFalse + - ReviewMark_Enforce + + - id: ReviewMark-Cmd-Dir + title: The tool shall support --dir flag to set the working directory for file operations. + justification: | + Allows users to target an evidence store or project directory without changing + the process working directory, enabling consistent scripting and CI/CD usage + without requiring a cd command before invoking the tool. + tests: + - Context_Create_DirFlag_SetsWorkingDirectory + - Context_Create_NoArguments_WorkingDirectoryIsNull + - Context_Create_DirFlag_MissingValue_ThrowsArgumentException + - ReviewMark_WorkingDirectoryOverride + + - id: ReviewMark-Cmd-Elaborate + title: The tool shall support --elaborate flag to print a Markdown elaboration of a review set. + justification: | + When preparing for a review, the reviewer needs the review set ID, its current + fingerprint, and the full sorted list of files to be reviewed. The --elaborate + command provides this information formatted as Markdown so it can be copied + directly into review documentation. + tests: + - Context_Create_ElaborateFlag_SetsElaborateId + - Context_Create_NoArguments_ElaborateIdIsNull + - Context_Create_ElaborateFlag_WithoutValue_ThrowsArgumentException + - ReviewMarkConfiguration_ElaborateReviewSet_ValidId_ReturnsElaboration + - ReviewMarkConfiguration_ElaborateReviewSet_UnknownId_ThrowsArgumentException + - ReviewMarkConfiguration_ElaborateReviewSet_NullId_ThrowsArgumentException + - ReviewMarkConfiguration_ElaborateReviewSet_MarkdownDepth_UsedForHeadings + - ReviewMarkConfiguration_ElaborateReviewSet_MarkdownDepthAbove5_Throws + - ReviewMarkConfiguration_ElaborateReviewSet_ContainsFullFingerprint + - Program_Run_WithHelpFlag_IncludesElaborateOption + - Program_Run_WithElaborateFlag_OutputsElaboration + - Program_Run_WithElaborateFlag_UnknownId_ReportsError + - ReviewMark_Elaborate + + - id: ReviewMark-Cmd-Lint + title: The tool shall support --lint flag to validate the definition file and report issues. + justification: | + Users need a way to verify that the .reviewmark.yaml configuration file is valid + before running the main tool, providing clear error messages about the cause and + location of any issues. + tests: + - Context_Create_LintFlag_SetsLintTrue + - Context_Create_NoArguments_LintIsFalse + - Program_Run_WithHelpFlag_IncludesLintOption + - Program_Run_WithLintFlag_ValidConfig_ReportsSuccess + - Program_Run_WithLintFlag_MissingConfig_ReportsError + - Program_Run_WithLintFlag_DuplicateIds_ReportsError + - Program_Run_WithLintFlag_UnknownSourceType_ReportsError + - Program_Run_WithLintFlag_CorruptedYaml_ReportsError + - Program_Run_WithLintFlag_MissingEvidenceSource_ReportsError + - Program_Run_WithLintFlag_MultipleErrors_ReportsAll + - ReviewMarkConfiguration_Load_InvalidYaml_ErrorIncludesFilenameAndLine + - ReviewMarkConfiguration_Load_MissingEvidenceSource_ErrorIncludesFilename + - ReviewMarkConfiguration_Lint_MultipleErrors_ReturnsAll + - ReviewMark_Lint diff --git a/docs/reqstream/configuration-requirements.yaml b/docs/reqstream/configuration-requirements.yaml new file mode 100644 index 0000000..4c3ad54 --- /dev/null +++ b/docs/reqstream/configuration-requirements.yaml @@ -0,0 +1,30 @@ +--- +# ReviewMarkConfiguration Software Unit Requirements +# +# PURPOSE: +# - Define requirements for the ReviewMarkConfiguration software unit +# - This unit parses the .reviewmark.yaml definition file into an in-memory model +# - It computes SHA256 fingerprints for review-sets and generates plan/report Markdown + +sections: + - title: ReviewMarkConfiguration Unit Requirements + requirements: + - id: ReviewMark-Config-Reading + title: The tool shall read and parse the .reviewmark.yaml file into an in-memory configuration model. + justification: | + Enables the tool to read its configuration from the standard `.reviewmark.yaml` file, + exposing needs-review patterns, evidence source, and review set definitions. Review sets + support SHA256 content-based fingerprinting to detect changes to covered files. + tests: + - ReviewMarkConfiguration_Parse_NullYaml_ThrowsArgumentNullException + - ReviewMarkConfiguration_Parse_ValidYaml_ReturnsConfiguration + - ReviewMarkConfiguration_Parse_NeedsReviewPatterns_ParsedCorrectly + - ReviewMarkConfiguration_Parse_EvidenceSource_ParsedCorrectly + - ReviewMarkConfiguration_Parse_Reviews_ParsedCorrectly + - ReviewMarkConfiguration_Parse_EvidenceSourceWithCredentials_ParsedCorrectly + - ReviewMarkConfiguration_GetNeedsReviewFiles_ReturnsMatchingFiles + - ReviewSet_GetFingerprint_SameContent_ReturnsSameFingerprint + - ReviewSet_GetFingerprint_DifferentContent_ReturnsDifferentFingerprint + - ReviewSet_GetFingerprint_RenameFile_ReturnsSameFingerprint + - ReviewMarkConfiguration_Load_NonExistentFile_ThrowsException + - ReviewMarkConfiguration_Load_FileshareRelativeLocation_ResolvesToAbsolutePath diff --git a/docs/reqstream/index-requirements.yaml b/docs/reqstream/index-requirements.yaml new file mode 100644 index 0000000..0abf7b8 --- /dev/null +++ b/docs/reqstream/index-requirements.yaml @@ -0,0 +1,66 @@ +--- +# Index Software Unit Requirements +# +# PURPOSE: +# - Define requirements for the ReviewIndex software unit +# - This unit loads review evidence from an EvidenceSource (none/fileshare/url) +# - It also scans PDF files to extract embedded review metadata for indexing + +sections: + - title: Index Unit Requirements + requirements: + - id: ReviewMark-Index-EvidenceSource + title: The tool shall load a ReviewIndex from an EvidenceSource supporting none, fileshare, and url types. + justification: | + The tool must be able to load review evidence index data from the EvidenceSource + specified in its configuration. Three source types are supported: `none` returns an + empty index immediately (useful during initial project setup), `fileshare` loads + the index JSON from a local or network file path, and `url` downloads it over + HTTP(S) with optional Basic-auth credentials read from environment variables. + An internal overload accepting an HttpClient enables unit testing via a fake + HttpMessageHandler without network access. + tests: + - ReviewIndex_Load_EvidenceSource_NullSource_ThrowsArgumentNullException + - ReviewIndex_Load_EvidenceSource_UnknownType_ThrowsInvalidOperationException + - ReviewIndex_Load_EvidenceSource_None_ReturnsEmptyIndex + - ReviewIndex_Load_EvidenceSource_None_HttpClientOverload_ReturnsEmptyIndex + - ReviewIndex_Load_EvidenceSource_Fileshare_LoadsFromFile + - ReviewIndex_Load_EvidenceSource_Fileshare_NonExistentFile_ThrowsInvalidOperationException + - ReviewIndex_Load_EvidenceSource_Fileshare_InvalidJson_ThrowsInvalidOperationException + - ReviewIndex_Load_EvidenceSource_Fileshare_EmptyReviews_ReturnsEmptyIndex + - ReviewIndex_Load_EvidenceSource_Fileshare_ValidJson_ReturnsPopulatedIndex + - ReviewIndex_Load_EvidenceSource_Fileshare_MissingRequiredFields_SkipsInvalidEntries + - ReviewIndex_Load_EvidenceSource_Url_SuccessResponse_LoadsIndex + - ReviewIndex_Load_EvidenceSource_Url_NotFoundResponse_ThrowsInvalidOperationException + - ReviewIndex_Load_EvidenceSource_Url_InvalidJson_ThrowsInvalidOperationException + - ReviewIndex_Load_EvidenceSource_NullHttpClient_ThrowsArgumentNullException + + - id: ReviewMark-EvidenceSource-None + title: The tool shall support a 'none' evidence source type that provides no review evidence. + justification: | + When a project is first starting out, it should be able to set the evidence-source + to 'none' until an evidence store is provisioned. The 'none' type requires no + location field and always returns an empty index, allowing the tool to run without + error during initial repository setup. + tests: + - ReviewIndex_Load_EvidenceSource_None_ReturnsEmptyIndex + - ReviewIndex_Load_EvidenceSource_None_HttpClientOverload_ReturnsEmptyIndex + - ReviewMarkConfiguration_Parse_NoneEvidenceSource_ParsedCorrectly + - ReviewMarkConfiguration_Parse_NoneEvidenceSource_NoLocationRequired + - ReviewMarkConfiguration_Lint_NoneEvidenceSource_NoErrors + + - id: ReviewMark-Index-PdfParsing + title: The tool shall parse PDF metadata from the Keywords field when indexing evidence files. + justification: | + When scanning PDF evidence files, the tool must read the standard PDF Keywords + field and extract space-separated `name=value` pairs. All four fields — id, + fingerprint, date, and result — are required for an entry to be indexed; PDFs + whose Keywords field is missing any of these fields (or is entirely absent) must + be skipped with a warning, ensuring the index only contains complete, valid entries. + tests: + - ReviewIndex_Scan_PdfWithValidMetadata_PopulatesIndex + - ReviewIndex_Scan_PdfWithMissingId_SkipsWithWarning + - ReviewIndex_Scan_PdfWithMissingFingerprint_SkipsWithWarning + - ReviewIndex_Scan_PdfWithMissingDate_SkipsWithWarning + - ReviewIndex_Scan_PdfWithMissingResult_SkipsWithWarning + - ReviewIndex_Scan_PdfWithNoKeywords_SkipsWithWarning diff --git a/docs/reqstream/ots-requirements.yaml b/docs/reqstream/ots-requirements.yaml new file mode 100644 index 0000000..b763998 --- /dev/null +++ b/docs/reqstream/ots-requirements.yaml @@ -0,0 +1,102 @@ +--- +# OTS (Off-the-Shelf) Software Requirements +# +# PURPOSE: +# - Define requirements for third-party components used by ReviewMark +# - OTS requirements document which capabilities the project depends on +# - Tests verify the OTS component provides the required behavior in this environment + +sections: + - title: OTS Software Requirements + sections: + - title: MSTest + requirements: + - id: ReviewMark-OTS-MSTest + title: MSTest shall execute unit tests and report results. + justification: | + MSTest (MSTest.TestFramework and MSTest.TestAdapter) is the unit-testing framework used + by the project. It discovers and runs all test methods and writes TRX result files that + feed into coverage reporting and requirements traceability. Passing tests confirm the + framework is functioning correctly. + tags: [ots] + tests: + - Context_Create_NoArguments_ReturnsDefaultContext + - Context_Create_VersionFlag_SetsVersionTrue + - Context_Create_HelpFlag_SetsHelpTrue + - Context_Create_SilentFlag_SetsSilentTrue + - Context_Create_ValidateFlag_SetsValidateTrue + - Context_Create_ResultsFlag_SetsResultsFile + - Context_Create_LogFlag_OpensLogFile + - Context_Create_UnknownArgument_ThrowsArgumentException + - Context_Create_ShortVersionFlag_SetsVersionTrue + + - title: ReqStream + requirements: + - id: ReviewMark-OTS-ReqStream + title: ReqStream shall enforce that every requirement is linked to passing test evidence. + justification: | + DemaConsulting.ReqStream processes requirements.yaml and the TRX test-result files to + produce a requirements report, justifications document, and traceability matrix. When + run with --enforce, it exits with a non-zero code if any requirement lacks test evidence, + making unproven requirements a build-breaking condition. A successful pipeline run with + --enforce proves all requirements are covered and that ReqStream is functioning. + tags: [ots] + tests: + - ReqStream_EnforcementMode + + - title: BuildMark + requirements: + - id: ReviewMark-OTS-BuildMark + title: BuildMark shall generate build-notes documentation from GitHub Actions metadata. + justification: | + DemaConsulting.BuildMark queries the GitHub API to capture workflow run details and + renders them as a markdown build-notes document included in the release artifacts. + It runs as part of the same CI pipeline that produces the TRX test results, so a + successful pipeline run is evidence that BuildMark executed without error. + tags: [ots] + tests: + - BuildMark_MarkdownReportGeneration + + - title: VersionMark + requirements: + - id: ReviewMark-OTS-VersionMark + title: VersionMark shall publish captured tool-version information. + justification: | + DemaConsulting.VersionMark reads version metadata for each dotnet tool used in the + pipeline and writes a versions markdown document included in the release artifacts. + It runs in the same CI pipeline that produces the TRX test results, so a successful + pipeline run is evidence that VersionMark executed without error. + tags: [ots] + tests: + - VersionMark_CapturesVersions + - VersionMark_GeneratesMarkdownReport + + - title: SarifMark + requirements: + - id: ReviewMark-OTS-SarifMark + title: SarifMark shall convert CodeQL SARIF results into a markdown report. + justification: | + DemaConsulting.SarifMark reads the SARIF output produced by CodeQL code scanning and + renders it as a human-readable markdown document included in the release artifacts. + It runs in the same CI pipeline that produces the TRX test results, so a successful + pipeline run is evidence that SarifMark executed without error. + tags: [ots] + tests: + - SarifMark_SarifReading + - SarifMark_MarkdownReportGeneration + + - title: SonarMark + requirements: + - id: ReviewMark-OTS-SonarMark + title: SonarMark shall generate a SonarCloud quality report. + justification: | + DemaConsulting.SonarMark retrieves quality-gate and metrics data from SonarCloud and + renders it as a markdown document included in the release artifacts. It runs in the + same CI pipeline that produces the TRX test results, so a successful pipeline run is + evidence that SonarMark executed without error. + tags: [ots] + tests: + - SonarMark_QualityGateRetrieval + - SonarMark_IssuesRetrieval + - SonarMark_HotSpotsRetrieval + - SonarMark_MarkdownReportGeneration diff --git a/docs/reqstream/platform-requirements.yaml b/docs/reqstream/platform-requirements.yaml new file mode 100644 index 0000000..a80c0ba --- /dev/null +++ b/docs/reqstream/platform-requirements.yaml @@ -0,0 +1,104 @@ +--- +# Platform Support Requirements +# +# PURPOSE: +# - Define requirements for cross-platform support +# - These requirements verify the tool builds and runs on all supported operating systems +# and .NET runtime versions +# - Tests are linked with source filters to ensure results come from specific platforms + +sections: + - title: Platform Support Requirements + requirements: + - id: ReviewMark-Platform-Windows + title: The tool shall build and run on Windows platforms. + justification: | + DEMA Consulting tools must support Windows as a major development platform. + tests: + # Tests link to "windows" to ensure results come from Windows platform + - "windows@ReviewMark_VersionDisplay" + - "windows@ReviewMark_HelpDisplay" + - "windows@ReviewMark_ReviewPlanGeneration" + - "windows@ReviewMark_ReviewReportGeneration" + - "windows@ReviewMark_IndexScan" + - "windows@ReviewMark_Enforce" + - "windows@ReviewMark_WorkingDirectoryOverride" + - "windows@ReviewMark_Elaborate" + - "windows@ReviewMark_Lint" + + - id: ReviewMark-Platform-Linux + title: The tool shall build and run on Linux platforms. + justification: | + DEMA Consulting tools must support Linux for CI/CD and containerized environments. + tests: + # Tests link to "ubuntu" to ensure results come from Linux platform + - "ubuntu@ReviewMark_VersionDisplay" + - "ubuntu@ReviewMark_HelpDisplay" + - "ubuntu@ReviewMark_ReviewPlanGeneration" + - "ubuntu@ReviewMark_ReviewReportGeneration" + - "ubuntu@ReviewMark_IndexScan" + - "ubuntu@ReviewMark_Enforce" + - "ubuntu@ReviewMark_WorkingDirectoryOverride" + - "ubuntu@ReviewMark_Elaborate" + - "ubuntu@ReviewMark_Lint" + + - id: ReviewMark-Platform-MacOS + title: The tool shall build and run on macOS platforms. + justification: | + DEMA Consulting tools must support macOS for developers using Apple platforms. + tests: + # Tests link to "macos" to ensure results come from macOS platform + - "macos@ReviewMark_VersionDisplay" + - "macos@ReviewMark_HelpDisplay" + - "macos@ReviewMark_ReviewPlanGeneration" + - "macos@ReviewMark_ReviewReportGeneration" + - "macos@ReviewMark_IndexScan" + - "macos@ReviewMark_Enforce" + - "macos@ReviewMark_WorkingDirectoryOverride" + - "macos@ReviewMark_Elaborate" + - "macos@ReviewMark_Lint" + + - id: ReviewMark-Platform-Net8 + title: The tool shall support .NET 8 runtime. + justification: | + .NET 8 is an LTS release providing long-term stability for enterprise users. + tests: + - "dotnet8.x@ReviewMark_VersionDisplay" + - "dotnet8.x@ReviewMark_HelpDisplay" + - "dotnet8.x@ReviewMark_ReviewPlanGeneration" + - "dotnet8.x@ReviewMark_ReviewReportGeneration" + - "dotnet8.x@ReviewMark_IndexScan" + - "dotnet8.x@ReviewMark_Enforce" + - "dotnet8.x@ReviewMark_WorkingDirectoryOverride" + - "dotnet8.x@ReviewMark_Elaborate" + - "dotnet8.x@ReviewMark_Lint" + + - id: ReviewMark-Platform-Net9 + title: The tool shall support .NET 9 runtime. + justification: | + .NET 9 support enables users to leverage the latest .NET features. + tests: + - "dotnet9.x@ReviewMark_VersionDisplay" + - "dotnet9.x@ReviewMark_HelpDisplay" + - "dotnet9.x@ReviewMark_ReviewPlanGeneration" + - "dotnet9.x@ReviewMark_ReviewReportGeneration" + - "dotnet9.x@ReviewMark_IndexScan" + - "dotnet9.x@ReviewMark_Enforce" + - "dotnet9.x@ReviewMark_WorkingDirectoryOverride" + - "dotnet9.x@ReviewMark_Elaborate" + - "dotnet9.x@ReviewMark_Lint" + + - id: ReviewMark-Platform-Net10 + title: The tool shall support .NET 10 runtime. + justification: | + .NET 10 support ensures the tool remains compatible with the latest .NET ecosystem. + tests: + - "dotnet10.x@ReviewMark_VersionDisplay" + - "dotnet10.x@ReviewMark_HelpDisplay" + - "dotnet10.x@ReviewMark_ReviewPlanGeneration" + - "dotnet10.x@ReviewMark_ReviewReportGeneration" + - "dotnet10.x@ReviewMark_IndexScan" + - "dotnet10.x@ReviewMark_Enforce" + - "dotnet10.x@ReviewMark_WorkingDirectoryOverride" + - "dotnet10.x@ReviewMark_Elaborate" + - "dotnet10.x@ReviewMark_Lint" diff --git a/docs/requirements/definition.yaml b/docs/requirements/definition.yaml deleted file mode 100644 index a0f3371..0000000 --- a/docs/requirements/definition.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -resource-path: - - docs/requirements - - docs/template -input-files: - - docs/requirements/title.txt - - docs/requirements/introduction.md - - docs/requirements/requirements.md -template: template.html -table-of-contents: true -number-sections: true diff --git a/docs/requirements_doc/definition.yaml b/docs/requirements_doc/definition.yaml new file mode 100644 index 0000000..0f4ccd2 --- /dev/null +++ b/docs/requirements_doc/definition.yaml @@ -0,0 +1,12 @@ +--- +resource-path: + - docs/requirements_doc + - docs/template +input-files: + - docs/requirements_doc/title.txt + - docs/requirements_doc/introduction.md + - docs/requirements_doc/requirements.md + - docs/requirements_doc/justifications.md +template: template.html +table-of-contents: true +number-sections: true diff --git a/docs/requirements/introduction.md b/docs/requirements_doc/introduction.md similarity index 100% rename from docs/requirements/introduction.md rename to docs/requirements_doc/introduction.md diff --git a/docs/requirements/title.txt b/docs/requirements_doc/title.txt similarity index 100% rename from docs/requirements/title.txt rename to docs/requirements_doc/title.txt diff --git a/docs/requirements_report/definition.yaml b/docs/requirements_report/definition.yaml new file mode 100644 index 0000000..918a645 --- /dev/null +++ b/docs/requirements_report/definition.yaml @@ -0,0 +1,11 @@ +--- +resource-path: + - docs/requirements_report + - docs/template +input-files: + - docs/requirements_report/title.txt + - docs/requirements_report/introduction.md + - docs/requirements_report/trace_matrix.md +template: template.html +table-of-contents: true +number-sections: true diff --git a/docs/tracematrix/introduction.md b/docs/requirements_report/introduction.md similarity index 100% rename from docs/tracematrix/introduction.md rename to docs/requirements_report/introduction.md diff --git a/docs/tracematrix/title.txt b/docs/requirements_report/title.txt similarity index 100% rename from docs/tracematrix/title.txt rename to docs/requirements_report/title.txt diff --git a/docs/tracematrix/definition.yaml b/docs/tracematrix/definition.yaml deleted file mode 100644 index ba93d57..0000000 --- a/docs/tracematrix/definition.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -resource-path: - - docs/tracematrix - - docs/template -input-files: - - docs/tracematrix/title.txt - - docs/tracematrix/introduction.md - - docs/tracematrix/tracematrix.md -template: template.html -table-of-contents: true -number-sections: true diff --git a/requirements.yaml b/requirements.yaml index 7e3984c..3654661 100644 --- a/requirements.yaml +++ b/requirements.yaml @@ -1,522 +1,31 @@ -# Requirements Testing Strategy +# Root Requirements File # -# This project uses three categories of tests to verify requirements: +# PURPOSE: +# - Serve as the entry point for ReqStream requirement processing +# - Include all reviewable requirement files from docs/reqstream/ # -# 1. Unit Tests - Run locally via "dotnet test" -# 2. Self-Validation Tests - Run locally via "--validate" -# 3. Platform Tests - Run via CI/CD across OS/runtime matrix +# USAGE: +# - Run ReqStream against this file to process all requirements: # -# NOTE: Running "reqstream --enforce" with only local test results (unit tests -# and local self-validation) is expected to show some unsatisfied requirements. -# Platform-specific requirements require test results from CI/CD runs across -# the full OS and runtime matrix. +# dotnet reqstream \ +# --requirements requirements.yaml \ +# --tests "artifacts/**/*.trx" \ +# --report docs/requirements_doc/requirements.md \ +# --justifications docs/requirements_doc/justifications.md \ +# --matrix docs/requirements_report/trace_matrix.md \ +# --enforce # -# Test links can include a source filter prefix (e.g. "windows@", "ubuntu@", "net8.0@", +# - Add new requirement files under docs/reqstream/ and include them here +# +# NOTE: Test links can include a source filter prefix (e.g. "windows@", "ubuntu@", "net8.0@", # "dotnet8.x@") to restrict which test results count as evidence for a requirement. This # is critical for platform and framework requirements - removing these filters invalidates # the evidence-based proof. # -# Source filter prefixes: -# windows@TestName - proves the test passed on a Windows platform -# ubuntu@TestName - proves the test passed on a Linux (Ubuntu) platform -# macos@TestName - proves the test passed on a macOS platform -# net8.0@TestName - proves the test passed under the .NET 8 target framework -# net9.0@TestName - proves the test passed under the .NET 9 target framework -# net10.0@TestName - proves the test passed under the .NET 10 target framework -# dotnet8.x@TestName - proves the self-validation test ran with .NET 8.x runtime -# dotnet9.x@TestName - proves the self-validation test ran with .NET 9.x runtime -# dotnet10.x@TestName - proves the self-validation test ran with .NET 10.x runtime -# --- -sections: - - title: ReviewMark Requirements - sections: - - title: Command-Line Interface - requirements: - - id: ReviewMark-Cmd-Context - title: The tool shall implement a Context class for command-line argument handling. - justification: | - Provides a standardized approach to command-line argument parsing and output - handling across all DEMA Consulting DotNet Tools. - tests: - - Context_Create_NoArguments_ReturnsDefaultContext - - Context_Create_VersionFlag_SetsVersionTrue - - Context_Create_HelpFlag_SetsHelpTrue - - Context_Create_SilentFlag_SetsSilentTrue - - Context_Create_ValidateFlag_SetsValidateTrue - - Context_Create_ResultsFlag_SetsResultsFile - - Context_Create_LogFlag_OpensLogFile - - - id: ReviewMark-Cmd-Version - title: The tool shall support -v and --version flags to display version information. - justification: | - Users need to quickly identify the version of the tool they are using for - troubleshooting and compatibility verification. - tests: - - Context_Create_VersionFlag_SetsVersionTrue - - Context_Create_ShortVersionFlag_SetsVersionTrue - - Program_Run_WithVersionFlag_DisplaysVersionOnly - - Program_Version_ReturnsNonEmptyString - - IntegrationTest_VersionFlag_OutputsVersion - - - id: ReviewMark-Cmd-Help - title: The tool shall support -?, -h, and --help flags to display usage information. - justification: | - Users need access to command-line usage documentation without requiring - external resources. - tests: - - Context_Create_HelpFlag_SetsHelpTrue - - Context_Create_ShortHelpFlag_H_SetsHelpTrue - - Context_Create_ShortHelpFlag_Question_SetsHelpTrue - - Program_Run_WithHelpFlag_DisplaysUsageInformation - - IntegrationTest_HelpFlag_OutputsUsageInformation - - - id: ReviewMark-Cmd-Silent - title: The tool shall support --silent flag to suppress console output. - justification: | - Enables automated scripts and CI/CD pipelines to run the tool without - cluttering output logs. - tests: - - Context_Create_SilentFlag_SetsSilentTrue - - Context_WriteLine_Silent_DoesNotWriteToConsole - - IntegrationTest_SilentFlag_SuppressesOutput - - - id: ReviewMark-Cmd-Validate - title: The tool shall support --validate flag to run self-validation tests. - justification: | - Provides a built-in mechanism to verify the tool is functioning correctly - in the deployment environment. - tests: - - Context_Create_ValidateFlag_SetsValidateTrue - - Program_Run_WithValidateFlag_RunsValidation - - IntegrationTest_ValidateFlag_RunsValidation - - - id: ReviewMark-Cmd-Results - title: The tool shall support --results flag to write validation results in TRX or JUnit format. - justification: | - Enables integration with CI/CD systems that expect standard test result formats. - tests: - - Context_Create_ResultsFlag_SetsResultsFile - - IntegrationTest_ValidateWithResults_GeneratesTrxFile - - IntegrationTest_ValidateWithResults_GeneratesJUnitFile - - - id: ReviewMark-Cmd-Log - title: The tool shall support --log flag to write output to a log file. - justification: | - Provides persistent logging for debugging and audit trails. - tests: - - Context_Create_LogFlag_OpensLogFile - - IntegrationTest_LogFlag_WritesOutputToFile - - - id: ReviewMark-Cmd-ErrorOutput - title: The tool shall write error messages to stderr. - justification: | - Error messages must be written to stderr so they remain visible to the user - without polluting stdout, which consumers may pipe or redirect for data capture. - tests: - - Context_WriteError_NotSilent_WritesToConsole - - IntegrationTest_UnknownArgument_ReturnsError - - - id: ReviewMark-Cmd-InvalidArgs - title: The tool shall reject unknown or malformed command-line arguments with a descriptive error. - justification: | - Providing clear feedback for invalid arguments helps users quickly correct - mistakes and prevents silent misconfiguration. - tests: - - Context_Create_UnknownArgument_ThrowsArgumentException - - Context_Create_LogFlag_WithoutValue_ThrowsArgumentException - - Context_Create_ResultsFlag_WithoutValue_ThrowsArgumentException - - IntegrationTest_UnknownArgument_ReturnsError - - - id: ReviewMark-Cmd-ExitCode - title: The tool shall return a non-zero exit code on failure. - justification: | - Callers (scripts, CI/CD pipelines) must be able to detect failure conditions - programmatically via the process exit code. - tests: - - Context_WriteError_SetsErrorExitCode - - IntegrationTest_UnknownArgument_ReturnsError - - - id: ReviewMark-Cmd-Definition - title: The tool shall support --definition flag to specify the definition YAML file. - justification: | - Users must be able to specify the path to the .reviewmark.yaml definition file, - which configures needs-review patterns, evidence source, and review set definitions. - tests: - - Context_Create_DefinitionFlag_SetsDefinitionFile - - Context_Create_DefinitionFlag_WithoutValue_ThrowsArgumentException - - ReviewMark_ReviewPlanGeneration - - ReviewMark_ReviewReportGeneration - - - id: ReviewMark-Cmd-Plan - title: The tool shall support --plan flag to write the review plan to a Markdown file. - justification: | - Enables automated generation of a review plan document that lists all review sets - and coverage status, suitable for inclusion in release documentation. - tests: - - Context_Create_PlanFlag_SetsPlanFile - - ReviewMark_ReviewPlanGeneration - - - id: ReviewMark-Cmd-PlanDepth - title: The tool shall support --plan-depth flag to set the Markdown heading depth for the review plan. - justification: | - Allows the review plan to be embedded at any heading level within a larger - Markdown document, with a default depth of 1 when not specified. - tests: - - Context_Create_PlanDepthFlag_SetsPlanDepth - - Context_Create_PlanDepthFlag_WithInvalidValue_ThrowsArgumentException - - Context_Create_PlanDepthFlag_WithZeroValue_ThrowsArgumentException - - Context_Create_NoArguments_PlanDepthDefaultsToOne - - - id: ReviewMark-Cmd-Report - title: The tool shall support --report flag to write the review report to a Markdown file. - justification: | - Enables automated generation of a review report document showing the current - status of each review set against the evidence index, suitable for release documentation. - tests: - - Context_Create_ReportFlag_SetsReportFile - - ReviewMark_ReviewReportGeneration - - - id: ReviewMark-Cmd-ReportDepth - title: The tool shall support --report-depth flag to set the Markdown heading depth for the review report. - justification: | - Allows the review report to be embedded at any heading level within a larger - Markdown document, with a default depth of 1 when not specified. - tests: - - Context_Create_ReportDepthFlag_SetsReportDepth - - Context_Create_NoArguments_ReportDepthDefaultsToOne - - - id: ReviewMark-Cmd-Index - title: The tool shall support --index flag to scan PDF evidence files matching a glob path and write - index.json. - justification: | - Provides a mechanism to regenerate the review evidence index from scanned PDF - files, reading embedded metadata from each PDF's Keywords field to populate - the index with review IDs, fingerprints, dates, results, and file names. - tests: - - Context_Create_IndexFlag_AddsIndexPath - - Context_Create_IndexFlag_MultipleTimes_AddsAllPaths - - Context_Create_NoArguments_IndexPathsEmpty - - ReviewMark_IndexScan - - - id: ReviewMark-Cmd-Enforce - title: The tool shall support --enforce flag to exit with a non-zero code when there are review issues. - justification: | - Enables CI/CD pipelines to block downstream stages when review sets are failed, - stale, or missing, or when files requiring review are not covered by any review-set. - Without --enforce the tool generates the plan and report but exits with code 0. - tests: - - Context_Create_EnforceFlag_SetsEnforceTrue - - Context_Create_NoArguments_EnforceFalse - - ReviewMark_Enforce - - - id: ReviewMark-Cmd-Dir - title: The tool shall support --dir flag to set the working directory for file operations. - justification: | - Allows users to target an evidence store or project directory without changing - the process working directory, enabling consistent scripting and CI/CD usage - without requiring a cd command before invoking the tool. - tests: - - Context_Create_DirFlag_SetsWorkingDirectory - - Context_Create_NoArguments_WorkingDirectoryIsNull - - Context_Create_DirFlag_MissingValue_ThrowsArgumentException - - ReviewMark_WorkingDirectoryOverride - - - id: ReviewMark-Cmd-Elaborate - title: The tool shall support --elaborate flag to print a Markdown elaboration of a review set. - justification: | - When preparing for a review, the reviewer needs the review set ID, its current - fingerprint, and the full sorted list of files to be reviewed. The --elaborate - command provides this information formatted as Markdown so it can be copied - directly into review documentation. - tests: - - Context_Create_ElaborateFlag_SetsElaborateId - - Context_Create_NoArguments_ElaborateIdIsNull - - Context_Create_ElaborateFlag_WithoutValue_ThrowsArgumentException - - ReviewMarkConfiguration_ElaborateReviewSet_ValidId_ReturnsElaboration - - ReviewMarkConfiguration_ElaborateReviewSet_UnknownId_ThrowsArgumentException - - ReviewMarkConfiguration_ElaborateReviewSet_NullId_ThrowsArgumentException - - ReviewMarkConfiguration_ElaborateReviewSet_MarkdownDepth_UsedForHeadings - - ReviewMarkConfiguration_ElaborateReviewSet_MarkdownDepthAbove5_Throws - - ReviewMarkConfiguration_ElaborateReviewSet_ContainsFullFingerprint - - Program_Run_WithHelpFlag_IncludesElaborateOption - - Program_Run_WithElaborateFlag_OutputsElaboration - - Program_Run_WithElaborateFlag_UnknownId_ReportsError - - ReviewMark_Elaborate - - - id: ReviewMark-Cmd-Lint - title: The tool shall support --lint flag to validate the definition file and report issues. - justification: | - Users need a way to verify that the .reviewmark.yaml configuration file is valid - before running the main tool, providing clear error messages about the cause and - location of any issues. - tests: - - Context_Create_LintFlag_SetsLintTrue - - Context_Create_NoArguments_LintIsFalse - - Program_Run_WithHelpFlag_IncludesLintOption - - Program_Run_WithLintFlag_ValidConfig_ReportsSuccess - - Program_Run_WithLintFlag_MissingConfig_ReportsError - - Program_Run_WithLintFlag_DuplicateIds_ReportsError - - Program_Run_WithLintFlag_UnknownSourceType_ReportsError - - Program_Run_WithLintFlag_CorruptedYaml_ReportsError - - Program_Run_WithLintFlag_MissingEvidenceSource_ReportsError - - Program_Run_WithLintFlag_MultipleErrors_ReportsAll - - ReviewMarkConfiguration_Load_InvalidYaml_ErrorIncludesFilenameAndLine - - ReviewMarkConfiguration_Load_MissingEvidenceSource_ErrorIncludesFilename - - ReviewMarkConfiguration_Lint_MultipleErrors_ReturnsAll - - ReviewMark_Lint - - - title: Configuration Reading - requirements: - - id: ReviewMark-Config-Reading - title: The tool shall read and parse the .reviewmark.yaml file into an in-memory configuration model. - justification: | - Enables the tool to read its configuration from the standard `.reviewmark.yaml` file, - exposing needs-review patterns, evidence source, and review set definitions. Review sets - support SHA256 content-based fingerprinting to detect changes to covered files. - tests: - - ReviewMarkConfiguration_Parse_NullYaml_ThrowsArgumentNullException - - ReviewMarkConfiguration_Parse_ValidYaml_ReturnsConfiguration - - ReviewMarkConfiguration_Parse_NeedsReviewPatterns_ParsedCorrectly - - ReviewMarkConfiguration_Parse_EvidenceSource_ParsedCorrectly - - ReviewMarkConfiguration_Parse_Reviews_ParsedCorrectly - - ReviewMarkConfiguration_Parse_EvidenceSourceWithCredentials_ParsedCorrectly - - ReviewMarkConfiguration_GetNeedsReviewFiles_ReturnsMatchingFiles - - ReviewSet_GetFingerprint_SameContent_ReturnsSameFingerprint - - ReviewSet_GetFingerprint_DifferentContent_ReturnsDifferentFingerprint - - ReviewSet_GetFingerprint_RenameFile_ReturnsSameFingerprint - - ReviewMarkConfiguration_Load_NonExistentFile_ThrowsException - - ReviewMarkConfiguration_Load_FileshareRelativeLocation_ResolvesToAbsolutePath - - - id: ReviewMark-Index-EvidenceSource - title: The tool shall load a ReviewIndex from an EvidenceSource supporting none, fileshare, and url types. - justification: | - The tool must be able to load review evidence index data from the EvidenceSource - specified in its configuration. Three source types are supported: `none` returns an - empty index immediately (useful during initial project setup), `fileshare` loads - the index JSON from a local or network file path, and `url` downloads it over - HTTP(S) with optional Basic-auth credentials read from environment variables. - An internal overload accepting an HttpClient enables unit testing via a fake - HttpMessageHandler without network access. - tests: - - ReviewIndex_Load_EvidenceSource_NullSource_ThrowsArgumentNullException - - ReviewIndex_Load_EvidenceSource_UnknownType_ThrowsInvalidOperationException - - ReviewIndex_Load_EvidenceSource_None_ReturnsEmptyIndex - - ReviewIndex_Load_EvidenceSource_None_HttpClientOverload_ReturnsEmptyIndex - - ReviewIndex_Load_EvidenceSource_Fileshare_LoadsFromFile - - ReviewIndex_Load_EvidenceSource_Fileshare_NonExistentFile_ThrowsInvalidOperationException - - ReviewIndex_Load_EvidenceSource_Fileshare_InvalidJson_ThrowsInvalidOperationException - - ReviewIndex_Load_EvidenceSource_Fileshare_EmptyReviews_ReturnsEmptyIndex - - ReviewIndex_Load_EvidenceSource_Fileshare_ValidJson_ReturnsPopulatedIndex - - ReviewIndex_Load_EvidenceSource_Fileshare_MissingRequiredFields_SkipsInvalidEntries - - ReviewIndex_Load_EvidenceSource_Url_SuccessResponse_LoadsIndex - - ReviewIndex_Load_EvidenceSource_Url_NotFoundResponse_ThrowsInvalidOperationException - - ReviewIndex_Load_EvidenceSource_Url_InvalidJson_ThrowsInvalidOperationException - - ReviewIndex_Load_EvidenceSource_NullHttpClient_ThrowsArgumentNullException - - - id: ReviewMark-EvidenceSource-None - title: The tool shall support a 'none' evidence source type that provides no review evidence. - justification: | - When a project is first starting out, it should be able to set the evidence-source - to 'none' until an evidence store is provisioned. The 'none' type requires no - location field and always returns an empty index, allowing the tool to run without - error during initial repository setup. - tests: - - ReviewIndex_Load_EvidenceSource_None_ReturnsEmptyIndex - - ReviewIndex_Load_EvidenceSource_None_HttpClientOverload_ReturnsEmptyIndex - - ReviewMarkConfiguration_Parse_NoneEvidenceSource_ParsedCorrectly - - ReviewMarkConfiguration_Parse_NoneEvidenceSource_NoLocationRequired - - ReviewMarkConfiguration_Lint_NoneEvidenceSource_NoErrors - - - id: ReviewMark-Index-PdfParsing - title: The tool shall parse PDF metadata from the Keywords field when indexing evidence files. - justification: | - When scanning PDF evidence files, the tool must read the standard PDF Keywords - field and extract space-separated `name=value` pairs. All four fields — id, - fingerprint, date, and result — are required for an entry to be indexed; PDFs - whose Keywords field is missing any of these fields (or is entirely absent) must - be skipped with a warning, ensuring the index only contains complete, valid entries. - tests: - - ReviewIndex_Scan_PdfWithValidMetadata_PopulatesIndex - - ReviewIndex_Scan_PdfWithMissingId_SkipsWithWarning - - ReviewIndex_Scan_PdfWithMissingFingerprint_SkipsWithWarning - - ReviewIndex_Scan_PdfWithMissingDate_SkipsWithWarning - - ReviewIndex_Scan_PdfWithMissingResult_SkipsWithWarning - - ReviewIndex_Scan_PdfWithNoKeywords_SkipsWithWarning - - - title: Platform Support - requirements: - - id: ReviewMark-Platform-Windows - title: The tool shall build and run on Windows platforms. - justification: | - DEMA Consulting tools must support Windows as a major development platform. - tests: - # Tests link to "windows" to ensure results come from Windows platform - - "windows@ReviewMark_VersionDisplay" - - "windows@ReviewMark_HelpDisplay" - - "windows@ReviewMark_ReviewPlanGeneration" - - "windows@ReviewMark_ReviewReportGeneration" - - "windows@ReviewMark_IndexScan" - - "windows@ReviewMark_Enforce" - - "windows@ReviewMark_WorkingDirectoryOverride" - - "windows@ReviewMark_Elaborate" - - "windows@ReviewMark_Lint" - - - id: ReviewMark-Platform-Linux - title: The tool shall build and run on Linux platforms. - justification: | - DEMA Consulting tools must support Linux for CI/CD and containerized environments. - tests: - # Tests link to "ubuntu" to ensure results come from Linux platform - - "ubuntu@ReviewMark_VersionDisplay" - - "ubuntu@ReviewMark_HelpDisplay" - - "ubuntu@ReviewMark_ReviewPlanGeneration" - - "ubuntu@ReviewMark_ReviewReportGeneration" - - "ubuntu@ReviewMark_IndexScan" - - "ubuntu@ReviewMark_Enforce" - - "ubuntu@ReviewMark_WorkingDirectoryOverride" - - "ubuntu@ReviewMark_Elaborate" - - "ubuntu@ReviewMark_Lint" - - - id: ReviewMark-Platform-MacOS - title: The tool shall build and run on macOS platforms. - justification: | - DEMA Consulting tools must support macOS for developers using Apple platforms. - tests: - # Tests link to "macos" to ensure results come from macOS platform - - "macos@ReviewMark_VersionDisplay" - - "macos@ReviewMark_HelpDisplay" - - "macos@ReviewMark_ReviewPlanGeneration" - - "macos@ReviewMark_ReviewReportGeneration" - - "macos@ReviewMark_IndexScan" - - "macos@ReviewMark_Enforce" - - "macos@ReviewMark_WorkingDirectoryOverride" - - "macos@ReviewMark_Elaborate" - - "macos@ReviewMark_Lint" - - - id: ReviewMark-Platform-Net8 - title: The tool shall support .NET 8 runtime. - justification: | - .NET 8 is an LTS release providing long-term stability for enterprise users. - tests: - - "dotnet8.x@ReviewMark_VersionDisplay" - - "dotnet8.x@ReviewMark_HelpDisplay" - - "dotnet8.x@ReviewMark_ReviewPlanGeneration" - - "dotnet8.x@ReviewMark_ReviewReportGeneration" - - "dotnet8.x@ReviewMark_IndexScan" - - "dotnet8.x@ReviewMark_Enforce" - - "dotnet8.x@ReviewMark_WorkingDirectoryOverride" - - "dotnet8.x@ReviewMark_Elaborate" - - "dotnet8.x@ReviewMark_Lint" - - - id: ReviewMark-Platform-Net9 - title: The tool shall support .NET 9 runtime. - justification: | - .NET 9 support enables users to leverage the latest .NET features. - tests: - - "dotnet9.x@ReviewMark_VersionDisplay" - - "dotnet9.x@ReviewMark_HelpDisplay" - - "dotnet9.x@ReviewMark_ReviewPlanGeneration" - - "dotnet9.x@ReviewMark_ReviewReportGeneration" - - "dotnet9.x@ReviewMark_IndexScan" - - "dotnet9.x@ReviewMark_Enforce" - - "dotnet9.x@ReviewMark_WorkingDirectoryOverride" - - "dotnet9.x@ReviewMark_Elaborate" - - "dotnet9.x@ReviewMark_Lint" - - - id: ReviewMark-Platform-Net10 - title: The tool shall support .NET 10 runtime. - justification: | - .NET 10 support ensures the tool remains compatible with the latest .NET ecosystem. - tests: - - "dotnet10.x@ReviewMark_VersionDisplay" - - "dotnet10.x@ReviewMark_HelpDisplay" - - "dotnet10.x@ReviewMark_ReviewPlanGeneration" - - "dotnet10.x@ReviewMark_ReviewReportGeneration" - - "dotnet10.x@ReviewMark_IndexScan" - - "dotnet10.x@ReviewMark_Enforce" - - "dotnet10.x@ReviewMark_WorkingDirectoryOverride" - - "dotnet10.x@ReviewMark_Elaborate" - - "dotnet10.x@ReviewMark_Lint" - - - title: OTS Software - requirements: - - id: ReviewMark-OTS-MSTest - title: MSTest shall execute unit tests and report results. - justification: | - MSTest (MSTest.TestFramework and MSTest.TestAdapter) is the unit-testing framework used - by the project. It discovers and runs all test methods and writes TRX result files that - feed into coverage reporting and requirements traceability. Passing tests confirm the - framework is functioning correctly. - tags: [ots] - tests: - - Context_Create_NoArguments_ReturnsDefaultContext - - Context_Create_VersionFlag_SetsVersionTrue - - Context_Create_HelpFlag_SetsHelpTrue - - Context_Create_SilentFlag_SetsSilentTrue - - Context_Create_ValidateFlag_SetsValidateTrue - - Context_Create_ResultsFlag_SetsResultsFile - - Context_Create_LogFlag_OpensLogFile - - Context_Create_UnknownArgument_ThrowsArgumentException - - Context_Create_ShortVersionFlag_SetsVersionTrue - - - id: ReviewMark-OTS-ReqStream - title: ReqStream shall enforce that every requirement is linked to passing test evidence. - justification: | - DemaConsulting.ReqStream processes requirements.yaml and the TRX test-result files to - produce a requirements report, justifications document, and traceability matrix. When - run with --enforce, it exits with a non-zero code if any requirement lacks test evidence, - making unproven requirements a build-breaking condition. A successful pipeline run with - --enforce proves all requirements are covered and that ReqStream is functioning. - tags: [ots] - tests: - - ReqStream_EnforcementMode - - - id: ReviewMark-OTS-BuildMark - title: BuildMark shall generate build-notes documentation from GitHub Actions metadata. - justification: | - DemaConsulting.BuildMark queries the GitHub API to capture workflow run details and - renders them as a markdown build-notes document included in the release artifacts. - It runs as part of the same CI pipeline that produces the TRX test results, so a - successful pipeline run is evidence that BuildMark executed without error. - tags: [ots] - tests: - - BuildMark_MarkdownReportGeneration - - - id: ReviewMark-OTS-VersionMark - title: VersionMark shall publish captured tool-version information. - justification: | - DemaConsulting.VersionMark reads version metadata for each dotnet tool used in the - pipeline and writes a versions markdown document included in the release artifacts. - It runs in the same CI pipeline that produces the TRX test results, so a successful - pipeline run is evidence that VersionMark executed without error. - tags: [ots] - tests: - - VersionMark_CapturesVersions - - VersionMark_GeneratesMarkdownReport - - - id: ReviewMark-OTS-SarifMark - title: SarifMark shall convert CodeQL SARIF results into a markdown report. - justification: | - DemaConsulting.SarifMark reads the SARIF output produced by CodeQL code scanning and - renders it as a human-readable markdown document included in the release artifacts. - It runs in the same CI pipeline that produces the TRX test results, so a successful - pipeline run is evidence that SarifMark executed without error. - tags: [ots] - tests: - - SarifMark_SarifReading - - SarifMark_MarkdownReportGeneration - - - id: ReviewMark-OTS-SonarMark - title: SonarMark shall generate a SonarCloud quality report. - justification: | - DemaConsulting.SonarMark retrieves quality-gate and metrics data from SonarCloud and - renders it as a markdown document included in the release artifacts. It runs in the - same CI pipeline that produces the TRX test results, so a successful pipeline run is - evidence that SonarMark executed without error. - tags: [ots] - tests: - - SonarMark_QualityGateRetrieval - - SonarMark_IssuesRetrieval - - SonarMark_HotSpotsRetrieval - - SonarMark_MarkdownReportGeneration +includes: + - docs/reqstream/cli-requirements.yaml + - docs/reqstream/configuration-requirements.yaml + - docs/reqstream/index-requirements.yaml + - docs/reqstream/platform-requirements.yaml + - docs/reqstream/ots-requirements.yaml From 168e65f44867ad2e30c9e93e85781bd98827acbb Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Mar 2026 18:36:14 -0400 Subject: [PATCH 07/19] Add software design documentation, requirements files, and expand review-sets (#31) * Initial plan * Add design documentation and update review-sets in .reviewmark.yaml Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/a5158598-c9e7-4185-a00d-0b6c7069931e Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Address PR review feedback: add requirements files, ValidationTests, fix review-sets and diagram Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/11aefa92-cb5d-46a4-ad14-fd7c21807456 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Rename requirements files per convention and split OTS into per-component files Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/041a3a1a-f5b2-4bec-b0d4-135c272431e2 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Add unit-context/program requirements, rename Index.cs to ReviewIndex.cs, convert diagrams to text Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/2cd94fd3-6efa-4e14-87c6-aaa5861db6d6 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Add ReviewEvidence record documentation to review-index.md Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/5bc38306-41ce-46fc-ab70-c8424822b0b7 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Fix requirements test linkages to match actual test method names Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/69ec20b8-a181-4b63-b73e-400ec7abd055 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Update docs/design/review-mark-configuration.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update docs/design/program.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update docs/reqstream/unit-program.yaml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update docs/design/review-index.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update docs/design/review-index.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update docs/design/context.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update docs/design/system.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update docs/design/definition.yaml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update docs/reqstream/unit-path-helpers.yaml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update docs/design/review-mark-configuration.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Revise design docs to focus on WHAT/WHY rather than implementation HOW Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/1632c6d8-1a27-4cfa-a722-72ae2f98ecd2 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Co-authored-by: Malcolm Nixon Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .reviewmark.yaml | 91 ++++++-- docs/design/context.md | 59 +++++ docs/design/definition.yaml | 18 ++ docs/design/glob-matcher.md | 32 +++ docs/design/introduction.md | 56 +++++ docs/design/path-helpers.md | 35 +++ docs/design/program.md | 47 ++++ docs/design/review-index.md | 83 +++++++ docs/design/review-mark-configuration.md | 75 +++++++ docs/design/system.md | 73 +++++++ docs/design/title.txt | 13 ++ docs/design/validation.md | 44 ++++ docs/reqstream/ots-buildmark.yaml | 20 ++ docs/reqstream/ots-mstest.yaml | 28 +++ docs/reqstream/ots-reqstream.yaml | 21 ++ docs/reqstream/ots-requirements.yaml | 102 --------- docs/reqstream/ots-sarifmark.yaml | 21 ++ docs/reqstream/ots-sonarmark.yaml | 23 ++ docs/reqstream/ots-versionmark.yaml | 21 ++ docs/reqstream/reviewmark-system.yaml | 67 ++++++ ...i-requirements.yaml => subsystem-cli.yaml} | 0 ...ents.yaml => subsystem-configuration.yaml} | 0 docs/reqstream/unit-context.yaml | 41 ++++ docs/reqstream/unit-glob-matcher.yaml | 29 +++ docs/reqstream/unit-path-helpers.yaml | 24 +++ docs/reqstream/unit-program.yaml | 45 ++++ ...quirements.yaml => unit-review-index.yaml} | 2 +- docs/reqstream/unit-validation.yaml | 34 +++ requirements.yaml | 19 +- .../{Index.cs => ReviewIndex.cs} | 0 .../ValidationTests.cs | 202 ++++++++++++++++++ 31 files changed, 1200 insertions(+), 125 deletions(-) create mode 100644 docs/design/context.md create mode 100644 docs/design/definition.yaml create mode 100644 docs/design/glob-matcher.md create mode 100644 docs/design/introduction.md create mode 100644 docs/design/path-helpers.md create mode 100644 docs/design/program.md create mode 100644 docs/design/review-index.md create mode 100644 docs/design/review-mark-configuration.md create mode 100644 docs/design/system.md create mode 100644 docs/design/title.txt create mode 100644 docs/design/validation.md create mode 100644 docs/reqstream/ots-buildmark.yaml create mode 100644 docs/reqstream/ots-mstest.yaml create mode 100644 docs/reqstream/ots-reqstream.yaml delete mode 100644 docs/reqstream/ots-requirements.yaml create mode 100644 docs/reqstream/ots-sarifmark.yaml create mode 100644 docs/reqstream/ots-sonarmark.yaml create mode 100644 docs/reqstream/ots-versionmark.yaml create mode 100644 docs/reqstream/reviewmark-system.yaml rename docs/reqstream/{cli-requirements.yaml => subsystem-cli.yaml} (100%) rename docs/reqstream/{configuration-requirements.yaml => subsystem-configuration.yaml} (100%) create mode 100644 docs/reqstream/unit-context.yaml create mode 100644 docs/reqstream/unit-glob-matcher.yaml create mode 100644 docs/reqstream/unit-path-helpers.yaml create mode 100644 docs/reqstream/unit-program.yaml rename docs/reqstream/{index-requirements.yaml => unit-review-index.yaml} (99%) create mode 100644 docs/reqstream/unit-validation.yaml rename src/DemaConsulting.ReviewMark/{Index.cs => ReviewIndex.cs} (100%) create mode 100644 test/DemaConsulting.ReviewMark.Tests/ValidationTests.cs diff --git a/.reviewmark.yaml b/.reviewmark.yaml index d964e3c..a2c7b54 100644 --- a/.reviewmark.yaml +++ b/.reviewmark.yaml @@ -7,7 +7,9 @@ # Processed in order; prefix a pattern with '!' to exclude. needs-review: - "**/*.cs" # All C# source and test files + - "requirements.yaml" # Root requirements file - "docs/reqstream/*.yaml" # Per-software-item requirements files + - "docs/design/*.md" # Software design documents - "!**/obj/**" # Exclude build output - "!**/bin/**" # Exclude build output @@ -30,51 +32,104 @@ reviews: - id: ReviewMark-Context title: Review of Context software unit (command-line argument handling) paths: - - "docs/reqstream/cli-requirements.yaml" # requirements + - "docs/reqstream/unit-context.yaml" # requirements + - "docs/design/context.md" # design - "src/**/Context.cs" # implementation - "test/**/ContextTests.cs" # tests - id: ReviewMark-GlobMatcher title: Review of GlobMatcher software unit (file pattern matching) paths: - - "src/**/GlobMatcher.cs" # implementation - - "test/**/GlobMatcherTests.cs" # tests + - "docs/reqstream/unit-glob-matcher.yaml" # requirements + - "docs/design/glob-matcher.md" # design + - "src/**/GlobMatcher.cs" # implementation + - "test/**/GlobMatcherTests.cs" # tests - - id: ReviewMark-Index - title: Review of Index software unit (review evidence indexing) + - id: ReviewMark-ReviewIndex + title: Review of ReviewIndex software unit (review evidence indexing) paths: - - "docs/reqstream/index-requirements.yaml" # requirements - - "src/**/Index.cs" # implementation + - "docs/reqstream/unit-review-index.yaml" # requirements + - "docs/design/review-index.md" # design + - "src/**/ReviewIndex.cs" # implementation - "test/**/IndexTests.cs" # tests - id: ReviewMark-PathHelpers title: Review of PathHelpers software unit (file path utilities) paths: - - "src/**/PathHelpers.cs" # implementation - - "test/**/PathHelpersTests.cs" # tests + - "docs/reqstream/unit-path-helpers.yaml" # requirements + - "docs/design/path-helpers.md" # design + - "src/**/PathHelpers.cs" # implementation + - "test/**/PathHelpersTests.cs" # tests - id: ReviewMark-Program title: Review of Program software unit (main entry point and tool orchestration) paths: - - "docs/reqstream/cli-requirements.yaml" # requirements - - "docs/reqstream/platform-requirements.yaml" # platform requirements + - "docs/reqstream/unit-program.yaml" # requirements + - "docs/design/program.md" # design - "docs/guide/guide.md" # user guide - "src/**/Program.cs" # implementation - "test/**/ProgramTests.cs" # unit tests - - "test/**/IntegrationTests.cs" # integration tests - - "test/**/Runner.cs" # test infrastructure - "test/**/TestDirectory.cs" # test infrastructure - - "test/**/AssemblyInfo.cs" # test infrastructure - id: ReviewMark-Configuration title: Review of ReviewMarkConfiguration software unit (configuration parsing and processing) paths: - - "docs/reqstream/configuration-requirements.yaml" # requirements - - "src/**/ReviewMarkConfiguration.cs" # implementation - - "test/**/ReviewMarkConfigurationTests.cs" # tests + - "docs/reqstream/subsystem-configuration.yaml" # requirements + - "docs/design/review-mark-configuration.md" # design + - "src/**/ReviewMarkConfiguration.cs" # implementation + - "test/**/ReviewMarkConfigurationTests.cs" # tests - id: ReviewMark-Validation title: Review of Validation software unit (self-validation test execution) paths: - - "docs/reqstream/ots-requirements.yaml" # OTS requirements verified by self-validation + - "docs/reqstream/unit-validation.yaml" # requirements + - "docs/design/validation.md" # design - "src/**/Validation.cs" # implementation + - "test/**/ValidationTests.cs" # tests + + # Special review-sets + - id: ReviewMark-System + title: Review of ReviewMark system-level behavior, platform support, and integration + paths: + - "docs/reqstream/reviewmark-system.yaml" # system requirements + - "docs/reqstream/platform-requirements.yaml" # platform requirements + - "docs/design/introduction.md" # design introduction and architecture + - "docs/design/system.md" # system design + - "test/**/IntegrationTests.cs" # integration tests + - "test/**/Runner.cs" # test infrastructure + - "test/**/AssemblyInfo.cs" # test infrastructure + + - id: ReviewMark-Design + title: Review of all ReviewMark design documentation + paths: + - "docs/reqstream/platform-requirements.yaml" # platform requirements + - "docs/design/introduction.md" # design introduction and architecture + - "docs/design/system.md" # system design + - "docs/design/context.md" # Context design + - "docs/design/glob-matcher.md" # GlobMatcher design + - "docs/design/review-index.md" # ReviewIndex design + - "docs/design/path-helpers.md" # PathHelpers design + - "docs/design/program.md" # Program design + - "docs/design/review-mark-configuration.md" # ReviewMarkConfiguration design + - "docs/design/validation.md" # Validation design + + - id: ReviewMark-AllRequirements + title: Review of all ReviewMark requirements files + paths: + - "requirements.yaml" # root requirements file + - "docs/reqstream/reviewmark-system.yaml" # system-level requirements + - "docs/reqstream/subsystem-cli.yaml" # CLI subsystem requirements + - "docs/reqstream/subsystem-configuration.yaml" # Configuration subsystem requirements + - "docs/reqstream/unit-context.yaml" # Context unit requirements + - "docs/reqstream/unit-program.yaml" # Program unit requirements + - "docs/reqstream/unit-review-index.yaml" # ReviewIndex unit requirements + - "docs/reqstream/unit-glob-matcher.yaml" # GlobMatcher unit requirements + - "docs/reqstream/unit-path-helpers.yaml" # PathHelpers unit requirements + - "docs/reqstream/unit-validation.yaml" # Validation unit requirements + - "docs/reqstream/platform-requirements.yaml" # Platform support requirements + - "docs/reqstream/ots-mstest.yaml" # MSTest OTS requirements + - "docs/reqstream/ots-reqstream.yaml" # ReqStream OTS requirements + - "docs/reqstream/ots-buildmark.yaml" # BuildMark OTS requirements + - "docs/reqstream/ots-versionmark.yaml" # VersionMark OTS requirements + - "docs/reqstream/ots-sarifmark.yaml" # SarifMark OTS requirements + - "docs/reqstream/ots-sonarmark.yaml" # SonarMark OTS requirements diff --git a/docs/design/context.md b/docs/design/context.md new file mode 100644 index 0000000..eed5e1e --- /dev/null +++ b/docs/design/context.md @@ -0,0 +1,59 @@ +# Context + +## Purpose + +The `Context` software unit is responsible for parsing command-line arguments and +providing a unified interface for output and logging throughout the tool. It acts as +the primary configuration carrier, passing parsed options from the CLI entry point +to all processing subsystems. + +## Properties + +The following properties are populated by `Context.Create()` from the command-line +arguments: + +| Property | Type | Description | +| -------- | ---- | ----------- | +| `Version` | bool | Requests version display | +| `Help` | bool | Requests help display | +| `Silent` | bool | Suppresses console output | +| `Validate` | bool | Requests self-validation run | +| `Lint` | bool | Requests configuration linting | +| `ResultsFile` | string? | Path for TRX/JUnit test results output | +| `DefinitionFile` | string | Path to the `.reviewmark.yaml` configuration | +| `PlanFile` | string? | Output path for the Review Plan document | +| `PlanDepth` | int | Heading depth for the Review Plan | +| `ReportFile` | string? | Output path for the Review Report document | +| `ReportDepth` | int | Heading depth for the Review Report | +| `IndexPaths` | string[]? | Paths to scan when building an evidence index | +| `WorkingDirectory` | string | Base directory for resolving relative paths | +| `Enforce` | bool | Fail if any review-set is not Current | +| `Elaborate` | bool | Expand file lists in generated documents | + +## Argument Parsing + +`Context.Create(string[] args)` is a factory method that processes the argument +array sequentially, recognizing both flag arguments (e.g., `--validate`) and +value arguments (e.g., `--plan `). Unrecognized or unsupported arguments +cause `Context.ParseArgument` to throw an `ArgumentException`, which callers of +`Context.Create` are expected to handle and surface as a CLI error. The resulting +`Context` instance holds the fully parsed state when argument parsing succeeds. + +## Output Methods + +| Method | Description | +| ------ | ----------- | +| `WriteLine(string)` | Writes a line to the console (unless `Silent` is set) and to the log file | +| `WriteError(string)` | Writes an error line to the console and to the log file | + +## Exit Code + +`Context.ExitCode` reflects the current error status of the tool run. It is set to +a non-zero value when an error is detected. The value of `ExitCode` is returned from +`Program.Main()` as the process exit code. + +## Logging + +When a log file path is provided via the relevant CLI argument, `Context` opens and +holds the log file handle for the duration of the tool run. All output written through +`WriteLine` and `WriteError` is duplicated to the log file. diff --git a/docs/design/definition.yaml b/docs/design/definition.yaml new file mode 100644 index 0000000..1b115fd --- /dev/null +++ b/docs/design/definition.yaml @@ -0,0 +1,18 @@ +--- +resource-path: + - docs/design + - docs/template +input-files: + - docs/design/title.txt + - docs/design/introduction.md + - docs/design/system.md + - docs/design/context.md + - docs/design/glob-matcher.md + - docs/design/review-index.md + - docs/design/path-helpers.md + - docs/design/program.md + - docs/design/review-mark-configuration.md + - docs/design/validation.md +template: template.html +table-of-contents: true +number-sections: true diff --git a/docs/design/glob-matcher.md b/docs/design/glob-matcher.md new file mode 100644 index 0000000..71c9a1a --- /dev/null +++ b/docs/design/glob-matcher.md @@ -0,0 +1,32 @@ +# GlobMatcher + +## Purpose + +The `GlobMatcher` software unit resolves an ordered list of glob patterns into a +concrete, sorted list of file paths relative to a base directory. It provides the +file enumeration primitive used by the Configuration subsystem to expand the +`needs-review` and `review-set` file lists defined in `.reviewmark.yaml`. + +## Algorithm + +`GlobMatcher.GetMatchingFiles(baseDirectory, patterns)` processes patterns in the +order they are declared. Patterns prefixed with `!` are exclusion patterns; all +others are inclusion patterns. Each inclusion pattern adds matching paths to the +result set; each exclusion pattern removes matching paths from the result set. +Because patterns are applied in declaration order, a later pattern can re-include +files excluded by an earlier one, or exclude files included by an earlier one. The +`**` wildcard matches any number of path segments, enabling recursive matching. +After all patterns are processed, the result set is sorted and returned. + +## Return Value + +The method returns a sorted list of relative file paths. Path separators are +normalized to forward slashes regardless of the host operating system, ensuring +consistent fingerprint computation across platforms. + +## Usage + +`GlobMatcher.GetMatchingFiles()` is called by `ReviewMarkConfiguration` to resolve: + +- The `needs-review` file list, which represents all files subject to review +- Each `review-set` file list, which represents the files covered by a specific review record diff --git a/docs/design/introduction.md b/docs/design/introduction.md new file mode 100644 index 0000000..648be94 --- /dev/null +++ b/docs/design/introduction.md @@ -0,0 +1,56 @@ +# Introduction + +This document describes the software design for the ReviewMark project. + +## Purpose + +ReviewMark is a .NET command-line tool for automated file-review evidence management +in regulated environments. It computes cryptographic fingerprints of defined file-sets, +queries a review evidence store for corresponding review records, and produces compliance +documents on each CI/CD run. + +This design document describes the internal architecture, subsystems, and software units +that together implement the ReviewMark tool. It is intended to support development, +review, and maintenance activities. + +## Scope + +This design document covers: + +- The software system decomposition into subsystems and software units +- The responsibilities and interfaces of each software unit +- The algorithms and data flows used for fingerprinting, evidence lookup, and document generation +- The self-validation framework + +This document does not cover: + +- External CI/CD pipeline configuration +- Evidence store setup or administration +- Requirements traceability (see the Requirements Specification) + +## Software Architecture + +The following diagram shows the decomposition of the ReviewMark software system into +subsystems and software units. + +```text +ReviewMark (Software System) +├── CLI Subsystem +│ ├── Program (Software Unit) +│ └── Context (Software Unit) +├── Configuration Subsystem +│ ├── ReviewMarkConfiguration (Software Unit) +│ └── GlobMatcher (Software Unit) +├── Index Subsystem +│ ├── ReviewIndex (Software Unit) +│ └── PathHelpers (Software Unit) +└── Validation (Software Unit) +``` + +## Audience + +This document is intended for: + +- Software developers working on ReviewMark +- Quality assurance teams performing design verification +- Project stakeholders reviewing architectural decisions diff --git a/docs/design/path-helpers.md b/docs/design/path-helpers.md new file mode 100644 index 0000000..942cae2 --- /dev/null +++ b/docs/design/path-helpers.md @@ -0,0 +1,35 @@ +# PathHelpers + +## Purpose + +The `PathHelpers` software unit provides safe path construction utilities that +prevent path traversal attacks. It is used by the Index subsystem when constructing +file system paths to evidence PDF files referenced in the evidence index. + +## SafePathCombine() + +`PathHelpers.SafePathCombine(basePath, relativePath)` combines a trusted base path +with an untrusted relative path from the evidence index, validating that the result +does not escape the base directory. + +The validation steps are: + +1. Reject any relative path that contains `..` segments (explicit traversal attempt). +2. Reject any relative path that is rooted (absolute path supplied where a relative one is required). +3. Combine the base path and relative path. +4. Verify that the combined path still begins with the base path (catches edge cases + such as platform-specific path normalization that might otherwise bypass the + earlier checks). +5. Return the combined path. + +The double-check strategy (pre-validation of segments plus post-combination +verification) defends against edge cases such as URL-encoded separators or +platform-specific path normalization that might otherwise bypass a single check. + +## Security Rationale + +Evidence index files may be loaded from external sources (file shares or URLs). +The `file` field in each index record is supplied by the evidence store and must +be treated as untrusted input. Without path validation, a maliciously crafted +index could direct the tool to read or reference files outside the intended +evidence directory. `SafePathCombine` eliminates this attack surface. diff --git a/docs/design/program.md b/docs/design/program.md new file mode 100644 index 0000000..6d260cc --- /dev/null +++ b/docs/design/program.md @@ -0,0 +1,47 @@ +# Program + +## Purpose + +The `Program` software unit is the main entry point of the ReviewMark tool. It is +responsible for constructing the execution context, dispatching to the appropriate +processing logic based on parsed flags, and returning a meaningful exit code to the +calling process. + +## Version Property + +`Program.Version` returns the tool version string. The version is embedded at build +time from the assembly metadata and follows semantic versioning conventions. + +## Main() Method + +`Program.Main(string[] args)` is the process entry point. It: + +1. Constructs a `Context` instance via `Context.Create(args)` inside a `using` block +2. Calls `Program.Run(Context)` to perform the requested operation +3. Returns `Context.ExitCode` as the process exit code + +Any unexpected exception that escapes `Run()` is logged to the standard error stream +via `Console.Error` and then rethrown. As a result, the process terminates due to the +unhandled exception and the final exit code is determined by the .NET runtime rather +than by `Program.Main` explicitly returning a non-zero value. + +## Run() Dispatch Logic + +`Program.Run(Context)` evaluates the parsed flags in the following priority order, +executing the first matching action and returning: + +1. If `--version` — print version and return +2. If `--help` — print banner and return +3. If `--validate` — run self-validation and return +4. If `--lint` — run configuration lint and return +5. If `--index` paths provided — scan and write evidence index, then return +6. Otherwise — generate Review Plan and/or Review Report and return + +Only one top-level action is performed per invocation. Actions later in the priority +order are not reached if an earlier flag is set. + +## PrintBanner() + +`Program.PrintBanner(Context)` writes the help text to the console via +`Context.WriteLine()`. The banner lists all supported flags and arguments with brief +descriptions. diff --git a/docs/design/review-index.md b/docs/design/review-index.md new file mode 100644 index 0000000..1a3ef95 --- /dev/null +++ b/docs/design/review-index.md @@ -0,0 +1,83 @@ +# ReviewIndex + +## Purpose + +The `ReviewIndex` software unit manages the loading, querying, and creation of the review +evidence index. It abstracts the evidence store behind a uniform interface so that +the rest of the tool does not need to know whether evidence is stored on a fileshare, +served over HTTP, or absent entirely. + +## ReviewEvidence Record + +`ReviewEvidence` is an immutable record that holds the in-memory representation of a +single review record once the index has been loaded or scanned. + +| Property | Type | Description | +| -------- | ---- | ----------- | +| `Id` | string | The review-set identifier | +| `Fingerprint` | string | The SHA-256 fingerprint of the reviewed files | +| `Date` | string | The date of the review (e.g. `2026-02-14`) | +| `Result` | string | The review outcome (`pass` or `fail`) | +| `File` | string | The file name of the review evidence PDF | + +The `ReviewIndex` holds these records in a two-level +`Dictionary>` keyed first by `Id` and +then by `Fingerprint`, which enables O(1) lookup by both fields simultaneously. + +## Evidence Index Format + +The evidence index is a JSON file (`index.json`) containing an array of review records. +Each record has the following fields: + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `id` | string | Unique identifier for the review record (matches the review-set `id` in `.reviewmark.yaml`) | +| `fingerprint` | string | SHA-256 fingerprint of the file-set at time of review | +| `date` | string | Date the review was conducted | +| `result` | string | Review outcome (`pass` or `fail`) | +| `file` | string | Relative path to the PDF evidence file | + +## ReviewIndex.Load() + +`ReviewIndex.Load(EvidenceSource)` selects a loading strategy based on the evidence +source type: + +| Source Type | Behavior | +| ----------- | -------- | +| `none` | Returns an empty index (equivalent to `ReviewIndex.Empty()`) | +| `fileshare` | Reads `index.json` from the specified file path | +| `url` | Downloads `index.json` from the specified HTTP or HTTPS URL | + +## ReviewIndex.Scan() + +`ReviewIndex.Scan(directory, patterns)` scans a directory for PDF files matching +the given glob patterns. For each PDF file found, it reads embedded metadata to +extract the review record fields and returns a populated in-memory `ReviewIndex`. +The caller (e.g., `Program`) is responsible for choosing an output path and calling +`Save(...)` on the returned index to produce `index.json` as part of the `--index` +workflow. + +## ReviewIndex.Empty() + +`ReviewIndex.Empty()` returns an index with no records. It is used when the evidence +source type is `none`, resulting in all review-sets being reported as Missing. + +## ReviewIndex.GetStatus() + +`ReviewIndex.GetStatus(id, fingerprint)` determines the review status of a +review-set by looking up the `id` in the loaded index: + +1. Look up `id` in the index + - If not found — return `Missing` +2. Check if there is a record whose `Fingerprint` matches the supplied `fingerprint` + - If no matching fingerprint exists — return `Stale` + - If a matching fingerprint exists: + - If the `Result` is `pass` — return `Current` + - If the `Result` is not `pass` — return `Failed` + +| Status | Meaning | +| ------ | ------- | +| `Current` | The review record matches the current fingerprint and has a passing result | +| `Failed` | The review record matches the current fingerprint but the result is not passing | +| `Stale` | A record exists for the id but the fingerprint does not match the current one | +| `Missing` | No review record exists for the id | diff --git a/docs/design/review-mark-configuration.md b/docs/design/review-mark-configuration.md new file mode 100644 index 0000000..9692eb1 --- /dev/null +++ b/docs/design/review-mark-configuration.md @@ -0,0 +1,75 @@ +# ReviewMarkConfiguration + +## Purpose + +The `ReviewMarkConfiguration` software unit is responsible for parsing the +`.reviewmark.yaml` configuration file and performing all review-set processing. +It coordinates file enumeration, fingerprint computation, evidence lookup, and +the generation of the Review Plan and Review Report compliance documents. + +## Configuration Model + +The `.reviewmark.yaml` file is deserialized into the following model: + +| Class | Description | +| ----- | ----------- | +| `ReviewMarkYaml` | Root configuration object containing the evidence source and review list | +| `EvidenceSourceYaml` | Describes how to locate the evidence index (`type`, `location`, optional `credentials`) | +| `ReviewYaml` | Describes a single review-set (`id`, `title`, file patterns) | + +## ReviewMarkConfiguration.Load() + +`ReviewMarkConfiguration.Load(definitionFile, workingDirectory)` reads and +deserializes the YAML file, resolves all glob patterns relative to the working +directory, computes fingerprints for each review-set, loads the evidence index, +and returns a fully initialized configuration object ready for plan/report generation. + +## Fingerprinting Algorithm + +The fingerprint for a review-set uniquely identifies the exact content of its file-set. +The algorithm is: + +1. For each file in the review-set, read its contents and compute a SHA-256 hash. +2. Collect all per-file hashes and sort them lexicographically. +3. Concatenate the sorted hashes and compute a SHA-256 hash of the result. +4. Return the final hash as a hex string — this is the review-set fingerprint. + +Sorting the per-file hashes before combining them ensures that the fingerprint is +sensitive to content changes but not to the order in which files happen to be +enumerated by the operating system. + +## Review Plan Generation + +The Review Plan is generated by `ReviewMarkConfiguration.PublishReviewPlan()`. It produces +a Markdown document that lists every file in the `needs-review` file-set and, for +each file, identifies which review-sets provide coverage. + +- The `--plan-depth` argument controls the heading level used for sections +- The `--elaborate` flag expands the file list for each review-set inline + +## Review Report Generation + +The Review Report is generated by `ReviewMarkConfiguration.PublishReviewReport()`. It +produces a Markdown document that lists every review-set with its current status. + +For each review-set the report includes: + +- The review-set `id` and `title` +- The current fingerprint of the file-set +- The review status: `Current`, `Stale`, `Missing`, or `Failed` + +Status is determined by looking up the current fingerprint in the loaded evidence +index to establish whether a passing, failing, stale, or missing review result exists. + +- The `--report-depth` argument controls the heading level used for sections +- The `--elaborate` flag expands the list of files covered by each review-set + +## Linting + +`ReviewMarkConfiguration.Lint(Context)` validates the loaded configuration for +correctness. Lint checks include: + +- All review-set `id` values are unique +- All glob patterns resolve to at least one file +- The `needs-review` file-set is non-empty +- All files in the `needs-review` set are covered by at least one review-set diff --git a/docs/design/system.md b/docs/design/system.md new file mode 100644 index 0000000..0f37a4f --- /dev/null +++ b/docs/design/system.md @@ -0,0 +1,73 @@ +# System Design + +This section describes the high-level behavior of the ReviewMark system and the workflow +that connects its subsystems. + +## Overview + +ReviewMark automates the evidence-gathering step of software review processes used in +regulated environments. On each CI/CD run, it determines which files are subject to +review, identifies the review evidence that covers them, and generates two compliance +documents: a Review Plan and a Review Report. + +## Main Workflow + +The following steps describe the end-to-end processing flow. + +1. Parse CLI arguments +2. Load `.reviewmark.yaml` +3. Resolve file lists via glob patterns +4. Compute SHA-256 fingerprints +5. Load evidence index + - `none` — use an empty index (no evidence store configured) + - `fileshare` — load `index.json` from a local or network file path + - `url` — download `index.json` from an HTTP or HTTPS URL +6. Generate Review Plan and/or Review Report +7. If `--enforce` flag is set: + - If all review-sets are Current — return success + - Otherwise — return a non-zero exit code + +## Evidence Source Types + +ReviewMark supports three evidence source types, configured in `.reviewmark.yaml`: + +| Source Type | Description | +| ----------- | ----------- | +| `none` | No evidence store; all review-sets are treated as missing | +| `fileshare` | Evidence index loaded from a local or network file path | +| `url` | Evidence index loaded from an HTTP or HTTPS URL | + +## Output Documents + +### Review Plan + +The Review Plan lists every file that is subject to review and identifies which +review-sets provide coverage for each file. It is generated by the `--plan` flag +and written to a configurable output path. + +### Review Report + +The Review Report lists every review-set defined in the configuration, the current +fingerprint of its file-set, and the review status (Current, Stale, Missing, or Failed). +It is generated by the `--report` flag and written to a configurable output path. + +The statuses have the following meanings: + +- **Current** — Evidence exists for the current fingerprint and the recorded result is `pass`. +- **Stale** — Evidence exists, but it corresponds to an older fingerprint than the current one. +- **Missing** — No evidence exists for this review-set. +- **Failed** — Evidence exists for the current fingerprint, but the recorded result is not `pass`. + +## Enforcement + +When the `--enforce` flag is set, ReviewMark returns a non-zero exit code if any +review-set does not have Current status (i.e., is Stale, Missing, or Failed). This allows +CI/CD pipelines to fail builds when review coverage is incomplete, out of date, or has +failed results for the current fingerprint. + +## Index Management + +The `--index` flag causes ReviewMark to scan a directory for PDF evidence files and +write an `index.json` file suitable for use as a fileshare evidence source. This +supports workflows where review PDFs are stored alongside source code or on a +shared network location. diff --git a/docs/design/title.txt b/docs/design/title.txt new file mode 100644 index 0000000..d140ba3 --- /dev/null +++ b/docs/design/title.txt @@ -0,0 +1,13 @@ +--- +title: ReviewMark Design +subtitle: Software Design Document for ReviewMark +author: DEMA Consulting +description: Software Design Document for ReviewMark +lang: en-US +keywords: + - ReviewMark + - Design + - Software Architecture + - .NET + - Command-Line Tool +--- diff --git a/docs/design/validation.md b/docs/design/validation.md new file mode 100644 index 0000000..04ff878 --- /dev/null +++ b/docs/design/validation.md @@ -0,0 +1,44 @@ +# Validation + +## Purpose + +The `Validation` software unit implements the self-validation framework for +ReviewMark. Self-validation allows the tool to verify its own correct operation +in a target environment, which is a requirement for regulated deployment contexts +where the tool itself is part of a qualified software chain. + +## Validation.Run() + +`Validation.Run(Context)` orchestrates all self-validation tests. It: + +1. Creates a test suite using the `DemaConsulting.TestResults` library +2. Executes each test case in sequence +3. Writes results to the configured output file (TRX or JUnit format) if `ResultsFile` is set +4. Writes a summary table and per-test results to the console via `Context.WriteLine()` +5. Sets `Context.ExitCode` to a non-zero value if any test fails + +## Test Output Format + +Results are written using the `DemaConsulting.TestResults` library, which supports +both TRX (Visual Studio Test Results) and JUnit XML output formats. The output format +is inferred from the file extension of `ResultsFile`. + +## Test Coverage + +The self-validation suite covers the following scenarios: + +- **Version display**: Tool correctly reports its version +- **Help display**: Tool correctly displays help text +- **Plan generation**: Review Plan is generated correctly for a known configuration +- **Report generation**: Review Report is generated correctly for a known configuration +- **Index scanning**: Evidence index is created correctly by scanning a directory +- **Enforce mode**: Tool returns non-zero exit code when enforce mode detects uncovered review sets +- **Working directory override**: Relative paths are resolved correctly when the working directory is overridden +- **Elaborate mode**: File lists are expanded in generated documents when elaborate mode is active +- **Lint mode**: Configuration errors are detected correctly + +## Console Output + +In addition to the structured results file, `Validation.Run()` writes a human-readable +summary to the console. The summary includes a table of all tests with their pass/fail +status, followed by detailed output for any failing tests to aid diagnosis. diff --git a/docs/reqstream/ots-buildmark.yaml b/docs/reqstream/ots-buildmark.yaml new file mode 100644 index 0000000..d59a4a7 --- /dev/null +++ b/docs/reqstream/ots-buildmark.yaml @@ -0,0 +1,20 @@ +--- +# BuildMark OTS Requirements +# +# PURPOSE: +# - Define requirements for the BuildMark off-the-shelf documentation generation tool +# - BuildMark generates build-notes documentation from GitHub Actions metadata + +sections: + - title: BuildMark OTS Requirements + requirements: + - id: ReviewMark-OTS-BuildMark + title: BuildMark shall generate build-notes documentation from GitHub Actions metadata. + justification: | + DemaConsulting.BuildMark queries the GitHub API to capture workflow run details and + renders them as a markdown build-notes document included in the release artifacts. + It runs as part of the same CI pipeline that produces the TRX test results, so a + successful pipeline run is evidence that BuildMark executed without error. + tags: [ots] + tests: + - BuildMark_MarkdownReportGeneration diff --git a/docs/reqstream/ots-mstest.yaml b/docs/reqstream/ots-mstest.yaml new file mode 100644 index 0000000..98dd61a --- /dev/null +++ b/docs/reqstream/ots-mstest.yaml @@ -0,0 +1,28 @@ +--- +# MSTest OTS Requirements +# +# PURPOSE: +# - Define requirements for the MSTest off-the-shelf testing framework +# - MSTest is used to discover, execute, and report unit test results + +sections: + - title: MSTest OTS Requirements + requirements: + - id: ReviewMark-OTS-MSTest + title: MSTest shall execute unit tests and report results. + justification: | + MSTest (MSTest.TestFramework and MSTest.TestAdapter) is the unit-testing framework used + by the project. It discovers and runs all test methods and writes TRX result files that + feed into coverage reporting and requirements traceability. Passing tests confirm the + framework is functioning correctly. + tags: [ots] + tests: + - Context_Create_NoArguments_ReturnsDefaultContext + - Context_Create_VersionFlag_SetsVersionTrue + - Context_Create_HelpFlag_SetsHelpTrue + - Context_Create_SilentFlag_SetsSilentTrue + - Context_Create_ValidateFlag_SetsValidateTrue + - Context_Create_ResultsFlag_SetsResultsFile + - Context_Create_LogFlag_OpensLogFile + - Context_Create_UnknownArgument_ThrowsArgumentException + - Context_Create_ShortVersionFlag_SetsVersionTrue diff --git a/docs/reqstream/ots-reqstream.yaml b/docs/reqstream/ots-reqstream.yaml new file mode 100644 index 0000000..908a75f --- /dev/null +++ b/docs/reqstream/ots-reqstream.yaml @@ -0,0 +1,21 @@ +--- +# ReqStream OTS Requirements +# +# PURPOSE: +# - Define requirements for the ReqStream off-the-shelf requirements traceability tool +# - ReqStream validates that every requirement is linked to passing test evidence + +sections: + - title: ReqStream OTS Requirements + requirements: + - id: ReviewMark-OTS-ReqStream + title: ReqStream shall enforce that every requirement is linked to passing test evidence. + justification: | + DemaConsulting.ReqStream processes requirements.yaml and the TRX test-result files to + produce a requirements report, justifications document, and traceability matrix. When + run with --enforce, it exits with a non-zero code if any requirement lacks test evidence, + making unproven requirements a build-breaking condition. A successful pipeline run with + --enforce proves all requirements are covered and that ReqStream is functioning. + tags: [ots] + tests: + - ReqStream_EnforcementMode diff --git a/docs/reqstream/ots-requirements.yaml b/docs/reqstream/ots-requirements.yaml deleted file mode 100644 index b763998..0000000 --- a/docs/reqstream/ots-requirements.yaml +++ /dev/null @@ -1,102 +0,0 @@ ---- -# OTS (Off-the-Shelf) Software Requirements -# -# PURPOSE: -# - Define requirements for third-party components used by ReviewMark -# - OTS requirements document which capabilities the project depends on -# - Tests verify the OTS component provides the required behavior in this environment - -sections: - - title: OTS Software Requirements - sections: - - title: MSTest - requirements: - - id: ReviewMark-OTS-MSTest - title: MSTest shall execute unit tests and report results. - justification: | - MSTest (MSTest.TestFramework and MSTest.TestAdapter) is the unit-testing framework used - by the project. It discovers and runs all test methods and writes TRX result files that - feed into coverage reporting and requirements traceability. Passing tests confirm the - framework is functioning correctly. - tags: [ots] - tests: - - Context_Create_NoArguments_ReturnsDefaultContext - - Context_Create_VersionFlag_SetsVersionTrue - - Context_Create_HelpFlag_SetsHelpTrue - - Context_Create_SilentFlag_SetsSilentTrue - - Context_Create_ValidateFlag_SetsValidateTrue - - Context_Create_ResultsFlag_SetsResultsFile - - Context_Create_LogFlag_OpensLogFile - - Context_Create_UnknownArgument_ThrowsArgumentException - - Context_Create_ShortVersionFlag_SetsVersionTrue - - - title: ReqStream - requirements: - - id: ReviewMark-OTS-ReqStream - title: ReqStream shall enforce that every requirement is linked to passing test evidence. - justification: | - DemaConsulting.ReqStream processes requirements.yaml and the TRX test-result files to - produce a requirements report, justifications document, and traceability matrix. When - run with --enforce, it exits with a non-zero code if any requirement lacks test evidence, - making unproven requirements a build-breaking condition. A successful pipeline run with - --enforce proves all requirements are covered and that ReqStream is functioning. - tags: [ots] - tests: - - ReqStream_EnforcementMode - - - title: BuildMark - requirements: - - id: ReviewMark-OTS-BuildMark - title: BuildMark shall generate build-notes documentation from GitHub Actions metadata. - justification: | - DemaConsulting.BuildMark queries the GitHub API to capture workflow run details and - renders them as a markdown build-notes document included in the release artifacts. - It runs as part of the same CI pipeline that produces the TRX test results, so a - successful pipeline run is evidence that BuildMark executed without error. - tags: [ots] - tests: - - BuildMark_MarkdownReportGeneration - - - title: VersionMark - requirements: - - id: ReviewMark-OTS-VersionMark - title: VersionMark shall publish captured tool-version information. - justification: | - DemaConsulting.VersionMark reads version metadata for each dotnet tool used in the - pipeline and writes a versions markdown document included in the release artifacts. - It runs in the same CI pipeline that produces the TRX test results, so a successful - pipeline run is evidence that VersionMark executed without error. - tags: [ots] - tests: - - VersionMark_CapturesVersions - - VersionMark_GeneratesMarkdownReport - - - title: SarifMark - requirements: - - id: ReviewMark-OTS-SarifMark - title: SarifMark shall convert CodeQL SARIF results into a markdown report. - justification: | - DemaConsulting.SarifMark reads the SARIF output produced by CodeQL code scanning and - renders it as a human-readable markdown document included in the release artifacts. - It runs in the same CI pipeline that produces the TRX test results, so a successful - pipeline run is evidence that SarifMark executed without error. - tags: [ots] - tests: - - SarifMark_SarifReading - - SarifMark_MarkdownReportGeneration - - - title: SonarMark - requirements: - - id: ReviewMark-OTS-SonarMark - title: SonarMark shall generate a SonarCloud quality report. - justification: | - DemaConsulting.SonarMark retrieves quality-gate and metrics data from SonarCloud and - renders it as a markdown document included in the release artifacts. It runs in the - same CI pipeline that produces the TRX test results, so a successful pipeline run is - evidence that SonarMark executed without error. - tags: [ots] - tests: - - SonarMark_QualityGateRetrieval - - SonarMark_IssuesRetrieval - - SonarMark_HotSpotsRetrieval - - SonarMark_MarkdownReportGeneration diff --git a/docs/reqstream/ots-sarifmark.yaml b/docs/reqstream/ots-sarifmark.yaml new file mode 100644 index 0000000..c49a525 --- /dev/null +++ b/docs/reqstream/ots-sarifmark.yaml @@ -0,0 +1,21 @@ +--- +# SarifMark OTS Requirements +# +# PURPOSE: +# - Define requirements for the SarifMark off-the-shelf SARIF reporting tool +# - SarifMark converts CodeQL SARIF results into a human-readable markdown report + +sections: + - title: SarifMark OTS Requirements + requirements: + - id: ReviewMark-OTS-SarifMark + title: SarifMark shall convert CodeQL SARIF results into a markdown report. + justification: | + DemaConsulting.SarifMark reads the SARIF output produced by CodeQL code scanning and + renders it as a human-readable markdown document included in the release artifacts. + It runs in the same CI pipeline that produces the TRX test results, so a successful + pipeline run is evidence that SarifMark executed without error. + tags: [ots] + tests: + - SarifMark_SarifReading + - SarifMark_MarkdownReportGeneration diff --git a/docs/reqstream/ots-sonarmark.yaml b/docs/reqstream/ots-sonarmark.yaml new file mode 100644 index 0000000..791d57e --- /dev/null +++ b/docs/reqstream/ots-sonarmark.yaml @@ -0,0 +1,23 @@ +--- +# SonarMark OTS Requirements +# +# PURPOSE: +# - Define requirements for the SonarMark off-the-shelf SonarCloud reporting tool +# - SonarMark generates a SonarCloud quality report as part of release artifacts + +sections: + - title: SonarMark OTS Requirements + requirements: + - id: ReviewMark-OTS-SonarMark + title: SonarMark shall generate a SonarCloud quality report. + justification: | + DemaConsulting.SonarMark retrieves quality-gate and metrics data from SonarCloud and + renders it as a markdown document included in the release artifacts. It runs in the + same CI pipeline that produces the TRX test results, so a successful pipeline run is + evidence that SonarMark executed without error. + tags: [ots] + tests: + - SonarMark_QualityGateRetrieval + - SonarMark_IssuesRetrieval + - SonarMark_HotSpotsRetrieval + - SonarMark_MarkdownReportGeneration diff --git a/docs/reqstream/ots-versionmark.yaml b/docs/reqstream/ots-versionmark.yaml new file mode 100644 index 0000000..58f0928 --- /dev/null +++ b/docs/reqstream/ots-versionmark.yaml @@ -0,0 +1,21 @@ +--- +# VersionMark OTS Requirements +# +# PURPOSE: +# - Define requirements for the VersionMark off-the-shelf tool-version documentation tool +# - VersionMark publishes captured tool-version information as part of release artifacts + +sections: + - title: VersionMark OTS Requirements + requirements: + - id: ReviewMark-OTS-VersionMark + title: VersionMark shall publish captured tool-version information. + justification: | + DemaConsulting.VersionMark reads version metadata for each dotnet tool used in the + pipeline and writes a versions markdown document included in the release artifacts. + It runs in the same CI pipeline that produces the TRX test results, so a successful + pipeline run is evidence that VersionMark executed without error. + tags: [ots] + tests: + - VersionMark_CapturesVersions + - VersionMark_GeneratesMarkdownReport diff --git a/docs/reqstream/reviewmark-system.yaml b/docs/reqstream/reviewmark-system.yaml new file mode 100644 index 0000000..5e51e6e --- /dev/null +++ b/docs/reqstream/reviewmark-system.yaml @@ -0,0 +1,67 @@ +--- +# ReviewMark System-Level Requirements +# +# PURPOSE: +# - Define system-level requirements describing what end-users need the ReviewMark tool to provide +# - These requirements capture the externally visible behavior of the complete ReviewMark system +# - Unit-level requirements (per-class behavior) are in the individual *-requirements.yaml files + +sections: + - title: System-Level Requirements + requirements: + - id: ReviewMark-System-ReviewPlan + title: >- + The tool shall generate a Review Plan document listing all files requiring review and their + review-set coverage. + justification: | + In regulated environments, auditors require evidence that every file subject to review + is covered by at least one named review-set. The Review Plan document provides this + evidence automatically on each CI/CD run, replacing manual tracking spreadsheets. + tests: + - ReviewMark_ReviewPlanGeneration + + - id: ReviewMark-System-ReviewReport + title: The tool shall generate a Review Report document listing every review-set and its current review status. + justification: | + Auditors need evidence that the review evidence for each review-set is current — + that the reviewed files have not changed since the review was conducted. The Review + Report provides this evidence automatically, showing Current, Stale, or Missing + status for each review-set. + tests: + - ReviewMark_ReviewReportGeneration + + - id: ReviewMark-System-Enforce + title: The tool shall return a non-zero exit code when enforcement is enabled and any review-set is not current. + justification: | + CI/CD pipelines must be able to gate releases on review coverage. The --enforce flag + enables this by causing the tool to exit with a non-zero code when any review-set has + Stale or Missing status, making incomplete review coverage a build-breaking condition. + tests: + - ReviewMark_Enforce + + - id: ReviewMark-System-IndexScan + title: The tool shall scan PDF evidence files and write an index.json when the --index flag is provided. + justification: | + Review evidence PDFs contain embedded metadata (id, fingerprint, date, result) in their + Keywords field. The --index command scans a directory of such PDFs and writes an + index.json, enabling the evidence store to be refreshed after new review PDFs are added + without manual maintenance of the index file. + tests: + - ReviewMark_IndexScan + + - id: ReviewMark-System-Validate + title: The tool shall execute self-validation tests when the --validate flag is provided. + justification: | + Regulated environments require tool qualification evidence to demonstrate that the tool + functions correctly in its specific deployment environment. The --validate flag triggers + a built-in test suite that exercises core tool behaviors and produces a pass/fail report. + tests: + - ReviewMark_VersionDisplay + - ReviewMark_HelpDisplay + - ReviewMark_ReviewPlanGeneration + - ReviewMark_ReviewReportGeneration + - ReviewMark_IndexScan + - ReviewMark_Enforce + - ReviewMark_WorkingDirectoryOverride + - ReviewMark_Elaborate + - ReviewMark_Lint diff --git a/docs/reqstream/cli-requirements.yaml b/docs/reqstream/subsystem-cli.yaml similarity index 100% rename from docs/reqstream/cli-requirements.yaml rename to docs/reqstream/subsystem-cli.yaml diff --git a/docs/reqstream/configuration-requirements.yaml b/docs/reqstream/subsystem-configuration.yaml similarity index 100% rename from docs/reqstream/configuration-requirements.yaml rename to docs/reqstream/subsystem-configuration.yaml diff --git a/docs/reqstream/unit-context.yaml b/docs/reqstream/unit-context.yaml new file mode 100644 index 0000000..2c880de --- /dev/null +++ b/docs/reqstream/unit-context.yaml @@ -0,0 +1,41 @@ +--- +# Context Software Unit Requirements +# +# PURPOSE: +# - Define requirements for the Context software unit +# - This unit parses command-line arguments into an in-memory context object +# - It also provides unified output and logging across the tool + +sections: + - title: Context Unit Requirements + requirements: + - id: ReviewMark-Context-Parsing + title: The Context unit shall parse command-line arguments into a strongly-typed Context object. + justification: | + All downstream processing reads options from the Context object rather than + directly from the raw argument array. The Context.Create factory method processes + arguments sequentially, recognizing flag and value arguments, and returns a fully + initialized Context. Unknown arguments must raise an ArgumentException so the + caller can report a clear error message. + tests: + - Context_Create_NoArguments_ReturnsDefaultContext + - Context_Create_VersionFlag_SetsVersionTrue + - Context_Create_HelpFlag_SetsHelpTrue + - Context_Create_SilentFlag_SetsSilentTrue + - Context_Create_ValidateFlag_SetsValidateTrue + - Context_Create_ResultsFlag_SetsResultsFile + - Context_Create_LogFlag_OpensLogFile + - Context_Create_UnknownArgument_ThrowsArgumentException + - Context_Create_ShortVersionFlag_SetsVersionTrue + + - id: ReviewMark-Context-Output + title: The Context unit shall provide WriteLine and WriteError methods for unified output and logging. + justification: | + All output goes through Context so that the --silent flag is honoured and + optionally duplicated to a log file opened by the --log flag. WriteError must + additionally set the error exit code so that the process exits with a non-zero + status when any error is reported. + tests: + - Context_WriteError_NotSilent_WritesToConsole + - Context_WriteError_SetsErrorExitCode + - Context_WriteLine_Silent_DoesNotWriteToConsole diff --git a/docs/reqstream/unit-glob-matcher.yaml b/docs/reqstream/unit-glob-matcher.yaml new file mode 100644 index 0000000..2529257 --- /dev/null +++ b/docs/reqstream/unit-glob-matcher.yaml @@ -0,0 +1,29 @@ +--- +# GlobMatcher Software Unit Requirements +# +# PURPOSE: +# - Define requirements for the GlobMatcher software unit +# - This unit resolves ordered include/exclude glob patterns to a list of files +# - It is used by ReviewMarkConfiguration to resolve needs-review and review-set file lists + +sections: + - title: GlobMatcher Unit Requirements + requirements: + - id: ReviewMark-GlobMatcher-IncludeExclude + title: >- + The GlobMatcher shall resolve ordered include and exclude glob patterns to a sorted list of + relative file paths. + justification: | + Review-set and needs-review configurations specify files using ordered glob patterns, + where patterns prefixed with '!' are exclusions. The GlobMatcher must apply these + patterns in declaration order so that a later include can re-add files removed by an + earlier exclude, and vice versa. The result must be sorted to ensure deterministic + fingerprinting regardless of filesystem iteration order. + tests: + - GlobMatcher_GetMatchingFiles_SingleIncludePattern_ReturnsMatchingFiles + - GlobMatcher_GetMatchingFiles_ExcludePattern_ExcludesMatchingFiles + - GlobMatcher_GetMatchingFiles_ReIncludeAfterExclude_ReturnsReIncludedFiles + - GlobMatcher_GetMatchingFiles_IncludeAndExclude_ReturnsFilteredFiles + - GlobMatcher_GetMatchingFiles_NullBaseDirectory_ThrowsArgumentNullException + - GlobMatcher_GetMatchingFiles_NullPatterns_ThrowsArgumentNullException + - GlobMatcher_GetMatchingFiles_NoMatchingFiles_ReturnsEmptyList diff --git a/docs/reqstream/unit-path-helpers.yaml b/docs/reqstream/unit-path-helpers.yaml new file mode 100644 index 0000000..f9295bc --- /dev/null +++ b/docs/reqstream/unit-path-helpers.yaml @@ -0,0 +1,24 @@ +--- +# PathHelpers Software Unit Requirements +# +# PURPOSE: +# - Define requirements for the PathHelpers software unit +# - This unit provides safe path operations that prevent path traversal attacks +# - It is used by ReviewIndex.cs and Validation.cs when constructing file paths + +sections: + - title: PathHelpers Unit Requirements + requirements: + - id: ReviewMark-PathHelpers-SafeCombine + title: The PathHelpers shall safely combine a base path and a relative path, rejecting path traversal attempts. + justification: | + When constructing file paths from user-supplied or externally-sourced components + (such as relative paths read from an evidence index), the tool must prevent path + traversal attacks. SafePathCombine validates that the relative path does not + contain '..' sequences or absolute path components, and performs a defense-in-depth + check that the resolved combined path remains under the base directory. + tests: + - PathHelpers_SafePathCombine_ValidPaths_CombinesCorrectly + - PathHelpers_SafePathCombine_PathTraversalWithDoubleDots_ThrowsArgumentException + - PathHelpers_SafePathCombine_AbsolutePath_ThrowsArgumentException + - PathHelpers_SafePathCombine_NestedPaths_CombinesCorrectly diff --git a/docs/reqstream/unit-program.yaml b/docs/reqstream/unit-program.yaml new file mode 100644 index 0000000..356f383 --- /dev/null +++ b/docs/reqstream/unit-program.yaml @@ -0,0 +1,45 @@ +--- +# Program Software Unit Requirements +# +# PURPOSE: +# - Define requirements for the Program software unit +# - This unit is the main entry point and top-level orchestrator of the tool +# - It dispatches to processing logic based on parsed CLI flags + +sections: + - title: Program Unit Requirements + requirements: + - id: ReviewMark-Program-EntryPoint + title: >- + The Program unit shall construct a Context, dispatch to the appropriate operation, + and return the Context exit code as the process exit code. + justification: | + Program.Main is the process entry point. It must create the execution context, + call Program.Run to perform the requested operation, and return the exit code + from the context so that callers can detect success or failure programmatically. + Unexpected exceptions are written to error output and then rethrown, so callers + may observe either a normal exit code or a process termination due to an + unhandled exception. + tests: + - Program_Run_WithVersionFlag_DisplaysVersionOnly + - Program_Version_ReturnsNonEmptyString + - Program_Run_WithHelpFlag_DisplaysUsageInformation + + - id: ReviewMark-Program-Dispatch + title: >- + The Program unit shall dispatch to exactly one operation per invocation based on + the priority order of CLI flags. + justification: | + --version, --help, --validate, --lint, --index, and plan/report operations must + be evaluated in a fixed priority order so that the behavior is predictable and + documented. Only the first matching flag action is executed; later flags are + not reached. + tests: + - Program_Run_WithVersionFlag_DisplaysVersionOnly + - Program_Run_WithHelpFlag_DisplaysUsageInformation + - Program_Run_WithValidateFlag_RunsValidation + - Program_Run_WithLintFlag_ValidConfig_ReportsSuccess + - Program_Run_WithHelpFlag_IncludesElaborateOption + - Program_Run_WithHelpFlag_IncludesLintOption + - Program_Run_WithElaborateFlag_OutputsElaboration + - Program_Run_WithElaborateFlag_UnknownId_ReportsError diff --git a/docs/reqstream/index-requirements.yaml b/docs/reqstream/unit-review-index.yaml similarity index 99% rename from docs/reqstream/index-requirements.yaml rename to docs/reqstream/unit-review-index.yaml index 0abf7b8..ad05a88 100644 --- a/docs/reqstream/index-requirements.yaml +++ b/docs/reqstream/unit-review-index.yaml @@ -1,5 +1,5 @@ --- -# Index Software Unit Requirements +# ReviewIndex Software Unit Requirements # # PURPOSE: # - Define requirements for the ReviewIndex software unit diff --git a/docs/reqstream/unit-validation.yaml b/docs/reqstream/unit-validation.yaml new file mode 100644 index 0000000..622ccd1 --- /dev/null +++ b/docs/reqstream/unit-validation.yaml @@ -0,0 +1,34 @@ +--- +# Validation Software Unit Requirements +# +# PURPOSE: +# - Define requirements for the Validation software unit +# - This unit provides self-validation test execution for regulated environments +# - Self-validation proves the tool is functioning correctly in its deployment environment + +sections: + - title: Validation Unit Requirements + requirements: + - id: ReviewMark-Validation-Run + title: The tool shall execute self-validation tests and report results when the --validate flag is provided. + justification: | + In regulated environments, tool qualification evidence is required to demonstrate + that the tool functions correctly in its deployment environment. Self-validation + runs a suite of functional tests covering core behaviors and reports pass/fail + results with a summary count, giving quality assurance teams the evidence they need. + tests: + - Validation_Run_NullContext_ThrowsArgumentNullException + - Validation_Run_WritesValidationHeader + - Validation_Run_WritesSummaryWithTotalTests + - Validation_Run_AllTestsPass_ExitCodeIsZero + + - id: ReviewMark-Validation-ResultsFile + title: The tool shall write self-validation results to a TRX or JUnit XML file when --results is provided. + justification: | + CI/CD pipelines and requirements traceability tools (such as ReqStream) consume + test result files in standard formats. By supporting both TRX (MSTest) and JUnit + XML output, the self-validation results can be fed directly into pipeline tooling + without additional conversion steps. + tests: + - Validation_Run_WithTrxResultsFile_WritesFile + - Validation_Run_WithXmlResultsFile_WritesFile diff --git a/requirements.yaml b/requirements.yaml index 3654661..341b6db 100644 --- a/requirements.yaml +++ b/requirements.yaml @@ -24,8 +24,19 @@ # --- includes: - - docs/reqstream/cli-requirements.yaml - - docs/reqstream/configuration-requirements.yaml - - docs/reqstream/index-requirements.yaml + - docs/reqstream/reviewmark-system.yaml + - docs/reqstream/subsystem-cli.yaml + - docs/reqstream/subsystem-configuration.yaml + - docs/reqstream/unit-context.yaml + - docs/reqstream/unit-program.yaml + - docs/reqstream/unit-review-index.yaml + - docs/reqstream/unit-glob-matcher.yaml + - docs/reqstream/unit-path-helpers.yaml + - docs/reqstream/unit-validation.yaml - docs/reqstream/platform-requirements.yaml - - docs/reqstream/ots-requirements.yaml + - docs/reqstream/ots-mstest.yaml + - docs/reqstream/ots-reqstream.yaml + - docs/reqstream/ots-buildmark.yaml + - docs/reqstream/ots-versionmark.yaml + - docs/reqstream/ots-sarifmark.yaml + - docs/reqstream/ots-sonarmark.yaml diff --git a/src/DemaConsulting.ReviewMark/Index.cs b/src/DemaConsulting.ReviewMark/ReviewIndex.cs similarity index 100% rename from src/DemaConsulting.ReviewMark/Index.cs rename to src/DemaConsulting.ReviewMark/ReviewIndex.cs diff --git a/test/DemaConsulting.ReviewMark.Tests/ValidationTests.cs b/test/DemaConsulting.ReviewMark.Tests/ValidationTests.cs new file mode 100644 index 0000000..d7d9e03 --- /dev/null +++ b/test/DemaConsulting.ReviewMark.Tests/ValidationTests.cs @@ -0,0 +1,202 @@ +// Copyright (c) DEMA Consulting +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +namespace DemaConsulting.ReviewMark.Tests; + +/// +/// Unit tests for the class. +/// +[TestClass] +public class ValidationTests +{ + /// + /// Test that Run throws ArgumentNullException when context is null. + /// + [TestMethod] + public void Validation_Run_NullContext_ThrowsArgumentNullException() + { + // Act & Assert + Assert.Throws(() => Validation.Run(null!)); + } + + /// + /// Test that Run writes a validation header containing system information. + /// + [TestMethod] + public void Validation_Run_WritesValidationHeader() + { + // Arrange + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--validate"]); + + // Act + Validation.Run(context); + + // Assert — output contains the markdown header and table headings + var output = outWriter.ToString(); + Assert.Contains("DEMA Consulting ReviewMark", output); + Assert.Contains("Tool Version", output); + Assert.Contains("Machine Name", output); + } + finally + { + Console.SetOut(originalOut); + } + } + + /// + /// Test that Run writes a summary with a total test count. + /// + [TestMethod] + public void Validation_Run_WritesSummaryWithTotalTests() + { + // Arrange + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--validate"]); + + // Act + Validation.Run(context); + + // Assert — output contains the summary section + var output = outWriter.ToString(); + Assert.Contains("Total Tests:", output); + Assert.Contains("Passed:", output); + Assert.Contains("Failed:", output); + } + finally + { + Console.SetOut(originalOut); + } + } + + /// + /// Test that Run returns a zero exit code when all tests pass. + /// + [TestMethod] + public void Validation_Run_AllTestsPass_ExitCodeIsZero() + { + // Arrange + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--validate"]); + + // Act + Validation.Run(context); + + // Assert — exit code is zero (no errors) + Assert.AreEqual(0, context.ExitCode); + } + finally + { + Console.SetOut(originalOut); + } + } + + /// + /// Test that Run writes results to a TRX file when --results is provided with a .trx extension. + /// + [TestMethod] + public void Validation_Run_WithTrxResultsFile_WritesFile() + { + // Arrange + var resultsFile = Path.Combine(Path.GetTempPath(), $"reviewmark-validation-{Guid.NewGuid()}.trx"); + try + { + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--validate", "--results", resultsFile]); + + // Act + Validation.Run(context); + + // Assert — results file exists and has content + Assert.IsTrue(File.Exists(resultsFile), "TRX results file was not created"); + var content = File.ReadAllText(resultsFile); + Assert.IsFalse(string.IsNullOrWhiteSpace(content), "TRX results file is empty"); + Assert.Contains("TestRun", content); + } + finally + { + Console.SetOut(originalOut); + } + } + finally + { + if (File.Exists(resultsFile)) + { + File.Delete(resultsFile); + } + } + } + + /// + /// Test that Run writes results to a JUnit XML file when --results is provided with a .xml extension. + /// + [TestMethod] + public void Validation_Run_WithXmlResultsFile_WritesFile() + { + // Arrange + var resultsFile = Path.Combine(Path.GetTempPath(), $"reviewmark-validation-{Guid.NewGuid()}.xml"); + try + { + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--validate", "--results", resultsFile]); + + // Act + Validation.Run(context); + + // Assert — results file exists and has content + Assert.IsTrue(File.Exists(resultsFile), "XML results file was not created"); + var content = File.ReadAllText(resultsFile); + Assert.IsFalse(string.IsNullOrWhiteSpace(content), "XML results file is empty"); + Assert.Contains("testsuites", content); + } + finally + { + Console.SetOut(originalOut); + } + } + finally + { + if (File.Exists(resultsFile)) + { + File.Delete(resultsFile); + } + } + } +} From e07fa7fd664019cf92971dea878ed3cd697ceda2 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sun, 29 Mar 2026 23:32:05 -0400 Subject: [PATCH 08/19] Apply TemplateDotNetTool PRs #76, #77, and #78 - Standards-driven agent restructure and docs reorganization (#34) * Initial plan * Apply TemplateDotNetTool PR #76 - Agent and lint cleanup Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/5e347cdf-1fe1-49e2-81c0-7f3ad8fa994c Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Apply TemplateDotNetTool PR #77 - Add Markdown format requirements to technical-documentation.md Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/80fe8d02-e5bc-44a7-ab7f-d0cec390ce31 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Rename docs/guide to docs/user_guide and guide.md to introduction.md Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/c4adc73c-69b1-4b10-b8e5-c37e5d901ffa Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --- .cspell.yaml | 1 + .github/agents/code-quality.agent.md | 216 ---------- .github/agents/code-review.agent.md | 73 ++-- .github/agents/developer.agent.md | 49 +++ .github/agents/implementation.agent.md | 93 +++++ .github/agents/quality.agent.md | 125 ++++++ .github/agents/repo-consistency.agent.md | 71 +++- .github/agents/requirements.agent.md | 387 ------------------ .github/agents/software-developer.agent.md | 253 ------------ .github/agents/technical-writer.agent.md | 258 ------------ .github/agents/test-developer.agent.md | 299 -------------- .github/standards/csharp-language.md | 86 ++++ .github/standards/csharp-testing.md | 119 ++++++ .github/standards/reqstream-usage.md | 146 +++++++ .github/standards/reviewmark-usage.md | 151 +++++++ .github/standards/software-items.md | 45 ++ .github/standards/technical-documentation.md | 172 ++++++++ .github/workflows/build.yaml | 7 +- .gitignore | 1 + .markdownlint-cli2.yaml | 6 + .reviewmark.yaml | 2 +- .yamllint.yaml | 1 + AGENTS.md | 81 ++-- README.md | 2 +- docs/{guide => user_guide}/definition.yaml | 6 +- .../guide.md => user_guide/introduction.md} | 0 docs/{guide => user_guide}/title.txt | 0 lint.bat | 6 +- lint.sh | 6 +- 29 files changed, 1158 insertions(+), 1504 deletions(-) delete mode 100644 .github/agents/code-quality.agent.md create mode 100644 .github/agents/developer.agent.md create mode 100644 .github/agents/implementation.agent.md create mode 100644 .github/agents/quality.agent.md delete mode 100644 .github/agents/requirements.agent.md delete mode 100644 .github/agents/software-developer.agent.md delete mode 100644 .github/agents/technical-writer.agent.md delete mode 100644 .github/agents/test-developer.agent.md create mode 100644 .github/standards/csharp-language.md create mode 100644 .github/standards/csharp-testing.md create mode 100644 .github/standards/reqstream-usage.md create mode 100644 .github/standards/reviewmark-usage.md create mode 100644 .github/standards/software-items.md create mode 100644 .github/standards/technical-documentation.md rename docs/{guide => user_guide}/definition.yaml (58%) rename docs/{guide/guide.md => user_guide/introduction.md} (100%) rename docs/{guide => user_guide}/title.txt (100%) diff --git a/.cspell.yaml b/.cspell.yaml index e40779a..d5bd59c 100644 --- a/.cspell.yaml +++ b/.cspell.yaml @@ -101,6 +101,7 @@ ignorePaths: - "**/third-party/**" - "**/3rd-party/**" - "**/AGENT_REPORT_*.md" + - "**/.agent-logs/**" - "**/bin/**" - "**/obj/**" - package-lock.json diff --git a/.github/agents/code-quality.agent.md b/.github/agents/code-quality.agent.md deleted file mode 100644 index 4c15c87..0000000 --- a/.github/agents/code-quality.agent.md +++ /dev/null @@ -1,216 +0,0 @@ ---- -name: code-quality -description: Ensures code quality through comprehensive linting and static analysis. -tools: [read, search, edit, execute, github, agent] -user-invocable: true ---- - -# Code Quality Agent - -Enforce comprehensive quality standards through linting, static analysis, -security scanning, and Continuous Compliance gate verification. - -## Reporting - -If detailed documentation of code quality analysis is needed, create a report using the -filename pattern `AGENT_REPORT_quality_analysis.md` to document quality metrics, -identified patterns, and improvement recommendations. - -## When to Invoke This Agent - -Use the Code Quality Agent for: - -- Enforcing all quality gates before merge/release -- Running and resolving linting issues across all file types -- Ensuring static analysis passes with zero blockers -- Verifying security scanning results and addressing vulnerabilities -- Validating Continuous Compliance requirements -- Maintaining lint scripts and linting tool infrastructure -- Troubleshooting quality gate failures in CI/CD - -## Primary Responsibilities - -**Quality Enforcement Context**: Code quality is enforced through CI pipelines -and automated workflows. Your role is to analyze, validate, and ensure quality -standards are met using existing tools and infrastructure, not to create new -enforcement mechanisms or helper scripts. - -### Comprehensive Quality Gate Enforcement - -The project MUST be: - -- **Secure**: Zero security vulnerabilities (CodeQL, SonarQube) -- **Maintainable**: Clean, formatted, documented code with zero warnings -- **Compliant**: Requirements traceability enforced, file reviews current -- **Correct**: Does what requirements specify with passing tests - -### Universal Quality Gates (ALL Must Pass) - -#### 1. Linting Standards (Zero Tolerance) - -**Primary Interface**: Use the comprehensive linting scripts for all routine checks: - -```bash -# Run comprehensive linting suite -./lint.sh # Unix/Linux/macOS -# or -lint.bat # Windows -``` - -**Note**: The @code-quality agent is responsible for maintaining the `lint.sh`/`lint.bat` scripts. - -#### 2. Build Quality (Zero Warnings) - -All builds must be configured to treat warnings as errors. -This ensures that compiler warnings are addressed immediately rather than accumulating as technical debt. - -#### 3. Static Analysis (Zero Blockers) - -- **SonarQube/SonarCloud**: Code quality and security analysis -- **CodeQL**: Security vulnerability scanning (SARIF output) -- **Language Analyzers**: Microsoft.CodeAnalysis.NetAnalyzers, SonarAnalyzer.CSharp -- **Custom Rules**: Project-specific quality rules - -#### 4. Continuous Compliance Verification - -```bash -# Requirements traceability enforcement -dotnet reqstream \ - --requirements requirements.yaml \ - --tests "test-results/**/*.trx" \ - --enforce - -# File review status enforcement (uses .reviewmark.yaml) -dotnet reviewmark --enforce -``` - -#### 5. Test Quality & Coverage - -- All tests must pass (zero failures) -- Requirements coverage enforced (no uncovered requirements) -- Test result artifacts properly generated (TRX, JUnit XML) - -## Comprehensive Tool Configuration - -**The @code-quality agent is responsible for maintaining the repository's linting -infrastructure, specifically the `lint.sh`/`lint.bat` scripts.** - -### Lint Script Maintenance - -When updating tool versions or maintaining linting infrastructure, -modify the lint scripts: - -- **`lint.sh`** - Unix/Linux/macOS comprehensive linting script -- **`lint.bat`** - Windows comprehensive linting script - -**IMPORTANT**: Modifications should be limited to tool version updates, -path corrections, or infrastructure improvements. Do not modify enforcement -standards, rule configurations, or quality thresholds as these define -compliance requirements. - -These scripts automatically handle: - -- Node.js tool installation (markdownlint-cli2, cspell) -- Python virtual environment setup and yamllint installation -- Tool execution with proper error handling and reporting - -### Static Analysis Integration - -#### SonarQube Quality Profile - -- **Reliability**: A rating (zero bugs) -- **Security**: A rating (zero vulnerabilities) -- **Maintainability**: A rating (zero code smells for new code) -- **Coverage**: Minimum threshold (typically 80%+ for new code) -- **Duplication**: Maximum threshold (typically <3% for new code) - -#### CodeQL Security Scanning - -- **Schedule**: On every push and pull request -- **Language Coverage**: All supported languages in repository -- **SARIF Output**: Integration with GitHub Security tab -- **Blocking**: Pipeline fails on HIGH/CRITICAL findings - -## Quality Gate Execution Workflow - -### 1. Pre-Merge Quality Gates - -```bash -# Run comprehensive linting suite -./lint.sh # Unix/Linux/macOS -# or -lint.bat # Windows - -# Build with warnings as errors -dotnet build --configuration Release --no-restore /p:TreatWarningsAsErrors=true - -# Run static analysis -dotnet sonarscanner begin /k:"project-key" -dotnet build -dotnet test --collect:"XPlat Code Coverage" -dotnet sonarscanner end - -# Verify requirements compliance -dotnet reqstream --requirements requirements.yaml --tests "**/*.trx" --enforce -``` - -### 2. Security Gate Validation - -```bash -# CodeQL analysis (automated in GitHub Actions) -codeql database create --language=csharp -codeql database analyze --format=sarif-latest --output=results.sarif - -# Dependency vulnerability scanning -dotnet list package --vulnerable --include-transitive -npm audit --audit-level=moderate # if Node.js dependencies -``` - -### 3. Documentation & Compliance Gates - -```bash -# File review status validation -dotnet reviewmark --definition .reviewmark.yaml --enforce - -# Generate compliance documentation -dotnet buildmark --tools tools.yaml --output docs/build_notes.md -dotnet reqstream --report docs/requirements_doc/requirements.md --justifications docs/requirements_doc/justifications.md -``` - -## Cross-Agent Coordination - -### Hand-off to Other Agents - -- If code quality issues need to be fixed, then call the @software-developer agent with the **request** to fix code - quality, security, or linting issues with **context** of specific quality gate failures and - **additional instructions** to maintain coding standards. -- If test coverage needs improvement or tests are failing, then call the @test-developer agent with the **request** - to improve test coverage or fix failing tests with **context** of current coverage metrics and failing test details. -- If documentation linting fails or documentation is missing, then call the @technical-writer agent with the - **request** to fix documentation linting or generate missing docs with **context** of specific linting failures and - documentation gaps. -- If requirements traceability fails, then call the @requirements agent with the **request** to address requirements - traceability failures with **context** of enforcement errors and missing test linkages. - -## Compliance Verification Checklist - -### Before Approving Any Changes - -1. **Linting**: All linting tools pass (markdownlint, cspell, yamllint, language linters) -2. **Build**: Zero warnings, zero errors in all configurations -3. **Static Analysis**: SonarQube quality gate GREEN, CodeQL no HIGH/CRITICAL findings -4. **Requirements**: ReqStream enforcement passes, all requirements covered -5. **Tests**: All tests pass, adequate coverage maintained -6. **Documentation**: All generated docs current, spell-check passes -7. **Security**: No vulnerability findings in dependencies or code -8. **File Reviews**: All reviewable files have current reviews (if applicable) - -## Don't Do These Things - -- **Never disable quality checks** to make builds pass (fix the underlying issue) -- **Never ignore security warnings** without documented risk acceptance -- **Never skip requirements enforcement** for "quick fixes" -- **Never modify functional code** without appropriate developer agent involvement -- **Never lower quality thresholds** without compliance team approval -- **Never commit with linting failures** (CI should block this) -- **Never bypass static analysis** findings without documented justification diff --git a/.github/agents/code-review.agent.md b/.github/agents/code-review.agent.md index fb01a20..f28a9b7 100644 --- a/.github/agents/code-review.agent.md +++ b/.github/agents/code-review.agent.md @@ -1,46 +1,73 @@ --- name: code-review -description: Assists in performing formal file reviews. -tools: [read, search, edit, execute, github, web, agent] +description: Agent for performing formal reviews user-invocable: true --- # Code Review Agent -Execute comprehensive code reviews with emphasis on structured compliance verification and file review status -requirements. +This agent runs the formal review based on the review-set it's told to perform. -## Reporting +# Formal Review Steps -Create a report using the filename pattern `AGENT_REPORT_code_review_[review-set].md` -(e.g., `AGENT_REPORT_code_review_auth-module.md`) to document review criteria, identified issues, and recommendations -for the specific review-set. - -## Review Steps +Formal reviews are a quality enforcement mechanism, and as such MUST be performed using the following four steps: 1. Download the to get the checklist to fill in 2. Use `dotnet reviewmark --elaborate [review-set]` to get the files to review 3. Review the files all together -4. Populate the checklist with the findings to make the report - -## Hand-off to Other Agents - -Only attempt to apply review fixes if requested. +4. Populate the checklist with the findings to `.agent-logs/reviews/review-report-[review-set].md` of the project. -- If code quality, logic, or structural issues need fixing, call the @software-developer agent -- If test coverage gaps or quality issues are identified, call the @test-developer agent -- If documentation accuracy or completeness issues are found, call the @technical-writer agent -- If quality gate verification is needed after fixes, call the @code-quality agent -- If requirements traceability issues are discovered, call the @requirements agent +# Don't Do These Things -## Don't Do These Things - -- **Never modify code during review** (document findings only, delegate fixes) +- **Never modify code during review** (document findings only) - **Never skip applicable checklist items** (comprehensive review required) - **Never approve reviews with unresolved critical findings** - **Never bypass review status requirements** for compliance - **Never conduct reviews without proper documentation** - **Never ignore security or compliance findings** - **Never approve without verifying all quality gates** + +# Reporting + +Upon completion create a summary in `.agent-logs/[agent-name]-[subject]-[unique-id].md` +of the project consisting of: + +```markdown +# Code Review Report + +**Result**: + +## Review Summary + +- **Review Set**: [Review set name/identifier] +- **Review Report File**: [Name of detailed review report generated] +- **Files Reviewed**: [Count and list of files reviewed] +- **Review Template Used**: [Template source and version] + +## Review Results + +- **Overall Conclusion**: [Summary of review results] +- **Critical Issues**: [Count of critical findings] +- **High Issues**: [Count of high severity findings] +- **Medium Issues**: [Count of medium severity findings] +- **Low Issues**: [Count of low severity findings] + +## Issue Details + +[For each issue found, include:] +- **File**: [File name and line number where applicable] +- **Issue Type**: [Security, logic error, compliance violation, etc.] +- **Severity**: [Critical/High/Medium/Low] +- **Description**: [Issue description] +- **Recommendation**: [Specific remediation recommendation] + +## Compliance Status + +- **Review Status**: [Complete/Incomplete with reasoning] +- **Quality Gates**: [Status of review checklist items] +- **Approval Status**: [Approved/Rejected with justification] +``` + +Return summary to caller. diff --git a/.github/agents/developer.agent.md b/.github/agents/developer.agent.md new file mode 100644 index 0000000..955f9e9 --- /dev/null +++ b/.github/agents/developer.agent.md @@ -0,0 +1,49 @@ +--- +name: developer +description: > + General-purpose software development agent that applies appropriate standards + based on the work being performed. +user-invocable: true +--- + +# Developer Agent + +Perform software development tasks by determining and applying appropriate DEMA Consulting standards from `.github/standards/`. + +# Standards-Based Workflow + +1. **Analyze the request** to identify scope: languages, file types, requirements, testing, reviews +2. **Read relevant standards** from `.github/standards/` as defined in AGENTS.md based on work performed +3. **Apply loaded standards** throughout development process +4. **Execute work** following standards requirements and quality checks +5. **Generate completion report** with results and compliance status + +# Reporting + +Upon completion create a summary in `.agent-logs/[agent-name]-[subject]-[unique-id].md` +of the project consisting of: + +```markdown +# Developer Agent Report + +**Result**: + +## Work Summary + +- **Files Modified**: [List of files created/modified/deleted] +- **Languages Detected**: [Languages identified] +- **Standards Applied**: [Standards files consulted] + +## Tooling Executed + +- **Language Tools**: [Compilers, linters, formatters used] +- **Compliance Tools**: [ReqStream, ReviewMark tools used] +- **Validation Results**: [Tool execution results] + +## Compliance Status + +- **Quality Checks**: [Standards quality checks status] +- **Issues Resolved**: [Any problems encountered and resolved] +``` + +Return this summary to the caller. diff --git a/.github/agents/implementation.agent.md b/.github/agents/implementation.agent.md new file mode 100644 index 0000000..767c66d --- /dev/null +++ b/.github/agents/implementation.agent.md @@ -0,0 +1,93 @@ +--- +name: implementation +description: Orchestrator agent that manages quality implementations through a formal state machine workflow. +user-invocable: true +--- + +# Implementation Agent + +Orchestrate quality implementations through a formal state machine workflow +that ensures research, development, and quality validation are performed +systematically. + +# State Machine Workflow + +**MANDATORY**: This agent MUST follow the orchestration process below to ensure +the quality of the implementation. The process consists of the following +states: + +- **RESEARCH** - performs initial analysis +- **DEVELOPMENT** - develops the implementation changes +- **QUALITY** - performs quality validation +- **REPORT** - generates final implementation report + +The state-transitions include retrying a limited number of times, using a 'retry-count' +counting how many retries have occurred. + +## RESEARCH State (start) + +Call the built-in @explore sub-agent with: + +- **context**: the user's request and any current quality findings +- **goal**: analyze the implementation state and develop a plan to implement the request + +Once the explore sub-agent finishes, transition to the DEVELOPMENT state. + +## DEVELOPMENT State + +Call the @developer sub-agent with: + +- **context** the user's request and the current implementation plan +- **goal** implement the user's request and any identified quality fixes + +Once the developer sub-agent finishes: + +- IF developer SUCCEEDED: Transition to QUALITY state to check the quality of the work +- IF developer FAILED: Transition to REPORT state to report the failure + +## QUALITY State + +Call the @quality sub-agent with: + +- **context** the user's request and the current implementation report +- **goal** check the quality of the work performed for any issues + +Once the quality sub-agent finishes: + +- IF quality SUCCEEDED: Transition to REPORT state to report completion +- IF quality FAILED and retry-count < 3: Transition to RESEARCH state to plan quality fixes +- IF quality FAILED and retry-count >= 3: Transition to REPORT state to report failure + +### REPORT State (end) + +Upon completion create a summary in `.agent-logs/[agent-name]-[subject]-[unique-id].md` +of the project consisting of: + +```markdown +# Implementation Orchestration Report + +**Result**: +**Final State**: +**Retry Count**: + +## State Machine Execution + +- **Research Results**: [Summary of explore agent findings] +- **Development Results**: [Summary of developer agent results] +- **Quality Results**: [Summary of quality agent results] +- **State Transitions**: [Log of state changes and decisions] + +## Sub-Agent Coordination + +- **Explore Agent**: [Research findings and context] +- **Developer Agent**: [Development status and files modified] +- **Quality Agent**: [Validation results and compliance status] + +## Final Status + +- **Implementation Success**: [Overall completion status] +- **Quality Compliance**: [Final quality validation status] +- **Issues Resolved**: [Problems encountered and resolution attempts] +``` + +Return this summary to the caller. diff --git a/.github/agents/quality.agent.md b/.github/agents/quality.agent.md new file mode 100644 index 0000000..4dd6902 --- /dev/null +++ b/.github/agents/quality.agent.md @@ -0,0 +1,125 @@ +--- +name: quality +description: > + Quality assurance agent that grades developer work against DEMA Consulting + standards and Continuous Compliance practices. +user-invocable: true +--- + +# Quality Agent + +Grade and validate software development work by ensuring compliance with +DEMA Consulting standards and Continuous Compliance practices. + +# Standards-Based Quality Assessment + +This assessment is a quality control system of the project and MUST be performed. + +1. **Analyze completed work** to identify scope and changes made +2. **Read relevant standards** from `.github/standards/` as defined in AGENTS.md based on work performed +3. **Execute comprehensive quality checks** across all compliance areas - EVERY checkbox item must be evaluated +4. **Validate tool compliance** using ReqStream, ReviewMark, and language tools +5. **Generate quality assessment report** with findings and recommendations + +## Requirements Compliance + +- [ ] Were requirements updated to reflect functional changes? +- [ ] Were new requirements created for new features? +- [ ] Do requirement IDs follow semantic naming standards? +- [ ] Were source filters applied appropriately for platform-specific requirements? +- [ ] Does ReqStream enforcement pass without errors? +- [ ] Is requirements traceability maintained to tests? + +## Design Documentation Compliance + +- [ ] Were design documents updated for architectural changes? +- [ ] Were new design artifacts created for new components? +- [ ] Are design decisions documented with rationale? +- [ ] Is system/subsystem/unit categorization maintained? +- [ ] Is design-to-implementation traceability preserved? + +## Code Quality Compliance + +- [ ] Are language-specific standards followed (from applicable standards files)? +- [ ] Are quality checks from standards files satisfied? +- [ ] Is code properly categorized (system/subsystem/unit/OTS)? +- [ ] Is appropriate separation of concerns maintained? +- [ ] Was language-specific tooling executed and passing? + +## Testing Compliance + +- [ ] Were tests created/updated for all functional changes? +- [ ] Is test coverage maintained for all requirements? +- [ ] Are testing standards followed (AAA pattern, etc.)? +- [ ] Does test categorization align with code structure? +- [ ] Do all tests pass without failures? + +## Review Management Compliance + +- [ ] Were review-sets updated to include new/modified files? +- [ ] Do file patterns follow include-then-exclude approach? +- [ ] Is review scope appropriate for change magnitude? +- [ ] Was ReviewMark tooling executed and passing? +- [ ] Were review artifacts generated correctly? + +## Documentation Compliance + +- [ ] Was README.md updated for user-facing changes? +- [ ] Were user guides updated for feature changes? +- [ ] Does API documentation reflect code changes? +- [ ] Was compliance documentation generated? +- [ ] Does documentation follow standards formatting? +- [ ] Is documentation organized under `docs/` following standard folder structure? +- [ ] Do Pandoc collections include proper `introduction.md` files with Purpose and Scope sections? +- [ ] Are auto-generated markdown files left unmodified? +- [ ] Do README.md files use absolute URLs and include concrete examples? +- [ ] Is documentation integrated into ReviewMark review-sets for formal review? + +## Process Compliance + +- [ ] Was Continuous Compliance workflow followed? +- [ ] Did all quality gates execute successfully? +- [ ] Were appropriate tools used for validation? +- [ ] Were standards consistently applied across work? +- [ ] Was compliance evidence generated and preserved? + +# Reporting + +Upon completion create a summary in `.agent-logs/[agent-name]-[subject]-[unique-id].md` +of the project consisting of: + +```markdown +# Quality Assessment Report + +**Result**: +**Overall Grade**: + +## Assessment Summary + +- **Work Reviewed**: [Description of work assessed] +- **Standards Applied**: [Standards files used for assessment] +- **Categories Evaluated**: [Quality check categories assessed] + +## Quality Check Results + +- **Requirements Compliance**: - [Summary] +- **Design Documentation**: - [Summary] +- **Code Quality**: - [Summary] +- **Testing Compliance**: - [Summary] +- **Review Management**: - [Summary] +- **Documentation**: - [Summary] +- **Process Compliance**: - [Summary] + +## Findings + +- **Issues Found**: [List of compliance issues] +- **Recommendations**: [Suggested improvements] +- **Tools Executed**: [Quality tools used for validation] + +## Compliance Status + +- **Standards Adherence**: [Overall compliance rating] +- **Quality Gates**: [Status of automated quality checks] +``` + +Return this summary to the caller. diff --git a/.github/agents/repo-consistency.agent.md b/.github/agents/repo-consistency.agent.md index 8591e2f..dfaf702 100644 --- a/.github/agents/repo-consistency.agent.md +++ b/.github/agents/repo-consistency.agent.md @@ -1,7 +1,8 @@ --- name: repo-consistency -description: Ensures downstream repositories remain consistent with the TemplateDotNetTool template patterns and best practices. -tools: [read, search, edit, execute, github, agent] +description: > + Ensures downstream repositories remain consistent with the TemplateDotNetTool + template patterns and best practices. user-invocable: true --- @@ -10,19 +11,26 @@ user-invocable: true Maintain consistency between downstream projects and the TemplateDotNetTool template, ensuring repositories benefit from template evolution while respecting project-specific customizations. -## Reporting +# Consistency Workflow (MANDATORY) -If detailed documentation of consistency analysis is needed, create a report using the filename pattern -`AGENT_REPORT_consistency_[repo_name].md` (e.g., `AGENT_REPORT_consistency_MyTool.md`) to document -consistency gaps, template evolution updates, and recommended changes for the specific repository. +**CRITICAL**: This agent MUST follow these steps systematically to ensure proper template consistency analysis: -## Consistency Steps +1. **Fetch Recent Template Changes**: Use GitHub search to fetch the 20 most recently merged PRs + (`is:pr is:merged sort:updated-desc`) from +2. **Analyze Template Evolution**: For each relevant PR, determine the intent and scope of changes + (what files were modified, what improvements were made) +3. **Assess Downstream Applicability**: Evaluate which template changes would benefit this repository + while respecting project-specific customizations +4. **Apply Appropriate Updates**: Implement applicable template improvements with proper translation for project context +5. **Validate Consistency**: Verify that applied changes maintain functionality and follow project patterns -1. Fetch the 20 most recently merged PRs (`is:pr is:merged sort:updated-desc`) from -2. Determine the intent of the template pull requests (what changes were performed to which files) -3. Apply missing changes to this repository's files (if appropriate and with translation) +## Key Principles -## Don't Do These Things +- **Evolutionary Consistency**: Template improvements should enhance downstream projects systematically +- **Intelligent Customization Respect**: Distinguish valid customizations from unintentional drift +- **Incremental Template Adoption**: Support phased adoption of template improvements based on project capacity + +# Don't Do These Things - **Never recommend changes without understanding project context** (some differences are intentional) - **Never flag valid project-specific customizations** as consistency problems @@ -32,8 +40,41 @@ consistency gaps, template evolution updates, and recommended changes for the sp - **Never skip validation** of preserved functionality after template alignment - **Never assume all template patterns apply universally** (assess project-specific needs) -## Key Principles +# Reporting -- **Evolutionary Consistency**: Template improvements should enhance downstream projects systematically -- **Intelligent Customization Respect**: Distinguished valid customizations from unintentional drift -- **Incremental Template Adoption**: Support phased adoption of template improvements based on project capacity +Upon completion create a summary in `.agent-logs/[agent-name]-[subject]-[unique-id].md` +of the project consisting of: + +```markdown +# Repo Consistency Report + +**Result**: + +## Consistency Analysis + +- **Template PRs Analyzed**: [Number and timeframe of PRs reviewed] +- **Template Changes Identified**: [Count and types of template improvements] +- **Applicable Updates**: [Changes determined suitable for this repository] +- **Project Customizations Preserved**: [Valid differences maintained] + +## Template Evolution Applied + +- **Files Modified**: [List of files updated for template consistency] +- **Improvements Adopted**: [Specific template enhancements implemented] +- **Configuration Updates**: [Tool configurations, workflows, or standards updated] + +## Consistency Status + +- **Template Alignment**: [Overall consistency rating with template] +- **Customization Respect**: [How project-specific needs were preserved] +- **Functionality Validation**: [Verification that changes don't break existing features] +- **Future Consistency**: [Recommendations for ongoing template alignment] + +## Issues Resolved + +- **Drift Corrections**: [Template drift issues addressed] +- **Enhancement Adoptions**: [Template improvements successfully integrated] +- **Validation Results**: [Testing and validation outcomes] +``` + +Return this summary to the caller. diff --git a/.github/agents/requirements.agent.md b/.github/agents/requirements.agent.md deleted file mode 100644 index bfd0a30..0000000 --- a/.github/agents/requirements.agent.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -name: requirements -description: Develops requirements and ensures appropriate test coverage. -tools: [read, search, edit, execute, github, web, agent] -user-invocable: true ---- - -# Requirements Agent - -Develop and maintain high-quality requirements with comprehensive test coverage linkage following Continuous -Compliance methodology for automated evidence generation and audit compliance. - -## Reporting - -If detailed documentation of requirements analysis is needed, create a report using the filename pattern -`AGENT_REPORT_requirements.md` to document requirement mappings, gap analysis, and traceability results. - -## When to Invoke This Agent - -Use the Requirements Agent for: - -- Creating new requirements in organized `docs/reqstream/` structure -- Establishing subsystem and software unit requirement files for independent review -- Reviewing and improving existing requirements quality and organization -- Ensuring proper requirements-to-test traceability -- Validating requirements enforcement in CI/CD pipelines -- Differentiating requirements from design/implementation details - -## Continuous Compliance Methodology - -### Core Principles - -The @requirements agent implements the Continuous Compliance methodology -, which provides automated compliance evidence -generation through structured requirements management: - -- **📚 Complete Methodology Documentation:** -- **📋 Detailed Requirements Guidelines:** - -- **🔧 ReqStream Tool Documentation:** - -#### Automated Evidence Generation - -- **Requirements Traceability**: Automated linking between requirements and test evidence -- **Compliance Reports**: Generated documentation for audit and regulatory compliance -- **Quality Gate Enforcement**: Pipeline failures prevent non-compliant code from merging -- **Platform-Specific Evidence**: Source filters ensure correct testing environment validation - -#### Continuous Compliance Benefits - -- **Audit Trail**: Complete requirements-to-implementation traceability -- **Regulatory Support**: Meets medical device, aerospace, automotive compliance standards -- **Quality Assurance**: Automated verification prevents compliance gaps -- **Documentation**: Generated reports reduce manual documentation overhead - -## Primary Responsibilities - -### Requirements Engineering Excellence - -- Focus on **observable behavior and characteristics**, not implementation details -- Write clear, testable requirements with measurable acceptance criteria -- Ensure semantic requirement IDs (`Project-Section-ShortDesc` format preferred over `REQ-042`) -- Include comprehensive justification explaining business/regulatory rationale -- Maintain hierarchical requirement structure with proper parent-child relationships - -### Requirements Organization for Review-Sets - -Organize requirements into separate files under `docs/reqstream/` to enable independent review processes: - -#### Subsystem-Level Requirements - -- **File Pattern**: `{subsystem}-subsystem.yaml` (e.g., `auth-subsystem.yaml`) -- **Content Focus**: High-level subsystem behavior, interfaces, and integration requirements -- **Review Scope**: Architectural and subsystem design reviews -- **Team Assignment**: Can be reviewed independently by subsystem teams - -#### Software Unit Requirements - -- **File Pattern**: `{subsystem}-{class}-class.yaml` (e.g., `auth-passwordvalidator-class.yaml`) -- **Content Focus**: Individual class behavior, method contracts, and invariants -- **Review Scope**: Code-level implementation reviews -- **Team Assignment**: Enable focused class-level review processes - -#### OTS Software Requirements - -- **File Pattern**: `ots-{component}.yaml` (e.g., `ots-systemtextjson.yaml`) -- **Content Focus**: Required functionality from third-party components, libraries, and frameworks -- **Review Scope**: Dependency validation and integration testing reviews -- **Team Assignment**: Can be reviewed by teams responsible for external dependency management -- **Section Structure**: Must use "OTS Software Requirements" as top-level section with component subsections: - -```yaml -sections: - - title: OTS Software Requirements - sections: - - title: System.Text.Json - requirements: - - id: Project-SystemTextJson-ReadJson - title: System.Text.Json shall be able to read JSON files. - # ... requirements for this OTS component - - title: NUnit - requirements: - - id: Project-NUnit-ParameterizedTests - title: NUnit shall support parameterized test methods. - # ... requirements for this OTS component -``` - -#### Benefits for Continuous Compliance - -- **Parallel Review Workflows**: Multiple teams can review different subsystems, classes, and OTS components simultaneously -- **Granular Status Tracking**: Review status maintained at subsystem, class, and OTS dependency level -- **Scalable Organization**: Supports large projects without requirement file conflicts -- **Independent Evidence**: Each file provides focused compliance evidence -- **Dependency Management**: OTS requirements enable systematic third-party component validation - -### Continuous Compliance Enforcement - -Following the Continuous Compliance methodology , -requirements management operates on these enforcement principles: - -#### Traceability Requirements (ENFORCED) - -- **Mandatory Coverage**: ALL requirements MUST link to passing tests - CI pipeline fails otherwise -- **Automated Verification**: `dotnet reqstream --enforce` validates complete traceability -- **Evidence Chain**: Requirements → Tests → Results → Documentation must be unbroken -- **Platform Compliance**: Source filters ensure correct testing environment evidence - -#### Quality Gate Integration - -- **Pipeline Enforcement**: CI/CD fails on any requirements without test coverage -- **Documentation Generation**: Automated requirements reports for audit compliance -- **Regulatory Support**: Meets FDA, DO-178C, ISO 26262, and other regulatory standards -- **Continuous Monitoring**: Every build verifies requirements compliance status - -#### Compliance Documentation - -Per Continuous Compliance requirements documentation -: - -- **Requirements Reports**: Generated documentation showing all requirements and their status -- **Justifications**: Business and regulatory rationale for each requirement -- **Trace Matrix**: Complete mapping of requirements to test evidence -- **Audit Trails**: Historical compliance evidence for regulatory reviews - -### Test Coverage Strategy & Linking - -#### Coverage Rules - -- **Requirements coverage**: Mandatory for all stated requirements -- **Test flexibility**: Not all tests need requirement links (corner cases, design validation, failure scenarios allowed) -- **Platform evidence**: Use source filters for platform/framework-specific requirements - -#### Source Filter Patterns (CRITICAL - DO NOT REMOVE) - -```yaml -tests: - - "windows@TestMethodName" # Windows platform evidence only - - "ubuntu@TestMethodName" # Linux (Ubuntu) platform evidence only - - "net8.0@TestMethodName" # .NET 8 runtime evidence only - - "net9.0@TestMethodName" # .NET 9 runtime evidence only - - "net10.0@TestMethodName" # .NET 10 runtime evidence only - - "TestMethodName" # Any platform evidence acceptable -``` - -**WARNING**: Removing source filters invalidates platform-specific compliance evidence and may cause audit failures. - -### Quality Gate Verification - -Before completing any requirements work, verify: - -#### 1. Requirements Quality - -- [ ] Semantic IDs follow `Project-Section-ShortDesc` pattern -- [ ] Clear, testable acceptance criteria defined -- [ ] Comprehensive justification provided -- [ ] Observable behavior specified (not implementation details) - -#### 2. Traceability Compliance - -- [ ] All requirements linked to appropriate tests -- [ ] Source filters applied for platform-specific requirements -- [ ] ReqStream enforcement passes: `dotnet reqstream --enforce` -- [ ] Generated reports current (requirements, justifications, trace matrix) - -#### 3. CI/CD Integration - -- [ ] Requirements files pass yamllint validation -- [ ] Test result formats compatible with ReqStream (TRX, JUnit XML) -- [ ] Pipeline configured with `--enforce` flag -- [ ] Build fails appropriately on coverage gaps - -## ReqStream Tool Integration - -### ReqStream Overview - -ReqStream is the core tool for implementing Continuous Compliance requirements management: - -**🔧 ReqStream Repository:** - -#### Key Capabilities - -- **Traceability Enforcement**: `dotnet reqstream --enforce` validates all requirements have test coverage -- **Multi-Format Support**: Handles TRX, JUnit XML, and other test result formats -- **Report Generation**: Creates requirements reports, justifications, and trace matrices -- **Source Filtering**: Validates platform-specific testing requirements -- **CI/CD Integration**: Provides exit codes for pipeline quality gates - -#### Essential ReqStream Commands - -```bash -# Validate requirements traceability (use in CI/CD) -dotnet reqstream --requirements requirements.yaml --tests "test-results/**/*.trx" --enforce - -# Generate requirements documentation (for publication) -dotnet reqstream --requirements requirements.yaml --report docs/requirements_doc/requirements.md - -# Generate justifications report (for publication) -dotnet reqstream --requirements requirements.yaml --justifications docs/requirements_doc/justifications.md - -# Generate trace matrix -dotnet reqstream --requirements requirements.yaml --tests "test-results/**/*.trx" --matrix docs/requirements_report/trace_matrix.md -``` - -### Required Tools & Configuration - -- **ReqStream**: Core requirements traceability and enforcement (`dotnet tool install DemaConsulting.ReqStream`) -- **yamllint**: YAML structure validation for requirements files -- **cspell**: Spell-checking for requirement text and justifications - -### Standard File Structure for Review-Set Organization - -```text -requirements.yaml # Root requirements file with includes only -docs/ - reqstream/ # Organized requirements files for independent review - # System-level requirements - system-requirements.yaml - - # Subsystem requirements (enable subsystem review-sets) - auth-subsystem.yaml # Authentication subsystem requirements - data-subsystem.yaml # Data management subsystem requirements - ui-subsystem.yaml # User interface subsystem requirements - - # Software unit requirements (enable class-level review-sets) - auth-passwordvalidator-class.yaml # PasswordValidator class requirements - data-repository-class.yaml # Repository pattern class requirements - ui-controller-class.yaml # UI Controller class requirements - - # OTS Software requirements (enable dependency review-sets) - ots-systemtextjson.yaml # System.Text.Json OTS requirements - ots-nunit.yaml # NUnit framework OTS requirements - ots-entityframework.yaml # Entity Framework OTS requirements - - requirements_doc/ # Pandoc document folder for requirements publication - definition.yaml # Document content definition - title.txt # Document metadata - requirements.md # Auto-generated requirements report - justifications.md # Auto-generated justifications - - requirements_report/ # Pandoc document folder for requirements testing publication - definition.yaml # Document content definition - title.txt # Document metadata - trace_matrix.md # Auto-generated trace matrix -``` - -#### Review-Set Benefits - -This file organization enables independent review workflows: - -- **Subsystem Reviews**: Each subsystem file can be reviewed independently by different teams -- **Software Unit Reviews**: Class-level requirements enable focused code reviews -- **OTS Dependency Reviews**: Third-party component requirements enable systematic dependency validation -- **Parallel Development**: Teams can work on requirements without conflicts -- **Granular Tracking**: Review status tracking per subsystem, software unit, and OTS dependency -- **Scalable Organization**: Supports large projects with multiple development teams - -#### Root Requirements File Structure - -```yaml -# requirements.yaml - Root configuration with includes only -includes: - # System and subsystem requirements - - docs/reqstream/system-requirements.yaml - - docs/reqstream/auth-subsystem.yaml - - docs/reqstream/data-subsystem.yaml - - docs/reqstream/ui-subsystem.yaml - # Software unit requirements (classes) - - docs/reqstream/auth-passwordvalidator-class.yaml - - docs/reqstream/data-repository-class.yaml - - docs/reqstream/ui-controller-class.yaml - # OTS Software requirements (third-party components) - - docs/reqstream/ots-systemtextjson.yaml - - docs/reqstream/ots-nunit.yaml - - docs/reqstream/ots-entityframework.yaml -``` - -## Continuous Compliance Best Practices - -### Requirements Quality Standards - -Following Continuous Compliance requirements guidelines -: - -#### 1. **Observable Behavior Focus** - -- Requirements specify WHAT the system shall do, not HOW it should be implemented -- Focus on externally observable characteristics and behavior -- Avoid implementation details, design constraints, or technology choices - -#### 2. **Testable Acceptance Criteria** - -- Each requirement must have clear, measurable acceptance criteria -- Requirements must be verifiable through automated or manual testing -- Ambiguous or untestable requirements cause compliance failures - -#### 3. **Comprehensive Justification** - -- Business rationale explaining why the requirement exists -- Regulatory or standard references where applicable -- Risk mitigation or quality improvement justification - -#### 4. **Semantic Requirement IDs** - -- Use meaningful IDs: `TestProject-CommandLine-DisplayHelp` instead of `REQ-042` -- Follow `Project-Section-ShortDesc` pattern for clarity -- Enable better requirement organization and traceability - -### Platform-Specific Requirements - -Critical for regulatory compliance in multi-platform environments: - -#### Source Filter Implementation - -```yaml -requirements: - - id: Platform-Windows-Compatibility - title: Windows Platform Support - description: The software shall operate on Windows 10 and later versions - tests: - - windows@PlatformTests.TestWindowsCompatibility # MUST run on Windows - - - id: Target-IAR-Build - title: IAR Compiler Compatibility - description: The firmware shall compile successfully with IAR C compiler - tests: - - iar@CompilerTests.TestIarBuild # MUST use IAR toolchain -``` - -**WARNING**: Source filters are REQUIRED for platform-specific compliance evidence. -Removing them invalidates regulatory audit trails. - -## Cross-Agent Coordination - -### Hand-off to Other Agents - -- If features need to be implemented to satisfy requirements, then call the @software-developer agent with the - **request** to implement features that satisfy requirements with **context** of specific requirement details - and **goal** of requirement compliance. -- If tests need to be created to validate requirements, then call the @test-developer agent with the **request** - to create tests that validate requirements with **context** of requirement specifications and - **additional instructions** for traceability setup. -- If requirements traceability needs to be enforced in CI/CD, then call the @code-quality agent with the **request** - to enforce requirements traceability in CI/CD with **context** of current enforcement status and **goal** of - automated compliance verification. -- If requirements documentation needs generation or maintenance, then call the @technical-writer agent with the - **request** to generate and maintain requirements documentation with **context** of current requirements and - **goal** of regulatory compliance documentation. - -## Compliance Verification Checklist - -### Before Completing Work - -1. **Requirement Quality**: Clear, testable, with proper justification -2. **Test Linkage**: All requirements have appropriate test coverage -3. **Source Filters**: Platform requirements have correct source filters -4. **Tool Validation**: yamllint, ReqStream enforcement passing -5. **Documentation**: Generated reports current and accessible -6. **CI Integration**: Pipeline properly configured for enforcement - -## Don't Do These Things - -- Create requirements without test linkage (CI will fail) -- Remove source filters from platform-specific requirements (breaks compliance) -- Mix implementation details with requirements (separate concerns) -- Skip justification text (required for compliance audits) -- Change test code directly (delegate to @test-developer agent) -- Modify CI/CD enforcement thresholds without compliance review diff --git a/.github/agents/software-developer.agent.md b/.github/agents/software-developer.agent.md deleted file mode 100644 index 891f281..0000000 --- a/.github/agents/software-developer.agent.md +++ /dev/null @@ -1,253 +0,0 @@ ---- -name: software-developer -description: Writes production code and self-validation tests. -tools: [read, search, edit, execute, github, agent] -user-invocable: true ---- - -# Software Developer Agent - -Develop production code with emphasis on testability, clarity, and compliance integration. - -## Reporting - -If detailed documentation of development work is needed, create a report using the filename pattern -`AGENT_REPORT_development.md` to document code changes, design decisions, and implementation details. - -## When to Invoke This Agent - -Use the Software Developer Agent for: - -- Implementing production code features and APIs -- Refactoring existing code for testability and maintainability -- Creating self-validation and demonstration code -- Implementing requirement-driven functionality -- Code architecture and design decisions -- Integration with Continuous Compliance tooling - -## Primary Responsibilities - -### Literate Programming Style (MANDATORY) - -Write all code in **literate style** for maximum clarity and maintainability. - -#### Literate Style Rules - -- **Intent Comments:** - Every paragraph starts with a comment explaining intent (not mechanics) -- **Logical Separation:** - Blank lines separate logical code paragraphs -- **Purpose Over Process:** - Comments describe why, code shows how -- **Standalone Clarity:** - Reading comments alone should explain the algorithm/approach -- **Verification Support:** - Code can be verified against the literate comments for correctness - -#### Examples - -**C# Example:** - -```csharp -// Validate input parameters to prevent downstream errors -if (string.IsNullOrEmpty(input)) -{ - throw new ArgumentException("Input cannot be null or empty", nameof(input)); -} - -// Transform input data using the configured processing pipeline -var processedData = ProcessingPipeline.Transform(input); - -// Apply business rules and validation logic -var validatedResults = BusinessRuleEngine.ValidateAndProcess(processedData); - -// Return formatted results matching the expected output contract -return OutputFormatter.Format(validatedResults); -``` - -**C++ Example:** - -```cpp -// Acquire exclusive hardware access using RAII pattern -std::lock_guard hardwareLock(m_hardwareMutex); - -// Validate sensor data integrity before processing -if (!sensorData.IsValid() || sensorData.GetTimestamp() < m_lastValidTimestamp) -{ - throw std::invalid_argument("Sensor data failed integrity validation"); -} - -// Apply hardware-specific calibration coefficients -auto calibratedReading = ApplyCalibration(sensorData.GetRawValue(), - m_calibrationCoefficients); - -// Filter noise using moving average with bounds checking -const auto filteredValue = m_noiseFilter.ApplyFilter(calibratedReading); -if (filteredValue < kMinOperationalThreshold || filteredValue > kMaxOperationalThreshold) -{ - LogWarning("Filtered sensor value outside operational range"); -} - -// Package result with quality metadata for downstream consumers -return SensorResult{filteredValue, CalculateQualityMetric(sensorData), - std::chrono::steady_clock::now()}; -``` - -### Design for Testability & Compliance - -#### Code Architecture Principles - -- **Single Responsibility**: Functions with focused, testable purposes -- **Dependency Injection**: External dependencies injected for testing -- **Pure Functions**: Minimize side effects and hidden state -- **Clear Interfaces**: Well-defined API contracts -- **Separation of Concerns**: Business logic separate from infrastructure - -#### Compliance-Ready Code Structure - -- **Documentation Standards**: Language-specific documentation required on ALL members for compliance -- **Error Handling**: Comprehensive error cases with appropriate logging -- **Configuration**: Externalize settings for different compliance environments -- **Traceability**: Code comments linking back to requirements where applicable - -### Quality Gate Verification - -Before completing any code changes, verify: - -#### 1. Code Quality Standards - -- [ ] Zero compiler warnings (`TreatWarningsAsErrors=true`) -- [ ] Follows `.editorconfig` and `.clang-format` formatting rules -- [ ] All code follows literate programming style -- [ ] Language-specific documentation complete on all members (XML for C#, Doxygen for C++) -- [ ] Passes static analysis (SonarQube, CodeQL, language analyzers) - -#### 2. Testability & Design - -- [ ] Functions have single, clear responsibilities -- [ ] External dependencies are injectable/mockable -- [ ] Code is structured for unit testing -- [ ] Error handling covers expected failure scenarios -- [ ] Configuration externalized from business logic - -#### 3. Compliance Integration - -- [ ] Code supports requirements traceability -- [ ] Logging/telemetry appropriate for audit trails -- [ ] Security considerations addressed (input validation, authorization) -- [ ] Platform compatibility maintained for multi-platform requirements - -## Tool Integration Requirements - -### Required Development Tools - -- **Language Formatters**: Applied via `.editorconfig`, `.clang-format` -- **Static Analyzers**: Microsoft.CodeAnalysis.NetAnalyzers, SonarAnalyzer.CSharp -- **Security Scanning**: CodeQL integration for vulnerability detection -- **Documentation**: XML docs generation for API documentation - -### Code Quality Tools Integration - -- **SonarQube/SonarCloud**: Continuous code quality monitoring -- **Build Integration**: Warnings as errors enforcement -- **IDE Integration**: Real-time feedback on code quality issues -- **CI/CD Integration**: Automated quality gate enforcement - -## Cross-Agent Coordination - -### Hand-off to Other Agents - -- If comprehensive tests need to be created for implemented functionality, then call the @test-developer agent with the - **request** to create comprehensive tests for implemented functionality with **context** of new code changes and - **goal** of achieving adequate test coverage. -- If quality gates and linting requirements need verification, then call the @code-quality agent with the **request** - to verify all quality gates and linting requirements with **context** of completed implementation and **goal** of - compliance verification. -- If documentation needs updating to reflect code changes, then call the @technical-writer agent with the **request** - to update documentation reflecting code changes with **context** of specific implementation changes and - **additional instructions** for maintaining documentation currency. -- If implementation validation against requirements is needed, then call the @requirements agent with the **request** - to validate implementation satisfies requirements with **context** of completed functionality and **goal** of - requirements compliance verification. - -## Implementation Standards by Language - -### C# Development - -#### C# Documentation Standards - -- **XML Documentation**: Required on ALL members (public/internal/private) with spaces after `///` -- **Standard XML Tags**: Use ``, ``, ``, `` -- **Compliance**: XML docs support automated compliance documentation generation - -**Example:** - -```csharp -/// -/// Processes user input data according to business rules -/// -/// User input data to process -/// Processed result with validation status -/// Thrown when input is invalid -public ProcessingResult ProcessUserData(UserData userData) -{ - // Validate input parameters meet business rule constraints - if (!InputValidator.IsValid(userData)) - { - throw new ArgumentException("User data does not meet validation requirements"); - } - - // Apply business transformation logic - var transformedData = BusinessEngine.Transform(userData); - - // Return structured result with success indicators - return new ProcessingResult(transformedData, ProcessingStatus.Success); -} -``` - -### C++ Development - -#### C++ Documentation Standards - -- **Doxygen Documentation**: Required on ALL members (public/protected/private) -- **Standard Doxygen Tags**: Use `@brief`, `@param`, `@return`, `@throws` -- **Compliance**: Doxygen comments support automated API documentation and compliance reports - -**Example:** - -```cpp -/// @brief Processes sensor data and validates against specifications -/// @param sensorReading Raw sensor data from hardware interface -/// @return Processed measurement with validation status -/// @throws std::invalid_argument if sensor reading is out of range -ProcessedMeasurement ProcessSensorData(const SensorReading& sensorReading) -{ - // Validate sensor reading falls within expected operational range - if (!IsValidSensorReading(sensorReading)) - { - throw std::invalid_argument("Sensor reading outside valid operational range"); - } - - // Apply calibration and filtering algorithms - auto calibratedValue = CalibrationEngine::Apply(sensorReading); - - // Return measurement with quality indicators - return ProcessedMeasurement{calibratedValue, MeasurementQuality::Valid}; -} -``` - -## Compliance Verification Checklist - -### Before Completing Implementation - -1. **Code Quality**: Zero warnings, passes all static analysis -2. **Documentation**: Comprehensive XML documentation (C#) or Doxygen comments (C++) on ALL members -3. **Testability**: Code structured for comprehensive testing -4. **Security**: Input validation, error handling, authorization checks -5. **Traceability**: Implementation traceable to requirements -6. **Standards**: Follows all coding standards and formatting rules - -## Don't Do These Things - -- Skip literate programming comments (mandatory for all code) -- Disable compiler warnings to make builds pass -- Create untestable code with hidden dependencies -- Skip XML documentation (C#) or Doxygen comments (C++) on any members -- Implement functionality without requirement traceability -- Ignore static analysis or security scanning results -- Write monolithic functions with multiple responsibilities diff --git a/.github/agents/technical-writer.agent.md b/.github/agents/technical-writer.agent.md deleted file mode 100644 index 0e1832e..0000000 --- a/.github/agents/technical-writer.agent.md +++ /dev/null @@ -1,258 +0,0 @@ ---- -name: technical-writer -description: Ensures documentation is accurate and complete. -tools: [read, search, edit, execute, github, agent] -user-invocable: true ---- - -# Technical Writer Agent - -Create and maintain clear, accurate, and -compliance-ready documentation following regulatory best practices and Continuous Compliance standards. - -## Reporting - -If detailed documentation of writing and editing activities is needed, -create a report using the filename pattern `AGENT_REPORT_documentation.md` to document content changes, -style decisions, and editorial processes. - -## When to Invoke This Agent - -Use the Technical Writer Agent for: - -- Creating and updating project documentation (README, guides, specifications) -- Ensuring documentation accuracy, completeness, and compliance -- Implementing regulatory documentation best practices -- Managing auto-generated compliance documentation -- Applying markdown linting and style standards - -## Primary Responsibilities - -### Continuous Compliance Documentation Standards - -#### Auto-Generated Documentation (CRITICAL - Do Not Edit Manually) - -```yaml -docs/ - requirements_doc/ - requirements.md # Generated by ReqStream - justifications.md # Generated by ReqStream - requirements_report/ - trace_matrix.md # Generated by ReqStream - build_notes.md # Generated by BuildMark - build_notes/ - versions.md # Generated by VersionMark - code_quality/ - sonar-quality.md # Generated by SonarMark - codeql-quality.md # Generated by SarifMark - code_review_plan/ - plan.md # Generated by ReviewMark - code_review_report/ - report.md # Generated by ReviewMark -``` - -**WARNING**: These files are regenerated on every CI/CD run. Manual edits will be lost. - -#### Project Documentation - -- **README.md**: Project overview, installation, usage -- **docs/*.md**: Architecture, design, user guides - -#### Code Documentation Coordination - -- **XML Documentation (C#)** and **Doxygen Comments (C++)**: Can be read and reviewed by @technical-writer agent for - accuracy and completeness -- **Code Comment Updates**: Must be performed by @software-developer agent, which maintains the proper formatting - rules and language-specific standards -- **Documentation Review**: @technical-writer agent verifies that code documentation aligns with overall project - documentation standard - -### Documentation Quality Standards - -#### Regulatory Documentation Excellence - -- **Purpose Statements**: Clear problem definition and document scope -- **Scope Boundaries**: Explicit inclusion/exclusion criteria -- **Traceability**: Links to requirements, tests, and implementation -- **Version Control**: Proper change tracking and approval workflows -- **Audience Targeting**: Appropriate detail level for intended readers - -#### Compliance-Ready Structure - -```markdown -# Document Title - -## Purpose - -[Why this document exists, what problem it solves] - -## Scope - -[What is covered, what is explicitly out of scope] - -## References - -[Links to related requirements, specifications, standards] - -# [Content sections organized logically] -``` - -#### Content Longevity Principles - -**Avoid Transitory Information**: Long-term documentation should not include information that becomes stale quickly: - -- **❌ Avoid**: Tool version numbers, specific counts (requirements, tests, files), current dates, "latest" references -- **❌ Examples**: "Currently using Node.js 18.2.1", "The system has 47 requirements", "As of March 2024" -- **✅ Instead**: Reference auto-generated reports, use relative descriptions, focus on stable concepts -- **✅ Examples**: "See docs/build_notes.md for current tool versions", "The requirements are organized by subsystem", - "The architecture follows..." - -**Exception**: Include transitory information only when documenting specific releases, version history, or -when the temporal context is the document's purpose. - -## Comprehensive Markdown & Documentation Standards - -### Link Style Rules by File Type - -#### Published Documents (README.md & Pandoc Document Structure) - -```markdown - -For more information, see [Continuous Compliance](https://github.com/demaconsulting/ContinuousCompliance). -Visit our website at https://docs.example.com/project-name -``` - -**CRITICAL**: Published documents (README.md and -any document in a Pandoc Document Structure) must use absolute URLs for all external links. -Relative links will break when documents are published, distributed as packages, or converted to PDF/other formats. - -**Published Document Types:** - -- README.md (shipped in packages and releases) -- Documents processed by Pandoc (typically in `docs/` with YAML frontmatter) -- Any document intended for standalone distribution - -#### AI Agent Files (`.github/agents/*.md`) - -```markdown - -For more information, see [Continuous Compliance](https://github.com/demaconsulting/ContinuousCompliance). -``` - -#### All Other Markdown Files - -```markdown - -For details, see the [Requirements Documentation][req-docs] and [Quality Standards][quality]. - -[req-docs]: https://github.com/demaconsulting/ContinuousCompliance/raw/refs/heads/main/docs/requirements.md -[quality]: https://github.com/demaconsulting/ContinuousCompliance/raw/refs/heads/main/docs/quality.md -``` - -### Documentation Linting Requirements - -Documentation formatting and spelling issues are automatically detected and reported by the project's lint scripts. -Run the repository's linting infrastructure to identify and resolve any documentation quality issues. - -### Pandoc Document Generation - -#### Pandoc Document Structure - -```yaml -docs/ - doc_folder/ - definition.yaml # Pandoc content definition - title.txt # Document metadata - introduction.md # Document introduction - sections/ # Individual content sections - sub-section.md # Sub-section document -``` - -#### Integration with CI/CD Pipeline - -```yaml -# Typical pipeline integration -- name: Generate Documentation - run: | - pandoc --metadata-file=docs/title.txt \ - --defaults=docs/definition.yaml \ - --output=docs/complete-document.pdf -``` - -### Diagram Integration Standards - -#### Mermaid Diagrams for Markdown - -Use **Mermaid diagrams** for all embedded diagrams in Markdown documents: - -```mermaid -graph TD - A[User Request] --> B[Auth Service] - B --> C[Business Logic] - C --> D[Data Layer] - D --> E[Database] -``` - -### Benefits of Mermaid Integration - -- **Version Control**: Diagrams stored as text, enabling proper diff tracking -- **Maintainability**: Easy to update diagrams alongside code changes -- **Consistency**: Standardized diagram styling across all documentation -- **Tooling Support**: Rendered automatically in GitHub, documentation sites, and modern editors -- **Accessibility**: Text-based format supports screen readers and accessibility tools - -## Quality Gate Verification - -### Documentation Linting Checklist - -- [ ] markdownlint-cli2 passes with zero errors -- [ ] cspell passes with zero spelling errors -- [ ] yamllint passes for any YAML content -- [ ] Links are functional and use correct style -- [ ] Generated documents compile without errors - -### Content Quality Standards - -- [ ] Purpose and scope clearly defined -- [ ] Audience-appropriate detail level -- [ ] Traceability to requirements maintained -- [ ] Examples and code snippets tested -- [ ] Cross-references accurate and current - -## Cross-Agent Coordination - -### Hand-off to Other Agents - -- If code examples, API documentation, or code comments need updating, then call the @software-developer agent with - the **request** to update code examples, API documentation, and code comments (XML/Doxygen) with **context** of - documentation requirements and **additional instructions** for maintaining code-documentation consistency. -- If documentation linting and quality checks need to be run, then call the @code-quality agent with the **request** - to run documentation linting and quality checks with **context** of updated documentation and **goal** of compliance - verification. -- If test procedures and coverage need documentation, then call the @test-developer agent with the **request** to - document test procedures and coverage with **context** of current test suite and **goal** of comprehensive test - documentation. - -## Compliance Verification Checklist - -### Before Completing Documentation Work - -1. **Linting**: All documentation passes markdownlint-cli2, cspell -2. **Structure**: Purpose and scope clearly defined -3. **Traceability**: Links to requirements, tests, code maintained -4. **Accuracy**: Content reflects current implementation -5. **Completeness**: All sections required for compliance included -6. **Generation**: Auto-generated docs compile successfully -7. **Links**: All references functional and use correct style -8. **Spelling**: Technical terms added to .cspell.yaml dictionary - -## Don't Do These Things - -- **Never edit auto-generated documentation** manually (will be overwritten) -- **Never edit code comments directly** (XML/Doxygen comments should be updated by @software-developer agent) -- **Never skip purpose and scope sections** in regulatory documents -- **Never ignore spelling errors** (add terms to .cspell.yaml instead) -- **Never use incorrect link styles** for file types (breaks tooling) -- **Never commit documentation** without linting verification -- **Never skip traceability links** in compliance-critical documents -- **Never document non-existent features** (code is source of truth) diff --git a/.github/agents/test-developer.agent.md b/.github/agents/test-developer.agent.md deleted file mode 100644 index 0c7f94b..0000000 --- a/.github/agents/test-developer.agent.md +++ /dev/null @@ -1,299 +0,0 @@ ---- -name: test-developer -description: Writes unit and integration tests. -tools: [read, search, edit, execute, github, agent] -user-invocable: true ---- - -# Test Developer Agent - -Develop comprehensive unit and integration tests with emphasis on requirements coverage and -Continuous Compliance verification. - -## Reporting - -If detailed documentation of testing activities is needed, -create a report using the filename pattern `AGENT_REPORT_testing.md` to document test strategies, coverage analysis, -and validation results. - -## When to Invoke This Agent - -Use the Test Developer Agent for: - -- Creating unit tests for new functionality -- Writing integration tests for component interactions -- Improving test coverage for compliance requirements -- Implementing AAA (Arrange-Act-Assert) pattern tests -- Generating platform-specific test evidence -- Upgrading legacy test suites to modern standards - -## Primary Responsibilities - -### Comprehensive Test Coverage Strategy - -#### Requirements Coverage (MANDATORY) - -- **All requirements MUST have linked tests** - Enforced by ReqStream -- **Platform-specific tests** must generate evidence with source filters -- **Test result formats** must be compatible (TRX, JUnit XML) -- **Coverage tracking** for audit and compliance purposes - -#### Test Type Strategy - -- **Unit Tests**: Individual component/function behavior -- **Integration Tests**: Component interaction and data flow -- **Platform Tests**: Platform-specific functionality validation -- **Validation Tests**: Self-validation and compliance verification - -### AAA Pattern Implementation (MANDATORY) - -All tests MUST follow Arrange-Act-Assert pattern for clarity and maintainability: - -```csharp -[TestMethod] -public void UserService_CreateUser_ValidInput_ReturnsSuccessResult() -{ - // Arrange - Set up test data and dependencies - var mockRepository = Substitute.For(); - var mockValidator = Substitute.For(); - var userService = new UserService(mockRepository, mockValidator); - var validUserData = new UserData - { - Name = "John Doe", - Email = "john@example.com" - }; - - // Act - Execute the system under test - var result = userService.CreateUser(validUserData); - - // Assert - Verify expected outcomes - Assert.IsTrue(result.IsSuccess); - Assert.AreEqual("John Doe", result.CreatedUser.Name); - mockRepository.Received(1).Save(Arg.Any()); -} -``` - -### Test Naming Standards - -#### C# Test Naming - -```csharp -// Pattern: ClassName_MethodUnderTest_Scenario_ExpectedBehavior -UserService_CreateUser_ValidInput_ReturnsSuccessResult() -UserService_CreateUser_InvalidEmail_ThrowsArgumentException() -UserService_CreateUser_DuplicateUser_ReturnsFailureResult() -``` - -#### C++ Test Naming - -```cpp -// Pattern: test_object_scenario_expected -test_user_service_valid_input_returns_success() -test_user_service_invalid_email_throws_exception() -test_user_service_duplicate_user_returns_failure() -``` - -## Quality Gate Verification - -### Test Quality Standards - -- [ ] All tests follow AAA pattern consistently -- [ ] Test names clearly describe scenario and expected outcome -- [ ] Each test validates single, specific behavior -- [ ] Both happy path and edge cases covered -- [ ] Platform-specific tests generate appropriate evidence -- [ ] Test results in standard formats (TRX, JUnit XML) - -### Requirements Traceability - -- [ ] Tests linked to specific requirements in requirements.yaml -- [ ] Source filters applied for platform-specific requirements -- [ ] Test coverage adequate for all stated requirements -- [ ] ReqStream validation passes with linked tests - -### Test Framework Standards - -#### C# Testing (MSTest V4) - -```csharp -[TestClass] -public class UserServiceTests -{ - private IUserRepository mockRepository; - private IValidator mockValidator; - - [TestInitialize] - public void Setup() - { - mockRepository = Substitute.For(); - mockValidator = Substitute.For(); - } - - [TestMethod] - public void UserService_ValidateUser_ValidData_ReturnsTrue() - { - // AAA implementation - } - - [TestCleanup] - public void Cleanup() - { - // Test cleanup if needed - } -} -``` - -#### C++ Testing (MSTest C++ / IAR Port) - -```cpp -TEST_CLASS(UserServiceTests) -{ - TEST_METHOD(test_user_service_validate_user_valid_data_returns_true) - { - // Arrange - setup test data - UserService service; - UserData validData{"John Doe", "john@example.com"}; - - // Act - execute test - bool result = service.ValidateUser(validData); - - // Assert - verify results - Assert::IsTrue(result); - } -}; -``` - -## Cross-Agent Coordination - -### Hand-off to Other Agents - -- If test quality gates and coverage metrics need verification, then call the @code-quality agent with the **request** - to verify test quality gates and coverage metrics with **context** of current test results and **goal** of meeting - coverage requirements. -- If test linkage needs to satisfy requirements traceability, then call the @requirements agent with the **request** - to ensure test linkage satisfies requirements traceability with **context** of test coverage and - **additional instructions** for maintaining traceability compliance. -- If testable code structure improvements are needed, then call the @software-developer agent with the **request** to - improve testable code structure with **context** of testing challenges and **goal** of enhanced testability. - -## Testing Infrastructure Requirements - -### Required Testing Tools - -```xml - - - - - - -``` - -### Test Result Generation - -```bash -# Generate test results with coverage -dotnet test --collect:"XPlat Code Coverage" --logger trx --results-directory TestResults - -# Platform-specific test execution -dotnet test --configuration Release --framework net8.0-windows --logger "trx;LogFileName=windows-tests.trx" -``` - -### CI/CD Integration - -```yaml -# Typical CI pipeline test stage -- name: Run Tests - run: | - dotnet test --configuration Release \ - --collect:"XPlat Code Coverage" \ - --logger trx \ - --results-directory TestResults \ - --verbosity normal - -- name: Upload Test Results - uses: actions/upload-artifact@v7 - with: - name: test-results - path: TestResults/**/*.trx -``` - -## Test Development Patterns - -### Comprehensive Test Coverage - -```csharp -[TestClass] -public class CalculatorTests -{ - [TestMethod] - public void Calculator_Add_PositiveNumbers_ReturnsSum() - { - // Happy path test - } - - [TestMethod] - public void Calculator_Add_NegativeNumbers_ReturnsSum() - { - // Edge case test - } - - [TestMethod] - public void Calculator_Divide_ByZero_ThrowsException() - { - // Error condition test - } - - [TestMethod] - public void Calculator_Divide_MaxValues_HandlesOverflow() - { - // Boundary condition test - } -} -``` - -### Mock and Dependency Testing - -```csharp -[TestMethod] -public void OrderService_ProcessOrder_ValidOrder_CallsPaymentService() -{ - // Arrange - Setup mocks and dependencies - var mockPaymentService = Substitute.For(); - var mockInventoryService = Substitute.For(); - var orderService = new OrderService(mockPaymentService, mockInventoryService); - - var testOrder = new Order { ProductId = 1, Quantity = 2, CustomerId = 123 }; - - // Act - Execute the system under test - var result = orderService.ProcessOrder(testOrder); - - // Assert - Verify interactions and outcomes - Assert.IsTrue(result.Success); - mockPaymentService.Received(1).ProcessPayment(Arg.Any()); - mockInventoryService.Received(1).ReserveItems(1, 2); -} -``` - -## Compliance Verification Checklist - -### Before Completing Test Work - -1. **AAA Pattern**: All tests follow Arrange-Act-Assert structure consistently -2. **Naming**: Test names clearly describe scenario and expected behavior -3. **Coverage**: Requirements coverage adequate, platform tests have source filters -4. **Quality**: Tests pass consistently, no flaky or unreliable tests -5. **Documentation**: Test intent and coverage clearly documented -6. **Integration**: Test results compatible with ReqStream and CI/CD pipeline -7. **Standards**: Follows framework-specific testing patterns and conventions - -## Don't Do These Things - -- **Never skip AAA pattern** in test structure (mandatory for consistency) -- **Never create tests without clear names** (must describe scenario/expectation) -- **Never write flaky tests** that pass/fail inconsistently -- **Never test implementation details** (test behavior, not internal mechanics) -- **Never skip edge cases** and error conditions -- **Never create tests without requirements linkage** (for compliance requirements) -- **Never ignore platform-specific test evidence** requirements -- **Never commit failing tests** (all tests must pass before merge) diff --git a/.github/standards/csharp-language.md b/.github/standards/csharp-language.md new file mode 100644 index 0000000..880544a --- /dev/null +++ b/.github/standards/csharp-language.md @@ -0,0 +1,86 @@ +# C# Language Coding Standards + +This document defines DEMA Consulting standards for C# software development +within Continuous Compliance environments. + +## Literate Programming Style (MANDATORY) + +Write all C# code in literate style because regulatory environments require +code that can be independently verified against requirements by reviewers. + +- **Intent Comments**: Start every code paragraph with a comment explaining + intent (not mechanics). Enables verification that code matches requirements. +- **Logical Separation**: Use blank lines to separate logical code paragraphs. + Makes algorithm structure visible to reviewers. +- **Purpose Over Process**: Comments describe why, code shows how. Separates + business logic from implementation details. +- **Standalone Clarity**: Reading comments alone should explain the algorithm + approach. Supports independent code review. + +### Example + +```csharp +// Validate input parameters to prevent downstream errors +if (string.IsNullOrEmpty(input)) +{ + throw new ArgumentException("Input cannot be null or empty", nameof(input)); +} + +// Transform input data using the configured processing pipeline +var processedData = ProcessingPipeline.Transform(input); + +// Apply business rules and validation logic +var validatedResults = BusinessRuleEngine.ValidateAndProcess(processedData); + +// Return formatted results matching the expected output contract +return OutputFormatter.Format(validatedResults); +``` + +## XML Documentation (MANDATORY) + +Document ALL members (public, internal, private) with XML comments because +compliance documentation is auto-generated from source code comments and review +agents need to validate implementation against documented intent. + +## Dependency Management + +Structure code for testability because all functionality must be validated +through automated tests linked to requirements. + +### Rules + +- **Inject Dependencies**: Use constructor injection for all external dependencies. + Enables mocking for unit tests. +- **Avoid Static Dependencies**: Use dependency injection instead of static + calls. Makes code testable in isolation. +- **Single Responsibility**: Each class should have one reason to change. + Simplifies testing and requirements traceability. +- **Pure Functions**: Minimize side effects and hidden state. Makes behavior + predictable and testable. + +## Error Handling + +Implement comprehensive error handling because failures must be logged for +audit trails and compliance reporting. + +- **Validate Inputs**: Check all parameters and throw appropriate exceptions + with clear messages +- **Use Typed Exceptions**: Throw specific exception types + (`ArgumentException`, `InvalidOperationException`) for different error + conditions +- **Include Context**: Exception messages should include enough information + for troubleshooting +- **Log Appropriately**: Use structured logging for audit trails in regulated + environments + +## Quality Checks + +Before submitting C# code, verify: + +- [ ] Code follows Literate Programming Style rules (intent comments, logical separation) +- [ ] XML documentation on ALL members with required tags +- [ ] Dependencies injected via constructor (no static dependencies) +- [ ] Single responsibility principle followed (one reason to change) +- [ ] Input validation with typed exceptions and clear messages +- [ ] Zero compiler warnings with `TreatWarningsAsErrors=true` +- [ ] Compatible with ReqStream requirements traceability diff --git a/.github/standards/csharp-testing.md b/.github/standards/csharp-testing.md new file mode 100644 index 0000000..6cee284 --- /dev/null +++ b/.github/standards/csharp-testing.md @@ -0,0 +1,119 @@ +# C# Testing Standards (MSTest) + +This document defines DEMA Consulting standards for C# test development using +MSTest within Continuous Compliance environments. + +# AAA Pattern Implementation (MANDATORY) + +Structure all tests using Arrange-Act-Assert pattern because regulatory reviews +require clear test logic that can be independently verified against +requirements. + +```csharp +[TestMethod] +public void ServiceName_MethodName_Scenario_ExpectedBehavior() +{ + // Arrange - (description) + // TODO: Set up test data, mocks, and system under test. + + // Act - (description) + // TODO: Execute the action being tested + + // Assert - (description) + // TODO: Verify expected outcomes and interactions +} +``` + +# Test Naming Standards + +Use descriptive test names because test names appear in requirements traceability matrices and compliance reports. + +- **Pattern**: `ClassName_MethodUnderTest_Scenario_ExpectedBehavior` +- **Descriptive Scenarios**: Clearly describe the input condition being tested +- **Expected Behavior**: State the expected outcome or exception + +## Examples + +- `UserValidator_ValidateEmail_ValidFormat_ReturnsTrue` +- `UserValidator_ValidateEmail_InvalidFormat_ThrowsArgumentException` +- `PaymentProcessor_ProcessPayment_InsufficientFunds_ReturnsFailureResult` + +# Requirements Coverage + +Link tests to requirements because every requirement must have passing test evidence for compliance validation. + +- **ReqStream Integration**: Tests must be linkable in requirements YAML files +- **Platform Filters**: Use source filters for platform-specific requirements (`windows@TestName`) +- **TRX Format**: Generate test results in TRX format for ReqStream compatibility +- **Coverage Completeness**: Test both success paths and error conditions + +# Mock Dependencies + +Mock external dependencies using NSubstitute (preferred) because tests must run in isolation to generate +reliable evidence. + +- **Isolate System Under Test**: Mock all external dependencies (databases, web services, file systems) +- **Verify Interactions**: Assert that expected method calls occurred with correct parameters +- **Predictable Behavior**: Set up mocks to return known values for consistent test results + +# MSTest V4 Antipatterns + +Avoid these common MSTest V4 patterns because they produce poor error messages or cause tests to be silently ignored. + +# Avoid Assertions in Catch Blocks (MSTEST0058) + +Instead of wrapping code in try/catch and asserting in the catch block, use `Assert.ThrowsExactly()`: + +```csharp +var ex = Assert.ThrowsExactly(() => SomeWork()); +Assert.Contains("Some message", ex.Message); +``` + +# Avoid Assert.IsTrue/IsFalse for Equality Checks + +Use `Assert.AreEqual`/`Assert.AreNotEqual` instead, as they provide better failure messages: + +```csharp +// ❌ Bad: Assert.IsTrue(result == expected); +// ✅ Good: Assert.AreEqual(expected, result); +``` + +# Avoid Non-Public Test Classes and Methods + +Test classes and `[TestMethod]` methods must be `public` or they will be silently ignored: + +```csharp +// ❌ Bad: internal class MyTests +// ✅ Good: public class MyTests +``` + +# Avoid Assert.IsTrue for Collection Count + +Use `Assert.HasCount` for count assertions: + +```csharp +// ❌ Bad: Assert.IsTrue(collection.Count == 3); +// ✅ Good: Assert.HasCount(3, collection); +``` + +# Avoid Assert.IsTrue for String Prefix Checks + +Use `Assert.StartsWith` instead, as it produces clearer failure messages: + +```csharp +// ❌ Bad: Assert.IsTrue(value.StartsWith("prefix")); +// ✅ Good: Assert.StartsWith("prefix", value); +``` + +# Quality Checks + +Before submitting C# tests, verify: + +- [ ] All tests follow AAA pattern with clear section comments +- [ ] Test names follow `ClassName_MethodUnderTest_Scenario_ExpectedBehavior` +- [ ] Each test verifies single, specific behavior (no shared state) +- [ ] Both success and failure scenarios covered including edge cases +- [ ] External dependencies mocked with NSubstitute or equivalent +- [ ] Tests linked to requirements with source filters where needed +- [ ] Test results generate TRX format for ReqStream compatibility +- [ ] MSTest V4 antipatterns avoided (proper assertions, public visibility, etc.) diff --git a/.github/standards/reqstream-usage.md b/.github/standards/reqstream-usage.md new file mode 100644 index 0000000..3f99929 --- /dev/null +++ b/.github/standards/reqstream-usage.md @@ -0,0 +1,146 @@ +# ReqStream Requirements Management Standards + +This document defines DEMA Consulting standards for requirements management +using ReqStream within Continuous Compliance environments. + +# Core Principles + +ReqStream implements Continuous Compliance methodology for automated evidence +generation: + +- **Requirements Traceability**: Every requirement MUST link to passing tests +- **Platform Evidence**: Source filters ensure correct testing environment + validation +- **Quality Gate Enforcement**: CI/CD fails on requirements without test + coverage +- **Audit Documentation**: Generated reports provide compliance evidence + +# Requirements Organization + +Organize requirements into separate files under `docs/reqstream/` for +independent review: + +```text +requirements.yaml # Root file (includes only) +docs/reqstream/ + {project}-system.yaml # System-level requirements + platform-requirements.yaml # Platform support requirements + subsystem-{subsystem}.yaml # Subsystem requirements + unit-{unit}.yaml # Unit (class) requirements + ots-{component}.yaml # OTS software item requirements +``` + +# Requirements File Format + +```yaml +sections: + - title: Functional Requirements + requirements: + - id: Project-Component-Feature + title: The system shall perform the required function. + justification: | + Business rationale explaining why this requirement exists. + Include regulatory or standard references where applicable. + tests: + - TestMethodName + - windows@PlatformSpecificTest # Source filter for platform evidence +``` + +# OTS Software Requirements + +Document third-party component requirements with specific section structure: + +```yaml +sections: + - title: OTS Software Requirements + sections: + - title: System.Text.Json + requirements: + - id: Project-SystemTextJson-ReadJson + title: System.Text.Json shall be able to read JSON files. + tests: + - JsonReaderTests.TestReadValidJson +``` + +# Semantic IDs (MANDATORY) + +Use meaningful IDs following `Project-Section-ShortDesc` pattern: + +- **Good**: `TemplateTool-Core-DisplayHelp` +- **Bad**: `REQ-042` (requires lookup to understand) + +# Requirement Best Practices + +Requirements specify WHAT the system shall do, not HOW: + +- Focus on externally observable characteristics and behavior +- Avoid implementation details, design constraints, or technology choices +- Each requirement must have clear, testable acceptance criteria + +Include business rationale for each requirement: + +- Business need or regulatory requirement +- Risk mitigation or quality improvement +- Standard or regulation references + +# Source Filter Requirements (CRITICAL) + +Platform-specific requirements MUST use source filters for compliance evidence: + +```yaml +tests: + - "windows@TestMethodName" # Windows platform evidence only + - "ubuntu@TestMethodName" # Linux platform evidence only + - "net8.0@TestMethodName" # .NET 8 runtime evidence only + - "TestMethodName" # Any platform evidence acceptable +``` + +**WARNING**: Removing source filters invalidates platform-specific compliance +evidence. + +# ReqStream Commands + +Essential ReqStream commands for Continuous Compliance: + +```bash +# Lint requirement files for issues (run before use) +dotnet reqstream \ + --requirements requirements.yaml \ + --lint + +# Enforce requirements traceability (use in CI/CD) +dotnet reqstream \ + --requirements requirements.yaml \ + --tests "artifacts/**/*.trx" \ + --enforce + +# Generate requirements report +dotnet reqstream \ + --requirements requirements.yaml \ + --report docs/requirements_doc/requirements.md + +# Generate justifications report +dotnet reqstream \ + --requirements requirements.yaml \ + --justifications docs/requirements_doc/justifications.md + +# Generate trace matrix +dotnet reqstream \ + --requirements requirements.yaml \ + --tests "artifacts/**/*.trx" \ + --matrix docs/requirements_report/trace_matrix.md +``` + +# Quality Checks + +Before submitting requirements, verify: + +- [ ] All requirements have semantic IDs (`Project-Section-Feature` pattern) +- [ ] Every requirement links to at least one passing test +- [ ] Platform-specific requirements use source filters (`platform@TestName`) +- [ ] Requirements specify observable behavior (WHAT), not implementation (HOW) +- [ ] Comprehensive justification explains business/regulatory need +- [ ] Files organized under `docs/reqstream/` following naming patterns +- [ ] Valid YAML syntax passes yamllint validation +- [ ] ReqStream enforcement passes: `dotnet reqstream --enforce` +- [ ] Test result formats compatible (TRX, JUnit XML) diff --git a/.github/standards/reviewmark-usage.md b/.github/standards/reviewmark-usage.md new file mode 100644 index 0000000..bdabd1d --- /dev/null +++ b/.github/standards/reviewmark-usage.md @@ -0,0 +1,151 @@ +# ReviewMark File Review Standards + +This document defines DEMA Consulting standards for managing file reviews using +ReviewMark within Continuous Compliance environments. + +# Core Purpose + +ReviewMark automates file review tracking using cryptographic fingerprints to +ensure: + +- Every file requiring review is covered by a current, valid review +- Reviews become stale when files change, triggering re-review +- Complete audit trail of review coverage for regulatory compliance + +# Review Definition Structure + +Configure reviews in `.reviewmark.yaml` at repository root: + +```yaml +# Patterns identifying all files that require review +needs-review: + # Include core development artifacts + - "**/*.cs" # All C# source and test files + - "**/*.md" # Requirements and design documentation + - "docs/reqstream/**/*.yaml" # Requirements files only + + # Exclude build output and generated content + - "!**/obj/**" # Exclude build output + - "!**/bin/**" # Exclude binary output + - "!**/generated/**" # Exclude auto-generated files + +# Source of review evidence +evidence-source: + type: none + +# Named review-sets grouping related files +reviews: + - id: MyProduct-PasswordValidator + title: Password Validator Unit Review + paths: + - "src/Auth/PasswordValidator.cs" + - "docs/reqstream/auth-passwordvalidator-class.yaml" + - "test/Auth/PasswordValidatorTests.cs" + - "docs/design/password-validation.md" + + - id: MyProduct-AllRequirements + title: All Requirements Review + paths: + - "requirements.yaml" + - "docs/reqstream/**/*.yaml" +``` + +# Review-Set Organization + +Organize review-sets using standard patterns to ensure comprehensive coverage +and consistent review processes: + +## [Project]-System Review + +Reviews system integration and operational validation: + +- **Files**: System-level requirements, design introduction, system design documents, integration tests +- **Purpose**: Validates system operates as designed and meets overall requirements +- **Example**: `TemplateTool-System` + +## [Product]-Design Review + +Reviews architectural and design consistency: + +- **Files**: System-level requirements, platform requirements, all design documents +- **Purpose**: Ensures design completeness and architectural coherence +- **Example**: `MyProduct-Design` + +## [Product]-AllRequirements Review + +Reviews requirements quality and traceability: + +- **Files**: All requirement files including root `requirements.yaml` +- **Purpose**: Validates requirements structure, IDs, justifications, and test linkage +- **Example**: `MyProduct-AllRequirements` + +## [Product]-[Unit] Review + +Reviews individual software unit implementation: + +- **Files**: Unit requirements, design documents, source code, unit tests +- **Purpose**: Validates unit meets requirements and is properly implemented +- **Example**: `MyProduct-PasswordValidator`, `MyProduct-ConfigParser` + +## [Product]-[Subsystem] Review + +Reviews subsystem architecture and interfaces: + +- **Files**: Subsystem requirements, design documents, integration tests (usually no source code) +- **Purpose**: Validates subsystem behavior and interface compliance +- **Example**: `MyProduct-Authentication`, `MyProduct-DataLayer` + +# ReviewMark Commands + +Essential ReviewMark commands for Continuous Compliance: + +```bash +# Lint review configuration for issues (run before use) +dotnet reviewmark \ + --lint + +# Generate review plan (shows coverage) +dotnet reviewmark \ + --plan docs/code_review_plan/plan.md + +# Generate review report (shows status) +dotnet reviewmark \ + --report docs/code_review_report/report.md + +# Enforce review compliance (use in CI/CD) +dotnet reviewmark \ + --plan docs/code_review_plan/plan.md \ + --report docs/code_review_report/report.md \ + --enforce +``` + +# File Pattern Best Practices + +Use "include-then-exclude" approach for `needs-review` patterns because it +ensures comprehensive coverage while removing unwanted files: + +## Include-Then-Exclude Strategy + +1. **Start broad**: Include all files of potential interest with generous patterns +2. **Exclude overreach**: Use `!` patterns to remove build output, generated files, and temporary files +3. **Test patterns**: Verify patterns match intended files using `dotnet reviewmark --elaborate` + +## Pattern Guidelines + +- **Be generous with includes**: Better to include too much initially than miss important files +- **Be specific with excludes**: Target exact paths and patterns that should never be reviewed +- **Order matters**: Patterns are processed sequentially, excludes override earlier includes + +# Quality Checks + +Before submitting ReviewMark configuration, verify: + +- [ ] `.reviewmark.yaml` exists at repository root with proper structure +- [ ] `needs-review` patterns cover requirements, design, code, and tests with proper exclusions +- [ ] Each review-set has unique `id` and groups architecturally related files +- [ ] File patterns use correct glob syntax and match intended files +- [ ] Evidence source properly configured (`none` for dev, `url` for production) +- [ ] Environment variables used for credentials (never hardcoded) +- [ ] ReviewMark enforcement configured: `dotnet reviewmark --enforce` +- [ ] Generated documents accessible for compliance auditing +- [ ] Review-set organization follows standard patterns ([Product]-[Unit], [Product]-Design, etc.) diff --git a/.github/standards/software-items.md b/.github/standards/software-items.md new file mode 100644 index 0000000..7991add --- /dev/null +++ b/.github/standards/software-items.md @@ -0,0 +1,45 @@ +# Software Items Definition Standards + +This document defines DEMA Consulting standards for categorizing software +items within Continuous Compliance environments because proper categorization +determines requirements management approach, testing strategy, and review +scope. + +# Software Item Categories + +Categorize all software into four primary groups: + +- **Software System**: Complete deliverable product including all components + and external interfaces +- **Software Subsystem**: Major architectural component with well-defined + interfaces and responsibilities +- **Software Unit**: Individual class, function, or tightly coupled set of + functions that can be tested in isolation +- **OTS Software Item**: Third-party component (library, framework, tool) + providing functionality not developed in-house + +# Categorization Guidelines + +Choose the appropriate category based on scope and testability: + +## Software System + +- Represents the entire product boundary +- Tested through system integration and end-to-end tests + +## Software Subsystem + +- Major architectural boundary (authentication, data layer, UI, communications) +- Tested through subsystem integration tests + +## Software Unit + +- Smallest independently testable component +- Tested through unit tests with mocked dependencies +- Typically a single class or cohesive set of functions + +## OTS Software Item + +- External dependency not developed in-house +- Tested through integration tests proving required functionality works +- Examples: System.Text.Json, Entity Framework, third-party APIs diff --git a/.github/standards/technical-documentation.md b/.github/standards/technical-documentation.md new file mode 100644 index 0000000..f09ee83 --- /dev/null +++ b/.github/standards/technical-documentation.md @@ -0,0 +1,172 @@ +# Technical Documentation Standards + +This document defines DEMA Consulting standards for technical documentation +within Continuous Compliance environments. + +# Core Principles + +Technical documentation serves as compliance evidence and must be structured +for regulatory review: + +- **Regulatory Compliance**: Documentation provides audit evidence and must be + current, accurate, and traceable to implementation +- **Agent-Readable Format**: Documentation may be processed by AI agents and + must follow consistent structure and formatting +- **Auto-Generation Support**: Compliance reports are generated automatically + and manual documentation must integrate seamlessly +- **Review Integration**: Documentation follows ReviewMark patterns for formal + review tracking + +# Documentation Organization + +Structure documentation under `docs/` following standard patterns for +consistency and tool compatibility: + +```text +docs/ + build_notes.md # Generated by BuildMark + build_notes/ # Auto-generated build notes + versions.md # Generated by VersionMark + code_review_plan/ # Auto-generated review plans + plan.md # Generated by ReviewMark + code_review_report/ # Auto-generated review reports + report.md # Generated by ReviewMark + design/ # Design documentation + introduction.md # Design overview + system.md # System architecture + {component}.md # Component-specific designs + reqstream/ # Requirements source files + {project}-system.yaml # System requirements + platform-requirements.yaml # Platform requirements + subsystem-{name}.yaml # Subsystem requirements + unit-{name}.yaml # Unit requirements + ots-{name}.yaml # OTS requirements + requirements_doc/ # Auto-generated requirements reports + requirements.md # Generated by ReqStream + justifications.md # Generated by ReqStream + requirements_report/ # Auto-generated trace matrices + trace_matrix.md # Generated by ReqStream + user_guide/ # User-facing documentation + introduction.md # User guide overview + {section}.md # User guide sections +``` + +# Pandoc Document Structure (MANDATORY) + +All document collections processed by Pandoc MUST include: + +- `definition.yaml` - specifying the files to include +- `title.txt` - document metadata +- `introduction.md` - document introduction +- `{sections}.md` - additional document sections + +## Introduction File Format + +```markdown +# Introduction + +Brief overview of the document collection purpose and audience. + +## Purpose + +Clear statement of why this documentation exists and what problem it solves. +Include regulatory or business drivers where applicable. + +## Scope + +Define what is covered and what is explicitly excluded from this documentation. +Specify version, system boundaries, and applicability constraints. +``` + +## Document Ordering + +List documents in logical reading order in Pandoc configuration because +readers need coherent information flow from general to specific topics. + +# Writing Guidelines + +Write technical documentation for clarity and compliance verification: + +- **Clear and Concise**: Use direct language and avoid unnecessary complexity. + Regulatory reviewers must understand content quickly. +- **Structured Sections**: Use consistent heading hierarchy and section + organization. Enables automated processing and review. +- **Specific Examples**: Include concrete examples with actual values rather + than placeholders. Supports implementation verification. +- **Current Information**: Keep documentation synchronized with code changes. + Outdated documentation invalidates compliance evidence. +- **Traceable Content**: Link documentation to requirements and implementation + where applicable for audit trails. + +# Markdown Format Requirements + +Markdown documentation in this repository must follow the formatting standards +defined in `.markdownlint-cli2.yaml` (subject to any exclusions configured there) +for consistency and professional presentation: + +- **120 Character Line Limit**: Keep lines 120 characters or fewer for readability. + Break long lines naturally at punctuation or logical breaks. +- **No Trailing Whitespace**: Remove all trailing spaces and tabs from line + endings to prevent formatting inconsistencies. +- **Blank Lines Around Headings**: Include a blank line both before and after + each heading to improve document structure and readability. +- **Blank Lines Around Lists**: Include a blank line both before and after + numbered and bullet lists to ensure proper rendering and visual separation. +- **ATX-Style Headers**: Use `#` syntax for headers instead of underline style + for consistency across all documentation. +- **Consistent List Indentation**: Use 2-space indentation for nested list + items to maintain uniform formatting. + +# Auto-Generated Content (CRITICAL) + +**NEVER modify auto-generated markdown files** because changes will be +overwritten and break compliance automation: + +- **Read-Only Files**: Generated reports under `docs/requirements_doc/`, + `docs/requirements_report/`, `docs/code_review_plan/`, and + `docs/code_review_report/` are regenerated on every build +- **Source Modification**: Update source files (requirements YAML, code + comments) instead of generated output +- **Tool Integration**: Generated content integrates with CI/CD pipelines and + manual changes disrupt automation + +# README.md Best Practices + +Structure README.md for both human readers and AI agent processing: + +## Content Requirements + +- **Project Overview**: Clear description of what the software does and why it exists +- **Installation Instructions**: Step-by-step setup with specific version requirements +- **Usage Examples**: Concrete examples with expected outputs, not just syntax +- **API Documentation**: Links to detailed API docs or inline examples for key functions +- **Contributing Guidelines**: Link to CONTRIBUTING.md with development setup +- **License Information**: Clear license statement with link to LICENSE file + +## Agent-Friendly Formatting + +- **Absolute URLs**: Use full GitHub URLs (not relative paths) for links because + agents may process README content outside repository context +- **Structured Sections**: Use consistent heading hierarchy for automated parsing +- **Code Block Languages**: Specify language for syntax highlighting and tool processing +- **Clear Prerequisites**: List exact version requirements and dependencies + +## Quality Guidelines + +- **Scannable Structure**: Use bullet points, headings, and short paragraphs +- **Current Examples**: Verify all code examples work with current version +- **Link Validation**: Ensure all external links are accessible and current +- **Consistent Tone**: Professional, helpful tone appropriate for technical audience + +# Quality Checks + +Before submitting technical documentation, verify: + +- [ ] Documentation organized under `docs/` following standard folder structure +- [ ] Pandoc collections include `introduction.md` with Purpose and Scope sections +- [ ] Content follows clear and concise writing guidelines with specific examples +- [ ] No modifications made to auto-generated markdown files in compliance folders +- [ ] README.md includes all required sections with absolute URLs and concrete examples +- [ ] Documentation integrated into ReviewMark review-sets for formal review +- [ ] Links validated and external references accessible +- [ ] Content synchronized with current code implementation and requirements diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8d64b2d..67a9d51 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -534,7 +534,6 @@ jobs: # TODO: Add --enforce once reviews branch is populated with review evidence PDFs and index.json run: > reviewmark - --definition .reviewmark.yaml --plan docs/code_review_plan/plan.md --plan-depth 1 --report docs/code_review_report/report.md @@ -600,11 +599,11 @@ jobs: shell: bash run: > dotnet pandoc - --defaults docs/guide/definition.yaml + --defaults docs/user_guide/definition.yaml --filter node_modules/.bin/mermaid-filter.cmd --metadata version="${{ inputs.version }}" --metadata date="$(date +'%Y-%m-%d')" - --output docs/guide/guide.html + --output docs/user_guide/introduction.html - name: Generate Code Quality HTML with Pandoc shell: bash @@ -671,7 +670,7 @@ jobs: run: > dotnet weasyprint --pdf-variant pdf/a-3u - docs/guide/guide.html + docs/user_guide/introduction.html "docs/ReviewMark User Guide.pdf" - name: Generate Code Quality PDF with Weasyprint diff --git a/.gitignore b/.gitignore index 48dc886..2d385e3 100644 --- a/.gitignore +++ b/.gitignore @@ -117,3 +117,4 @@ versionmark-*.json # Agent report files AGENT_REPORT_*.md +.agent-logs/ diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml index 04f1f80..4532ba3 100644 --- a/.markdownlint-cli2.yaml +++ b/.markdownlint-cli2.yaml @@ -11,6 +11,11 @@ # - Do not relax rules to accommodate existing non-compliant files # - Consistency across repositories is critical for documentation quality +noBanner: true + +# Disable the progress indicator on stdout +noProgress: true + config: # Enable all default rules default: true @@ -45,3 +50,4 @@ ignores: - "**/third-party/**" - "**/3rd-party/**" - "**/AGENT_REPORT_*.md" + - "**/.agent-logs/**" diff --git a/.reviewmark.yaml b/.reviewmark.yaml index a2c7b54..da57ad5 100644 --- a/.reviewmark.yaml +++ b/.reviewmark.yaml @@ -66,7 +66,7 @@ reviews: paths: - "docs/reqstream/unit-program.yaml" # requirements - "docs/design/program.md" # design - - "docs/guide/guide.md" # user guide + - "docs/user_guide/introduction.md" # user guide - "src/**/Program.cs" # implementation - "test/**/ProgramTests.cs" # unit tests - "test/**/TestDirectory.cs" # test infrastructure diff --git a/.yamllint.yaml b/.yamllint.yaml index 947ca60..061321b 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -12,6 +12,7 @@ ignore: | thirdparty/ third-party/ 3rd-party/ + .agent-logs/ rules: # Allow 'on:' in GitHub Actions workflows (not a boolean value) diff --git a/AGENTS.md b/AGENTS.md index c0d6359..04a9589 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,38 +3,44 @@ Comprehensive guidance for AI agents working on ReviewMark - a tool for automated file-review evidence management in regulated environments. +## Standards Application (ALL Agents Must Follow) + +Before performing any work, agents must read and apply the relevant standards from `.github/standards/`: + +- **`csharp-language.md`** - For C# code development (literate programming, XML docs, dependency injection) +- **`csharp-testing.md`** - For C# test development (AAA pattern, naming, MSTest anti-patterns) +- **`reqstream-usage.md`** - For requirements management (traceability, semantic IDs, source filters) +- **`reviewmark-usage.md`** - For file review management (review-sets, file patterns, enforcement) +- **`software-items.md`** - For software categorization (system/subsystem/unit/OTS classification) +- **`technical-documentation.md`** - For documentation creation and maintenance (structure, Pandoc, README best practices) + +Load only the standards relevant to your specific task scope and apply their +quality checks and guidelines throughout your work. + +## Agent Delegation Guidelines + +The default agent should handle simple, straightforward tasks directly. +Delegate to specialized agents only for specific scenarios: + +- **Light development work** (small fixes, simple features) → Call @developer agent +- **Light quality checking** (linting, basic validation) → Call @quality agent +- **Formal feature implementation** (complex, multi-step) → Call the `@implementation` agent +- **Formal bug resolution** (complex debugging, systematic fixes) → Call the `@implementation` agent +- **Formal reviews** (compliance verification, detailed analysis) → Call @code-review agent +- **Template consistency** (downstream repository alignment) → Call @repo-consistency agent + ## Available Specialized Agents -- **requirements** - Develops requirements and ensures test coverage linkage -- **technical-writer** - Creates accurate documentation following regulatory best practices -- **software-developer** - Writes production code and self-validation tests in literate style -- **test-developer** - Creates unit and integration tests following AAA pattern -- **code-quality** - Enforces linting, static analysis, and security standards -- **code-review** - Assists in performing formal file reviews -- **repo-consistency** - Ensures downstream repositories remain consistent with template patterns - -## Agent Selection - -- To fix a bug, call the @software-developer agent with the **context** of the bug details and **goal** of resolving - the issue while maintaining code quality. -- To add a new feature, call the @requirements agent with the **request** to define feature requirements and **context** - of business needs and **goal** of comprehensive requirement specification. -- To write or fix tests, call the @test-developer agent with the **context** of the functionality to be tested and - **goal** of achieving comprehensive test coverage. -- To update documentation, call the @technical-writer agent with the **context** of changes requiring documentation and - **goal** of maintaining current and accurate documentation. -- To manage requirements and traceability, call the @requirements agent with the **context** of requirement changes and - **goal** of maintaining compliance traceability. -- To resolve quality or linting issues, call the @code-quality agent with the **context** of quality gate failures and - **goal** of achieving compliance standards. -- To update linting tools or scripts, call the @code-quality agent with the **context** of tool requirements and - **goal** of maintaining quality infrastructure. -- To address security alerts or scanning issues, call the @code-quality agent with the **context** of security findings - and **goal** of resolving vulnerabilities. -- To perform file reviews, call the @code-review agent with the **context** of files requiring review and **goal** of - compliance verification. -- To ensure template consistency, call the @repo-consistency agent with the **context** of downstream repository - and **goal** of maintaining template alignment. +- **code-review** - Agent for performing formal reviews using standardized + review processes +- **developer** - General-purpose software development agent that applies + appropriate standards based on the work being performed +- **implementation** - Orchestrator agent that manages quality implementations + through a formal state machine workflow +- **quality** - Quality assurance agent that grades developer work against DEMA + Consulting standards and Continuous Compliance practices +- **repo-consistency** - Ensures downstream repositories remain consistent with + the TemplateDotNetTool template patterns and best practices ## Quality Gate Enforcement (ALL Agents Must Verify) @@ -134,7 +140,7 @@ build.bat # Windows ## Documentation -- **User Guide**: `docs/guide/guide.md` +- **User Guide**: `docs/user_guide/introduction.md` - **Requirements**: `requirements.yaml` -> auto-generated docs - **Build Notes**: Auto-generated via BuildMark - **Code Quality**: Auto-generated via CodeQL and SonarMark @@ -171,11 +177,10 @@ dotnet pack --configuration Release ## Agent Report Files -When agents need to write report files to communicate with each other or the user, follow these guidelines: +Upon completion, create a report file at `.agent-logs/[agent-name]-[subject]-[unique-id].md` that includes: + +- A concise summary of the work performed +- Any important decisions made and their rationale +- Follow-up items, open questions, or TODOs -- **Naming Convention**: Use the pattern `AGENT_REPORT_xxxx.md` (e.g., `AGENT_REPORT_analysis.md`, `AGENT_REPORT_results.md`) -- **Purpose**: These files are for temporary inter-agent communication and should not be committed -- **Exclusions**: Files matching `AGENT_REPORT_*.md` are automatically: - - Excluded from git (via .gitignore) - - Excluded from markdown linting - - Excluded from spell checking +Store agent logs in the `.agent-logs/` folder so they are ignored via `.gitignore` and excluded from linting and commits. diff --git a/README.md b/README.md index 9613b2a..1bb807f 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,6 @@ By contributing to this project, you agree that your contributions will be licen [link-quality]: https://sonarcloud.io/dashboard?id=demaconsulting_ReviewMark [link-security]: https://sonarcloud.io/dashboard?id=demaconsulting_ReviewMark [link-nuget]: https://www.nuget.org/packages/DemaConsulting.ReviewMark -[link-guide]: https://github.com/demaconsulting/ReviewMark/blob/main/docs/guide/guide.md +[link-guide]: https://github.com/demaconsulting/ReviewMark/blob/main/docs/user_guide/introduction.md [link-theory-of-operations]: https://github.com/demaconsulting/ReviewMark/blob/main/THEORY-OF-OPERATIONS.md [link-continuous-compliance]: https://github.com/demaconsulting/ContinuousCompliance diff --git a/docs/guide/definition.yaml b/docs/user_guide/definition.yaml similarity index 58% rename from docs/guide/definition.yaml rename to docs/user_guide/definition.yaml index 19f05ce..01a2e76 100644 --- a/docs/guide/definition.yaml +++ b/docs/user_guide/definition.yaml @@ -1,10 +1,10 @@ --- resource-path: - - docs/guide + - docs/user_guide - docs/template input-files: - - docs/guide/title.txt - - docs/guide/guide.md + - docs/user_guide/title.txt + - docs/user_guide/introduction.md template: template.html table-of-contents: true number-sections: true diff --git a/docs/guide/guide.md b/docs/user_guide/introduction.md similarity index 100% rename from docs/guide/guide.md rename to docs/user_guide/introduction.md diff --git a/docs/guide/title.txt b/docs/user_guide/title.txt similarity index 100% rename from docs/guide/title.txt rename to docs/user_guide/title.txt diff --git a/lint.bat b/lint.bat index f94b53d..c7440d4 100644 --- a/lint.bat +++ b/lint.bat @@ -12,17 +12,17 @@ REM - Agents execute this script to identify files needing fixes set "LINT_ERROR=0" REM Install npm dependencies -call npm install +call npm install --silent REM Create Python virtual environment (for yamllint) if missing if not exist ".venv\Scripts\activate.bat" ( python -m venv .venv ) call .venv\Scripts\activate.bat -pip install -r pip-requirements.txt +pip install -r pip-requirements.txt --quiet --disable-pip-version-check REM Run spell check -call npx cspell --no-progress --no-color "**/*.{md,yaml,yml,json,cs,cpp,hpp,h,txt}" +call npx cspell --no-progress --no-color --quiet "**/*.{md,yaml,yml,json,cs,cpp,hpp,h,txt}" if errorlevel 1 set "LINT_ERROR=1" REM Run markdownlint check diff --git a/lint.sh b/lint.sh index 7d8116b..c567e09 100755 --- a/lint.sh +++ b/lint.sh @@ -11,17 +11,17 @@ lint_error=0 # Install npm dependencies -npm install +npm install --silent # Create Python virtual environment (for yamllint) if [ ! -d ".venv" ]; then python -m venv .venv fi source .venv/bin/activate -pip install -r pip-requirements.txt +pip install -r pip-requirements.txt --quiet --disable-pip-version-check # Run spell check -npx cspell --no-progress --no-color "**/*.{md,yaml,yml,json,cs,cpp,hpp,h,txt}" || lint_error=1 +npx cspell --no-progress --no-color --quiet "**/*.{md,yaml,yml,json,cs,cpp,hpp,h,txt}" || lint_error=1 # Run markdownlint check npx markdownlint-cli2 "**/*.md" || lint_error=1 From 79803603cb7a1bd6a5a14c48ad5d67356eddf739 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 06:20:57 -0400 Subject: [PATCH 09/19] Bump demaconsulting.pandoctool from 3.9.0 to 3.9.0.2 (#36) --- updated-dependencies: - dependency-name: demaconsulting.pandoctool dependency-version: 3.9.0.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .config/dotnet-tools.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index e1f510b..da5068e 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -9,7 +9,7 @@ ] }, "demaconsulting.pandoctool": { - "version": "3.9.0", + "version": "3.9.0.2", "commands": [ "pandoc" ] From 392113fa26fe83064cccbddf9dc2b8332bdedecc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 06:21:27 -0400 Subject: [PATCH 10/19] Bump the nuget-dependencies group with 5 updates (#35) Bumps demaconsulting.reqstream from 1.4.1 to 1.5.0 Bumps demaconsulting.sonarmark from 1.2.0 to 1.3.0 Bumps demaconsulting.versionmark from 1.0.0 to 1.1.0 Bumps Polyfill from 9.22.0 to 9.23.0 Bumps SonarAnalyzer.CSharp from 10.21.0.135717 to 10.22.0.136894 --- updated-dependencies: - dependency-name: demaconsulting.reqstream dependency-version: 1.5.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: nuget-dependencies - dependency-name: demaconsulting.sonarmark dependency-version: 1.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: nuget-dependencies - dependency-name: demaconsulting.versionmark dependency-version: 1.1.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: nuget-dependencies - dependency-name: Polyfill dependency-version: 9.23.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: nuget-dependencies - dependency-name: SonarAnalyzer.CSharp dependency-version: 10.22.0.136894 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: nuget-dependencies - dependency-name: SonarAnalyzer.CSharp dependency-version: 10.22.0.136894 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: nuget-dependencies ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .config/dotnet-tools.json | 6 +++--- .../DemaConsulting.ReviewMark.csproj | 4 ++-- .../DemaConsulting.ReviewMark.Tests.csproj | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index da5068e..9f771a4 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -27,13 +27,13 @@ ] }, "demaconsulting.sonarmark": { - "version": "1.2.0", + "version": "1.3.0", "commands": [ "sonarmark" ] }, "demaconsulting.reqstream": { - "version": "1.4.1", + "version": "1.5.0", "commands": [ "reqstream" ] @@ -45,7 +45,7 @@ ] }, "demaconsulting.versionmark": { - "version": "1.0.0", + "version": "1.1.0", "commands": [ "versionmark" ] diff --git a/src/DemaConsulting.ReviewMark/DemaConsulting.ReviewMark.csproj b/src/DemaConsulting.ReviewMark/DemaConsulting.ReviewMark.csproj index bf8e20b..f747b99 100644 --- a/src/DemaConsulting.ReviewMark/DemaConsulting.ReviewMark.csproj +++ b/src/DemaConsulting.ReviewMark/DemaConsulting.ReviewMark.csproj @@ -58,7 +58,7 @@ - + @@ -72,7 +72,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/test/DemaConsulting.ReviewMark.Tests/DemaConsulting.ReviewMark.Tests.csproj b/test/DemaConsulting.ReviewMark.Tests/DemaConsulting.ReviewMark.Tests.csproj index e961571..9c9d7a8 100644 --- a/test/DemaConsulting.ReviewMark.Tests/DemaConsulting.ReviewMark.Tests.csproj +++ b/test/DemaConsulting.ReviewMark.Tests/DemaConsulting.ReviewMark.Tests.csproj @@ -50,7 +50,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive From c38d2dd98641f5446fb3991dcd22353b90a3ed30 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 31 Mar 2026 10:41:52 -0400 Subject: [PATCH 11/19] Restructure ReviewMark into subsystem folders following ReqStream template pattern (#37) * Initial plan * Restructure source into subsystem folder layout - Move source files into Cli/, Configuration/, Indexing/, SelfTest/ subsystem folders - Move test files into matching subsystem folders - Move design docs into matching subsystem folders under docs/design/ - Update namespaces: DemaConsulting.ReviewMark.{Cli,Configuration,Indexing,SelfTest} - Add using directives to cross-subsystem consumers - Update docs/design/definition.yaml with new input-file paths - Rewrite docs/design/introduction.md with subsystem-aware content - Create docs/reqstream/subsystem-indexing.yaml - Create docs/reqstream/subsystem-self-test.yaml - Update requirements.yaml to include new subsystem requirement files - Update .reviewmark.yaml with new review-sets and updated paths Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Fix lint errors: correct import ordering and yaml line length Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Use glob pattern for design docs in ReviewMark-Design review-set Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/4ae917d8-dada-479b-a86a-e2ef2dd74fde Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Reorganize docs/reqstream into subsystem subdirectories mirroring docs/design Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/20e6ebb2-c034-4d2d-bea2-2f9a1d21613d Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Update .reviewmark.yaml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Co-authored-by: Malcolm Nixon Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .reviewmark.yaml | 126 +++++++++--------- docs/design/{ => cli}/context.md | 0 .../{ => configuration}/glob-matcher.md | 0 .../review-mark-configuration.md | 0 docs/design/definition.yaml | 20 ++- docs/design/{ => indexing}/path-helpers.md | 0 docs/design/{ => indexing}/review-index.md | 0 docs/design/introduction.md | 112 +++++++++++----- docs/design/{ => self-test}/validation.md | 0 docs/reqstream/{ => cli}/subsystem-cli.yaml | 0 docs/reqstream/{ => cli}/unit-context.yaml | 0 .../subsystem-configuration.yaml | 0 .../unit-glob-matcher.yaml | 0 .../indexing/subsystem-indexing.yaml | 57 ++++++++ .../{ => indexing}/unit-path-helpers.yaml | 0 .../{ => indexing}/unit-review-index.yaml | 0 docs/reqstream/{ => ots}/ots-buildmark.yaml | 0 docs/reqstream/{ => ots}/ots-mstest.yaml | 0 docs/reqstream/{ => ots}/ots-reqstream.yaml | 0 docs/reqstream/{ => ots}/ots-sarifmark.yaml | 0 docs/reqstream/{ => ots}/ots-sonarmark.yaml | 0 docs/reqstream/{ => ots}/ots-versionmark.yaml | 0 .../self-test/subsystem-self-test.yaml | 38 ++++++ .../{ => self-test}/unit-validation.yaml | 0 requirements.yaml | 30 +++-- .../{ => Cli}/Context.cs | 2 +- .../{ => Configuration}/GlobMatcher.cs | 2 +- .../ReviewMarkConfiguration.cs | 3 +- .../{ => Indexing}/PathHelpers.cs | 2 +- .../{ => Indexing}/ReviewIndex.cs | 3 +- src/DemaConsulting.ReviewMark/Program.cs | 4 + .../{ => SelfTest}/Validation.cs | 4 +- .../{ => Cli}/ContextTests.cs | 4 +- .../{ => Configuration}/GlobMatcherTests.cs | 5 +- .../ReviewMarkConfigurationTests.cs | 5 +- .../{ => Indexing}/IndexTests.cs | 4 +- .../{ => Indexing}/PathHelpersTests.cs | 4 +- .../IntegrationTests.cs | 2 + .../ProgramTests.cs | 3 + .../{ => SelfTest}/ValidationTests.cs | 5 +- .../TestDirectory.cs | 2 + 41 files changed, 304 insertions(+), 133 deletions(-) rename docs/design/{ => cli}/context.md (100%) rename docs/design/{ => configuration}/glob-matcher.md (100%) rename docs/design/{ => configuration}/review-mark-configuration.md (100%) rename docs/design/{ => indexing}/path-helpers.md (100%) rename docs/design/{ => indexing}/review-index.md (100%) rename docs/design/{ => self-test}/validation.md (100%) rename docs/reqstream/{ => cli}/subsystem-cli.yaml (100%) rename docs/reqstream/{ => cli}/unit-context.yaml (100%) rename docs/reqstream/{ => configuration}/subsystem-configuration.yaml (100%) rename docs/reqstream/{ => configuration}/unit-glob-matcher.yaml (100%) create mode 100644 docs/reqstream/indexing/subsystem-indexing.yaml rename docs/reqstream/{ => indexing}/unit-path-helpers.yaml (100%) rename docs/reqstream/{ => indexing}/unit-review-index.yaml (100%) rename docs/reqstream/{ => ots}/ots-buildmark.yaml (100%) rename docs/reqstream/{ => ots}/ots-mstest.yaml (100%) rename docs/reqstream/{ => ots}/ots-reqstream.yaml (100%) rename docs/reqstream/{ => ots}/ots-sarifmark.yaml (100%) rename docs/reqstream/{ => ots}/ots-sonarmark.yaml (100%) rename docs/reqstream/{ => ots}/ots-versionmark.yaml (100%) create mode 100644 docs/reqstream/self-test/subsystem-self-test.yaml rename docs/reqstream/{ => self-test}/unit-validation.yaml (100%) rename src/DemaConsulting.ReviewMark/{ => Cli}/Context.cs (99%) rename src/DemaConsulting.ReviewMark/{ => Configuration}/GlobMatcher.cs (98%) rename src/DemaConsulting.ReviewMark/{ => Configuration}/ReviewMarkConfiguration.cs (99%) rename src/DemaConsulting.ReviewMark/{ => Indexing}/PathHelpers.cs (98%) rename src/DemaConsulting.ReviewMark/{ => Indexing}/ReviewIndex.cs (99%) rename src/DemaConsulting.ReviewMark/{ => SelfTest}/Validation.cs (99%) rename test/DemaConsulting.ReviewMark.Tests/{ => Cli}/ContextTests.cs (99%) rename test/DemaConsulting.ReviewMark.Tests/{ => Configuration}/GlobMatcherTests.cs (98%) rename test/DemaConsulting.ReviewMark.Tests/{ => Configuration}/ReviewMarkConfigurationTests.cs (99%) rename test/DemaConsulting.ReviewMark.Tests/{ => Indexing}/IndexTests.cs (99%) rename test/DemaConsulting.ReviewMark.Tests/{ => Indexing}/PathHelpersTests.cs (98%) rename test/DemaConsulting.ReviewMark.Tests/{ => SelfTest}/ValidationTests.cs (98%) diff --git a/.reviewmark.yaml b/.reviewmark.yaml index da57ad5..3ad6044 100644 --- a/.reviewmark.yaml +++ b/.reviewmark.yaml @@ -8,8 +8,8 @@ needs-review: - "**/*.cs" # All C# source and test files - "requirements.yaml" # Root requirements file - - "docs/reqstream/*.yaml" # Per-software-item requirements files - - "docs/design/*.md" # Software design documents + - "docs/reqstream/**/*.yaml" # Per-software-item requirements files + - "docs/design/**/*.md" # Software design documents (including subsystem folders) - "!**/obj/**" # Exclude build output - "!**/bin/**" # Exclude build output @@ -28,64 +28,85 @@ evidence-source: # - source: what the code actually does # - tests: which behaviors are verified and how reviews: - # Software unit reviews - one per class + # Software unit reviews - one per unit + - id: ReviewMark-Program + title: Review of Program software unit (main entry point and tool orchestration) + paths: + - "docs/reqstream/unit-program.yaml" # requirements + - "docs/design/program.md" # design + - "docs/user_guide/introduction.md" # user guide + - "src/**/Program.cs" # implementation + - "test/**/ProgramTests.cs" # unit tests + - "test/**/TestDirectory.cs" # test infrastructure + - id: ReviewMark-Context title: Review of Context software unit (command-line argument handling) paths: - - "docs/reqstream/unit-context.yaml" # requirements - - "docs/design/context.md" # design - - "src/**/Context.cs" # implementation - - "test/**/ContextTests.cs" # tests + - "docs/reqstream/cli/unit-context.yaml" # requirements + - "docs/design/cli/context.md" # design + - "src/**/Cli/Context.cs" # implementation + - "test/**/Cli/ContextTests.cs" # tests + + - id: ReviewMark-ReviewMarkConfiguration + title: Review of ReviewMarkConfiguration software unit (configuration parsing and processing) + paths: + - "docs/reqstream/configuration/unit-review-mark-configuration.yaml" # requirements + - "docs/design/configuration/review-mark-configuration.md" # design + - "src/**/Configuration/ReviewMarkConfiguration.cs" # implementation + - "test/**/Configuration/ReviewMarkConfigurationTests.cs" # tests - id: ReviewMark-GlobMatcher title: Review of GlobMatcher software unit (file pattern matching) paths: - - "docs/reqstream/unit-glob-matcher.yaml" # requirements - - "docs/design/glob-matcher.md" # design - - "src/**/GlobMatcher.cs" # implementation - - "test/**/GlobMatcherTests.cs" # tests + - "docs/reqstream/configuration/unit-glob-matcher.yaml" # requirements + - "docs/design/configuration/glob-matcher.md" # design + - "src/**/Configuration/GlobMatcher.cs" # implementation + - "test/**/Configuration/GlobMatcherTests.cs" # tests - id: ReviewMark-ReviewIndex title: Review of ReviewIndex software unit (review evidence indexing) paths: - - "docs/reqstream/unit-review-index.yaml" # requirements - - "docs/design/review-index.md" # design - - "src/**/ReviewIndex.cs" # implementation - - "test/**/IndexTests.cs" # tests + - "docs/reqstream/indexing/unit-review-index.yaml" # requirements + - "docs/design/indexing/review-index.md" # design + - "src/**/Indexing/ReviewIndex.cs" # implementation + - "test/**/Indexing/IndexTests.cs" # tests - id: ReviewMark-PathHelpers title: Review of PathHelpers software unit (file path utilities) paths: - - "docs/reqstream/unit-path-helpers.yaml" # requirements - - "docs/design/path-helpers.md" # design - - "src/**/PathHelpers.cs" # implementation - - "test/**/PathHelpersTests.cs" # tests + - "docs/reqstream/indexing/unit-path-helpers.yaml" # requirements + - "docs/design/indexing/path-helpers.md" # design + - "src/**/Indexing/PathHelpers.cs" # implementation + - "test/**/Indexing/PathHelpersTests.cs" # tests - - id: ReviewMark-Program - title: Review of Program software unit (main entry point and tool orchestration) + - id: ReviewMark-Validation + title: Review of Validation software unit (self-validation test execution) paths: - - "docs/reqstream/unit-program.yaml" # requirements - - "docs/design/program.md" # design - - "docs/user_guide/introduction.md" # user guide - - "src/**/Program.cs" # implementation - - "test/**/ProgramTests.cs" # unit tests - - "test/**/TestDirectory.cs" # test infrastructure + - "docs/reqstream/self-test/unit-validation.yaml" # requirements + - "docs/design/self-test/validation.md" # design + - "src/**/SelfTest/Validation.cs" # implementation + - "test/**/SelfTest/ValidationTests.cs" # tests - - id: ReviewMark-Configuration - title: Review of ReviewMarkConfiguration software unit (configuration parsing and processing) + # Subsystem reviews + - id: ReviewMark-Cli + title: Review of Cli subsystem (command-line interface) paths: - - "docs/reqstream/subsystem-configuration.yaml" # requirements - - "docs/design/review-mark-configuration.md" # design - - "src/**/ReviewMarkConfiguration.cs" # implementation - - "test/**/ReviewMarkConfigurationTests.cs" # tests + - "docs/reqstream/cli/subsystem-cli.yaml" # subsystem requirements + - "docs/design/cli/context.md" # Context design + - "docs/design/program.md" # Program design - - id: ReviewMark-Validation - title: Review of Validation software unit (self-validation test execution) + - id: ReviewMark-Indexing + title: Review of Indexing subsystem (review evidence loading and path utilities) + paths: + - "docs/reqstream/indexing/subsystem-indexing.yaml" # subsystem requirements + - "docs/design/indexing/review-index.md" # ReviewIndex design + - "docs/design/indexing/path-helpers.md" # PathHelpers design + + - id: ReviewMark-SelfTest + title: Review of SelfTest subsystem (self-validation) paths: - - "docs/reqstream/unit-validation.yaml" # requirements - - "docs/design/validation.md" # design - - "src/**/Validation.cs" # implementation - - "test/**/ValidationTests.cs" # tests + - "docs/reqstream/self-test/subsystem-self-test.yaml" # subsystem requirements + - "docs/design/self-test/validation.md" # Validation design # Special review-sets - id: ReviewMark-System @@ -103,33 +124,10 @@ reviews: title: Review of all ReviewMark design documentation paths: - "docs/reqstream/platform-requirements.yaml" # platform requirements - - "docs/design/introduction.md" # design introduction and architecture - - "docs/design/system.md" # system design - - "docs/design/context.md" # Context design - - "docs/design/glob-matcher.md" # GlobMatcher design - - "docs/design/review-index.md" # ReviewIndex design - - "docs/design/path-helpers.md" # PathHelpers design - - "docs/design/program.md" # Program design - - "docs/design/review-mark-configuration.md" # ReviewMarkConfiguration design - - "docs/design/validation.md" # Validation design + - "docs/design/**/*.md" # all design documents - id: ReviewMark-AllRequirements title: Review of all ReviewMark requirements files paths: - "requirements.yaml" # root requirements file - - "docs/reqstream/reviewmark-system.yaml" # system-level requirements - - "docs/reqstream/subsystem-cli.yaml" # CLI subsystem requirements - - "docs/reqstream/subsystem-configuration.yaml" # Configuration subsystem requirements - - "docs/reqstream/unit-context.yaml" # Context unit requirements - - "docs/reqstream/unit-program.yaml" # Program unit requirements - - "docs/reqstream/unit-review-index.yaml" # ReviewIndex unit requirements - - "docs/reqstream/unit-glob-matcher.yaml" # GlobMatcher unit requirements - - "docs/reqstream/unit-path-helpers.yaml" # PathHelpers unit requirements - - "docs/reqstream/unit-validation.yaml" # Validation unit requirements - - "docs/reqstream/platform-requirements.yaml" # Platform support requirements - - "docs/reqstream/ots-mstest.yaml" # MSTest OTS requirements - - "docs/reqstream/ots-reqstream.yaml" # ReqStream OTS requirements - - "docs/reqstream/ots-buildmark.yaml" # BuildMark OTS requirements - - "docs/reqstream/ots-versionmark.yaml" # VersionMark OTS requirements - - "docs/reqstream/ots-sarifmark.yaml" # SarifMark OTS requirements - - "docs/reqstream/ots-sonarmark.yaml" # SonarMark OTS requirements + - "docs/reqstream/**/*.yaml" # all requirements files diff --git a/docs/design/context.md b/docs/design/cli/context.md similarity index 100% rename from docs/design/context.md rename to docs/design/cli/context.md diff --git a/docs/design/glob-matcher.md b/docs/design/configuration/glob-matcher.md similarity index 100% rename from docs/design/glob-matcher.md rename to docs/design/configuration/glob-matcher.md diff --git a/docs/design/review-mark-configuration.md b/docs/design/configuration/review-mark-configuration.md similarity index 100% rename from docs/design/review-mark-configuration.md rename to docs/design/configuration/review-mark-configuration.md diff --git a/docs/design/definition.yaml b/docs/design/definition.yaml index 1b115fd..2a99aa1 100644 --- a/docs/design/definition.yaml +++ b/docs/design/definition.yaml @@ -1,18 +1,26 @@ --- resource-path: - docs/design + - docs/design/cli + - docs/design/configuration + - docs/design/indexing + - docs/design/self-test - docs/template + input-files: - docs/design/title.txt - docs/design/introduction.md - docs/design/system.md - - docs/design/context.md - - docs/design/glob-matcher.md - - docs/design/review-index.md - - docs/design/path-helpers.md - docs/design/program.md - - docs/design/review-mark-configuration.md - - docs/design/validation.md + - docs/design/cli/context.md + - docs/design/configuration/glob-matcher.md + - docs/design/configuration/review-mark-configuration.md + - docs/design/indexing/review-index.md + - docs/design/indexing/path-helpers.md + - docs/design/self-test/validation.md + template: template.html + table-of-contents: true + number-sections: true diff --git a/docs/design/path-helpers.md b/docs/design/indexing/path-helpers.md similarity index 100% rename from docs/design/path-helpers.md rename to docs/design/indexing/path-helpers.md diff --git a/docs/design/review-index.md b/docs/design/indexing/review-index.md similarity index 100% rename from docs/design/review-index.md rename to docs/design/indexing/review-index.md diff --git a/docs/design/introduction.md b/docs/design/introduction.md index 648be94..d48b144 100644 --- a/docs/design/introduction.md +++ b/docs/design/introduction.md @@ -1,56 +1,94 @@ # Introduction -This document describes the software design for the ReviewMark project. +This document provides the detailed design for the ReviewMark tool, a .NET command-line +application for automated file-review evidence management in regulated environments. ## Purpose -ReviewMark is a .NET command-line tool for automated file-review evidence management -in regulated environments. It computes cryptographic fingerprints of defined file-sets, -queries a review evidence store for corresponding review records, and produces compliance -documents on each CI/CD run. - -This design document describes the internal architecture, subsystems, and software units -that together implement the ReviewMark tool. It is intended to support development, -review, and maintenance activities. +The purpose of this document is to describe the internal design of each software unit that +comprises ReviewMark. It captures data models, algorithms, key methods, and inter-unit +interactions at a level of detail sufficient for formal code review, compliance verification, +and future maintenance. The document does not restate requirements; it explains how they are +realized. ## Scope -This design document covers: +This document covers the detailed design of the following software units: -- The software system decomposition into subsystems and software units -- The responsibilities and interfaces of each software unit -- The algorithms and data flows used for fingerprinting, evidence lookup, and document generation -- The self-validation framework +- **Program** — entry point and execution orchestrator (`Program.cs`) +- **Context** — command-line argument parser and I/O owner (`Cli/Context.cs`) +- **ReviewMarkConfiguration** — YAML configuration parser and review-set processor (`Configuration/ReviewMarkConfiguration.cs`) +- **GlobMatcher** — file pattern matching using glob syntax (`Configuration/GlobMatcher.cs`) +- **ReviewIndex** — review evidence loader and query engine (`Indexing/ReviewIndex.cs`) +- **PathHelpers** — file path utilities (`Indexing/PathHelpers.cs`) +- **Validation** — self-validation test runner (`SelfTest/Validation.cs`) -This document does not cover: +The following topics are out of scope: -- External CI/CD pipeline configuration -- Evidence store setup or administration -- Requirements traceability (see the Requirements Specification) +- External library internals (YamlDotNet, PDFsharp, DemaConsulting.TestResults) +- Build pipeline configuration +- Deployment and packaging -## Software Architecture +## Software Structure -The following diagram shows the decomposition of the ReviewMark software system into -subsystems and software units. +The following tree shows how the ReviewMark software items are organized across the system, +subsystem, and unit levels: ```text -ReviewMark (Software System) -├── CLI Subsystem -│ ├── Program (Software Unit) -│ └── Context (Software Unit) -├── Configuration Subsystem -│ ├── ReviewMarkConfiguration (Software Unit) -│ └── GlobMatcher (Software Unit) -├── Index Subsystem -│ ├── ReviewIndex (Software Unit) -│ └── PathHelpers (Software Unit) -└── Validation (Software Unit) +ReviewMark (System) +├── Program (Unit) +├── Cli (Subsystem) +│ └── Context (Unit) +├── Configuration (Subsystem) +│ ├── ReviewMarkConfiguration (Unit) +│ └── GlobMatcher (Unit) +├── Indexing (Subsystem) +│ ├── ReviewIndex (Unit) +│ └── PathHelpers (Unit) +└── SelfTest (Subsystem) + └── Validation (Unit) ``` -## Audience +Each unit is described in detail in its own chapter within this document. + +## Folder Layout + +The source code folder structure mirrors the top-level subsystem breakdown above, giving +reviewers an explicit navigation aid from design to code: + +```text +src/DemaConsulting.ReviewMark/ +├── Program.cs — entry point and execution orchestrator +├── Cli/ +│ └── Context.cs — command-line argument parser and I/O owner +├── Configuration/ +│ ├── ReviewMarkConfiguration.cs — YAML configuration parser and review-set processor +│ └── GlobMatcher.cs — file pattern matching using glob syntax +├── Indexing/ +│ ├── ReviewIndex.cs — review evidence loader and query engine +│ └── PathHelpers.cs — file path utilities +└── SelfTest/ + └── Validation.cs — self-validation test runner +``` + +The test project mirrors the same layout under `test/DemaConsulting.ReviewMark.Tests/`. + +## Document Conventions + +Throughout this document: + +- Class names, method names, property names, and file names appear in `monospace` font. +- The word **shall** denotes a design constraint that the implementation must satisfy. +- Section headings within each unit chapter follow a consistent structure: overview, data model, + methods/algorithms, and interactions with other units. +- Text tables are used in preference to diagrams, which may not render in all PDF viewers. + +## References -This document is intended for: +- [ReviewMark Architecture][arch] +- [ReviewMark User Guide][guide] +- [ReviewMark Repository][repo] -- Software developers working on ReviewMark -- Quality assurance teams performing design verification -- Project stakeholders reviewing architectural decisions +[arch]: ../../THEORY-OF-OPERATIONS.md +[guide]: ../../README.md +[repo]: https://github.com/demaconsulting/ReviewMark diff --git a/docs/design/validation.md b/docs/design/self-test/validation.md similarity index 100% rename from docs/design/validation.md rename to docs/design/self-test/validation.md diff --git a/docs/reqstream/subsystem-cli.yaml b/docs/reqstream/cli/subsystem-cli.yaml similarity index 100% rename from docs/reqstream/subsystem-cli.yaml rename to docs/reqstream/cli/subsystem-cli.yaml diff --git a/docs/reqstream/unit-context.yaml b/docs/reqstream/cli/unit-context.yaml similarity index 100% rename from docs/reqstream/unit-context.yaml rename to docs/reqstream/cli/unit-context.yaml diff --git a/docs/reqstream/subsystem-configuration.yaml b/docs/reqstream/configuration/subsystem-configuration.yaml similarity index 100% rename from docs/reqstream/subsystem-configuration.yaml rename to docs/reqstream/configuration/subsystem-configuration.yaml diff --git a/docs/reqstream/unit-glob-matcher.yaml b/docs/reqstream/configuration/unit-glob-matcher.yaml similarity index 100% rename from docs/reqstream/unit-glob-matcher.yaml rename to docs/reqstream/configuration/unit-glob-matcher.yaml diff --git a/docs/reqstream/indexing/subsystem-indexing.yaml b/docs/reqstream/indexing/subsystem-indexing.yaml new file mode 100644 index 0000000..31a2072 --- /dev/null +++ b/docs/reqstream/indexing/subsystem-indexing.yaml @@ -0,0 +1,57 @@ +--- +# Indexing Subsystem Requirements +# +# PURPOSE: +# - Define requirements for the ReviewMark Indexing subsystem +# - The Indexing subsystem spans ReviewIndex.cs (evidence loading) and PathHelpers.cs (path utilities) +# - Subsystem requirements describe the externally visible evidence-loading behavior + +sections: + - title: Indexing Subsystem Requirements + requirements: + - id: ReviewMark-Indexing-LoadEvidence + title: >- + The tool shall load review evidence from a configured EvidenceSource + supporting none, fileshare, and url types. + justification: | + The Indexing subsystem must support multiple evidence source types to accommodate + different deployment environments. The 'none' type allows the tool to operate + during initial project setup without an evidence store. The 'fileshare' type + supports loading from a local or network file path. The 'url' type supports + downloading evidence over HTTP(S), enabling centralized evidence stores accessible + from any CI/CD environment. + tests: + - ReviewIndex_Load_EvidenceSource_None_ReturnsEmptyIndex + - ReviewIndex_Load_EvidenceSource_None_HttpClientOverload_ReturnsEmptyIndex + - ReviewIndex_Load_EvidenceSource_Fileshare_LoadsFromFile + - ReviewIndex_Load_EvidenceSource_Fileshare_ValidJson_ReturnsPopulatedIndex + - ReviewIndex_Load_EvidenceSource_Url_SuccessResponse_LoadsIndex + + - id: ReviewMark-Indexing-ScanPdfEvidence + title: The tool shall scan PDF evidence files and extract embedded review metadata to build an index. + justification: | + Review evidence is stored as PDF files with metadata embedded in the Keywords + field. The Indexing subsystem must be able to scan directories for PDF files + and extract the review ID, fingerprint, date, and result from each file to + populate the evidence index used for report generation. + tests: + - ReviewIndex_Scan_PdfWithValidMetadata_PopulatesIndex + - ReviewIndex_Scan_MultiplePdfs_PopulatesAllEntries + - ReviewIndex_Scan_NoMatchingFiles_LeavesIndexEmpty + - ReviewIndex_Scan_ClearsExistingEntries + + - id: ReviewMark-Indexing-SafePathCombine + title: The tool shall combine file paths safely, rejecting path traversal sequences. + justification: | + Path traversal sequences (such as '..') in file paths could allow access to + files outside the intended directory. The Indexing subsystem must reject such + sequences to prevent unintended file system access in both evidence scanning + and index file operations. + tests: + - PathHelpers_SafePathCombine_ValidPaths_CombinesCorrectly + - PathHelpers_SafePathCombine_PathTraversalWithDoubleDots_ThrowsArgumentException + - PathHelpers_SafePathCombine_DoubleDotsInMiddle_ThrowsArgumentException + - PathHelpers_SafePathCombine_AbsolutePath_ThrowsArgumentException + - PathHelpers_SafePathCombine_CurrentDirectoryReference_CombinesCorrectly + - PathHelpers_SafePathCombine_NestedPaths_CombinesCorrectly + - PathHelpers_SafePathCombine_EmptyRelativePath_ReturnsBasePath diff --git a/docs/reqstream/unit-path-helpers.yaml b/docs/reqstream/indexing/unit-path-helpers.yaml similarity index 100% rename from docs/reqstream/unit-path-helpers.yaml rename to docs/reqstream/indexing/unit-path-helpers.yaml diff --git a/docs/reqstream/unit-review-index.yaml b/docs/reqstream/indexing/unit-review-index.yaml similarity index 100% rename from docs/reqstream/unit-review-index.yaml rename to docs/reqstream/indexing/unit-review-index.yaml diff --git a/docs/reqstream/ots-buildmark.yaml b/docs/reqstream/ots/ots-buildmark.yaml similarity index 100% rename from docs/reqstream/ots-buildmark.yaml rename to docs/reqstream/ots/ots-buildmark.yaml diff --git a/docs/reqstream/ots-mstest.yaml b/docs/reqstream/ots/ots-mstest.yaml similarity index 100% rename from docs/reqstream/ots-mstest.yaml rename to docs/reqstream/ots/ots-mstest.yaml diff --git a/docs/reqstream/ots-reqstream.yaml b/docs/reqstream/ots/ots-reqstream.yaml similarity index 100% rename from docs/reqstream/ots-reqstream.yaml rename to docs/reqstream/ots/ots-reqstream.yaml diff --git a/docs/reqstream/ots-sarifmark.yaml b/docs/reqstream/ots/ots-sarifmark.yaml similarity index 100% rename from docs/reqstream/ots-sarifmark.yaml rename to docs/reqstream/ots/ots-sarifmark.yaml diff --git a/docs/reqstream/ots-sonarmark.yaml b/docs/reqstream/ots/ots-sonarmark.yaml similarity index 100% rename from docs/reqstream/ots-sonarmark.yaml rename to docs/reqstream/ots/ots-sonarmark.yaml diff --git a/docs/reqstream/ots-versionmark.yaml b/docs/reqstream/ots/ots-versionmark.yaml similarity index 100% rename from docs/reqstream/ots-versionmark.yaml rename to docs/reqstream/ots/ots-versionmark.yaml diff --git a/docs/reqstream/self-test/subsystem-self-test.yaml b/docs/reqstream/self-test/subsystem-self-test.yaml new file mode 100644 index 0000000..f4e1bfa --- /dev/null +++ b/docs/reqstream/self-test/subsystem-self-test.yaml @@ -0,0 +1,38 @@ +--- +# SelfTest Subsystem Requirements +# +# PURPOSE: +# - Define requirements for the ReviewMark SelfTest subsystem +# - The SelfTest subsystem spans Validation.cs (self-validation test execution) +# - Subsystem requirements describe the self-validation mechanism for tool qualification +# in regulated environments + +sections: + - title: SelfTest Subsystem Requirements + requirements: + - id: ReviewMark-SelfTest-Qualification + title: The tool shall provide a self-validation mechanism to qualify the tool in its deployment environment. + justification: | + In regulated environments, tool qualification evidence is required to demonstrate + that the tool functions correctly in its deployment environment before it is used + to generate compliance artifacts. The SelfTest subsystem provides a built-in + self-validation suite that exercises core behaviors and produces a pass/fail + summary, enabling quality assurance teams to obtain tool qualification evidence + without requiring a separate test harness. + tests: + - Validation_Run_NullContext_ThrowsArgumentNullException + - Validation_Run_WritesValidationHeader + - Validation_Run_WritesSummaryWithTotalTests + - Validation_Run_AllTestsPass_ExitCodeIsZero + + - id: ReviewMark-SelfTest-ResultsOutput + title: The tool shall write self-validation results to a standard test result file when --results is provided. + justification: | + CI/CD pipelines and requirements traceability tools (such as ReqStream) consume + test result files in standard formats. By supporting both TRX (MSTest) and JUnit + XML output, the SelfTest subsystem enables self-validation results to be fed + directly into pipeline tooling and traceability reports without additional + conversion steps, satisfying audit trail requirements. + tests: + - Validation_Run_WithTrxResultsFile_WritesFile + - Validation_Run_WithXmlResultsFile_WritesFile diff --git a/docs/reqstream/unit-validation.yaml b/docs/reqstream/self-test/unit-validation.yaml similarity index 100% rename from docs/reqstream/unit-validation.yaml rename to docs/reqstream/self-test/unit-validation.yaml diff --git a/requirements.yaml b/requirements.yaml index 341b6db..133b3ed 100644 --- a/requirements.yaml +++ b/requirements.yaml @@ -25,18 +25,20 @@ --- includes: - docs/reqstream/reviewmark-system.yaml - - docs/reqstream/subsystem-cli.yaml - - docs/reqstream/subsystem-configuration.yaml - - docs/reqstream/unit-context.yaml - - docs/reqstream/unit-program.yaml - - docs/reqstream/unit-review-index.yaml - - docs/reqstream/unit-glob-matcher.yaml - - docs/reqstream/unit-path-helpers.yaml - - docs/reqstream/unit-validation.yaml - docs/reqstream/platform-requirements.yaml - - docs/reqstream/ots-mstest.yaml - - docs/reqstream/ots-reqstream.yaml - - docs/reqstream/ots-buildmark.yaml - - docs/reqstream/ots-versionmark.yaml - - docs/reqstream/ots-sarifmark.yaml - - docs/reqstream/ots-sonarmark.yaml + - docs/reqstream/unit-program.yaml + - docs/reqstream/cli/subsystem-cli.yaml + - docs/reqstream/cli/unit-context.yaml + - docs/reqstream/configuration/subsystem-configuration.yaml + - docs/reqstream/configuration/unit-glob-matcher.yaml + - docs/reqstream/indexing/subsystem-indexing.yaml + - docs/reqstream/indexing/unit-review-index.yaml + - docs/reqstream/indexing/unit-path-helpers.yaml + - docs/reqstream/self-test/subsystem-self-test.yaml + - docs/reqstream/self-test/unit-validation.yaml + - docs/reqstream/ots/ots-mstest.yaml + - docs/reqstream/ots/ots-reqstream.yaml + - docs/reqstream/ots/ots-buildmark.yaml + - docs/reqstream/ots/ots-versionmark.yaml + - docs/reqstream/ots/ots-sarifmark.yaml + - docs/reqstream/ots/ots-sonarmark.yaml diff --git a/src/DemaConsulting.ReviewMark/Context.cs b/src/DemaConsulting.ReviewMark/Cli/Context.cs similarity index 99% rename from src/DemaConsulting.ReviewMark/Context.cs rename to src/DemaConsulting.ReviewMark/Cli/Context.cs index e2315e7..c329ee6 100644 --- a/src/DemaConsulting.ReviewMark/Context.cs +++ b/src/DemaConsulting.ReviewMark/Cli/Context.cs @@ -18,7 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -namespace DemaConsulting.ReviewMark; +namespace DemaConsulting.ReviewMark.Cli; /// /// Context class that handles command-line arguments and program output. diff --git a/src/DemaConsulting.ReviewMark/GlobMatcher.cs b/src/DemaConsulting.ReviewMark/Configuration/GlobMatcher.cs similarity index 98% rename from src/DemaConsulting.ReviewMark/GlobMatcher.cs rename to src/DemaConsulting.ReviewMark/Configuration/GlobMatcher.cs index 45d1863..04ba230 100644 --- a/src/DemaConsulting.ReviewMark/GlobMatcher.cs +++ b/src/DemaConsulting.ReviewMark/Configuration/GlobMatcher.cs @@ -20,7 +20,7 @@ using Microsoft.Extensions.FileSystemGlobbing; -namespace DemaConsulting.ReviewMark; +namespace DemaConsulting.ReviewMark.Configuration; /// /// Provides glob-based file matching utilities. diff --git a/src/DemaConsulting.ReviewMark/ReviewMarkConfiguration.cs b/src/DemaConsulting.ReviewMark/Configuration/ReviewMarkConfiguration.cs similarity index 99% rename from src/DemaConsulting.ReviewMark/ReviewMarkConfiguration.cs rename to src/DemaConsulting.ReviewMark/Configuration/ReviewMarkConfiguration.cs index 726a1e5..f481599 100644 --- a/src/DemaConsulting.ReviewMark/ReviewMarkConfiguration.cs +++ b/src/DemaConsulting.ReviewMark/Configuration/ReviewMarkConfiguration.cs @@ -20,11 +20,12 @@ using System.Security.Cryptography; using System.Text; +using DemaConsulting.ReviewMark.Indexing; using YamlDotNet.Core; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; -namespace DemaConsulting.ReviewMark; +namespace DemaConsulting.ReviewMark.Configuration; // --------------------------------------------------------------------------- // Internal YAML deserialization models diff --git a/src/DemaConsulting.ReviewMark/PathHelpers.cs b/src/DemaConsulting.ReviewMark/Indexing/PathHelpers.cs similarity index 98% rename from src/DemaConsulting.ReviewMark/PathHelpers.cs rename to src/DemaConsulting.ReviewMark/Indexing/PathHelpers.cs index dffa821..22cc3d8 100644 --- a/src/DemaConsulting.ReviewMark/PathHelpers.cs +++ b/src/DemaConsulting.ReviewMark/Indexing/PathHelpers.cs @@ -18,7 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -namespace DemaConsulting.ReviewMark; +namespace DemaConsulting.ReviewMark.Indexing; /// /// Helper utilities for safe path operations. diff --git a/src/DemaConsulting.ReviewMark/ReviewIndex.cs b/src/DemaConsulting.ReviewMark/Indexing/ReviewIndex.cs similarity index 99% rename from src/DemaConsulting.ReviewMark/ReviewIndex.cs rename to src/DemaConsulting.ReviewMark/Indexing/ReviewIndex.cs index 347b8f6..a492eed 100644 --- a/src/DemaConsulting.ReviewMark/ReviewIndex.cs +++ b/src/DemaConsulting.ReviewMark/Indexing/ReviewIndex.cs @@ -23,10 +23,11 @@ using System.Text; using System.Text.Json; using System.Text.Json.Serialization; +using DemaConsulting.ReviewMark.Configuration; using PdfSharp.Pdf; using PdfSharp.Pdf.IO; -namespace DemaConsulting.ReviewMark; +namespace DemaConsulting.ReviewMark.Indexing; // --------------------------------------------------------------------------- // Internal JSON deserialization models diff --git a/src/DemaConsulting.ReviewMark/Program.cs b/src/DemaConsulting.ReviewMark/Program.cs index 061adf5..5759f32 100644 --- a/src/DemaConsulting.ReviewMark/Program.cs +++ b/src/DemaConsulting.ReviewMark/Program.cs @@ -19,6 +19,10 @@ // SOFTWARE. using System.Reflection; +using DemaConsulting.ReviewMark.Cli; +using DemaConsulting.ReviewMark.Configuration; +using DemaConsulting.ReviewMark.Indexing; +using DemaConsulting.ReviewMark.SelfTest; namespace DemaConsulting.ReviewMark; diff --git a/src/DemaConsulting.ReviewMark/Validation.cs b/src/DemaConsulting.ReviewMark/SelfTest/Validation.cs similarity index 99% rename from src/DemaConsulting.ReviewMark/Validation.cs rename to src/DemaConsulting.ReviewMark/SelfTest/Validation.cs index d7e0568..6fd8197 100644 --- a/src/DemaConsulting.ReviewMark/Validation.cs +++ b/src/DemaConsulting.ReviewMark/SelfTest/Validation.cs @@ -20,9 +20,11 @@ using System.Runtime.InteropServices; using System.Text.RegularExpressions; +using DemaConsulting.ReviewMark.Cli; +using DemaConsulting.ReviewMark.Indexing; using DemaConsulting.TestResults.IO; -namespace DemaConsulting.ReviewMark; +namespace DemaConsulting.ReviewMark.SelfTest; /// /// Provides self-validation functionality for ReviewMark. diff --git a/test/DemaConsulting.ReviewMark.Tests/ContextTests.cs b/test/DemaConsulting.ReviewMark.Tests/Cli/ContextTests.cs similarity index 99% rename from test/DemaConsulting.ReviewMark.Tests/ContextTests.cs rename to test/DemaConsulting.ReviewMark.Tests/Cli/ContextTests.cs index e360756..1cb256a 100644 --- a/test/DemaConsulting.ReviewMark.Tests/ContextTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/Cli/ContextTests.cs @@ -18,7 +18,9 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -namespace DemaConsulting.ReviewMark.Tests; +using DemaConsulting.ReviewMark.Cli; + +namespace DemaConsulting.ReviewMark.Tests.Cli; /// /// Unit tests for the Context class. diff --git a/test/DemaConsulting.ReviewMark.Tests/GlobMatcherTests.cs b/test/DemaConsulting.ReviewMark.Tests/Configuration/GlobMatcherTests.cs similarity index 98% rename from test/DemaConsulting.ReviewMark.Tests/GlobMatcherTests.cs rename to test/DemaConsulting.ReviewMark.Tests/Configuration/GlobMatcherTests.cs index c9dc0b0..4407af9 100644 --- a/test/DemaConsulting.ReviewMark.Tests/GlobMatcherTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/Configuration/GlobMatcherTests.cs @@ -18,7 +18,10 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -namespace DemaConsulting.ReviewMark.Tests; +using DemaConsulting.ReviewMark.Configuration; +using DemaConsulting.ReviewMark.Indexing; + +namespace DemaConsulting.ReviewMark.Tests.Configuration; /// /// Unit tests for the class. diff --git a/test/DemaConsulting.ReviewMark.Tests/ReviewMarkConfigurationTests.cs b/test/DemaConsulting.ReviewMark.Tests/Configuration/ReviewMarkConfigurationTests.cs similarity index 99% rename from test/DemaConsulting.ReviewMark.Tests/ReviewMarkConfigurationTests.cs rename to test/DemaConsulting.ReviewMark.Tests/Configuration/ReviewMarkConfigurationTests.cs index 1e2095b..5d01a12 100644 --- a/test/DemaConsulting.ReviewMark.Tests/ReviewMarkConfigurationTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/Configuration/ReviewMarkConfigurationTests.cs @@ -18,7 +18,10 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -namespace DemaConsulting.ReviewMark.Tests; +using DemaConsulting.ReviewMark.Configuration; +using DemaConsulting.ReviewMark.Indexing; + +namespace DemaConsulting.ReviewMark.Tests.Configuration; /// /// Unit tests for , , diff --git a/test/DemaConsulting.ReviewMark.Tests/IndexTests.cs b/test/DemaConsulting.ReviewMark.Tests/Indexing/IndexTests.cs similarity index 99% rename from test/DemaConsulting.ReviewMark.Tests/IndexTests.cs rename to test/DemaConsulting.ReviewMark.Tests/Indexing/IndexTests.cs index e287413..0223c94 100644 --- a/test/DemaConsulting.ReviewMark.Tests/IndexTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/Indexing/IndexTests.cs @@ -20,9 +20,11 @@ using System.Net.Http; using System.Text; +using DemaConsulting.ReviewMark.Configuration; +using DemaConsulting.ReviewMark.Indexing; using PdfSharp.Pdf; -namespace DemaConsulting.ReviewMark.Tests; +namespace DemaConsulting.ReviewMark.Tests.Indexing; /// /// Unit tests for the class and record. diff --git a/test/DemaConsulting.ReviewMark.Tests/PathHelpersTests.cs b/test/DemaConsulting.ReviewMark.Tests/Indexing/PathHelpersTests.cs similarity index 98% rename from test/DemaConsulting.ReviewMark.Tests/PathHelpersTests.cs rename to test/DemaConsulting.ReviewMark.Tests/Indexing/PathHelpersTests.cs index e399e24..f20f1ee 100644 --- a/test/DemaConsulting.ReviewMark.Tests/PathHelpersTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/Indexing/PathHelpersTests.cs @@ -18,7 +18,9 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -namespace DemaConsulting.ReviewMark.Tests; +using DemaConsulting.ReviewMark.Indexing; + +namespace DemaConsulting.ReviewMark.Tests.Indexing; /// /// Tests for the PathHelpers class. diff --git a/test/DemaConsulting.ReviewMark.Tests/IntegrationTests.cs b/test/DemaConsulting.ReviewMark.Tests/IntegrationTests.cs index f164ce2..0863757 100644 --- a/test/DemaConsulting.ReviewMark.Tests/IntegrationTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/IntegrationTests.cs @@ -18,6 +18,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +using DemaConsulting.ReviewMark.Indexing; + namespace DemaConsulting.ReviewMark.Tests; /// diff --git a/test/DemaConsulting.ReviewMark.Tests/ProgramTests.cs b/test/DemaConsulting.ReviewMark.Tests/ProgramTests.cs index ee250bc..6a2e0da 100644 --- a/test/DemaConsulting.ReviewMark.Tests/ProgramTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/ProgramTests.cs @@ -18,6 +18,9 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +using DemaConsulting.ReviewMark.Cli; +using DemaConsulting.ReviewMark.Indexing; + namespace DemaConsulting.ReviewMark.Tests; /// diff --git a/test/DemaConsulting.ReviewMark.Tests/ValidationTests.cs b/test/DemaConsulting.ReviewMark.Tests/SelfTest/ValidationTests.cs similarity index 98% rename from test/DemaConsulting.ReviewMark.Tests/ValidationTests.cs rename to test/DemaConsulting.ReviewMark.Tests/SelfTest/ValidationTests.cs index d7d9e03..1949dd6 100644 --- a/test/DemaConsulting.ReviewMark.Tests/ValidationTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/SelfTest/ValidationTests.cs @@ -18,7 +18,10 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -namespace DemaConsulting.ReviewMark.Tests; +using DemaConsulting.ReviewMark.Cli; +using DemaConsulting.ReviewMark.SelfTest; + +namespace DemaConsulting.ReviewMark.Tests.SelfTest; /// /// Unit tests for the class. diff --git a/test/DemaConsulting.ReviewMark.Tests/TestDirectory.cs b/test/DemaConsulting.ReviewMark.Tests/TestDirectory.cs index b94ab9e..b5e5da1 100644 --- a/test/DemaConsulting.ReviewMark.Tests/TestDirectory.cs +++ b/test/DemaConsulting.ReviewMark.Tests/TestDirectory.cs @@ -18,6 +18,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +using DemaConsulting.ReviewMark.Indexing; + namespace DemaConsulting.ReviewMark.Tests; /// From 539dfcfe279e5c684d1d29a95afd55541c13ce85 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 31 Mar 2026 13:36:30 -0400 Subject: [PATCH 12/19] Formal review of all 13 review-sets; apply findings (#38) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial plan * fix: address code review findings from formal review of all review-sets Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/78f46014-06de-48bf-85c7-0edbb7983736 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * fix: correct status coverage gaps in user guide and system requirements Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/3a13bcb6-e2e3-4c73-9f48-566e4c5c8f96 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Update docs/design/cli/context.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix: apply PR review suggestions — rephrase location doc and expand enforce justification Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/5adf8c75-f585-4404-b78d-e5314af3906c Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Co-authored-by: Malcolm Nixon Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/design/cli/context.md | 4 +-- docs/design/program.md | 21 ++++++++++----- docs/reqstream/reviewmark-system.yaml | 11 +++++--- docs/user_guide/introduction.md | 5 ++-- .../Cli/ContextTests.cs | 26 +++++++++---------- 5 files changed, 39 insertions(+), 28 deletions(-) diff --git a/docs/design/cli/context.md b/docs/design/cli/context.md index eed5e1e..50827f0 100644 --- a/docs/design/cli/context.md +++ b/docs/design/cli/context.md @@ -26,9 +26,9 @@ arguments: | `ReportFile` | string? | Output path for the Review Report document | | `ReportDepth` | int | Heading depth for the Review Report | | `IndexPaths` | string[]? | Paths to scan when building an evidence index | -| `WorkingDirectory` | string | Base directory for resolving relative paths | +| `WorkingDirectory` | string? | Base directory for resolving relative paths | | `Enforce` | bool | Fail if any review-set is not Current | -| `Elaborate` | bool | Expand file lists in generated documents | +| `ElaborateId` | string? | Review-set ID to elaborate, or null if `--elaborate` was not specified | ## Argument Parsing diff --git a/docs/design/program.md b/docs/design/program.md index 6d260cc..e0e93cd 100644 --- a/docs/design/program.md +++ b/docs/design/program.md @@ -31,17 +31,24 @@ than by `Program.Main` explicitly returning a non-zero value. executing the first matching action and returning: 1. If `--version` — print version and return -2. If `--help` — print banner and return -3. If `--validate` — run self-validation and return -4. If `--lint` — run configuration lint and return -5. If `--index` paths provided — scan and write evidence index, then return -6. Otherwise — generate Review Plan and/or Review Report and return +2. Print application banner +3. If `--help` — print help and return +4. If `--validate` — run self-validation and return +5. If `--lint` — run configuration lint and return +6. Otherwise — run main tool logic (index scanning and/or Review Plan/Report/Elaborate) +The application banner (step 2) is always printed unless `--version` is specified. Only one top-level action is performed per invocation. Actions later in the priority order are not reached if an earlier flag is set. ## PrintBanner() -`Program.PrintBanner(Context)` writes the help text to the console via -`Context.WriteLine()`. The banner lists all supported flags and arguments with brief +`Program.PrintBanner(Context)` writes the application name, version, and copyright +notice to the console via `Context.WriteLine()`. The banner is printed for every +invocation except `--version`. + +## PrintHelp() + +`Program.PrintHelp(Context)` writes usage information to the console via +`Context.WriteLine()`. The help text lists all supported flags and arguments with brief descriptions. diff --git a/docs/reqstream/reviewmark-system.yaml b/docs/reqstream/reviewmark-system.yaml index 5e51e6e..9e890ed 100644 --- a/docs/reqstream/reviewmark-system.yaml +++ b/docs/reqstream/reviewmark-system.yaml @@ -25,8 +25,8 @@ sections: justification: | Auditors need evidence that the review evidence for each review-set is current — that the reviewed files have not changed since the review was conducted. The Review - Report provides this evidence automatically, showing Current, Stale, or Missing - status for each review-set. + Report provides this evidence automatically, showing Current, Stale, Missing, or + Failed status for each review-set. tests: - ReviewMark_ReviewReportGeneration @@ -34,8 +34,11 @@ sections: title: The tool shall return a non-zero exit code when enforcement is enabled and any review-set is not current. justification: | CI/CD pipelines must be able to gate releases on review coverage. The --enforce flag - enables this by causing the tool to exit with a non-zero code when any review-set has - Stale or Missing status, making incomplete review coverage a build-breaking condition. + enables this by causing the tool to exit with a non-zero code in two situations: when + the Review Plan shows that files matching needs-review are not covered by any + review-set, or when the Review Report shows that any review-set has Stale, Missing, or + Failed status. This makes incomplete file coverage, out-of-date reviews, and failed + reviews all build-breaking conditions. tests: - ReviewMark_Enforce diff --git a/docs/user_guide/introduction.md b/docs/user_guide/introduction.md index de6ca85..be0ecd5 100644 --- a/docs/user_guide/introduction.md +++ b/docs/user_guide/introduction.md @@ -152,8 +152,9 @@ Lint checks the following: - **File readability** — the definition file exists and can be read. - **YAML syntax** — the file is valid YAML; syntax errors include the filename and line number. -- **`evidence-source` block** — the block is present, has a `type` field (`url` or `fileshare`), - and has a `location` field. +- **`evidence-source` block** — the block is present and has a `type` field (`none`, `url`, or + `fileshare`); when `type` is `url` or `fileshare`, it must also include a `location` field + (no `location` field is used with `type: none`). - **Review sets** — each set has an `id`, a `title`, and at least one `paths` entry. - **Duplicate IDs** — no two review sets share the same `id`. diff --git a/test/DemaConsulting.ReviewMark.Tests/Cli/ContextTests.cs b/test/DemaConsulting.ReviewMark.Tests/Cli/ContextTests.cs index 1cb256a..2c94bd3 100644 --- a/test/DemaConsulting.ReviewMark.Tests/Cli/ContextTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/Cli/ContextTests.cs @@ -200,7 +200,7 @@ public void Context_Create_LogFlag_OpensLogFile() public void Context_Create_UnknownArgument_ThrowsArgumentException() { // Act & Assert - var exception = Assert.Throws(() => Context.Create(["--unknown"])); + var exception = Assert.ThrowsExactly(() => Context.Create(["--unknown"])); Assert.Contains("Unsupported argument", exception.Message); } @@ -211,7 +211,7 @@ public void Context_Create_UnknownArgument_ThrowsArgumentException() public void Context_Create_LogFlag_WithoutValue_ThrowsArgumentException() { // Act & Assert - var exception = Assert.Throws(() => Context.Create(["--log"])); + var exception = Assert.ThrowsExactly(() => Context.Create(["--log"])); Assert.Contains("--log", exception.Message); } @@ -222,7 +222,7 @@ public void Context_Create_LogFlag_WithoutValue_ThrowsArgumentException() public void Context_Create_ResultsFlag_WithoutValue_ThrowsArgumentException() { // Act & Assert - var exception = Assert.Throws(() => Context.Create(["--results"])); + var exception = Assert.ThrowsExactly(() => Context.Create(["--results"])); Assert.Contains("--results", exception.Message); } @@ -412,7 +412,7 @@ public void Context_Create_DefinitionFlag_SetsDefinitionFile() public void Context_Create_DefinitionFlag_WithoutValue_ThrowsArgumentException() { // Act & Assert - --definition with no following value should throw and include the flag name in the message - var exception = Assert.Throws(() => Context.Create(["--definition"])); + var exception = Assert.ThrowsExactly(() => Context.Create(["--definition"])); Assert.Contains("--definition", exception.Message); } @@ -452,7 +452,7 @@ public void Context_Create_PlanDepthFlag_SetsPlanDepth() public void Context_Create_PlanDepthFlag_WithInvalidValue_ThrowsArgumentException() { // Act & Assert - --plan-depth with a non-numeric value should throw - Assert.Throws(() => Context.Create(["--plan-depth", "not-a-number"])); + Assert.ThrowsExactly(() => Context.Create(["--plan-depth", "not-a-number"])); } /// @@ -463,7 +463,7 @@ public void Context_Create_PlanDepthFlag_WithInvalidValue_ThrowsArgumentExceptio public void Context_Create_PlanDepthFlag_WithZeroValue_ThrowsArgumentException() { // Act & Assert - --plan-depth requires a positive integer; zero is not valid - Assert.Throws(() => Context.Create(["--plan-depth", "0"])); + Assert.ThrowsExactly(() => Context.Create(["--plan-depth", "0"])); } /// @@ -501,7 +501,7 @@ public void Context_Create_ReportDepthFlag_SetsReportDepth() public void Context_Create_ReportDepthFlag_NonNumeric_ThrowsArgumentException() { // Act & Assert - creating a context with a non-numeric report depth should fail validation - Assert.Throws(() => Context.Create(["--report-depth", "abc"])); + Assert.ThrowsExactly(() => Context.Create(["--report-depth", "abc"])); } /// @@ -511,7 +511,7 @@ public void Context_Create_ReportDepthFlag_NonNumeric_ThrowsArgumentException() public void Context_Create_ReportDepthFlag_Zero_ThrowsArgumentException() { // Act & Assert - creating a context with a report depth of 0 should fail validation - Assert.Throws(() => Context.Create(["--report-depth", "0"])); + Assert.ThrowsExactly(() => Context.Create(["--report-depth", "0"])); } /// @@ -521,7 +521,7 @@ public void Context_Create_ReportDepthFlag_Zero_ThrowsArgumentException() public void Context_Create_ReportDepthFlag_MissingValue_ThrowsArgumentException() { // Act & Assert - creating a context with --report-depth but no value should fail validation - Assert.Throws(() => Context.Create(["--report-depth"])); + Assert.ThrowsExactly(() => Context.Create(["--report-depth"])); } /// @@ -628,7 +628,7 @@ public void Context_Create_NoArguments_EnforceFalse() public void Context_Create_PlanDepthFlag_WithValueGreaterThanFive_ThrowsArgumentException() { // Act & Assert - --plan-depth cannot exceed 5 (max heading depth supported) - Assert.Throws(() => Context.Create(["--plan-depth", "6"])); + Assert.ThrowsExactly(() => Context.Create(["--plan-depth", "6"])); } /// @@ -638,7 +638,7 @@ public void Context_Create_PlanDepthFlag_WithValueGreaterThanFive_ThrowsArgument public void Context_Create_ReportDepthFlag_WithValueGreaterThanFive_ThrowsArgumentException() { // Act & Assert - --report-depth cannot exceed 5 (max heading depth supported) - Assert.Throws(() => Context.Create(["--report-depth", "6"])); + Assert.ThrowsExactly(() => Context.Create(["--report-depth", "6"])); } /// @@ -675,7 +675,7 @@ public void Context_Create_NoArguments_WorkingDirectoryIsNull() public void Context_Create_DirFlag_MissingValue_ThrowsArgumentException() { // Act & Assert - --dir without a path value should throw - Assert.Throws(() => Context.Create(["--dir"])); + Assert.ThrowsExactly(() => Context.Create(["--dir"])); } /// @@ -712,7 +712,7 @@ public void Context_Create_NoArguments_ElaborateIdIsNull() public void Context_Create_ElaborateFlag_WithoutValue_ThrowsArgumentException() { // Act & Assert - --elaborate without an ID argument should throw - Assert.Throws(() => Context.Create(["--elaborate"])); + Assert.ThrowsExactly(() => Context.Create(["--elaborate"])); } /// From f40bd3b2709acee1bb7bf43f0f187db6431dfcab Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 2 Apr 2026 09:52:44 -0400 Subject: [PATCH 13/19] Sync AGENTS.md, agent files, and standards from TemplateDotNetTool (#39) - Updated AGENTS.md with latest template content - Updated .github/agents/*.md (code-review, developer, implementation, quality, repo-consistency) - Updated .github/standards/*.md (csharp-testing, reqstream-usage, reviewmark-usage, technical-documentation) - Added .github/standards/design-documentation.md (new in template) Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --- .github/agents/code-review.agent.md | 6 +- .github/agents/developer.agent.md | 4 +- .github/agents/implementation.agent.md | 12 +- .github/agents/quality.agent.md | 162 ++++++++-------- .github/agents/repo-consistency.agent.md | 4 +- .github/standards/csharp-testing.md | 4 +- .github/standards/design-documentation.md | 142 ++++++++++++++ .github/standards/reqstream-usage.md | 73 +++++--- .github/standards/reviewmark-usage.md | 81 ++++---- .github/standards/technical-documentation.md | 15 +- AGENTS.md | 184 +++++++++---------- 11 files changed, 422 insertions(+), 265 deletions(-) create mode 100644 .github/standards/design-documentation.md diff --git a/.github/agents/code-review.agent.md b/.github/agents/code-review.agent.md index f28a9b7..cee797f 100644 --- a/.github/agents/code-review.agent.md +++ b/.github/agents/code-review.agent.md @@ -17,7 +17,7 @@ Formal reviews are a quality enforcement mechanism, and as such MUST be performe to get the checklist to fill in 2. Use `dotnet reviewmark --elaborate [review-set]` to get the files to review 3. Review the files all together -4. Populate the checklist with the findings to `.agent-logs/reviews/review-report-[review-set].md` of the project. +4. Populate the checklist with the findings to `.agent-logs/reviews/review-report-{review-set}.md` of the project. # Don't Do These Things @@ -31,13 +31,13 @@ Formal reviews are a quality enforcement mechanism, and as such MUST be performe # Reporting -Upon completion create a summary in `.agent-logs/[agent-name]-[subject]-[unique-id].md` +Upon completion create a summary in `.agent-logs/{agent-name}-{subject}-{unique-id}.md` of the project consisting of: ```markdown # Code Review Report -**Result**: +**Result**: (SUCCEEDED|FAILED) ## Review Summary diff --git a/.github/agents/developer.agent.md b/.github/agents/developer.agent.md index 955f9e9..2028f79 100644 --- a/.github/agents/developer.agent.md +++ b/.github/agents/developer.agent.md @@ -20,13 +20,13 @@ Perform software development tasks by determining and applying appropriate DEMA # Reporting -Upon completion create a summary in `.agent-logs/[agent-name]-[subject]-[unique-id].md` +Upon completion create a summary in `.agent-logs/{agent-name}-{subject}-{unique-id}.md` of the project consisting of: ```markdown # Developer Agent Report -**Result**: +**Result**: (SUCCEEDED|FAILED) ## Work Summary diff --git a/.github/agents/implementation.agent.md b/.github/agents/implementation.agent.md index 767c66d..91b44d7 100644 --- a/.github/agents/implementation.agent.md +++ b/.github/agents/implementation.agent.md @@ -26,7 +26,7 @@ counting how many retries have occurred. ## RESEARCH State (start) -Call the built-in @explore sub-agent with: +Call the built-in explore sub-agent with: - **context**: the user's request and any current quality findings - **goal**: analyze the implementation state and develop a plan to implement the request @@ -35,7 +35,7 @@ Once the explore sub-agent finishes, transition to the DEVELOPMENT state. ## DEVELOPMENT State -Call the @developer sub-agent with: +Call the developer sub-agent with: - **context** the user's request and the current implementation plan - **goal** implement the user's request and any identified quality fixes @@ -47,7 +47,7 @@ Once the developer sub-agent finishes: ## QUALITY State -Call the @quality sub-agent with: +Call the quality sub-agent with: - **context** the user's request and the current implementation report - **goal** check the quality of the work performed for any issues @@ -60,14 +60,14 @@ Once the quality sub-agent finishes: ### REPORT State (end) -Upon completion create a summary in `.agent-logs/[agent-name]-[subject]-[unique-id].md` +Upon completion create a summary in `.agent-logs/{agent-name}-{subject}-{unique-id}.md` of the project consisting of: ```markdown # Implementation Orchestration Report -**Result**: -**Final State**: +**Result**: (SUCCEEDED|FAILED) +**Final State**: (RESEARCH|DEVELOPMENT|QUALITY|REPORT) **Retry Count**: ## State Machine Execution diff --git a/.github/agents/quality.agent.md b/.github/agents/quality.agent.md index 4dd6902..a7b57d4 100644 --- a/.github/agents/quality.agent.md +++ b/.github/agents/quality.agent.md @@ -13,86 +13,24 @@ DEMA Consulting standards and Continuous Compliance practices. # Standards-Based Quality Assessment -This assessment is a quality control system of the project and MUST be performed. +This assessment is a quality control system of the project and MUST be performed systematically. 1. **Analyze completed work** to identify scope and changes made 2. **Read relevant standards** from `.github/standards/` as defined in AGENTS.md based on work performed -3. **Execute comprehensive quality checks** across all compliance areas - EVERY checkbox item must be evaluated +3. **Execute comprehensive quality assessment** using the structured evaluation criteria in the reporting template 4. **Validate tool compliance** using ReqStream, ReviewMark, and language tools 5. **Generate quality assessment report** with findings and recommendations -## Requirements Compliance - -- [ ] Were requirements updated to reflect functional changes? -- [ ] Were new requirements created for new features? -- [ ] Do requirement IDs follow semantic naming standards? -- [ ] Were source filters applied appropriately for platform-specific requirements? -- [ ] Does ReqStream enforcement pass without errors? -- [ ] Is requirements traceability maintained to tests? - -## Design Documentation Compliance - -- [ ] Were design documents updated for architectural changes? -- [ ] Were new design artifacts created for new components? -- [ ] Are design decisions documented with rationale? -- [ ] Is system/subsystem/unit categorization maintained? -- [ ] Is design-to-implementation traceability preserved? - -## Code Quality Compliance - -- [ ] Are language-specific standards followed (from applicable standards files)? -- [ ] Are quality checks from standards files satisfied? -- [ ] Is code properly categorized (system/subsystem/unit/OTS)? -- [ ] Is appropriate separation of concerns maintained? -- [ ] Was language-specific tooling executed and passing? - -## Testing Compliance - -- [ ] Were tests created/updated for all functional changes? -- [ ] Is test coverage maintained for all requirements? -- [ ] Are testing standards followed (AAA pattern, etc.)? -- [ ] Does test categorization align with code structure? -- [ ] Do all tests pass without failures? - -## Review Management Compliance - -- [ ] Were review-sets updated to include new/modified files? -- [ ] Do file patterns follow include-then-exclude approach? -- [ ] Is review scope appropriate for change magnitude? -- [ ] Was ReviewMark tooling executed and passing? -- [ ] Were review artifacts generated correctly? - -## Documentation Compliance - -- [ ] Was README.md updated for user-facing changes? -- [ ] Were user guides updated for feature changes? -- [ ] Does API documentation reflect code changes? -- [ ] Was compliance documentation generated? -- [ ] Does documentation follow standards formatting? -- [ ] Is documentation organized under `docs/` following standard folder structure? -- [ ] Do Pandoc collections include proper `introduction.md` files with Purpose and Scope sections? -- [ ] Are auto-generated markdown files left unmodified? -- [ ] Do README.md files use absolute URLs and include concrete examples? -- [ ] Is documentation integrated into ReviewMark review-sets for formal review? - -## Process Compliance - -- [ ] Was Continuous Compliance workflow followed? -- [ ] Did all quality gates execute successfully? -- [ ] Were appropriate tools used for validation? -- [ ] Were standards consistently applied across work? -- [ ] Was compliance evidence generated and preserved? - # Reporting -Upon completion create a summary in `.agent-logs/[agent-name]-[subject]-[unique-id].md` +Upon completion create a summary in `.agent-logs/{agent-name}-{subject}-{unique-id}.md` of the project consisting of: ```markdown # Quality Assessment Report -**Result**: -**Overall Grade**: +**Result**: (SUCCEEDED|FAILED) +**Overall Grade**: (PASS|FAIL|NEEDS_WORK) ## Assessment Summary @@ -100,26 +38,84 @@ of the project consisting of: - **Standards Applied**: [Standards files used for assessment] - **Categories Evaluated**: [Quality check categories assessed] -## Quality Check Results - -- **Requirements Compliance**: - [Summary] -- **Design Documentation**: - [Summary] -- **Code Quality**: - [Summary] -- **Testing Compliance**: - [Summary] -- **Review Management**: - [Summary] -- **Documentation**: - [Summary] -- **Process Compliance**: - [Summary] - -## Findings - -- **Issues Found**: [List of compliance issues] -- **Recommendations**: [Suggested improvements] +## Requirements Compliance: (PASS|FAIL|N/A) + +- Were requirements updated to reflect functional changes? (PASS|FAIL|N/A) - [Evidence] +- Were new requirements created for new features? (PASS|FAIL|N/A) - [Evidence] +- Do requirement IDs follow semantic naming standards? (PASS|FAIL|N/A) - [Evidence] +- Do requirement files follow kebab-case naming convention? (PASS|FAIL|N/A) - [Evidence] +- Are requirement files organized under `docs/reqstream/` with proper folder structure? (PASS|FAIL|N/A) - [Evidence] +- Are OTS requirements properly placed in `docs/reqstream/ots/` subfolder? (PASS|FAIL|N/A) - [Evidence] +- Were source filters applied appropriately for platform-specific requirements? (PASS|FAIL|N/A) - [Evidence] +- Does ReqStream enforcement pass without errors? (PASS|FAIL|N/A) - [Evidence] +- Is requirements traceability maintained to tests? (PASS|FAIL|N/A) - [Evidence] + +## Design Documentation Compliance: (PASS|FAIL|N/A) + +- Were design documents updated for architectural changes? (PASS|FAIL|N/A) - [Evidence] +- Were new design artifacts created for new components? (PASS|FAIL|N/A) - [Evidence] +- Do design folder names use kebab-case convention matching source structure? (PASS|FAIL|N/A) - [Evidence] +- Are design files properly named ({subsystem-name}.md, {unit-name}.md patterns)? (PASS|FAIL|N/A) - [Evidence] +- Is `docs/design/introduction.md` present with required Software Structure section? (PASS|FAIL|N/A) - [Evidence] +- Are design decisions documented with rationale? (PASS|FAIL|N/A) - [Evidence] +- Is system/subsystem/unit categorization maintained? (PASS|FAIL|N/A) - [Evidence] +- Is design-to-implementation traceability preserved? (PASS|FAIL|N/A) - [Evidence] + +## Code Quality Compliance: (PASS|FAIL|N/A) + +- Are language-specific standards followed (from applicable standards files)? (PASS|FAIL|N/A) - [Evidence] +- Are quality checks from standards files satisfied? (PASS|FAIL|N/A) - [Evidence] +- Is code properly categorized (system/subsystem/unit/OTS)? (PASS|FAIL|N/A) - [Evidence] +- Is appropriate separation of concerns maintained? (PASS|FAIL|N/A) - [Evidence] +- Was language-specific tooling executed and passing? (PASS|FAIL|N/A) - [Evidence] + +## Testing Compliance: (PASS|FAIL|N/A) + +- Were tests created/updated for all functional changes? (PASS|FAIL|N/A) - [Evidence] +- Is test coverage maintained for all requirements? (PASS|FAIL|N/A) - [Evidence] +- Are testing standards followed (AAA pattern, etc.)? (PASS|FAIL|N/A) - [Evidence] +- Does test categorization align with code structure? (PASS|FAIL|N/A) - [Evidence] +- Do all tests pass without failures? (PASS|FAIL|N/A) - [Evidence] + +## Review Management Compliance: (PASS|FAIL|N/A) + +- Were review-sets updated to include new/modified files? (PASS|FAIL|N/A) - [Evidence] +- Do file patterns follow include-then-exclude approach? (PASS|FAIL|N/A) - [Evidence] +- Is review scope appropriate for change magnitude? (PASS|FAIL|N/A) - [Evidence] +- Was ReviewMark tooling executed and passing? (PASS|FAIL|N/A) - [Evidence] +- Were review artifacts generated correctly? (PASS|FAIL|N/A) - [Evidence] + +## Documentation Compliance: (PASS|FAIL|N/A) + +- Was README.md updated for user-facing changes? (PASS|FAIL|N/A) - [Evidence] +- Were user guides updated for feature changes? (PASS|FAIL|N/A) - [Evidence] +- Does API documentation reflect code changes? (PASS|FAIL|N/A) - [Evidence] +- Was compliance documentation generated? (PASS|FAIL|N/A) - [Evidence] +- Does documentation follow standards formatting? (PASS|FAIL|N/A) - [Evidence] +- Is documentation organized under `docs/` following standard folder structure? (PASS|FAIL|N/A) - [Evidence] +- Do Pandoc collections include proper `introduction.md` with Purpose and Scope sections? (PASS|FAIL|N/A) - [Evidence] +- Are auto-generated markdown files left unmodified? (PASS|FAIL|N/A) - [Evidence] +- Do README.md files use absolute URLs and include concrete examples? (PASS|FAIL|N/A) - [Evidence] +- Is documentation integrated into ReviewMark review-sets for formal review? (PASS|FAIL|N/A) - [Evidence] + +## Process Compliance: (PASS|FAIL|N/A) + +- Was Continuous Compliance workflow followed? (PASS|FAIL|N/A) - [Evidence] +- Did all quality gates execute successfully? (PASS|FAIL|N/A) - [Evidence] +- Were appropriate tools used for validation? (PASS|FAIL|N/A) - [Evidence] +- Were standards consistently applied across work? (PASS|FAIL|N/A) - [Evidence] +- Was compliance evidence generated and preserved? (PASS|FAIL|N/A) - [Evidence] + +## Overall Findings + +- **Critical Issues**: [Count and description of critical findings] +- **Recommendations**: [Suggested improvements and next steps] - **Tools Executed**: [Quality tools used for validation] ## Compliance Status -- **Standards Adherence**: [Overall compliance rating] -- **Quality Gates**: [Status of automated quality checks] +- **Standards Adherence**: [Overall compliance rating with specific standards] +- **Quality Gates**: [Status of automated quality checks with tool outputs] ``` Return this summary to the caller. diff --git a/.github/agents/repo-consistency.agent.md b/.github/agents/repo-consistency.agent.md index dfaf702..b0f93d2 100644 --- a/.github/agents/repo-consistency.agent.md +++ b/.github/agents/repo-consistency.agent.md @@ -42,13 +42,13 @@ benefit from template evolution while respecting project-specific customizations # Reporting -Upon completion create a summary in `.agent-logs/[agent-name]-[subject]-[unique-id].md` +Upon completion create a summary in `.agent-logs/{agent-name}-{subject}-{unique-id}.md` of the project consisting of: ```markdown # Repo Consistency Report -**Result**: +**Result**: (SUCCEEDED|FAILED) ## Consistency Analysis diff --git a/.github/standards/csharp-testing.md b/.github/standards/csharp-testing.md index 6cee284..2f26520 100644 --- a/.github/standards/csharp-testing.md +++ b/.github/standards/csharp-testing.md @@ -56,7 +56,7 @@ reliable evidence. - **Verify Interactions**: Assert that expected method calls occurred with correct parameters - **Predictable Behavior**: Set up mocks to return known values for consistent test results -# MSTest V4 Antipatterns +# MSTest V4 Anti-patterns Avoid these common MSTest V4 patterns because they produce poor error messages or cause tests to be silently ignored. @@ -116,4 +116,4 @@ Before submitting C# tests, verify: - [ ] External dependencies mocked with NSubstitute or equivalent - [ ] Tests linked to requirements with source filters where needed - [ ] Test results generate TRX format for ReqStream compatibility -- [ ] MSTest V4 antipatterns avoided (proper assertions, public visibility, etc.) +- [ ] MSTest V4 anti-patterns avoided (proper assertions, public visibility, etc.) diff --git a/.github/standards/design-documentation.md b/.github/standards/design-documentation.md new file mode 100644 index 0000000..6312275 --- /dev/null +++ b/.github/standards/design-documentation.md @@ -0,0 +1,142 @@ +# Design Documentation Standards + +This document defines DEMA Consulting standards for design documentation +within Continuous Compliance environments, extending the general technical +documentation standards with specific requirements for software design +artifacts. + +# Core Principles + +Design documentation serves as the bridge between requirements and +implementation, providing detailed technical specifications that enable: + +- **Formal Code Review**: Reviewers can verify implementation matches design +- **Compliance Evidence**: Auditors can trace requirements through design to code +- **Maintenance Support**: Developers can understand system structure and interactions +- **Quality Assurance**: Testing teams can validate against detailed specifications + +# Required Structure and Documents + +Design documentation must be organized under `docs/design/` with folder structure +mirroring source code organization because reviewers need clear navigation from +design to implementation: + +```text +docs/design/ +├── introduction.md # Design overview with software structure +├── system.md # System-level design documentation +├── {subsystem-name}/ # Subsystem design documents (kebab-case folder names) +│ ├── {subsystem-name}.md # Subsystem overview and design +│ └── {unit-name}.md # Unit-level design documents +└── {unit-name}.md # Top-level unit design documents (if not in subsystem) +``` + +## introduction.md (MANDATORY) + +The `introduction.md` file serves as the design entry point and MUST include +these sections because auditors need clear scope boundaries and architectural +overview: + +### Purpose Section + +Clear statement of the design document's purpose, audience, and regulatory +or compliance drivers. + +### Scope Section + +Define what software items are covered and what is explicitly excluded. +Specify version boundaries and applicability constraints. + +### Software Structure Section (MANDATORY) + +Include a text-based tree diagram showing the software organization across +System, Subsystem, and Unit levels. Agents MUST read `software-items.md` +to understand these classifications before creating this section. + +Example format: + +```text +ProjectName (System) +├── ComponentA (Subsystem) +│ ├── ClassX (Unit) +│ └── ClassY (Unit) +├── ComponentB (Subsystem) +│ └── ClassZ (Unit) +└── UtilityClass (Unit) +``` + +### Folder Layout Section (MANDATORY) + +Include a text-based tree diagram showing how the source code folders +mirror the software structure, with file paths and brief descriptions. + +Example format: + +```text +src/ProjectName/ +├── ComponentA/ +│ ├── ClassX.cs — brief description +│ └── ClassY.cs — brief description +├── ComponentB/ +│ └── ClassZ.cs — brief description +└── UtilityClass.cs — brief description +``` + +## system.md (MANDATORY) + +The `system.md` file contains system-level design documentation including: + +- System architecture and major components +- External interfaces and dependencies +- Data flow and control flow +- System-wide design constraints and decisions +- Integration patterns and communication protocols + +## Subsystem and Unit Design Documents + +For each subsystem identified in the software structure: + +- Create a kebab-case folder matching the subsystem name (enables automated tooling) +- Include `{subsystem-name}.md` with subsystem overview and design +- Include unit design documents for complex units within the subsystem + +For significant units requiring detailed design: + +- Document data models, algorithms, and key methods +- Describe interactions with other units +- Include sufficient detail for formal code review +- Place in appropriate subsystem folder or at design root level + +# Software Items Integration (CRITICAL) + +Before creating design documentation, agents MUST: + +1. **Read `.github/standards/software-items.md`** to understand System/Subsystem/Unit classifications +2. **Apply proper categorization** when creating software structure diagrams +3. **Ensure consistency** between software structure and folder layout +4. **Validate mapping** from design categories to source code organization + +# Writing Guidelines + +Design documentation must be technical and specific because it serves as the +implementation specification for formal code review: + +- **Implementation Detail**: Provide sufficient detail for code review and implementation +- **Architectural Clarity**: Clearly define component boundaries and interfaces +- **Traceability**: Link to requirements where applicable using ReqStream patterns +- **Concrete Examples**: Use actual class names, method signatures, and data structures +- **Current Information**: Keep synchronized with code changes and refactoring + +# Quality Checks + +Before submitting design documentation, verify: + +- [ ] `introduction.md` includes both Software Structure and Folder Layout sections +- [ ] Software structure correctly categorizes items as System/Subsystem/Unit per `software-items.md` +- [ ] Folder layout matches actual source code organization +- [ ] `system.md` provides comprehensive system-level design +- [ ] Subsystem documentation folders use kebab-case names while mirroring source subsystem names and structure +- [ ] Design documents contain sufficient implementation detail +- [ ] All documents follow technical documentation formatting standards +- [ ] Content is current with implementation and requirements +- [ ] Documents are integrated into ReviewMark review-sets for formal review diff --git a/.github/standards/reqstream-usage.md b/.github/standards/reqstream-usage.md index 3f99929..aa75a1f 100644 --- a/.github/standards/reqstream-usage.md +++ b/.github/standards/reqstream-usage.md @@ -15,28 +15,54 @@ generation: coverage - **Audit Documentation**: Generated reports provide compliance evidence +# Software Items Integration (CRITICAL) + +Before creating requirements files, agents MUST: + +1. **Read `.github/standards/software-items.md`** to understand System/Subsystem/Unit/OTS classifications +2. **Apply proper categorization** when organizing requirements files +3. **Mirror source code structure** in requirements folder organization + # Requirements Organization -Organize requirements into separate files under `docs/reqstream/` for -independent review: +Organize requirements into separate files under `docs/reqstream/` mirroring +the source code structure because reviewers need clear navigation from +requirements to design to implementation: ```text -requirements.yaml # Root file (includes only) +requirements.yaml # Root file (includes only) docs/reqstream/ - {project}-system.yaml # System-level requirements - platform-requirements.yaml # Platform support requirements - subsystem-{subsystem}.yaml # Subsystem requirements - unit-{unit}.yaml # Unit (class) requirements - ots-{component}.yaml # OTS software item requirements +├── system.yaml # System-level requirements +├── platform-requirements.yaml # Platform support requirements +├── {subsystem-name}/ # Subsystem requirements (kebab-case folders) +│ └── {subsystem-name}.yaml # Requirements for this subsystem +├── {unit-name}.yaml # Unit requirements (for top-level units) +└── ots/ # OTS software item requirements + └── {ots-name}.yaml # Requirements for OTS components ``` +The folder structure MUST mirror the source code organization to maintain +consistency with design documentation and enable automated tooling. + +# Requirement Hierarchies and Links + +When linking requirements between different software item levels, links MUST +only flow downward in the hierarchy to maintain clear traceability: + +- **System requirements** → may link to subsystem or unit requirements +- **Subsystem requirements** → may link to unit requirements within that subsystem +- **Unit requirements** → should NOT link to higher-level requirements + +This prevents circular dependencies and ensures clear hierarchical relationships +for compliance auditing. + # Requirements File Format ```yaml sections: - title: Functional Requirements requirements: - - id: Project-Component-Feature + - id: Project-Subsystem-Feature title: The system shall perform the required function. justification: | Business rationale explaining why this requirement exists. @@ -46,9 +72,15 @@ sections: - windows@PlatformSpecificTest # Source filter for platform evidence ``` +Requirements specify WHAT the system shall do, not HOW, because implementation +details belong in design documentation while requirements focus on externally +observable behavior with clear, testable acceptance criteria. + # OTS Software Requirements -Document third-party component requirements with specific section structure: +Document third-party component requirements in the `docs/reqstream/ots/` folder +with nested sections because auditors need clear separation between in-house +and external component evidence: ```yaml sections: @@ -64,25 +96,12 @@ sections: # Semantic IDs (MANDATORY) -Use meaningful IDs following `Project-Section-ShortDesc` pattern: +Use meaningful IDs following `Project-Section-ShortDesc` pattern because +auditors need to understand requirements without cross-referencing: - **Good**: `TemplateTool-Core-DisplayHelp` - **Bad**: `REQ-042` (requires lookup to understand) -# Requirement Best Practices - -Requirements specify WHAT the system shall do, not HOW: - -- Focus on externally observable characteristics and behavior -- Avoid implementation details, design constraints, or technology choices -- Each requirement must have clear, testable acceptance criteria - -Include business rationale for each requirement: - -- Business need or regulatory requirement -- Risk mitigation or quality improvement -- Standard or regulation references - # Source Filter Requirements (CRITICAL) Platform-specific requirements MUST use source filters for compliance evidence: @@ -140,7 +159,9 @@ Before submitting requirements, verify: - [ ] Platform-specific requirements use source filters (`platform@TestName`) - [ ] Requirements specify observable behavior (WHAT), not implementation (HOW) - [ ] Comprehensive justification explains business/regulatory need -- [ ] Files organized under `docs/reqstream/` following naming patterns +- [ ] Files organized under `docs/reqstream/` following folder structure patterns +- [ ] Subsystem folders use kebab-case naming matching source code +- [ ] OTS requirements placed in `ots/` subfolder - [ ] Valid YAML syntax passes yamllint validation - [ ] ReqStream enforcement passes: `dotnet reqstream --enforce` - [ ] Test result formats compatible (TRX, JUnit XML) diff --git a/.github/standards/reviewmark-usage.md b/.github/standards/reviewmark-usage.md index bdabd1d..a40179f 100644 --- a/.github/standards/reviewmark-usage.md +++ b/.github/standards/reviewmark-usage.md @@ -20,9 +20,10 @@ Configure reviews in `.reviewmark.yaml` at repository root: # Patterns identifying all files that require review needs-review: # Include core development artifacts + - "requirements.yaml" # Root requirements file + - "docs/reqstream/**/*.yaml" # Requirements files + - "docs/design/**/*.md" # Design documentation - "**/*.cs" # All C# source and test files - - "**/*.md" # Requirements and design documentation - - "docs/reqstream/**/*.yaml" # Requirements files only # Exclude build output and generated content - "!**/obj/**" # Exclude build output @@ -38,10 +39,10 @@ reviews: - id: MyProduct-PasswordValidator title: Password Validator Unit Review paths: - - "src/Auth/PasswordValidator.cs" - - "docs/reqstream/auth-passwordvalidator-class.yaml" - - "test/Auth/PasswordValidatorTests.cs" - - "docs/design/password-validation.md" + - "docs/reqstream/authentication/password-validator.yaml" + - "docs/design/authentication/password-validator.md" + - "src/{ProjectName}/Authentication/PasswordValidator.cs" + - "test/{ProjectName}.Tests/Authentication/PasswordValidatorTests.cs" - id: MyProduct-AllRequirements title: All Requirements Review @@ -59,7 +60,9 @@ and consistent review processes: Reviews system integration and operational validation: -- **Files**: System-level requirements, design introduction, system design documents, integration tests +- **Files**: System requirements (`docs/reqstream/system.yaml`), design introduction + (`docs/design/introduction.md`), system design (`docs/design/system.md`), + integration tests - **Purpose**: Validates system operates as designed and meets overall requirements - **Example**: `TemplateTool-System` @@ -67,7 +70,7 @@ Reviews system integration and operational validation: Reviews architectural and design consistency: -- **Files**: System-level requirements, platform requirements, all design documents +- **Files**: System requirements, platform requirements, all design documents under `docs/design/` - **Purpose**: Ensures design completeness and architectural coherence - **Example**: `MyProduct-Design` @@ -75,7 +78,7 @@ Reviews architectural and design consistency: Reviews requirements quality and traceability: -- **Files**: All requirement files including root `requirements.yaml` +- **Files**: All requirement files including root `requirements.yaml` and all files under `docs/reqstream/` - **Purpose**: Validates requirements structure, IDs, justifications, and test linkage - **Example**: `MyProduct-AllRequirements` @@ -85,6 +88,11 @@ Reviews individual software unit implementation: - **Files**: Unit requirements, design documents, source code, unit tests - **Purpose**: Validates unit meets requirements and is properly implemented +- **File Path Pattern**: + - Requirements: `docs/reqstream/{subsystem-name}/{unit-name}.yaml` or `docs/reqstream/{unit-name}.yaml` + - Design: `docs/design/{subsystem-name}/{unit-name}.md` or `docs/design/{unit-name}.md` + - Source: `src/{ProjectName}/{SubsystemName}/{UnitName}.cs` + - Tests: `test/{ProjectName}.Tests/{SubsystemName}/{UnitName}Tests.cs` - **Example**: `MyProduct-PasswordValidator`, `MyProduct-ConfigParser` ## [Product]-[Subsystem] Review @@ -93,48 +101,48 @@ Reviews subsystem architecture and interfaces: - **Files**: Subsystem requirements, design documents, integration tests (usually no source code) - **Purpose**: Validates subsystem behavior and interface compliance +- **File Path Pattern**: + - Requirements: `docs/reqstream/{subsystem-name}/{subsystem-name}.yaml` + - Design: `docs/design/{subsystem-name}/{subsystem-name}.md` + - Tests: `test/{ProjectName}.Tests/{SubsystemName}Integration/` or similar - **Example**: `MyProduct-Authentication`, `MyProduct-DataLayer` -# ReviewMark Commands - -Essential ReviewMark commands for Continuous Compliance: - -```bash -# Lint review configuration for issues (run before use) -dotnet reviewmark \ - --lint +## [Product]-OTS Review -# Generate review plan (shows coverage) -dotnet reviewmark \ - --plan docs/code_review_plan/plan.md - -# Generate review report (shows status) -dotnet reviewmark \ - --report docs/code_review_report/report.md +Reviews OTS (Off-The-Shelf) software integration: -# Enforce review compliance (use in CI/CD) -dotnet reviewmark \ - --plan docs/code_review_plan/plan.md \ - --report docs/code_review_report/report.md \ - --enforce -``` +- **Files**: OTS requirements and integration test evidence +- **Purpose**: Validates OTS components meet integration requirements +- **File Path Pattern**: + - Requirements: `docs/reqstream/ots/{ots-name}.yaml` + - Tests: Integration tests proving OTS functionality +- **Example**: `MyProduct-SystemTextJson`, `MyProduct-EntityFramework` # File Pattern Best Practices Use "include-then-exclude" approach for `needs-review` patterns because it ensures comprehensive coverage while removing unwanted files: -## Include-Then-Exclude Strategy - 1. **Start broad**: Include all files of potential interest with generous patterns 2. **Exclude overreach**: Use `!` patterns to remove build output, generated files, and temporary files 3. **Test patterns**: Verify patterns match intended files using `dotnet reviewmark --elaborate` -## Pattern Guidelines +**Order matters**: Patterns are processed sequentially, excludes override earlier includes. + +# ReviewMark Commands + +Essential ReviewMark commands for Continuous Compliance: + +```bash +# Lint review configuration for issues (run before use) +dotnet reviewmark --lint -- **Be generous with includes**: Better to include too much initially than miss important files -- **Be specific with excludes**: Target exact paths and patterns that should never be reviewed -- **Order matters**: Patterns are processed sequentially, excludes override earlier includes +# Generate review plan and report (use in CI/CD) +dotnet reviewmark \ + --plan docs/code_review_plan/plan.md \ + --report docs/code_review_report/report.md \ + --enforce +``` # Quality Checks @@ -144,6 +152,7 @@ Before submitting ReviewMark configuration, verify: - [ ] `needs-review` patterns cover requirements, design, code, and tests with proper exclusions - [ ] Each review-set has unique `id` and groups architecturally related files - [ ] File patterns use correct glob syntax and match intended files +- [ ] File paths reflect current naming conventions (kebab-case design/requirements folders, PascalCase source folders) - [ ] Evidence source properly configured (`none` for dev, `url` for production) - [ ] Environment variables used for credentials (never hardcoded) - [ ] ReviewMark enforcement configured: `dotnet reviewmark --enforce` diff --git a/.github/standards/technical-documentation.md b/.github/standards/technical-documentation.md index f09ee83..c117aa2 100644 --- a/.github/standards/technical-documentation.md +++ b/.github/standards/technical-documentation.md @@ -34,13 +34,18 @@ docs/ design/ # Design documentation introduction.md # Design overview system.md # System architecture - {component}.md # Component-specific designs + {subsystem-name}/ # Subsystem design folder + {subsystem-name}.md # Subsystem-specific designs + {unit-name}.md # Unit-specific designs + {unit-name}.md # Top-level unit design reqstream/ # Requirements source files - {project}-system.yaml # System requirements + system.yaml # System requirements platform-requirements.yaml # Platform requirements - subsystem-{name}.yaml # Subsystem requirements - unit-{name}.yaml # Unit requirements - ots-{name}.yaml # OTS requirements + {subsystem-name}/ # Subsystem requirements folder + {subsystem-name}.yaml # Subsystem requirements + {unit-name}.yaml # Unit requirements + ots/ # OTS requirement files + {ots-name}.yaml # OTS requirements requirements_doc/ # Auto-generated requirements reports requirements.md # Generated by ReqStream justifications.md # Generated by ReqStream diff --git a/AGENTS.md b/AGENTS.md index 04a9589..c884c2e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,6 @@ # Agent Quick Reference -Comprehensive guidance for AI agents working on ReviewMark - a tool for automated -file-review evidence management in regulated environments. +Comprehensive guidance for AI agents working on repositories following Continuous Compliance practices. ## Standards Application (ALL Agents Must Follow) @@ -9,6 +8,7 @@ Before performing any work, agents must read and apply the relevant standards fr - **`csharp-language.md`** - For C# code development (literate programming, XML docs, dependency injection) - **`csharp-testing.md`** - For C# test development (AAA pattern, naming, MSTest anti-patterns) +- **`design-documentation.md`** - For design documentation (software structure diagrams, system.md, subsystem organization) - **`reqstream-usage.md`** - For requirements management (traceability, semantic IDs, source filters) - **`reviewmark-usage.md`** - For file review management (review-sets, file patterns, enforcement) - **`software-items.md`** - For software categorization (system/subsystem/unit/OTS classification) @@ -22,12 +22,12 @@ quality checks and guidelines throughout your work. The default agent should handle simple, straightforward tasks directly. Delegate to specialized agents only for specific scenarios: -- **Light development work** (small fixes, simple features) → Call @developer agent -- **Light quality checking** (linting, basic validation) → Call @quality agent -- **Formal feature implementation** (complex, multi-step) → Call the `@implementation` agent -- **Formal bug resolution** (complex debugging, systematic fixes) → Call the `@implementation` agent -- **Formal reviews** (compliance verification, detailed analysis) → Call @code-review agent -- **Template consistency** (downstream repository alignment) → Call @repo-consistency agent +- **Light development work** (small fixes, simple features) → Call developer agent +- **Light quality checking** (linting, basic validation) → Call quality agent +- **Formal feature implementation** (complex, multi-step) → Call the `implementation` agent +- **Formal bug resolution** (complex debugging, systematic fixes) → Call the `implementation` agent +- **Formal reviews** (compliance verification, detailed analysis) → Call code-review agent +- **Template consistency** (downstream repository alignment) → Call repo-consistency agent ## Available Specialized Agents @@ -55,129 +55,113 @@ modification policies in header comments. 6. **Documentation Currency**: All docs current and generated 7. **File Review Status**: All reviewable files have current reviews -## Tech Stack +## Continuous Compliance Overview -- C# (latest), .NET 8.0/9.0/10.0, dotnet CLI, NuGet +This repository follows the DEMA Consulting Continuous Compliance + approach, which enforces quality and +compliance gates on every CI/CD run instead of as a last-mile activity. -## Key Files +### Core Principles -- **`requirements.yaml`** - All requirements with test linkage (enforced via `dotnet reqstream --enforce`) -- **`.editorconfig`** - Code style (file-scoped namespaces, 4-space indent, UTF-8, LF endings) -- **`.cspell.yaml`, `.markdownlint-cli2.yaml`, `.yamllint.yaml`** - Linting configs +- **Requirements Traceability**: Every requirement MUST link to passing tests +- **Quality Gates**: All quality checks must pass before merge +- **Documentation Currency**: All docs auto-generated and kept current +- **Automated Evidence**: Full audit trail generated with every build -### Spell check word list policy +## Required Compliance Tools -**Never** add a word to the `.cspell.yaml` word list in order to silence a spell-checking failure. -Doing so defeats the purpose of spell-checking and reduces the quality of the repository. +### Linting Tools (ALL Must Pass) -- If cspell flags a word that is **misspelled**, fix the spelling in the source file. -- If cspell flags a word that is a **genuine technical term** (tool name, project identifier, etc.) and is - spelled correctly, raise a **proposal** (e.g. comment in a pull request) explaining why the word - should be added. The proposal must be reviewed and approved before the word is added to the list. +- **markdownlint-cli2**: Markdown style and formatting enforcement +- **cspell**: Spell-checking across all text files (use `.cspell.yaml` for technical terms) +- **yamllint**: YAML structure and formatting validation +- **Language-specific linters**: Based on repository technology stack -## Requirements +### Quality Analysis -- All requirements MUST be linked to tests (prefer `ReviewMark_*` self-validation tests) -- Not all tests need to be linked to requirements (tests may exist for corner cases, design testing, failure-testing, etc.) -- Enforced in CI: `dotnet reqstream --requirements requirements.yaml --tests "test-results/**/*.trx" --enforce` -- When adding features: add requirement + link to test +- **SonarQube/SonarCloud**: Code quality and security analysis +- **CodeQL**: Security vulnerability scanning (produces SARIF output) +- **Static analyzers**: Microsoft.CodeAnalysis.NetAnalyzers, SonarAnalyzer.CSharp, etc. -## Test Source Filters +### Requirements & Compliance -Test links in `requirements.yaml` can include a source filter prefix to restrict which test results count as -evidence. This is critical for platform and framework requirements - **do not remove these filters**. +- **ReqStream**: Requirements traceability enforcement (`dotnet reqstream --enforce`) +- **ReviewMark**: File review status enforcement +- **BuildMark**: Tool version documentation +- **VersionMark**: Version tracking across CI/CD jobs -- `windows@TestName` - proves the test passed on a Windows platform -- `ubuntu@TestName` - proves the test passed on a Linux (Ubuntu) platform -- `macos@TestName` - proves the test passed on a macOS platform -- `net8.0@TestName` - proves the test passed under the .NET 8 target framework -- `net9.0@TestName` - proves the test passed under the .NET 9 target framework -- `net10.0@TestName` - proves the test passed under the .NET 10 target framework -- `dotnet8.x@TestName` - proves the self-validation test ran on a machine with .NET 8.x runtime -- `dotnet9.x@TestName` - proves the self-validation test ran on a machine with .NET 9.x runtime -- `dotnet10.x@TestName` - proves the self-validation test ran on a machine with .NET 10.x runtime +## Project Structure Template -Without the source filter, a test result from any platform/framework satisfies the requirement. Adding the filter -ensures the CI evidence comes specifically from the required environment. +- `docs/` - Documentation and compliance artifacts + - `design/` - Detailed design documents + - `introduction.md` - System/Subsystem/Unit breakdown for this repository + - `reqstream/` - Subsystem requirements YAML files (included by root requirements.yaml) + - Auto-generated reports (requirements, justifications, trace matrix) +- `src/{ProjectName}/` - Source code projects +- `test/{ProjectName}.Tests/` - Test projects +- `.github/workflows/` - CI/CD pipeline definitions (build.yaml, build_on_push.yaml, release.yaml) +- Configuration files: `.editorconfig`, `.clang-format`, `nuget.config`, `.reviewmark.yaml`, etc. -## Testing +## Key Configuration Files -- **Test Naming**: `ReviewMark_MethodUnderTest_Scenario` for self-validation tests -- **Self-Validation**: All tests run via `--validate` flag and can output TRX/JUnit format -- **Test Framework**: Uses DemaConsulting.TestResults library for test result generation +### Essential Files (Repository-Specific) -## Code Style +- **`lint.sh` / `lint.bat`** - Cross-platform comprehensive linting scripts +- **`.editorconfig`** - Code formatting rules +- **`.clang-format`** - C/C++ formatting (if applicable) +- **`.cspell.yaml`** - Spell-check configuration and technical term dictionary +- **`.markdownlint-cli2.yaml`** - Markdown linting rules +- **`.yamllint.yaml`** - YAML linting configuration +- **`nuget.config`** - NuGet package sources (if .NET) +- **`package.json`** - Node.js dependencies for linting tools -- **XML Docs**: On ALL members (public/internal/private) with spaces after `///` in summaries -- **Errors**: `ArgumentException` for parsing, `InvalidOperationException` for runtime issues -- **Namespace**: File-scoped namespaces only -- **Using Statements**: Top of file only (no nested using declarations except for IDisposable) -- **String Formatting**: Use interpolated strings ($"") for clarity +### Compliance Files -## Project Structure +- **`requirements.yaml`** - Root requirements file with includes +- **`.reviewmark.yaml`** - File review definitions and tracking +- CI/CD pipeline files with quality gate enforcement -- **Context.cs**: Handles command-line argument parsing, logging, and output -- **Program.cs**: Main entry point with version/help/validation routing -- **Validation.cs**: Self-validation tests with TRX/JUnit output support +## Continuous Compliance Workflow -## Build and Test +### CI/CD Pipeline Stages (Standard) -```bash -# Build the project -dotnet build --configuration Release +1. **Lint**: `./lint.sh` or `lint.bat` - comprehensive linting suite +2. **Build**: Compile with warnings as errors +3. **Analyze**: SonarQube/SonarCloud, CodeQL security scanning +4. **Test**: Execute all tests, generate coverage reports +5. **Validate**: Tool self-validation tests +6. **Document**: Generate requirements reports, trace matrix, build notes +7. **Enforce**: Requirements traceability, file review status +8. **Publish**: Generate final documentation (Pandoc → PDF) -# Run unit tests -dotnet test --configuration Release +### Quality Gate Enforcement -# Run self-validation -dotnet run --project src/DemaConsulting.ReviewMark \ - --configuration Release --framework net10.0 --no-build -- --validate +All stages must pass before merge. Pipeline fails immediately on: -# Use convenience scripts -./build.sh # Linux/macOS -build.bat # Windows -``` +- Any linting errors +- Build warnings or errors +- Security vulnerabilities (CodeQL) +- Requirements without test coverage +- Outdated file reviews +- Missing documentation -## Documentation +## Continuous Compliance Requirements -- **User Guide**: `docs/user_guide/introduction.md` -- **Requirements**: `requirements.yaml` -> auto-generated docs -- **Build Notes**: Auto-generated via BuildMark -- **Code Quality**: Auto-generated via CodeQL and SonarMark -- **Trace Matrix**: Auto-generated via ReqStream -- **CHANGELOG.md**: Not present - changes are captured in the auto-generated build notes +This repository follows continuous compliance practices from DEMA Consulting +Continuous Compliance . -## Markdown Link Style +### Core Requirements Traceability Rules -- **AI agent markdown files** (`.github/agents/*.md`): Use inline links `[text](url)` so URLs are visible in agent context -- **README.md**: Use absolute URLs (shipped in NuGet package) -- **All other markdown files**: Use reference-style links `[text][ref]` with `[ref]: url` at document end +- **ALL requirements MUST be linked to tests** - Enforced in CI via `dotnet reqstream --enforce` +- **NOT all tests need requirement links** - Tests may exist for corner cases, design validation, failure scenarios +- **Source filters are critical** - Platform/framework requirements need specific test evidence -## CI/CD - -- **Quality Checks**: Markdown lint, spell check, YAML lint -- **Build**: Multi-platform (Windows/Linux/macOS) -- **CodeQL**: Security scanning -- **Integration Tests**: .NET 8/9/10 on Windows/Linux/macOS -- **Documentation**: Auto-generated via Pandoc + Weasyprint - -## Common Tasks - -```bash -# Format code -dotnet format - -# Run all linters -./lint.sh # Linux/macOS -lint.bat # Windows - -# Pack as NuGet tool -dotnet pack --configuration Release -``` +For detailed requirements format, test linkage patterns, and ReqStream +integration, call the developer agent with requirements management context. ## Agent Report Files -Upon completion, create a report file at `.agent-logs/[agent-name]-[subject]-[unique-id].md` that includes: +Upon completion, create a report file at `.agent-logs/{agent-name}-{subject}-{unique-id}.md` that includes: - A concise summary of the work performed - Any important decisions made and their rationale From cb37c3d3fd778580a7f1aded996ec1e5bbde4d4a Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 2 Apr 2026 17:26:09 -0400 Subject: [PATCH 14/19] feat: unified Load mechanism with linting support and ReportIssues helper (#41) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial plan * feat: implement LoadWithLinting unified load+lint mechanism - Add LintSeverity enum, LintIssue record, ReviewMarkLoadResult record - Add ReviewMarkConfiguration.LoadWithLinting() as the unified mechanism - Refactor Load() and Lint() to delegate to LoadWithLinting() - Update Program.RunLintLogic and RunDefinitionLogic to use LoadWithLinting - Add 6 new tests (167 total passing) - Update design docs and requirements Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/72f2e0f3-5b69-43b1-bfe0-820655e63846 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * feat: rename LoadWithLinting→Load, remove old Load/Lint wrappers, add ReportIssues(Context) - Rename LoadWithLinting to Load (the one clean API for all callers) - Remove old exception-throwing Load and string-returning Lint wrappers - Add ReviewMarkLoadResult.ReportIssues(Context) helper — eliminates the duplicated foreach loops in RunLintLogic and RunDefinitionLogic - Update all tests: old exception/string-list tests migrated to result API, LoadWithLinting_* tests renamed to Load_*, new ReportIssues test added - Update design doc and requirements Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/6ea03d7a-aae2-4774-adad-ac3f8c98c0de Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * fix: update reqstream test names after Load/Lint rename The previous rename of test methods broke the reqstream traceability: - ReviewMarkConfiguration_Load_InvalidYaml_ErrorIncludesFilenameAndLine → ReviewMarkConfiguration_Load_InvalidYaml_ReturnsNullConfigWithErrorIssue - ReviewMarkConfiguration_Load_MissingEvidenceSource_ErrorIncludesFilename → ReviewMarkConfiguration_Load_MissingEvidenceSource_ReturnsNullConfigWithErrorIssue - ReviewMarkConfiguration_Lint_MultipleErrors_ReturnsAll → ReviewMarkConfiguration_Load_MultipleErrors_ReturnsAllIssues - ReviewMarkConfiguration_Lint_NoneEvidenceSource_NoErrors → ReviewMarkConfiguration_Load_NoneEvidenceSource_NoIssues Updated in subsystem-cli.yaml and unit-review-index.yaml to satisfy ReviewMark-Cmd-Lint and ReviewMark-EvidenceSource-None requirements. Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/ccf913ed-b0f1-4487-8964-1c5468d72010 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --- .../review-mark-configuration.md | 25 +- docs/reqstream/cli/subsystem-cli.yaml | 6 +- .../subsystem-configuration.yaml | 18 +- .../reqstream/indexing/unit-review-index.yaml | 2 +- .../Configuration/ReviewMarkConfiguration.cs | 243 +++++++++++------- src/DemaConsulting.ReviewMark/Program.cs | 28 +- .../ReviewMarkConfigurationTests.cs | 138 +++++++--- .../ProgramTests.cs | 37 +++ 8 files changed, 339 insertions(+), 158 deletions(-) diff --git a/docs/design/configuration/review-mark-configuration.md b/docs/design/configuration/review-mark-configuration.md index 9692eb1..b220af2 100644 --- a/docs/design/configuration/review-mark-configuration.md +++ b/docs/design/configuration/review-mark-configuration.md @@ -19,10 +19,18 @@ The `.reviewmark.yaml` file is deserialized into the following model: ## ReviewMarkConfiguration.Load() -`ReviewMarkConfiguration.Load(definitionFile, workingDirectory)` reads and -deserializes the YAML file, resolves all glob patterns relative to the working -directory, computes fingerprints for each review-set, loads the evidence index, -and returns a fully initialized configuration object ready for plan/report generation. +`ReviewMarkConfiguration.Load(filePath)` is the unified loading mechanism that performs +both configuration parsing and linting in a single pass. It returns a `ReviewMarkLoadResult` +containing: + +- `Configuration`: the loaded `ReviewMarkConfiguration`, or `null` if any error-level issues + were detected. +- `Issues`: a read-only list of `LintIssue` records, each with a `Location`, `Severity` + (`LintSeverity.Error` or `LintSeverity.Warning`), and `Description`. + +Errors result in a `null` configuration so callers can distinguish between a completely +invalid file and a file with only warnings. `LintIssue.ToString()` formats each issue as +`{location}: {severity}: {description}`, matching standard linting tool output conventions. ## Fingerprinting Algorithm @@ -66,10 +74,9 @@ index to establish whether a passing, failing, stale, or missing review result e ## Linting -`ReviewMarkConfiguration.Lint(Context)` validates the loaded configuration for -correctness. Lint checks include: +`ReviewMarkConfiguration.Load(filePath)` accumulates all detectable issues in a single pass +without stopping at the first error. Lint checks include: +- Missing or invalid `evidence-source` block and fields - All review-set `id` values are unique -- All glob patterns resolve to at least one file -- The `needs-review` file-set is non-empty -- All files in the `needs-review` set are covered by at least one review-set +- Each review-set has required `id`, `title`, and `paths` fields diff --git a/docs/reqstream/cli/subsystem-cli.yaml b/docs/reqstream/cli/subsystem-cli.yaml index 71e9bdc..ed013bf 100644 --- a/docs/reqstream/cli/subsystem-cli.yaml +++ b/docs/reqstream/cli/subsystem-cli.yaml @@ -237,7 +237,7 @@ sections: - Program_Run_WithLintFlag_CorruptedYaml_ReportsError - Program_Run_WithLintFlag_MissingEvidenceSource_ReportsError - Program_Run_WithLintFlag_MultipleErrors_ReportsAll - - ReviewMarkConfiguration_Load_InvalidYaml_ErrorIncludesFilenameAndLine - - ReviewMarkConfiguration_Load_MissingEvidenceSource_ErrorIncludesFilename - - ReviewMarkConfiguration_Lint_MultipleErrors_ReturnsAll + - ReviewMarkConfiguration_Load_InvalidYaml_ReturnsNullConfigWithErrorIssue + - ReviewMarkConfiguration_Load_MissingEvidenceSource_ReturnsNullConfigWithErrorIssue + - ReviewMarkConfiguration_Load_MultipleErrors_ReturnsAllIssues - ReviewMark_Lint diff --git a/docs/reqstream/configuration/subsystem-configuration.yaml b/docs/reqstream/configuration/subsystem-configuration.yaml index 4c3ad54..80c5b92 100644 --- a/docs/reqstream/configuration/subsystem-configuration.yaml +++ b/docs/reqstream/configuration/subsystem-configuration.yaml @@ -26,5 +26,21 @@ sections: - ReviewSet_GetFingerprint_SameContent_ReturnsSameFingerprint - ReviewSet_GetFingerprint_DifferentContent_ReturnsDifferentFingerprint - ReviewSet_GetFingerprint_RenameFile_ReturnsSameFingerprint - - ReviewMarkConfiguration_Load_NonExistentFile_ThrowsException + - ReviewMarkConfiguration_Load_NonExistentFile_ReturnsNullConfigWithErrorIssue - ReviewMarkConfiguration_Load_FileshareRelativeLocation_ResolvesToAbsolutePath + - id: ReviewMark-Config-Loading + title: The Load mechanism shall perform linting and return both the configuration and lint issues. + justification: | + Enables a single-pass loading mechanism that combines configuration parsing and linting, + returning a ReviewMarkLoadResult with both the configuration (or null on error) and + all detected LintIssue records. This allows callers to receive comprehensive diagnostics + without performing two separate operations. + tests: + - ReviewMarkConfiguration_Load_ValidFile_ReturnsConfigurationAndNoIssues + - ReviewMarkConfiguration_Load_NonExistentFile_ReturnsNullConfigWithErrorIssue + - ReviewMarkConfiguration_Load_InvalidYaml_ReturnsNullConfigWithErrorIssue + - ReviewMarkConfiguration_Load_MissingEvidenceSource_ReturnsNullConfigWithErrorIssue + - ReviewMarkConfiguration_Load_MultipleErrors_ReturnsAllIssues + - ReviewMarkConfiguration_Load_NoneEvidenceSource_NoIssues + - ReviewMarkLoadResult_ReportIssues_RoutesIssuesToContext + - Program_Run_WithDefinitionFlag_InvalidConfig_ReportsLintError diff --git a/docs/reqstream/indexing/unit-review-index.yaml b/docs/reqstream/indexing/unit-review-index.yaml index ad05a88..052c327 100644 --- a/docs/reqstream/indexing/unit-review-index.yaml +++ b/docs/reqstream/indexing/unit-review-index.yaml @@ -47,7 +47,7 @@ sections: - ReviewIndex_Load_EvidenceSource_None_HttpClientOverload_ReturnsEmptyIndex - ReviewMarkConfiguration_Parse_NoneEvidenceSource_ParsedCorrectly - ReviewMarkConfiguration_Parse_NoneEvidenceSource_NoLocationRequired - - ReviewMarkConfiguration_Lint_NoneEvidenceSource_NoErrors + - ReviewMarkConfiguration_Load_NoneEvidenceSource_NoIssues - id: ReviewMark-Index-PdfParsing title: The tool shall parse PDF metadata from the Keywords field when indexing evidence files. diff --git a/src/DemaConsulting.ReviewMark/Configuration/ReviewMarkConfiguration.cs b/src/DemaConsulting.ReviewMark/Configuration/ReviewMarkConfiguration.cs index f481599..9a1241d 100644 --- a/src/DemaConsulting.ReviewMark/Configuration/ReviewMarkConfiguration.cs +++ b/src/DemaConsulting.ReviewMark/Configuration/ReviewMarkConfiguration.cs @@ -20,6 +20,7 @@ using System.Security.Cryptography; using System.Text; +using DemaConsulting.ReviewMark.Cli; using DemaConsulting.ReviewMark.Indexing; using YamlDotNet.Core; using YamlDotNet.Serialization; @@ -406,6 +407,68 @@ internal sealed record ReviewReportResult(string Markdown, bool HasIssues); /// The generated Markdown content. internal sealed record ElaborateResult(string Markdown); +/// +/// Severity level of a lint issue. +/// +internal enum LintSeverity +{ + /// Informational warning — does not prevent configuration use. + Warning, + + /// Fatal error — prevents configuration use. + Error +} + +/// +/// A single lint issue detected when loading or validating a .reviewmark.yaml file. +/// +/// +/// The file path (and optionally :line:column) where the issue was detected. +/// +/// The severity of the issue. +/// A human-readable description of the issue. +internal sealed record LintIssue(string Location, LintSeverity Severity, string Description) +{ + /// + public override string ToString() => + $"{Location}: {Severity.ToString().ToLowerInvariant()}: {Description}"; +} + +/// +/// The result of . +/// +/// +/// The loaded configuration, or null if any error-level lint issues were detected. +/// +/// +/// All lint issues (errors and warnings) detected during loading. May be empty when the +/// file is valid. +/// +internal sealed record ReviewMarkLoadResult( + ReviewMarkConfiguration? Configuration, + IReadOnlyList Issues) +{ + /// + /// Reports all lint issues to the supplied , routing errors + /// to and warnings to . + /// + /// The context to report issues to. + internal void ReportIssues(Context context) + { + foreach (var issue in Issues) + { + if (issue.Severity == LintSeverity.Error) + { + context.WriteError(issue.ToString()); + } + else + { + context.WriteLine(issue.ToString()); + } + } + } +} + /// /// Represents the parsed contents of a .reviewmark.yaml configuration file. /// @@ -443,86 +506,16 @@ internal ReviewMarkConfiguration( } /// - /// Loads and parses a .reviewmark.yaml file from disk. - /// - /// Absolute or relative path to the configuration file. - /// A populated instance. - /// Thrown when is null or empty. - /// - /// Thrown when the file cannot be read, the YAML is invalid, or required configuration fields are - /// missing. The exception message always identifies the problematic file and, for YAML syntax - /// errors, the line and column number. - /// - internal static ReviewMarkConfiguration Load(string filePath) - { - // Validate the file path argument - if (string.IsNullOrWhiteSpace(filePath)) - { - throw new ArgumentException("File path must not be null or empty.", nameof(filePath)); - } - - // Read the file contents and wrap any file-system exception with useful context. - // Generic catch is justified here: Expected exceptions include IOException (and its subtypes - // such as FileNotFoundException, DirectoryNotFoundException, PathTooLongException), - // UnauthorizedAccessException, ArgumentException (invalid path characters), - // NotSupportedException, and other file-system exceptions. - string yaml; - try - { - yaml = File.ReadAllText(filePath); - } - catch (Exception ex) when (ex is not InvalidOperationException) - { - throw new InvalidOperationException($"Failed to read configuration file '{filePath}': {ex.Message}", ex); - } - - // Deserialize the raw YAML model, embedding the file path and line number in any parse error. - var raw = ReviewMarkConfigurationHelpers.DeserializeRaw(yaml, filePath); - - // Determine the base directory for resolving relative fileshare locations. - var baseDirectory = Path.GetDirectoryName(Path.GetFullPath(filePath)) - ?? throw new InvalidOperationException($"Cannot determine base directory for configuration file '{filePath}'."); - - // Validate the raw model, embedding the file path in any semantic error. - ReviewMarkConfiguration config; - try - { - config = ReviewMarkConfigurationHelpers.BuildConfiguration(raw); - } - catch (ArgumentException ex) - { - throw new InvalidOperationException($"Invalid configuration in '{filePath}': {ex.Message}", ex); - } - - // Resolve relative fileshare locations against the config file's directory so that - // a relative location (e.g., "index.json") works correctly regardless of the process - // working directory. - if (string.Equals(config.EvidenceSource.Type, "fileshare", StringComparison.OrdinalIgnoreCase) && - !Path.IsPathRooted(config.EvidenceSource.Location)) - { - var absoluteLocation = Path.GetFullPath(config.EvidenceSource.Location, baseDirectory); - return new ReviewMarkConfiguration( - config.NeedsReviewPatterns, - config.EvidenceSource with { Location = absoluteLocation }, - config.Reviews); - } - - return config; - } - - /// - /// Lints a .reviewmark.yaml file and returns all detected issues. - /// Unlike , this method does not stop at the first error; - /// it accumulates every detectable problem and returns them all so the caller - /// can report a complete list in a single pass. + /// Loads and lints a .reviewmark.yaml file, returning both the parsed + /// configuration and all detected issues in a single pass. /// /// Absolute or relative path to the configuration file. /// - /// A read-only list of error messages. The list is empty when the file is - /// structurally and semantically valid. + /// A containing the configuration (or null if + /// any error-level issues were detected) and the complete list of lint issues. /// /// Thrown when is null or empty. - internal static IReadOnlyList Lint(string filePath) + internal static ReviewMarkLoadResult Load(string filePath) { // Validate the file path argument if (string.IsNullOrWhiteSpace(filePath)) @@ -530,7 +523,7 @@ internal static IReadOnlyList Lint(string filePath) throw new ArgumentException("File path must not be null or empty.", nameof(filePath)); } - var errors = new List(); + var issues = new List(); // Try to read the file; if this fails we cannot continue. string yaml; @@ -540,8 +533,8 @@ internal static IReadOnlyList Lint(string filePath) } catch (Exception ex) when (ex is not InvalidOperationException) { - errors.Add($"{filePath}: error: {ex.Message}"); - return errors; + issues.Add(new LintIssue(filePath, LintSeverity.Error, ex.Message)); + return new ReviewMarkLoadResult(null, issues); } // Try to parse the raw YAML model; if this fails we cannot do semantic checks. @@ -554,39 +547,50 @@ internal static IReadOnlyList Lint(string filePath) } catch (InvalidOperationException ex) when (ex.InnerException is YamlException yamlEx) { - errors.Add($"{filePath}:{yamlEx.Start.Line}:{yamlEx.Start.Column}: error: {yamlEx.Message}"); - return errors; + issues.Add(new LintIssue( + $"{filePath}:{yamlEx.Start.Line}:{yamlEx.Start.Column}", + LintSeverity.Error, + $"at line {yamlEx.Start.Line}, column {yamlEx.Start.Column}: {yamlEx.Message}")); + return new ReviewMarkLoadResult(null, issues); } catch (InvalidOperationException ex) { - errors.Add($"{filePath}: error: {ex.Message}"); - return errors; + issues.Add(new LintIssue(filePath, LintSeverity.Error, ex.Message)); + return new ReviewMarkLoadResult(null, issues); } // Validate the evidence-source block, collecting all field-level errors. var es = raw.EvidenceSource; if (es == null) { - errors.Add( - $"{filePath}: error: Configuration is missing required 'evidence-source' block."); + issues.Add(new LintIssue( + filePath, + LintSeverity.Error, + "Configuration is missing required 'evidence-source' block.")); } else { if (string.IsNullOrWhiteSpace(es.Type)) { - errors.Add( - $"{filePath}: error: 'evidence-source' is missing a required 'type' field."); + issues.Add(new LintIssue( + filePath, + LintSeverity.Error, + "'evidence-source' is missing a required 'type' field.")); } else if (!ReviewMarkConfigurationHelpers.IsSupportedEvidenceSourceType(es.Type)) { - errors.Add( - $"{filePath}: error: 'evidence-source' type '{es.Type}' is not supported (must be 'none', 'url', or 'fileshare')."); + issues.Add(new LintIssue( + filePath, + LintSeverity.Error, + $"'evidence-source' type '{es.Type}' is not supported (must be 'none', 'url', or 'fileshare').")); } if (string.IsNullOrWhiteSpace(es.Location) && !string.Equals(es.Type, "none", StringComparison.OrdinalIgnoreCase)) { - errors.Add( - $"{filePath}: error: 'evidence-source' is missing a required 'location' field."); + issues.Add(new LintIssue( + filePath, + LintSeverity.Error, + "'evidence-source' is missing a required 'location' field.")); } } @@ -602,13 +606,17 @@ internal static IReadOnlyList Lint(string filePath) if (string.IsNullOrWhiteSpace(r.Id)) { - errors.Add( - $"{filePath}: error: Review set at index {i} is missing a required 'id' field."); + issues.Add(new LintIssue( + filePath, + LintSeverity.Error, + $"Review set at index {i} is missing a required 'id' field.")); } else if (seenIds.TryGetValue(r.Id, out var firstIndex)) { - errors.Add( - $"{filePath}: error: reviews[{i}] has duplicate ID '{r.Id}' (first defined at reviews[{firstIndex}])."); + issues.Add(new LintIssue( + filePath, + LintSeverity.Error, + $"reviews[{i}] has duplicate ID '{r.Id}' (first defined at reviews[{firstIndex}]).")); } else { @@ -617,18 +625,55 @@ internal static IReadOnlyList Lint(string filePath) if (string.IsNullOrWhiteSpace(r.Title)) { - errors.Add( - $"{filePath}: error: Review set at index {i} is missing a required 'title' field."); + issues.Add(new LintIssue( + filePath, + LintSeverity.Error, + $"Review set at index {i} is missing a required 'title' field.")); } if (r.Paths == null || !r.Paths.Any(p => !string.IsNullOrWhiteSpace(p))) { - errors.Add( - $"{filePath}: error: Review set at index {i} is missing required 'paths' entries."); + issues.Add(new LintIssue( + filePath, + LintSeverity.Error, + $"Review set at index {i} is missing required 'paths' entries.")); } } - return errors; + // If any error-level issues were found, return null configuration + if (issues.Any(i => i.Severity == LintSeverity.Error)) + { + return new ReviewMarkLoadResult(null, issues); + } + + // Build configuration from the validated raw model + var config = ReviewMarkConfigurationHelpers.BuildConfiguration(raw); + + // Determine the base directory for resolving relative fileshare locations. + var baseDirectory = Path.GetDirectoryName(Path.GetFullPath(filePath)); + if (baseDirectory == null) + { + issues.Add(new LintIssue( + filePath, + LintSeverity.Error, + $"Cannot determine base directory for configuration file '{filePath}'.")); + return new ReviewMarkLoadResult(null, issues); + } + + // Resolve relative fileshare locations against the config file's directory so that + // a relative location (e.g., "index.json") works correctly regardless of the process + // working directory. + if (string.Equals(config.EvidenceSource.Type, "fileshare", StringComparison.OrdinalIgnoreCase) && + !Path.IsPathRooted(config.EvidenceSource.Location)) + { + var absoluteLocation = Path.GetFullPath(config.EvidenceSource.Location, baseDirectory); + config = new ReviewMarkConfiguration( + config.NeedsReviewPatterns, + config.EvidenceSource with { Location = absoluteLocation }, + config.Reviews); + } + + return new ReviewMarkLoadResult(config, issues); } /// diff --git a/src/DemaConsulting.ReviewMark/Program.cs b/src/DemaConsulting.ReviewMark/Program.cs index 5759f32..89ed3f7 100644 --- a/src/DemaConsulting.ReviewMark/Program.cs +++ b/src/DemaConsulting.ReviewMark/Program.cs @@ -176,15 +176,12 @@ private static void RunLintLogic(Context context) var directory = context.WorkingDirectory ?? Directory.GetCurrentDirectory(); var definitionFile = context.DefinitionFile ?? PathHelpers.SafePathCombine(directory, ".reviewmark.yaml"); - // Lint the file, collecting all detectable errors in one pass. - var errors = ReviewMarkConfiguration.Lint(definitionFile); - foreach (var error in errors) - { - context.WriteError(error); - } + // Load and lint the file in one pass, collecting all detectable issues. + var result = ReviewMarkConfiguration.Load(definitionFile); + result.ReportIssues(context); // Report overall result - if (errors.Count == 0) + if (result.Issues.Count == 0) { context.WriteLine($"{definitionFile}: No issues found"); } @@ -259,8 +256,19 @@ private static void RunIndexLogic(Context context, string directory) /// The path to the definition YAML file. private static void RunDefinitionLogic(Context context, string directory, string definitionFile) { - // Load the configuration from the definition file - var config = ReviewMarkConfiguration.Load(definitionFile); + // Load the configuration with integrated linting + var loadResult = ReviewMarkConfiguration.Load(definitionFile); + + // Always report any lint issues found during loading + loadResult.ReportIssues(context); + + // If the configuration could not be loaded, stop here + if (loadResult.Configuration == null) + { + return; + } + + var config = loadResult.Configuration; // Handle --plan: generate and write the review plan if (context.PlanFile != null) @@ -281,7 +289,7 @@ private static void RunDefinitionLogic(Context context, string directory, string HandleIssues(context, reportResult.HasIssues, "Review report has review issues."); } - // Handle --elaborate: generate and print the review set elaboration + // Handle --elaborate if (context.ElaborateId != null) { try diff --git a/test/DemaConsulting.ReviewMark.Tests/Configuration/ReviewMarkConfigurationTests.cs b/test/DemaConsulting.ReviewMark.Tests/Configuration/ReviewMarkConfigurationTests.cs index 5d01a12..9f214bb 100644 --- a/test/DemaConsulting.ReviewMark.Tests/Configuration/ReviewMarkConfigurationTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/Configuration/ReviewMarkConfigurationTests.cs @@ -18,6 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +using DemaConsulting.ReviewMark.Cli; using DemaConsulting.ReviewMark.Configuration; using DemaConsulting.ReviewMark.Indexing; @@ -291,41 +292,50 @@ public void ReviewSet_GetFingerprint_RenameFile_ReturnsSameFingerprint() } /// - /// Test that Load throws when the specified file does not exist. + /// Test that Load returns null configuration with an error issue when the file does not exist. /// [TestMethod] - public void ReviewMarkConfiguration_Load_NonExistentFile_ThrowsException() + public void ReviewMarkConfiguration_Load_NonExistentFile_ReturnsNullConfigWithErrorIssue() { // Arrange — a path within the test directory that does not exist var nonExistentPath = PathHelpers.SafePathCombine(_testDirectory, ".reviewmark.yaml"); - // Act & Assert - Assert.Throws(() => - ReviewMarkConfiguration.Load(nonExistentPath)); + // Act + var result = ReviewMarkConfiguration.Load(nonExistentPath); + + // Assert — configuration is null and one error issue is reported + Assert.IsNull(result.Configuration); + Assert.AreEqual(1, result.Issues.Count); + Assert.AreEqual(LintSeverity.Error, result.Issues[0].Severity); } /// - /// Test that Load includes the file name in the error message when YAML is invalid. + /// Test that Load returns null configuration with an error issue naming file and line when YAML is invalid. /// [TestMethod] - public void ReviewMarkConfiguration_Load_InvalidYaml_ErrorIncludesFilenameAndLine() + public void ReviewMarkConfiguration_Load_InvalidYaml_ReturnsNullConfigWithErrorIssue() { // Arrange — write a configuration file with invalid YAML syntax var configPath = PathHelpers.SafePathCombine(_testDirectory, ".reviewmark.yaml"); File.WriteAllText(configPath, "{{{invalid yaml"); - // Act & Assert - var ex = Assert.Throws(() => - ReviewMarkConfiguration.Load(configPath)); - Assert.Contains(".reviewmark.yaml", ex.Message); - Assert.Contains("at line", ex.Message); + // Act + var result = ReviewMarkConfiguration.Load(configPath); + + // Assert — configuration is null, one error issue naming file and line + Assert.IsNull(result.Configuration); + Assert.AreEqual(1, result.Issues.Count); + Assert.AreEqual(LintSeverity.Error, result.Issues[0].Severity); + Assert.Contains(".reviewmark.yaml", result.Issues[0].Location); + Assert.Contains("at line", result.Issues[0].Description); } /// - /// Test that Load includes the file name in the error message when required fields are missing. + /// Test that Load returns null configuration with an error issue naming the file and missing field + /// when required fields are missing. /// [TestMethod] - public void ReviewMarkConfiguration_Load_MissingEvidenceSource_ErrorIncludesFilename() + public void ReviewMarkConfiguration_Load_MissingEvidenceSource_ReturnsNullConfigWithErrorIssue() { // Arrange — write a valid YAML file that is missing the required evidence-source block var configPath = PathHelpers.SafePathCombine(_testDirectory, ".reviewmark.yaml"); @@ -339,19 +349,22 @@ public void ReviewMarkConfiguration_Load_MissingEvidenceSource_ErrorIncludesFile - "src/**/*.cs" """); - // Act & Assert - var ex = Assert.Throws(() => - ReviewMarkConfiguration.Load(configPath)); - Assert.Contains(".reviewmark.yaml", ex.Message); - Assert.Contains("evidence-source", ex.Message); + // Act + var result = ReviewMarkConfiguration.Load(configPath); + + // Assert — configuration is null and error mentions evidence-source + Assert.IsNull(result.Configuration); + Assert.AreEqual(1, result.Issues.Count); + Assert.AreEqual(LintSeverity.Error, result.Issues[0].Severity); + Assert.Contains("evidence-source", result.Issues[0].Description); } /// - /// Test that Lint returns all errors from a file with multiple detectable issues + /// Test that Load returns all issues from a file with multiple detectable errors /// (missing evidence-source AND duplicate review IDs) without stopping at the first. /// [TestMethod] - public void ReviewMarkConfiguration_Lint_MultipleErrors_ReturnsAll() + public void ReviewMarkConfiguration_Load_MultipleErrors_ReturnsAllIssues() { // Arrange — write a YAML file missing evidence-source and containing duplicate IDs var configPath = PathHelpers.SafePathCombine(_testDirectory, ".reviewmark.yaml"); @@ -370,13 +383,16 @@ public void ReviewMarkConfiguration_Lint_MultipleErrors_ReturnsAll() """); // Act - var errors = ReviewMarkConfiguration.Lint(configPath); - - // Assert — both the missing evidence-source error and the duplicate ID error are returned - Assert.AreEqual(2, errors.Count); - Assert.IsTrue(errors.Any(e => e.Contains("evidence-source")), + var result = ReviewMarkConfiguration.Load(configPath); + + // Assert — configuration is null and both errors are reported + Assert.IsNull(result.Configuration); + Assert.AreEqual(2, result.Issues.Count); + Assert.IsTrue(result.Issues.All(i => i.Severity == LintSeverity.Error), + "Expected all issues to have error severity."); + Assert.IsTrue(result.Issues.Any(i => i.Description.Contains("evidence-source")), "Expected an error about missing evidence-source."); - Assert.IsTrue(errors.Any(e => e.Contains("duplicate ID") && e.Contains("Core-Logic")), + Assert.IsTrue(result.Issues.Any(i => i.Description.Contains("duplicate ID") && i.Description.Contains("Core-Logic")), "Expected an error about duplicate ID 'Core-Logic'."); } @@ -402,11 +418,12 @@ public void ReviewMarkConfiguration_Load_FileshareRelativeLocation_ResolvesToAbs """); // Act - load the configuration - var config = ReviewMarkConfiguration.Load(configPath); + var result = ReviewMarkConfiguration.Load(configPath); // Assert — relative location is resolved to an absolute path under the config directory - Assert.IsTrue(Path.IsPathRooted(config.EvidenceSource.Location)); - Assert.AreEqual(PathHelpers.SafePathCombine(_testDirectory, "index.json"), config.EvidenceSource.Location); + Assert.IsNotNull(result.Configuration); + Assert.IsTrue(Path.IsPathRooted(result.Configuration.EvidenceSource.Location)); + Assert.AreEqual(PathHelpers.SafePathCombine(_testDirectory, "index.json"), result.Configuration.EvidenceSource.Location); } /// @@ -454,11 +471,11 @@ public void ReviewMarkConfiguration_Parse_NoneEvidenceSource_NoLocationRequired( } /// - /// Test that Lint does not report an error when the evidence-source type is none + /// Test that Load does not report an issue when the evidence-source type is none /// and no location field is present. /// [TestMethod] - public void ReviewMarkConfiguration_Lint_NoneEvidenceSource_NoErrors() + public void ReviewMarkConfiguration_Load_NoneEvidenceSource_NoIssues() { // Arrange — write a valid config with a none evidence source var configPath = PathHelpers.SafePathCombine(_testDirectory, ".reviewmark.yaml"); @@ -473,10 +490,11 @@ public void ReviewMarkConfiguration_Lint_NoneEvidenceSource_NoErrors() """); // Act - var errors = ReviewMarkConfiguration.Lint(configPath); + var result = ReviewMarkConfiguration.Load(configPath); - // Assert — no errors for a valid none source - Assert.HasCount(0, errors); + // Assert — no issues and configuration is non-null for a valid none source + Assert.IsNotNull(result.Configuration); + Assert.HasCount(0, result.Issues); } // ------------------------------------------------------------------------- @@ -885,4 +903,54 @@ public void ReviewMarkConfiguration_ElaborateReviewSet_ContainsFullFingerprint() Assert.Contains(expectedFingerprint, result.Markdown); Assert.AreEqual(64, expectedFingerprint.Length); } + + /// + /// Test that Load on a valid file returns configuration and no issues. + /// + [TestMethod] + public void ReviewMarkConfiguration_Load_ValidFile_ReturnsConfigurationAndNoIssues() + { + // Arrange — write a valid configuration file + var configPath = PathHelpers.SafePathCombine(_testDirectory, ".reviewmark.yaml"); + File.WriteAllText(configPath, MinimalYaml); + + // Act + var result = ReviewMarkConfiguration.Load(configPath); + + // Assert — configuration is non-null and no issues are reported + Assert.IsNotNull(result.Configuration); + Assert.HasCount(0, result.Issues); + } + + /// + /// Test that ReportIssues routes errors to WriteError and warnings to WriteLine via Context. + /// + [TestMethod] + public void ReviewMarkLoadResult_ReportIssues_RoutesIssuesToContext() + { + // Arrange — a result with one warning and one error; capture output via a log file + var logFile = PathHelpers.SafePathCombine(_testDirectory, "report.log"); + var issues = new List + { + new("file.yaml", LintSeverity.Warning, "A warning message"), + new("file.yaml", LintSeverity.Error, "An error message") + }; + var result = new ReviewMarkLoadResult(null, issues); + + // Act — dispose context before reading log to release the file handle on Windows + int exitCode; + using (var context = Context.Create(["--silent", "--log", logFile])) + { + result.ReportIssues(context); + exitCode = context.ExitCode; + } + + // Assert — error sets exit code; both messages appear in the log + Assert.AreEqual(1, exitCode); + var log = File.ReadAllText(logFile); + Assert.Contains("warning", log); + Assert.Contains("A warning message", log); + Assert.Contains("error", log); + Assert.Contains("An error message", log); + } } diff --git a/test/DemaConsulting.ReviewMark.Tests/ProgramTests.cs b/test/DemaConsulting.ReviewMark.Tests/ProgramTests.cs index 6a2e0da..9eff37f 100644 --- a/test/DemaConsulting.ReviewMark.Tests/ProgramTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/ProgramTests.cs @@ -580,4 +580,41 @@ public void Program_Run_WithLintFlag_MultipleErrors_ReportsAll() Assert.Contains("duplicate ID", logContent); Assert.Contains("Core-Logic", logContent); } + + /// + /// Test that Run with --definition flag pointing to an invalid config reports lint errors and exits with code 1. + /// + [TestMethod] + public void Program_Run_WithDefinitionFlag_InvalidConfig_ReportsLintError() + { + // Arrange — create a definition file with no evidence-source block + using var tempDir = new TestDirectory(); + var definitionFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "definition.yaml"); + File.WriteAllText(definitionFile, """ + needs-review: + - "src/**/*.cs" + reviews: + - id: Core-Logic + title: Review of core business logic + paths: + - "src/**/*.cs" + """); + + var planFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "plan.md"); + var logFile = PathHelpers.SafePathCombine(tempDir.DirectoryPath, "test.log"); + + // Act — dispose the context before reading the log to release the file handle on Windows + int exitCode; + using (var context = Context.Create(["--silent", "--log", logFile, "--definition", definitionFile, "--plan", planFile])) + { + Program.Run(context); + exitCode = context.ExitCode; + } + + // Assert — non-zero exit code and log contains error mentioning evidence-source + var logContent = File.ReadAllText(logFile); + Assert.AreEqual(1, exitCode); + Assert.Contains("error:", logContent); + Assert.Contains("evidence-source", logContent); + } } From 8be65f13543cf558937b2cfd78a46b26cb163123 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Apr 2026 17:42:23 -0400 Subject: [PATCH 15/19] restructure: move design and reqstream files into review-mark system folder (#43) * Binary-copy updated files from TemplateDotNetTool Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/2f33b6eb-2560-4958-a7a8-8bc20814d740 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * restructure: move design and reqstream files into review-mark system folder - Move docs/design files into docs/design/review-mark/ system folder - Move docs/reqstream files into docs/reqstream/review-mark/ system folder - Rename reqstream files to remove subsystem-/unit- prefixes - Create missing subsystem overview design files (cli.md, configuration.md, indexing.md, self-test.md) - Update requirements.yaml includes to new paths - Update .reviewmark.yaml: new paths, renamed IDs, add ReviewMark-Configuration subsystem - Update docs/design/introduction.md Folder Layout section - Update docs/design/definition.yaml resource-path and input-files - Complies with updated .github/standards Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Update docs/design/review-mark/cli/cli.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix: split configuration requirements, add tests to subsystem reviews, fix lint issues Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/c5dea3ca-a569-475d-acf6-c15f9ca66bc0 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * refine: use class-scoped titles for unit requirements Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/c5dea3ca-a569-475d-acf6-c15f9ca66bc0 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * sync: update PathHelpers to use post-combine containment check from template Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/3983415c-b3a4-43dc-8d11-0ddd968e0198 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Co-authored-by: Malcolm Nixon Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/agents/developer.agent.md | 3 +- .github/agents/implementation.agent.md | 10 +- .github/agents/quality.agent.md | 14 ++ .github/standards/design-documentation.md | 63 +++--- .github/standards/reqstream-usage.md | 32 ++- .github/standards/reviewmark-usage.md | 155 ++++++-------- .github/standards/technical-documentation.md | 63 +++--- .reviewmark.yaml | 132 ++++++------ AGENTS.md | 189 +++++++----------- docs/design/definition.yaml | 29 +-- docs/design/indexing/path-helpers.md | 35 ---- docs/design/introduction.md | 24 +++ docs/design/review-mark/cli/cli.md | 21 ++ docs/design/{ => review-mark}/cli/context.md | 0 .../configuration/configuration.md | 22 ++ .../configuration/glob-matcher.md | 0 .../review-mark-configuration.md | 0 docs/design/review-mark/indexing/indexing.md | 21 ++ .../review-mark/indexing/path-helpers.md | 54 +++++ .../indexing/review-index.md | 0 docs/design/{ => review-mark}/program.md | 0 .../{system.md => review-mark/review-mark.md} | 0 .../design/review-mark/self-test/self-test.md | 20 ++ .../{ => review-mark}/self-test/validation.md | 0 .../cli/cli.yaml} | 0 .../cli/context.yaml} | 0 .../configuration/configuration.yaml | 61 ++++++ .../configuration/glob-matcher.yaml} | 0 .../review-mark-configuration.yaml} | 6 +- .../indexing/indexing.yaml} | 0 .../indexing/path-helpers.yaml} | 7 +- .../indexing/review-index.yaml} | 0 .../platform-requirements.yaml | 0 .../program.yaml} | 0 .../review-mark.yaml} | 0 .../self-test/self-test.yaml} | 0 .../self-test/validation.yaml} | 0 lint.bat | 51 +++-- lint.sh | 49 +++-- requirements.yaml | 25 +-- .../Indexing/PathHelpers.cs | 38 ++-- 41 files changed, 661 insertions(+), 463 deletions(-) delete mode 100644 docs/design/indexing/path-helpers.md create mode 100644 docs/design/review-mark/cli/cli.md rename docs/design/{ => review-mark}/cli/context.md (100%) create mode 100644 docs/design/review-mark/configuration/configuration.md rename docs/design/{ => review-mark}/configuration/glob-matcher.md (100%) rename docs/design/{ => review-mark}/configuration/review-mark-configuration.md (100%) create mode 100644 docs/design/review-mark/indexing/indexing.md create mode 100644 docs/design/review-mark/indexing/path-helpers.md rename docs/design/{ => review-mark}/indexing/review-index.md (100%) rename docs/design/{ => review-mark}/program.md (100%) rename docs/design/{system.md => review-mark/review-mark.md} (100%) create mode 100644 docs/design/review-mark/self-test/self-test.md rename docs/design/{ => review-mark}/self-test/validation.md (100%) rename docs/reqstream/{cli/subsystem-cli.yaml => review-mark/cli/cli.yaml} (100%) rename docs/reqstream/{cli/unit-context.yaml => review-mark/cli/context.yaml} (100%) create mode 100644 docs/reqstream/review-mark/configuration/configuration.yaml rename docs/reqstream/{configuration/unit-glob-matcher.yaml => review-mark/configuration/glob-matcher.yaml} (100%) rename docs/reqstream/{configuration/subsystem-configuration.yaml => review-mark/configuration/review-mark-configuration.yaml} (91%) rename docs/reqstream/{indexing/subsystem-indexing.yaml => review-mark/indexing/indexing.yaml} (100%) rename docs/reqstream/{indexing/unit-path-helpers.yaml => review-mark/indexing/path-helpers.yaml} (75%) rename docs/reqstream/{indexing/unit-review-index.yaml => review-mark/indexing/review-index.yaml} (100%) rename docs/reqstream/{ => review-mark}/platform-requirements.yaml (100%) rename docs/reqstream/{unit-program.yaml => review-mark/program.yaml} (100%) rename docs/reqstream/{reviewmark-system.yaml => review-mark/review-mark.yaml} (100%) rename docs/reqstream/{self-test/subsystem-self-test.yaml => review-mark/self-test/self-test.yaml} (100%) rename docs/reqstream/{self-test/unit-validation.yaml => review-mark/self-test/validation.yaml} (100%) diff --git a/.github/agents/developer.agent.md b/.github/agents/developer.agent.md index 2028f79..d936129 100644 --- a/.github/agents/developer.agent.md +++ b/.github/agents/developer.agent.md @@ -16,7 +16,8 @@ Perform software development tasks by determining and applying appropriate DEMA 2. **Read relevant standards** from `.github/standards/` as defined in AGENTS.md based on work performed 3. **Apply loaded standards** throughout development process 4. **Execute work** following standards requirements and quality checks -5. **Generate completion report** with results and compliance status +5. **Lint fixes** follow the linting process before performing quality gates +6. **Generate completion report** with results and compliance status # Reporting diff --git a/.github/agents/implementation.agent.md b/.github/agents/implementation.agent.md index 91b44d7..35cc1c8 100644 --- a/.github/agents/implementation.agent.md +++ b/.github/agents/implementation.agent.md @@ -28,7 +28,7 @@ counting how many retries have occurred. Call the built-in explore sub-agent with: -- **context**: the user's request and any current quality findings +- **context**: the user's request + any previous quality findings + retry context - **goal**: analyze the implementation state and develop a plan to implement the request Once the explore sub-agent finishes, transition to the DEVELOPMENT state. @@ -37,7 +37,7 @@ Once the explore sub-agent finishes, transition to the DEVELOPMENT state. Call the developer sub-agent with: -- **context** the user's request and the current implementation plan +- **context** the user's request + research plan + specific quality issues to address (if retry) - **goal** implement the user's request and any identified quality fixes Once the developer sub-agent finishes: @@ -49,7 +49,7 @@ Once the developer sub-agent finishes: Call the quality sub-agent with: -- **context** the user's request and the current implementation report +- **context** the user's request + development summary + files changed + previous issues (if any) - **goal** check the quality of the work performed for any issues Once the quality sub-agent finishes: @@ -73,7 +73,7 @@ of the project consisting of: ## State Machine Execution - **Research Results**: [Summary of explore agent findings] -- **Development Results**: [Summary of developer agent results] +- **Development Results**: [Summary of developer agent results] - **Quality Results**: [Summary of quality agent results] - **State Transitions**: [Log of state changes and decisions] @@ -86,7 +86,7 @@ of the project consisting of: ## Final Status - **Implementation Success**: [Overall completion status] -- **Quality Compliance**: [Final quality validation status] +- **Quality Compliance**: [Final quality validation status] - **Issues Resolved**: [Problems encountered and resolution attempts] ``` diff --git a/.github/agents/quality.agent.md b/.github/agents/quality.agent.md index a7b57d4..691a17d 100644 --- a/.github/agents/quality.agent.md +++ b/.github/agents/quality.agent.md @@ -26,6 +26,13 @@ This assessment is a quality control system of the project and MUST be performed Upon completion create a summary in `.agent-logs/{agent-name}-{subject}-{unique-id}.md` of the project consisting of: +The **Result** field MUST reflect the quality validation outcome for orchestrator decision-making: + +- **Result: SUCCEEDED** - Only when Overall Grade is PASS (all compliance requirements met) +- **Result: FAILED** - When Overall Grade is FAIL or NEEDS_WORK (compliance failures present) + +This ensures orchestrators properly halt workflows when quality gates fail. + ```markdown # Quality Assessment Report @@ -98,6 +105,13 @@ of the project consisting of: - Do README.md files use absolute URLs and include concrete examples? (PASS|FAIL|N/A) - [Evidence] - Is documentation integrated into ReviewMark review-sets for formal review? (PASS|FAIL|N/A) - [Evidence] +## Software Item Completeness: (PASS|FAIL|N/A) + +- Does every identified software unit have its own requirements file? (PASS|FAIL|N/A) - [Evidence] +- Does every identified software unit have its own design document? (PASS|FAIL|N/A) - [Evidence] +- Does every identified subsystem have its own requirements file? (PASS|FAIL|N/A) - [Evidence] +- Does every identified subsystem have its own design document? (PASS|FAIL|N/A) - [Evidence] + ## Process Compliance: (PASS|FAIL|N/A) - Was Continuous Compliance workflow followed? (PASS|FAIL|N/A) - [Evidence] diff --git a/.github/standards/design-documentation.md b/.github/standards/design-documentation.md index 6312275..e14cd30 100644 --- a/.github/standards/design-documentation.md +++ b/.github/standards/design-documentation.md @@ -23,12 +23,13 @@ design to implementation: ```text docs/design/ -├── introduction.md # Design overview with software structure -├── system.md # System-level design documentation -├── {subsystem-name}/ # Subsystem design documents (kebab-case folder names) -│ ├── {subsystem-name}.md # Subsystem overview and design -│ └── {unit-name}.md # Unit-level design documents -└── {unit-name}.md # Top-level unit design documents (if not in subsystem) +├── introduction.md # Design overview with software structure +└── {system-name}/ # System-level design folder (one per system) + ├── {system-name}.md # System-level design documentation + ├── {subsystem-name}/ # Subsystem design documents (kebab-case folder names) + │ ├── {subsystem-name}.md # Subsystem overview and design + │ └── {unit-name}.md # Unit-level design documents + └── {unit-name}.md # Top-level unit design documents (if not in subsystem) ``` ## introduction.md (MANDATORY) @@ -56,13 +57,16 @@ to understand these classifications before creating this section. Example format: ```text -ProjectName (System) +Project1Name (System) ├── ComponentA (Subsystem) │ ├── ClassX (Unit) │ └── ClassY (Unit) ├── ComponentB (Subsystem) │ └── ClassZ (Unit) └── UtilityClass (Unit) + +Project2Name (System) +└── HelperClass (Unit) ``` ### Folder Layout Section (MANDATORY) @@ -73,24 +77,29 @@ mirror the software structure, with file paths and brief descriptions. Example format: ```text -src/ProjectName/ +src/Project1Name/ ├── ComponentA/ -│ ├── ClassX.cs — brief description -│ └── ClassY.cs — brief description +│ ├── ClassX.cs — Core business logic handler +│ └── ClassY.cs — Data validation service ├── ComponentB/ -│ └── ClassZ.cs — brief description -└── UtilityClass.cs — brief description +│ └── ClassZ.cs — Integration interface +└── UtilityClass.cs — Common utility functions + +src/Project2Name/ +└── HelperClass.cs — Helper functions ``` -## system.md (MANDATORY) +## System Design Documentation (MANDATORY) -The `system.md` file contains system-level design documentation including: +For each system identified in the repository: -- System architecture and major components -- External interfaces and dependencies -- Data flow and control flow -- System-wide design constraints and decisions -- Integration patterns and communication protocols +- Create a kebab-case folder matching the system name +- Include `{system-name}.md` with system-level design documentation such as: + - System architecture and major components + - External interfaces and dependencies + - Data flow and control flow + - System-wide design constraints and decisions + - Integration patterns and communication protocols ## Subsystem and Unit Design Documents @@ -98,9 +107,9 @@ For each subsystem identified in the software structure: - Create a kebab-case folder matching the subsystem name (enables automated tooling) - Include `{subsystem-name}.md` with subsystem overview and design -- Include unit design documents for complex units within the subsystem +- Include unit design documents for ALL units within the subsystem -For significant units requiring detailed design: +For every unit identified in the software structure: - Document data models, algorithms, and key methods - Describe interactions with other units @@ -124,8 +133,10 @@ implementation specification for formal code review: - **Implementation Detail**: Provide sufficient detail for code review and implementation - **Architectural Clarity**: Clearly define component boundaries and interfaces - **Traceability**: Link to requirements where applicable using ReqStream patterns -- **Concrete Examples**: Use actual class names, method signatures, and data structures -- **Current Information**: Keep synchronized with code changes and refactoring + +# Mermaid Diagram Integration + +Use Mermaid diagrams to supplement text descriptions (diagrams must not replace text content). # Quality Checks @@ -133,10 +144,10 @@ Before submitting design documentation, verify: - [ ] `introduction.md` includes both Software Structure and Folder Layout sections - [ ] Software structure correctly categorizes items as System/Subsystem/Unit per `software-items.md` -- [ ] Folder layout matches actual source code organization -- [ ] `system.md` provides comprehensive system-level design +- [ ] Folder layout mirrors software structure organization +- [ ] Design documents provide sufficient detail for code review +- [ ] System documentation provides comprehensive system-level design - [ ] Subsystem documentation folders use kebab-case names while mirroring source subsystem names and structure -- [ ] Design documents contain sufficient implementation detail - [ ] All documents follow technical documentation formatting standards - [ ] Content is current with implementation and requirements - [ ] Documents are integrated into ReviewMark review-sets for formal review diff --git a/.github/standards/reqstream-usage.md b/.github/standards/reqstream-usage.md index aa75a1f..bd8c739 100644 --- a/.github/standards/reqstream-usage.md +++ b/.github/standards/reqstream-usage.md @@ -30,15 +30,17 @@ the source code structure because reviewers need clear navigation from requirements to design to implementation: ```text -requirements.yaml # Root file (includes only) +requirements.yaml # Root file (includes only) docs/reqstream/ -├── system.yaml # System-level requirements -├── platform-requirements.yaml # Platform support requirements -├── {subsystem-name}/ # Subsystem requirements (kebab-case folders) -│ └── {subsystem-name}.yaml # Requirements for this subsystem -├── {unit-name}.yaml # Unit requirements (for top-level units) -└── ots/ # OTS software item requirements - └── {ots-name}.yaml # Requirements for OTS components +├── {system-name}/ # System-level requirements folder (one per system) +│ ├── {system-name}.yaml # System-level requirements +│ ├── platform-requirements.yaml # Platform support requirements +│ ├── {subsystem-name}/ # Subsystem requirements (kebab-case folders) +│ │ ├── {subsystem-name}.yaml # Requirements for this subsystem +│ │ └── {unit-name}.yaml # Requirements for units within this subsystem +│ └── {unit-name}.yaml # Requirements for top-level units (outside subsystems) +└── ots/ # OTS software items folder + └── {ots-name}.yaml # Requirements for OTS components ``` The folder structure MUST mirror the source code organization to maintain @@ -62,7 +64,7 @@ for compliance auditing. sections: - title: Functional Requirements requirements: - - id: Project-Subsystem-Feature + - id: System-Subsystem-Feature title: The system shall perform the required function. justification: | Business rationale explaining why this requirement exists. @@ -88,7 +90,7 @@ sections: sections: - title: System.Text.Json requirements: - - id: Project-SystemTextJson-ReadJson + - id: TemplateTool-SystemTextJson-ReadJson title: System.Text.Json shall be able to read JSON files. tests: - JsonReaderTests.TestReadValidJson @@ -96,7 +98,7 @@ sections: # Semantic IDs (MANDATORY) -Use meaningful IDs following `Project-Section-ShortDesc` pattern because +Use meaningful IDs following `System-Section-ShortDesc` pattern because auditors need to understand requirements without cross-referencing: - **Good**: `TemplateTool-Core-DisplayHelp` @@ -127,12 +129,6 @@ dotnet reqstream \ --requirements requirements.yaml \ --lint -# Enforce requirements traceability (use in CI/CD) -dotnet reqstream \ - --requirements requirements.yaml \ - --tests "artifacts/**/*.trx" \ - --enforce - # Generate requirements report dotnet reqstream \ --requirements requirements.yaml \ @@ -154,7 +150,7 @@ dotnet reqstream \ Before submitting requirements, verify: -- [ ] All requirements have semantic IDs (`Project-Section-Feature` pattern) +- [ ] All requirements have semantic IDs (`System-Section-Feature` pattern) - [ ] Every requirement links to at least one passing test - [ ] Platform-specific requirements use source filters (`platform@TestName`) - [ ] Requirements specify observable behavior (WHAT), not implementation (HOW) diff --git a/.github/standards/reviewmark-usage.md b/.github/standards/reviewmark-usage.md index a40179f..2fdaa19 100644 --- a/.github/standards/reviewmark-usage.md +++ b/.github/standards/reviewmark-usage.md @@ -1,16 +1,24 @@ -# ReviewMark File Review Standards +# ReviewMark Usage Standard -This document defines DEMA Consulting standards for managing file reviews using -ReviewMark within Continuous Compliance environments. +## Purpose -# Core Purpose +ReviewMark manages file review status enforcement and formal review processes. It tracks which files need +review, organizes them into review-sets, and generates review plans and reports. -ReviewMark automates file review tracking using cryptographic fingerprints to -ensure: +## Key Commands -- Every file requiring review is covered by a current, valid review -- Reviews become stale when files change, triggering re-review -- Complete audit trail of review coverage for regulatory compliance +- **Lint Configuration**: `dotnet reviewmark --lint` +- **Elaborate Review-Set**: `dotnet reviewmark --elaborate [review-set]` +- **Generate Plan**: `dotnet reviewmark --plan docs/code_review_plan/plan.md` +- **Generate Report**: `dotnet reviewmark --report docs/code_review_report/report.md` + +## Repository Structure + +Required repository items for ReviewMark operation: + +- `.reviewmark.yaml` - Configuration for review-sets, file-patterns, and review evidence-source. +- `docs/code_review_plan/` - Review planning artifacts +- `docs/code_review_report/` - Review status reports # Review Definition Structure @@ -19,36 +27,25 @@ Configure reviews in `.reviewmark.yaml` at repository root: ```yaml # Patterns identifying all files that require review needs-review: - # Include core development artifacts - - "requirements.yaml" # Root requirements file - - "docs/reqstream/**/*.yaml" # Requirements files - - "docs/design/**/*.md" # Design documentation - - "**/*.cs" # All C# source and test files - - # Exclude build output and generated content - - "!**/obj/**" # Exclude build output - - "!**/bin/**" # Exclude binary output - - "!**/generated/**" # Exclude auto-generated files + # Include source code (adjust file extensions for your repo) + - "**/*.cs" # C# source files + - "**/*.cpp" # C++ source files + - "**/*.hpp" # C++ header files + - "!**/bin/**" # Generated source in build outputs + - "!**/obj/**" # Generated source in build intermediates + + # Include requirement files + - "requirements.yaml" # Root requirements file + - "docs/reqstream/**/*.yaml" # Requirements files + + # Include critical documentation files + - "README.md" # Root level README + - "docs/user_guide/**/*.md" # User guide + - "docs/design/**/*.md" # Design documentation # Source of review evidence evidence-source: type: none - -# Named review-sets grouping related files -reviews: - - id: MyProduct-PasswordValidator - title: Password Validator Unit Review - paths: - - "docs/reqstream/authentication/password-validator.yaml" - - "docs/design/authentication/password-validator.md" - - "src/{ProjectName}/Authentication/PasswordValidator.cs" - - "test/{ProjectName}.Tests/Authentication/PasswordValidatorTests.cs" - - - id: MyProduct-AllRequirements - title: All Requirements Review - paths: - - "requirements.yaml" - - "docs/reqstream/**/*.yaml" ``` # Review-Set Organization @@ -56,93 +53,56 @@ reviews: Organize review-sets using standard patterns to ensure comprehensive coverage and consistent review processes: -## [Project]-System Review +## [System]-Architecture Review (one per system) -Reviews system integration and operational validation: +Reviews system architecture and operational validation: -- **Files**: System requirements (`docs/reqstream/system.yaml`), design introduction - (`docs/design/introduction.md`), system design (`docs/design/system.md`), +- **Files**: System requirements (`docs/reqstream/{system-name}/{system-name}.yaml`), design introduction + (`docs/design/introduction.md`), system design (`docs/design/{system-name}/{system-name}.md`), integration tests - **Purpose**: Validates system operates as designed and meets overall requirements -- **Example**: `TemplateTool-System` +- **Example**: `SomeSystem-Architecture` -## [Product]-Design Review +## [System]-Design Review Reviews architectural and design consistency: - **Files**: System requirements, platform requirements, all design documents under `docs/design/` - **Purpose**: Ensures design completeness and architectural coherence -- **Example**: `MyProduct-Design` +- **Example**: `SomeSystem-Design` -## [Product]-AllRequirements Review +## [System]-AllRequirements Review Reviews requirements quality and traceability: -- **Files**: All requirement files including root `requirements.yaml` and all files under `docs/reqstream/` +- **Files**: All requirement files including root `requirements.yaml` and all files under `docs/reqstream/{system-name}/` - **Purpose**: Validates requirements structure, IDs, justifications, and test linkage -- **Example**: `MyProduct-AllRequirements` - -## [Product]-[Unit] Review - -Reviews individual software unit implementation: - -- **Files**: Unit requirements, design documents, source code, unit tests -- **Purpose**: Validates unit meets requirements and is properly implemented -- **File Path Pattern**: - - Requirements: `docs/reqstream/{subsystem-name}/{unit-name}.yaml` or `docs/reqstream/{unit-name}.yaml` - - Design: `docs/design/{subsystem-name}/{unit-name}.md` or `docs/design/{unit-name}.md` - - Source: `src/{ProjectName}/{SubsystemName}/{UnitName}.cs` - - Tests: `test/{ProjectName}.Tests/{SubsystemName}/{UnitName}Tests.cs` -- **Example**: `MyProduct-PasswordValidator`, `MyProduct-ConfigParser` +- **Example**: `SomeSystem-AllRequirements` -## [Product]-[Subsystem] Review +## [System]-[Subsystem] Review Reviews subsystem architecture and interfaces: - **Files**: Subsystem requirements, design documents, integration tests (usually no source code) - **Purpose**: Validates subsystem behavior and interface compliance - **File Path Pattern**: - - Requirements: `docs/reqstream/{subsystem-name}/{subsystem-name}.yaml` - - Design: `docs/design/{subsystem-name}/{subsystem-name}.md` - - Tests: `test/{ProjectName}.Tests/{SubsystemName}Integration/` or similar -- **Example**: `MyProduct-Authentication`, `MyProduct-DataLayer` + - Requirements: `docs/reqstream/{system-name}/{subsystem-name}/{subsystem-name}.yaml` + - Design: `docs/design/{system-name}/{subsystem-name}/{subsystem-name}.md` + - Tests: `test/{SystemName}.Tests/{SubsystemName}/{SubsystemName}*` or similar +- **Example**: `SomeSystem-Authentication`, `SomeSystem-DataLayer` -## [Product]-OTS Review +## [System]-[Subsystem]-[Unit] Review -Reviews OTS (Off-The-Shelf) software integration: +Reviews individual software unit implementation: -- **Files**: OTS requirements and integration test evidence -- **Purpose**: Validates OTS components meet integration requirements +- **Files**: Unit requirements, design documents, source code, unit tests +- **Purpose**: Validates unit meets requirements and is properly implemented - **File Path Pattern**: - - Requirements: `docs/reqstream/ots/{ots-name}.yaml` - - Tests: Integration tests proving OTS functionality -- **Example**: `MyProduct-SystemTextJson`, `MyProduct-EntityFramework` - -# File Pattern Best Practices - -Use "include-then-exclude" approach for `needs-review` patterns because it -ensures comprehensive coverage while removing unwanted files: - -1. **Start broad**: Include all files of potential interest with generous patterns -2. **Exclude overreach**: Use `!` patterns to remove build output, generated files, and temporary files -3. **Test patterns**: Verify patterns match intended files using `dotnet reviewmark --elaborate` - -**Order matters**: Patterns are processed sequentially, excludes override earlier includes. - -# ReviewMark Commands - -Essential ReviewMark commands for Continuous Compliance: - -```bash -# Lint review configuration for issues (run before use) -dotnet reviewmark --lint - -# Generate review plan and report (use in CI/CD) -dotnet reviewmark \ - --plan docs/code_review_plan/plan.md \ - --report docs/code_review_report/report.md \ - --enforce -``` + - Requirements: `docs/reqstream/{system-name}/{subsystem-name}/{unit-name}.yaml` or `docs/reqstream/{system-name}/{unit-name}.yaml` + - Design: `docs/design/{system-name}/{subsystem-name}/{unit-name}.md` or `docs/design/{system-name}/{unit-name}.md` + - Source: `src/{SystemName}/{SubsystemName}/{UnitName}.cs` + - Tests: `test/{SystemName}.Tests/{SubsystemName}/{UnitName}Tests.cs` +- **Example**: `SomeSystem-Authentication-PasswordValidator`, `SomeSystem-DataLayer-ConfigParser` # Quality Checks @@ -155,6 +115,5 @@ Before submitting ReviewMark configuration, verify: - [ ] File paths reflect current naming conventions (kebab-case design/requirements folders, PascalCase source folders) - [ ] Evidence source properly configured (`none` for dev, `url` for production) - [ ] Environment variables used for credentials (never hardcoded) -- [ ] ReviewMark enforcement configured: `dotnet reviewmark --enforce` - [ ] Generated documents accessible for compliance auditing -- [ ] Review-set organization follows standard patterns ([Product]-[Unit], [Product]-Design, etc.) +- [ ] Review-set organization follows standard patterns ([System]-[Subsystem], [System]-Design, etc.) diff --git a/.github/standards/technical-documentation.md b/.github/standards/technical-documentation.md index c117aa2..5bcc937 100644 --- a/.github/standards/technical-documentation.md +++ b/.github/standards/technical-documentation.md @@ -24,36 +24,39 @@ consistency and tool compatibility: ```text docs/ - build_notes.md # Generated by BuildMark - build_notes/ # Auto-generated build notes - versions.md # Generated by VersionMark - code_review_plan/ # Auto-generated review plans - plan.md # Generated by ReviewMark - code_review_report/ # Auto-generated review reports - report.md # Generated by ReviewMark - design/ # Design documentation - introduction.md # Design overview - system.md # System architecture - {subsystem-name}/ # Subsystem design folder - {subsystem-name}.md # Subsystem-specific designs - {unit-name}.md # Unit-specific designs - {unit-name}.md # Top-level unit design - reqstream/ # Requirements source files - system.yaml # System requirements - platform-requirements.yaml # Platform requirements - {subsystem-name}/ # Subsystem requirements folder - {subsystem-name}.yaml # Subsystem requirements - {unit-name}.yaml # Unit requirements - ots/ # OTS requirement files - {ots-name}.yaml # OTS requirements - requirements_doc/ # Auto-generated requirements reports - requirements.md # Generated by ReqStream - justifications.md # Generated by ReqStream - requirements_report/ # Auto-generated trace matrices - trace_matrix.md # Generated by ReqStream - user_guide/ # User-facing documentation - introduction.md # User guide overview - {section}.md # User guide sections + build_notes.md # Generated by BuildMark + build_notes/ # Auto-generated build notes + versions.md # Generated by VersionMark + code_review_plan/ # Auto-generated review plans + plan.md # Generated by ReviewMark + code_review_report/ # Auto-generated review reports + report.md # Generated by ReviewMark + design/ # Design documentation + introduction.md # Design overview + {system-name}/ # System architecture folder + {system-name}.md # System architecture + {subsystem-name}/ # Subsystem design folder + {subsystem-name}.md # Subsystem-specific designs + {unit-name}.md # Unit-specific designs + {unit-name}.md # Top-level unit design + reqstream/ # Requirements source files + {system-name}/ # System requirements folder + {system-name}.yaml # System requirements + platform-requirements.yaml # Platform requirements + {subsystem-name}/ # Subsystem requirements folder + {subsystem-name}.yaml # Subsystem requirements + {unit-name}.yaml # Unit-specific requirements + {unit-name}.yaml # Top-level unit requirements + ots/ # OTS requirement files + {ots-name}.yaml # OTS requirements + requirements_doc/ # Auto-generated requirements reports + requirements.md # Generated by ReqStream + justifications.md # Generated by ReqStream + requirements_report/ # Auto-generated trace matrices + trace_matrix.md # Generated by ReqStream + user_guide/ # User-facing documentation + introduction.md # User guide overview + {section}.md # User guide sections ``` # Pandoc Document Structure (MANDATORY) diff --git a/.reviewmark.yaml b/.reviewmark.yaml index 3ad6044..bb0f42c 100644 --- a/.reviewmark.yaml +++ b/.reviewmark.yaml @@ -32,102 +32,120 @@ reviews: - id: ReviewMark-Program title: Review of Program software unit (main entry point and tool orchestration) paths: - - "docs/reqstream/unit-program.yaml" # requirements - - "docs/design/program.md" # design - - "docs/user_guide/introduction.md" # user guide - - "src/**/Program.cs" # implementation - - "test/**/ProgramTests.cs" # unit tests - - "test/**/TestDirectory.cs" # test infrastructure - - - id: ReviewMark-Context + - "docs/reqstream/review-mark/program.yaml" # requirements + - "docs/design/review-mark/program.md" # design + - "docs/user_guide/introduction.md" # user guide + - "src/**/Program.cs" # implementation + - "test/**/ProgramTests.cs" # unit tests + - "test/**/TestDirectory.cs" # test infrastructure + + - id: ReviewMark-Cli-Context title: Review of Context software unit (command-line argument handling) paths: - - "docs/reqstream/cli/unit-context.yaml" # requirements - - "docs/design/cli/context.md" # design - - "src/**/Cli/Context.cs" # implementation - - "test/**/Cli/ContextTests.cs" # tests + - "docs/reqstream/review-mark/cli/context.yaml" # requirements + - "docs/design/review-mark/cli/context.md" # design + - "src/**/Cli/Context.cs" # implementation + - "test/**/Cli/ContextTests.cs" # tests - - id: ReviewMark-ReviewMarkConfiguration + - id: ReviewMark-Configuration-ReviewMarkConfiguration title: Review of ReviewMarkConfiguration software unit (configuration parsing and processing) paths: - - "docs/reqstream/configuration/unit-review-mark-configuration.yaml" # requirements - - "docs/design/configuration/review-mark-configuration.md" # design - - "src/**/Configuration/ReviewMarkConfiguration.cs" # implementation - - "test/**/Configuration/ReviewMarkConfigurationTests.cs" # tests + - "docs/reqstream/review-mark/configuration/review-mark-configuration.yaml" # requirements + - "docs/design/review-mark/configuration/review-mark-configuration.md" # design + - "src/**/Configuration/ReviewMarkConfiguration.cs" # implementation + - "test/**/Configuration/ReviewMarkConfigurationTests.cs" # tests - - id: ReviewMark-GlobMatcher + - id: ReviewMark-Configuration-GlobMatcher title: Review of GlobMatcher software unit (file pattern matching) paths: - - "docs/reqstream/configuration/unit-glob-matcher.yaml" # requirements - - "docs/design/configuration/glob-matcher.md" # design - - "src/**/Configuration/GlobMatcher.cs" # implementation - - "test/**/Configuration/GlobMatcherTests.cs" # tests + - "docs/reqstream/review-mark/configuration/glob-matcher.yaml" # requirements + - "docs/design/review-mark/configuration/glob-matcher.md" # design + - "src/**/Configuration/GlobMatcher.cs" # implementation + - "test/**/Configuration/GlobMatcherTests.cs" # tests - - id: ReviewMark-ReviewIndex + - id: ReviewMark-Indexing-ReviewIndex title: Review of ReviewIndex software unit (review evidence indexing) paths: - - "docs/reqstream/indexing/unit-review-index.yaml" # requirements - - "docs/design/indexing/review-index.md" # design - - "src/**/Indexing/ReviewIndex.cs" # implementation - - "test/**/Indexing/IndexTests.cs" # tests + - "docs/reqstream/review-mark/indexing/review-index.yaml" # requirements + - "docs/design/review-mark/indexing/review-index.md" # design + - "src/**/Indexing/ReviewIndex.cs" # implementation + - "test/**/Indexing/IndexTests.cs" # tests - - id: ReviewMark-PathHelpers + - id: ReviewMark-Indexing-PathHelpers title: Review of PathHelpers software unit (file path utilities) paths: - - "docs/reqstream/indexing/unit-path-helpers.yaml" # requirements - - "docs/design/indexing/path-helpers.md" # design - - "src/**/Indexing/PathHelpers.cs" # implementation - - "test/**/Indexing/PathHelpersTests.cs" # tests + - "docs/reqstream/review-mark/indexing/path-helpers.yaml" # requirements + - "docs/design/review-mark/indexing/path-helpers.md" # design + - "src/**/Indexing/PathHelpers.cs" # implementation + - "test/**/Indexing/PathHelpersTests.cs" # tests - - id: ReviewMark-Validation + - id: ReviewMark-SelfTest-Validation title: Review of Validation software unit (self-validation test execution) paths: - - "docs/reqstream/self-test/unit-validation.yaml" # requirements - - "docs/design/self-test/validation.md" # design - - "src/**/SelfTest/Validation.cs" # implementation - - "test/**/SelfTest/ValidationTests.cs" # tests + - "docs/reqstream/review-mark/self-test/validation.yaml" # requirements + - "docs/design/review-mark/self-test/validation.md" # design + - "src/**/SelfTest/Validation.cs" # implementation + - "test/**/SelfTest/ValidationTests.cs" # tests # Subsystem reviews - id: ReviewMark-Cli title: Review of Cli subsystem (command-line interface) paths: - - "docs/reqstream/cli/subsystem-cli.yaml" # subsystem requirements - - "docs/design/cli/context.md" # Context design - - "docs/design/program.md" # Program design + - "docs/reqstream/review-mark/cli/cli.yaml" # subsystem requirements + - "docs/design/review-mark/cli/cli.md" # Cli subsystem design + - "docs/design/review-mark/cli/context.md" # Context design + - "docs/design/review-mark/program.md" # Program design + - "test/**/Cli/ContextTests.cs" # Context unit tests + - "test/**/ProgramTests.cs" # Program unit tests + + - id: ReviewMark-Configuration + title: Review of Configuration subsystem (configuration parsing and file pattern matching) + paths: + - "docs/reqstream/review-mark/configuration/configuration.yaml" # subsystem requirements + - "docs/design/review-mark/configuration/configuration.md" # Configuration subsystem design + - "docs/design/review-mark/configuration/review-mark-configuration.md" # ReviewMarkConfiguration design + - "docs/design/review-mark/configuration/glob-matcher.md" # GlobMatcher design + - "test/**/Configuration/ReviewMarkConfigurationTests.cs" # ReviewMarkConfiguration tests + - "test/**/Configuration/GlobMatcherTests.cs" # GlobMatcher tests - id: ReviewMark-Indexing title: Review of Indexing subsystem (review evidence loading and path utilities) paths: - - "docs/reqstream/indexing/subsystem-indexing.yaml" # subsystem requirements - - "docs/design/indexing/review-index.md" # ReviewIndex design - - "docs/design/indexing/path-helpers.md" # PathHelpers design + - "docs/reqstream/review-mark/indexing/indexing.yaml" # subsystem requirements + - "docs/design/review-mark/indexing/indexing.md" # Indexing subsystem design + - "docs/design/review-mark/indexing/review-index.md" # ReviewIndex design + - "docs/design/review-mark/indexing/path-helpers.md" # PathHelpers design + - "test/**/Indexing/IndexTests.cs" # ReviewIndex tests + - "test/**/Indexing/PathHelpersTests.cs" # PathHelpers tests - id: ReviewMark-SelfTest title: Review of SelfTest subsystem (self-validation) paths: - - "docs/reqstream/self-test/subsystem-self-test.yaml" # subsystem requirements - - "docs/design/self-test/validation.md" # Validation design + - "docs/reqstream/review-mark/self-test/self-test.yaml" # subsystem requirements + - "docs/design/review-mark/self-test/self-test.md" # SelfTest subsystem design + - "docs/design/review-mark/self-test/validation.md" # Validation design + - "test/**/SelfTest/ValidationTests.cs" # Validation tests # Special review-sets - - id: ReviewMark-System + - id: ReviewMark-Architecture title: Review of ReviewMark system-level behavior, platform support, and integration paths: - - "docs/reqstream/reviewmark-system.yaml" # system requirements - - "docs/reqstream/platform-requirements.yaml" # platform requirements - - "docs/design/introduction.md" # design introduction and architecture - - "docs/design/system.md" # system design - - "test/**/IntegrationTests.cs" # integration tests - - "test/**/Runner.cs" # test infrastructure - - "test/**/AssemblyInfo.cs" # test infrastructure + - "docs/reqstream/review-mark/review-mark.yaml" # system requirements + - "docs/reqstream/review-mark/platform-requirements.yaml" # platform requirements + - "docs/design/introduction.md" # design introduction and architecture + - "docs/design/review-mark/review-mark.md" # system design + - "test/**/IntegrationTests.cs" # integration tests + - "test/**/Runner.cs" # test infrastructure + - "test/**/AssemblyInfo.cs" # test infrastructure - id: ReviewMark-Design title: Review of all ReviewMark design documentation paths: - - "docs/reqstream/platform-requirements.yaml" # platform requirements - - "docs/design/**/*.md" # all design documents + - "docs/reqstream/review-mark/platform-requirements.yaml" # platform requirements + - "docs/design/**/*.md" # all design documents - id: ReviewMark-AllRequirements title: Review of all ReviewMark requirements files paths: - - "requirements.yaml" # root requirements file - - "docs/reqstream/**/*.yaml" # all requirements files + - "requirements.yaml" # root requirements file + - "docs/reqstream/**/*.yaml" # all requirements files diff --git a/AGENTS.md b/AGENTS.md index c884c2e..87fc5c7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,43 @@ Comprehensive guidance for AI agents working on repositories following Continuous Compliance practices. -## Standards Application (ALL Agents Must Follow) +# Project Structure + +The following is the basic folder structure of the project. Agents should use this information when searching for +existing files and to know where to make new files. + +```text +├── docs/ +│ ├── build_notes/ +│ ├── code_quality/ +│ ├── code_review_plan/ +│ ├── code_review_report/ +│ ├── design/ +│ ├── requirements_doc/ +│ ├── requirements_report/ +│ └── reqstream/ +├── src/ +│ └── / +└── test/ + └── / +``` + +# Key Configuration Files + +- **`.config/dotnet-tools.json`** - Local tool manifest for Continuous Compliance tools +- **`.editorconfig`** - Code formatting rules +- **`.clang-format`** - C/C++ formatting (if applicable) +- **`.cspell.yaml`** - Spell-check configuration and technical term dictionary +- **`.markdownlint-cli2.yaml`** - Markdown linting rules +- **`.yamllint.yaml`** - YAML linting configuration +- **`.reviewmark.yaml`** - File review definitions and tracking +- **`nuget.config`** - NuGet package sources (if .NET) +- **`package.json`** - Node.js dependencies for linting tools +- **`requirements.yaml`** - Root requirements file with includes +- **`pip-requirements.txt`** - Python dependencies for yamllint +- **`lint.sh` / `lint.bat`** - Cross-platform comprehensive linting scripts + +# Standards Application (ALL Agents Must Follow) Before performing any work, agents must read and apply the relevant standards from `.github/standards/`: @@ -17,24 +53,23 @@ Before performing any work, agents must read and apply the relevant standards fr Load only the standards relevant to your specific task scope and apply their quality checks and guidelines throughout your work. -## Agent Delegation Guidelines +# Agent Delegation Guidelines The default agent should handle simple, straightforward tasks directly. Delegate to specialized agents only for specific scenarios: -- **Light development work** (small fixes, simple features) → Call developer agent -- **Light quality checking** (linting, basic validation) → Call quality agent -- **Formal feature implementation** (complex, multi-step) → Call the `implementation` agent -- **Formal bug resolution** (complex debugging, systematic fixes) → Call the `implementation` agent -- **Formal reviews** (compliance verification, detailed analysis) → Call code-review agent -- **Template consistency** (downstream repository alignment) → Call repo-consistency agent +- **Light development work** (small fixes, simple features) → Call the developer agent +- **Light quality checking** (linting, basic validation) → Call the quality agent +- **Formal feature implementation** (complex, multi-step) → Call the implementation agent +- **Formal bug resolution** (complex debugging, systematic fixes) → Call the implementation agent +- **Formal reviews** (compliance verification, detailed analysis) → Call the code-review agent +- **Template consistency** (downstream repository alignment) → Call the repo-consistency agent ## Available Specialized Agents -- **code-review** - Agent for performing formal reviews using standardized - review processes -- **developer** - General-purpose software development agent that applies - appropriate standards based on the work being performed +- **developer** - General-purpose software development agent that applies appropriate + standards based on the work being performed +- **code-review** - Agent for performing formal reviews using standardized review processes - **implementation** - Orchestrator agent that manages quality implementations through a formal state machine workflow - **quality** - Quality assurance agent that grades developer work against DEMA @@ -42,129 +77,49 @@ Delegate to specialized agents only for specific scenarios: - **repo-consistency** - Ensures downstream repositories remain consistent with the TemplateDotNetTool template patterns and best practices -## Quality Gate Enforcement (ALL Agents Must Verify) +# Linting (Required Before Quality Gates) + +1. **Markdown Auto-fix**: `npx markdownlint-cli2 --fix **/*.md` (fixes most markdown issues except line length) +2. **Dotnet Auto-fix**: `dotnet format` (reformats .NET languages) +3. **Run full check**: `lint.bat` (Windows) or `lint.sh` (Unix) +4. **Fix remaining**: Address line length, spelling, YAML syntax manually +5. **Verify clean**: Re-run until 0 errors before quality validation + +## Linting Tools (ALL Must Pass) + +- **markdownlint-cli2**: Markdown style and formatting enforcement +- **cspell**: Spell-checking across all text files (use `.cspell.yaml` for technical terms) +- **yamllint**: YAML structure and formatting validation +- **Language-specific linters**: Based on repository technology stack + +# Quality Gate Enforcement (ALL Agents Must Verify) Configuration files and scripts are self-documenting with their design intent and modification policies in header comments. -1. **Linting Standards**: `./lint.sh` (Unix) or `lint.bat` (Windows) - comprehensive linting suite -2. **Build Quality**: Zero warnings (`TreatWarningsAsErrors=true`) -3. **Static Analysis**: SonarQube/CodeQL passing with no blockers -4. **Requirements Traceability**: `dotnet reqstream --enforce` passing -5. **Test Coverage**: All requirements linked to passing tests -6. **Documentation Currency**: All docs current and generated -7. **File Review Status**: All reviewable files have current reviews +1. **Build Quality**: Zero warnings (`TreatWarningsAsErrors=true`) +2. **Static Analysis**: SonarQube/CodeQL passing with no blockers +3. **Requirements Traceability**: `dotnet reqstream --enforce` passing +4. **Test Coverage**: All requirements linked to passing tests +5. **Documentation Currency**: All docs current and generated +6. **File Review Status**: All reviewable files have current reviews -## Continuous Compliance Overview +# Continuous Compliance Overview This repository follows the DEMA Consulting Continuous Compliance approach, which enforces quality and compliance gates on every CI/CD run instead of as a last-mile activity. -### Core Principles +## Core Principles - **Requirements Traceability**: Every requirement MUST link to passing tests - **Quality Gates**: All quality checks must pass before merge - **Documentation Currency**: All docs auto-generated and kept current - **Automated Evidence**: Full audit trail generated with every build -## Required Compliance Tools - -### Linting Tools (ALL Must Pass) - -- **markdownlint-cli2**: Markdown style and formatting enforcement -- **cspell**: Spell-checking across all text files (use `.cspell.yaml` for technical terms) -- **yamllint**: YAML structure and formatting validation -- **Language-specific linters**: Based on repository technology stack - -### Quality Analysis - -- **SonarQube/SonarCloud**: Code quality and security analysis -- **CodeQL**: Security vulnerability scanning (produces SARIF output) -- **Static analyzers**: Microsoft.CodeAnalysis.NetAnalyzers, SonarAnalyzer.CSharp, etc. - -### Requirements & Compliance +## Requirements & Compliance - **ReqStream**: Requirements traceability enforcement (`dotnet reqstream --enforce`) - **ReviewMark**: File review status enforcement - **BuildMark**: Tool version documentation - **VersionMark**: Version tracking across CI/CD jobs - -## Project Structure Template - -- `docs/` - Documentation and compliance artifacts - - `design/` - Detailed design documents - - `introduction.md` - System/Subsystem/Unit breakdown for this repository - - `reqstream/` - Subsystem requirements YAML files (included by root requirements.yaml) - - Auto-generated reports (requirements, justifications, trace matrix) -- `src/{ProjectName}/` - Source code projects -- `test/{ProjectName}.Tests/` - Test projects -- `.github/workflows/` - CI/CD pipeline definitions (build.yaml, build_on_push.yaml, release.yaml) -- Configuration files: `.editorconfig`, `.clang-format`, `nuget.config`, `.reviewmark.yaml`, etc. - -## Key Configuration Files - -### Essential Files (Repository-Specific) - -- **`lint.sh` / `lint.bat`** - Cross-platform comprehensive linting scripts -- **`.editorconfig`** - Code formatting rules -- **`.clang-format`** - C/C++ formatting (if applicable) -- **`.cspell.yaml`** - Spell-check configuration and technical term dictionary -- **`.markdownlint-cli2.yaml`** - Markdown linting rules -- **`.yamllint.yaml`** - YAML linting configuration -- **`nuget.config`** - NuGet package sources (if .NET) -- **`package.json`** - Node.js dependencies for linting tools - -### Compliance Files - -- **`requirements.yaml`** - Root requirements file with includes -- **`.reviewmark.yaml`** - File review definitions and tracking -- CI/CD pipeline files with quality gate enforcement - -## Continuous Compliance Workflow - -### CI/CD Pipeline Stages (Standard) - -1. **Lint**: `./lint.sh` or `lint.bat` - comprehensive linting suite -2. **Build**: Compile with warnings as errors -3. **Analyze**: SonarQube/SonarCloud, CodeQL security scanning -4. **Test**: Execute all tests, generate coverage reports -5. **Validate**: Tool self-validation tests -6. **Document**: Generate requirements reports, trace matrix, build notes -7. **Enforce**: Requirements traceability, file review status -8. **Publish**: Generate final documentation (Pandoc → PDF) - -### Quality Gate Enforcement - -All stages must pass before merge. Pipeline fails immediately on: - -- Any linting errors -- Build warnings or errors -- Security vulnerabilities (CodeQL) -- Requirements without test coverage -- Outdated file reviews -- Missing documentation - -## Continuous Compliance Requirements - -This repository follows continuous compliance practices from DEMA Consulting -Continuous Compliance . - -### Core Requirements Traceability Rules - -- **ALL requirements MUST be linked to tests** - Enforced in CI via `dotnet reqstream --enforce` -- **NOT all tests need requirement links** - Tests may exist for corner cases, design validation, failure scenarios -- **Source filters are critical** - Platform/framework requirements need specific test evidence - -For detailed requirements format, test linkage patterns, and ReqStream -integration, call the developer agent with requirements management context. - -## Agent Report Files - -Upon completion, create a report file at `.agent-logs/{agent-name}-{subject}-{unique-id}.md` that includes: - -- A concise summary of the work performed -- Any important decisions made and their rationale -- Follow-up items, open questions, or TODOs - -Store agent logs in the `.agent-logs/` folder so they are ignored via `.gitignore` and excluded from linting and commits. diff --git a/docs/design/definition.yaml b/docs/design/definition.yaml index 2a99aa1..23b5011 100644 --- a/docs/design/definition.yaml +++ b/docs/design/definition.yaml @@ -1,23 +1,28 @@ --- resource-path: - docs/design - - docs/design/cli - - docs/design/configuration - - docs/design/indexing - - docs/design/self-test + - docs/design/review-mark + - docs/design/review-mark/cli + - docs/design/review-mark/configuration + - docs/design/review-mark/indexing + - docs/design/review-mark/self-test - docs/template input-files: - docs/design/title.txt - docs/design/introduction.md - - docs/design/system.md - - docs/design/program.md - - docs/design/cli/context.md - - docs/design/configuration/glob-matcher.md - - docs/design/configuration/review-mark-configuration.md - - docs/design/indexing/review-index.md - - docs/design/indexing/path-helpers.md - - docs/design/self-test/validation.md + - docs/design/review-mark/review-mark.md + - docs/design/review-mark/program.md + - docs/design/review-mark/cli/cli.md + - docs/design/review-mark/cli/context.md + - docs/design/review-mark/configuration/configuration.md + - docs/design/review-mark/configuration/glob-matcher.md + - docs/design/review-mark/configuration/review-mark-configuration.md + - docs/design/review-mark/indexing/indexing.md + - docs/design/review-mark/indexing/review-index.md + - docs/design/review-mark/indexing/path-helpers.md + - docs/design/review-mark/self-test/self-test.md + - docs/design/review-mark/self-test/validation.md template: template.html diff --git a/docs/design/indexing/path-helpers.md b/docs/design/indexing/path-helpers.md deleted file mode 100644 index 942cae2..0000000 --- a/docs/design/indexing/path-helpers.md +++ /dev/null @@ -1,35 +0,0 @@ -# PathHelpers - -## Purpose - -The `PathHelpers` software unit provides safe path construction utilities that -prevent path traversal attacks. It is used by the Index subsystem when constructing -file system paths to evidence PDF files referenced in the evidence index. - -## SafePathCombine() - -`PathHelpers.SafePathCombine(basePath, relativePath)` combines a trusted base path -with an untrusted relative path from the evidence index, validating that the result -does not escape the base directory. - -The validation steps are: - -1. Reject any relative path that contains `..` segments (explicit traversal attempt). -2. Reject any relative path that is rooted (absolute path supplied where a relative one is required). -3. Combine the base path and relative path. -4. Verify that the combined path still begins with the base path (catches edge cases - such as platform-specific path normalization that might otherwise bypass the - earlier checks). -5. Return the combined path. - -The double-check strategy (pre-validation of segments plus post-combination -verification) defends against edge cases such as URL-encoded separators or -platform-specific path normalization that might otherwise bypass a single check. - -## Security Rationale - -Evidence index files may be loaded from external sources (file shares or URLs). -The `file` field in each index record is supplied by the evidence store and must -be treated as untrusted input. Without path validation, a maliciously crafted -index could direct the tool to read or reference files outside the intended -evidence directory. `SafePathCombine` eliminates this attack surface. diff --git a/docs/design/introduction.md b/docs/design/introduction.md index d48b144..e8a2c2b 100644 --- a/docs/design/introduction.md +++ b/docs/design/introduction.md @@ -73,6 +73,30 @@ src/DemaConsulting.ReviewMark/ The test project mirrors the same layout under `test/DemaConsulting.ReviewMark.Tests/`. +The design documentation follows the same hierarchy under `docs/design/review-mark/`: + +```text +docs/design/ +├── introduction.md — this document (software structure and folder layout) +└── review-mark/ + ├── review-mark.md — system-level design + ├── program.md — Program unit design + ├── cli/ + │ ├── cli.md — Cli subsystem overview + │ └── context.md — Context unit design + ├── configuration/ + │ ├── configuration.md — Configuration subsystem overview + │ ├── review-mark-configuration.md — ReviewMarkConfiguration unit design + │ └── glob-matcher.md — GlobMatcher unit design + ├── indexing/ + │ ├── indexing.md — Indexing subsystem overview + │ ├── review-index.md — ReviewIndex unit design + │ └── path-helpers.md — PathHelpers unit design + └── self-test/ + ├── self-test.md — SelfTest subsystem overview + └── validation.md — Validation unit design +``` + ## Document Conventions Throughout this document: diff --git a/docs/design/review-mark/cli/cli.md b/docs/design/review-mark/cli/cli.md new file mode 100644 index 0000000..378ca33 --- /dev/null +++ b/docs/design/review-mark/cli/cli.md @@ -0,0 +1,21 @@ +# Cli Subsystem + +## Overview + +The Cli subsystem is responsible for parsing and owning the command-line interface of +ReviewMark. It exposes a single software unit — Context — that processes the raw +`string[] args` array into a structured set of properties consumed by the rest of the +tool. + +## Responsibilities + +- Parse all supported command-line flags and arguments into a typed `Context` object +- Validate that no unrecognized arguments are supplied +- Own the output channels (stdout and optional log file) and the process exit code +- Propagate the `--silent` flag to suppress non-error output + +## Units + +| Unit | Source File | Purpose | +|---------|--------------------------|----------------------------------------------| +| Context | `Cli/Context.cs` | Command-line argument parser and I/O owner | diff --git a/docs/design/cli/context.md b/docs/design/review-mark/cli/context.md similarity index 100% rename from docs/design/cli/context.md rename to docs/design/review-mark/cli/context.md diff --git a/docs/design/review-mark/configuration/configuration.md b/docs/design/review-mark/configuration/configuration.md new file mode 100644 index 0000000..47d8df8 --- /dev/null +++ b/docs/design/review-mark/configuration/configuration.md @@ -0,0 +1,22 @@ +# Configuration Subsystem + +## Overview + +The Configuration subsystem is responsible for loading, validating, and processing the +ReviewMark YAML configuration file (`.reviewmark.yaml`). It also provides the +file-pattern-matching capability used to resolve glob patterns into concrete file lists. + +## Responsibilities + +- Deserialize `.reviewmark.yaml` into a strongly-typed configuration model +- Lint the loaded configuration and report any structural errors or warnings +- Resolve `needs-review` and per-review-set `paths` glob patterns into sorted file lists +- Compute SHA-256 fingerprints across resolved file sets +- Generate Review Plan and Review Report markdown documents + +## Units + +| Unit | Source File | Purpose | +| --- | --- | --- | +| ReviewMarkConfiguration | `Configuration/ReviewMarkConfiguration.cs` | YAML parser and review-set processor | +| GlobMatcher | `Configuration/GlobMatcher.cs` | File pattern matching using glob syntax | diff --git a/docs/design/configuration/glob-matcher.md b/docs/design/review-mark/configuration/glob-matcher.md similarity index 100% rename from docs/design/configuration/glob-matcher.md rename to docs/design/review-mark/configuration/glob-matcher.md diff --git a/docs/design/configuration/review-mark-configuration.md b/docs/design/review-mark/configuration/review-mark-configuration.md similarity index 100% rename from docs/design/configuration/review-mark-configuration.md rename to docs/design/review-mark/configuration/review-mark-configuration.md diff --git a/docs/design/review-mark/indexing/indexing.md b/docs/design/review-mark/indexing/indexing.md new file mode 100644 index 0000000..c971a5d --- /dev/null +++ b/docs/design/review-mark/indexing/indexing.md @@ -0,0 +1,21 @@ +# Indexing Subsystem + +## Overview + +The Indexing subsystem is responsible for loading review evidence from an external index +and for safe file-path manipulation. It provides the lookup engine that determines whether +each review-set is Current, Stale, Missing, or Failed. + +## Responsibilities + +- Load the evidence index from a `none`, `fileshare`, or `url` source +- Scan a set of PDF files, extract structured metadata from the Keywords field, and + produce an `index.json` evidence index +- Provide safe path-combination utilities that prevent directory-traversal attacks + +## Units + +| Unit | Source File | Purpose | +|---------------|--------------------------------|------------------------------------------------------| +| ReviewIndex | `Indexing/ReviewIndex.cs` | Review evidence loader and query engine | +| PathHelpers | `Indexing/PathHelpers.cs` | File path utilities (safe path combination) | diff --git a/docs/design/review-mark/indexing/path-helpers.md b/docs/design/review-mark/indexing/path-helpers.md new file mode 100644 index 0000000..ae98cfe --- /dev/null +++ b/docs/design/review-mark/indexing/path-helpers.md @@ -0,0 +1,54 @@ +# PathHelpers Design + +## Overview + +`PathHelpers` is a static utility class that provides a safe path-combination method. It +protects callers against path-traversal attacks by verifying the resolved combined path stays +within the base directory. Note that `Path.GetFullPath` normalizes `.`/`..` segments but does +not resolve symlinks or reparse points, so this check guards against string-level traversal +only. + +## Class Structure + +### SafePathCombine Method + +```csharp +internal static string SafePathCombine(string basePath, string relativePath) +``` + +Combines `basePath` and `relativePath` safely, ensuring the resulting path remains within +the base directory. + +**Validation steps:** + +1. Reject null inputs via `ArgumentNullException.ThrowIfNull`. +2. Combine the paths with `Path.Combine` to produce the candidate path (preserving the + caller's relative/absolute style). +3. Resolve both `basePath` and the candidate to absolute form with `Path.GetFullPath`. +4. Compute `Path.GetRelativePath(absoluteBase, absoluteCombined)` and reject the input if + the result is exactly `".."`, starts with `".."` followed by `Path.DirectorySeparatorChar` + or `Path.AltDirectorySeparatorChar`, or is itself rooted (absolute), which would indicate + the combined path escapes the base directory. + +## Design Decisions + +- **`Path.GetRelativePath` for containment check**: Using `GetRelativePath` to verify + containment handles root paths (e.g. `/`, `C:\`), platform case-sensitivity, and + directory-separator normalization natively. The containment test should treat `..` as an + escaping segment only when it is the entire relative result or is followed by a directory + separator, avoiding false positives for valid in-base names such as `..data`. +- **Post-combine canonical-path check**: Resolving paths after combining handles all traversal + patterns — `../`, embedded `/../`, absolute-path overrides, and platform edge cases — + without fragile pre-combine string inspection of `relativePath`. +- **ArgumentException on invalid input**: Callers receive a specific `ArgumentException` + identifying `relativePath` as the problematic parameter, making debugging straightforward. +- **No logging or error accumulation**: `SafePathCombine` is a pure utility method that throws + on invalid input; it does not interact with the `Context` or any output mechanism. + +## Security Rationale + +Evidence index files may be loaded from external sources (file shares or URLs). +The `file` field in each index record is supplied by the evidence store and must +be treated as untrusted input. Without path validation, a maliciously crafted +index could direct the tool to read or reference files outside the intended +evidence directory. `SafePathCombine` eliminates this attack surface. diff --git a/docs/design/indexing/review-index.md b/docs/design/review-mark/indexing/review-index.md similarity index 100% rename from docs/design/indexing/review-index.md rename to docs/design/review-mark/indexing/review-index.md diff --git a/docs/design/program.md b/docs/design/review-mark/program.md similarity index 100% rename from docs/design/program.md rename to docs/design/review-mark/program.md diff --git a/docs/design/system.md b/docs/design/review-mark/review-mark.md similarity index 100% rename from docs/design/system.md rename to docs/design/review-mark/review-mark.md diff --git a/docs/design/review-mark/self-test/self-test.md b/docs/design/review-mark/self-test/self-test.md new file mode 100644 index 0000000..891c081 --- /dev/null +++ b/docs/design/review-mark/self-test/self-test.md @@ -0,0 +1,20 @@ +# SelfTest Subsystem + +## Overview + +The SelfTest subsystem provides a self-validation framework that allows ReviewMark to +qualify itself as a tool for use in regulated environments. It executes a built-in suite +of integration tests against a temporary working directory and reports the results. + +## Responsibilities + +- Orchestrate the execution of the built-in validation test suite +- Write test results to a TRX or JUnit XML file for ingestion by CI pipelines +- Output a human-readable summary table to the console +- Set the process exit code to reflect overall pass/fail status + +## Units + +| Unit | Source File | Purpose | +|------------|---------------------------|--------------------------------------------------| +| Validation | `SelfTest/Validation.cs` | Self-validation test runner | diff --git a/docs/design/self-test/validation.md b/docs/design/review-mark/self-test/validation.md similarity index 100% rename from docs/design/self-test/validation.md rename to docs/design/review-mark/self-test/validation.md diff --git a/docs/reqstream/cli/subsystem-cli.yaml b/docs/reqstream/review-mark/cli/cli.yaml similarity index 100% rename from docs/reqstream/cli/subsystem-cli.yaml rename to docs/reqstream/review-mark/cli/cli.yaml diff --git a/docs/reqstream/cli/unit-context.yaml b/docs/reqstream/review-mark/cli/context.yaml similarity index 100% rename from docs/reqstream/cli/unit-context.yaml rename to docs/reqstream/review-mark/cli/context.yaml diff --git a/docs/reqstream/review-mark/configuration/configuration.yaml b/docs/reqstream/review-mark/configuration/configuration.yaml new file mode 100644 index 0000000..9e974cc --- /dev/null +++ b/docs/reqstream/review-mark/configuration/configuration.yaml @@ -0,0 +1,61 @@ +--- +# Configuration Subsystem Requirements +# +# PURPOSE: +# - Define requirements for the ReviewMark Configuration subsystem +# - The Configuration subsystem spans ReviewMarkConfiguration.cs (config loading and processing) +# and GlobMatcher.cs (file pattern matching) +# - Subsystem requirements describe the externally visible configuration capabilities + +sections: + - title: Configuration Subsystem Requirements + requirements: + - id: ReviewMark-Configuration-NeedsReview + title: The tool shall identify all files requiring review by resolving needs-review glob patterns. + justification: | + Users configure which files require review using glob patterns. The Configuration + subsystem must resolve these patterns to a concrete list of files, applying includes + and excludes in declaration order, so that ReviewMark can detect uncovered files + and generate accurate review plans. + tests: + - ReviewMarkConfiguration_GetNeedsReviewFiles_ReturnsMatchingFiles + + - id: ReviewMark-Configuration-Fingerprinting + title: The tool shall compute SHA-256 fingerprints for review-sets to detect file changes. + justification: | + Review-set fingerprints are the mechanism by which ReviewMark detects that files + have changed since the last review. The SHA-256 fingerprint must be based on file + content rather than names alone, so that renamed files do not invalidate the + fingerprint, and changed content always produces a new fingerprint. + tests: + - ReviewSet_GetFingerprint_SameContent_ReturnsSameFingerprint + - ReviewSet_GetFingerprint_DifferentContent_ReturnsDifferentFingerprint + - ReviewSet_GetFingerprint_RenameFile_ReturnsSameFingerprint + + - id: ReviewMark-Configuration-PlanGeneration + title: The tool shall generate a Review Plan Markdown document listing review-set coverage. + justification: | + The Review Plan is a compliance artifact that documents which review-sets exist + and what files they cover. It enables auditors to verify that all relevant files + are included in at least one review-set before reviews are conducted. + tests: + - ReviewMark_ReviewPlanGeneration + + - id: ReviewMark-Configuration-ReportGeneration + title: The tool shall generate a Review Report Markdown document showing review-set status. + justification: | + The Review Report is a compliance artifact that documents the current review status + of each review-set (Current, Stale, Missing, or Failed), enabling auditors to + confirm that all review-sets have current evidence before a release. + tests: + - ReviewMark_ReviewReportGeneration + + - id: ReviewMark-Configuration-Elaboration + title: The tool shall elaborate a review-set by providing its ID, fingerprint, and file list. + justification: | + When preparing for a code review, the reviewer needs the review set ID, its current + fingerprint, and the full sorted list of files to be reviewed. The elaboration + command provides this formatted as Markdown so it can be copied directly into + review documentation. + tests: + - ReviewMark_Elaborate diff --git a/docs/reqstream/configuration/unit-glob-matcher.yaml b/docs/reqstream/review-mark/configuration/glob-matcher.yaml similarity index 100% rename from docs/reqstream/configuration/unit-glob-matcher.yaml rename to docs/reqstream/review-mark/configuration/glob-matcher.yaml diff --git a/docs/reqstream/configuration/subsystem-configuration.yaml b/docs/reqstream/review-mark/configuration/review-mark-configuration.yaml similarity index 91% rename from docs/reqstream/configuration/subsystem-configuration.yaml rename to docs/reqstream/review-mark/configuration/review-mark-configuration.yaml index 80c5b92..f4e9545 100644 --- a/docs/reqstream/configuration/subsystem-configuration.yaml +++ b/docs/reqstream/review-mark/configuration/review-mark-configuration.yaml @@ -10,7 +10,9 @@ sections: - title: ReviewMarkConfiguration Unit Requirements requirements: - id: ReviewMark-Config-Reading - title: The tool shall read and parse the .reviewmark.yaml file into an in-memory configuration model. + title: >- + ReviewMarkConfiguration shall read and parse the .reviewmark.yaml file into an in-memory + configuration model. justification: | Enables the tool to read its configuration from the standard `.reviewmark.yaml` file, exposing needs-review patterns, evidence source, and review set definitions. Review sets @@ -29,7 +31,7 @@ sections: - ReviewMarkConfiguration_Load_NonExistentFile_ReturnsNullConfigWithErrorIssue - ReviewMarkConfiguration_Load_FileshareRelativeLocation_ResolvesToAbsolutePath - id: ReviewMark-Config-Loading - title: The Load mechanism shall perform linting and return both the configuration and lint issues. + title: ReviewMarkConfiguration.Load shall perform linting and return both the configuration and lint issues. justification: | Enables a single-pass loading mechanism that combines configuration parsing and linting, returning a ReviewMarkLoadResult with both the configuration (or null on error) and diff --git a/docs/reqstream/indexing/subsystem-indexing.yaml b/docs/reqstream/review-mark/indexing/indexing.yaml similarity index 100% rename from docs/reqstream/indexing/subsystem-indexing.yaml rename to docs/reqstream/review-mark/indexing/indexing.yaml diff --git a/docs/reqstream/indexing/unit-path-helpers.yaml b/docs/reqstream/review-mark/indexing/path-helpers.yaml similarity index 75% rename from docs/reqstream/indexing/unit-path-helpers.yaml rename to docs/reqstream/review-mark/indexing/path-helpers.yaml index f9295bc..9193174 100644 --- a/docs/reqstream/indexing/unit-path-helpers.yaml +++ b/docs/reqstream/review-mark/indexing/path-helpers.yaml @@ -14,9 +14,10 @@ sections: justification: | When constructing file paths from user-supplied or externally-sourced components (such as relative paths read from an evidence index), the tool must prevent path - traversal attacks. SafePathCombine validates that the relative path does not - contain '..' sequences or absolute path components, and performs a defense-in-depth - check that the resolved combined path remains under the base directory. + traversal attacks. SafePathCombine combines the paths and then resolves both to + absolute form, using Path.GetRelativePath to verify the combined path remains + within the base directory. This post-combine canonical-path check handles all + traversal patterns without fragile pre-combine string inspection. tests: - PathHelpers_SafePathCombine_ValidPaths_CombinesCorrectly - PathHelpers_SafePathCombine_PathTraversalWithDoubleDots_ThrowsArgumentException diff --git a/docs/reqstream/indexing/unit-review-index.yaml b/docs/reqstream/review-mark/indexing/review-index.yaml similarity index 100% rename from docs/reqstream/indexing/unit-review-index.yaml rename to docs/reqstream/review-mark/indexing/review-index.yaml diff --git a/docs/reqstream/platform-requirements.yaml b/docs/reqstream/review-mark/platform-requirements.yaml similarity index 100% rename from docs/reqstream/platform-requirements.yaml rename to docs/reqstream/review-mark/platform-requirements.yaml diff --git a/docs/reqstream/unit-program.yaml b/docs/reqstream/review-mark/program.yaml similarity index 100% rename from docs/reqstream/unit-program.yaml rename to docs/reqstream/review-mark/program.yaml diff --git a/docs/reqstream/reviewmark-system.yaml b/docs/reqstream/review-mark/review-mark.yaml similarity index 100% rename from docs/reqstream/reviewmark-system.yaml rename to docs/reqstream/review-mark/review-mark.yaml diff --git a/docs/reqstream/self-test/subsystem-self-test.yaml b/docs/reqstream/review-mark/self-test/self-test.yaml similarity index 100% rename from docs/reqstream/self-test/subsystem-self-test.yaml rename to docs/reqstream/review-mark/self-test/self-test.yaml diff --git a/docs/reqstream/self-test/unit-validation.yaml b/docs/reqstream/review-mark/self-test/validation.yaml similarity index 100% rename from docs/reqstream/self-test/unit-validation.yaml rename to docs/reqstream/review-mark/self-test/validation.yaml diff --git a/lint.bat b/lint.bat index c7440d4..433421b 100644 --- a/lint.bat +++ b/lint.bat @@ -2,7 +2,7 @@ setlocal REM Comprehensive Linting Script -REM +REM REM PURPOSE: REM - Run ALL lint checks when executed (no options or modes) REM - Output lint failures directly for agent parsing @@ -11,30 +11,55 @@ REM - Agents execute this script to identify files needing fixes set "LINT_ERROR=0" -REM Install npm dependencies -call npm install --silent +REM === PYTHON SECTION === + +REM Create python venv if necessary +if not exist ".venv\Scripts\activate.bat" python -m venv .venv +if errorlevel 1 goto skip_python -REM Create Python virtual environment (for yamllint) if missing -if not exist ".venv\Scripts\activate.bat" ( - python -m venv .venv -) +REM Activate python venv call .venv\Scripts\activate.bat +if errorlevel 1 goto skip_python + +REM Install python tools pip install -r pip-requirements.txt --quiet --disable-pip-version-check +if errorlevel 1 goto skip_python + +REM Run yamllint +yamllint . +if errorlevel 1 set "LINT_ERROR=1" +goto npm_section + +:skip_python +set "LINT_ERROR=1" + +REM === NPM SECTION === -REM Run spell check +:npm_section + +REM Install npm dependencies +call npm install --silent +if errorlevel 1 goto skip_npm + +REM Run cspell call npx cspell --no-progress --no-color --quiet "**/*.{md,yaml,yml,json,cs,cpp,hpp,h,txt}" if errorlevel 1 set "LINT_ERROR=1" -REM Run markdownlint check +REM Run markdownlint-cli2 call npx markdownlint-cli2 "**/*.md" if errorlevel 1 set "LINT_ERROR=1" +goto dotnet_section -REM Run yamllint check -yamllint . -if errorlevel 1 set "LINT_ERROR=1" +:skip_npm +set "LINT_ERROR=1" + +REM === DOTNET SECTION === + +:dotnet_section -REM Run .NET formatting check (verifies no changes are needed) +REM Run dotnet format dotnet format --verify-no-changes if errorlevel 1 set "LINT_ERROR=1" +REM Report result exit /b %LINT_ERROR% diff --git a/lint.sh b/lint.sh index c567e09..13ac584 100755 --- a/lint.sh +++ b/lint.sh @@ -1,7 +1,7 @@ #!/bin/bash # Comprehensive Linting Script -# +# # PURPOSE: # - Run ALL lint checks when executed (no options or modes) # - Output lint failures directly for agent parsing @@ -10,26 +10,47 @@ lint_error=0 -# Install npm dependencies -npm install --silent +# === PYTHON SECTION === -# Create Python virtual environment (for yamllint) +# Create python venv if necessary if [ ! -d ".venv" ]; then - python -m venv .venv + python -m venv .venv || { lint_error=1; skip_python=1; } +fi + +# Activate python venv +if [ "$skip_python" != "1" ]; then + source .venv/bin/activate || { lint_error=1; skip_python=1; } +fi + +# Install python tools +if [ "$skip_python" != "1" ]; then + pip install -r pip-requirements.txt --quiet --disable-pip-version-check || { lint_error=1; skip_python=1; } +fi + +# Run yamllint +if [ "$skip_python" != "1" ]; then + yamllint . || lint_error=1 fi -source .venv/bin/activate -pip install -r pip-requirements.txt --quiet --disable-pip-version-check -# Run spell check -npx cspell --no-progress --no-color --quiet "**/*.{md,yaml,yml,json,cs,cpp,hpp,h,txt}" || lint_error=1 +# === NPM SECTION === + +# Install npm dependencies +npm install --silent || { lint_error=1; skip_npm=1; } + +# Run cspell +if [ "$skip_npm" != "1" ]; then + npx cspell --no-progress --no-color --quiet "**/*.{md,yaml,yml,json,cs,cpp,hpp,h,txt}" || lint_error=1 +fi -# Run markdownlint check -npx markdownlint-cli2 "**/*.md" || lint_error=1 +# Run markdownlint-cli2 +if [ "$skip_npm" != "1" ]; then + npx markdownlint-cli2 "**/*.md" || lint_error=1 +fi -# Run yamllint check -yamllint . || lint_error=1 +# === DOTNET SECTION === -# Run .NET formatting check (verifies no changes are needed) +# Run dotnet format dotnet format --verify-no-changes || lint_error=1 +# Report result exit $lint_error diff --git a/requirements.yaml b/requirements.yaml index 133b3ed..4383a7d 100644 --- a/requirements.yaml +++ b/requirements.yaml @@ -24,18 +24,19 @@ # --- includes: - - docs/reqstream/reviewmark-system.yaml - - docs/reqstream/platform-requirements.yaml - - docs/reqstream/unit-program.yaml - - docs/reqstream/cli/subsystem-cli.yaml - - docs/reqstream/cli/unit-context.yaml - - docs/reqstream/configuration/subsystem-configuration.yaml - - docs/reqstream/configuration/unit-glob-matcher.yaml - - docs/reqstream/indexing/subsystem-indexing.yaml - - docs/reqstream/indexing/unit-review-index.yaml - - docs/reqstream/indexing/unit-path-helpers.yaml - - docs/reqstream/self-test/subsystem-self-test.yaml - - docs/reqstream/self-test/unit-validation.yaml + - docs/reqstream/review-mark/review-mark.yaml + - docs/reqstream/review-mark/platform-requirements.yaml + - docs/reqstream/review-mark/program.yaml + - docs/reqstream/review-mark/cli/cli.yaml + - docs/reqstream/review-mark/cli/context.yaml + - docs/reqstream/review-mark/configuration/configuration.yaml + - docs/reqstream/review-mark/configuration/review-mark-configuration.yaml + - docs/reqstream/review-mark/configuration/glob-matcher.yaml + - docs/reqstream/review-mark/indexing/indexing.yaml + - docs/reqstream/review-mark/indexing/review-index.yaml + - docs/reqstream/review-mark/indexing/path-helpers.yaml + - docs/reqstream/review-mark/self-test/self-test.yaml + - docs/reqstream/review-mark/self-test/validation.yaml - docs/reqstream/ots/ots-mstest.yaml - docs/reqstream/ots/ots-reqstream.yaml - docs/reqstream/ots/ots-buildmark.yaml diff --git a/src/DemaConsulting.ReviewMark/Indexing/PathHelpers.cs b/src/DemaConsulting.ReviewMark/Indexing/PathHelpers.cs index 22cc3d8..7527551 100644 --- a/src/DemaConsulting.ReviewMark/Indexing/PathHelpers.cs +++ b/src/DemaConsulting.ReviewMark/Indexing/PathHelpers.cs @@ -26,39 +26,37 @@ namespace DemaConsulting.ReviewMark.Indexing; internal static class PathHelpers { /// - /// Safely combines two paths, ensuring the second path doesn't contain path traversal sequences. + /// Safely combines two paths, ensuring the resolved combined path stays within the base directory. /// /// The base path. /// The relative path to combine. /// The combined path. - /// Thrown when relativePath contains invalid characters or path traversal sequences. + /// Thrown when or is . + /// + /// Thrown when the resolved combined path escapes the base directory, or when a supplied path is invalid. + /// + /// Thrown when a supplied path contains an unsupported format. + /// Thrown when the combined or resolved path exceeds the system-defined maximum length. internal static string SafePathCombine(string basePath, string relativePath) { // Validate inputs ArgumentNullException.ThrowIfNull(basePath); ArgumentNullException.ThrowIfNull(relativePath); - // Ensure the relative path doesn't contain path traversal sequences - if (relativePath.Contains("..") || Path.IsPathRooted(relativePath)) - { - throw new ArgumentException($"Invalid path component: {relativePath}", nameof(relativePath)); - } - - // This call to Path.Combine is safe because we've validated that: - // 1. relativePath doesn't contain ".." (path traversal) - // 2. relativePath is not an absolute path (IsPathRooted check) - // This ensures the combined path will always be under basePath + // Combine the paths (preserves the caller's relative/absolute style) var combinedPath = Path.Combine(basePath, relativePath); - // Additional security validation: ensure the combined path is still under the base path. - // This defense-in-depth approach protects against edge cases that might bypass the - // initial validation, ensuring the final path stays within the intended directory. - var fullBasePath = Path.GetFullPath(basePath); - var fullCombinedPath = Path.GetFullPath(combinedPath); + // Security check: resolve both paths to absolute form and verify the combined + // path is still inside the base directory. Path.GetRelativePath handles root + // paths, platform case-sensitivity, and directory-separator normalization natively. + var absoluteBase = Path.GetFullPath(basePath); + var absoluteCombined = Path.GetFullPath(combinedPath); + var checkRelative = Path.GetRelativePath(absoluteBase, absoluteCombined); - // Use GetRelativePath to verify the relationship between paths - var relativeCheck = Path.GetRelativePath(fullBasePath, fullCombinedPath); - if (relativeCheck.StartsWith("..") || Path.IsPathRooted(relativeCheck)) + if (string.Equals(checkRelative, "..", StringComparison.Ordinal) + || checkRelative.StartsWith(".." + Path.DirectorySeparatorChar, StringComparison.Ordinal) + || checkRelative.StartsWith(".." + Path.AltDirectorySeparatorChar, StringComparison.Ordinal) + || Path.IsPathRooted(checkRelative)) { throw new ArgumentException($"Invalid path component: {relativePath}", nameof(relativePath)); } From f658cc508e1fe512ce9099a9a92ae93752ddb02b Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Apr 2026 19:45:57 -0400 Subject: [PATCH 16/19] Fix documentation, requirements, and subsystem test suite issues identified in formal review (#44) * fix: address formal review issues in documentation and requirements files Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Add subsystem test suites and requirements children links - Add CliTests.cs: subsystem integration tests for CLI (Context + Program) - Add ConfigurationTests.cs: subsystem integration tests for Configuration - Add IndexingTests.cs: subsystem integration tests for Indexing - Add SelfTestTests.cs: subsystem integration tests for SelfTest - Update cli.yaml: add children links and new subsystem test references - Update configuration.yaml: add children links and new subsystem test references - Update indexing.yaml: add children links and new subsystem test references - Update self-test.yaml: add children links and new subsystem test references - Update .reviewmark.yaml: update subsystem review-sets to include new test files Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * fix: add 'selftest' to cspell word list Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * fix: shorten MD013-violating table row in review-mark-configuration.md Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/4e67c1ba-0bd1-4537-b878-2b85e42f4d22 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --- .cspell.yaml | 1 + .reviewmark.yaml | 18 +- docs/design/review-mark/cli/context.md | 2 +- .../review-mark-configuration.md | 12 +- .../review-mark/indexing/path-helpers.md | 2 +- .../review-mark/indexing/review-index.md | 47 +++-- .../review-mark/self-test/validation.md | 2 +- docs/reqstream/review-mark/cli/cli.yaml | 24 +++ .../configuration/configuration.yaml | 8 + .../configuration/glob-matcher.yaml | 2 + .../review-mark-configuration.yaml | 1 - .../review-mark/indexing/indexing.yaml | 10 + .../review-mark/indexing/path-helpers.yaml | 3 + .../review-mark/self-test/self-test.yaml | 4 + .../Cli/CliTests.cs | 148 ++++++++++++++ .../Configuration/ConfigurationTests.cs | 180 ++++++++++++++++++ .../Indexing/IndexingTests.cs | 145 ++++++++++++++ .../SelfTest/SelfTestTests.cs | 97 ++++++++++ 18 files changed, 668 insertions(+), 38 deletions(-) create mode 100644 test/DemaConsulting.ReviewMark.Tests/Cli/CliTests.cs create mode 100644 test/DemaConsulting.ReviewMark.Tests/Configuration/ConfigurationTests.cs create mode 100644 test/DemaConsulting.ReviewMark.Tests/Indexing/IndexingTests.cs create mode 100644 test/DemaConsulting.ReviewMark.Tests/SelfTest/SelfTestTests.cs diff --git a/.cspell.yaml b/.cspell.yaml index d5bd59c..d519dc9 100644 --- a/.cspell.yaml +++ b/.cspell.yaml @@ -82,6 +82,7 @@ words: - spdx - streetsidesoftware - testname + - selftest - trace_matrix - triaging - Trivy diff --git a/.reviewmark.yaml b/.reviewmark.yaml index bb0f42c..d071e17 100644 --- a/.reviewmark.yaml +++ b/.reviewmark.yaml @@ -93,38 +93,28 @@ reviews: paths: - "docs/reqstream/review-mark/cli/cli.yaml" # subsystem requirements - "docs/design/review-mark/cli/cli.md" # Cli subsystem design - - "docs/design/review-mark/cli/context.md" # Context design - - "docs/design/review-mark/program.md" # Program design - - "test/**/Cli/ContextTests.cs" # Context unit tests - - "test/**/ProgramTests.cs" # Program unit tests + - "test/**/Cli/CliTests.cs" # Cli subsystem tests - id: ReviewMark-Configuration title: Review of Configuration subsystem (configuration parsing and file pattern matching) paths: - "docs/reqstream/review-mark/configuration/configuration.yaml" # subsystem requirements - "docs/design/review-mark/configuration/configuration.md" # Configuration subsystem design - - "docs/design/review-mark/configuration/review-mark-configuration.md" # ReviewMarkConfiguration design - - "docs/design/review-mark/configuration/glob-matcher.md" # GlobMatcher design - - "test/**/Configuration/ReviewMarkConfigurationTests.cs" # ReviewMarkConfiguration tests - - "test/**/Configuration/GlobMatcherTests.cs" # GlobMatcher tests + - "test/**/Configuration/ConfigurationTests.cs" # Configuration subsystem tests - id: ReviewMark-Indexing title: Review of Indexing subsystem (review evidence loading and path utilities) paths: - "docs/reqstream/review-mark/indexing/indexing.yaml" # subsystem requirements - "docs/design/review-mark/indexing/indexing.md" # Indexing subsystem design - - "docs/design/review-mark/indexing/review-index.md" # ReviewIndex design - - "docs/design/review-mark/indexing/path-helpers.md" # PathHelpers design - - "test/**/Indexing/IndexTests.cs" # ReviewIndex tests - - "test/**/Indexing/PathHelpersTests.cs" # PathHelpers tests + - "test/**/Indexing/IndexingTests.cs" # Indexing subsystem tests - id: ReviewMark-SelfTest title: Review of SelfTest subsystem (self-validation) paths: - "docs/reqstream/review-mark/self-test/self-test.yaml" # subsystem requirements - "docs/design/review-mark/self-test/self-test.md" # SelfTest subsystem design - - "docs/design/review-mark/self-test/validation.md" # Validation design - - "test/**/SelfTest/ValidationTests.cs" # Validation tests + - "test/**/SelfTest/SelfTestTests.cs" # SelfTest subsystem tests # Special review-sets - id: ReviewMark-Architecture diff --git a/docs/design/review-mark/cli/context.md b/docs/design/review-mark/cli/context.md index 50827f0..e794389 100644 --- a/docs/design/review-mark/cli/context.md +++ b/docs/design/review-mark/cli/context.md @@ -20,7 +20,7 @@ arguments: | `Validate` | bool | Requests self-validation run | | `Lint` | bool | Requests configuration linting | | `ResultsFile` | string? | Path for TRX/JUnit test results output | -| `DefinitionFile` | string | Path to the `.reviewmark.yaml` configuration | +| `DefinitionFile` | string? | Path to the `.reviewmark.yaml` configuration | | `PlanFile` | string? | Output path for the Review Plan document | | `PlanDepth` | int | Heading depth for the Review Plan | | `ReportFile` | string? | Output path for the Review Report document | diff --git a/docs/design/review-mark/configuration/review-mark-configuration.md b/docs/design/review-mark/configuration/review-mark-configuration.md index b220af2..64fc688 100644 --- a/docs/design/review-mark/configuration/review-mark-configuration.md +++ b/docs/design/review-mark/configuration/review-mark-configuration.md @@ -17,6 +17,16 @@ The `.reviewmark.yaml` file is deserialized into the following model: | `EvidenceSourceYaml` | Describes how to locate the evidence index (`type`, `location`, optional `credentials`) | | `ReviewYaml` | Describes a single review-set (`id`, `title`, file patterns) | +### Evidence Source Types + +The `type` field of `EvidenceSourceYaml` controls how the evidence index is located: + +| Type | Description | +| ---- | ----------- | +| `none` | No evidence index. The `location` field is optional and ignored. All review-sets are reported as Missing. | +| `fileshare` | The evidence index is read from the file path specified in `location`. | +| `url` | The evidence index is downloaded from the HTTP or HTTPS URL specified in `location`. | + ## ReviewMarkConfiguration.Load() `ReviewMarkConfiguration.Load(filePath)` is the unified loading mechanism that performs @@ -38,7 +48,7 @@ The fingerprint for a review-set uniquely identifies the exact content of its fi The algorithm is: 1. For each file in the review-set, read its contents and compute a SHA-256 hash. -2. Collect all per-file hashes and sort them lexicographically. +2. Convert each hash to a lowercase hex string, then collect all per-file hashes and sort them lexicographically. 3. Concatenate the sorted hashes and compute a SHA-256 hash of the result. 4. Return the final hash as a hex string — this is the review-set fingerprint. diff --git a/docs/design/review-mark/indexing/path-helpers.md b/docs/design/review-mark/indexing/path-helpers.md index ae98cfe..b2aecad 100644 --- a/docs/design/review-mark/indexing/path-helpers.md +++ b/docs/design/review-mark/indexing/path-helpers.md @@ -1,4 +1,4 @@ -# PathHelpers Design +# PathHelpers ## Overview diff --git a/docs/design/review-mark/indexing/review-index.md b/docs/design/review-mark/indexing/review-index.md index 1a3ef95..4a3dd2e 100644 --- a/docs/design/review-mark/indexing/review-index.md +++ b/docs/design/review-mark/indexing/review-index.md @@ -62,22 +62,31 @@ workflow. `ReviewIndex.Empty()` returns an index with no records. It is used when the evidence source type is `none`, resulting in all review-sets being reported as Missing. -## ReviewIndex.GetStatus() - -`ReviewIndex.GetStatus(id, fingerprint)` determines the review status of a -review-set by looking up the `id` in the loaded index: - -1. Look up `id` in the index - - If not found — return `Missing` -2. Check if there is a record whose `Fingerprint` matches the supplied `fingerprint` - - If no matching fingerprint exists — return `Stale` - - If a matching fingerprint exists: - - If the `Result` is `pass` — return `Current` - - If the `Result` is not `pass` — return `Failed` - -| Status | Meaning | -| ------ | ------- | -| `Current` | The review record matches the current fingerprint and has a passing result | -| `Failed` | The review record matches the current fingerprint but the result is not passing | -| `Stale` | A record exists for the id but the fingerprint does not match the current one | -| `Missing` | No review record exists for the id | +## ReviewIndex.Save() + +`ReviewIndex` provides two overloads for persisting the index to `index.json` format: + +- `Save(string filePath)` — writes the serialized index to the specified file path +- `Save(Stream stream)` — writes the serialized index to the provided stream + +Both overloads serialize all `ReviewEvidence` records in the index to JSON format. +The `Save(string filePath)` overload is used by the `--index` workflow in `Program` +to write the output file after scanning. + +## ReviewIndex.GetEvidence() + +`ReviewIndex.GetEvidence(string id, string fingerprint)` returns the `ReviewEvidence` +record whose `Id` matches `id` and whose `Fingerprint` matches `fingerprint`, or `null` +if no such record exists. + +## ReviewIndex.HasId() + +`ReviewIndex.HasId(string id)` returns `true` if the index contains at least one record +with the given `id`, regardless of fingerprint. Returns `false` if no record exists for +the id. + +## ReviewIndex.GetAllForId() + +`ReviewIndex.GetAllForId(string id)` returns all `ReviewEvidence` records that have the +given `id`, as an enumerable collection. Returns an empty collection if no records exist +for the id. diff --git a/docs/design/review-mark/self-test/validation.md b/docs/design/review-mark/self-test/validation.md index 04ff878..3ff0ba0 100644 --- a/docs/design/review-mark/self-test/validation.md +++ b/docs/design/review-mark/self-test/validation.md @@ -15,7 +15,7 @@ where the tool itself is part of a qualified software chain. 2. Executes each test case in sequence 3. Writes results to the configured output file (TRX or JUnit format) if `ResultsFile` is set 4. Writes a summary table and per-test results to the console via `Context.WriteLine()` -5. Sets `Context.ExitCode` to a non-zero value if any test fails +5. Calls `Context.WriteError()` when any test fails, which causes `Context.ExitCode` to return a non-zero value ## Test Output Format diff --git a/docs/reqstream/review-mark/cli/cli.yaml b/docs/reqstream/review-mark/cli/cli.yaml index ed013bf..d1976ac 100644 --- a/docs/reqstream/review-mark/cli/cli.yaml +++ b/docs/reqstream/review-mark/cli/cli.yaml @@ -22,6 +22,7 @@ sections: - Context_Create_ValidateFlag_SetsValidateTrue - Context_Create_ResultsFlag_SetsResultsFile - Context_Create_LogFlag_OpensLogFile + children: [ReviewMark-Context-Parsing, ReviewMark-Context-Output] - id: ReviewMark-Cmd-Version title: The tool shall support -v and --version flags to display version information. @@ -34,6 +35,8 @@ sections: - Program_Run_WithVersionFlag_DisplaysVersionOnly - Program_Version_ReturnsNonEmptyString - IntegrationTest_VersionFlag_OutputsVersion + - Cli_VersionFlag_OutputsVersionOnly + children: [ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-Help title: The tool shall support -?, -h, and --help flags to display usage information. @@ -46,6 +49,8 @@ sections: - Context_Create_ShortHelpFlag_Question_SetsHelpTrue - Program_Run_WithHelpFlag_DisplaysUsageInformation - IntegrationTest_HelpFlag_OutputsUsageInformation + - Cli_HelpFlag_OutputsUsageInformation + children: [ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-Silent title: The tool shall support --silent flag to suppress console output. @@ -56,6 +61,8 @@ sections: - Context_Create_SilentFlag_SetsSilentTrue - Context_WriteLine_Silent_DoesNotWriteToConsole - IntegrationTest_SilentFlag_SuppressesOutput + - Cli_SilentFlag_SuppressesOutput + children: [ReviewMark-Context-Output] - id: ReviewMark-Cmd-Validate title: The tool shall support --validate flag to run self-validation tests. @@ -66,6 +73,8 @@ sections: - Context_Create_ValidateFlag_SetsValidateTrue - Program_Run_WithValidateFlag_RunsValidation - IntegrationTest_ValidateFlag_RunsValidation + - Cli_ValidateFlag_RunsValidation + children: [ReviewMark-Program-Dispatch, ReviewMark-Validation-Run] - id: ReviewMark-Cmd-Results title: The tool shall support --results flag to write validation results in TRX or JUnit format. @@ -75,6 +84,7 @@ sections: - Context_Create_ResultsFlag_SetsResultsFile - IntegrationTest_ValidateWithResults_GeneratesTrxFile - IntegrationTest_ValidateWithResults_GeneratesJUnitFile + children: [ReviewMark-Validation-ResultsFile] - id: ReviewMark-Cmd-Log title: The tool shall support --log flag to write output to a log file. @@ -83,6 +93,7 @@ sections: tests: - Context_Create_LogFlag_OpensLogFile - IntegrationTest_LogFlag_WritesOutputToFile + children: [ReviewMark-Context-Output] - id: ReviewMark-Cmd-ErrorOutput title: The tool shall write error messages to stderr. @@ -92,6 +103,7 @@ sections: tests: - Context_WriteError_NotSilent_WritesToConsole - IntegrationTest_UnknownArgument_ReturnsError + children: [ReviewMark-Context-Output] - id: ReviewMark-Cmd-InvalidArgs title: The tool shall reject unknown or malformed command-line arguments with a descriptive error. @@ -103,6 +115,7 @@ sections: - Context_Create_LogFlag_WithoutValue_ThrowsArgumentException - Context_Create_ResultsFlag_WithoutValue_ThrowsArgumentException - IntegrationTest_UnknownArgument_ReturnsError + children: [ReviewMark-Context-Parsing] - id: ReviewMark-Cmd-ExitCode title: The tool shall return a non-zero exit code on failure. @@ -112,6 +125,7 @@ sections: tests: - Context_WriteError_SetsErrorExitCode - IntegrationTest_UnknownArgument_ReturnsError + children: [ReviewMark-Context-Output] - id: ReviewMark-Cmd-Definition title: The tool shall support --definition flag to specify the definition YAML file. @@ -123,6 +137,7 @@ sections: - Context_Create_DefinitionFlag_WithoutValue_ThrowsArgumentException - ReviewMark_ReviewPlanGeneration - ReviewMark_ReviewReportGeneration + children: [ReviewMark-Config-Loading, ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-Plan title: The tool shall support --plan flag to write the review plan to a Markdown file. @@ -132,6 +147,7 @@ sections: tests: - Context_Create_PlanFlag_SetsPlanFile - ReviewMark_ReviewPlanGeneration + children: [ReviewMark-Config-Reading, ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-PlanDepth title: The tool shall support --plan-depth flag to set the Markdown heading depth for the review plan. @@ -143,6 +159,7 @@ sections: - Context_Create_PlanDepthFlag_WithInvalidValue_ThrowsArgumentException - Context_Create_PlanDepthFlag_WithZeroValue_ThrowsArgumentException - Context_Create_NoArguments_PlanDepthDefaultsToOne + children: [ReviewMark-Context-Parsing] - id: ReviewMark-Cmd-Report title: The tool shall support --report flag to write the review report to a Markdown file. @@ -152,6 +169,7 @@ sections: tests: - Context_Create_ReportFlag_SetsReportFile - ReviewMark_ReviewReportGeneration + children: [ReviewMark-Config-Reading, ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-ReportDepth title: The tool shall support --report-depth flag to set the Markdown heading depth for the review report. @@ -161,6 +179,7 @@ sections: tests: - Context_Create_ReportDepthFlag_SetsReportDepth - Context_Create_NoArguments_ReportDepthDefaultsToOne + children: [ReviewMark-Context-Parsing] - id: ReviewMark-Cmd-Index title: The tool shall support --index flag to scan PDF evidence files matching a glob path and write @@ -174,6 +193,7 @@ sections: - Context_Create_IndexFlag_MultipleTimes_AddsAllPaths - Context_Create_NoArguments_IndexPathsEmpty - ReviewMark_IndexScan + children: [ReviewMark-Index-PdfParsing, ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-Enforce title: The tool shall support --enforce flag to exit with a non-zero code when there are review issues. @@ -185,6 +205,7 @@ sections: - Context_Create_EnforceFlag_SetsEnforceTrue - Context_Create_NoArguments_EnforceFalse - ReviewMark_Enforce + children: [ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-Dir title: The tool shall support --dir flag to set the working directory for file operations. @@ -197,6 +218,7 @@ sections: - Context_Create_NoArguments_WorkingDirectoryIsNull - Context_Create_DirFlag_MissingValue_ThrowsArgumentException - ReviewMark_WorkingDirectoryOverride + children: [ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-Elaborate title: The tool shall support --elaborate flag to print a Markdown elaboration of a review set. @@ -219,6 +241,7 @@ sections: - Program_Run_WithElaborateFlag_OutputsElaboration - Program_Run_WithElaborateFlag_UnknownId_ReportsError - ReviewMark_Elaborate + children: [ReviewMark-Config-Reading, ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-Lint title: The tool shall support --lint flag to validate the definition file and report issues. @@ -241,3 +264,4 @@ sections: - ReviewMarkConfiguration_Load_MissingEvidenceSource_ReturnsNullConfigWithErrorIssue - ReviewMarkConfiguration_Load_MultipleErrors_ReturnsAllIssues - ReviewMark_Lint + children: [ReviewMark-Config-Loading, ReviewMark-Program-Dispatch] diff --git a/docs/reqstream/review-mark/configuration/configuration.yaml b/docs/reqstream/review-mark/configuration/configuration.yaml index 9e974cc..a634e22 100644 --- a/docs/reqstream/review-mark/configuration/configuration.yaml +++ b/docs/reqstream/review-mark/configuration/configuration.yaml @@ -19,6 +19,8 @@ sections: and generate accurate review plans. tests: - ReviewMarkConfiguration_GetNeedsReviewFiles_ReturnsMatchingFiles + - Configuration_LoadConfig_ResolvesNeedsReviewFiles + children: [ReviewMark-Config-Reading, ReviewMark-GlobMatcher-IncludeExclude] - id: ReviewMark-Configuration-Fingerprinting title: The tool shall compute SHA-256 fingerprints for review-sets to detect file changes. @@ -31,6 +33,8 @@ sections: - ReviewSet_GetFingerprint_SameContent_ReturnsSameFingerprint - ReviewSet_GetFingerprint_DifferentContent_ReturnsDifferentFingerprint - ReviewSet_GetFingerprint_RenameFile_ReturnsSameFingerprint + - Configuration_LoadConfig_FingerprintReflectsFileContent + children: [ReviewMark-Config-Reading] - id: ReviewMark-Configuration-PlanGeneration title: The tool shall generate a Review Plan Markdown document listing review-set coverage. @@ -40,6 +44,8 @@ sections: are included in at least one review-set before reviews are conducted. tests: - ReviewMark_ReviewPlanGeneration + - Configuration_LoadConfig_PlanGenerationSucceeds + children: [ReviewMark-Config-Reading, ReviewMark-Config-Loading] - id: ReviewMark-Configuration-ReportGeneration title: The tool shall generate a Review Report Markdown document showing review-set status. @@ -49,6 +55,7 @@ sections: confirm that all review-sets have current evidence before a release. tests: - ReviewMark_ReviewReportGeneration + children: [ReviewMark-Config-Reading, ReviewMark-Config-Loading] - id: ReviewMark-Configuration-Elaboration title: The tool shall elaborate a review-set by providing its ID, fingerprint, and file list. @@ -59,3 +66,4 @@ sections: review documentation. tests: - ReviewMark_Elaborate + children: [ReviewMark-Config-Reading] diff --git a/docs/reqstream/review-mark/configuration/glob-matcher.yaml b/docs/reqstream/review-mark/configuration/glob-matcher.yaml index 2529257..4beb586 100644 --- a/docs/reqstream/review-mark/configuration/glob-matcher.yaml +++ b/docs/reqstream/review-mark/configuration/glob-matcher.yaml @@ -27,3 +27,5 @@ sections: - GlobMatcher_GetMatchingFiles_NullBaseDirectory_ThrowsArgumentNullException - GlobMatcher_GetMatchingFiles_NullPatterns_ThrowsArgumentNullException - GlobMatcher_GetMatchingFiles_NoMatchingFiles_ReturnsEmptyList + - GlobMatcher_GetMatchingFiles_EmptyPatterns_ReturnsEmptyList + - GlobMatcher_GetMatchingFiles_MultipleIncludePatterns_ReturnsAllMatching diff --git a/docs/reqstream/review-mark/configuration/review-mark-configuration.yaml b/docs/reqstream/review-mark/configuration/review-mark-configuration.yaml index f4e9545..ef5bd0b 100644 --- a/docs/reqstream/review-mark/configuration/review-mark-configuration.yaml +++ b/docs/reqstream/review-mark/configuration/review-mark-configuration.yaml @@ -28,7 +28,6 @@ sections: - ReviewSet_GetFingerprint_SameContent_ReturnsSameFingerprint - ReviewSet_GetFingerprint_DifferentContent_ReturnsDifferentFingerprint - ReviewSet_GetFingerprint_RenameFile_ReturnsSameFingerprint - - ReviewMarkConfiguration_Load_NonExistentFile_ReturnsNullConfigWithErrorIssue - ReviewMarkConfiguration_Load_FileshareRelativeLocation_ResolvesToAbsolutePath - id: ReviewMark-Config-Loading title: ReviewMarkConfiguration.Load shall perform linting and return both the configuration and lint issues. diff --git a/docs/reqstream/review-mark/indexing/indexing.yaml b/docs/reqstream/review-mark/indexing/indexing.yaml index 31a2072..e82871d 100644 --- a/docs/reqstream/review-mark/indexing/indexing.yaml +++ b/docs/reqstream/review-mark/indexing/indexing.yaml @@ -26,6 +26,8 @@ sections: - ReviewIndex_Load_EvidenceSource_Fileshare_LoadsFromFile - ReviewIndex_Load_EvidenceSource_Fileshare_ValidJson_ReturnsPopulatedIndex - ReviewIndex_Load_EvidenceSource_Url_SuccessResponse_LoadsIndex + - Indexing_SafePathCombine_WithIndexPath_LoadsIndex + children: [ReviewMark-Index-EvidenceSource, ReviewMark-EvidenceSource-None] - id: ReviewMark-Indexing-ScanPdfEvidence title: The tool shall scan PDF evidence files and extract embedded review metadata to build an index. @@ -39,6 +41,13 @@ sections: - ReviewIndex_Scan_MultiplePdfs_PopulatesAllEntries - ReviewIndex_Scan_NoMatchingFiles_LeavesIndexEmpty - ReviewIndex_Scan_ClearsExistingEntries + - ReviewIndex_Scan_PdfWithMissingId_SkipsWithWarning + - ReviewIndex_Scan_PdfWithMissingFingerprint_SkipsWithWarning + - ReviewIndex_Scan_PdfWithMissingDate_SkipsWithWarning + - ReviewIndex_Scan_PdfWithMissingResult_SkipsWithWarning + - ReviewIndex_Scan_PdfWithNoKeywords_SkipsWithWarning + - Indexing_ReviewIndex_SaveAndLoad_RoundTrip + children: [ReviewMark-Index-PdfParsing] - id: ReviewMark-Indexing-SafePathCombine title: The tool shall combine file paths safely, rejecting path traversal sequences. @@ -55,3 +64,4 @@ sections: - PathHelpers_SafePathCombine_CurrentDirectoryReference_CombinesCorrectly - PathHelpers_SafePathCombine_NestedPaths_CombinesCorrectly - PathHelpers_SafePathCombine_EmptyRelativePath_ReturnsBasePath + children: [ReviewMark-PathHelpers-SafeCombine] diff --git a/docs/reqstream/review-mark/indexing/path-helpers.yaml b/docs/reqstream/review-mark/indexing/path-helpers.yaml index 9193174..91690f7 100644 --- a/docs/reqstream/review-mark/indexing/path-helpers.yaml +++ b/docs/reqstream/review-mark/indexing/path-helpers.yaml @@ -23,3 +23,6 @@ sections: - PathHelpers_SafePathCombine_PathTraversalWithDoubleDots_ThrowsArgumentException - PathHelpers_SafePathCombine_AbsolutePath_ThrowsArgumentException - PathHelpers_SafePathCombine_NestedPaths_CombinesCorrectly + - PathHelpers_SafePathCombine_DoubleDotsInMiddle_ThrowsArgumentException + - PathHelpers_SafePathCombine_CurrentDirectoryReference_CombinesCorrectly + - PathHelpers_SafePathCombine_EmptyRelativePath_ReturnsBasePath diff --git a/docs/reqstream/review-mark/self-test/self-test.yaml b/docs/reqstream/review-mark/self-test/self-test.yaml index f4e1bfa..4af7563 100644 --- a/docs/reqstream/review-mark/self-test/self-test.yaml +++ b/docs/reqstream/review-mark/self-test/self-test.yaml @@ -24,6 +24,8 @@ sections: - Validation_Run_WritesValidationHeader - Validation_Run_WritesSummaryWithTotalTests - Validation_Run_AllTestsPass_ExitCodeIsZero + - SelfTest_Run_AllTestsPass_ExitCodeIsZero + children: [ReviewMark-Validation-Run] - id: ReviewMark-SelfTest-ResultsOutput title: The tool shall write self-validation results to a standard test result file when --results is provided. @@ -36,3 +38,5 @@ sections: tests: - Validation_Run_WithTrxResultsFile_WritesFile - Validation_Run_WithXmlResultsFile_WritesFile + - SelfTest_Run_GeneratesResultsFile + children: [ReviewMark-Validation-ResultsFile] diff --git a/test/DemaConsulting.ReviewMark.Tests/Cli/CliTests.cs b/test/DemaConsulting.ReviewMark.Tests/Cli/CliTests.cs new file mode 100644 index 0000000..6d1e07c --- /dev/null +++ b/test/DemaConsulting.ReviewMark.Tests/Cli/CliTests.cs @@ -0,0 +1,148 @@ +// Copyright (c) DEMA Consulting +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +using DemaConsulting.ReviewMark.Cli; + +namespace DemaConsulting.ReviewMark.Tests.Cli; + +/// +/// Subsystem integration tests for the CLI subsystem (Context + Program). +/// +[TestClass] +public class CliTests +{ + /// + /// Test that the CLI correctly outputs only the version string when --version is supplied. + /// + [TestMethod] + public void Cli_VersionFlag_OutputsVersionOnly() + { + // Arrange + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--version"]); + + // Act + Program.Run(context); + + // Assert — output is the version string with no banner or copyright + var output = outWriter.ToString(); + Assert.AreEqual(Program.Version, output.Trim()); + Assert.DoesNotContain("Copyright", output); + } + finally + { + Console.SetOut(originalOut); + } + } + + /// + /// Test that the CLI outputs usage information when --help is supplied. + /// + [TestMethod] + public void Cli_HelpFlag_OutputsUsageInformation() + { + // Arrange + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--help"]); + + // Act + Program.Run(context); + + // Assert — output contains usage and options sections + var output = outWriter.ToString(); + Assert.Contains("Usage:", output); + Assert.Contains("Options:", output); + Assert.Contains("--version", output); + Assert.Contains("--help", output); + } + finally + { + Console.SetOut(originalOut); + } + } + + /// + /// Test that the CLI runs self-validation when --validate is supplied. + /// + [TestMethod] + public void Cli_ValidateFlag_RunsValidation() + { + // Arrange + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--validate"]); + + // Act + Program.Run(context); + + // Assert — output contains validation summary and exit code is zero + var output = outWriter.ToString(); + Assert.Contains("Total Tests:", output); + Assert.AreEqual(0, context.ExitCode); + } + finally + { + Console.SetOut(originalOut); + } + } + + /// + /// Test that the CLI suppresses all console output when --silent is supplied. + /// + [TestMethod] + public void Cli_SilentFlag_SuppressesOutput() + { + // Arrange + var originalOut = Console.Out; + var originalError = Console.Error; + try + { + using var outWriter = new StringWriter(); + using var errWriter = new StringWriter(); + Console.SetOut(outWriter); + Console.SetError(errWriter); + using var context = Context.Create(["--silent"]); + + // Act + Program.Run(context); + + // Assert — no output written to stdout or stderr; exit code is zero + Assert.AreEqual(string.Empty, outWriter.ToString()); + Assert.AreEqual(string.Empty, errWriter.ToString()); + Assert.AreEqual(0, context.ExitCode); + } + finally + { + Console.SetOut(originalOut); + Console.SetError(originalError); + } + } +} diff --git a/test/DemaConsulting.ReviewMark.Tests/Configuration/ConfigurationTests.cs b/test/DemaConsulting.ReviewMark.Tests/Configuration/ConfigurationTests.cs new file mode 100644 index 0000000..563117a --- /dev/null +++ b/test/DemaConsulting.ReviewMark.Tests/Configuration/ConfigurationTests.cs @@ -0,0 +1,180 @@ +// Copyright (c) DEMA Consulting +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +using DemaConsulting.ReviewMark.Configuration; +using DemaConsulting.ReviewMark.Indexing; + +namespace DemaConsulting.ReviewMark.Tests.Configuration; + +/// +/// Subsystem integration tests for the Configuration subsystem +/// (ReviewMarkConfiguration + GlobMatcher working together). +/// +[TestClass] +public class ConfigurationTests +{ + /// + /// Unique temporary directory created before each test and deleted after. + /// + private string _testDirectory = string.Empty; + + /// + /// Creates a fresh GUID-based temporary directory before each test. + /// + [TestInitialize] + public void TestInitialize() + { + _testDirectory = PathHelpers.SafePathCombine( + Path.GetTempPath(), + $"ConfigurationTests_{Guid.NewGuid()}"); + Directory.CreateDirectory(_testDirectory); + } + + /// + /// Deletes the temporary directory and all its contents after each test. + /// + [TestCleanup] + public void TestCleanup() + { + if (Directory.Exists(_testDirectory)) + { + Directory.Delete(_testDirectory, recursive: true); + } + } + + /// + /// Test that loading a configuration with needs-review glob patterns correctly resolves matching files. + /// + [TestMethod] + public void Configuration_LoadConfig_ResolvesNeedsReviewFiles() + { + // Arrange + var srcDir = PathHelpers.SafePathCombine(_testDirectory, "src"); + Directory.CreateDirectory(srcDir); + File.WriteAllText(PathHelpers.SafePathCombine(srcDir, "Main.cs"), "class Main {}"); + File.WriteAllText(PathHelpers.SafePathCombine(srcDir, "Helper.cs"), "class Helper {}"); + + var indexFile = PathHelpers.SafePathCombine(_testDirectory, "index.json"); + File.WriteAllText(indexFile, """{"reviews":[]}"""); + + var definitionFile = PathHelpers.SafePathCombine(_testDirectory, ".reviewmark.yaml"); + File.WriteAllText(definitionFile, $""" + needs-review: + - "src/**/*.cs" + evidence-source: + type: fileshare + location: {indexFile} + reviews: + - id: Core-Logic + title: Core logic review + paths: + - "src/**/*.cs" + """); + + // Act + var result = ReviewMarkConfiguration.Load(definitionFile); + + // Assert + Assert.IsNotNull(result.Configuration); + var files = result.Configuration.GetNeedsReviewFiles(_testDirectory); + Assert.AreEqual(2, files.Count); + } + + /// + /// Test that modifying a file changes the review-set fingerprint. + /// + [TestMethod] + public void Configuration_LoadConfig_FingerprintReflectsFileContent() + { + // Arrange + var srcDir = PathHelpers.SafePathCombine(_testDirectory, "src"); + Directory.CreateDirectory(srcDir); + var sourceFile = PathHelpers.SafePathCombine(srcDir, "Main.cs"); + File.WriteAllText(sourceFile, "class Main {}"); + + var indexFile = PathHelpers.SafePathCombine(_testDirectory, "index.json"); + File.WriteAllText(indexFile, """{"reviews":[]}"""); + + var definitionFile = PathHelpers.SafePathCombine(_testDirectory, ".reviewmark.yaml"); + File.WriteAllText(definitionFile, $""" + needs-review: + - "src/**/*.cs" + evidence-source: + type: fileshare + location: {indexFile} + reviews: + - id: Core-Logic + title: Core logic review + paths: + - "src/**/*.cs" + """); + + // Act — load before and after modifying the source file + var result1 = ReviewMarkConfiguration.Load(definitionFile); + Assert.IsNotNull(result1.Configuration); + var fingerprint1 = result1.Configuration.Reviews[0].GetFingerprint(_testDirectory); + + File.WriteAllText(sourceFile, "class Main { void Modified() {} }"); + + var result2 = ReviewMarkConfiguration.Load(definitionFile); + Assert.IsNotNull(result2.Configuration); + var fingerprint2 = result2.Configuration.Reviews[0].GetFingerprint(_testDirectory); + + // Assert — fingerprints differ after content change + Assert.AreNotEqual(fingerprint1, fingerprint2); + } + + /// + /// Test that generating a review plan succeeds and includes the review set ID. + /// + [TestMethod] + public void Configuration_LoadConfig_PlanGenerationSucceeds() + { + // Arrange + var srcDir = PathHelpers.SafePathCombine(_testDirectory, "src"); + Directory.CreateDirectory(srcDir); + File.WriteAllText(PathHelpers.SafePathCombine(srcDir, "Main.cs"), "class Main {}"); + + var indexFile = PathHelpers.SafePathCombine(_testDirectory, "index.json"); + File.WriteAllText(indexFile, """{"reviews":[]}"""); + + var definitionFile = PathHelpers.SafePathCombine(_testDirectory, ".reviewmark.yaml"); + File.WriteAllText(definitionFile, $""" + needs-review: + - "src/**/*.cs" + evidence-source: + type: fileshare + location: {indexFile} + reviews: + - id: Core-Logic + title: Core logic review + paths: + - "src/**/*.cs" + """); + + // Act + var result = ReviewMarkConfiguration.Load(definitionFile); + Assert.IsNotNull(result.Configuration); + var planResult = result.Configuration.PublishReviewPlan(_testDirectory); + + // Assert + Assert.Contains("Core-Logic", planResult.Markdown); + } +} diff --git a/test/DemaConsulting.ReviewMark.Tests/Indexing/IndexingTests.cs b/test/DemaConsulting.ReviewMark.Tests/Indexing/IndexingTests.cs new file mode 100644 index 0000000..466005b --- /dev/null +++ b/test/DemaConsulting.ReviewMark.Tests/Indexing/IndexingTests.cs @@ -0,0 +1,145 @@ +// Copyright (c) DEMA Consulting +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +using DemaConsulting.ReviewMark.Configuration; +using DemaConsulting.ReviewMark.Indexing; + +namespace DemaConsulting.ReviewMark.Tests.Indexing; + +/// +/// Subsystem integration tests for the Indexing subsystem +/// (ReviewIndex + PathHelpers working together). +/// +[TestClass] +public class IndexingTests +{ + /// + /// Unique temporary directory created before each test and deleted after. + /// + private string _testDirectory = string.Empty; + + /// + /// Creates a fresh GUID-based temporary directory before each test. + /// + [TestInitialize] + public void TestInitialize() + { + _testDirectory = PathHelpers.SafePathCombine( + Path.GetTempPath(), + $"IndexingTests_{Guid.NewGuid()}"); + Directory.CreateDirectory(_testDirectory); + } + + /// + /// Deletes the temporary directory and all its contents after each test. + /// + [TestCleanup] + public void TestCleanup() + { + if (Directory.Exists(_testDirectory)) + { + Directory.Delete(_testDirectory, recursive: true); + } + } + + /// + /// Test that SafePathCombine with a subdirectory segment resolves to a valid index path + /// that can be loaded by ReviewIndex. + /// + [TestMethod] + public void Indexing_SafePathCombine_WithIndexPath_LoadsIndex() + { + // Arrange + var evidenceDir = PathHelpers.SafePathCombine(_testDirectory, "evidence"); + Directory.CreateDirectory(evidenceDir); + + var indexFile = PathHelpers.SafePathCombine(evidenceDir, "index.json"); + File.WriteAllText(indexFile, """ + { + "reviews": [ + { + "id": "Test-Review", + "fingerprint": "abc123", + "date": "2024-01-01", + "result": "pass", + "file": "test.pdf" + } + ] + } + """); + + var combinedPath = PathHelpers.SafePathCombine(_testDirectory, "evidence/index.json"); + var source = new EvidenceSource("fileshare", combinedPath, null, null); + + // Act + var index = ReviewIndex.Load(source); + + // Assert + Assert.IsTrue(index.HasId("Test-Review")); + var evidence = index.GetEvidence("Test-Review", "abc123"); + Assert.IsNotNull(evidence); + } + + /// + /// Test that a ReviewIndex can be saved and reloaded with all entries preserved. + /// + [TestMethod] + public void Indexing_ReviewIndex_SaveAndLoad_RoundTrip() + { + // Arrange + var indexFile = PathHelpers.SafePathCombine(_testDirectory, "index.json"); + File.WriteAllText(indexFile, """ + { + "reviews": [ + { + "id": "Review-Alpha", + "fingerprint": "fp001", + "date": "2024-06-01", + "result": "pass", + "file": "alpha.pdf" + }, + { + "id": "Review-Beta", + "fingerprint": "fp002", + "date": "2024-06-02", + "result": "pass", + "file": "beta.pdf" + } + ] + } + """); + + var source = new EvidenceSource("fileshare", indexFile, null, null); + + // Act — load, save to a new file, then reload + var index1 = ReviewIndex.Load(source); + var savedFile = PathHelpers.SafePathCombine(_testDirectory, "index-copy.json"); + index1.Save(savedFile); + + var source2 = new EvidenceSource("fileshare", savedFile, null, null); + var index2 = ReviewIndex.Load(source2); + + // Assert — all entries survive the round-trip + Assert.IsTrue(index2.HasId("Review-Alpha")); + Assert.IsTrue(index2.HasId("Review-Beta")); + Assert.IsNotNull(index2.GetEvidence("Review-Alpha", "fp001")); + Assert.IsNotNull(index2.GetEvidence("Review-Beta", "fp002")); + } +} diff --git a/test/DemaConsulting.ReviewMark.Tests/SelfTest/SelfTestTests.cs b/test/DemaConsulting.ReviewMark.Tests/SelfTest/SelfTestTests.cs new file mode 100644 index 0000000..829064c --- /dev/null +++ b/test/DemaConsulting.ReviewMark.Tests/SelfTest/SelfTestTests.cs @@ -0,0 +1,97 @@ +// Copyright (c) DEMA Consulting +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +using DemaConsulting.ReviewMark.Cli; +using DemaConsulting.ReviewMark.SelfTest; + +namespace DemaConsulting.ReviewMark.Tests.SelfTest; + +/// +/// Subsystem integration tests for the SelfTest subsystem. +/// +[TestClass] +public class SelfTestTests +{ + /// + /// Test that running self-validation passes all tests and exits with code zero. + /// + [TestMethod] + public void SelfTest_Run_AllTestsPass_ExitCodeIsZero() + { + // Arrange + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--validate"]); + + // Act + Validation.Run(context); + + // Assert + Assert.AreEqual(0, context.ExitCode); + Assert.Contains("Total Tests:", outWriter.ToString()); + } + finally + { + Console.SetOut(originalOut); + } + } + + /// + /// Test that running self-validation with --results creates a TRX results file. + /// + [TestMethod] + public void SelfTest_Run_GeneratesResultsFile() + { + // Arrange + var resultsFile = Path.Combine(Path.GetTempPath(), $"reviewmark-selftest-{Guid.NewGuid()}.trx"); + try + { + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--validate", "--results", resultsFile]); + + // Act + Validation.Run(context); + + // Assert + Assert.IsTrue(File.Exists(resultsFile), "Results file was not created"); + var content = File.ReadAllText(resultsFile); + Assert.Contains("TestRun", content); + } + finally + { + Console.SetOut(originalOut); + } + } + finally + { + if (File.Exists(resultsFile)) + { + File.Delete(resultsFile); + } + } + } +} From 69d920590768868c148bf77dccdc89e498cfb062 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Fri, 3 Apr 2026 23:29:16 -0400 Subject: [PATCH 17/19] Fix review issues: I/O error handling in Program.cs, console color in Context.cs, missing test refs in cli.yaml (#45) Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/ac56b6a1-fc95-41b9-8f21-2a884f7253a0 Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --- docs/reqstream/review-mark/cli/cli.yaml | 5 +++++ src/DemaConsulting.ReviewMark/Cli/Context.cs | 12 +++++++++--- src/DemaConsulting.ReviewMark/Program.cs | 20 ++++++++++++++++++-- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/docs/reqstream/review-mark/cli/cli.yaml b/docs/reqstream/review-mark/cli/cli.yaml index d1976ac..673b6ec 100644 --- a/docs/reqstream/review-mark/cli/cli.yaml +++ b/docs/reqstream/review-mark/cli/cli.yaml @@ -158,6 +158,7 @@ sections: - Context_Create_PlanDepthFlag_SetsPlanDepth - Context_Create_PlanDepthFlag_WithInvalidValue_ThrowsArgumentException - Context_Create_PlanDepthFlag_WithZeroValue_ThrowsArgumentException + - Context_Create_PlanDepthFlag_WithValueGreaterThanFive_ThrowsArgumentException - Context_Create_NoArguments_PlanDepthDefaultsToOne children: [ReviewMark-Context-Parsing] @@ -178,6 +179,10 @@ sections: Markdown document, with a default depth of 1 when not specified. tests: - Context_Create_ReportDepthFlag_SetsReportDepth + - Context_Create_ReportDepthFlag_MissingValue_ThrowsArgumentException + - Context_Create_ReportDepthFlag_NonNumeric_ThrowsArgumentException + - Context_Create_ReportDepthFlag_Zero_ThrowsArgumentException + - Context_Create_ReportDepthFlag_WithValueGreaterThanFive_ThrowsArgumentException - Context_Create_NoArguments_ReportDepthDefaultsToOne children: [ReviewMark-Context-Parsing] diff --git a/src/DemaConsulting.ReviewMark/Cli/Context.cs b/src/DemaConsulting.ReviewMark/Cli/Context.cs index c329ee6..f1e58f7 100644 --- a/src/DemaConsulting.ReviewMark/Cli/Context.cs +++ b/src/DemaConsulting.ReviewMark/Cli/Context.cs @@ -468,9 +468,15 @@ public void WriteError(string message) if (!Silent) { var previousColor = Console.ForegroundColor; - Console.ForegroundColor = ConsoleColor.Red; - Console.Error.WriteLine(message); - Console.ForegroundColor = previousColor; + try + { + Console.ForegroundColor = ConsoleColor.Red; + Console.Error.WriteLine(message); + } + finally + { + Console.ForegroundColor = previousColor; + } } // Write to log file if logging is enabled diff --git a/src/DemaConsulting.ReviewMark/Program.cs b/src/DemaConsulting.ReviewMark/Program.cs index 89ed3f7..e3b7454 100644 --- a/src/DemaConsulting.ReviewMark/Program.cs +++ b/src/DemaConsulting.ReviewMark/Program.cs @@ -274,7 +274,15 @@ private static void RunDefinitionLogic(Context context, string directory, string if (context.PlanFile != null) { var planResult = config.PublishReviewPlan(directory, context.PlanDepth); - File.WriteAllText(context.PlanFile, planResult.Markdown); + try + { + File.WriteAllText(context.PlanFile, planResult.Markdown); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or DirectoryNotFoundException) + { + throw new InvalidOperationException($"Failed to write review plan to '{context.PlanFile}': {ex.Message}", ex); + } + context.WriteLine($"Review plan written to {context.PlanFile}"); HandleIssues(context, planResult.HasIssues, "Review plan has coverage issues."); } @@ -284,7 +292,15 @@ private static void RunDefinitionLogic(Context context, string directory, string { var index = ReviewIndex.Load(config.EvidenceSource); var reportResult = config.PublishReviewReport(index, directory, context.ReportDepth); - File.WriteAllText(context.ReportFile, reportResult.Markdown); + try + { + File.WriteAllText(context.ReportFile, reportResult.Markdown); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or DirectoryNotFoundException) + { + throw new InvalidOperationException($"Failed to write review report to '{context.ReportFile}': {ex.Message}", ex); + } + context.WriteLine($"Review report written to {context.ReportFile}"); HandleIssues(context, reportResult.HasIssues, "Review report has review issues."); } From 6b61889ff62bb3c878ae4db1c704131f2bbdba6d Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sun, 5 Apr 2026 08:32:04 -0400 Subject: [PATCH 18/19] Add IntegrationTest_* and Cli_* tests; update reqstream YAML for test-linkage compliance (#46) * Sync .github/agents and .github/standards from TemplateDotNetTool Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/5fe50d02-c30f-45d3-a137-438007a8d9b6 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Add IntegrationTest_* and Cli_* test methods; update reqstream YAML for test-linkage compliance - Add 7 new IntegrationTest_* methods to IntegrationTests.cs covering: ReviewPlanGeneration, ReviewReportGeneration, Enforce, IndexScan, WorkingDirectoryOverride, Elaborate, Lint - Add 13 new Cli_* methods to CliTests.cs covering: ResultsFlag, LogFlag, ErrorOutput, InvalidArgs, ExitCode, DefinitionFlag, PlanFlag, ReportFlag, EnforceFlag, DirFlag, ElaborateFlag, LintFlag, IndexFlag - Update docs/reqstream/review-mark/review-mark.yaml: system requirements now use only IntegrationTest_* tests and add children: links to subsystem IDs - Update docs/reqstream/review-mark/cli/cli.yaml: subsystem requirements now use only Cli_* tests; add new requirements for all CLI flags - Update docs/reqstream/review-mark/configuration/configuration.yaml: subsystem requirements use only Configuration_* tests - Update docs/reqstream/review-mark/indexing/indexing.yaml: subsystem requirements use only Indexing_* tests - Update docs/reqstream/review-mark/self-test/self-test.yaml: subsystem requirements use only SelfTest_* tests - Update docs/reqstream/review-mark/configuration/review-mark-configuration.yaml: remove Program_* test from ReviewMark-Config-Loading - Update docs/reqstream/review-mark/indexing/review-index.yaml: remove ReviewMarkConfiguration_* tests from ReviewMark-EvidenceSource-None - Update .reviewmark.yaml: add Purpose review; update ReviewMark-Architecture, ReviewMark-Design, ReviewMark-AllRequirements; remove user_guide and TestDirectory.cs from ReviewMark-Program review Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * Fix review set spelling: 'review set' -> 'review-set' in test comments Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * fix: address quality findings - fix cspell wdir, add missing system reqs, reorder review-sets Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * fix: dispose context before reading log file in Cli_LogFlag_WritesOutputToFile Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/faf4bc52-4fa8-491e-a753-863b18c1e6a6 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * fix: apply all PR review feedback - temp file leaks, ErrorOutput, depth tests, traceability tests Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/bb1deea5-108a-46ac-9875-9199ef6993e9 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * refactor: simplify GetMethod call and use specific heading text in depth assertions Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/bb1deea5-108a-46ac-9875-9199ef6993e9 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --- .github/agents/code-review.agent.md | 39 +- .github/agents/developer.agent.md | 16 +- .github/agents/implementation.agent.md | 20 +- .github/agents/quality.agent.md | 118 +-- .github/agents/repo-consistency.agent.md | 28 +- .github/standards/csharp-testing.md | 15 +- .github/standards/reqstream-usage.md | 16 +- .github/standards/reviewmark-usage.md | 127 +++- .github/standards/software-items.md | 5 + .reviewmark.yaml | 121 +-- docs/reqstream/review-mark/cli/cli.yaml | 109 +-- .../configuration/configuration.yaml | 9 +- .../review-mark-configuration.yaml | 1 - .../review-mark/indexing/indexing.yaml | 23 +- .../review-mark/indexing/review-index.yaml | 3 - docs/reqstream/review-mark/review-mark.yaml | 122 ++- .../review-mark/self-test/self-test.yaml | 6 - .../Cli/CliTests.cs | 707 ++++++++++++++++++ .../Configuration/ConfigurationTests.cs | 75 ++ .../Indexing/IndexingTests.cs | 19 + .../IntegrationTests.cs | 333 +++++++++ 21 files changed, 1555 insertions(+), 357 deletions(-) diff --git a/.github/agents/code-review.agent.md b/.github/agents/code-review.agent.md index cee797f..bb48e5c 100644 --- a/.github/agents/code-review.agent.md +++ b/.github/agents/code-review.agent.md @@ -15,7 +15,7 @@ Formal reviews are a quality enforcement mechanism, and as such MUST be performe 1. Download the to get the checklist to fill in -2. Use `dotnet reviewmark --elaborate [review-set]` to get the files to review +2. Use `dotnet reviewmark --elaborate {review-set}` to get the files to review 3. Review the files all together 4. Populate the checklist with the findings to `.agent-logs/reviews/review-report-{review-set}.md` of the project. @@ -41,33 +41,34 @@ of the project consisting of: ## Review Summary -- **Review Set**: [Review set name/identifier] -- **Review Report File**: [Name of detailed review report generated] -- **Files Reviewed**: [Count and list of files reviewed] -- **Review Template Used**: [Template source and version] +- **Review Set**: {Review set name/identifier} +- **Review Report File**: {Name of detailed review report generated} +- **Files Reviewed**: {Count and list of files reviewed} +- **Review Template Used**: {Template source and version} ## Review Results -- **Overall Conclusion**: [Summary of review results] -- **Critical Issues**: [Count of critical findings] -- **High Issues**: [Count of high severity findings] -- **Medium Issues**: [Count of medium severity findings] -- **Low Issues**: [Count of low severity findings] +- **Overall Conclusion**: {Summary of review results} +- **Critical Issues**: {Count of critical findings} +- **High Issues**: {Count of high severity findings} +- **Medium Issues**: {Count of medium severity findings} +- **Low Issues**: {Count of low severity findings} ## Issue Details -[For each issue found, include:] -- **File**: [File name and line number where applicable] -- **Issue Type**: [Security, logic error, compliance violation, etc.] -- **Severity**: [Critical/High/Medium/Low] -- **Description**: [Issue description] -- **Recommendation**: [Specific remediation recommendation] +For each issue found, include: + +- **File**: {File name and line number where applicable} +- **Issue Type**: {Security, logic error, compliance violation, etc.} +- **Severity**: {Critical/High/Medium/Low} +- **Description**: {Issue description} +- **Recommendation**: {Specific remediation recommendation} ## Compliance Status -- **Review Status**: [Complete/Incomplete with reasoning] -- **Quality Gates**: [Status of review checklist items] -- **Approval Status**: [Approved/Rejected with justification] +- **Review Status**: {Complete/Incomplete with reasoning} +- **Quality Gates**: {Status of review checklist items} +- **Approval Status**: {Approved/Rejected with justification} ``` Return summary to caller. diff --git a/.github/agents/developer.agent.md b/.github/agents/developer.agent.md index d936129..2671008 100644 --- a/.github/agents/developer.agent.md +++ b/.github/agents/developer.agent.md @@ -31,20 +31,20 @@ of the project consisting of: ## Work Summary -- **Files Modified**: [List of files created/modified/deleted] -- **Languages Detected**: [Languages identified] -- **Standards Applied**: [Standards files consulted] +- **Files Modified**: {List of files created/modified/deleted} +- **Languages Detected**: {Languages identified} +- **Standards Applied**: {Standards files consulted} ## Tooling Executed -- **Language Tools**: [Compilers, linters, formatters used] -- **Compliance Tools**: [ReqStream, ReviewMark tools used] -- **Validation Results**: [Tool execution results] +- **Language Tools**: {Compilers, linters, formatters used} +- **Compliance Tools**: {ReqStream, ReviewMark tools used} +- **Validation Results**: {Tool execution results} ## Compliance Status -- **Quality Checks**: [Standards quality checks status] -- **Issues Resolved**: [Any problems encountered and resolved] +- **Quality Checks**: {Standards quality checks status} +- **Issues Resolved**: {Any problems encountered and resolved} ``` Return this summary to the caller. diff --git a/.github/agents/implementation.agent.md b/.github/agents/implementation.agent.md index 35cc1c8..03603a4 100644 --- a/.github/agents/implementation.agent.md +++ b/.github/agents/implementation.agent.md @@ -72,22 +72,22 @@ of the project consisting of: ## State Machine Execution -- **Research Results**: [Summary of explore agent findings] -- **Development Results**: [Summary of developer agent results] -- **Quality Results**: [Summary of quality agent results] -- **State Transitions**: [Log of state changes and decisions] +- **Research Results**: {Summary of explore agent findings} +- **Development Results**: {Summary of developer agent results} +- **Quality Results**: {Summary of quality agent results} +- **State Transitions**: {Log of state changes and decisions} ## Sub-Agent Coordination -- **Explore Agent**: [Research findings and context] -- **Developer Agent**: [Development status and files modified] -- **Quality Agent**: [Validation results and compliance status] +- **Explore Agent**: {Research findings and context} +- **Developer Agent**: {Development status and files modified} +- **Quality Agent**: {Validation results and compliance status} ## Final Status -- **Implementation Success**: [Overall completion status] -- **Quality Compliance**: [Final quality validation status] -- **Issues Resolved**: [Problems encountered and resolution attempts] +- **Implementation Success**: {Overall completion status} +- **Quality Compliance**: {Final quality validation status} +- **Issues Resolved**: {Problems encountered and resolution attempts} ``` Return this summary to the caller. diff --git a/.github/agents/quality.agent.md b/.github/agents/quality.agent.md index 691a17d..18fc7c6 100644 --- a/.github/agents/quality.agent.md +++ b/.github/agents/quality.agent.md @@ -41,95 +41,95 @@ This ensures orchestrators properly halt workflows when quality gates fail. ## Assessment Summary -- **Work Reviewed**: [Description of work assessed] -- **Standards Applied**: [Standards files used for assessment] -- **Categories Evaluated**: [Quality check categories assessed] +- **Work Reviewed**: {Description of work assessed} +- **Standards Applied**: {Standards files used for assessment} +- **Categories Evaluated**: {Quality check categories assessed} ## Requirements Compliance: (PASS|FAIL|N/A) -- Were requirements updated to reflect functional changes? (PASS|FAIL|N/A) - [Evidence] -- Were new requirements created for new features? (PASS|FAIL|N/A) - [Evidence] -- Do requirement IDs follow semantic naming standards? (PASS|FAIL|N/A) - [Evidence] -- Do requirement files follow kebab-case naming convention? (PASS|FAIL|N/A) - [Evidence] -- Are requirement files organized under `docs/reqstream/` with proper folder structure? (PASS|FAIL|N/A) - [Evidence] -- Are OTS requirements properly placed in `docs/reqstream/ots/` subfolder? (PASS|FAIL|N/A) - [Evidence] -- Were source filters applied appropriately for platform-specific requirements? (PASS|FAIL|N/A) - [Evidence] -- Does ReqStream enforcement pass without errors? (PASS|FAIL|N/A) - [Evidence] -- Is requirements traceability maintained to tests? (PASS|FAIL|N/A) - [Evidence] +- Were requirements updated to reflect functional changes? (PASS|FAIL|N/A) - {Evidence} +- Were new requirements created for new features? (PASS|FAIL|N/A) - {Evidence} +- Do requirement IDs follow semantic naming standards? (PASS|FAIL|N/A) - {Evidence} +- Do requirement files follow kebab-case naming convention? (PASS|FAIL|N/A) - {Evidence} +- Are requirement files organized under `docs/reqstream/` with proper folder structure? (PASS|FAIL|N/A) - {Evidence} +- Are OTS requirements properly placed in `docs/reqstream/ots/` subfolder? (PASS|FAIL|N/A) - {Evidence} +- Were source filters applied appropriately for platform-specific requirements? (PASS|FAIL|N/A) - {Evidence} +- Does ReqStream enforcement pass without errors? (PASS|FAIL|N/A) - {Evidence} +- Is requirements traceability maintained to tests? (PASS|FAIL|N/A) - {Evidence} ## Design Documentation Compliance: (PASS|FAIL|N/A) -- Were design documents updated for architectural changes? (PASS|FAIL|N/A) - [Evidence] -- Were new design artifacts created for new components? (PASS|FAIL|N/A) - [Evidence] -- Do design folder names use kebab-case convention matching source structure? (PASS|FAIL|N/A) - [Evidence] -- Are design files properly named ({subsystem-name}.md, {unit-name}.md patterns)? (PASS|FAIL|N/A) - [Evidence] -- Is `docs/design/introduction.md` present with required Software Structure section? (PASS|FAIL|N/A) - [Evidence] -- Are design decisions documented with rationale? (PASS|FAIL|N/A) - [Evidence] -- Is system/subsystem/unit categorization maintained? (PASS|FAIL|N/A) - [Evidence] -- Is design-to-implementation traceability preserved? (PASS|FAIL|N/A) - [Evidence] +- Were design documents updated for architectural changes? (PASS|FAIL|N/A) - {Evidence} +- Were new design artifacts created for new components? (PASS|FAIL|N/A) - {Evidence} +- Do design folder names use kebab-case convention matching source structure? (PASS|FAIL|N/A) - {Evidence} +- Are design files properly named ({subsystem-name}.md, {unit-name}.md patterns)? (PASS|FAIL|N/A) - {Evidence} +- Is `docs/design/introduction.md` present with required Software Structure section? (PASS|FAIL|N/A) - {Evidence} +- Are design decisions documented with rationale? (PASS|FAIL|N/A) - {Evidence} +- Is system/subsystem/unit categorization maintained? (PASS|FAIL|N/A) - {Evidence} +- Is design-to-implementation traceability preserved? (PASS|FAIL|N/A) - {Evidence} ## Code Quality Compliance: (PASS|FAIL|N/A) -- Are language-specific standards followed (from applicable standards files)? (PASS|FAIL|N/A) - [Evidence] -- Are quality checks from standards files satisfied? (PASS|FAIL|N/A) - [Evidence] -- Is code properly categorized (system/subsystem/unit/OTS)? (PASS|FAIL|N/A) - [Evidence] -- Is appropriate separation of concerns maintained? (PASS|FAIL|N/A) - [Evidence] -- Was language-specific tooling executed and passing? (PASS|FAIL|N/A) - [Evidence] +- Are language-specific standards followed (from applicable standards files)? (PASS|FAIL|N/A) - {Evidence} +- Are quality checks from standards files satisfied? (PASS|FAIL|N/A) - {Evidence} +- Is code properly categorized (system/subsystem/unit/OTS)? (PASS|FAIL|N/A) - {Evidence} +- Is appropriate separation of concerns maintained? (PASS|FAIL|N/A) - {Evidence} +- Was language-specific tooling executed and passing? (PASS|FAIL|N/A) - {Evidence} ## Testing Compliance: (PASS|FAIL|N/A) -- Were tests created/updated for all functional changes? (PASS|FAIL|N/A) - [Evidence] -- Is test coverage maintained for all requirements? (PASS|FAIL|N/A) - [Evidence] -- Are testing standards followed (AAA pattern, etc.)? (PASS|FAIL|N/A) - [Evidence] -- Does test categorization align with code structure? (PASS|FAIL|N/A) - [Evidence] -- Do all tests pass without failures? (PASS|FAIL|N/A) - [Evidence] +- Were tests created/updated for all functional changes? (PASS|FAIL|N/A) - {Evidence} +- Is test coverage maintained for all requirements? (PASS|FAIL|N/A) - {Evidence} +- Are testing standards followed (AAA pattern, etc.)? (PASS|FAIL|N/A) - {Evidence} +- Does test categorization align with code structure? (PASS|FAIL|N/A) - {Evidence} +- Do all tests pass without failures? (PASS|FAIL|N/A) - {Evidence} ## Review Management Compliance: (PASS|FAIL|N/A) -- Were review-sets updated to include new/modified files? (PASS|FAIL|N/A) - [Evidence] -- Do file patterns follow include-then-exclude approach? (PASS|FAIL|N/A) - [Evidence] -- Is review scope appropriate for change magnitude? (PASS|FAIL|N/A) - [Evidence] -- Was ReviewMark tooling executed and passing? (PASS|FAIL|N/A) - [Evidence] -- Were review artifacts generated correctly? (PASS|FAIL|N/A) - [Evidence] +- Were review-sets updated for structural changes (new/deleted systems, subsystems, or units)? (PASS|FAIL|N/A) - {Evidence} +- Do file patterns follow include-then-exclude approach? (PASS|FAIL|N/A) - {Evidence} +- Is review scope appropriate for change magnitude? (PASS|FAIL|N/A) - {Evidence} +- Was ReviewMark tooling executed and passing? (PASS|FAIL|N/A) - {Evidence} +- Were review artifacts generated correctly? (PASS|FAIL|N/A) - {Evidence} ## Documentation Compliance: (PASS|FAIL|N/A) -- Was README.md updated for user-facing changes? (PASS|FAIL|N/A) - [Evidence] -- Were user guides updated for feature changes? (PASS|FAIL|N/A) - [Evidence] -- Does API documentation reflect code changes? (PASS|FAIL|N/A) - [Evidence] -- Was compliance documentation generated? (PASS|FAIL|N/A) - [Evidence] -- Does documentation follow standards formatting? (PASS|FAIL|N/A) - [Evidence] -- Is documentation organized under `docs/` following standard folder structure? (PASS|FAIL|N/A) - [Evidence] -- Do Pandoc collections include proper `introduction.md` with Purpose and Scope sections? (PASS|FAIL|N/A) - [Evidence] -- Are auto-generated markdown files left unmodified? (PASS|FAIL|N/A) - [Evidence] -- Do README.md files use absolute URLs and include concrete examples? (PASS|FAIL|N/A) - [Evidence] -- Is documentation integrated into ReviewMark review-sets for formal review? (PASS|FAIL|N/A) - [Evidence] +- Was README.md updated for user-facing changes? (PASS|FAIL|N/A) - {Evidence} +- Were user guides updated for feature changes? (PASS|FAIL|N/A) - {Evidence} +- Does API documentation reflect code changes? (PASS|FAIL|N/A) - {Evidence} +- Was compliance documentation generated? (PASS|FAIL|N/A) - {Evidence} +- Does documentation follow standards formatting? (PASS|FAIL|N/A) - {Evidence} +- Is documentation organized under `docs/` following standard folder structure? (PASS|FAIL|N/A) - {Evidence} +- Do Pandoc collections include proper `introduction.md` with Purpose and Scope sections? (PASS|FAIL|N/A) - {Evidence} +- Are auto-generated markdown files left unmodified? (PASS|FAIL|N/A) - {Evidence} +- Do README.md files use absolute URLs and include concrete examples? (PASS|FAIL|N/A) - {Evidence} +- Is documentation integrated into ReviewMark review-sets for formal review? (PASS|FAIL|N/A) - {Evidence} ## Software Item Completeness: (PASS|FAIL|N/A) -- Does every identified software unit have its own requirements file? (PASS|FAIL|N/A) - [Evidence] -- Does every identified software unit have its own design document? (PASS|FAIL|N/A) - [Evidence] -- Does every identified subsystem have its own requirements file? (PASS|FAIL|N/A) - [Evidence] -- Does every identified subsystem have its own design document? (PASS|FAIL|N/A) - [Evidence] +- Does every identified software unit have its own requirements file? (PASS|FAIL|N/A) - {Evidence} +- Does every identified software unit have its own design document? (PASS|FAIL|N/A) - {Evidence} +- Does every identified subsystem have its own requirements file? (PASS|FAIL|N/A) - {Evidence} +- Does every identified subsystem have its own design document? (PASS|FAIL|N/A) - {Evidence} ## Process Compliance: (PASS|FAIL|N/A) -- Was Continuous Compliance workflow followed? (PASS|FAIL|N/A) - [Evidence] -- Did all quality gates execute successfully? (PASS|FAIL|N/A) - [Evidence] -- Were appropriate tools used for validation? (PASS|FAIL|N/A) - [Evidence] -- Were standards consistently applied across work? (PASS|FAIL|N/A) - [Evidence] -- Was compliance evidence generated and preserved? (PASS|FAIL|N/A) - [Evidence] +- Was Continuous Compliance workflow followed? (PASS|FAIL|N/A) - {Evidence} +- Did all quality gates execute successfully? (PASS|FAIL|N/A) - {Evidence} +- Were appropriate tools used for validation? (PASS|FAIL|N/A) - {Evidence} +- Were standards consistently applied across work? (PASS|FAIL|N/A) - {Evidence} +- Was compliance evidence generated and preserved? (PASS|FAIL|N/A) - {Evidence} ## Overall Findings -- **Critical Issues**: [Count and description of critical findings] -- **Recommendations**: [Suggested improvements and next steps] -- **Tools Executed**: [Quality tools used for validation] +- **Critical Issues**: {Count and description of critical findings} +- **Recommendations**: {Suggested improvements and next steps} +- **Tools Executed**: {Quality tools used for validation} ## Compliance Status -- **Standards Adherence**: [Overall compliance rating with specific standards] -- **Quality Gates**: [Status of automated quality checks with tool outputs] +- **Standards Adherence**: {Overall compliance rating with specific standards} +- **Quality Gates**: {Status of automated quality checks with tool outputs} ``` Return this summary to the caller. diff --git a/.github/agents/repo-consistency.agent.md b/.github/agents/repo-consistency.agent.md index b0f93d2..b623895 100644 --- a/.github/agents/repo-consistency.agent.md +++ b/.github/agents/repo-consistency.agent.md @@ -52,29 +52,29 @@ of the project consisting of: ## Consistency Analysis -- **Template PRs Analyzed**: [Number and timeframe of PRs reviewed] -- **Template Changes Identified**: [Count and types of template improvements] -- **Applicable Updates**: [Changes determined suitable for this repository] -- **Project Customizations Preserved**: [Valid differences maintained] +- **Template PRs Analyzed**: {Number and timeframe of PRs reviewed} +- **Template Changes Identified**: {Count and types of template improvements} +- **Applicable Updates**: {Changes determined suitable for this repository} +- **Project Customizations Preserved**: {Valid differences maintained} ## Template Evolution Applied -- **Files Modified**: [List of files updated for template consistency] -- **Improvements Adopted**: [Specific template enhancements implemented] -- **Configuration Updates**: [Tool configurations, workflows, or standards updated] +- **Files Modified**: {List of files updated for template consistency} +- **Improvements Adopted**: {Specific template enhancements implemented} +- **Configuration Updates**: {Tool configurations, workflows, or standards updated} ## Consistency Status -- **Template Alignment**: [Overall consistency rating with template] -- **Customization Respect**: [How project-specific needs were preserved] -- **Functionality Validation**: [Verification that changes don't break existing features] -- **Future Consistency**: [Recommendations for ongoing template alignment] +- **Template Alignment**: {Overall consistency rating with template} +- **Customization Respect**: {How project-specific needs were preserved} +- **Functionality Validation**: {Verification that changes don't break existing features} +- **Future Consistency**: {Recommendations for ongoing template alignment} ## Issues Resolved -- **Drift Corrections**: [Template drift issues addressed] -- **Enhancement Adoptions**: [Template improvements successfully integrated] -- **Validation Results**: [Testing and validation outcomes] +- **Drift Corrections**: {Template drift issues addressed} +- **Enhancement Adoptions**: {Template improvements successfully integrated} +- **Validation Results**: {Testing and validation outcomes} ``` Return this summary to the caller. diff --git a/.github/standards/csharp-testing.md b/.github/standards/csharp-testing.md index 2f26520..f96a3c3 100644 --- a/.github/standards/csharp-testing.md +++ b/.github/standards/csharp-testing.md @@ -13,14 +13,11 @@ requirements. [TestMethod] public void ServiceName_MethodName_Scenario_ExpectedBehavior() { - // Arrange - (description) - // TODO: Set up test data, mocks, and system under test. + // Arrange: description of setup (omit if nothing to set up) - // Act - (description) - // TODO: Execute the action being tested + // Act: description of action (can combine with Assert when action occurs within assertion) - // Assert - (description) - // TODO: Verify expected outcomes and interactions + // Assert: description of verification } ``` @@ -28,7 +25,9 @@ public void ServiceName_MethodName_Scenario_ExpectedBehavior() Use descriptive test names because test names appear in requirements traceability matrices and compliance reports. -- **Pattern**: `ClassName_MethodUnderTest_Scenario_ExpectedBehavior` +- **System tests**: `{SystemName}_{Functionality}_{Scenario}_{ExpectedBehavior}` +- **Subsystem tests**: `{SubsystemName}_{Functionality}_{Scenario}_{ExpectedBehavior}` +- **Unit tests**: `{ClassName}_{MethodUnderTest}_{Scenario}_{ExpectedBehavior}` - **Descriptive Scenarios**: Clearly describe the input condition being tested - **Expected Behavior**: State the expected outcome or exception @@ -110,7 +109,7 @@ Use `Assert.StartsWith` instead, as it produces clearer failure messages: Before submitting C# tests, verify: - [ ] All tests follow AAA pattern with clear section comments -- [ ] Test names follow `ClassName_MethodUnderTest_Scenario_ExpectedBehavior` +- [ ] Test names follow hierarchical patterns defined in Test Naming Standards section - [ ] Each test verifies single, specific behavior (no shared state) - [ ] Both success and failure scenarios covered including edge cases - [ ] External dependencies mocked with NSubstitute or equivalent diff --git a/.github/standards/reqstream-usage.md b/.github/standards/reqstream-usage.md index bd8c739..ff3bc95 100644 --- a/.github/standards/reqstream-usage.md +++ b/.github/standards/reqstream-usage.md @@ -58,6 +58,17 @@ only flow downward in the hierarchy to maintain clear traceability: This prevents circular dependencies and ensures clear hierarchical relationships for compliance auditing. +# Test Linkage Hierarchy + +Requirements MUST link to tests at their own level to maintain proper test scope: + +- **System requirements** → link ONLY to system-level integration tests +- **Subsystem requirements** → link ONLY to subsystem-level tests +- **Unit requirements** → link ONLY to unit-level tests + +Lower-level tests validate implementation details, while higher-level requirements +are validated through integration behavior at their architectural level. + # Requirements File Format ```yaml @@ -69,7 +80,9 @@ sections: justification: | Business rationale explaining why this requirement exists. Include regulatory or standard references where applicable. - tests: + children: # Links to child requirements (optional) + - ChildSystem-Feature-Behavior + tests: # Links to test methods (required) - TestMethodName - windows@PlatformSpecificTest # Source filter for platform evidence ``` @@ -158,6 +171,7 @@ Before submitting requirements, verify: - [ ] Files organized under `docs/reqstream/` following folder structure patterns - [ ] Subsystem folders use kebab-case naming matching source code - [ ] OTS requirements placed in `ots/` subfolder +- [ ] Every software unit has requirements file, design doc, and tests - [ ] Valid YAML syntax passes yamllint validation - [ ] ReqStream enforcement passes: `dotnet reqstream --enforce` - [ ] Test result formats compatible (TRX, JUnit XML) diff --git a/.github/standards/reviewmark-usage.md b/.github/standards/reviewmark-usage.md index 2fdaa19..e2e380a 100644 --- a/.github/standards/reviewmark-usage.md +++ b/.github/standards/reviewmark-usage.md @@ -8,7 +8,7 @@ review, organizes them into review-sets, and generates review plans and reports. ## Key Commands - **Lint Configuration**: `dotnet reviewmark --lint` -- **Elaborate Review-Set**: `dotnet reviewmark --elaborate [review-set]` +- **Elaborate Review-Set**: `dotnet reviewmark --elaborate {review-set}` - **Generate Plan**: `dotnet reviewmark --plan docs/code_review_plan/plan.md` - **Generate Report**: `dotnet reviewmark --report docs/code_review_report/report.md` @@ -29,7 +29,7 @@ Configure reviews in `.reviewmark.yaml` at repository root: needs-review: # Include source code (adjust file extensions for your repo) - "**/*.cs" # C# source files - - "**/*.cpp" # C++ source files + - "**/*.cpp" # C++ source files - "**/*.hpp" # C++ header files - "!**/bin/**" # Generated source in build outputs - "!**/obj/**" # Generated source in build intermediates @@ -48,72 +48,121 @@ evidence-source: type: none ``` +# Review-Set Design Principles + +When constructing review-sets, follow these principles to maintain manageable scope and effective compliance evidence: + +- **Hierarchical Scope**: Higher-level reviews exclude lower-level implementation details, relying instead on design + documents to describe what components they use. System reviews exclude subsystem/unit details, subsystem reviews + exclude unit source code, only unit reviews include actual implementation. +- **Single Focus**: Each review-set proves one specific compliance question (user promises, system architecture, + design consistency, etc.) +- **Context Management**: Keep file counts manageable to prevent context overflow while maintaining complete coverage + through the hierarchy + # Review-Set Organization -Organize review-sets using standard patterns to ensure comprehensive coverage -and consistent review processes: +Organize review-sets using these standard patterns to ensure comprehensive coverage +while keeping each review manageable in scope: + +**Note**: File path patterns shown below use C# naming conventions (PascalCase, `.cs` extensions). +Other languages should adapt these patterns to their conventions (e.g., C++ might use +`snake_case` with `.cpp`/`.hpp` extensions). + +## `Purpose` Review (only one per repository) + +Reviews user-facing capabilities and system promises: + +- **Purpose**: Proves that the systems provide the capabilities the user is being told about +- **Title**: "Review that Advertised Features Match System Design" +- **Scope**: Excludes subsystem and unit files, relying on system-level design documents + to describe what subsystems and units they use +- **File Path Patterns**: + - README: `README.md` + - User guide: `docs/user_guide/**/*.md` + - System requirements: `docs/reqstream/{system-name}/{system-name}.yaml` + - Design introduction: `docs/design/introduction.md` + - System design: `docs/design/{system-name}/{system-name}.md` -## [System]-Architecture Review (one per system) +## `{System}-Architecture` Review (one per system) Reviews system architecture and operational validation: -- **Files**: System requirements (`docs/reqstream/{system-name}/{system-name}.yaml`), design introduction - (`docs/design/introduction.md`), system design (`docs/design/{system-name}/{system-name}.md`), - integration tests -- **Purpose**: Validates system operates as designed and meets overall requirements -- **Example**: `SomeSystem-Architecture` +- **Purpose**: Proves that the system is designed and tested to satisfy its requirements +- **Title**: "Review that {System} Architecture Satisfies Requirements" +- **Scope**: Excludes subsystem and unit files, relying on system-level design to describe + what subsystems and units it uses +- **File Path Patterns**: + - System requirements: `docs/reqstream/{system-name}/{system-name}.yaml` + - Design introduction: `docs/design/introduction.md` + - System design: `docs/design/{system-name}/{system-name}.md` + - System integration tests: `test/{SystemName}.Tests/{SystemName}Tests.cs` -## [System]-Design Review +## `{System}-Design` Review (one per system) Reviews architectural and design consistency: -- **Files**: System requirements, platform requirements, all design documents under `docs/design/` -- **Purpose**: Ensures design completeness and architectural coherence -- **Example**: `SomeSystem-Design` +- **Purpose**: Proves the system design is consistent and complete +- **Title**: "Review that {System} Design is Consistent and Complete" +- **Scope**: Only brings in top-level requirements and relies on brevity of design documentation +- **File Path Patterns**: + - System requirements: `docs/reqstream/{system-name}/{system-name}.yaml` + - Platform requirements: `docs/reqstream/{system-name}/platform-requirements.yaml` + - Design introduction: `docs/design/introduction.md` + - System design files: `docs/design/{system-name}/**/*.md` -## [System]-AllRequirements Review +## `{System}-AllRequirements` Review (one per system) Reviews requirements quality and traceability: -- **Files**: All requirement files including root `requirements.yaml` and all files under `docs/reqstream/{system-name}/` -- **Purpose**: Validates requirements structure, IDs, justifications, and test linkage -- **Example**: `SomeSystem-AllRequirements` +- **Purpose**: Proves the requirements are consistent and complete +- **Title**: "Review that All {System} Requirements are Complete" +- **Scope**: Only brings in requirements files to keep review manageable +- **File Path Patterns**: + - Root requirements: `requirements.yaml` + - System requirements: `docs/reqstream/{system-name}/**/*.yaml` + - OTS requirements: `docs/reqstream/ots/**/*.yaml` (if applicable) -## [System]-[Subsystem] Review +## `{System}-{Subsystem}` Review (one per subsystem) Reviews subsystem architecture and interfaces: -- **Files**: Subsystem requirements, design documents, integration tests (usually no source code) -- **Purpose**: Validates subsystem behavior and interface compliance -- **File Path Pattern**: +- **Purpose**: Proves that the subsystem is designed and tested to satisfy its requirements +- **Title**: "Review that {System} {Subsystem} Satisfies Subsystem Requirements" +- **Scope**: Excludes units under the subsystem, relying on subsystem design to describe + what units it uses +- **File Path Patterns**: - Requirements: `docs/reqstream/{system-name}/{subsystem-name}/{subsystem-name}.yaml` - Design: `docs/design/{system-name}/{subsystem-name}/{subsystem-name}.md` - - Tests: `test/{SystemName}.Tests/{SubsystemName}/{SubsystemName}*` or similar -- **Example**: `SomeSystem-Authentication`, `SomeSystem-DataLayer` + - Tests: `test/{SystemName}.Tests/{SubsystemName}/{SubsystemName}Tests.cs` -## [System]-[Subsystem]-[Unit] Review +## `{System}-{Subsystem}-{Unit}` Review (one per unit) Reviews individual software unit implementation: -- **Files**: Unit requirements, design documents, source code, unit tests -- **Purpose**: Validates unit meets requirements and is properly implemented -- **File Path Pattern**: - - Requirements: `docs/reqstream/{system-name}/{subsystem-name}/{unit-name}.yaml` or `docs/reqstream/{system-name}/{unit-name}.yaml` - - Design: `docs/design/{system-name}/{subsystem-name}/{unit-name}.md` or `docs/design/{system-name}/{unit-name}.md` - - Source: `src/{SystemName}/{SubsystemName}/{UnitName}.cs` - - Tests: `test/{SystemName}.Tests/{SubsystemName}/{UnitName}Tests.cs` -- **Example**: `SomeSystem-Authentication-PasswordValidator`, `SomeSystem-DataLayer-ConfigParser` +- **Purpose**: Proves the unit is designed, implemented, and tested to satisfy its requirements +- **Title**: "Review that {System} {Subsystem} {Unit} Implementation is Correct" +- **Scope**: Complete unit review including all artifacts +- **File Path Patterns**: + - Requirements: `docs/reqstream/{system-name}/{subsystem-name}/{unit-name}.yaml` or + `docs/reqstream/{system-name}/{unit-name}.yaml` + - Design: `docs/design/{system-name}/{subsystem-name}/{unit-name}.md` or + `docs/design/{system-name}/{unit-name}.md` + - Source: `src/{SystemName}/{SubsystemName}/{UnitName}.cs` or `src/{SystemName}/{UnitName}.cs` + - Tests: `test/{SystemName}.Tests/{SubsystemName}/{UnitName}Tests.cs` or + `test/{SystemName}.Tests/{UnitName}Tests.cs` # Quality Checks Before submitting ReviewMark configuration, verify: - [ ] `.reviewmark.yaml` exists at repository root with proper structure -- [ ] `needs-review` patterns cover requirements, design, code, and tests with proper exclusions -- [ ] Each review-set has unique `id` and groups architecturally related files +- [ ] Review-set organization follows the standard hierarchy patterns +- [ ] Purpose review-set includes README.md, user guide, system requirements, design introduction, and system design files +- [ ] System-level reviews follow hierarchical scope principle (exclude subsystem/unit details) +- [ ] Subsystem reviews follow hierarchical scope principle (exclude unit source code) +- [ ] Only unit reviews include actual source code files +- [ ] Each review-set focuses on a single compliance question (single focus principle) - [ ] File patterns use correct glob syntax and match intended files -- [ ] File paths reflect current naming conventions (kebab-case design/requirements folders, PascalCase source folders) +- [ ] Review-set file counts remain manageable (context management principle) - [ ] Evidence source properly configured (`none` for dev, `url` for production) -- [ ] Environment variables used for credentials (never hardcoded) -- [ ] Generated documents accessible for compliance auditing -- [ ] Review-set organization follows standard patterns ([System]-[Subsystem], [System]-Design, etc.) diff --git a/.github/standards/software-items.md b/.github/standards/software-items.md index 7991add..ce7e328 100644 --- a/.github/standards/software-items.md +++ b/.github/standards/software-items.md @@ -18,6 +18,11 @@ Categorize all software into four primary groups: - **OTS Software Item**: Third-party component (library, framework, tool) providing functionality not developed in-house +**Naming**: When names collide in hierarchy, add descriptive suffix to higher-level entity: + +- System: Application/Library/System (e.g. TestResults → TestResultsLibrary) +- Subsystem: Subsystem (e.g. Linter → LinterSubsystem) + # Categorization Guidelines Choose the appropriate category based on scope and testability: diff --git a/.reviewmark.yaml b/.reviewmark.yaml index d071e17..fa5b4a6 100644 --- a/.reviewmark.yaml +++ b/.reviewmark.yaml @@ -28,16 +28,80 @@ evidence-source: # - source: what the code actually does # - tests: which behaviors are verified and how reviews: + # Purpose review - proves advertised features match system design + - id: Purpose + title: Review that Advertised Features Match System Design + paths: + - "README.md" + - "docs/user_guide/**/*.md" + - "docs/reqstream/review-mark/review-mark.yaml" + - "docs/design/introduction.md" + - "docs/design/review-mark/review-mark.md" + + # Special review-sets (system-level) + - id: ReviewMark-Architecture + title: Review of ReviewMark system-level behavior, platform support, and integration + paths: + - "docs/reqstream/review-mark/review-mark.yaml" # system requirements + - "docs/reqstream/review-mark/platform-requirements.yaml" # platform requirements + - "docs/design/introduction.md" # design introduction and architecture + - "docs/design/review-mark/review-mark.md" # system design + - "test/**/IntegrationTests.cs" # integration tests + - "test/**/Runner.cs" # test infrastructure + - "test/**/AssemblyInfo.cs" # test infrastructure + + - id: ReviewMark-Design + title: Review of all ReviewMark design documentation + paths: + - "docs/reqstream/review-mark/review-mark.yaml" # system requirements + - "docs/reqstream/review-mark/platform-requirements.yaml" # platform requirements + - "docs/design/introduction.md" # design introduction + - "docs/design/review-mark/**/*.md" # system design documents + + - id: ReviewMark-AllRequirements + title: Review of all ReviewMark requirements files + paths: + - "requirements.yaml" # root requirements file + - "docs/reqstream/review-mark/**/*.yaml" # all review-mark requirements files + - "docs/reqstream/ots/**/*.yaml" # all OTS requirements files + + # Subsystem reviews - one per subsystem (no unit source code) + - id: ReviewMark-Cli + title: Review of Cli subsystem (command-line interface) + paths: + - "docs/reqstream/review-mark/cli/cli.yaml" # subsystem requirements + - "docs/design/review-mark/cli/cli.md" # Cli subsystem design + - "test/**/Cli/CliTests.cs" # Cli subsystem tests + + - id: ReviewMark-Configuration + title: Review of Configuration subsystem (configuration parsing and file pattern matching) + paths: + - "docs/reqstream/review-mark/configuration/configuration.yaml" # subsystem requirements + - "docs/design/review-mark/configuration/configuration.md" # Configuration subsystem design + - "test/**/Configuration/ConfigurationTests.cs" # Configuration subsystem tests + + - id: ReviewMark-Indexing + title: Review of Indexing subsystem (review evidence loading and path utilities) + paths: + - "docs/reqstream/review-mark/indexing/indexing.yaml" # subsystem requirements + - "docs/design/review-mark/indexing/indexing.md" # Indexing subsystem design + - "test/**/Indexing/IndexingTests.cs" # Indexing subsystem tests + + - id: ReviewMark-SelfTest + title: Review of SelfTest subsystem (self-validation) + paths: + - "docs/reqstream/review-mark/self-test/self-test.yaml" # subsystem requirements + - "docs/design/review-mark/self-test/self-test.md" # SelfTest subsystem design + - "test/**/SelfTest/SelfTestTests.cs" # SelfTest subsystem tests + # Software unit reviews - one per unit - id: ReviewMark-Program title: Review of Program software unit (main entry point and tool orchestration) paths: - "docs/reqstream/review-mark/program.yaml" # requirements - "docs/design/review-mark/program.md" # design - - "docs/user_guide/introduction.md" # user guide - "src/**/Program.cs" # implementation - "test/**/ProgramTests.cs" # unit tests - - "test/**/TestDirectory.cs" # test infrastructure - id: ReviewMark-Cli-Context title: Review of Context software unit (command-line argument handling) @@ -86,56 +150,3 @@ reviews: - "docs/design/review-mark/self-test/validation.md" # design - "src/**/SelfTest/Validation.cs" # implementation - "test/**/SelfTest/ValidationTests.cs" # tests - - # Subsystem reviews - - id: ReviewMark-Cli - title: Review of Cli subsystem (command-line interface) - paths: - - "docs/reqstream/review-mark/cli/cli.yaml" # subsystem requirements - - "docs/design/review-mark/cli/cli.md" # Cli subsystem design - - "test/**/Cli/CliTests.cs" # Cli subsystem tests - - - id: ReviewMark-Configuration - title: Review of Configuration subsystem (configuration parsing and file pattern matching) - paths: - - "docs/reqstream/review-mark/configuration/configuration.yaml" # subsystem requirements - - "docs/design/review-mark/configuration/configuration.md" # Configuration subsystem design - - "test/**/Configuration/ConfigurationTests.cs" # Configuration subsystem tests - - - id: ReviewMark-Indexing - title: Review of Indexing subsystem (review evidence loading and path utilities) - paths: - - "docs/reqstream/review-mark/indexing/indexing.yaml" # subsystem requirements - - "docs/design/review-mark/indexing/indexing.md" # Indexing subsystem design - - "test/**/Indexing/IndexingTests.cs" # Indexing subsystem tests - - - id: ReviewMark-SelfTest - title: Review of SelfTest subsystem (self-validation) - paths: - - "docs/reqstream/review-mark/self-test/self-test.yaml" # subsystem requirements - - "docs/design/review-mark/self-test/self-test.md" # SelfTest subsystem design - - "test/**/SelfTest/SelfTestTests.cs" # SelfTest subsystem tests - - # Special review-sets - - id: ReviewMark-Architecture - title: Review of ReviewMark system-level behavior, platform support, and integration - paths: - - "docs/reqstream/review-mark/review-mark.yaml" # system requirements - - "docs/reqstream/review-mark/platform-requirements.yaml" # platform requirements - - "docs/design/introduction.md" # design introduction and architecture - - "docs/design/review-mark/review-mark.md" # system design - - "test/**/IntegrationTests.cs" # integration tests - - "test/**/Runner.cs" # test infrastructure - - "test/**/AssemblyInfo.cs" # test infrastructure - - - id: ReviewMark-Design - title: Review of all ReviewMark design documentation - paths: - - "docs/reqstream/review-mark/platform-requirements.yaml" # platform requirements - - "docs/design/**/*.md" # all design documents - - - id: ReviewMark-AllRequirements - title: Review of all ReviewMark requirements files - paths: - - "requirements.yaml" # root requirements file - - "docs/reqstream/**/*.yaml" # all requirements files diff --git a/docs/reqstream/review-mark/cli/cli.yaml b/docs/reqstream/review-mark/cli/cli.yaml index 673b6ec..20c7290 100644 --- a/docs/reqstream/review-mark/cli/cli.yaml +++ b/docs/reqstream/review-mark/cli/cli.yaml @@ -15,13 +15,7 @@ sections: Provides a standardized approach to command-line argument parsing and output handling across all DEMA Consulting DotNet Tools. tests: - - Context_Create_NoArguments_ReturnsDefaultContext - - Context_Create_VersionFlag_SetsVersionTrue - - Context_Create_HelpFlag_SetsHelpTrue - - Context_Create_SilentFlag_SetsSilentTrue - - Context_Create_ValidateFlag_SetsValidateTrue - - Context_Create_ResultsFlag_SetsResultsFile - - Context_Create_LogFlag_OpensLogFile + - Cli_VersionFlag_OutputsVersionOnly children: [ReviewMark-Context-Parsing, ReviewMark-Context-Output] - id: ReviewMark-Cmd-Version @@ -30,11 +24,6 @@ sections: Users need to quickly identify the version of the tool they are using for troubleshooting and compatibility verification. tests: - - Context_Create_VersionFlag_SetsVersionTrue - - Context_Create_ShortVersionFlag_SetsVersionTrue - - Program_Run_WithVersionFlag_DisplaysVersionOnly - - Program_Version_ReturnsNonEmptyString - - IntegrationTest_VersionFlag_OutputsVersion - Cli_VersionFlag_OutputsVersionOnly children: [ReviewMark-Program-Dispatch] @@ -44,11 +33,6 @@ sections: Users need access to command-line usage documentation without requiring external resources. tests: - - Context_Create_HelpFlag_SetsHelpTrue - - Context_Create_ShortHelpFlag_H_SetsHelpTrue - - Context_Create_ShortHelpFlag_Question_SetsHelpTrue - - Program_Run_WithHelpFlag_DisplaysUsageInformation - - IntegrationTest_HelpFlag_OutputsUsageInformation - Cli_HelpFlag_OutputsUsageInformation children: [ReviewMark-Program-Dispatch] @@ -58,9 +42,6 @@ sections: Enables automated scripts and CI/CD pipelines to run the tool without cluttering output logs. tests: - - Context_Create_SilentFlag_SetsSilentTrue - - Context_WriteLine_Silent_DoesNotWriteToConsole - - IntegrationTest_SilentFlag_SuppressesOutput - Cli_SilentFlag_SuppressesOutput children: [ReviewMark-Context-Output] @@ -70,9 +51,6 @@ sections: Provides a built-in mechanism to verify the tool is functioning correctly in the deployment environment. tests: - - Context_Create_ValidateFlag_SetsValidateTrue - - Program_Run_WithValidateFlag_RunsValidation - - IntegrationTest_ValidateFlag_RunsValidation - Cli_ValidateFlag_RunsValidation children: [ReviewMark-Program-Dispatch, ReviewMark-Validation-Run] @@ -81,9 +59,7 @@ sections: justification: | Enables integration with CI/CD systems that expect standard test result formats. tests: - - Context_Create_ResultsFlag_SetsResultsFile - - IntegrationTest_ValidateWithResults_GeneratesTrxFile - - IntegrationTest_ValidateWithResults_GeneratesJUnitFile + - Cli_ResultsFlag_GeneratesTrxFile children: [ReviewMark-Validation-ResultsFile] - id: ReviewMark-Cmd-Log @@ -91,8 +67,7 @@ sections: justification: | Provides persistent logging for debugging and audit trails. tests: - - Context_Create_LogFlag_OpensLogFile - - IntegrationTest_LogFlag_WritesOutputToFile + - Cli_LogFlag_WritesOutputToFile children: [ReviewMark-Context-Output] - id: ReviewMark-Cmd-ErrorOutput @@ -101,8 +76,7 @@ sections: Error messages must be written to stderr so they remain visible to the user without polluting stdout, which consumers may pipe or redirect for data capture. tests: - - Context_WriteError_NotSilent_WritesToConsole - - IntegrationTest_UnknownArgument_ReturnsError + - Cli_ErrorOutput_WritesToStderr children: [ReviewMark-Context-Output] - id: ReviewMark-Cmd-InvalidArgs @@ -111,10 +85,7 @@ sections: Providing clear feedback for invalid arguments helps users quickly correct mistakes and prevents silent misconfiguration. tests: - - Context_Create_UnknownArgument_ThrowsArgumentException - - Context_Create_LogFlag_WithoutValue_ThrowsArgumentException - - Context_Create_ResultsFlag_WithoutValue_ThrowsArgumentException - - IntegrationTest_UnknownArgument_ReturnsError + - Cli_InvalidArgs_ReturnsNonZeroExitCode children: [ReviewMark-Context-Parsing] - id: ReviewMark-Cmd-ExitCode @@ -123,8 +94,7 @@ sections: Callers (scripts, CI/CD pipelines) must be able to detect failure conditions programmatically via the process exit code. tests: - - Context_WriteError_SetsErrorExitCode - - IntegrationTest_UnknownArgument_ReturnsError + - Cli_ExitCode_ReturnsNonZeroOnError children: [ReviewMark-Context-Output] - id: ReviewMark-Cmd-Definition @@ -133,10 +103,7 @@ sections: Users must be able to specify the path to the .reviewmark.yaml definition file, which configures needs-review patterns, evidence source, and review set definitions. tests: - - Context_Create_DefinitionFlag_SetsDefinitionFile - - Context_Create_DefinitionFlag_WithoutValue_ThrowsArgumentException - - ReviewMark_ReviewPlanGeneration - - ReviewMark_ReviewReportGeneration + - Cli_DefinitionFlag_LoadsSpecifiedFile children: [ReviewMark-Config-Loading, ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-Plan @@ -145,8 +112,7 @@ sections: Enables automated generation of a review plan document that lists all review sets and coverage status, suitable for inclusion in release documentation. tests: - - Context_Create_PlanFlag_SetsPlanFile - - ReviewMark_ReviewPlanGeneration + - Cli_PlanFlag_GeneratesReviewPlan children: [ReviewMark-Config-Reading, ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-PlanDepth @@ -155,11 +121,7 @@ sections: Allows the review plan to be embedded at any heading level within a larger Markdown document, with a default depth of 1 when not specified. tests: - - Context_Create_PlanDepthFlag_SetsPlanDepth - - Context_Create_PlanDepthFlag_WithInvalidValue_ThrowsArgumentException - - Context_Create_PlanDepthFlag_WithZeroValue_ThrowsArgumentException - - Context_Create_PlanDepthFlag_WithValueGreaterThanFive_ThrowsArgumentException - - Context_Create_NoArguments_PlanDepthDefaultsToOne + - Cli_PlanDepthFlag_SetsHeadingDepth children: [ReviewMark-Context-Parsing] - id: ReviewMark-Cmd-Report @@ -168,8 +130,7 @@ sections: Enables automated generation of a review report document showing the current status of each review set against the evidence index, suitable for release documentation. tests: - - Context_Create_ReportFlag_SetsReportFile - - ReviewMark_ReviewReportGeneration + - Cli_ReportFlag_GeneratesReviewReport children: [ReviewMark-Config-Reading, ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-ReportDepth @@ -178,12 +139,7 @@ sections: Allows the review report to be embedded at any heading level within a larger Markdown document, with a default depth of 1 when not specified. tests: - - Context_Create_ReportDepthFlag_SetsReportDepth - - Context_Create_ReportDepthFlag_MissingValue_ThrowsArgumentException - - Context_Create_ReportDepthFlag_NonNumeric_ThrowsArgumentException - - Context_Create_ReportDepthFlag_Zero_ThrowsArgumentException - - Context_Create_ReportDepthFlag_WithValueGreaterThanFive_ThrowsArgumentException - - Context_Create_NoArguments_ReportDepthDefaultsToOne + - Cli_ReportDepthFlag_SetsHeadingDepth children: [ReviewMark-Context-Parsing] - id: ReviewMark-Cmd-Index @@ -194,10 +150,7 @@ sections: files, reading embedded metadata from each PDF's Keywords field to populate the index with review IDs, fingerprints, dates, results, and file names. tests: - - Context_Create_IndexFlag_AddsIndexPath - - Context_Create_IndexFlag_MultipleTimes_AddsAllPaths - - Context_Create_NoArguments_IndexPathsEmpty - - ReviewMark_IndexScan + - Cli_IndexFlag_CreatesIndexJson children: [ReviewMark-Index-PdfParsing, ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-Enforce @@ -207,9 +160,7 @@ sections: stale, or missing, or when files requiring review are not covered by any review-set. Without --enforce the tool generates the plan and report but exits with code 0. tests: - - Context_Create_EnforceFlag_SetsEnforceTrue - - Context_Create_NoArguments_EnforceFalse - - ReviewMark_Enforce + - Cli_EnforceFlag_ExitsNonZeroWhenNotCurrent children: [ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-Dir @@ -219,10 +170,7 @@ sections: the process working directory, enabling consistent scripting and CI/CD usage without requiring a cd command before invoking the tool. tests: - - Context_Create_DirFlag_SetsWorkingDirectory - - Context_Create_NoArguments_WorkingDirectoryIsNull - - Context_Create_DirFlag_MissingValue_ThrowsArgumentException - - ReviewMark_WorkingDirectoryOverride + - Cli_DirFlag_SetsWorkingDirectory children: [ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-Elaborate @@ -233,19 +181,7 @@ sections: command provides this information formatted as Markdown so it can be copied directly into review documentation. tests: - - Context_Create_ElaborateFlag_SetsElaborateId - - Context_Create_NoArguments_ElaborateIdIsNull - - Context_Create_ElaborateFlag_WithoutValue_ThrowsArgumentException - - ReviewMarkConfiguration_ElaborateReviewSet_ValidId_ReturnsElaboration - - ReviewMarkConfiguration_ElaborateReviewSet_UnknownId_ThrowsArgumentException - - ReviewMarkConfiguration_ElaborateReviewSet_NullId_ThrowsArgumentException - - ReviewMarkConfiguration_ElaborateReviewSet_MarkdownDepth_UsedForHeadings - - ReviewMarkConfiguration_ElaborateReviewSet_MarkdownDepthAbove5_Throws - - ReviewMarkConfiguration_ElaborateReviewSet_ContainsFullFingerprint - - Program_Run_WithHelpFlag_IncludesElaborateOption - - Program_Run_WithElaborateFlag_OutputsElaboration - - Program_Run_WithElaborateFlag_UnknownId_ReportsError - - ReviewMark_Elaborate + - Cli_ElaborateFlag_OutputsElaboration children: [ReviewMark-Config-Reading, ReviewMark-Program-Dispatch] - id: ReviewMark-Cmd-Lint @@ -255,18 +191,5 @@ sections: before running the main tool, providing clear error messages about the cause and location of any issues. tests: - - Context_Create_LintFlag_SetsLintTrue - - Context_Create_NoArguments_LintIsFalse - - Program_Run_WithHelpFlag_IncludesLintOption - - Program_Run_WithLintFlag_ValidConfig_ReportsSuccess - - Program_Run_WithLintFlag_MissingConfig_ReportsError - - Program_Run_WithLintFlag_DuplicateIds_ReportsError - - Program_Run_WithLintFlag_UnknownSourceType_ReportsError - - Program_Run_WithLintFlag_CorruptedYaml_ReportsError - - Program_Run_WithLintFlag_MissingEvidenceSource_ReportsError - - Program_Run_WithLintFlag_MultipleErrors_ReportsAll - - ReviewMarkConfiguration_Load_InvalidYaml_ReturnsNullConfigWithErrorIssue - - ReviewMarkConfiguration_Load_MissingEvidenceSource_ReturnsNullConfigWithErrorIssue - - ReviewMarkConfiguration_Load_MultipleErrors_ReturnsAllIssues - - ReviewMark_Lint + - Cli_LintFlag_ReportsSuccess children: [ReviewMark-Config-Loading, ReviewMark-Program-Dispatch] diff --git a/docs/reqstream/review-mark/configuration/configuration.yaml b/docs/reqstream/review-mark/configuration/configuration.yaml index a634e22..5ebdda3 100644 --- a/docs/reqstream/review-mark/configuration/configuration.yaml +++ b/docs/reqstream/review-mark/configuration/configuration.yaml @@ -18,7 +18,6 @@ sections: and excludes in declaration order, so that ReviewMark can detect uncovered files and generate accurate review plans. tests: - - ReviewMarkConfiguration_GetNeedsReviewFiles_ReturnsMatchingFiles - Configuration_LoadConfig_ResolvesNeedsReviewFiles children: [ReviewMark-Config-Reading, ReviewMark-GlobMatcher-IncludeExclude] @@ -30,9 +29,6 @@ sections: content rather than names alone, so that renamed files do not invalidate the fingerprint, and changed content always produces a new fingerprint. tests: - - ReviewSet_GetFingerprint_SameContent_ReturnsSameFingerprint - - ReviewSet_GetFingerprint_DifferentContent_ReturnsDifferentFingerprint - - ReviewSet_GetFingerprint_RenameFile_ReturnsSameFingerprint - Configuration_LoadConfig_FingerprintReflectsFileContent children: [ReviewMark-Config-Reading] @@ -43,7 +39,6 @@ sections: and what files they cover. It enables auditors to verify that all relevant files are included in at least one review-set before reviews are conducted. tests: - - ReviewMark_ReviewPlanGeneration - Configuration_LoadConfig_PlanGenerationSucceeds children: [ReviewMark-Config-Reading, ReviewMark-Config-Loading] @@ -54,7 +49,7 @@ sections: of each review-set (Current, Stale, Missing, or Failed), enabling auditors to confirm that all review-sets have current evidence before a release. tests: - - ReviewMark_ReviewReportGeneration + - Configuration_LoadConfig_ReportGenerationSucceeds children: [ReviewMark-Config-Reading, ReviewMark-Config-Loading] - id: ReviewMark-Configuration-Elaboration @@ -65,5 +60,5 @@ sections: command provides this formatted as Markdown so it can be copied directly into review documentation. tests: - - ReviewMark_Elaborate + - Configuration_LoadConfig_ElaborationSucceeds children: [ReviewMark-Config-Reading] diff --git a/docs/reqstream/review-mark/configuration/review-mark-configuration.yaml b/docs/reqstream/review-mark/configuration/review-mark-configuration.yaml index ef5bd0b..ee92a7b 100644 --- a/docs/reqstream/review-mark/configuration/review-mark-configuration.yaml +++ b/docs/reqstream/review-mark/configuration/review-mark-configuration.yaml @@ -44,4 +44,3 @@ sections: - ReviewMarkConfiguration_Load_MultipleErrors_ReturnsAllIssues - ReviewMarkConfiguration_Load_NoneEvidenceSource_NoIssues - ReviewMarkLoadResult_ReportIssues_RoutesIssuesToContext - - Program_Run_WithDefinitionFlag_InvalidConfig_ReportsLintError diff --git a/docs/reqstream/review-mark/indexing/indexing.yaml b/docs/reqstream/review-mark/indexing/indexing.yaml index e82871d..3b4ba91 100644 --- a/docs/reqstream/review-mark/indexing/indexing.yaml +++ b/docs/reqstream/review-mark/indexing/indexing.yaml @@ -21,11 +21,6 @@ sections: downloading evidence over HTTP(S), enabling centralized evidence stores accessible from any CI/CD environment. tests: - - ReviewIndex_Load_EvidenceSource_None_ReturnsEmptyIndex - - ReviewIndex_Load_EvidenceSource_None_HttpClientOverload_ReturnsEmptyIndex - - ReviewIndex_Load_EvidenceSource_Fileshare_LoadsFromFile - - ReviewIndex_Load_EvidenceSource_Fileshare_ValidJson_ReturnsPopulatedIndex - - ReviewIndex_Load_EvidenceSource_Url_SuccessResponse_LoadsIndex - Indexing_SafePathCombine_WithIndexPath_LoadsIndex children: [ReviewMark-Index-EvidenceSource, ReviewMark-EvidenceSource-None] @@ -37,15 +32,6 @@ sections: and extract the review ID, fingerprint, date, and result from each file to populate the evidence index used for report generation. tests: - - ReviewIndex_Scan_PdfWithValidMetadata_PopulatesIndex - - ReviewIndex_Scan_MultiplePdfs_PopulatesAllEntries - - ReviewIndex_Scan_NoMatchingFiles_LeavesIndexEmpty - - ReviewIndex_Scan_ClearsExistingEntries - - ReviewIndex_Scan_PdfWithMissingId_SkipsWithWarning - - ReviewIndex_Scan_PdfWithMissingFingerprint_SkipsWithWarning - - ReviewIndex_Scan_PdfWithMissingDate_SkipsWithWarning - - ReviewIndex_Scan_PdfWithMissingResult_SkipsWithWarning - - ReviewIndex_Scan_PdfWithNoKeywords_SkipsWithWarning - Indexing_ReviewIndex_SaveAndLoad_RoundTrip children: [ReviewMark-Index-PdfParsing] @@ -57,11 +43,6 @@ sections: sequences to prevent unintended file system access in both evidence scanning and index file operations. tests: - - PathHelpers_SafePathCombine_ValidPaths_CombinesCorrectly - - PathHelpers_SafePathCombine_PathTraversalWithDoubleDots_ThrowsArgumentException - - PathHelpers_SafePathCombine_DoubleDotsInMiddle_ThrowsArgumentException - - PathHelpers_SafePathCombine_AbsolutePath_ThrowsArgumentException - - PathHelpers_SafePathCombine_CurrentDirectoryReference_CombinesCorrectly - - PathHelpers_SafePathCombine_NestedPaths_CombinesCorrectly - - PathHelpers_SafePathCombine_EmptyRelativePath_ReturnsBasePath + - Indexing_SafePathCombine_WithIndexPath_LoadsIndex + - Indexing_SafePathCombine_WithTraversalInputs_Throws children: [ReviewMark-PathHelpers-SafeCombine] diff --git a/docs/reqstream/review-mark/indexing/review-index.yaml b/docs/reqstream/review-mark/indexing/review-index.yaml index 052c327..1c8c779 100644 --- a/docs/reqstream/review-mark/indexing/review-index.yaml +++ b/docs/reqstream/review-mark/indexing/review-index.yaml @@ -45,9 +45,6 @@ sections: tests: - ReviewIndex_Load_EvidenceSource_None_ReturnsEmptyIndex - ReviewIndex_Load_EvidenceSource_None_HttpClientOverload_ReturnsEmptyIndex - - ReviewMarkConfiguration_Parse_NoneEvidenceSource_ParsedCorrectly - - ReviewMarkConfiguration_Parse_NoneEvidenceSource_NoLocationRequired - - ReviewMarkConfiguration_Load_NoneEvidenceSource_NoIssues - id: ReviewMark-Index-PdfParsing title: The tool shall parse PDF metadata from the Keywords field when indexing evidence files. diff --git a/docs/reqstream/review-mark/review-mark.yaml b/docs/reqstream/review-mark/review-mark.yaml index 9e890ed..1e79eb0 100644 --- a/docs/reqstream/review-mark/review-mark.yaml +++ b/docs/reqstream/review-mark/review-mark.yaml @@ -18,7 +18,10 @@ sections: is covered by at least one named review-set. The Review Plan document provides this evidence automatically on each CI/CD run, replacing manual tracking spreadsheets. tests: - - ReviewMark_ReviewPlanGeneration + - IntegrationTest_ReviewPlanGeneration + children: + - ReviewMark-Cmd-Plan + - ReviewMark-Configuration-PlanGeneration - id: ReviewMark-System-ReviewReport title: The tool shall generate a Review Report document listing every review-set and its current review status. @@ -28,7 +31,10 @@ sections: Report provides this evidence automatically, showing Current, Stale, Missing, or Failed status for each review-set. tests: - - ReviewMark_ReviewReportGeneration + - IntegrationTest_ReviewReportGeneration + children: + - ReviewMark-Cmd-Report + - ReviewMark-Configuration-ReportGeneration - id: ReviewMark-System-Enforce title: The tool shall return a non-zero exit code when enforcement is enabled and any review-set is not current. @@ -40,7 +46,9 @@ sections: Failed status. This makes incomplete file coverage, out-of-date reviews, and failed reviews all build-breaking conditions. tests: - - ReviewMark_Enforce + - IntegrationTest_Enforce + children: + - ReviewMark-Cmd-Enforce - id: ReviewMark-System-IndexScan title: The tool shall scan PDF evidence files and write an index.json when the --index flag is provided. @@ -50,7 +58,10 @@ sections: index.json, enabling the evidence store to be refreshed after new review PDFs are added without manual maintenance of the index file. tests: - - ReviewMark_IndexScan + - IntegrationTest_IndexScan + children: + - ReviewMark-Cmd-Index + - ReviewMark-Indexing-ScanPdfEvidence - id: ReviewMark-System-Validate title: The tool shall execute self-validation tests when the --validate flag is provided. @@ -59,12 +70,97 @@ sections: functions correctly in its specific deployment environment. The --validate flag triggers a built-in test suite that exercises core tool behaviors and produces a pass/fail report. tests: - - ReviewMark_VersionDisplay - - ReviewMark_HelpDisplay - - ReviewMark_ReviewPlanGeneration - - ReviewMark_ReviewReportGeneration - - ReviewMark_IndexScan - - ReviewMark_Enforce - - ReviewMark_WorkingDirectoryOverride - - ReviewMark_Elaborate - - ReviewMark_Lint + - IntegrationTest_ValidateFlag_RunsValidation + children: + - ReviewMark-Cmd-Validate + - ReviewMark-SelfTest-Qualification + + - id: ReviewMark-System-Version + title: The tool shall display the version string when the --version flag is provided. + justification: | + Users need to quickly identify the version of the tool they are using for + troubleshooting and compatibility verification. + tests: + - IntegrationTest_VersionFlag_OutputsVersion + children: + - ReviewMark-Cmd-Version + + - id: ReviewMark-System-Help + title: The tool shall display usage information when the --help flag is provided. + justification: | + Users need access to command-line usage documentation without requiring external resources. + tests: + - IntegrationTest_HelpFlag_OutputsUsageInformation + children: + - ReviewMark-Cmd-Help + + - id: ReviewMark-System-WorkingDirectory + title: The tool shall support a --dir flag to set the working directory for file operations. + justification: | + Allows users to target an evidence store or project directory without changing + the process working directory, enabling consistent scripting and CI/CD usage. + tests: + - IntegrationTest_WorkingDirectoryOverride + children: + - ReviewMark-Cmd-Dir + + - id: ReviewMark-System-Elaborate + title: The tool shall print a Markdown elaboration of a review set when --elaborate is provided. + justification: | + When preparing for a review, the reviewer needs the review set ID, its current + fingerprint, and the full sorted list of files to be reviewed. + tests: + - IntegrationTest_Elaborate + children: + - ReviewMark-Cmd-Elaborate + + - id: ReviewMark-System-Lint + title: The tool shall validate the definition file and report issues when --lint is provided. + justification: | + Users need a way to verify that the .reviewmark.yaml configuration file is valid + before running the main tool. + tests: + - IntegrationTest_Lint + children: + - ReviewMark-Cmd-Lint + + - id: ReviewMark-System-Silent + title: The tool shall support --silent flag to suppress console output. + justification: | + Enables automated scripts and CI/CD pipelines to run the tool without cluttering + output logs when only the exit code is needed. + tests: + - IntegrationTest_SilentFlag_SuppressesOutput + children: + - ReviewMark-Cmd-Silent + + - id: ReviewMark-System-Log + title: The tool shall support --log flag to write output to a persistent log file. + justification: | + Provides persistent logging for debugging and audit trails when running in CI/CD + environments where console output may not be captured. + tests: + - IntegrationTest_LogFlag_WritesOutputToFile + children: + - ReviewMark-Cmd-Log + + - id: ReviewMark-System-InvalidArgs + title: The tool shall reject unknown command-line arguments with a non-zero exit code. + justification: | + Providing clear feedback for invalid arguments helps users quickly correct mistakes + and prevents silent misconfiguration in automated environments. + tests: + - IntegrationTest_UnknownArgument_ReturnsError + children: + - ReviewMark-Cmd-InvalidArgs + + - id: ReviewMark-System-Results + title: The tool shall write validation results to a standard test result file when --results is provided. + justification: | + Enables integration with CI/CD systems and requirements traceability tools that + expect standard TRX or JUnit XML test result formats. + tests: + - IntegrationTest_ValidateWithResults_GeneratesTrxFile + - IntegrationTest_ValidateWithResults_GeneratesJUnitFile + children: + - ReviewMark-Cmd-Results diff --git a/docs/reqstream/review-mark/self-test/self-test.yaml b/docs/reqstream/review-mark/self-test/self-test.yaml index 4af7563..750ba17 100644 --- a/docs/reqstream/review-mark/self-test/self-test.yaml +++ b/docs/reqstream/review-mark/self-test/self-test.yaml @@ -20,10 +20,6 @@ sections: summary, enabling quality assurance teams to obtain tool qualification evidence without requiring a separate test harness. tests: - - Validation_Run_NullContext_ThrowsArgumentNullException - - Validation_Run_WritesValidationHeader - - Validation_Run_WritesSummaryWithTotalTests - - Validation_Run_AllTestsPass_ExitCodeIsZero - SelfTest_Run_AllTestsPass_ExitCodeIsZero children: [ReviewMark-Validation-Run] @@ -36,7 +32,5 @@ sections: directly into pipeline tooling and traceability reports without additional conversion steps, satisfying audit trail requirements. tests: - - Validation_Run_WithTrxResultsFile_WritesFile - - Validation_Run_WithXmlResultsFile_WritesFile - SelfTest_Run_GeneratesResultsFile children: [ReviewMark-Validation-ResultsFile] diff --git a/test/DemaConsulting.ReviewMark.Tests/Cli/CliTests.cs b/test/DemaConsulting.ReviewMark.Tests/Cli/CliTests.cs index 6d1e07c..d274bc3 100644 --- a/test/DemaConsulting.ReviewMark.Tests/Cli/CliTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/Cli/CliTests.cs @@ -145,4 +145,711 @@ public void Cli_SilentFlag_SuppressesOutput() Console.SetError(originalError); } } + + /// + /// Test that --results flag generates a TRX file. + /// + [TestMethod] + public void Cli_ResultsFlag_GeneratesTrxFile() + { + // Arrange + var resultsFile = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid()}.trx"); + + try + { + using var context = Context.Create(["--validate", "--results", resultsFile]); + + // Act + Program.Run(context); + + // Assert — exit code is zero and results file contains TRX content + Assert.AreEqual(0, context.ExitCode); + Assert.IsTrue(File.Exists(resultsFile), "Results file was not created"); + var content = File.ReadAllText(resultsFile); + Assert.Contains(" + /// Test that --log flag writes output to a log file. + /// + [TestMethod] + public void Cli_LogFlag_WritesOutputToFile() + { + // Arrange + var logFile = Path.GetTempFileName(); + + try + { + int exitCode; + using (var context = Context.Create(["--log", logFile])) + { + // Act + Program.Run(context); + exitCode = context.ExitCode; + } + + // context is disposed here — log file is closed and safe to read + Assert.AreEqual(0, exitCode); + Assert.IsTrue(File.Exists(logFile), "Log file was not created"); + var logContent = File.ReadAllText(logFile); + Assert.Contains("ReviewMark version", logContent); + } + finally + { + if (File.Exists(logFile)) + { + File.Delete(logFile); + } + } + } + + /// + /// Test that unknown argument causes error output to stderr. + /// + [TestMethod] + public void Cli_ErrorOutput_WritesToStderr() + { + // Arrange + var originalError = Console.Error; + try + { + using var errWriter = new StringWriter(); + Console.SetError(errWriter); + + var mainMethod = typeof(Program).GetMethod( + "Main", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static); + + Assert.IsNotNull(mainMethod, "Could not find Program.Main(string[] args)."); + + // Act — invoke the real CLI entrypoint so invalid args are handled exactly + // as they are in production, including writing parse errors to stderr. + var result = mainMethod.Invoke(null, [new string[] { "--unknown-arg-xyz" }]); + var exitCode = result is int code ? code : 0; + + // Assert — invalid args should return a failure exit code and write an error to stderr + var stderr = errWriter.ToString(); + Assert.AreNotEqual(0, exitCode); + StringAssert.Contains(stderr, "Error:"); + StringAssert.Contains(stderr, "--unknown-arg-xyz"); + } + finally + { + Console.SetError(originalError); + } + } + + /// + /// Test that invalid arguments produce a non-zero exit code. + /// + [TestMethod] + public void Cli_InvalidArgs_ReturnsNonZeroExitCode() + { + // Arrange + Act — the full CLI (Context.Create in Main) catches ArgumentException and writes error + var originalOut = Console.Out; + var originalError = Console.Error; + try + { + using var outWriter = new StringWriter(); + using var errWriter = new StringWriter(); + Console.SetOut(outWriter); + Console.SetError(errWriter); + + // Simulate what Program.Main does: catch ArgumentException and use WriteError + int exitCode; + try + { + using var context = Context.Create(["--completely-invalid-arg"]); + Program.Run(context); + exitCode = context.ExitCode; + } + catch (ArgumentException ex) + { + // Program.Main writes this to a temporary context — simulate + using var errorContext = Context.Create([]); + errorContext.WriteError(ex.Message); + exitCode = errorContext.ExitCode; + } + + // Assert — non-zero exit code for invalid arguments + Assert.AreNotEqual(0, exitCode); + } + finally + { + Console.SetOut(originalOut); + Console.SetError(originalError); + } + } + + /// + /// Test that exit code is non-zero when an error occurs. + /// + [TestMethod] + public void Cli_ExitCode_ReturnsNonZeroOnError() + { + // Arrange + using var context = Context.Create([]); + + // Act — WriteError sets the exit code to 1 + context.WriteError("Simulated error for exit code test"); + + // Assert — exit code is non-zero + Assert.AreNotEqual(0, context.ExitCode); + } + + /// + /// Test that --definition flag loads the specified definition file. + /// + [TestMethod] + public void Cli_DefinitionFlag_LoadsSpecifiedFile() + { + // Arrange + var defFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".yaml")); + var planFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".md")); + + try + { + File.WriteAllText(defFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: none + reviews: + - id: Test-Review + title: Test review + paths: + - "src/**/*.cs" + """); + + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--definition", defFile, "--plan", planFile]); + + // Act + Program.Run(context); + + // Assert — exits with zero and plan file created from specified definition + Assert.AreEqual(0, context.ExitCode); + Assert.IsTrue(File.Exists(planFile), "Plan file was not created"); + } + finally + { + Console.SetOut(originalOut); + } + } + finally + { + if (File.Exists(defFile)) + { + File.Delete(defFile); + } + if (File.Exists(planFile)) + { + File.Delete(planFile); + } + } + } + + /// + /// Test that --plan flag generates a review plan file. + /// + [TestMethod] + public void Cli_PlanFlag_GeneratesReviewPlan() + { + // Arrange + var defFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".yaml")); + var planFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".md")); + + try + { + File.WriteAllText(defFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: none + reviews: + - id: Test-Review + title: Test review + paths: + - "src/**/*.cs" + """); + + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--definition", defFile, "--plan", planFile]); + + // Act + Program.Run(context); + + // Assert — plan file exists and contains review-set id + Assert.AreEqual(0, context.ExitCode); + Assert.IsTrue(File.Exists(planFile), "Plan file was not created"); + var planContent = File.ReadAllText(planFile); + Assert.Contains("Test-Review", planContent); + } + finally + { + Console.SetOut(originalOut); + } + } + finally + { + if (File.Exists(defFile)) + { + File.Delete(defFile); + } + if (File.Exists(planFile)) + { + File.Delete(planFile); + } + } + } + + /// + /// Test that --report flag generates a review report file. + /// + [TestMethod] + public void Cli_ReportFlag_GeneratesReviewReport() + { + // Arrange + var defFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".yaml")); + var reportFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".md")); + + try + { + File.WriteAllText(defFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: none + reviews: + - id: Test-Review + title: Test review + paths: + - "src/**/*.cs" + """); + + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--definition", defFile, "--report", reportFile]); + + // Act + Program.Run(context); + + // Assert — report file exists and contains review-set id + Assert.AreEqual(0, context.ExitCode); + Assert.IsTrue(File.Exists(reportFile), "Report file was not created"); + var reportContent = File.ReadAllText(reportFile); + Assert.Contains("Test-Review", reportContent); + } + finally + { + Console.SetOut(originalOut); + } + } + finally + { + if (File.Exists(defFile)) + { + File.Delete(defFile); + } + if (File.Exists(reportFile)) + { + File.Delete(reportFile); + } + } + } + + /// + /// Test that --enforce flag exits with non-zero when reviews are not current. + /// + [TestMethod] + public void Cli_EnforceFlag_ExitsNonZeroWhenNotCurrent() + { + // Arrange + var defFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".yaml")); + var reportFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".md")); + + try + { + File.WriteAllText(defFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: none + reviews: + - id: Test-Review + title: Test review + paths: + - "src/**/*.cs" + """); + + var originalOut = Console.Out; + var originalError = Console.Error; + try + { + using var outWriter = new StringWriter(); + using var errWriter = new StringWriter(); + Console.SetOut(outWriter); + Console.SetError(errWriter); + using var context = Context.Create(["--definition", defFile, "--report", reportFile, "--enforce"]); + + // Act + Program.Run(context); + + // Assert — non-zero exit code because evidence source is 'none' + Assert.AreNotEqual(0, context.ExitCode); + } + finally + { + Console.SetOut(originalOut); + Console.SetError(originalError); + } + } + finally + { + if (File.Exists(defFile)) + { + File.Delete(defFile); + } + if (File.Exists(reportFile)) + { + File.Delete(reportFile); + } + } + } + + /// + /// Test that --dir flag sets the working directory for file operations. + /// + [TestMethod] + public void Cli_DirFlag_SetsWorkingDirectory() + { + // Arrange — create a temp directory with a .reviewmark.yaml file + var tmpDir = Path.Combine(Path.GetTempPath(), $"reviewmark_cli_{Guid.NewGuid()}"); + Directory.CreateDirectory(tmpDir); + var defFile = Path.Combine(tmpDir, ".reviewmark.yaml"); + var planFile = Path.Combine(tmpDir, "plan.md"); + + try + { + File.WriteAllText(defFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: none + reviews: + - id: Test-Review + title: Test review + paths: + - "src/**/*.cs" + """); + + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--dir", tmpDir, "--plan", planFile]); + + // Act + Program.Run(context); + + // Assert — exits successfully using directory-relative definition file + Assert.AreEqual(0, context.ExitCode); + Assert.IsTrue(File.Exists(planFile), "Plan file was not created"); + } + finally + { + Console.SetOut(originalOut); + } + } + finally + { + if (Directory.Exists(tmpDir)) + { + Directory.Delete(tmpDir, recursive: true); + } + } + } + + /// + /// Test that --elaborate flag outputs elaboration for a valid review-set. + /// + [TestMethod] + public void Cli_ElaborateFlag_OutputsElaboration() + { + // Arrange + var defFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".yaml")); + + try + { + File.WriteAllText(defFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: none + reviews: + - id: Test-Review + title: Test review + paths: + - "src/**/*.cs" + """); + + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--definition", defFile, "--elaborate", "Test-Review"]); + + // Act + Program.Run(context); + + // Assert — exits successfully and output contains review-set id + Assert.AreEqual(0, context.ExitCode); + var output = outWriter.ToString(); + Assert.Contains("Test-Review", output); + } + finally + { + Console.SetOut(originalOut); + } + } + finally + { + if (File.Exists(defFile)) + { + File.Delete(defFile); + } + } + } + + /// + /// Test that --lint flag reports success for a valid config. + /// + [TestMethod] + public void Cli_LintFlag_ReportsSuccess() + { + // Arrange + var defFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".yaml")); + + try + { + File.WriteAllText(defFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: none + reviews: + - id: Test-Review + title: Test review + paths: + - "src/**/*.cs" + """); + + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--definition", defFile, "--lint"]); + + // Act + Program.Run(context); + + // Assert — exits successfully and reports no issues + Assert.AreEqual(0, context.ExitCode); + var output = outWriter.ToString(); + Assert.Contains("No issues found", output); + } + finally + { + Console.SetOut(originalOut); + } + } + finally + { + if (File.Exists(defFile)) + { + File.Delete(defFile); + } + } + } + + /// + /// Test that --index flag scans and creates index.json. + /// + [TestMethod] + public void Cli_IndexFlag_CreatesIndexJson() + { + // Arrange — create a temp directory to index + var tmpDir = Path.Combine(Path.GetTempPath(), $"reviewmark_index_{Guid.NewGuid()}"); + Directory.CreateDirectory(tmpDir); + var indexFile = Path.Combine(tmpDir, "index.json"); + + try + { + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create([ + "--dir", tmpDir, + "--index", Path.Combine(tmpDir, "**", "*.pdf")]); + + // Act + Program.Run(context); + + // Assert — exits successfully and index.json was created + Assert.AreEqual(0, context.ExitCode); + Assert.IsTrue(File.Exists(indexFile), "index.json was not created"); + } + finally + { + Console.SetOut(originalOut); + } + } + finally + { + if (Directory.Exists(tmpDir)) + { + Directory.Delete(tmpDir, recursive: true); + } + } + } + + /// + /// Test that --plan-depth flag sets the heading depth in the generated review plan. + /// + [TestMethod] + public void Cli_PlanDepthFlag_SetsHeadingDepth() + { + // Arrange + var defFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".yaml")); + var planFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".md")); + + try + { + File.WriteAllText(defFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: none + reviews: + - id: Test-Review + title: Test review + paths: + - "src/**/*.cs" + """); + + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--definition", defFile, "--plan", planFile, "--plan-depth", "2"]); + + // Act + Program.Run(context); + + // Assert — plan file uses ## (depth 2) headings + Assert.AreEqual(0, context.ExitCode); + Assert.IsTrue(File.Exists(planFile), "Plan file was not created"); + var planContent = File.ReadAllText(planFile); + StringAssert.Contains(planContent, "## Review Coverage"); + } + finally + { + Console.SetOut(originalOut); + } + } + finally + { + if (File.Exists(defFile)) + { + File.Delete(defFile); + } + if (File.Exists(planFile)) + { + File.Delete(planFile); + } + } + } + + /// + /// Test that --report-depth flag sets the heading depth in the generated review report. + /// + [TestMethod] + public void Cli_ReportDepthFlag_SetsHeadingDepth() + { + // Arrange + var defFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".yaml")); + var reportFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".md")); + + try + { + File.WriteAllText(defFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: none + reviews: + - id: Test-Review + title: Test review + paths: + - "src/**/*.cs" + """); + + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--definition", defFile, "--report", reportFile, "--report-depth", "2"]); + + // Act + Program.Run(context); + + // Assert — report file uses ## (depth 2) headings + Assert.AreEqual(0, context.ExitCode); + Assert.IsTrue(File.Exists(reportFile), "Report file was not created"); + var reportContent = File.ReadAllText(reportFile); + StringAssert.Contains(reportContent, "## Review Status"); + } + finally + { + Console.SetOut(originalOut); + } + } + finally + { + if (File.Exists(defFile)) + { + File.Delete(defFile); + } + if (File.Exists(reportFile)) + { + File.Delete(reportFile); + } + } + } } diff --git a/test/DemaConsulting.ReviewMark.Tests/Configuration/ConfigurationTests.cs b/test/DemaConsulting.ReviewMark.Tests/Configuration/ConfigurationTests.cs index 563117a..63cd75f 100644 --- a/test/DemaConsulting.ReviewMark.Tests/Configuration/ConfigurationTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/Configuration/ConfigurationTests.cs @@ -177,4 +177,79 @@ public void Configuration_LoadConfig_PlanGenerationSucceeds() // Assert Assert.Contains("Core-Logic", planResult.Markdown); } + + /// + /// Test that generating a review report succeeds and includes the review set ID. + /// + [TestMethod] + public void Configuration_LoadConfig_ReportGenerationSucceeds() + { + // Arrange + var srcDir = PathHelpers.SafePathCombine(_testDirectory, "src"); + Directory.CreateDirectory(srcDir); + File.WriteAllText(PathHelpers.SafePathCombine(srcDir, "Main.cs"), "class Main {}"); + + var indexFile = PathHelpers.SafePathCombine(_testDirectory, "index.json"); + File.WriteAllText(indexFile, """{"reviews":[]}"""); + + var definitionFile = PathHelpers.SafePathCombine(_testDirectory, ".reviewmark.yaml"); + File.WriteAllText(definitionFile, $""" + needs-review: + - "src/**/*.cs" + evidence-source: + type: fileshare + location: {indexFile} + reviews: + - id: Core-Logic + title: Core logic review + paths: + - "src/**/*.cs" + """); + + // Act + var result = ReviewMarkConfiguration.Load(definitionFile); + Assert.IsNotNull(result.Configuration); + var index = ReviewIndex.Load(result.Configuration.EvidenceSource); + var reportResult = result.Configuration.PublishReviewReport(index, _testDirectory); + + // Assert + Assert.Contains("Core-Logic", reportResult.Markdown); + } + + /// + /// Test that elaborating a review-set succeeds and includes the review set ID and fingerprint. + /// + [TestMethod] + public void Configuration_LoadConfig_ElaborationSucceeds() + { + // Arrange + var srcDir = PathHelpers.SafePathCombine(_testDirectory, "src"); + Directory.CreateDirectory(srcDir); + File.WriteAllText(PathHelpers.SafePathCombine(srcDir, "Main.cs"), "class Main {}"); + + var indexFile = PathHelpers.SafePathCombine(_testDirectory, "index.json"); + File.WriteAllText(indexFile, """{"reviews":[]}"""); + + var definitionFile = PathHelpers.SafePathCombine(_testDirectory, ".reviewmark.yaml"); + File.WriteAllText(definitionFile, $""" + needs-review: + - "src/**/*.cs" + evidence-source: + type: fileshare + location: {indexFile} + reviews: + - id: Core-Logic + title: Core logic review + paths: + - "src/**/*.cs" + """); + + // Act + var result = ReviewMarkConfiguration.Load(definitionFile); + Assert.IsNotNull(result.Configuration); + var elaborateResult = result.Configuration.ElaborateReviewSet("Core-Logic", _testDirectory); + + // Assert + Assert.Contains("Core-Logic", elaborateResult.Markdown); + } } diff --git a/test/DemaConsulting.ReviewMark.Tests/Indexing/IndexingTests.cs b/test/DemaConsulting.ReviewMark.Tests/Indexing/IndexingTests.cs index 466005b..ef881b9 100644 --- a/test/DemaConsulting.ReviewMark.Tests/Indexing/IndexingTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/Indexing/IndexingTests.cs @@ -142,4 +142,23 @@ public void Indexing_ReviewIndex_SaveAndLoad_RoundTrip() Assert.IsNotNull(index2.GetEvidence("Review-Alpha", "fp001")); Assert.IsNotNull(index2.GetEvidence("Review-Beta", "fp002")); } + + /// + /// Test that SafePathCombine throws for path traversal inputs, preventing directory escapes. + /// + [TestMethod] + public void Indexing_SafePathCombine_WithTraversalInputs_Throws() + { + // Arrange + var evidenceDir = PathHelpers.SafePathCombine(_testDirectory, "evidence"); + Directory.CreateDirectory(evidenceDir); + + // Act & Assert — double-dot traversal must be rejected + Assert.Throws(() => + PathHelpers.SafePathCombine(evidenceDir, "../../../etc/sensitive")); + + // Act & Assert — absolute path must be rejected + Assert.Throws(() => + PathHelpers.SafePathCombine(evidenceDir, Path.GetTempPath())); + } } diff --git a/test/DemaConsulting.ReviewMark.Tests/IntegrationTests.cs b/test/DemaConsulting.ReviewMark.Tests/IntegrationTests.cs index 0863757..dc99821 100644 --- a/test/DemaConsulting.ReviewMark.Tests/IntegrationTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/IntegrationTests.cs @@ -249,4 +249,337 @@ public void IntegrationTest_UnknownArgument_ReturnsError() Assert.AreNotEqual(0, exitCode); Assert.Contains("Error", output); } + + /// + /// Test that review plan generation writes a Markdown plan file. + /// + [TestMethod] + public void IntegrationTest_ReviewPlanGeneration() + { + // Arrange + var defFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".yaml")); + var planFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".md")); + + try + { + File.WriteAllText(defFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: none + reviews: + - id: Test-Review + title: Test review + paths: + - "src/**/*.cs" + """); + + // Act + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--definition", + defFile, + "--plan", + planFile); + + // Assert — exit succeeds and plan file contains review-set id + Assert.AreEqual(0, exitCode, $"Output: {output}"); + Assert.IsTrue(File.Exists(planFile), "Plan file was not created"); + var planContent = File.ReadAllText(planFile); + Assert.Contains("Test-Review", planContent); + } + finally + { + if (File.Exists(defFile)) + { + File.Delete(defFile); + } + if (File.Exists(planFile)) + { + File.Delete(planFile); + } + } + } + + /// + /// Test that review report generation writes a Markdown report file. + /// + [TestMethod] + public void IntegrationTest_ReviewReportGeneration() + { + // Arrange + var defFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".yaml")); + var reportFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".md")); + + try + { + File.WriteAllText(defFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: none + reviews: + - id: Test-Review + title: Test review + paths: + - "src/**/*.cs" + """); + + // Act + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--definition", + defFile, + "--report", + reportFile); + + // Assert — exit succeeds and report file contains review-set id + Assert.AreEqual(0, exitCode, $"Output: {output}"); + Assert.IsTrue(File.Exists(reportFile), "Report file was not created"); + var reportContent = File.ReadAllText(reportFile); + Assert.Contains("Test-Review", reportContent); + } + finally + { + if (File.Exists(defFile)) + { + File.Delete(defFile); + } + if (File.Exists(reportFile)) + { + File.Delete(reportFile); + } + } + } + + /// + /// Test that --enforce returns non-zero when reviews are not current. + /// + [TestMethod] + public void IntegrationTest_Enforce() + { + // Arrange + var defFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".yaml")); + var reportFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".md")); + + try + { + File.WriteAllText(defFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: none + reviews: + - id: Test-Review + title: Test review + paths: + - "src/**/*.cs" + """); + + // Act — enforce with no evidence returns non-zero exit code + var exitCode = Runner.Run( + out var _, + "dotnet", + _dllPath, + "--definition", + defFile, + "--report", + reportFile, + "--enforce"); + + // Assert — non-zero because evidence source is 'none' so no reviews are current + Assert.AreNotEqual(0, exitCode); + } + finally + { + if (File.Exists(defFile)) + { + File.Delete(defFile); + } + if (File.Exists(reportFile)) + { + File.Delete(reportFile); + } + } + } + + /// + /// Test that --index scans a directory and creates an index.json. + /// + [TestMethod] + public void IntegrationTest_IndexScan() + { + // Arrange — create a temp directory to index (with no PDF files) + var tmpDir = Path.Combine(Path.GetTempPath(), $"reviewmark_idx_{Guid.NewGuid()}"); + Directory.CreateDirectory(tmpDir); + var indexFile = Path.Combine(tmpDir, "index.json"); + + try + { + // Act — index the empty directory + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--dir", + tmpDir, + "--index", + Path.Combine(tmpDir, "**", "*.pdf")); + + // Assert — exits successfully and produces index.json + Assert.AreEqual(0, exitCode, $"Output: {output}"); + Assert.IsTrue(File.Exists(indexFile), "index.json was not created"); + } + finally + { + if (Directory.Exists(tmpDir)) + { + Directory.Delete(tmpDir, recursive: true); + } + } + } + + /// + /// Test that --dir sets the working directory for file operations. + /// + [TestMethod] + public void IntegrationTest_WorkingDirectoryOverride() + { + // Arrange — create a temp directory with a definition file + var tmpDir = Path.Combine(Path.GetTempPath(), $"reviewmark_work_{Guid.NewGuid()}"); + Directory.CreateDirectory(tmpDir); + var defFile = Path.Combine(tmpDir, ".reviewmark.yaml"); + var planFile = Path.Combine(tmpDir, "plan.md"); + + try + { + File.WriteAllText(defFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: none + reviews: + - id: Test-Review + title: Test review + paths: + - "src/**/*.cs" + """); + + // Act — use --dir to point to temp directory containing the definition file + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--dir", + tmpDir, + "--plan", + planFile); + + // Assert — exits successfully using the directory-relative definition file + Assert.AreEqual(0, exitCode, $"Output: {output}"); + Assert.IsTrue(File.Exists(planFile), "Plan file was not created"); + } + finally + { + if (Directory.Exists(tmpDir)) + { + Directory.Delete(tmpDir, recursive: true); + } + } + } + + /// + /// Test that --elaborate outputs elaboration for a valid review-set ID. + /// + [TestMethod] + public void IntegrationTest_Elaborate() + { + // Arrange + var defFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".yaml")); + + try + { + File.WriteAllText(defFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: none + reviews: + - id: Test-Review + title: Test review + paths: + - "src/**/*.cs" + """); + + // Act + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--definition", + defFile, + "--elaborate", + "Test-Review"); + + // Assert — exits successfully and output contains the review-set id + Assert.AreEqual(0, exitCode, $"Output: {output}"); + Assert.Contains("Test-Review", output); + } + finally + { + if (File.Exists(defFile)) + { + File.Delete(defFile); + } + } + } + + /// + /// Test that --lint with a valid config reports success. + /// + [TestMethod] + public void IntegrationTest_Lint() + { + // Arrange + var defFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".yaml")); + + try + { + File.WriteAllText(defFile, """ + needs-review: + - "src/**/*.cs" + evidence-source: + type: none + reviews: + - id: Test-Review + title: Test review + paths: + - "src/**/*.cs" + """); + + // Act + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--definition", + defFile, + "--lint"); + + // Assert — exits successfully and output reports no issues + Assert.AreEqual(0, exitCode, $"Output: {output}"); + Assert.Contains("No issues found", output); + } + finally + { + if (File.Exists(defFile)) + { + File.Delete(defFile); + } + } + } } From 4bfdc01eb517ba9cde376289fc961a75a0336eb1 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sun, 5 Apr 2026 10:11:01 -0400 Subject: [PATCH 19/19] =?UTF-8?q?fix:=20formal=20review=20fixes=20?= =?UTF-8?q?=E2=80=94=20results=20file=20directory=20creation=20and=20requi?= =?UTF-8?q?rements=20scope=20cleanup=20(#47)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: remove self-validation test refs from CLI/Configuration requirements; fix results file directory creation Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/0c41f310-04ab-4dc1-8f1e-c297f19312b5 Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> * refactor: use TestDirectory helper in Validation_Run_WithResultsFileInNewDirectory_CreatesDirectory test Agent-Logs-Url: https://github.com/demaconsulting/ReviewMark/sessions/4b5d1e5d-5412-46c0-a53c-0dc783a2cf7f Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Malcolmnixon <1863707+Malcolmnixon@users.noreply.github.com> Co-authored-by: Malcolm Nixon --- .../review-mark/self-test/validation.yaml | 1 + .../SelfTest/Validation.cs | 6 ++++ .../SelfTest/ValidationTests.cs | 31 +++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/docs/reqstream/review-mark/self-test/validation.yaml b/docs/reqstream/review-mark/self-test/validation.yaml index 622ccd1..2c6641d 100644 --- a/docs/reqstream/review-mark/self-test/validation.yaml +++ b/docs/reqstream/review-mark/self-test/validation.yaml @@ -32,3 +32,4 @@ sections: tests: - Validation_Run_WithTrxResultsFile_WritesFile - Validation_Run_WithXmlResultsFile_WritesFile + - Validation_Run_WithResultsFileInNewDirectory_CreatesDirectory diff --git a/src/DemaConsulting.ReviewMark/SelfTest/Validation.cs b/src/DemaConsulting.ReviewMark/SelfTest/Validation.cs index 6fd8197..60f66ef 100644 --- a/src/DemaConsulting.ReviewMark/SelfTest/Validation.cs +++ b/src/DemaConsulting.ReviewMark/SelfTest/Validation.cs @@ -531,6 +531,12 @@ private static void WriteResultsFile(Context context, DemaConsulting.TestResults return; } + var directory = Path.GetDirectoryName(context.ResultsFile); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + File.WriteAllText(context.ResultsFile, content); context.WriteLine($"Results written to {context.ResultsFile}"); } diff --git a/test/DemaConsulting.ReviewMark.Tests/SelfTest/ValidationTests.cs b/test/DemaConsulting.ReviewMark.Tests/SelfTest/ValidationTests.cs index 1949dd6..3576944 100644 --- a/test/DemaConsulting.ReviewMark.Tests/SelfTest/ValidationTests.cs +++ b/test/DemaConsulting.ReviewMark.Tests/SelfTest/ValidationTests.cs @@ -202,4 +202,35 @@ public void Validation_Run_WithXmlResultsFile_WritesFile() } } } + + /// + /// Test that Run creates the parent directory when --results specifies a path with a non-existent parent. + /// + [TestMethod] + public void Validation_Run_WithResultsFileInNewDirectory_CreatesDirectory() + { + // Arrange — use TestDirectory as the root; the 'output' subdirectory does not exist yet + using var tempDir = new TestDirectory(); + var subDir = Path.Combine(tempDir.DirectoryPath, "output"); + var resultsFile = Path.Combine(subDir, "results.trx"); + + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--validate", "--results", resultsFile]); + + // Act + Validation.Run(context); + + // Assert — directory and results file were created + Assert.IsTrue(Directory.Exists(subDir), "Parent directory was not created"); + Assert.IsTrue(File.Exists(resultsFile), "TRX results file was not created in new directory"); + } + finally + { + Console.SetOut(originalOut); + } + } }