diff --git a/.github/workflows/pr-image-tests.yaml b/.github/workflows/pr-image-tests.yaml
index cb85748c0f8..b15f8e02cea 100644
--- a/.github/workflows/pr-image-tests.yaml
+++ b/.github/workflows/pr-image-tests.yaml
@@ -7,6 +7,7 @@ on:
       - synchronize
     paths:
       - 'dockerfiles/Dockerfile'
+      - 'dockerfiles/Dockerfile.windows'
       - 'conf/**'
 
   workflow_dispatch:
@@ -79,3 +80,40 @@ jobs:
           # Ensure we disable buildkit
           DOCKER_BUILDKIT: 0
         shell: bash
+    pr-image-tests-build-windows-images:
+      name: PR - Docker windows build test, windows 2019 and 2022
+      runs-on: windows-${{ matrix.windows-base-version }}
+      strategy:
+        fail-fast: true
+        matrix:
+          windows-base-version: 
+            # https://github.com/fluent/fluent-bit/blob/1d366594a889624ec3003819fe18588aac3f17cd/dockerfiles/Dockerfile.windows#L3
+            - '2019'
+            - '2022'
+      permissions:
+        contents: read
+      steps:
+        - name: Checkout repository
+          uses: actions/checkout@v3
+
+        - name: Extract metadata from Github
+          id: meta
+          uses: docker/metadata-action@v4
+          with:
+            images: ${{ github.repository }}/pr-${{ github.event.pull_request.number }}
+            tags: |
+              type=sha
+            flavor: |
+              suffix=-windows-${{ matrix.windows-base-version }}
+    
+        - name: Build the windows images
+          id: build
+          run: |
+            docker build -t ${{ steps.meta.outputs.tags }} --build-arg WINDOWS_VERSION=ltsc${{ matrix.windows-base-version }} -f ./dockerfiles/Dockerfile.windows .
+
+        - name: Sanity check it runs
+          # We do this for a simple check of dependencies
+          run: |
+            docker run --rm -t ${{ steps.meta.outputs.tags }} --help
+          shell: bash
+
diff --git a/dockerfiles/Dockerfile.windows b/dockerfiles/Dockerfile.windows
index b3235232fc4..eae4bda3075 100644
--- a/dockerfiles/Dockerfile.windows
+++ b/dockerfiles/Dockerfile.windows
@@ -40,6 +40,7 @@ RUN Start-Process /local/vc_redist.x64.exe -ArgumentList '/install', '/quiet', '
 
 # Install Chocolatey and OpenSSL: https://github.com/StefanScherer/dockerfiles-windows/blob/main/openssl/Dockerfile
 ENV chocolateyUseWindowsCompression false
+ENV chocolateyVersion '1.4.0'
 RUN iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); `
     choco feature disable --name showDownloadProgress ; `
     choco install -y openssl;