diff --git a/.gitea/workflows/managed-validation.yml b/.gitea/workflows/managed-validation.yml index abe430e4..dcb89481 100644 --- a/.gitea/workflows/managed-validation.yml +++ b/.gitea/workflows/managed-validation.yml @@ -20,7 +20,9 @@ concurrency: jobs: full: - name: ${{ inputs.profile || 'full' }} + # Gitea Actions does not consistently evaluate the GitHub-style `||` + # expression for pull-request runs without workflow inputs. + name: Managed repository validation runs-on: ubuntu-latest timeout-minutes: 60 steps: diff --git a/.gitea/workflows/release-gates.yml b/.gitea/workflows/release-gates.yml index c174e473..16f5dd8a 100644 --- a/.gitea/workflows/release-gates.yml +++ b/.gitea/workflows/release-gates.yml @@ -61,7 +61,9 @@ jobs: docker compose config > artifacts/docker-compose.resolved.yml - name: Publish quality evidence if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + # Gitea Actions currently exposes the GHES-compatible artifact API; + # upload-artifact v4 deliberately refuses that API. + uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20 with: name: quality-evidence path: | @@ -98,7 +100,7 @@ jobs: npm audit --audit-level=high --json > ../artifacts/npm-audit.json - name: Publish dependency evidence if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20 with: name: dependency-audits path: | @@ -135,6 +137,9 @@ jobs: - name: Generate SPDX SBOM env: RELEASE_SHA: ${{ gitea.sha }} + GEOINTEL_IMAGE_ARCHIVE: artifacts/geointel-image.tar + GEOINTEL_KEEP_IMAGE_ARCHIVE: "true" + SYFT_PARALLELISM: "1" run: | IMAGE_ID="$(cat artifacts/image-id.txt)" test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID" @@ -142,14 +147,22 @@ jobs: - name: Enforce container vulnerability policy env: RELEASE_SHA: ${{ gitea.sha }} + GEOINTEL_IMAGE_ARCHIVE: artifacts/geointel-image.tar + GEOINTEL_KEEP_IMAGE_ARCHIVE: "true" run: | IMAGE_ID="$(cat artifacts/image-id.txt)" test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID" bash scripts/scan_container_image.sh "$IMAGE_ID" test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID" + - name: Remove temporary image archive + if: always() + run: >- + rm -f -- artifacts/geointel-image.tar + artifacts/geointel-image.tar.image-id + artifacts/geointel-image.tar.partial.* - name: Publish container evidence if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20 with: name: container-evidence path: | diff --git a/.github/workflows/release-gates.yml b/.github/workflows/release-gates.yml index ced85ca3..9debbe8b 100644 --- a/.github/workflows/release-gates.yml +++ b/.github/workflows/release-gates.yml @@ -133,6 +133,9 @@ jobs: - name: Generate SPDX SBOM env: RELEASE_SHA: ${{ github.sha }} + GEOINTEL_IMAGE_ARCHIVE: artifacts/geointel-image.tar + GEOINTEL_KEEP_IMAGE_ARCHIVE: "true" + SYFT_PARALLELISM: "1" run: | IMAGE_ID="$(cat artifacts/image-id.txt)" test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID" @@ -140,11 +143,19 @@ jobs: - name: Enforce container vulnerability policy env: RELEASE_SHA: ${{ github.sha }} + GEOINTEL_IMAGE_ARCHIVE: artifacts/geointel-image.tar + GEOINTEL_KEEP_IMAGE_ARCHIVE: "true" run: | IMAGE_ID="$(cat artifacts/image-id.txt)" test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID" bash scripts/scan_container_image.sh "$IMAGE_ID" test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID" + - name: Remove temporary image archive + if: always() + run: >- + rm -f -- artifacts/geointel-image.tar + artifacts/geointel-image.tar.image-id + artifacts/geointel-image.tar.partial.* - name: Publish container evidence if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 diff --git a/backend/tests/test_rc6_supply_chain.py b/backend/tests/test_rc6_supply_chain.py index 0f8684dc..b6ebf7b0 100644 --- a/backend/tests/test_rc6_supply_chain.py +++ b/backend/tests/test_rc6_supply_chain.py @@ -54,9 +54,19 @@ def test_ci_runs_complete_release_and_supply_chain_gates() -> None: assert 'scan_container_image.sh "$IMAGE_ID"' in workflow assert "generate_container_sbom.sh" in workflow assert "scan_container_image.sh" in workflow - assert "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" in workflow + assert "GEOINTEL_IMAGE_ARCHIVE: artifacts/geointel-image.tar" in workflow + assert "Remove temporary image archive" in workflow assert context in workflow + assert ( + "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + in read(".github/workflows/release-gates.yml") + ) + assert ( + "actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de" + in read(".gitea/workflows/release-gates.yml") + ) + def test_gitea_production_deploy_depends_on_every_release_gate() -> None: release = read(".gitea/workflows/release-gates.yml") @@ -72,17 +82,25 @@ def test_gitea_production_deploy_depends_on_every_release_gate() -> None: def test_release_workflows_pin_third_party_actions_to_reviewed_commits() -> None: - expected = ( + shared = ( "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683", "actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065", "actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020", - "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02", ) for path in (".gitea/workflows/release-gates.yml", ".github/workflows/release-gates.yml"): workflow = read(path) - for action in expected: + for action in shared: assert action in workflow + assert ( + "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + in read(".github/workflows/release-gates.yml") + ) + assert ( + "actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de" + in read(".gitea/workflows/release-gates.yml") + ) + def test_managed_validation_targets_the_actual_backend_and_frontend_projects() -> None: workflow = read(".gitea/workflows/managed-validation.yml") @@ -101,14 +119,23 @@ def test_scanner_images_are_versioned_and_digest_pinned() -> None: scan = read("scripts/scan_container_image.sh") assert "anchore/syft:v1.44.0@sha256:" in sbom + assert "--user 0:0" in sbom + assert 'docker save "$IMAGE_ID"' in sbom + assert '"docker-archive:$WORKDIR/$IMAGE_ARCHIVE"' in sbom + assert 'SYFT_PARALLELISM=${SYFT_PARALLELISM:-1}' in sbom + assert '--volumes-from "$HOSTNAME"' in sbom + assert 'ARCHIVE_ID_FILE="${IMAGE_ARCHIVE}.image-id"' in sbom + assert "/var/run/docker.sock" not in sbom assert "aquasec/trivy:0.70.0@sha256:" in scan assert "--severity HIGH,CRITICAL" in scan assert "--ignore-unfixed" in scan assert "--timeout 20m" in scan assert "--scanners vuln" in scan - assert 'ignored_container_ids' in scan - assert 'ignore_args=(-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro")' in scan - assert 'trivy_ignore_args=(--ignorefile "$CONTAINER_IGNORE_FILE")' in scan + assert 'docker save "$IMAGE_ID"' in scan + assert '--input "$WORKDIR/$IMAGE_ARCHIVE"' in scan + assert '--volumes-from "$HOSTNAME"' in scan + assert '--ignorefile "$CONTAINER_IGNORE_FILE"' in scan + assert "/var/run/docker.sock" not in scan assert "--skip-files /usr/local/bin/gosu" in scan assert "final filesystem replaces it with the audited setpriv shell wrapper" in scan assert "geointel-container-vulnerabilities.json" in scan diff --git a/backend/tests/test_sprint233_operational_completion.py b/backend/tests/test_sprint233_operational_completion.py index 0c0a4121..7f08b1b2 100644 --- a/backend/tests/test_sprint233_operational_completion.py +++ b/backend/tests/test_sprint233_operational_completion.py @@ -468,7 +468,9 @@ def test_detection_lab_only_receives_operational_imagery_rasters() -> None: assert "department_omgeving_thematic_raster" in capability_source assert "digitaal_vlaanderen_dhmv" in capability_source assert "vmm_flood_hazard" in capability_source - assert "dataset.dataset_type !== 'raster' || dataset.status !== 'ready'" in capability_source + assert "dataset.dataset_type !== 'raster'" in capability_source + assert "dataset.status !== 'ready'" in capability_source + assert "return datasetInferenceBlockReason(dataset) === null" in capability_source assert "const detectionRasterDatasets = useMemo(" in app_source assert "rasterDatasets: detectionRasterDatasets" in app_source assert "rasterDatasets={detectionRasterDatasets}" in app_source diff --git a/scripts/generate_container_sbom.sh b/scripts/generate_container_sbom.sh index 29708ba5..50a42abb 100644 --- a/scripts/generate_container_sbom.sh +++ b/scripts/generate_container_sbom.sh @@ -4,24 +4,67 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" TARGET_IMAGE="${1:-geointel-ci:local}" OUTPUT="${2:-artifacts/geointel-sbom.spdx.json}" +IMAGE_ARCHIVE="${GEOINTEL_IMAGE_ARCHIVE:-artifacts/geointel-image.tar}" +ARCHIVE_ID_FILE="${IMAGE_ARCHIVE}.image-id" +KEEP_ARCHIVE="${GEOINTEL_KEEP_IMAGE_ARCHIVE:-false}" SYFT_IMAGE="anchore/syft:v1.44.0@sha256:86fde6445b483d902fe011dd9f68c4987dd94e07da1e9edc004e3c2422650de6" +ARCHIVE_CREATED=false +PARTIAL_ARCHIVE="" -case "$OUTPUT" in - /*|*..*) - echo "SBOM output must be a repository-relative path without '..'." >&2 - exit 1 - ;; -esac +cleanup() { + if [ -n "$PARTIAL_ARCHIVE" ]; then + rm -f -- "$PARTIAL_ARCHIVE" + fi + if [ "$ARCHIVE_CREATED" = true ] && [ "$KEEP_ARCHIVE" != true ]; then + rm -f -- "$ROOT/$IMAGE_ARCHIVE" "$ROOT/$ARCHIVE_ID_FILE" + fi +} +trap cleanup EXIT + +for candidate in "$OUTPUT" "$IMAGE_ARCHIVE"; do + case "$candidate" in + /*|*..*) + echo "SBOM paths must be repository-relative and may not contain '..'." >&2 + exit 1 + ;; + esac +done + +IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$TARGET_IMAGE")" +test -n "$IMAGE_ID" +mkdir -p "$ROOT/$(dirname "$OUTPUT")" "$ROOT/$(dirname "$IMAGE_ARCHIVE")" + +ARCHIVE_ID="" +if [ -s "$ROOT/$ARCHIVE_ID_FILE" ]; then + ARCHIVE_ID="$(tr -d '[:space:]' < "$ROOT/$ARCHIVE_ID_FILE")" +fi +if [ ! -s "$ROOT/$IMAGE_ARCHIVE" ] || [ "$ARCHIVE_ID" != "$IMAGE_ID" ]; then + rm -f -- "$ROOT/$IMAGE_ARCHIVE" "$ROOT/$ARCHIVE_ID_FILE" + PARTIAL_ARCHIVE="$ROOT/${IMAGE_ARCHIVE}.partial.$$" + docker save "$IMAGE_ID" --output "$PARTIAL_ARCHIVE" + mv "$PARTIAL_ARCHIVE" "$ROOT/$IMAGE_ARCHIVE" + PARTIAL_ARCHIVE="" + printf '%s\n' "$IMAGE_ID" > "$ROOT/$ARCHIVE_ID_FILE" + ARCHIVE_CREATED=true +fi + +WORKDIR="/workspace" +WORKSPACE_ARGS=(-v "$ROOT:/workspace") +if [[ -n "${HOSTNAME:-}" ]] && docker inspect --type container "$HOSTNAME" >/dev/null 2>&1; then + # The Gitea job runs against TCP DinD. Reusing its workspace mounts keeps + # the archive inside the job boundary and avoids a privileged Docker API hop. + WORKDIR="$ROOT" + WORKSPACE_ARGS=(--volumes-from "$HOSTNAME") +fi -docker image inspect "$TARGET_IMAGE" >/dev/null -mkdir -p "$ROOT/$(dirname "$OUTPUT")" docker run --rm \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v "$ROOT:/workspace" \ - -w /workspace \ + --user 0:0 \ + -e "SYFT_PARALLELISM=${SYFT_PARALLELISM:-1}" \ + "${WORKSPACE_ARGS[@]}" \ + -w "$WORKDIR" \ "$SYFT_IMAGE" \ - "docker:$TARGET_IMAGE" \ + "docker-archive:$WORKDIR/$IMAGE_ARCHIVE" \ -o "spdx-json=$OUTPUT" test -s "$ROOT/$OUTPUT" -echo "SBOM written to $OUTPUT" +echo "SBOM written to $OUTPUT for $IMAGE_ID" diff --git a/scripts/scan_container_image.sh b/scripts/scan_container_image.sh index fdd443d3..95d46382 100644 --- a/scripts/scan_container_image.sh +++ b/scripts/scan_container_image.sh @@ -4,68 +4,113 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" TARGET_IMAGE="${1:-geointel-ci:local}" OUTPUT="${2:-artifacts/geointel-container-vulnerabilities.json}" +IMAGE_ARCHIVE="${GEOINTEL_IMAGE_ARCHIVE:-artifacts/geointel-image.tar}" +ARCHIVE_ID_FILE="${IMAGE_ARCHIVE}.image-id" +KEEP_ARCHIVE="${GEOINTEL_KEEP_IMAGE_ARCHIVE:-false}" TRIVY_IMAGE="aquasec/trivy:0.70.0@sha256:be1190afcb28352bfddc4ddeb71470835d16462af68d310f9f4bca710961a41e" CACHE_DIR="${GEOINTEL_TRIVY_CACHE:-$ROOT/.cache/trivy}" PYTHON_CMD="${PYTHON_BIN:-python3}" -IGNORE_FILE="$(mktemp)" -CONTAINER_IGNORE_FILE="/tmp/geointel-trivy-ignore.txt" -trap 'rm -f "$IGNORE_FILE"' EXIT +IGNORE_FILE="$CACHE_DIR/geointel-trivy-ignore.txt" +ARCHIVE_CREATED=false +PARTIAL_ARCHIVE="" -case "$OUTPUT" in - /*|*..*) - echo "Scan output must be a repository-relative path without '..'." >&2 +cleanup() { + rm -f -- "$IGNORE_FILE" + if [ -n "$PARTIAL_ARCHIVE" ]; then + rm -f -- "$PARTIAL_ARCHIVE" + fi + if [ "$ARCHIVE_CREATED" = true ] && [ "$KEEP_ARCHIVE" != true ]; then + rm -f -- "$ROOT/$IMAGE_ARCHIVE" "$ROOT/$ARCHIVE_ID_FILE" + fi +} +trap cleanup EXIT + +for candidate in "$OUTPUT" "$IMAGE_ARCHIVE"; do + case "$candidate" in + /*|*..*) + echo "Scan paths must be repository-relative and may not contain '..'." >&2 + exit 1 + ;; + esac +done + +IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$TARGET_IMAGE")" +test -n "$IMAGE_ID" +case "$CACHE_DIR" in + "$ROOT"/*) CACHE_RELATIVE="${CACHE_DIR#"$ROOT/"}" ;; + *) + echo "Trivy cache must remain inside the repository workspace." >&2 exit 1 ;; esac +case "$CACHE_RELATIVE" in + *..*) + echo "Trivy cache may not contain '..'." >&2 + exit 1 + ;; +esac +mkdir -p "$ROOT/$(dirname "$OUTPUT")" "$ROOT/$(dirname "$IMAGE_ARCHIVE")" "$CACHE_DIR" -docker image inspect "$TARGET_IMAGE" >/dev/null -mkdir -p "$ROOT/$(dirname "$OUTPUT")" "$CACHE_DIR" -"$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py" -mapfile -t ignored_container_ids < <( - "$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py" \ - --print-container-ids | tr -d '\r' -) -ignore_args=() -trivy_ignore_args=() -if [ "${#ignored_container_ids[@]}" -gt 0 ]; then - printf '%s\n' "${ignored_container_ids[@]}" > "$IGNORE_FILE" - ignore_args=(-v "$IGNORE_FILE:$CONTAINER_IGNORE_FILE:ro") - trivy_ignore_args=(--ignorefile "$CONTAINER_IGNORE_FILE") +ARCHIVE_ID="" +if [ -s "$ROOT/$ARCHIVE_ID_FILE" ]; then + ARCHIVE_ID="$(tr -d '[:space:]' < "$ROOT/$ARCHIVE_ID_FILE")" fi +if [ ! -s "$ROOT/$IMAGE_ARCHIVE" ] || [ "$ARCHIVE_ID" != "$IMAGE_ID" ]; then + rm -f -- "$ROOT/$IMAGE_ARCHIVE" "$ROOT/$ARCHIVE_ID_FILE" + PARTIAL_ARCHIVE="$ROOT/${IMAGE_ARCHIVE}.partial.$$" + docker save "$IMAGE_ID" --output "$PARTIAL_ARCHIVE" + mv "$PARTIAL_ARCHIVE" "$ROOT/$IMAGE_ARCHIVE" + PARTIAL_ARCHIVE="" + printf '%s\n' "$IMAGE_ID" > "$ROOT/$ARCHIVE_ID_FILE" + ARCHIVE_CREATED=true +fi + +WORKDIR="/workspace" +WORKSPACE_ARGS=(-v "$ROOT:/workspace") +if [[ -n "${HOSTNAME:-}" ]] && docker inspect --type container "$HOSTNAME" >/dev/null 2>&1; then + WORKDIR="$ROOT" + WORKSPACE_ARGS=(--volumes-from "$HOSTNAME") +fi +CONTAINER_CACHE_DIR="$WORKDIR/$CACHE_RELATIVE" +CONTAINER_IGNORE_FILE="$CONTAINER_CACHE_DIR/geointel-trivy-ignore.txt" + +"$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py" +"$PYTHON_CMD" "$ROOT/scripts/verify_security_exceptions.py" \ + --print-container-ids | tr -d '\r' > "$IGNORE_FILE" # Keep the complete report, including vulnerabilities without an available fix. docker run --rm \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v "$ROOT:/workspace" \ - -v "$CACHE_DIR:/root/.cache/trivy" \ + --user 0:0 \ + "${WORKSPACE_ARGS[@]}" \ "$TRIVY_IMAGE" \ image \ + --input "$WORKDIR/$IMAGE_ARCHIVE" \ --scanners vuln \ --timeout 20m \ --skip-version-check \ --format json \ - --output "/workspace/$OUTPUT" \ - "$TARGET_IMAGE" + --cache-dir "$CONTAINER_CACHE_DIR" \ + --output "$WORKDIR/$OUTPUT" # Release policy: fixed HIGH/CRITICAL findings block the build. Unfixed findings # remain visible in the full report and must be reviewed before release. The # postgres base image's Go gosu binary remains discoverable in a lower layer, # but the final filesystem replaces it with the audited setpriv shell wrapper. docker run --rm \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v "$CACHE_DIR:/root/.cache/trivy" \ - "${ignore_args[@]}" \ + --user 0:0 \ + "${WORKSPACE_ARGS[@]}" \ "$TRIVY_IMAGE" \ image \ + --input "$WORKDIR/$IMAGE_ARCHIVE" \ --scanners vuln \ --timeout 20m \ --skip-version-check \ --ignore-unfixed \ - "${trivy_ignore_args[@]}" \ + --cache-dir "$CONTAINER_CACHE_DIR" \ + --ignorefile "$CONTAINER_IGNORE_FILE" \ --skip-files /usr/local/bin/gosu \ --severity HIGH,CRITICAL \ - --exit-code 1 \ - "$TARGET_IMAGE" + --exit-code 1 test -s "$ROOT/$OUTPUT" -echo "Container vulnerability report written to $OUTPUT" +echo "Container vulnerability report written to $OUTPUT for $IMAGE_ID"