.github/.github/workflows/python-security.yml at main · ContextualWisdomLab/.github

296 lines (282 loc) · 13.6 KB
# Central Python security gate for every ContextualWisdomLab repo.
# Fills a governance gap left when duplicate LOCAL workflows were removed in
# favour of the central required workflows: the central Security Scan bundle
# covers supply-chain (osv / dependency-review / trivy) and posture (scorecard),
# but NOT Python source SAST (bandit) or the Python dependency audit
# (pip-audit) that some repos ran locally (naruon, bandscope,
# xtrmLLMBatchPython, contextual-orchestrator).
#   bandit    Python SAST   -> SARIF uploaded under category "bandit"
#   pip-audit dep audit     -> HARD gate by job result (like osv/trivy)
# Both jobs are CONDITIONAL on the repo actually containing Python, so
# non-Python repos are a no-op. Gating is by the JOB result, ref-independent,
# exactly like the trivy-fs / osv-scan jobs in security-scan.yml. The SARIF is
# uploaded under a DISTINCT category ("bandit"); it is NOT added to the
# code_scanning ruleset rule, which stays CodeQL-only on purpose (requiring
# multiple tools in that rule is unsatisfiable across PR head/merge refs), so
# it does not affect auto-merge.
# High sensitivity: bandit fails on MEDIUM+ severity & MEDIUM+ confidence;
# pip-audit fails on any known vulnerability.
name: Python Security
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review, closed]
    branches: [main, master, develop]
    branches: [main, master, develop]
  # Periodic full-repo coverage so non-PR drift is caught (the removed local
  # workflows ran on push + schedule).
  schedule:
    - cron: "17 3 * * 1"
  repository_dispatch:
    types: [python-security-scan]
concurrency:
  group: python-security-${{ github.event.pull_request.base.repo.full_name || github.repository }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true
permissions:
  contents: read
  detect-python:
    name: Detect Python
    if: github.event.action != 'closed'
    runs-on: ubuntu-24.04
    outputs:
      has_python: ${{ steps.detect.outputs.has_python }}
      has_manifest: ${{ steps.detect.outputs.has_manifest }}
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - name: Detect Python sources and dependency manifests
        id: detect
        run: |
          set -euo pipefail
          has_python=false
          if find . -type f -name '*.py' -not -path './.git/*' -print -quit | grep -q .; then
            has_python=true
          has_manifest=false
          if find . -type f \
              \( -name 'requirements*.txt' -o -name 'pyproject.toml' \
                 -o -name 'pylock.*.toml' \) \
              -not -path './.git/*' -print -quit | grep -q .; then
            has_manifest=true
          echo "has_python=${has_python}" >> "$GITHUB_OUTPUT"
          echo "has_manifest=${has_manifest}" >> "$GITHUB_OUTPUT"
    name: Bandit (Python SAST)
    needs: detect-python
    if: github.event.action != 'closed' && needs.detect-python.outputs.has_python == 'true'
    runs-on: ubuntu-24.04
    permissions:
      contents: read
      security-events: write
      actions: read
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
        with:
          egress-policy: audit
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"
      - name: Install bandit
        # Hash-pinned lock (uv-generated) satisfies Scorecard Pinned-Dependencies.
        run: python -m pip install --require-hashes -r requirements-bandit-ci-hashes.txt
      - name: Run bandit (SARIF)
        id: bandit
        run: |
          set +e
          bandit --recursive . \
            --severity-level medium \
            --confidence-level medium \
            --exclude ./.git,./.github,./node_modules,./.venv,./venv,./tests,./test \
            --format json \
            --output bandit-results.json
          echo "rc=$?" >> "$GITHUB_OUTPUT"
          set -e
          if [ ! -s bandit-results.json ]; then
            echo "::error::Bandit did not produce bandit-results.json; inspect the Bandit command output above."
            exit 2
          python - <<'PY'
          import json
          from pathlib import Path
          data = json.loads(Path("bandit-results.json").read_text(encoding="utf-8"))
          issues = data.get("results", [])
          rules = {}
          results = []
          levels = {"HIGH": "error", "MEDIUM": "warning", "LOW": "note"}
          for issue in issues:
              rule_id = issue.get("test_id") or "bandit"
              name = issue.get("test_name") or rule_id
              text = issue.get("issue_text") or "Bandit finding"
              severity = issue.get("issue_severity", "UNKNOWN")
              confidence = issue.get("issue_confidence", "UNKNOWN")
              filename = (issue.get("filename") or "").replace("\\", "/").lstrip("./")
              line = int(issue.get("line_number") or 1)
              message = f"{rule_id}: {text} (severity={severity}, confidence={confidence})"
              print(f"::error file={filename},line={line},title={rule_id}::{message}")
              rules.setdefault(
                  rule_id,
                      "id": rule_id,
                      "name": name,
                      "shortDescription": {"text": name},
                      "fullDescription": {"text": text},
                      "helpUri": issue.get("more_info", "https://bandit.readthedocs.io/"),
              results.append(
                      "ruleId": rule_id,
                      "level": levels.get(severity, "warning"),
                      "message": {"text": message},
                      "locations": [
                              "physicalLocation": {
                                  "artifactLocation": {"uri": filename},
                                  "region": {"startLine": line},
          print(f"Bandit findings at configured threshold: {len(issues)}")
          sarif = {
              "$schema": "https://json.schemastore.org/sarif-2.1.0.json",
              "version": "2.1.0",
              "runs": [
                      "tool": {
                          "driver": {
                              "name": "Bandit",
                              "informationUri": "https://bandit.readthedocs.io/",
                              "rules": list(rules.values()),
                      "results": results,
          Path("bandit-results.sarif").write_text(json.dumps(sarif, indent=2), encoding="utf-8")
      - name: Upload Bandit SARIF to code scanning
        id: upload_bandit_sarif
        if: always() && hashFiles('bandit-results.sarif') != ''
        # The explicit gate below still fails on every Medium+ Bandit result.
        continue-on-error: true
        uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
        with:
          sarif_file: bandit-results.sarif
          category: bandit
          wait-for-processing: false
      - name: Report Bandit SARIF upload failure
        if: steps.upload_bandit_sarif.outcome == 'failure'
        run: |
          echo "::warning::Bandit SARIF upload to code scanning failed after the local Bandit scan. The Bandit hard gate still follows the scan rc, so upload rate limits cannot hide MEDIUM+ findings."
      - name: Enforce bandit gate (fail on MEDIUM+ findings)
        if: steps.bandit.outputs.rc != '0'
        run: |
          echo "::error::Bandit found MEDIUM+ severity/confidence issues. See the 'bandit' code scanning category."
          exit 1
  pip-audit:
    name: pip-audit (Python dependency audit)
    needs: detect-python
    if: github.event.action != 'closed' && needs.detect-python.outputs.has_manifest == 'true'
    runs-on: ubuntu-24.04
    permissions:
      contents: read
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
        with:
          egress-policy: audit
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"
      - name: Install pip-audit
        # Hash-pinned lock (uv-generated) satisfies Scorecard Pinned-Dependencies.
        run: python -m pip install --require-hashes -r requirements-pip-audit-ci-hashes.txt
      - name: Run pip-audit (hard gate on any known vulnerability)
        run: |
          set -euo pipefail
          status=0
          audit_log="$(mktemp)"
          # Run one pip-audit invocation and classify a non-zero exit. pip-audit
          # 2.10.1 prints "Found N known vulnerabilit(y|ies) ... in N package(s)"
          # on stderr only when it has findings (pip_audit/_cli.py); any other
          # non-zero exit is an audit-service/transport failure (advisory query
          # traceback, resolver/_fatal error) and must not be reported as a
          # vulnerability finding (#2158). Both keep the gate closed.
          run_audit() {
            local label="$1"
            shift
            echo "::group::pip-audit ${label}"
            if pip-audit "$@" 2>&1 | tee "${audit_log}"; then
              echo "::endgroup::"
              return 0
            echo "::endgroup::"
            status=1
            if grep -Eq "Found [0-9]+ known vulnerabilit" "${audit_log}"; then
              echo "::error::pip-audit found known-vulnerable Python dependencies in ${label}. Remediate the pins listed above."
            else
              echo "::error::pip-audit could not complete for ${label}: $(grep -Ev '^[[:space:]]*$' "${audit_log}" | tail -n 1). This is an audit-service/transport failure, not a vulnerability finding; rerun the job before treating it as a security result."
          # Audit every discovered requirements file.
          while IFS= read -r req; do
            # A matching requirements-<tool>-ci-overrides.txt (a `uv pip compile --override`
            # input, e.g. requirements-strix-ci-overrides.txt) means the *-hashes.txt this
            # override applies to pins a version whose declared metadata range intentionally
            # conflicts with another pin in the same file (verified safe at override time, not a
            # resolution mistake). pip's own dependency resolver -- which pip-audit's default
            # `-r` mode still calls even for fully hash-pinned files -- fails on that same
            # declared-range conflict regardless of --require-hashes, and plain --no-deps does
            # not suppress it (confirmed: --no-deps only skips fetching undeclared transitive
            # packages, pip's resolver still cross-checks the packages that *are* listed
            # together). --disable-pip bypasses pip's resolver entirely and audits the exact
            # pins directly, but it requires every requirement to be an exact version (raises on
            # any bare range) -- true for the compiled *-hashes.txt, not necessarily true for the
            # hand-maintained raw input (e.g. requirements-strix-ci.txt intentionally leaves
            # protobuf as a range). So: hashed output files with an override get
            # --disable-pip --no-deps; their raw, non-hash input counterpart is skipped here
            # (it is never itself a `pip install --require-hashes` target -- only its compiled
            # *-hashes.txt is installed -- and that compiled file is the one audited with full
            # transitive coverage).
            base="${req%.txt}"
            unhashed_base="${base%-hashes}"
            if [ "$base" != "$unhashed_base" ] && [ -f "${unhashed_base}-overrides.txt" ]; then
              run_audit "-r ${req} (--disable-pip --no-deps: overridden lock)" --strict --desc=on --no-deps --disable-pip -r "${req}"
            elif [ "$base" = "$unhashed_base" ] && [ -f "${unhashed_base}-overrides.txt" ]; then
              echo "::notice::Skipping pip-audit for ${req}: it is the raw input to an overridden lock (${unhashed_base}-hashes.txt), never itself a pip install --require-hashes target, and its compiled hashes file is audited separately with full resolution."
            else
              run_audit "-r ${req}" --strict --desc=on -r "${req}"
          done < <(find . -type f -name 'requirements*.txt' -not -path './.git/*')
          # Audit the project itself when a PEP 621 / lock manifest exists.
          if find . -maxdepth 2 -type f \
              \( -name 'pyproject.toml' -o -name 'pylock.*.toml' \) \
              -not -path './.git/*' -print -quit | grep -q .; then
            run_audit ". (project manifest)" --strict --desc=on .
          if [ "${status}" != "0" ]; then
            echo "::error::pip-audit failed for at least one input; the per-input errors above say whether it was a finding or an audit-service failure."
            exit 1
FilesExpand file tree

python-security.yml

Latest commit

History

python-security.yml

File metadata and controls
