Implement modularized workflow validation and update build scripts - #3172

#!/bin/bash
# Verify which secrets are actually referenced by the reusable E2E workflow.
rg -n '\${{\s*secrets\.' .github/workflows/validate/e2e.yml

#!/bin/bash
set -euo pipefail

FILE=".github/workflows/validate/e2e.yml"

echo "== File =="
ls -la "$FILE"

echo
echo "== Literal references to reusable secrets (e.g. \${{ secrets.NAME }} ) =="
rg -nF '${{ secrets.' "$FILE" -C 2 || true

echo
echo "== All occurrences of 'secrets.' in the file (broader pattern) =="
rg -n 'secrets\.' "$FILE" -C 1 || true

echo
echo "== Extract unique secret names referenced =="
python3 - <<'PY'
import re, pathlib
p = pathlib.Path(".github/workflows/validate/e2e.yml")
txt = p.read_text()
names = sorted(set(re.findall(r"\$\{\{\s*secrets\.([A-Za-z0-9_]+)\s*\}\}", txt)))
print("\n".join(names) if names else "(none found)")
PY

#!/bin/bash
set -euo pipefail
FILE=".github/workflows/validate/e2e.yml"
python3 - <<'PY'
import re, pathlib
txt = pathlib.Path("$FILE").read_text()
# Match patterns like: ${{ secrets.NAME || ... }} and ${ { secrets.NAME }}
names = sorted(set(re.findall(r"\$\{\{\s*secrets\.([A-Za-z0-9_]+)", txt)))
print("\n".join(names) if names else "(none found)")
PY

#!/bin/bash
set -euo pipefail

FILE_E2E=".github/workflows/validate/e2e.yml"
FILE_PR=".github/workflows/pr-builder.yml"

echo "== Reusable workflow secrets references (unique names) =="
python3 - <<PY
import re, pathlib
txt = pathlib.Path("${FILE_E2E}").read_text()
names = sorted(set(re.findall(r"\$\{\{\s*secrets\.([A-Za-z0-9_]+)", txt)))
print("\n".join(names) if names else "(none found)")
PY

echo
echo "== Literal references to reusable secrets in e2e.yml =="
rg -nF '${{ secrets.' "$FILE_E2E" -C 1

echo
echo "== Confirm pr-builder.yml around validate-e2e =="
rg -n 'validate-e2e:' "$FILE_PR" -n -C 3

#!/bin/bash
set -euo pipefail

FILE_E2E=".github/workflows/validate/e2e.yml"
FILE_PR=".github/workflows/pr-builder.yml"

echo "== All literal occurrences of \${{ secrets. in e2e.yml =="
rg -nF '${{ secrets.' "$FILE_E2E"

echo
echo "== pr-builder.yml validate-e2e job secrets forwarding =="
rg -n "validate-e2e:|secrets:" "$FILE_PR" -n -C 2

# First, check if the file exists and inspect the content around lines 137 and 232
wc -l .github/workflows/validate/backend.yml

# Read the specific lines mentioned in the review
sed -n '130,145p' .github/workflows/validate/backend.yml
sed -n '225,240p' .github/workflows/validate/backend.yml

# Search for all occurrences of codecov-action in the file to understand context
rg 'codecov-action' .github/workflows/validate/backend.yml -B 2 -A 2

#!/bin/bash
set -euo pipefail

FILE=".github/workflows/validate/backend.yml"
echo "== Checking file exists =="
ls -la "$FILE"

echo
echo "== Showing lines around 160-190 =="
nl -ba "$FILE" | sed -n '140,210p'

echo
echo "== Searching for ':latest' in the workflow =="
rg -n --fixed-string ":latest" "$FILE" || true

echo
echo "== Searching for postgres/redis image references =="
rg -n "image:\s*(postgres|redis):" "$FILE" || true

#!/bin/bash
set -euo pipefail

FILE=".github/workflows/validate/backend.yml"

echo "== Showing lines around 160-190 (with line numbers via awk) =="
awk 'NR>=140 && NR<=210 {printf "%d\t%s\n", NR, $0}' "$FILE"

echo
echo "== Searching for ':latest' in the workflow =="
rg -n --fixed-string ":latest" "$FILE" || true

echo
echo "== Searching for postgres/redis image references =="
rg -n "image:\s*(postgres|redis):" "$FILE" || true

#!/bin/bash
set -euo pipefail

FILE=".github/workflows/validate/backend.yml"

echo "== Searching for all ':latest' occurrences in the workflow =="
rg -n ":latest" "$FILE" || true

echo
echo "== Showing exact service blocks for postgres/redis (lines 160-190) =="
awk 'NR>=155 && NR<=190 {printf "%d\t%s\n", NR, $0}' "$FILE"

#!/bin/bash
set -euo pipefail

# 1) Show the relevant parts of the workflow with line numbers
FILE=".github/workflows/validate/e2e.yml"
echo "== File: $FILE =="
if [ -f "$FILE" ]; then
  nl -ba "$FILE" | sed -n '1,220p'
else
  echo "Missing: $FILE"
  exit 1
fi

# 2) Find all checkout steps using the pinned v4 SHA in this repo (and confirm if persist-credentials is already set)
echo
echo "== All actions/checkout occurrences with the pinned SHA in .github/workflows =="
rg -n "actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5" .github/workflows -S || true

echo
echo "== Any checkout steps in this file that already set persist-credentials =="
rg -n "actions/checkout@|persist-credentials" "$FILE" -S || true

# 3) Check for any git operations that might require credentials/token after checkout
echo
echo "== git push / auth / GITHUB_TOKEN usage in this file =="
rg -n "git push|GITHUB_TOKEN|persist-credentials|checkout.*token|token:" "$FILE" -S || true

#!/bin/bash
set -euo pipefail

FILE=".github/workflows/validate/e2e.yml"
echo "== File: $FILE =="

if [ ! -f "$FILE" ]; then
  echo "Missing: $FILE"
  exit 1
fi

# Show enough of the workflow to include all checkout steps and nearby context
cat -n "$FILE" | sed -n '1,220p'

echo
echo "== All actions/checkout occurrences with the pinned SHA in .github/workflows =="
rg -n "actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5" .github/workflows -S || true

echo
echo "== Any checkout steps in this file that already set persist-credentials =="
rg -n "persist-credentials" "$FILE" -S || true
rg -n "uses: actions/checkout@" "$FILE" -S || true

echo
echo "== Any git push / auth / GITHUB_TOKEN usage in this file =="
rg -n "git push|GITHUB_TOKEN|persist-credentials|checkout.*token|token:" "$FILE" -S || true

#!/bin/bash
set -euo pipefail
FILE=".github/workflows/validate/e2e.yml"

echo "== Line count =="
wc -l "$FILE"

echo
echo "== actions/checkout@... occurrences in e2e.yml =="
rg -n "uses:\s*actions/checkout@" "$FILE" -S

echo
echo "== persist-credentials occurrences in e2e.yml =="
rg -n "persist-credentials" "$FILE" -S || true

#!/bin/bash
set -euo pipefail
FILE=".github/workflows/validate/e2e.yml"

echo "== git-related commands in e2e.yml =="
rg -n "(^|\s)git\s+(push|fetch|pull|clone|config)|GITHUB_TOKEN|token:" "$FILE" -S || true

echo
echo "== persist-credentials in the repo (to see if any pattern exists) =="
rg -n "persist-credentials\s*:\s*(false|true)" .github/workflows -S || true

#!/bin/bash
set -euo pipefail

FILE=".github/workflows/validate/e2e.yml"

echo "== File exists? =="
ls -la "$FILE" || true

echo
echo "== Snippet around lines 150-190 =="
nl -ba "$FILE" | sed -n '150,190p'

echo
echo "== Snippet around lines 250-300 (covers 266-277) =="
nl -ba "$FILE" | sed -n '250,300p'

echo
echo "== Snippet around lines 300-340 (covers 309-320) =="
nl -ba "$FILE" | sed -n '300,340p'

echo
echo "== Find all curl health/liveness probes =="
rg -n --fixed-string "health/liveness" "$FILE" || true

echo
echo "== Find all curl probes to localhost:3000 (if any) =="
rg -n --fixed-string "https://localhost:3000" "$FILE" || true

#!/bin/bash
set -euo pipefail

FILE=".github/workflows/validate/e2e.yml"

echo "== Snippet around lines 150-190 =="
awk 'NR>=150 && NR<=190 {printf "%5d  %s\n", NR, $0}' "$FILE"

echo
echo "== Snippet around lines 250-300 (covers 266-277) =="
awk 'NR>=250 && NR<=300 {printf "%5d  %s\n", NR, $0}' "$FILE"

echo
echo "== Snippet around lines 300-340 (covers 309-320) =="
awk 'NR>=300 && NR<=340 {printf "%5d  %s\n", NR, $0}' "$FILE"

echo
echo "== Find all curl health/liveness probes =="
rg -n "curl .*health/liveness" "$FILE" || true

echo
echo "== Find all curl probes to localhost:3000 =="
rg -n "curl .*https://localhost:3000" "$FILE" || true

echo
echo "== Check whether any uses curl -sSf anywhere in this workflow file =="
rg -n "curl .* -sSf" "$FILE" || true

#!/bin/bash
set -euo pipefail

# Show the workflow file around the referenced lines
FILE=".github/workflows/validate/frontend.yml"
echo "== File: $FILE =="
python3 - <<'PY'
import itertools
file_path=".github/workflows/validate/frontend.yml"
start=1
end=200
with open(file_path,'r',encoding='utf-8') as f:
    for i,line in enumerate(f,1):
        if 40 <= i <= 120:
            print(f"{i:4d}: {line.rstrip()}")
        if i>120:
            break
PY

# Extract any other jq/npm audit usage in the repo (to understand broader context)
echo
echo "== Search for npm audit --json usage =="
rg -n "npm audit --json" .github/workflows || true

echo
echo "== Search for jq parsing of .vulnerabilities[].via[] =="
rg -n "\\.vulnerabilities\\[\\]\\.via\\[\\]" .github/workflows || true

#!/bin/bash
set -euo pipefail

JQ_EXPR='
  [.vulnerabilities[].via[] | select(type == "object")
   | select(.severity == "high" or .severity == "critical")
   | select(.url | test($ignored | join("|")) | not)
  ] | length
'

# Reuse same jq invocation style as the workflow
JQ_BASE=(jq --argjson ignored '["ghsa-ignored-example"]' "$JQ_EXPR")

echo "== Case 1: Valid JSON with empty vulnerabilities =="
payload='{"vulnerabilities":{}}'
printf '%s\n' "$payload" | "${JQ_BASE[@]}"
echo "exit=$?"

echo "== Case 2: Missing vulnerabilities key =="
payload='{"meta":{}}'
printf '%s\n' "$payload" | "${JQ_BASE[@]}"
echo "exit=$?"

echo "== Case 3: vulnerabilities is null =="
payload='{"vulnerabilities":null}'
printf '%s\n' "$payload" | "${JQ_BASE[@]}"
echo "exit=$?"

echo "== Case 4: via is missing/empty =="
payload='{"vulnerabilities":{"a":{"via":[]}}}'
printf '%s\n' "$payload" | "${JQ_BASE[@]}"
echo "exit=$?"

echo "== Case 5: via is a string (shape mismatch) =="
payload='{"vulnerabilities":{"a":{"via":"not-an-array"}}}'
printf '%s\n' "$payload" | "${JQ_BASE[@]}"
echo "exit=$?"

echo "== Case 6: Invalid JSON (npm audit error text) =="
payload='npm audit failed'
# show jq exit code without stopping
set +e
printf '%s\n' "$payload" | "${JQ_BASE[@]}"
echo "jq_exit=$?"
set -e

echo "== Case 7: via contains object(s) with high/critical but url missing =="
payload='{"vulnerabilities":{"a":{"via":[{"severity":"high","name":"x"}]}}}'
printf '%s\n' "$payload" | "${JQ_BASE[@]}"
echo "exit=$?"

#!/bin/bash
set -euo pipefail

FILE=".github/workflows/validate/frontend.yml"

echo "== Show relevant workflow context around the audit loop =="
python3 - <<'PY'
import itertools
path=".github/workflows/validate/frontend.yml"
start=1
end=90
with open(path,'r',encoding='utf-8') as f:
    for i,line in enumerate(f,1):
        if 25 <= i <= 80:
            print(f"{i:4d}: {line.rstrip()}")
PY

echo
echo "== Re-run jq shape tests, capturing exit codes without aborting =="
JQ_EXPR='
  [.vulnerabilities[].via[] | select(type == "object")
   | select(.severity == "high" or .severity == "critical")
   | select(.url | test($ignored | join(\"|\")) | not)
  ] | length
'
IGNORED='["ghsa-ignored-example"]'

run_case () {
  local label="$1"
  local payload="$2"
  echo
  echo "-- $label --"
  set +e
  printf '%s\n' "$payload" | jq --argjson ignored "$IGNORED" "$JQ_EXPR"
  local code=$?
  echo "exit=$code"
  set -e
}

run_case "Case 1: valid JSON with empty vulnerabilities" '{"vulnerabilities":{}}'
run_case "Case 2: missing vulnerabilities key" '{"meta":{}}'
run_case "Case 3: vulnerabilities is null" '{"vulnerabilities":null}'
run_case "Case 4: via is []" '{"vulnerabilities":{"a":{"via":[]}}}'
run_case "Case 5: via is string (shape mismatch)" '{"vulnerabilities":{"a":{"via":"not-an-array"}}}'
run_case "Case 6: via contains object with high but url missing" '{"vulnerabilities":{"a":{"via":[{"severity":"high","name":"x"}]}}}'

echo
echo "== Invalid npm audit text (not JSON) -> jq behavior =="
set +e
printf '%s\n' "npm audit failed" | jq --argjson ignored "$IGNORED" "$JQ_EXPR" >/dev/null
echo "exit=$?"
set -e

#!/bin/bash
set -euo pipefail

IGNORED='["ghsa-ignored-example"]'

# Exact jq program from workflow (no extra escaping)
JQ_PROG='
  [.vulnerabilities[].via[] | select(type == "object")
   | select(.severity == "high" or .severity == "critical")
   | select(.url | test($ignored | join("|")) | not)
  ] | length
'

echo "== jq exit codes for shape variants (standalone) =="
run_jq () {
  local label="$1"
  local payload="$2"
  echo
  echo "-- $label --"
  set +e
  out="$(printf '%s\n' "$payload" | jq --argjson ignored "$IGNORED" "$JQ_PROG" 2>&1)"
  code=$?
  printf '%s\n' "$out"
  echo "exit=$code"
  set -e
}

run_jq "Case 1: vulnerabilities = {}" '{"vulnerabilities":{}}'
run_jq "Case 2: vulnerabilities missing" '{"meta":{}}'
run_jq "Case 3: vulnerabilities = null" '{"vulnerabilities":null}'
run_jq "Case 4: via = []" '{"vulnerabilities":{"a":{"via":[]}}}'
run_jq "Case 5: via = not array" '{"vulnerabilities":{"a":{"via":"not-an-array"}}}'
run_jq "Case 6: high/critical via object but missing url" '{"vulnerabilities":{"a":{"via":[{"severity":"high","name":"x"}]}}}'
run_jq "Case 7: invalid JSON text" 'npm audit failed'

echo
echo "== bash -e behavior when jq fails in command substitution =="
set +e
set -e

test_script () {
  local payload="$1"
  echo
  echo "-- payload: $payload --"
  set +e
  # emulates: REMAINING=$(echo "$AUDIT_RESULT" | jq ... )
  REMAINING=$(printf '%s\n' "$payload" | jq --argjson ignored "$IGNORED" "$JQ_PROG" )
  jq_code=$?
  echo "jq_exit=$jq_code REMAINING='${REMAINING}'"
  set -e
  echo "script continued"
}

set +e
test_script '{"vulnerabilities":{}}'
test_script '{"meta":{}}'
test_script '{"vulnerabilities":null}'
test_script '{"vulnerabilities":{"a":{"via":"not-an-array"}}}'
test_script 'npm audit failed'

set -e
echo
echo "done"

#!/bin/bash
set -e

echo "case: REMAINING empty"
REMAINING=''
if [ "$REMAINING" -gt 0 ]; then
  echo "in if"
else
  echo "in else (no exit)"
fi
echo "after if (should only print if not exited)"

#!/bin/bash
set -euo pipefail

FILE=".github/workflows/validate/frontend.yml"

echo "== Exists? =="
ls -l "$FILE" || exit 1

echo
echo "== Matching codecov action usage =="
rg -n --hidden --no-ignore-vcs "uses:\s*codecov/codecov-action@" "$FILE" || true

echo
echo "== Show lines around 245 and 257 =="
nl -ba "$FILE" | sed -n '230,270p'

#!/bin/bash
set -euo pipefail
FILE=".github/workflows/validate/frontend.yml"

echo "== Show lines around 245 and 257 =="
# Prefer cat -n; fallback to sed+grep for line numbers if needed.
if command -v cat >/dev/null 2>&1 && cat -n "$FILE" >/dev/null 2>&1; then
  cat -n "$FILE" | sed -n '235,265p'
else
  sed -n '235,265p' "$FILE" | nl -ba -w2 -s': ' || true
fi

echo
echo "== Full matching lines (with context) =="
rg -n "uses:\s*codecov/codecov-action@" "$FILE" -n -C 3