-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathimport_sources.sh
More file actions
167 lines (138 loc) · 5.77 KB
/
Copy pathimport_sources.sh
File metadata and controls
167 lines (138 loc) · 5.77 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
#!/usr/bin/env bash
# scripts/import_sources.sh
# -----------------------------------------------------------------------------
# Import remote repository docs into the hub repo.
#
# Behavior (idempotent):
# - For each entry in sources.json:
# * Shallow clone the repo with sparse-checkout limited to .subdir
# * Rsync the selected subtree into the local .mount path
# - Respects per-source "exclude" globs and per-repo ".indexignore"
# * Normalize front matter for Just the Docs via ensure_front_matter.py
# - Passes PROJECT_TITLE, REDIRECT_FROM (JSON), and NAV_ORDER env vars
#
# Inputs:
# sources.json entries:
# - name : short id
# - title : display name (defaults to name)
# - repo : org/repo
# - ref : branch or tag
# - subdir : subtree to import
# - mount : destination mount path in this repo
# - exclude[] : optional rsync exclude patterns
# - redirect_from[] : optional legacy slugs (or "redirects[]" for bw-compat)
# - nav_order : optional integer for navigation ordering
#
# Requirements: bash, git, jq, rsync
# Safe defaults: set -euo pipefail, strict quoting, basic path sanitization.
# -----------------------------------------------------------------------------
set -euo pipefail
IFS=$'\n\t'
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
readonly ROOT
readonly MANIFEST="${ROOT}/_data/sources.json"
# --- Utilities ----------------------------------------------------------------
die() { echo "error: $*" >&2; exit 1; }
require_cmd() {
command -v "$1" >/dev/null 2>&1 || die "$1 not found"
}
# Retry helper with exponential backoff.
retry_cmd() {
local label="$1"
shift
local max_retries="${IMPORT_MAX_RETRIES:-3}"
local base_delay="${IMPORT_RETRY_BASE_SECONDS:-2}"
local attempt=1
local delay="$base_delay"
while true; do
if "$@"; then
return 0
fi
if (( attempt >= max_retries )); then
echo "error: ${label} failed after ${attempt} attempts" >&2
return 1
fi
echo "warning: ${label} failed on attempt ${attempt}/${max_retries}; retrying in ${delay}s..." >&2
sleep "$delay"
attempt=$((attempt + 1))
delay=$((delay * 2))
done
}
clone_sparse_repo() {
local url="$1"
local ref="$2"
local tmp_abs="$3"
local sub="$4"
rm -rf -- "${tmp_abs:?}"
git clone --quiet --filter=blob:none --no-checkout --depth 1 --branch "$ref" "$url" "$tmp_abs"
git -C "$tmp_abs" sparse-checkout init --cone
git -C "$tmp_abs" sparse-checkout set "$sub"
git -C "$tmp_abs" checkout --quiet
}
sanitize_relpath() {
# Reject empty, absolute, current-dir, parent-traversal, and empty segments.
# Valid examples: "docs", "projects/foo"
# Invalid examples: ".", "..", "/abs", "a/../b", "a//b"
local p="${1:-}"
local seg=""
[[ -n "$p" && "$p" != /* && "$p" != "." && "$p" != ".." ]] || return 1
IFS='/' read -r -a _parts <<<"$p"
for seg in "${_parts[@]}"; do
[[ -n "$seg" && "$seg" != "." && "$seg" != ".." ]] || return 1
done
}
# --- Preflight -----------------------------------------------------------------
require_cmd jq
require_cmd rsync
require_cmd git
[[ -f "$MANIFEST" ]] || die "missing ${MANIFEST}"
mkdir -p "${ROOT}/_ext"
# --- Main ----------------------------------------------------------------------
jq -c '.sources[]' "$MANIFEST" | while IFS= read -r row; do
name="$(jq -er '.name // empty' <<<"$row")" || die "source entry missing name"
title="$(jq -r '.title // .name // empty' <<<"$row")"
repo="$(jq -er '.repo // empty' <<<"$row")" || die "source '${name}' missing repo"
ref="$(jq -er '.ref // empty' <<<"$row")" || die "source '${name}' missing ref"
sub="$(jq -er '.subdir // empty' <<<"$row")" || die "source '${name}' missing subdir"
mount="$(jq -er '.mount // empty' <<<"$row")" || die "source '${name}' missing mount"
[[ "$name" =~ ^[A-Za-z0-9._-]+$ ]] || die "invalid source name slug: $name"
[[ "$repo" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || die "invalid repo format: $repo"
sanitize_relpath "$sub" || die "invalid subdir: $sub"
sanitize_relpath "$mount" || die "invalid mount: $mount"
tmp="_ext/$name"
sanitize_relpath "$tmp" || die "invalid tmp: $tmp"
TMP_ABS="${ROOT}/${tmp}"
MOUNT_ABS="${ROOT}/${mount}"
SRC_ABS="${TMP_ABS}/${sub}"
[[ "${MOUNT_ABS}" != "${ROOT}" && "${MOUNT_ABS}" != "${ROOT}/" ]] \
|| die "refusing to sync into repo root for source '${name}'"
url="https://github.com/${repo}.git"
echo "::group::Clone ${repo}@${ref} -> ${tmp} (sparse: ${sub})"
retry_cmd "clone ${repo}@${ref}" clone_sparse_repo "$url" "$ref" "$TMP_ABS" "$sub" \
|| die "unable to import ${repo}@${ref}"
echo "::endgroup::"
echo "::group::Sync ${sub} -> ${mount}"
rm -rf -- "${MOUNT_ABS:?}"
mkdir -p "${MOUNT_ABS}"
# Build rsync filter list from manifest excludes
RSYNC_FILTERS=()
while IFS= read -r pat; do
[[ -n "$pat" ]] && RSYNC_FILTERS+=(--filter="- ${pat}")
done < <(jq -r '.exclude[]? // empty' <<<"$row")
rsync -a --delete \
--filter='dir-merge,- .indexignore' \
"${RSYNC_FILTERS[@]}" \
"${SRC_ABS%/}/" "${MOUNT_ABS%/}/"
echo "::endgroup::"
# Debug: list top-level imported Markdown files
echo "Imported files in ${mount}:"
find "${MOUNT_ABS}" -maxdepth 1 -type f -name '*.md' -printf ' %P\n' | sort || true
echo "::group::Normalize front matter for ${mount}"
# Accept either "redirect_from" or legacy "redirects"
redirect_from_json="$(jq -c '.redirect_from? // .redirects? // []' <<<"$row")"
nav_order="$(jq -r '.nav_order? // ""' <<<"$row")"
PROJECT_TITLE="$title" REDIRECT_FROM="$redirect_from_json" NAV_ORDER="$nav_order" \
python3 "${ROOT}/scripts/ensure_front_matter.py" "$MOUNT_ABS"
echo "::endgroup::"
done
# --- EOF -----------------------------------------------------------------------