java-profiler/.github/scripts/quarantine.py at main · DataDog/java-profiler

History

executable file

404 lines (337 loc) · 17.5 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

#!/usr/bin/env python3

"""The quarantine list: which failing tests do not turn CI red.

Its `validate` subcommand enforces the format, the ticket, and the review_by

date. The gating decision itself (find_entry(), covers(), applies_to()) is a

library used in-process by flake_report.py -- there is no CLI for it, so the

rule CI actually runs cannot drift from a separate CLI wrapper.

The paste-ready entry a PR comment proposes for a flaky test is rendered by

flake_summary.py's own call to format_entry() below, not by this module's CLI.

The list is a plain text table (see ddprof-test/quarantine.txt) rather than

JSON or YAML: it is edited by hand far more often than by machine, so real

comments, one-line diffs and clean `git blame` matter more than a schema. It

also has to parse inside the Alpine test containers, where PyYAML cannot be

assumed -- this needs nothing but str.split.

"""

import argparse

import datetime

import fnmatch

import os

import re

import sys

DEFAULT_LIST = os.path.join("ddprof-test", "quarantine.txt")

TICKET_RE = re.compile(r"^PROF-\d+$")

DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")

FIELDS = ("test", "ticket", "added", "review_by", "cells", "reason")

# Long enough not to be busywork, short enough that a quarantine outlives

# neither the release it was added in nor the memory of why.

DEFAULT_REVIEW_DAYS = 90

# A review_by further out than this is not a review date, it is a way to write

# "never" without saying so. Padded above DEFAULT_REVIEW_DAYS since a proposal

# is dated `added` at the moment it is written, and review_by is measured from

# whenever the entry is actually appended -- which is not the same day.

MAX_REVIEW_DAYS = DEFAULT_REVIEW_DAYS + 30

# The `test` field is an exact test id, optionally ending in a class-wide

# ".*" -- that is all covers() understands. Anything else (a bare "*", a "?",

# or a wildcard anywhere but as the final two characters) passes validate()

# today and then silently quarantines nothing at runtime.

BAD_TEST_WILDCARD_RE = re.compile(r"[*?]")

# Cell names are <libc>-<jdk>-<config>-<arch>. Only libc and arch are a closed

# set -- jdk and config come from the workflow inputs and grow without warning

# -- so those two are the only axes worth checking a glob against.

# "el7" names the GitLab functional job's Oracle Linux 7 runtime, which is

# technically glibc but must not share the "glibc" token: that job runs a

# different environment (container, kernel, network stack) than the GitHub

# Actions "glibc" matrix (Ubuntu), and the two can fail the same test for

# unrelated reasons. Sharing a cell string would let an entry meant to excuse

# one silently excuse the other.

KNOWN_ARCHES = ("amd64", "aarch64")

KNOWN_LIBCS = ("glibc", "musl", "el7")

# Anything that reads like an architecture. A glob naming one that CI never

# builds silently quarantines nothing, which is how "*arm64*" shipped in this

# file's own example: the arch is spelled aarch64.

ARCH_LIKE_RE = re.compile(r"(?:x86|x64|amd|arm|aarch|i386|ppc|s390)[\w_]*")

# A synthetic universe of cell names, used only to ask whether two entries'

# cell globs could both match the same real cell. Wide enough to catch a glob

# written against any axis (jdk, config, the libc/arch pair, or the slow/regular

# suite suffix) without having to enumerate the workflow's actual, ever-growing

# matrix. Over-inclusive on purpose: a synthetic cell that never occurs for

# real only makes overlap detection more conservative, never less. Under-

# inclusive is the dangerous direction -- cells_overlap() fails closed for a

# glob matching nothing synthetic, so a JDK variant missing from here makes two

# genuinely disjoint entries look like duplicates and fail validation.

_SYNTHETIC_JDK_BASES = ("8", "11", "17", "21", "25")

_SYNTHETIC_JDK_SUFFIXES = ("", "-orcl", "-j9", "-ibm", "-graal")

_SYNTHETIC_JDKS = tuple(

base + suffix

for base in _SYNTHETIC_JDK_BASES

for suffix in _SYNTHETIC_JDK_SUFFIXES

)

_SYNTHETIC_CONFIGS = ("debug", "release", "asan", "tsan")

_SYNTHETIC_SUITE_SUFFIXES = ("", "-slow")

SYNTHETIC_CELLS = tuple(

"{}-{}-{}-{}{}".format(libc, jdk, config, arch, suffix)

for libc in KNOWN_LIBCS

for jdk in _SYNTHETIC_JDKS

for config in _SYNTHETIC_CONFIGS

for arch in KNOWN_ARCHES

for suffix in _SYNTHETIC_SUITE_SUFFIXES

)

def _matches_any_synthetic_cell(globs):

return any(any(fnmatch.fnmatch(cell, g) for g in globs) for cell in SYNTHETIC_CELLS)

def cells_overlap(globs_a, globs_b):

"""Could some real cell match both sets of globs? No globs means every cell.

Equal glob lists always overlap without needing the synthetic universe,

which matters when a glob names an axis (like a jdk or config) that

SYNTHETIC_CELLS does not model. And when a glob's axis is genuinely

unmodelled -- it matches nothing in the synthetic universe at all -- this

fails closed (treats it as overlapping) rather than open: a duplicate that

cells_overlap cannot evaluate is exactly the case validate() must not wave

through, since find_entry() would still only honour the first entry.

"""

if not globs_a or not globs_b:

return True

if sorted(globs_a) == sorted(globs_b):

return True

if not _matches_any_synthetic_cell(globs_a) or not _matches_any_synthetic_cell(globs_b):

return True

return any(

any(fnmatch.fnmatch(cell, g) for g in globs_a)

and any(fnmatch.fnmatch(cell, g) for g in globs_b)

for cell in SYNTHETIC_CELLS

)

def parse(path):

"""([entry], [(line number, message)]) — entries and malformed lines.

Each entry carries `_line` so validate() can point at the offender.

"""

entries, errors = [], []

if not os.path.exists(path):

return entries, errors

with open(path) as handle:

for number, raw in enumerate(handle, start=1):

line = raw.strip()

if not line or line.startswith("#"):

continue

parts = [p.strip() for p in line.split("|")]

if len(parts) != len(FIELDS):

errors.append((number, "expected {} fields separated by '|', found {}".format(

len(FIELDS), len(parts))))

continue

entry = dict(zip(FIELDS, parts))

entry["cells"] = [c.strip() for c in entry["cells"].split(",")

if c.strip() and c.strip() != "-"]

entry["_line"] = number

entries.append(entry)

return entries, errors

def load(path):

"""Entries only, for callers that just need to match against the list."""

return parse(path)[0]

def applies_to(entry, cell):

"""Does this entry cover the given cell? No globs means everywhere."""

globs = entry.get("cells")

if not globs:

return True

return any(fnmatch.fnmatch(cell, g) for g in globs)

_INVOCATION_INDEX_RE = re.compile(r"^\[\d+\]$")

def normalise_test_id(test_id):

"""A JUnit XML test id reduced to the shape entries are written in.

Gradle writes a JUnit 5 @Test method as `name="method()"`, so the id built

from the report is `Class.method()`, while quarantine.txt documents -- and

a human writes -- `Class.method`. Stripping the parentheses here, in the

one function every caller's match goes through, keeps the documented shape

matching the id JUnit actually produces while reports and annotations go

on showing the real name.

"""

return test_id[:-2] if test_id.endswith("()") else test_id

def covers(entry, test_id):

pattern = entry["test"]

test_id = normalise_test_id(test_id)

if pattern.endswith(".*"):

# "covers every method in the class", per quarantine.txt -- so the

# remainder after the class name must be a single method segment. An

# unbounded prefix match would make "com.datadoghq.profiler.*" suspend

# the merge gate for the entire repository from one validate-clean

# line.

prefix = pattern[:-1]

if not test_id.startswith(prefix):

return False

return "." not in test_id[len(prefix):]

return test_id == pattern

def _parse_date(s):

"""A YYYY-MM-DD string as a date, or raises ValueError.

date.fromisoformat() needs Python 3.7+; this also has to run under

Python 3.6 (EL7's base-repo python3).

"""

return datetime.datetime.strptime(s, "%Y-%m-%d").date()

def is_expired(entry, today=None):

"""True once this entry's review_by date has passed.

validate-quarantine (quarantine.py's own CLI) only runs from ci.yml, but

find_entry() is called from every workflow that reuses run_tests_with_retry.sh

(nightly.yml, release-validated.yml included). Enforcing expiry here, at

match time, means a stale mute cannot keep quarantining a failure just

because the workflow that hit it never runs the separate validator.

"""

review_by = entry.get("review_by", "")

if not DATE_RE.match(review_by):

# Blank or not a date at all. parse() accepts both, and validate()

# only runs in PR CI, so treating an unreadable expiry as "never

# expires" would let the one entry nobody can review outlive every

# entry that can be. An expiry that cannot be read has passed.

return True

try:

due = _parse_date(review_by)

except ValueError:

return True

return due < (today or datetime.date.today())

def find_entry(entries, test_id, cell):

"""The first entry quarantining this test on this cell, or None.

Every caller that decides whether a failure gates goes through here, so the

matching rule cannot drift between the subcommand and flake_report.py. An

expired entry is treated as absent rather than as a hit, so it can never

excuse a failure outside the PR CI that happens to run validate-quarantine.

"""

return next(

(e for e in entries

if covers(e, test_id) and applies_to(e, cell) and not is_expired(e)),

None,

)

def format_entry(test, ticket, added, review_by, cells, reason):

return " | ".join([test, ticket, added, review_by, ",".join(cells) or "-", reason])

def cmd_validate(args):

# parse() tolerates a missing file so that matching still works before the

# first entry lands. Validation must not: "0 quarantined test(s), all

# valid" for a list that has been renamed or deleted would report success

# at the exact moment gating silently stopped applying everywhere.

if not os.path.exists(args.list):

print("::error::quarantine list '{}' does not exist".format(args.list))

return 1

entries, problems = parse(args.list)

today = datetime.date.today()

seen_by_name = []

def complain(line, message):

problems.append((line, message))

for entry in entries:

line = entry["_line"]

name = entry["test"]

for field in FIELDS:

if field == "cells":

continue # optional, normalised to [] above

if not entry[field]:

complain(line, "field '{}' is empty".format(field))

# Two entries shadow each other on cells where they overlap when either

# pattern covers() the other -- not just when the `test` strings are

# identical. A trailing ".*" entry covers individual methods too, and

# find_entry() only ever returns the first match, so the second

# entry's ticket and review_by silently never take effect on the

# cells the two share.

for prior in seen_by_name:

if not (covers(prior, entry["test"]) or covers(entry, prior["test"])):

continue

if cells_overlap(prior["cells"], entry["cells"]):

where = ", ".join(entry["cells"]) or "every cell"

complain(line, "'{}' is already quarantined (as '{}') for {} on line {}".format(

name, prior["test"], where, prior["_line"]))

break

seen_by_name.append(entry)

if entry["ticket"] and not TICKET_RE.match(entry["ticket"]):

complain(line, "ticket '{}' is not a PROF-<number>".format(entry["ticket"]))

for field in ("added", "review_by"):

if entry[field] and not DATE_RE.match(entry[field]):

complain(line, "{} '{}' is not YYYY-MM-DD".format(field, entry[field]))

stem = entry["test"][:-2] if entry["test"].endswith(".*") else entry["test"]

if entry["test"] and (not stem or BAD_TEST_WILDCARD_RE.search(stem)):

complain(line, (

"test pattern '{}' is not an exact id or a class-wide '<class>.*'; "

"covers() understands nothing else, so this would silently "

"quarantine nothing"

).format(entry["test"]))

# "<package>.*" is not a class. covers() scopes a trailing '.*' to one

# class's methods, so a package-level pattern quarantines nothing --

# while looking like it quarantines a great deal.

if entry["test"].endswith(".*") and stem:

last = stem.rsplit(".", 1)[-1]

if last and not last[:1].isupper():

complain(line, (

"test pattern '{}' reads as a package, not a class: a "

"trailing '.*' covers the methods of one class, so this "

"matches nothing. Name the class, or list its tests"

).format(entry["test"]))

if entry["test"].endswith("()"):

complain(line, (

"test '{}' carries the parentheses JUnit puts in its XML; "

"entries are written as <class>.<method>, and covers() "

"normalises the report's id to that shape -- drop the '()'"

).format(entry["test"]))

if _INVOCATION_INDEX_RE.match(entry["test"].rsplit(".", 1)[-1]):

complain(line, (

"test '{}' names an invocation index. @ParameterizedTest and "

"@RetryingTest invocations appear in the XML as '[1]', '[2]' "

"with no method name at all, so the index identifies neither "

"the method nor a stable case -- quarantine the class with "

"'<class>.*' instead"

).format(entry["test"]))

if entry["added"] and DATE_RE.match(entry["added"]):

try:

_parse_date(entry["added"])

except ValueError:

complain(line, "added '{}' is not a real calendar date".format(entry["added"]))

for pattern in entry["cells"]:

unknown_arch_tokens = [

t for t in ARCH_LIKE_RE.findall(pattern) if t not in KNOWN_ARCHES

]

# A token like "amd" or "aarch" (from "*amd*"/"*aarch*") is a

# legitimate abbreviation of a real arch and matches real cells;

# only complain when the glob, as actually evaluated by fnmatch,

# matches nothing in the synthetic universe -- that is what

# distinguishes a working glob from one like "*arm64*" that

# genuinely names an architecture CI never builds.

if unknown_arch_tokens and not _matches_any_synthetic_cell([pattern]):

complain(line, (

"cell glob '{}' names architecture '{}', which CI never "

"builds (cells end in {}); it would quarantine nothing"

).format(pattern, unknown_arch_tokens[0], " or ".join(KNOWN_ARCHES)))

head = pattern.split("-", 1)[0]

if head and "*" not in head and "?" not in head and head not in KNOWN_LIBCS:

complain(line, (

"cell glob '{}' starts with '{}'; cell names start with {}"

).format(pattern, head, " or ".join(KNOWN_LIBCS)))

if DATE_RE.match(entry["review_by"]):

try:

due = _parse_date(entry["review_by"])

except ValueError:

complain(line, "review_by '{}' is not a real calendar date".format(

entry["review_by"]))

else:

if due < today:

complain(line, (

"'{}' has been quarantined since {} and its review was due {} "

"({} days ago). Fix the test and delete this line, or renew "

"review_by with a note on {}."

).format(name, entry["added"], entry["review_by"],

(today - due).days, entry["ticket"] or "the ticket"))

elif due > today + datetime.timedelta(days=MAX_REVIEW_DAYS):

complain(line, (

"review_by '{}' is more than {} days out; that is not a "

"review date, it defeats the point of an expiring quarantine"

).format(entry["review_by"], MAX_REVIEW_DAYS))

for line, message in sorted(problems):

print("::error file={},line={}::{}".format(args.list, line, message))

if problems:

print("\n{} problem(s) in {}".format(len(problems), args.list), file=sys.stderr)

return 1

print("{}: {} quarantined test(s), all valid".format(args.list, len(entries)))

return 0

def main():

parser = argparse.ArgumentParser(description=__doc__)

parser.add_argument("--list", default=DEFAULT_LIST)

# required=True on add_subparsers() needs Python 3.7+; this also has to run

# under Python 3.6 (EL7's base-repo python3), so the check is manual.

sub = parser.add_subparsers(dest="command")

validate = sub.add_parser("validate", help="check the list's format and review dates")

validate.set_defaults(func=cmd_validate)

args = parser.parse_args()

if args.command is None:

parser.error("a command is required")

return args.func(args)

if __name__ == "__main__":

sys.exit(main())

FilesExpand file tree

quarantine.py

Latest commit

History

quarantine.py

File metadata and controls