package processing.mode.cpp; import processing.mode.cpp.FunctionPointerType; import processing.mode.cpp.FunctionSignatureType; import processing.mode.cpp.NamedType; import processing.mode.cpp.Param; import processing.mode.cpp.TypeRef; import java.util.ArrayList; import java.util.List; /** * Recursive-descent parser for CppMode's Java-flavored-C++ grammar subset. * * Design notes: * - Comments are consumed by skipComments() and attached as leadingComments * on the next real node produced, per the Lexer's comment-as-first-class- * token design (see Lexer notes). This is the direct replacement for the * old blankCommentsAndLiterals regex pass. * - Fail-fast: the first parse error throws ParseException immediately. * No error recovery is attempted (see ParseException's notes for why). * - Multi-declarator statements ("int rectX, rectY;") are desugared into * multiple single-name nodes at this layer -- see parseVariableDeclTail. */ public final class Parser { private final List tokens; private int pos = 0; public Parser(List tokens) { this.tokens = tokens; } public static CompilationUnit parse(String source) { List tokens = new CppLexer(source).tokenize(); return new Parser(tokens).parseCompilationUnit(); } /** Test-only entry point: parses a single TypeRef from source text in isolation. */ static TypeRef parseTypeRefFromString(String source) { List tokens = new CppLexer(source).tokenize(); return new Parser(tokens).parseTypeRef(); } // ----- Core cursor helpers ----------------------------------------- // // peek()/peek(ahead)/advance() transparently skip over comment tokens. // This is deliberate and fixes a real bug class found via real-corpus // testing: a line comment can legally appear in the MIDDLE of a // multi-line expression (e.g. a function call whose arguments are each // commented, confirmed real by Move_Eye.pde's // "camera(x, y, z, // eyeX, eyeY, eyeZ \n ...")). The original design // only collected leading comments at statement/declaration boundaries // via consumeLeadingComments(), which left every OTHER parse point // (inside parseArgList, parsePostfix, anywhere mid-expression) just as // comment-blind as the regex passes this parser exists to replace. // Routing comment-skipping through the cursor primitives themselves // fixes every call site at once instead of needing a fix at each one. // // consumeLeadingComments() still exists and is still called explicitly // at statement/declaration start -- it now serves only to COLLECT the // comment tokens for attachment to the next node's leadingComments // field, not to make parsing work correctly (that's now peek/advance's // job). It works correctly alongside this since skipCommentsAt() is // idempotent -- calling it twice in a row from the same position is a // no-op the second time. private void skipCommentsAt() { while (pos < tokens.size() && (tokens.get(pos).type() == CppLexerTokenType.LINE_COMMENT || tokens.get(pos).type() == CppLexerTokenType.BLOCK_COMMENT)) { pos++; } } private CppLexerToken peek() { int p = pos; while (p < tokens.size() && (tokens.get(p).type() == CppLexerTokenType.LINE_COMMENT || tokens.get(p).type() == CppLexerTokenType.BLOCK_COMMENT)) p++; return p < tokens.size() ? tokens.get(p) : tokens.get(tokens.size() - 1); } private CppLexerToken peek(int ahead) { // Walk forward from pos, skipping comment tokens, to find the // "ahead"-th real (non-comment) token after the current position. skipCommentsAt(); int p = pos; int remaining = ahead; while (remaining > 0 && p < tokens.size() - 1) { p++; while (p < tokens.size() && (tokens.get(p).type() == CppLexerTokenType.LINE_COMMENT || tokens.get(p).type() == CppLexerTokenType.BLOCK_COMMENT)) { p++; } remaining--; } return p < tokens.size() ? tokens.get(p) : tokens.get(tokens.size() - 1); } private CppLexerToken advance() { skipCommentsAt(); CppLexerToken t = tokens.get(pos); if (t.type() != CppLexerTokenType.EOF) pos++; return t; } private boolean isAtEnd() { return peek().type() == CppLexerTokenType.EOF; } private boolean check(CppLexerTokenType type) { return peek().type() == type; } private boolean checkKeyword(String kw) { return peek().isKeyword(kw); } private boolean checkPunct(String p) { return peek().isPunct(p); } private boolean checkOp(String op) { return peek().isOp(op); } /** True if the current token is the integer literal "0" specifically * (not any other literal or expression) -- used to recognize the * pure-virtual specifier "= 0" precisely, without accidentally * matching some other "= " shape. */ private boolean checkLiteralZero() { CppLexerToken t = peek(); return t.type() == CppLexerTokenType.INT_LITERAL && t.text().equals("0"); } private boolean matchKeyword(String kw) { if (checkKeyword(kw)) { advance(); return true; } return false; } private boolean matchPunct(String p) { if (checkPunct(p)) { advance(); return true; } return false; } private boolean matchOp(String op) { if (checkOp(op)) { advance(); return true; } return false; } private CppLexerToken expectPunct(String p) { if (checkPunct(p)) return advance(); throw error("expected '" + p + "' but found '" + peek().text() + "'"); } private CppLexerToken expectOp(String op) { if (checkOp(op)) return advance(); throw error("expected '" + op + "' but found '" + peek().text() + "'"); } private CppLexerToken expectKeyword(String kw) { if (checkKeyword(kw)) return advance(); throw error("expected '" + kw + "' but found '" + peek().text() + "'"); } /** * Set of KEYWORD-classified tokens that are NOT actually reserved * words in real Processing/Java/C++ -- they're lexed as keywords * purely so they can be recognized as TYPE names in a type-name * position (see Lexer's KEYWORDS set and its own comment on this), * but that classification incorrectly also blocked them from ever * being used as a declarator/variable NAME, which real Processing * code is free to do. * * Found via a real sketch (RayTracer.pde's "Vec3 color;" struct * field -- "color" the Processing pseudo-type, used as an ordinary * field name, which is completely legal Processing/C++ and not * something the language itself reserves) hitting * "expected identifier but found 'color'" outright. * * Deliberately a narrow, explicit allow-list -- NOT a general * relaxation letting any KEYWORD stand in for an identifier, which * would incorrectly let real reserved words ("int", "if", "class", * etc.) be used as names too. Only pseudo-type keywords that this * project itself introduced for type-name recognition belong here. */ private static final java.util.Set PSEUDO_TYPE_KEYWORDS_USABLE_AS_NAMES = java.util.Set.of( "color" ); private CppLexerToken expectIdentifier() { if (check(CppLexerTokenType.IDENTIFIER)) return advance(); if (check(CppLexerTokenType.KEYWORD) && PSEUDO_TYPE_KEYWORDS_USABLE_AS_NAMES.contains(peek().text())) { return advance(); } throw error("expected identifier but found '" + peek().text() + "'"); } private ParseException error(String message) { CppLexerToken t = peek(); return new ParseException(message, t.line(), t.col()); } /** * Consumes and discards any whitespace-adjacent comment tokens immediately * preceding the current position, returning them so the caller can attach * them as leadingComments on the node it's about to build. Must be called * at the start of every "parse one item/statement/declaration" entry point * so comments are never silently dropped nor mistaken for code. */ /** * Consumes and collects any comment tokens sitting at the current raw * position, for attachment to the next node's leadingComments field. * * Deliberately reads tokens.get(pos) directly rather than calling * peek()/check() -- those now transparently skip comment tokens (see * the cursor-helpers notes above, added after the Move_Eye.pde bug), * so they can never see a comment to report back here. This method is * the one place that still needs to look at raw, unfiltered token- * stream position. */ private List consumeLeadingComments() { List comments = new ArrayList<>(); while (pos < tokens.size() && (tokens.get(pos).type() == CppLexerTokenType.LINE_COMMENT || tokens.get(pos).type() == CppLexerTokenType.BLOCK_COMMENT)) { comments.add(tokens.get(pos)); pos++; } return comments; } // ----- Entry point --------------------------------------------------- public CompilationUnit parseCompilationUnit() { List items = new ArrayList<>(); consumeLeadingComments(); // leading file-level comments currently dropped // at EOF if nothing follows; fine for now while (!isAtEnd()) { List comments = consumeLeadingComments(); if (isAtEnd()) break; items.addAll(parseTopLevelItem(comments)); } return new CompilationUnit(items); } /** * Dispatches on lookahead to the appropriate top-level (or class-member, * since both positions share this same dispatch per TopLevelItem's design * notes) sub-parser. Returns a List since a single source statement can * desugar into multiple VariableDecl nodes (multi-declarator decls, * confirmed at both top-level and class-member scope by the Button and * Handles fixtures respectively) -- every other branch returns a * single-element list. */ private List parseTopLevelItem(List leadingComments) { consumeAttributes(); if (check(CppLexerTokenType.PREPROCESSOR_DIRECTIVE)) { CppLexerToken t = advance(); return List.of(new PreprocessorLine(t.text(), t.line(), t.col(), leadingComments)); } // C++20 concept definition: "concept Name = constraint-expr;" // "concept" is lexed as IDENTIFIER (not in the keyword set), so // use text comparison rather than checkKeyword. if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("concept")) { int startPos = pos; { int _d=0; while(!isAtEnd()){if(checkPunct("{"))_d++;else if(checkPunct("}")){if(_d==0){advance();break;}_d--;}else if(checkPunct(";")&&_d==0)break;advance();} matchPunct(";"); } // Reconstruct the raw text and emit as-is (concept defs are valid at // namespace scope and must be visible before use in template params). StringBuilder raw = new StringBuilder(); for (int i = startPos; i < pos - 1; i++) { if (i > startPos) raw.append(' '); raw.append(tokens.get(i).text()); } raw.append(';'); return List.of(new PreprocessorLine(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments)); } List templateParams = List.of(); // extern "C" / extern "C++" linkage specification -- consume verbatim if (checkKeyword("extern") && pos + 1 < tokens.size() && tokens.get(pos + 1).type() == CppLexerTokenType.STRING_LITERAL) { advance(); advance(); // consume extern "C++" if (checkPunct("{")) { // extern "C++" { ... } block -- parse contents as top-level items advance(); // consume { List result = new ArrayList<>(); while (!isAtEnd() && !checkPunct("}")) { List innerComments = consumeLeadingComments(); if (checkPunct("}")) break; result.addAll(parseTopLevelItem(innerComments)); } matchPunct("}"); return result; } else { // extern "C++" single-declaration return parseTopLevelItem(leadingComments); } } if (matchKeyword("template")) { // Explicit template instantiation: "template class Foo;" (no < after template) if (!checkOp("<")) { int startPos = pos - 1; while (!isAtEnd() && !checkPunct(";")) advance(); matchPunct(";"); StringBuilder raw = new StringBuilder(); for (int i = startPos; i < pos - 1; i++) { if (i > startPos) raw.append(" "); raw.append(tokens.get(i).text()); } raw.append(";"); return List.of(new PreprocessorLine(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments)); } templateParams = parseTemplateParamList(); consumeLeadingComments(); // C++20 concept definition: "template concept Foo = ..." if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("concept")) { int startPos = pos - templateParams.size() - 3; // approximate; use simpler reconstruction // Build the full concept declaration text StringBuilder raw = new StringBuilder("template<"); for (int i = 0; i < templateParams.size(); i++) { if (i > 0) raw.append(", "); raw.append(templateParams.get(i)); } raw.append("> "); // Capture from "concept" through ";" int conceptStart = pos; { int _d=0; while(!isAtEnd()){if(checkPunct("{"))_d++;else if(checkPunct("}")){if(_d==0){advance();break;}_d--;}else if(checkPunct(";")&&_d==0)break;advance();} } matchPunct(";"); for (int i = conceptStart; i < pos - 1; i++) { if (i > conceptStart) raw.append(' '); raw.append(tokens.get(i).text()); } raw.append(';'); return List.of(new PreprocessorLine(raw.toString(), tokens.get(conceptStart).line(), tokens.get(conceptStart).col(), leadingComments)); } // C++20 "requires" clause after template params. // "requires" is lexed as IDENTIFIER (not in the keyword set). if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("requires")) { advance(); // 'requires' int depth = 0; while (!isAtEnd()) { if (checkPunct("(") || checkPunct("[")) { depth++; advance(); continue; } if (checkPunct(")") || checkPunct("]")) { depth--; advance(); continue; } if (checkOp("<")) { depth++; advance(); continue; } if (checkOp(">")) { depth--; advance(); continue; } if (checkOp(">>")) { depth -= 2; advance(); continue; } if (checkPunct(";")) break; if (checkPunct("{")) { // Braced block in requires: consume it entirely int _bd = 0; while (!isAtEnd()) { if (checkPunct("{")) _bd++; else if (checkPunct("}")) { _bd--; if (_bd == 0) { advance(); break; } } advance(); } continue; } if (depth == 0) { if (checkKeyword("void") || checkKeyword("auto") || checkKeyword("int") || checkKeyword("float") || checkKeyword("bool") || checkKeyword("char") || checkKeyword("double")|| checkKeyword("long") || checkKeyword("class") || checkKeyword("struct") || checkKeyword("const") || checkKeyword("static") || checkKeyword("inline")|| checkKeyword("virtual") || checkKeyword("explicit")|| checkKeyword("friend")) { break; } if (check(CppLexerTokenType.IDENTIFIER) && pos + 1 < tokens.size()) { CppLexerToken next = tokens.get(pos + 1); if (next.type() == CppLexerTokenType.IDENTIFIER) break; } } advance(); } } // C++20 concept used as a constraint directly in the template head: // "template" -- Concept is an identifier, not typename/class. // This is handled in parseTemplateParamName already. } if (checkKeyword("class") || checkKeyword("struct")) { // Partial specialization: "template struct Foo { ... }" // After parsing the class/struct, check for a specialization arg list. // BUT: "struct TypeName funcName(...)" is a function with elaborated return type if (pos + 2 < tokens.size() && tokens.get(pos + 1).type() == CppLexerTokenType.IDENTIFIER && tokens.get(pos + 2).type() == CppLexerTokenType.IDENTIFIER && !tokens.get(pos + 1).isKeyword("class") && !tokens.get(pos + 1).isKeyword("struct")) { return parseFunctionOrVariable(leadingComments, templateParams, true); } return List.of(parseTypeDef(leadingComments, templateParams)); } if (checkKeyword("enum")) { return List.of(parseEnumDecl(leadingComments)); } if (checkKeyword("namespace") || (checkKeyword("inline") && pos + 1 < tokens.size() && tokens.get(pos + 1).isKeyword("namespace"))) { return List.of(parseNamespaceDecl(leadingComments)); } // typedef -- emit verbatim as-is (covers function-pointer typedefs, etc.) if (checkKeyword("typedef")) { int startPos = pos; { int _d=0; while(!isAtEnd()){if(checkPunct("{"))_d++;else if(checkPunct("}")){if(_d==0){advance();break;}_d--;}else if(checkPunct(";")&&_d==0)break;advance();} matchPunct(";"); } StringBuilder raw = new StringBuilder(); for (int i = startPos; i < pos - 1; i++) { if (i > startPos) raw.append(' '); raw.append(tokens.get(i).text()); } raw.append(';'); return List.of(new PreprocessorLine(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments)); } if (checkKeyword("typedef")) { int startPos = pos; { int _d=0; while(!isAtEnd()){if(checkPunct("{"))_d++;else if(checkPunct("}")){if(_d==0){advance();break;}_d--;}else if(checkPunct(";")&&_d==0)break;advance();} } matchPunct(";"); StringBuilder raw = new StringBuilder(); for (int i = startPos; i < pos - 1; i++) { if (i > startPos) raw.append(' '); raw.append(tokens.get(i).text()); } raw.append(';'); return List.of(new PreprocessorLine(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments)); } if (checkKeyword("typedef")) { int startPos = pos; { int _d=0; while(!isAtEnd()){if(checkPunct("{"))_d++;else if(checkPunct("}")){if(_d==0){advance();break;}_d--;}else if(checkPunct(";")&&_d==0)break;advance();} } matchPunct(";"); StringBuilder raw = new StringBuilder(); for (int i = startPos; i < pos - 1; i++) { if (i > startPos) raw.append(' '); raw.append(tokens.get(i).text()); } raw.append(';'); return List.of(new PreprocessorLine(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments)); } // static_assert at top level: emit verbatim as PreprocessorLine if (checkKeyword("static_assert")) { int startPos = pos; advance(); expectPunct("("); int _d=1; while (!isAtEnd() && _d > 0) { if (checkPunct("(")) _d++; else if (checkPunct(")")) _d--; advance(); } matchPunct(";"); StringBuilder raw = new StringBuilder(); for (int i = startPos; i < pos; i++) { if (i > startPos) raw.append(" "); raw.append(tokens.get(i).text()); } if (!raw.toString().endsWith(";")) raw.append(";"); CppLexerToken t0 = tokens.get(startPos); return List.of(new TopLevelStatement( new ExprStatement(new Identifier(raw.toString(), t0.line(), t0.col(), List.of()), t0.line(), t0.col(), List.of()), t0.line(), t0.col(), leadingComments)); } if (checkKeyword("using") && tokens.get(pos + 1).isKeyword("namespace")) { return List.of(parseUsingNamespaceDecl(leadingComments)); } // using T = Type; -- type alias. Consume to ';' and emit as-is. // template using Alias = ...; -- template alias, prepend template params. if (checkKeyword("using")) { int startPos = pos; { int _d=0; while(!isAtEnd()){if(checkPunct("{"))_d++;else if(checkPunct("}")){if(_d==0){advance();break;}_d--;}else if(checkPunct(";")&&_d==0)break;advance();} matchPunct(";"); } StringBuilder raw = new StringBuilder(); // Prepend template params if this is a template alias if (!templateParams.isEmpty()) { raw.append("template<"); for (int ti = 0; ti < templateParams.size(); ti++) { if (ti > 0) raw.append(", "); raw.append(templateParams.get(ti)); } raw.append("> "); } for (int i = startPos; i < pos - 1; i++) { if (i > startPos) raw.append(' '); raw.append(tokens.get(i).text()); } raw.append(';'); return List.of(new PreprocessorLine(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments)); } if (checkOp("~") || (checkKeyword("virtual") && tokens.get(pos + 1).isOp("~"))) { return List.of(parseFunctionOrConstructorOrDestructor(leadingComments, templateParams)); } // Global structured binding: "auto [a, b] = expr;" if (isStructuredBindingStart()) { Statement sb2 = parseStructuredBinding(leadingComments); return List.of(new TopLevelStatement(sb2, sb2.line(), sb2.col(), leadingComments)); } // Deduction guide: "Wrapper(const char*) -> Wrapper<:string>;" if (check(CppLexerTokenType.IDENTIFIER) && pos + 1 < tokens.size() && tokens.get(pos + 1).isPunct("(")) { int scan = pos + 2; int depth = 1; while (scan < tokens.size() && depth > 0) { if (tokens.get(scan).isPunct("(")) depth++; else if (tokens.get(scan).isPunct(")")) depth--; scan++; } if (scan < tokens.size() && tokens.get(scan).isPunct("->")) { int startPos = pos; while (!isAtEnd() && !checkPunct(";")) advance(); matchPunct(";"); // Reconstruct: include template params prefix if present StringBuilder raw = new StringBuilder(); if (!templateParams.isEmpty()) { raw.append("template<"); for (int i = 0; i < templateParams.size(); i++) { if (i > 0) raw.append(", "); raw.append(templateParams.get(i)); } raw.append("> "); } for (int i = startPos; i < pos - 1; i++) { if (i > startPos) raw.append(" "); raw.append(tokens.get(i).text()); } raw.append(";"); return List.of(new PreprocessorLine(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments)); } } return parseFunctionOrVariable(leadingComments, templateParams, true); } /** "template" -- returns just the param names. */ private List parseTemplateParamList() { // "template" already consumed by the caller; just consume "<...>" if (checkKeyword("template")) advance(); // in case called with template still present expectOp("<"); List params = new ArrayList<>(); // template<> is a valid explicit full specialization marker -- emit as empty list if (!checkOp(">")) { params.add(parseTemplateParamText()); while (matchPunct(",")) { params.add(parseTemplateParamText()); } } if (checkOp(">>")) { splitTrailingShiftIntoTwoCloseAngles(); } expectOp(">"); return params; } /** * Parses one template parameter and returns its FULL TEXT as a String, * preserving everything needed for CodeGen to emit it verbatim: * "typename T" * "typename... Args" * "class T" * "template class Container" * "auto V" * "int N = sizeof(T)" * "typename std::enable_if<:is_integral>::value, int>::type = 0" * "Numeric T" (concept-constrained) * * The AST stores List for template params. Previously it stored only * the param name, which lost all type/variadic/default information. Now it * stores the full text so CodeGen can round-trip it as "template". */ private String parseTemplateParamText() { int startPos = pos; // We'll capture tokens until we hit ',' or '>' or '>>' at depth 0. // Complex cases (template-template, SFINAE) are handled by parseTemplateParamName // for semantic purposes, but here we just need the text. parseTemplateParamName(); // advance pos past the parameter int endPos = pos; // Reconstruct the text from the tokens we consumed StringBuilder sb = new StringBuilder(); for (int i = startPos; i < endPos; i++) { if (i > startPos) { CppLexerToken prev = tokens.get(i - 1); CppLexerToken cur = tokens.get(i); // Add spacing between tokens where needed if (needsSpace(prev, cur)) sb.append(' '); } sb.append(tokens.get(i).text()); } return sb.toString(); } /** Whether two adjacent tokens need a space between them in output. */ private boolean needsSpace(CppLexerToken prev, CppLexerToken cur) { String p = prev.text(), c = cur.text(); // No space needed before/after punctuation pairs if (p.equals("<") || p.equals(">") || p.equals(">>") || p.equals("*") || p.equals("&") || p.equals("&&") || p.equals("::")) return false; if (c.equals(">") || c.equals(">>") || c.equals("<") || c.equals("::") || c.equals("(") || c.equals(")") || c.equals(",")) return false; // Space between two word tokens (keywords, identifiers) if ((prev.type() == CppLexerTokenType.KEYWORD || prev.type() == CppLexerTokenType.IDENTIFIER) && (cur.type() == CppLexerTokenType.KEYWORD || cur.type() == CppLexerTokenType.IDENTIFIER)) return true; // Space before/after ellipsis in most contexts if (p.equals("...") || c.equals("...")) return true; // Space after "=" for defaults if (p.equals("=") || c.equals("=")) return true; return false; } /** * Parses one template parameter in any of the C++ forms: * * typename T -- type param * class T -- type param (alternate spelling) * typename... Args -- variadic type param pack * class... Args -- variadic type param pack * template class C -- template-template param * template class... C -- variadic template-template param * int N -- non-type param * auto V -- non-type param (C++17 auto NTTP) * typename T = Default -- type param with default * int N = 42 -- non-type param with default * typename std::enable_if<...>::type = 0 -- SFINAE sink param * Concept T -- constrained type param (C++20) * * Returns just the name string (or a descriptive placeholder for complex * params the AST doesn't fully model -- consistent with existing practice * where template params are stored as strings, not as typed AST nodes). */ private String parseTemplateParamName() { // --- template-template parameter: "template<...> class Name" --- if (checkKeyword("template")) { parseTemplateParamList(); // consume the nested <...> (recursive) matchKeyword("class"); matchKeyword("typename"); // variadic template-template: "template class... C" if (checkPunct("...")) advance(); if (check(CppLexerTokenType.IDENTIFIER)) return advance().text(); return ""; } // --- concept-constrained type parameter: "Numeric T" --- if (check(CppLexerTokenType.IDENTIFIER) && pos + 1 < tokens.size() && tokens.get(pos + 1).type() == CppLexerTokenType.IDENTIFIER && pos + 2 < tokens.size() && (tokens.get(pos + 2).isOp(">") || tokens.get(pos + 2).isOp(">>") || tokens.get(pos + 2).isPunct(",") || tokens.get(pos + 2).isPunct("...") || tokens.get(pos + 2).isOp("="))) { advance(); if (checkPunct("...")) advance(); String name = advance().text(); if (matchOp("=")) parseTemplateDefaultValue(); return name; } // --- typename / class type parameter --- if (checkKeyword("typename") || checkKeyword("class")) { advance(); // consume typename/class // variadic pack: "typename... Args" if (checkPunct("...")) advance(); // SFINAE sink: "typename std::enable_if::type = 0" // After typename, if what follows is a qualified name (has :: or // template args), it's a non-type SFINAE param, not a plain // type-param name. Parse it as a full type and grab any default. if (check(CppLexerTokenType.IDENTIFIER) || check(CppLexerTokenType.KEYWORD)) { boolean isSfinaeType = checkPunct("::") || (pos + 1 < tokens.size() && ( tokens.get(pos + 1).isPunct("::") || tokens.get(pos + 1).isOp("<"))); if (isSfinaeType) { try { parseTypeRef(); } catch (ParseException ignored) {} if (matchOp("=")) parseTemplateDefaultValue(); return ""; } String name = advance().text(); if (matchOp("=")) parseTemplateDefaultValue(); return name; } // Anonymous typename with default: "typename = std::enable_if" if (matchOp("=")) parseTemplateDefaultValue(); return ""; } // --- auto NTTP: "auto V" or "auto... Vs" --- if (checkKeyword("auto")) { advance(); if (checkPunct("...")) advance(); String name = check(CppLexerTokenType.IDENTIFIER) ? advance().text() : ""; if (matchOp("=")) parseTemplateDefaultValue(); return name; } // --- Non-type parameter: "int N", "std::size_t N", "const char* S", // or member-pointer NTTP: "int T::* Field" TypeRef type = parseTypeRef(); if (checkPunct("...")) advance(); String name; if (check(CppLexerTokenType.IDENTIFIER)) { name = advance().text(); // Member pointer NTTP: "int T::* Field" -- consume ::* and actual name if (checkPunct("::") && pos + 1 < tokens.size() && tokens.get(pos + 1).isOp("*")) { advance(); advance(); // :: and * if (check(CppLexerTokenType.IDENTIFIER)) name = advance().text(); } } else if (checkPunct("(")) { // Member function pointer NTTP: "int (T::* Method)(int) const" // or pointer-to-array NTTP: "int (*Arr)[N]" advance(); // consume ( // consume optional Class:: qualifier if (check(CppLexerTokenType.IDENTIFIER) && pos + 1 < tokens.size() && tokens.get(pos + 1).isPunct("::")) { advance(); advance(); // Class :: } matchOp("*"); matchOp("&"); // * or & name = check(CppLexerTokenType.IDENTIFIER) ? advance().text() : ""; expectPunct(")"); // consume trailing (params) and optional const if (checkPunct("(")) { advance(); int d = 1; while (!isAtEnd() && d > 0) { if (checkPunct("(")) d++; else if (checkPunct(")")) d--; advance(); } matchKeyword("const"); } // consume trailing [dims] for pointer-to-array while (checkPunct("[")) { advance(); if (!checkPunct("]")) parseExpr(); expectPunct("]"); } } else { name = type instanceof NamedType nt ? nt.baseName() : ""; } // optional default value: "int N = 42" or "::type = 0" if (matchOp("=")) parseTemplateDefaultValue(); return name; } /** * Parses a class/struct declaration: "class Name [: public Base1, ...] { members };". * "public"/"private"/"protected" access specifiers within the body are * consumed and discarded (not modeled in the AST -- not yet confirmed * necessary by any semantic pass; if member visibility ever matters, * this is the place to start tracking it). */ private TypeDef parseTypeDef(List leadingComments, List templateParams) { CppLexerToken start = peek(); String kind = checkKeyword("class") ? "class" : "struct"; advance(); // alignas specifier: "struct alignas(64) CacheLine" if (checkKeyword("alignas") && pos + 1 < tokens.size() && tokens.get(pos + 1).isPunct("(")) { advance(); advance(); int _ad=1; while (!isAtEnd() && _ad > 0) { if (checkPunct("(")) _ad++; else if (checkPunct(")")) _ad--; advance(); } } // Also consume [[attributes]] before the name consumeAttributes(); String name = expectIdentifier().text(); // Partial or explicit specialization: "template struct Foo" // or "template<> struct TypeName" -- capture the specialization arg // text and fold it into the name so CodeGen emits "TypeName" correctly. if (checkOp("<")) { // Specialization args: "Foo" -- scan to matching > manually int argStart = pos; advance(); // consume < int depth = 1; while (!isAtEnd() && depth > 0) { if (checkOp("<")) { depth++; advance(); } else if (checkOp(">")) { depth--; advance(); } else if (checkOp(">>")) { depth -= 2; splitTrailingShiftIntoTwoCloseAngles(); advance(); // consume first > if (depth <= 0) { advance(); break; } // consume second > and exit advance(); // consume second > when still inside } else advance(); } int argEnd = pos; // Reconstruct the specialization suffix verbatim from tokens StringBuilder spec = new StringBuilder(); for (int i = argStart; i < argEnd; i++) spec.append(tokens.get(i).text()); name = name + spec.toString(); } // Forward declaration: "struct Node;" or "class Container;" with no body. // Also handles "template class Container;" with default args. if (checkPunct(";")) { advance(); return new TypeDef(kind, name, templateParams, List.of(), List.of(), start.line(), start.col(), leadingComments); } List baseClasses = new ArrayList<>(); if (matchPunct(":")) { baseClasses.add(parseBaseClassEntry()); while (matchPunct(",")) { baseClasses.add(parseBaseClassEntry()); } } expectPunct("{"); List members = new ArrayList<>(); while (!checkPunct("}")) { if (isAtEnd()) throw error("unexpected end of input inside class/struct body for '" + name + "'"); List comments = consumeLeadingComments(); if (checkPunct("}")) break; if (matchKeyword("public") || matchKeyword("private") || matchKeyword("protected")) { expectPunct(":"); continue; } members.addAll(parseClassMember(comments, name)); } expectPunct("}"); // The trailing ';' after a class/struct body is standard C++, but // confirmed OPTIONAL in real .pde input -- Scrollbar.pde's // "class HScrollbar { ... }" has no trailing ';' at all. Processing's // own preprocessing tolerates this; this parser does too rather than // hard-requiring strict C++ grammar here. matchPunct(";"); return new TypeDef(kind, name, templateParams, baseClasses, members, start.line(), start.col(), leadingComments); } /** "public BaseName" / "private BaseName" / "protected BaseName" / bare "BaseName". */ private String parseBaseClassEntry() { boolean isVirtualBase = matchKeyword("virtual"); matchKeyword("public"); if (!checkKeyword("public")) { matchKeyword("private"); matchKeyword("protected"); } if (!isVirtualBase) isVirtualBase = matchKeyword("virtual"); String name = parseQualifiedTypeName(); // Consume template args on base class name: "Base" (CRTP), // "std::enable_shared_from_this", etc. if (checkOp("<") && looksLikeTemplateArgList()) { StringBuilder sb = new StringBuilder(name); sb.append('<'); advance(); // '<' int depth = 1; while (!isAtEnd() && depth > 0) { if (checkOp("<")) depth++; else if (checkOp(">")) { depth--; if (depth == 0) break; } else if (checkOp(">>")) { depth -= 2; if (depth <= 0) { splitTrailingShiftIntoTwoCloseAngles(); break; } } sb.append(peek().text()); advance(); } expectOp(">"); sb.append('>'); name = sb.toString(); } boolean isPackExpansion = matchPunct("..."); // pack expansion in base class list: "Bases..." String suffix = isPackExpansion ? "..." : ""; return isVirtualBase ? "virtual " + name + suffix : name + suffix; } /** * A class member is almost always parseTopLevelItem's domain (field decl, * method, nested type) -- but constructors/destructors named after the * enclosing class need special handling parseTopLevelItem's generic * function/variable dispatch can't do alone (no return type at all, * possible initializer list), so this wraps that in enclosing-class * context. */ private List parseClassMember(List leadingComments, String enclosingClassName) { if (check(CppLexerTokenType.PREPROCESSOR_DIRECTIVE)) { CppLexerToken t = advance(); return List.of(new PreprocessorLine(t.text(), t.line(), t.col(), leadingComments)); } consumeAttributes(); // friend declarations: "friend class Foo;" or "friend void func(...);" // Consume entirely -- friend declarations don't produce AST members we need. if (checkKeyword("friend")) { advance(); // consume 'friend' // Friend function with a body: "friend T operator*(...) { ... }" // Must be parsed as a real function, not consumed to ';'. // Detect by scanning ahead: if we find '{' before ';', it has a body. int scan = pos; int depth = 0; boolean hasBody = false; while (scan < tokens.size()) { String txt = tokens.get(scan).text(); if (txt.equals("{")) { if (depth++ == 0) { hasBody = true; break; } } else if (txt.equals("}")) { if (--depth == 0) break; } else if (txt.equals(";") && depth == 0) break; scan++; } if (hasBody) { // Friend function WITH a body is a free function defined in-class // but semantically at namespace scope. Return it as a TopLevelItem // to be hoisted out of the class body. // We must NOT emit it as a class member since free functions with 2+ // parameters can't be member functions. List items = parseFunctionOrVariable(leadingComments, List.of(), false); return items; // will be added to class members and hoisted by ClassHoister } // Friend declaration (no body) -- consume to ';' { int _d=0; while(!isAtEnd()){if(checkPunct("{"))_d++;else if(checkPunct("}")){if(_d==0){advance();break;}_d--;}else if(checkPunct(";")&&_d==0)break;advance();} matchPunct(";"); } return List.of(); } List templateParams = List.of(); if (checkKeyword("template")) { templateParams = parseTemplateParamList(); consumeLeadingComments(); // Trailing requires clause on template member: "template\n requires Concept" if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("requires")) { advance(); // consume requires if (checkPunct("(")) { advance(); int _d=1; while(!isAtEnd()&&_d>0){if(checkPunct("("))_d++;else if(checkPunct(")"))_d--;advance();} } else { int _d=0; while(!isAtEnd()){if(checkOp("<"))_d++;else if(checkOp(">")&&_d>0)_d--;else if(_d==0&&(checkPunct("{;")||checkPunct(";")|| checkKeyword("auto")||checkKeyword("void")||checkKeyword("bool")||checkKeyword("int")||checkKeyword("float")||checkKeyword("const")||checkKeyword("inline")||checkKeyword("static")||checkKeyword("virtual")||checkKeyword("constexpr")||checkKeyword("explicit")||checkKeyword("operator")))break;advance();} } consumeLeadingComments(); } } if (checkKeyword("class") || checkKeyword("struct")) { // Anonymous struct: "struct { float x, y; } position;" if (pos + 1 < tokens.size() && tokens.get(pos + 1).isPunct("{")) { int anonStart = pos; advance(); // consume struct/class int bd = 0; while (!isAtEnd()) { if (checkPunct("{")) { bd++; advance(); } else if (checkPunct("}")) { bd--; advance(); if (bd == 0) break; } else advance(); } // pos is now after }, next is memberName then ; int beforeMember = pos; String memberName = check(CppLexerTokenType.IDENTIFIER) ? advance().text() : ""; matchPunct(";"); StringBuilder raw = new StringBuilder(); for (int i = anonStart; i < beforeMember; i++) { if (i > anonStart) raw.append(" "); raw.append(tokens.get(i).text()); } if (!memberName.isEmpty()) raw.append(" ").append(memberName); raw.append(";"); return List.of(new PreprocessorLine(raw.toString(), tokens.get(anonStart).line(), tokens.get(anonStart).col(), leadingComments)); } // "struct/class Name funcName(" -- elaborated return type, not a definition if (pos + 2 < tokens.size() && tokens.get(pos + 1).type() == CppLexerTokenType.IDENTIFIER && tokens.get(pos + 2).type() == CppLexerTokenType.IDENTIFIER) { return parseFunctionOrVariable(leadingComments, templateParams, false); } return List.of(parseTypeDef(leadingComments, templateParams)); } if (checkKeyword("enum")) { return List.of(parseEnumDecl(leadingComments)); } // static_assert inside class body if (checkKeyword("static_assert")) { int startPos = pos; advance(); expectPunct("("); int _d=1; while (!isAtEnd() && _d > 0) { if (checkPunct("(")) _d++; else if (checkPunct(")")) _d--; advance(); } matchPunct(";"); return List.of(new PreprocessorLine("// static_assert", tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments)); } // using X = Type; inside a struct/class body -- consume verbatim if (checkKeyword("using") && !tokens.get(pos + 1).isKeyword("namespace")) { int startPos = pos; { int _d=0; while(!isAtEnd()){if(checkPunct("{"))_d++;else if(checkPunct("}")){if(_d==0){advance();break;}_d--;}else if(checkPunct(";")&&_d==0)break;advance();} matchPunct(";"); } StringBuilder raw = new StringBuilder(); for (int i = startPos; i < pos - 1; i++) { if (i > startPos) raw.append(" "); raw.append(tokens.get(i).text()); } raw.append(";"); return List.of(new PreprocessorLine(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments)); } // Destructor dispatch: "~Name() {...}" or "virtual ~Name() {...}" -- // must look one token past an optional leading "virtual", since that // keyword (confirmed common on destructors, e.g. "virtual ~LSystem()") // precedes the '~' that would otherwise be the dispatch signal. if (checkOp("~") || (checkKeyword("virtual") && tokens.get(pos + 1).isOp("~"))) { return List.of(parseFunctionOrConstructorOrDestructor(leadingComments, templateParams)); } // Constructor dispatch: identifier matching the enclosing class name, // directly followed by '(' with no return type preceding it. // Also handle "constexpr ClassName(...)" -- consume constexpr first. // friend declaration: "friend class Foo;" or "friend Box makeBox(U v);" if (checkKeyword("friend")) { int startPos = pos; advance(); // consume friend // "friend class/struct Foo" -- forward decl, consume verbatim if (checkKeyword("class") || checkKeyword("struct")) { while (!isAtEnd() && !checkPunct(";")) advance(); matchPunct(";"); return List.of(new PreprocessorLine("// friend class", tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments)); } // "friend RetType funcName(...)" or "friend RetType operator...()" -- parse as function // Consume template params if present; also use outer templateParams if already consumed List friendTemplateParams = templateParams.isEmpty() ? List.of() : templateParams; if (matchKeyword("template")) friendTemplateParams = parseTemplateParamList(); return parseFunctionOrVariable(leadingComments, friendTemplateParams, false); } // explicit(...): conditional explicit specifier (C++20) if (checkKeyword("explicit") && pos + 1 < tokens.size() && tokens.get(pos + 1).isPunct("(")) { advance(); // consume explicit advance(); int _ed=1; // consume ( while (!isAtEnd() && _ed > 0) { if (checkPunct("(")) _ed++; else if (checkPunct(")")) _ed--; advance(); } } else { matchKeyword("explicit"); // plain explicit } boolean isConstexprCtor = false; if ((checkKeyword("constexpr") || checkKeyword("consteval")) && pos + 1 < tokens.size() && tokens.get(pos + 1).type() == CppLexerTokenType.IDENTIFIER && tokens.get(pos + 1).text().equals(enclosingClassName) && pos + 2 < tokens.size() && tokens.get(pos + 2).isPunct("(")) { advance(); // consume constexpr/consteval isConstexprCtor = true; } // Constructor: match enclosingClassName OR just its base name (for specializations like Grid) String enclosingBase = enclosingClassName.contains("<") ? enclosingClassName.substring(0, enclosingClassName.indexOf("<")) : enclosingClassName; if (check(CppLexerTokenType.IDENTIFIER) && (peek().text().equals(enclosingClassName) || peek().text().equals(enclosingBase)) && tokens.get(pos + 1).isPunct("(")) { FunctionDecl ctorFd = parseFunctionOrConstructorOrDestructor(leadingComments, templateParams); if (isConstexprCtor && !ctorFd.isConstexpr()) { // constexpr was consumed before dispatch; rebuild FunctionDecl with isConstexpr=true ctorFd = new FunctionDecl(ctorFd.returnType(), ctorFd.name(), ctorFd.templateParams(), ctorFd.params(), ctorFd.initializerList(), ctorFd.body(), ctorFd.isConstructor(), ctorFd.isDestructor(), ctorFd.isVirtual(), ctorFd.isOverride(), ctorFd.isConst(), true, ctorFd.isStatic(), ctorFd.isPureVirtual(), ctorFd.isDefault(), ctorFd.isDelete(), ctorFd.line(), ctorFd.col(), ctorFd.leadingComments()); } return List.of(ctorFd); } return parseFunctionOrVariable(leadingComments, templateParams, false); } /** * Parses a destructor ("~Name() { ... }") or a constructor (handled here * too since both share the "no return type, name/~name immediately * followed by '('" shape, including the optional virtual/override/const * qualifiers and, for constructors, an initializer list). */ private FunctionDecl parseFunctionOrConstructorOrDestructor(List leadingComments, List templateParams) { CppLexerToken start = peek(); boolean isVirtual = matchKeyword("virtual"); boolean isConstexprCtorMethod = matchKeyword("constexpr") || matchKeyword("consteval"); boolean isDestructor = matchOp("~"); String name = expectIdentifier().text(); if (isDestructor) name = "~" + name; List params = parseParamList(); boolean isConst = matchKeyword("const"); boolean isOverride = matchKeyword("override"); // order-flexible: also accept override before const, just in case if (!isConst) isConst = matchKeyword("const"); // Consume trailing qualifiers: noexcept, noexcept(expr), override, final, requires while (true) { if (checkKeyword("noexcept")) { advance(); if (checkPunct("(")) { advance(); int d=1; while(!isAtEnd()&&d>0){if(checkPunct("("))d++;else if(checkPunct(")"))d--;advance();} } } else if (checkKeyword("override")) { advance(); } else if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("final")) { advance(); } else if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("requires")) { advance(); if (checkPunct("(")) { advance(); int d=1; while(!isAtEnd()&&d>0){if(checkPunct("("))d++;else if(checkPunct(")"))d--;advance();} } else { while (!isAtEnd() && !checkPunct("{") && !checkPunct(";") && !checkOp("=")) { if (checkOp("<")) { advance(); int d=1; while(!isAtEnd()&&d>0){if(checkOp("<"))d++;else if(checkOp(">"))d--;advance();} } else advance(); } } } else break; } List initList = new ArrayList<>(); if (!isDestructor && matchPunct(":")) { FunctionDecl.ConstructorInit ci1 = parseConstructorInitEntry(); if (matchPunct("...")) ci1 = new FunctionDecl.ConstructorInit(ci1.memberName() + "...", ci1.args()); initList.add(ci1); while (matchPunct(",")) { FunctionDecl.ConstructorInit ci = parseConstructorInitEntry(); if (matchPunct("...")) ci = new FunctionDecl.ConstructorInit(ci.memberName() + "...", ci.args()); initList.add(ci); } } // Pure-virtual specifier ("= 0"), e.g. "virtual ~A() = 0;" -- a // real, valid C++ idiom for ensuring polymorphic deletion through // an abstract base. Found via a realistic builder-pattern sketch // using "virtual void draw() = 0;" on an abstract base class -- // confirmed real, ordinary OOP, not exotic. Must be checked // BEFORE the body/";" check below, since "= 0;" replaces both. boolean isPureVirtual = false, isDefault = false, isDelete = false; if (checkOp("=")) { int save = pos; advance(); if (checkKeyword("default")) { advance(); isDefault = true; } else if (checkKeyword("delete")) { advance(); isDelete = true; } else if (checkLiteralZero()) { advance(); isPureVirtual = true; } else { pos = save; } } Block body = checkPunct("{") ? parseBlock() : null; if (body == null && !isPureVirtual && !isDefault && !isDelete) expectPunct(";"); else if (isPureVirtual || isDefault || isDelete) expectPunct(";"); return new FunctionDecl(null, name, templateParams, params, initList, body, !isDestructor, isDestructor, isVirtual, isOverride, isConst, false, isConstexprCtorMethod, isPureVirtual, isDefault, isDelete, start.line(), start.col(), leadingComments); } private FunctionDecl.ConstructorInit parseConstructorInitEntry() { // Base class names can be qualified: "std::runtime_error(msg)" String memberName = parseQualifiedTypeName(); List args = new ArrayList<>(); if (checkPunct("{")) { // Brace initialization: "m{{1,0},{0,1}}" args.add(parseInitializerList()); } else { expectPunct("("); if (!checkPunct(")")) { Expr a0 = parseExpr(); if (matchPunct("...")) a0 = new PostfixExpr("...", a0, a0.line(), a0.col(), List.of()); args.add(a0); while (matchPunct(",")) { if (checkPunct(")")) break; Expr ai = parseExpr(); if (matchPunct("...")) ai = new PostfixExpr("...", ai, ai.line(), ai.col(), List.of()); args.add(ai); } } expectPunct(")"); } return new FunctionDecl.ConstructorInit(memberName, args); } /** * Handles every remaining top-level/class-member form: an ordinary * function declaration/definition, an operator overload, or a variable * declaration (possibly multi-declarator, desugared into multiple * VariableDecl nodes sharing one TypeRef). * * Disambiguation: parse [virtual] [static] [const-is-not-valid-here] * returnType, then a name (ordinary identifier OR "operator" followed by * an operator token, per the operator-overload fixture). If '(' follows * the name, it's a function; otherwise it's a variable declaration * (with possible comma-separated additional declarators). * * @param isTopLevel true when called from true file scope * (parseTopLevelItem), false when called from inside * a class/struct body (parseClassMember). Gates the * bare-statement (Processing static-mode) fallback, * which only makes sense at file scope -- a class * member is always a declaration, never a bare * statement. Without this gate, the fallback was * reachable from inside class bodies too, which * caused a real bug: "bool operator==(...) const {...}" * as a class member was misjudged as "not a * declaration" by the fallback's lookahead and * incorrectly routed into statement parsing instead * (found via the oop_features.cpp fixture). */ private List parseFunctionOrVariable(List leadingComments, List templateParams, boolean isTopLevel) { CppLexerToken start = peek(); // Bare top-level statement fallback (Processing "static mode" sketches, // confirmed real by the example corpus's Coordinates.pde -- a flat // sequence of statements like "size(640, 360);" with no enclosing // setup()/draw() at all). Distinguished from a real declaration by // lookahead: a declaration is "TypeName Identifier ...", whereas a // bare call statement is "identifier(" with nothing in between. This // check must run before any of virtual/static/const are consumed, // since none of those prefix a bare statement. Only ever applies at // true top level -- see isTopLevel notes above. if (isTopLevel && !looksLikeTopLevelDeclarationOrFunction()) { Statement stmt = parseStatement(List.of()); return List.of(new TopLevelStatement(stmt, start.line(), start.col(), leadingComments)); } boolean isVirtual = matchKeyword("virtual"); boolean isStatic = matchKeyword("static"); matchKeyword("inline"); matchKeyword("volatile"); // consume volatile qualifier // alignas(expr): consume alignment specifier before type if (checkKeyword("alignas") && pos + 1 < tokens.size() && tokens.get(pos + 1).isPunct("(")) { advance(); advance(); int _aad=1; while (!isAtEnd() && _aad > 0) { if (checkPunct("(")) _aad++; else if (checkPunct(")")) _aad--; advance(); } } boolean isConst = matchKeyword("const"); boolean isConstexprFn = matchKeyword("constexpr") || matchKeyword("consteval"); if (isConstexprFn && !isConst) isConst = true; matchKeyword("constinit"); if (!isVirtual) isVirtual = matchKeyword("virtual"); // constexpr virtual if (!isStatic) isStatic = matchKeyword("static"); if (!isConst) isConst = matchKeyword("const"); matchKeyword("volatile"); // volatile may appear after const // Raw function-pointer variable declaration: "Type (*name)(Params) = init;" if (looksLikeFunctionPointerVariable()) { TypeRef returnType = parseTypeRef(); NameAndFunctionPointerType decl = parseFunctionPointerDeclaratorTail(returnType); Expr initializer = null; if (matchOp("=")) { initializer = parseExpr(); } expectPunct(";"); return List.of(new VariableDecl(decl.type(), decl.name(), List.of(), initializer, isConst, isStatic, start.line(), start.col(), leadingComments)); } TypeRef type = parseTypeRef(isConstexprFn ? false : isConst); // constexpr should not mark return type as const // Constructor: if ( follows the type with no name AND we have leading constexpr/virtual, // the type name IS the function name (e.g. "constexpr RGBA(...)") String name; if (checkPunct("(") && type instanceof NamedType nt2 && !looksLikeFunctionPointerDeclarator() && (isConst || isVirtual || isStatic) // only when qualifier precedes && looksLikeParamList()) { name = nt2.baseName(); } else { name = parseFunctionOrVariableName(); } if (checkPunct("(") && !looksLikeFunctionPointerDeclarator() && looksLikeParamList()) { List params = parseParamList(); boolean isOverride = false; boolean isMethodConst = false; // trailing const/override may appear in either order for (int i = 0; i < 3; i++) { if (matchKeyword("const")) { isMethodConst = true; continue; } if (matchKeyword("volatile")) { continue; } // const volatile method if (matchKeyword("override")) { isOverride = true; continue; } } // Trailing return type: "auto f() -> ReturnType" // Capture it and use it as the actual return type (replacing "auto"). TypeRef trailingReturnType = null; if (checkPunct("->")) { advance(); trailingReturnType = parseTypeRef(); } TypeRef effectiveReturnType = (trailingReturnType != null) ? trailingReturnType : type; // Consume __attribute__((...)): GCC attribute syntax before trailing qualifiers if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("__attribute__")) { advance(); // __attribute__ if (checkPunct("(")) { advance(); int _ad=1; while(!isAtEnd()&&_ad>0){if(checkPunct("("))_ad++;else if(checkPunct(")"))_ad--;advance();} } } // Consume trailing qualifiers: noexcept, noexcept(expr), override, final, requires, &/&& while (true) { if (checkOp("&") || checkOp("&&")) { // Ref-qualifier on member function: "T f() &" or "T f() &&" // Consume and discard -- not represented in the AST currently. advance(); } else if (checkKeyword("noexcept")) { advance(); if (checkPunct("(")) { advance(); int d=1; while(!isAtEnd()&&d>0){if(checkPunct("("))d++;else if(checkPunct(")"))d--;advance();} } } else if (checkKeyword("override")) { isOverride = true; advance(); } else if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("final")) { advance(); } else if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("requires")) { // trailing requires clause: "requires expr" -- consume to { or ; advance(); if (checkPunct("(")) { advance(); int d=1; while(!isAtEnd()&&d>0){if(checkPunct("("))d++;else if(checkPunct(")"))d--;advance();} } else if (checkPunct("{")) { // requires { expr } body -- consume the whole braced block advance(); int d=1; while (!isAtEnd() && d > 0) { if (checkPunct("{")) d++; else if (checkPunct("}")) { if (--d == 0) { advance(); break; } } advance(); } } else { // bare expression like "requires std::is_arithmetic_v" // Also handle "requires requires { ... }" (nested) if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("requires")) { advance(); // consume inner requires } if (checkPunct("{")) { advance(); int d=1; while (!isAtEnd() && d > 0) { if (checkPunct("{")) d++; else if (checkPunct("}")) { if (--d == 0) { advance(); break; } } advance(); } } else { while (!isAtEnd() && !checkPunct("{") && !checkPunct(";") && !checkOp("=")) { if (checkOp("<")) { advance(); int d=1; while(!isAtEnd()&&d>0){if(checkOp("<"))d++;else if(checkOp(">"))d--;advance();} } else advance(); } } } } else break; } // Pure-virtual specifier ("= 0") or = default / = delete boolean isPureVirtual = false, isDefault = false, isDelete = false; if (checkOp("=")) { int save = pos; advance(); if (checkLiteralZero()) { advance(); isPureVirtual = true; } else if (checkKeyword("default")) { advance(); isDefault = true; } else if (checkKeyword("delete")) { advance(); isDelete = true; } else { pos = save; } } // Constructor initializer list: ": mem(val), mem2(val2)" if (matchPunct(":") && !checkPunct(":")) { // consume initializer list entries until { or ; int _d = 0; while (!isAtEnd()) { if (checkPunct("{") && _d == 0) break; if (checkPunct("(") || checkPunct("{")) _d++; else if (checkPunct(")") || checkPunct("}")) _d--; advance(); } } Block body = checkPunct("{") ? parseBlock() : null; if (body == null) expectPunct(";"); FunctionDecl fn = new FunctionDecl(effectiveReturnType, name, templateParams, params, List.of(), body, false, false, isVirtual, isOverride, isMethodConst, isConstexprFn, isStatic, isPureVirtual, isDefault, isDelete, start.line(), start.col(), leadingComments); return List.of(fn); } // Variable declaration, possibly multi-declarator. // Bit field: "unsigned int active : 1" or unnamed "unsigned int : 2" // Consume ": width" specifier. Width may be a literal or constexpr identifier. if (checkPunct(":") && pos + 1 < tokens.size() && (tokens.get(pos + 1).type() == CppLexerTokenType.INT_LITERAL || tokens.get(pos + 1).type() == CppLexerTokenType.IDENTIFIER)) { advance(); // consume ":" parseExpr(); // consume width expression } // GCC __attribute__((...)): may appear after variable name if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("__attribute__")) { advance(); if (checkPunct("(")) { advance(); int _ad=1; while(!isAtEnd()&&_ad>0){if(checkPunct("("))_ad++;else if(checkPunct(")"))_ad--;advance();} } } List result = new ArrayList<>(); result.add(parseOneTopLevelDeclarator(type, name, isConst, isStatic, templateParams, start, leadingComments)); while (matchPunct(",")) { // Each declarator after the first comma can carry its OWN // leading "*"/"&" markers, independent of every other // declarator on the line -- real, standard C++ syntax // ("int* a, b;" -- only a is a pointer; "int* a, *b;" -- // both are pointers, the "*" repeated per declarator). // Confirmed real and necessary by Scrollbar.pde's own // corrected declaration ("HScrollbar* hs1, * hs2;"). // // IMPORTANT: build each declarator's type from the base // type's NAME/TEMPLATE-ARGS/CONST ONLY -- never reuse // "type"'s own pointerDepth/isReference, which already // belongs EXCLUSIVELY to the first declarator (parseTypeRef // consumes a "*"/"&" immediately after the base type name, // before the first declarator's name is even parsed). A // first attempt at this fix added each subsequent // declarator's marker count ON TOP of the shared type's // existing pointerDepth, which produced "HScrollbar**" for // the second declarator in "HScrollbar* hs1, * hs2;" // instead of the correct "HScrollbar*" -- caught by // checking the RENDERED OUTPUT, not just parse success; // "does this parse" and "is the resulting AST actually // correct" are different questions, the same lesson from // the return-statement misparse bug found much earlier in // this project. int extraPointerDepth = 0; boolean extraIsReference = false; while (checkOp("*") || checkOp("&")) { if (matchOp("*")) extraPointerDepth++; else { matchOp("&"); extraIsReference = true; } } String nextName = expectIdentifier().text(); TypeRef declaratorType = type; if (type instanceof NamedType nt) { declaratorType = new NamedType(nt.baseName(), nt.templateArgs(), extraPointerDepth, extraIsReference, nt.isConst(), false); } result.add(parseOneTopLevelDeclarator(declaratorType, nextName, isConst, isStatic, List.of(), start, List.of())); } expectPunct(";"); return result; } /** * Lookahead-only: determines whether the upcoming tokens form a real * top-level declaration or function definition (possibly preceded by * virtual/static/const) rather than a bare statement. A declaration has * the shape "[virtual] [static] [const] TypeName Identifier ..."; a bare * statement -- the Processing static-mode case -- starts with an * identifier immediately followed by something that isn't another * identifier (most commonly '(' for a call, or an assignment operator * for a plain assignment to an already-declared global). */ private boolean looksLikeTopLevelDeclarationOrFunction() { int save = pos; try { matchKeyword("virtual"); matchKeyword("static"); matchKeyword("volatile"); matchKeyword("const"); matchKeyword("constexpr"); matchKeyword("consteval"); matchKeyword("constinit"); matchKeyword("inline"); matchKeyword("static"); // tolerate either order, mirroring the real parse path if (looksLikeFunctionPointerVariable()) return true; if (!(check(CppLexerTokenType.IDENTIFIER) || check(CppLexerTokenType.KEYWORD))) return false; TypeRef type; try { type = parseTypeRef(); } catch (ParseException e) { return false; } // "operator==" style names: 'operator' is lexed as a KEYWORD. // Two cases reach here: // (a) "operator" itself was parsed as the bare "type" (when // there's no separate return type before it -- not // actually valid C++ for operator overloads but tolerated // defensively), OR // (b) more commonly, a real return type was already consumed // ("bool", "Handle", etc.) and the CURRENT token is now // "operator" itself, e.g. "bool operator==(...)" -- // confirmed real and previously missed by the // oop_features.cpp fixture's Handle::operator==. // Either way, this is unambiguously a declaration. if ((type instanceof NamedType nt && nt.baseName().equals("operator")) || checkKeyword("operator")) { return true; } // Constructor: type name followed by ( -- "RGBA(...)" inside struct RGBA if (checkPunct("(")) return true; return check(CppLexerTokenType.IDENTIFIER); } finally { pos = save; } } /** * Parses an ordinary name, "operator" followed by an operator/comparison * token (e.g. "operator=="), or a "::"-qualified name for out-of-class * static member definitions (e.g. "Counter::count" in * "int Counter::count = 0;", confirmed real by the kitchen-sink fixture). */ private String parseFunctionOrVariableName() { if (checkKeyword("operator")) { CppLexerToken opKw = advance(); // Special multi-token operator names: // operator[] -- subscript // operator() -- function call // operator new / operator delete // operator== / operator+ / etc. -- single-token symbols if (checkPunct("[")) { advance(); // '[' expectPunct("]"); return opKw.text() + "[]"; } if (checkPunct("(")) { advance(); // '(' expectPunct(")"); return opKw.text() + "()"; } if (checkKeyword("new") || checkKeyword("delete")) { CppLexerToken kw = advance(); String opName = opKw.text() + " " + kw.text(); // operator new[] / operator delete[] if (checkPunct("[")) { advance(); expectPunct("]"); opName += "[]"; } return opName; } // Cast operator: "operator int", "operator float", "operator bool", etc. // ONLY fire for actual C++ primitive type keywords, not arbitrary // keywords that happen to follow "operator" in other contexts. // Using a small explicit set prevents keywords like "color", "auto", // "return", etc. from being misidentified as cast targets. if (check(CppLexerTokenType.KEYWORD) && CAST_OPERATOR_TYPE_KEYWORDS.contains(peek().text())) { CppLexerToken castTok = advance(); return opKw.text() + " " + castTok.text(); } // Also handle identifier type names used as cast targets (e.g. // "operator MyType()" where MyType is a user-defined class). // Only safe when the next token is an identifier (not a keyword), // so we don't misidentify keyword-named things like "color". if (check(CppLexerTokenType.IDENTIFIER)) { CppLexerToken castTok = advance(); return opKw.text() + " " + castTok.text(); } // User-defined literal operator: "operator"" _suffix" // The "" is a STRING_LITERAL token; the suffix is an IDENTIFIER. if (check(CppLexerTokenType.STRING_LITERAL) && peek().text().equals("\"\"")) { advance(); // consume "" // Optional suffix identifier (e.g. _deg, _kb, _v) if (check(CppLexerTokenType.IDENTIFIER) || (check(CppLexerTokenType.KEYWORD) && peek().text().startsWith("_"))) { CppLexerToken suffix = advance(); return opKw.text() + "\"\"" + suffix.text(); } return opKw.text() + "\"\""; } // Ordinary single-token operator, e.g. "==", "+", "<<". // Special case: spaceship operator "<=>" is lexed as "<=" then ">". CppLexerToken opTok = advance(); if (opTok.text().equals("<=") && checkOp(">")) { advance(); // consume the ">" return opKw.text() + "<=>"; } return opKw.text() + opTok.text(); } // Cast-operator case: "operator" was already consumed by parseTypeRef() // as NamedType("operator"), leaving a keyword type like "bool"/"int" as // the current token. "operator bool()" and "operator int()" reach here. // Guard against mistakenly eating real qualifiers, storage specifiers, OR // Processing-specific keywords like "color" that are legitimately used as // method names (e.g. "Builder& color(int r, int g, int b)" in the // builder pattern) -- these must fall through to expectIdentifier() below. if (check(CppLexerTokenType.KEYWORD) && !checkKeyword("const") && !checkKeyword("override") && !checkKeyword("virtual") && !checkKeyword("static") && !checkKeyword("inline") && !checkKeyword("explicit") && !checkKeyword("operator") && !PSEUDO_TYPE_KEYWORDS_USABLE_AS_NAMES.contains(peek().text())) { CppLexerToken castTok = advance(); String suffix = ""; while (checkOp("*") || checkOp("&")) suffix += advance().text(); return "operator " + castTok.text() + suffix; } // Unnamed bitfield: "unsigned int : 2" -- no identifier before ":" // Return empty name; the caller's bitfield-width consumer will handle ":". if (checkPunct(":") && pos + 1 < tokens.size() && tokens.get(pos + 1).type() == CppLexerTokenType.INT_LITERAL) { return ""; } String name = expectIdentifier().text(); // Member data pointer: "ClassName::* varName" -- e.g. "int Point::* ptr" // After consuming "Point" as the name, if :: follows and then * (not another identifier), // this is a pointer-to-member-data declarator, not a qualified name. if (checkPunct("::") && pos + 1 < tokens.size() && tokens.get(pos + 1).isOp("*")) { advance(); // "::" advance(); // "*" String memberName = expectIdentifier().text(); return name + "::* " + memberName; } while (checkPunct("::")) { advance(); name += "::" + expectIdentifier().text(); } // Variable template specialization: "zero" -- consume if (checkOp("<") && looksLikeTemplateArgList()) { StringBuilder sb = new StringBuilder(name); sb.append("<"); advance(); // consume < int depth = 1; while (!isAtEnd() && depth > 0) { if (checkOp("<")) { depth++; sb.append(advance().text()); } else if (checkOp(">")) { depth--; if (depth > 0) sb.append(advance().text()); else advance(); } else if (checkOp(">>")) { depth -= 2; splitTrailingShiftIntoTwoCloseAngles(); if (depth > 0) sb.append(advance().text()); else advance(); } else sb.append(advance().text()); } sb.append(">"); name = sb.toString(); } return name; } private VariableDecl parseOneTopLevelDeclarator(TypeRef type, String name, boolean isConst, boolean isStatic, List templateParams, CppLexerToken start, List leadingComments) { List dims = parseOptionalArrayDims(); // Bitfield declarator: "unsigned int active : 1" -- consume ": width" and discard. // The ":" is a PUNCT token; must be distinguished from inheritance ":" (which // appears at class scope before a type name) and ternary ":" (inside expressions). // Here we're inside a declarator so ":" followed by INT_LITERAL or IDENTIFIER // is unambiguously a bitfield width specifier. if (checkPunct(":") && pos + 1 < tokens.size() && (tokens.get(pos + 1).type() == CppLexerTokenType.INT_LITERAL || tokens.get(pos + 1).type() == CppLexerTokenType.IDENTIFIER)) { advance(); // consume ":" parseExpr(); // consume width expression (usually a literal, may be constexpr) } Expr initializer = null; if (matchOp("=")) { initializer = checkPunct("{") ? parseInitializerList() : parseExpr(); } else if (checkPunct("(")) { initializer = parseDirectInitAsCall(name); } else if (checkPunct("{")) { // Brace-direct-init: "Type name{args};" -- the unambiguous, // most-vexing-parse-proof sibling of paren-direct-init // ("Type name(args);"). Confirmed necessary by this project's // own CodeGen, which deliberately renders the self-named- // CallExpr direct-init shape using braces rather than parens // (see CodeGen.emitDeclaratorTail's notes on why) -- without // this branch, the parser couldn't read its own generated // output back, which a round-trip test caught immediately. // Represented with the exact same CallExpr shape as the paren // form (callee = an Identifier matching the declarator's own // name), since semantically they mean the same thing and // CodeGen already knows how to render that shape as braces. initializer = parseDirectInitAsCall(name); } return new VariableDecl(type, name, dims, initializer, isConst, isStatic, templateParams, start.line(), start.col(), leadingComments); } /** * Lookahead-only: from the "(" that follows a name (in * parseFunctionOrVariable, deciding between "this is a function decl" * and "this is direct-init construction syntax"), determines whether the * parenthesized content actually looks like a parameter list rather than * a constructor-call argument list. * * Confirmed necessary by Arctangent.pde's "Eye e1(250, 16, 120);" -- * direct-init with literal constructor arguments. A real parameter list * is either empty ("()") or starts with something type-shaped followed * by a parameter name; an argument list starting with a literal, * a unary operator, or anything else that can't start a TypeRef is * unambiguously NOT a parameter list. */ private boolean looksLikeParamList() { int save = pos; try { expectPunct("("); if (checkPunct(")")) return true; if (!(check(CppLexerTokenType.IDENTIFIER) || check(CppLexerTokenType.KEYWORD))) return false; // Skip explicit object param keyword: "this" if (checkKeyword("this")) advance(); if (checkPunct(")")) return true; if (!(check(CppLexerTokenType.IDENTIFIER) || check(CppLexerTokenType.KEYWORD))) return false; try { parseTypeRef(); } catch (ParseException e) { return false; } // Accept: named param, variadic pack "...", anonymous param ")", or // reference/pointer-to-array: "int (&arr)[10]" -- ( follows the type return check(CppLexerTokenType.IDENTIFIER) || checkPunct("...") // variadic: "Args... args" or "T..." || checkPunct(")") // anonymous param: "void f(int)" || checkPunct(",") // next param after anonymous || checkPunct("("); // reference/pointer-to-array param: "int (&arr)[10]" } finally { pos = save; } } private List parseParamList() { expectPunct("("); List params = new ArrayList<>(); if (!checkPunct(")")) { params.add(parseParam()); while (matchPunct(",")) { params.add(parseParam()); } } expectPunct(")"); return params; } /** * Lookahead for the raw function-pointer VARIABLE form specifically * (distinct from looksLikeFunctionPointerDeclarator, which checks * starting from the "(" itself once a return type has already been * consumed) -- this version starts from before the return type, so it * must first skip over a tentative TypeRef. */ private boolean looksLikeFunctionPointerVariable() { int save = pos; try { if (!(check(CppLexerTokenType.IDENTIFIER) || check(CppLexerTokenType.KEYWORD))) return false; try { parseTypeRef(); } catch (ParseException e) { return false; } return looksLikeFunctionPointerDeclarator(); } finally { pos = save; } } private EnumDecl parseEnumDecl(List leadingComments) { CppLexerToken start = expectKeyword("enum"); boolean isScoped = matchKeyword("class"); if (!isScoped) matchKeyword("struct"); // "enum struct" is also valid String name = expectIdentifier().text(); // Optional underlying type: "enum class Color : int" if (matchPunct(":")) { parseTypeRef(); // consume the underlying type, discard } expectPunct("{"); List values = new ArrayList<>(); if (!checkPunct("}")) { values.add(expectIdentifier().text()); if (matchOp("=")) parseExpr(); // optional initializer: "Red = 1" while (matchPunct(",")) { if (checkPunct("}")) break; // tolerate a trailing comma values.add(expectIdentifier().text()); if (matchOp("=")) parseExpr(); // optional initializer } } expectPunct("}"); expectPunct(";"); return new EnumDecl(name, isScoped, values, start.line(), start.col(), leadingComments); } private NamespaceDecl parseNamespaceDecl(List leadingComments) { CppLexerToken start = peek(); boolean isInlineNs = matchKeyword("inline"); // "inline namespace" expectKeyword("namespace"); StringBuilder nameBuilder = new StringBuilder(expectIdentifier().text()); while (checkPunct("::")) { advance(); nameBuilder.append("::").append(expectIdentifier().text()); } String name = nameBuilder.toString(); if (matchOp("=")) { StringBuilder alias = new StringBuilder(); while (!isAtEnd() && !checkPunct(";")) { alias.append(peek().text()); advance(); } matchPunct(";"); return new NamespaceDecl(name, false, List.of(), start.line(), start.col(), leadingComments); } expectPunct("{"); List items = new ArrayList<>(); while (!checkPunct("}")) { if (isAtEnd()) throw error("unexpected end of input inside namespace '" + name + "'"); List comments = consumeLeadingComments(); if (checkPunct("}")) break; items.addAll(parseTopLevelItem(comments)); } expectPunct("}"); return new NamespaceDecl(name, isInlineNs, items, start.line(), start.col(), leadingComments); } private UsingNamespaceDecl parseUsingNamespaceDecl(List leadingComments) { CppLexerToken start = expectKeyword("using"); expectKeyword("namespace"); String name = parseQualifiedTypeName(); expectPunct(";"); return new UsingNamespaceDecl(name, start.line(), start.col(), leadingComments); } // ----- Type references ------------------------------------------------ /** * Parses a TypeRef in any position: variable type, return type, param * type, template argument, cast target type, etc. * * Handles, in order: * 1. An optional leading "const". * 2. The raw C-style function-pointer special case: "Type (*)(Params)". * Detected by lookahead -- after parsing the return type, if we see * "(" "*" we know this is a function pointer declarator, not an * ordinary parenthesized expression (callers parsing a VariableDecl * pass the name through separately -- see parseFunctionPointerDecl * for the full "Type (*name)(Params)" variable-declaration form). * 3. An ordinary dotted/"::"-qualified base name (e.g. "std::string"). * 4. An optional "<...>" template argument list, where each argument * is itself parsed as either a TypeRef or -- specifically inside * std::function's argument list -- a bare function signature * "ReturnType(ParamType, ...)" with no name and no "(*)" (see * tryParseFunctionSignatureArg). * 5. Any number of trailing '*' (pointerDepth) and at most one trailing * '&' (isReference). */ private TypeRef parseTypeRef() { boolean isConst = matchKeyword("const"); return parseTypeRefAfterConst(isConst); } private TypeRef parseTypeRef(boolean leadingConst) { // Always consume "const" if present. If leadingConst=true, isConst is already // set; we still MUST consume the token so parseQualifiedTypeName doesn't see // "const" as the type name (e.g. "constexpr const char* p" -- constexpr sets // leadingConst=true, then "const" must still be consumed before "char"). // If const is explicitly present, always mark type as const regardless of leadingConst. boolean hasExplicitConst = matchKeyword("const"); return parseTypeRefAfterConst(leadingConst || hasExplicitConst); } private TypeRef parseTypeRefAfterConst(boolean isConst) { // typename/struct/class/enum X -- type elaboration or dependent type, consume prefix if (checkKeyword("typename")) advance(); else if (checkKeyword("struct") || checkKeyword("class") || checkKeyword("enum")) advance(); // decltype(expr) -- C++11 computed type if (checkKeyword("decltype")) { advance(); expectPunct("("); int d = 1; StringBuilder dtExpr = new StringBuilder("decltype("); while (!isAtEnd() && d > 0) { if (checkPunct("(")) { d++; dtExpr.append("("); advance(); } else if (checkPunct(")")) { d--; if (d > 0) dtExpr.append(")"); advance(); } else { dtExpr.append(peek().text()); advance(); } } dtExpr.append(")"); return new NamedType(dtExpr.toString(), List.of(), 0, false, isConst, false); } String baseName = parseQualifiedTypeName(); if (checkKeyword("auto")) { advance(); baseName = "auto"; } // C++20 concept-constrained auto if (matchKeyword("const")) isConst = true; // east-const: "int const" == "const int" matchKeyword("volatile"); // consume optional volatile after type name List templateArgs = List.of(); if (checkOp("<") && !checkOp("<>")) { if (pos + 1 < tokens.size() && tokens.get(pos + 1).isOp(">")) { // Explicit empty "<>" -- e.g. "Ec06Buf<> ec06_b" or "new Foo<>()". // Store sentinel so CodeGen emits "<>" rather than bare name. // (List.of() is indistinguishable from "no args" -- sentinel needed.) advance(); advance(); // consume < and > templateArgs = List.of(new NamedType("<>", List.of(), 0, false, false, false)); } else { templateArgs = parseTemplateArgList(); } // After template args, may have "::member" -- e.g. // "std::enable_if::type" or "std::is_pointer::value". // Fold these into the baseName as a qualified suffix so the full // type name round-trips correctly. while (checkPunct("::")) { advance(); // '::' if (check(CppLexerTokenType.IDENTIFIER) || check(CppLexerTokenType.KEYWORD)) { baseName = baseName + "<...>::" + advance().text(); // May have further template args after the member if (checkOp("<")) { try { parseTemplateArgList(); } catch (ParseException ignored) {} } } } } int pointerDepth = 0; while (matchOp("*")) { pointerDepth++; matchKeyword("const"); // east-const pointer: "int* const" -- consume trailing const matchKeyword("volatile"); // similarly for volatile } boolean isReference = matchOp("&"); // Only consume && as rvalue-ref if followed by a name or * (not = or binary operator context) boolean isRvalueRef = !isReference && checkOp("&&") && pos + 1 < tokens.size() && (tokens.get(pos + 1).type() == CppLexerTokenType.IDENTIFIER || tokens.get(pos + 1).type() == CppLexerTokenType.KEYWORD || tokens.get(pos + 1).isOp("*") || tokens.get(pos + 1).isPunct(")") || tokens.get(pos + 1).isPunct(",") || tokens.get(pos + 1).isPunct(">") || tokens.get(pos + 1).isPunct("...")); if (isRvalueRef) advance(); return new NamedType(baseName, templateArgs, pointerDepth, isReference, isConst, isRvalueRef); } /** * Parses a "::"-joined sequence of identifiers/keywords-used-as-typenames, * e.g. "std::string", "MyNamespace::Handle", or a single plain name like * "int" or "Handle". Accepts KEYWORD tokens too (not just IDENTIFIER) * since C++ builtin type keywords ("int", "float", "bool", "void", "auto", * "char", etc.) are lexed as KEYWORD, and "color" (a Processing-flavored * pseudo-keyword, see Lexer notes) needs the same treatment. */ /** * Parses a "::"-joined sequence of identifiers/keywords-used-as-typenames, * e.g. "std::string", "MyNamespace::Handle", or a single plain name like * "int" or "Handle". Also handles compound built-in integer type * keywords -- "signed"/"unsigned"/"long"/"short" composing with a * following base type keyword, e.g. "signed char", "unsigned int", * "long long", "unsigned long long" -- confirmed real and previously * missed entirely by the original corpus walk (found via real-corpus * testing on Datatype_Conversion.pde's "signed char b;"). */ private String parseQualifiedTypeName() { StringBuilder sb = new StringBuilder(); sb.append(parseTypeNameSegment()); // Compound integer-type keyword combinations: greedily consume any // run of signed/unsigned/long/short/int/char that follows, since // these only ever combine with each other (never with a separate // user type name in the same position) -- e.g. "unsigned long long // int" is valid C++, "unsigned long long ArrayList" never occurs. while ((check(CppLexerTokenType.KEYWORD)) && isIntegerTypeModifierOrBase(peek().text()) && isIntegerTypeModifierOrBase(sb.toString())) { sb.append(' ').append(parseTypeNameSegment()); } while (checkPunct("::")) { advance(); sb.append("::").append(parseTypeNameSegment()); } return sb.toString(); } private static final java.util.Set INTEGER_TYPE_WORDS = java.util.Set.of( "signed", "unsigned", "long", "short", "int", "char", "double", "float" ); /** True if every space-separated word in s is one of the integer-type-modifier/base words. */ private boolean isIntegerTypeModifierOrBase(String s) { for (String word : s.split(" ")) { if (!INTEGER_TYPE_WORDS.contains(word)) return false; } return true; } /** * Consumes tokens for a default template argument value or type expression, * stopping when it hits an unbalanced '>' or ',' or ';' at depth 0. * This is necessary because parseExpr() would consume '>' as a comparison * operator, swallowing the closing '>' of the template parameter list. */ private void parseTemplateDefaultValue() { // parenDepth: tracks () and [] nesting -- a > inside parens is a // comparison operator, not a template closer. e.g. (sizeof(T) > 2 ? 8 : 4) // angleDepth: tracks <> nesting from template args inside the default value // braceDepth: tracks {} so < inside lambdas/init-lists doesn't affect angle depth int parenDepth = 0; int angleDepth = 0; int braceDepth = 0; while (!isAtEnd()) { if (checkPunct("{")) { braceDepth++; advance(); continue; } if (checkPunct("}")) { if (braceDepth > 0) { braceDepth--; advance(); continue; } break; } if (checkPunct("(") || checkPunct("[")) { parenDepth++; advance(); continue; } if (checkPunct(")") || checkPunct("]")) { if (parenDepth == 0) break; // closing the outer param list parenDepth--; advance(); continue; } // < and > are only angle brackets when not inside parens/braces if (checkOp("<") && parenDepth == 0 && braceDepth == 0) { angleDepth++; advance(); continue; } if (checkOp(">") && parenDepth == 0 && braceDepth == 0) { if (angleDepth == 0) break; // closes the outer template param list angleDepth--; advance(); continue; } if (checkOp(">>") && parenDepth == 0 && braceDepth == 0) { if (angleDepth == 1) { // Split ">>" -- first closes inner angle, second closes outer splitTrailingShiftIntoTwoCloseAngles(); angleDepth--; advance(); // consume first ">" break; // second ">" left for outer list } if (angleDepth == 0) break; angleDepth -= 2; advance(); continue; } if (parenDepth == 0 && angleDepth == 0 && checkPunct(",")) break; if (parenDepth == 0 && angleDepth == 0 && checkPunct(";")) break; advance(); } } private String parseTypeNameSegment() { CppLexerToken t = peek(); if (t.type() == CppLexerTokenType.IDENTIFIER || t.type() == CppLexerTokenType.KEYWORD) { advance(); return t.text(); } throw error("expected a type name but found '" + t.text() + "'"); } /** * Parses "<Arg1, Arg2, ...>" where each Arg is either an ordinary * TypeRef or a bare function-signature (for std::function<Ret(Params)>). */ private List parseTemplateArgList() { expectOp("<"); List args = new ArrayList<>(); // Explicit empty "<>": store a sentinel so CodeGen emits "<>" not bare name if (checkOp(">") || checkOp(">>")) { if (checkOp(">>")) splitTrailingShiftIntoTwoCloseAngles(); expectOp(">"); args.add(new NamedType("<>", List.of(), 0, false, false, false)); return args; } if (!checkOp(">")) { TypeRef _a0 = parseTemplateArg(); if (matchPunct("...") && _a0 instanceof NamedType _nt0) _a0 = new NamedType(_nt0.baseName() + "...", _nt0.templateArgs(), _nt0.pointerDepth(), _nt0.isReference(), _nt0.isConst(), _nt0.isRvalueRef()); args.add(_a0); while (matchPunct(",")) { if (checkOp(">") || checkOp(">>")) break; TypeRef _ai = parseTemplateArg(); if (matchPunct("...") && _ai instanceof NamedType _nti) _ai = new NamedType(_nti.baseName() + "...", _nti.templateArgs(), _nti.pointerDepth(), _nti.isReference(), _nti.isConst(), _nti.isRvalueRef()); args.add(_ai); } } // Note: ">>" closing two nested template lists at once (e.g. // "Pair>") is lexed as a single ">>" OPERATOR token // by the lexer's longest-match rule, not as two separate '>' tokens -- // the corpus's "template angle brackets vs shift/comparison ambiguity" // smoke test confirmed ">>" lexes as one token. We must special-case // that here: a closing ">>" needs to close *this* list and leave a // single '>' behind for the enclosing list to consume. if (checkOp(">>")) { splitTrailingShiftIntoTwoCloseAngles(); } expectOp(">"); return args; } /** * Rewrites the current ">>" token into two consecutive ">" tokens in the * token stream, so nested template arg lists each consume exactly one * '>' as their closer. This mutates the token list in place rather than * re-lexing, which is simpler than threading "are we inside a template * list" state back into the lexer (the lexer has no such context, and * shouldn't need one -- this is a parser-level concern only). */ private void splitTrailingShiftIntoTwoCloseAngles() { CppLexerToken shift = tokens.get(pos); CppLexerToken first = new CppLexerToken(CppLexerTokenType.OPERATOR, ">", shift.line(), shift.col()); CppLexerToken second = new CppLexerToken(CppLexerTokenType.OPERATOR, ">", shift.line(), shift.col() + 1); tokens.set(pos, first); tokens.add(pos + 1, second); } /** Render a NamedType back to its source string including template args. */ private static String renderNamedTypeAsString(NamedType nt) { StringBuilder sb = new StringBuilder(nt.baseName()); if (!nt.templateArgs().isEmpty()) { sb.append("<"); for (int i = 0; i < nt.templateArgs().size(); i++) { if (i > 0) sb.append(", "); TypeRef a = nt.templateArgs().get(i); if (a instanceof NamedType na) sb.append(renderNamedTypeAsString(na)); else sb.append(a.toString()); } sb.append(">"); } if (nt.pointerDepth() > 0) sb.append("*".repeat(nt.pointerDepth())); if (nt.isReference()) sb.append("&"); return sb.toString(); } private TypeRef parseTemplateArg() { if (checkKeyword("true") || checkKeyword("false") || check(CppLexerTokenType.BOOL_LITERAL)) { String val = advance().text(); return new NamedType(val, List.of(), 0, false, false, false); } if (check(CppLexerTokenType.INT_LITERAL) || check(CppLexerTokenType.FLOAT_LITERAL) || check(CppLexerTokenType.CHAR_LITERAL)) { String val = advance().text(); return new NamedType(val, List.of(), 0, false, false, false); } if (checkOp("-") && pos + 1 < tokens.size() && (tokens.get(pos + 1).type() == CppLexerTokenType.INT_LITERAL || tokens.get(pos + 1).type() == CppLexerTokenType.FLOAT_LITERAL)) { advance(); String val = "-" + advance().text(); return new NamedType(val, List.of(), 0, false, false, false); } // Address-of non-type template argument: "&Widget::value", "&Class::method" if (checkOp("&")) { advance(); // consume & String name = parseQualifiedTypeName(); // Widget::value return new NamedType("&" + name, List.of(), 0, false, false, false); } // Logical negation: "!std::is_integral::value" in enable_if<!...> if (checkOp("!")) { advance(); // consume '!' parseTemplateDefaultValue(); // consume the rest of the expression return new NamedType("!...", List.of(), 0, false, false, false); } // decltype as template argument: "Pair" // Must be handled before parseTypeRef() which would consume "decltype" as a type. if (checkKeyword("decltype")) { advance(); // consume "decltype" expectPunct("("); int d = 1; StringBuilder dt = new StringBuilder("decltype("); while (!isAtEnd() && d > 0) { if (checkPunct("(")) { d++; dt.append("("); advance(); } else if (checkPunct(")")) { d--; if (d > 0) dt.append(")"); advance(); } else { dt.append(peek().text()); advance(); } } dt.append(")"); return new NamedType(dt.toString(), List.of(), 0, false, false, false); } // Pack expansion "..." trailing a type: "Ts..." in template args // Also handles standalone "..." as a variadic marker if (checkPunct("...")) { advance(); return new NamedType("...", List.of(), 0, false, false, false); } // sizeof expression: sizeof(T) or sizeof...(Args) if (checkKeyword("sizeof")) { int sStart = pos; advance(); StringBuilder sof = new StringBuilder("sizeof"); if (checkPunct("...")) { sof.append("..."); advance(); } if (checkPunct("(")) { sof.append("("); advance(); int d = 1; while (!isAtEnd() && d > 0) { if (checkPunct("(")) { d++; sof.append("("); advance(); } else if (checkPunct(")")) { d--; if (d > 0) sof.append(")"); advance(); } else { sof.append(peek().text()); advance(); } } sof.append(")"); } return new NamedType(sof.toString(), List.of(), 0, false, false, false); } TypeRef maybeReturnType = parseTypeRef(); // NTTP brace-init: "Point19{0.0f, 0.0f}" as template argument if (checkPunct("{") && maybeReturnType instanceof NamedType _nttp) { StringBuilder _nb = new StringBuilder(_nttp.baseName()).append("{"); int _nd = 1; advance(); while (!isAtEnd() && _nd > 0) { if (checkPunct("{")) { _nd++; _nb.append("{"); advance(); } else if (checkPunct("}")) { _nd--; if (_nd == 0) { _nb.append("}"); advance(); break; } _nb.append("}"); advance(); } else { _nb.append(peek().text()); advance(); } } return new NamedType(_nb.toString(), List.of(), 0, false, false, false); } // Compound boolean expression in template arg: "is_arithmetic_v && !is_same_v" // The && was consumed as rvalue-ref by parseTypeRef; check if ! or || follows // The && was already consumed by parseTypeRef as rvalue-ref boolean trailingRvalueRef = maybeReturnType instanceof NamedType ntrr && ntrr.isRvalueRef(); if (trailingRvalueRef || checkOp("||") || checkOp("!")) { // Strip the falsely-consumed && from the type if (trailingRvalueRef && maybeReturnType instanceof NamedType ntrr2) { maybeReturnType = new NamedType(ntrr2.baseName(), ntrr2.templateArgs(), ntrr2.pointerDepth(), ntrr2.isReference(), ntrr2.isConst(), false); } StringBuilder expr = new StringBuilder( maybeReturnType instanceof NamedType nt ? renderNamedTypeAsString(nt) : ""); if (trailingRvalueRef) expr.append(" && "); while (!isAtEnd()) { if (checkOp("&&")) { expr.append(" && "); advance(); } else if (checkOp("||")) { expr.append(" || "); advance(); } else if (checkOp("!")) { expr.append("!"); advance(); } else if (checkOp(">") || checkOp(">>") || checkPunct(",")) break; else if (checkPunct("(") || check(CppLexerTokenType.IDENTIFIER) || check(CppLexerTokenType.KEYWORD)) { TypeRef sub = parseTypeRef(); if (sub instanceof NamedType nts) expr.append(renderNamedTypeAsString(nts)); } else break; } return new NamedType(expr.toString(), List.of(), 0, false, false, false); } if (checkPunct("(")) { return parseFunctionSignatureTail(maybeReturnType); } // NTTP arithmetic expression: "N-1", "N*2", "N+1" etc. // After parsing the base type/value, consume arithmetic ops and operands if (maybeReturnType instanceof NamedType nt && !nt.baseName().equals("void")) { StringBuilder expr = new StringBuilder(nt.baseName()); while (!isAtEnd() && (checkOp("+") || checkOp("-") || checkOp("*") || checkOp("/") || checkOp("%"))) { expr.append(advance().text()); if (!isAtEnd() && (check(CppLexerTokenType.INT_LITERAL) || check(CppLexerTokenType.IDENTIFIER))) { expr.append(advance().text()); } } if (expr.length() > nt.baseName().length()) { return new NamedType(expr.toString(), List.of(), 0, false, false, false); } } return maybeReturnType; } /** Parses "(ParamType, ...)" given an already-parsed return type, producing a FunctionSignatureType. */ private FunctionSignatureType parseFunctionSignatureTail(TypeRef returnType) { expectPunct("("); List paramTypes = new ArrayList<>(); if (!checkPunct(")")) { paramTypes.add(parseTypeRef()); while (matchPunct(",")) { paramTypes.add(parseTypeRef()); } } expectPunct(")"); return new FunctionSignatureType(returnType, paramTypes); } /** * Detects whether the upcoming tokens form a raw C-style function pointer * declarator: "(" "*" IDENTIFIER ")" "(" -- used by declaration parsing to * decide whether a "(" right after a type belongs to this special form * rather than being parsed as part of an ordinary VariableDecl. Lookahead * only; does not consume. */ private boolean looksLikeFunctionPointerDeclarator() { if (!checkPunct("(")) return false; int i = 1; // Consume optional Class:: qualifier for member function pointers while (pos + i + 1 < tokens.size() && tokens.get(pos + i).type() == CppLexerTokenType.IDENTIFIER && tokens.get(pos + i + 1).isPunct("::")) { i += 2; } if (pos + i >= tokens.size()) return false; if (!tokens.get(pos + i).isOp("*") && !tokens.get(pos + i).isOp("&")) return false; i++; // past * or & if (pos + i >= tokens.size()) return false; // name (possibly followed by [N] for array-of-function-pointers) if (tokens.get(pos + i).type() != CppLexerTokenType.IDENTIFIER) return false; i++; // past name if (pos + i >= tokens.size()) return false; // Either ) directly followed by ( for params, or [N] then ) // A bare "(*name)" MUST be followed by "(" (param list) to be a function pointer. // "shader(*noiseShader);" is a call expression, not a fn ptr -- no "(" after ")". if (tokens.get(pos + i).isPunct(")")) { int j = i + 1; if (j >= tokens.size()) return false; // Function pointer: "void (*fp)(int)" -- "(" must follow // Array of fn ptrs: "void (*fp)[N]" -- "[" may follow return tokens.get(pos + j).isPunct("(") || tokens.get(pos + j).isPunct("["); } if (tokens.get(pos + i).isPunct("[")) return true; // array of fn ptrs return false; } /** * Parses the remainder of "Type (*name)(ParamTypes...)" given an * already-parsed return type, returning the declared variable's name and * its FunctionPointerType. Confirmed necessary by the corpus's * useRawFunctionPointer fixture. */ private NameAndFunctionPointerType parseFunctionPointerDeclaratorTail(TypeRef returnType) { expectPunct("("); // Consume optional Class:: qualifier for member function pointers: (Widget::*name) // Encode Class:: and & vs * into name so CodeGen can render correctly. StringBuilder classQual = new StringBuilder(); while (check(CppLexerTokenType.IDENTIFIER) && tokens.get(pos + 1).isPunct("::")) { classQual.append(advance().text()).append("::"); advance(); // consume "::" } boolean isRef = matchOp("&"); if (!isRef) matchOp("*"); // (&name) or (*name) String name = expectIdentifier().text(); // Encoded name: "Point::*memberFuncPtr" or "&refToRow" etc. String encodedName = classQual.length() > 0 ? classQual.toString() + (isRef ? "&" : "*") + name : (isRef ? "&" : "") + name; // Array-of-function-pointers: (*arr[5]) -- consume the [N] subscript // Encode dims into name now so CodeGen can emit "void (*arr[5])(int)" correctly. // parseOptionalArrayDims consumes them; we re-encode as "[N]" string suffix. List arrayDims = parseOptionalArrayDims(); StringBuilder arrayDimStr = new StringBuilder(); for (Expr dim : arrayDims) { arrayDimStr.append("["); if (dim != null) arrayDimStr.append(CodeGen.renderExpr(dim)); arrayDimStr.append("]"); } encodedName = encodedName + arrayDimStr.toString(); expectPunct(")"); // Pointer-to-array or reference-to-array: "int (*p)[20]" / "int (&r)[20]" // Encode array dims into name for CodeGen: "&refToRow[20]" if (checkPunct("[")) { StringBuilder dimStr = new StringBuilder(); while (checkPunct("[")) { advance(); dimStr.append("["); if (!checkPunct("]")) { dimStr.append(peek().text()); parseExpr(); } expectPunct("]"); dimStr.append("]"); } return new NameAndFunctionPointerType(encodedName + dimStr.toString(), new FunctionPointerType(returnType, List.of())); } expectPunct("("); List paramTypes = new ArrayList<>(); if (!checkPunct(")")) { paramTypes.add(parseTypeRef()); while (matchPunct(",")) { paramTypes.add(parseTypeRef()); } } expectPunct(")"); boolean isConstMethod = matchKeyword("const"); // trailing const on member function pointer // Encode const into name with sentinel __const__ for CodeGen if (isConstMethod) encodedName = encodedName + "__const__"; return new NameAndFunctionPointerType(encodedName, new FunctionPointerType(returnType, paramTypes)); } private record NameAndFunctionPointerType(String name, FunctionPointerType type) { } // ----- Expressions (precedence climbing, low to high) ----------------- // // assignment -> ternary (("=") assignment)? [right-assoc] // ternary -> logicalOr ("?" expr ":" ternary)? // logicalOr -> logicalAnd ("||" logicalAnd)* // logicalAnd -> bitOr ("&&" bitOr)* // bitOr -> bitXor ("|" bitXor)* // bitXor -> bitAnd ("^" bitAnd)* // bitAnd -> equality ("&" equality)* // equality -> relational (("=="|"!=") relational)* // relational -> shift (("<"|">"|"<="|">=") shift)* // shift -> additive (("<<"|">>") additive)* // additive -> multiplicative (("+"|"-") multiplicative)* // multiplicative -> unary (("*"|"/"|"%") unary)* // unary -> ("-"|"!"|"&"|"~"|"++"|"--") unary | postfix // postfix -> primary (call | member | index | "++" | "--")* // primary -> literal | identifier | scopedName | "(" expr ")" // | lambda | new | cast | initializerList private static final java.util.Set COMPOUND_ASSIGN_OPS = java.util.Set.of( "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=", "<<=", ">>=" ); Expr parseExpr() { return parseAssignment(); } private Expr parseAssignment() { Expr left = parseTernary(); if (checkOp("=")) { CppLexerToken t = advance(); Expr right = parseAssignment(); // right-associative return new AssignExpr(left, right, t.line(), t.col(), List.of()); } if (peek().type() == CppLexerTokenType.OPERATOR && COMPOUND_ASSIGN_OPS.contains(peek().text())) { CppLexerToken t = advance(); Expr right = parseAssignment(); return new BinaryExpr(t.text(), left, right, t.line(), t.col(), List.of()); } return left; } private Expr parseTernary() { // throw-expression: "throw expr" is valid as an expression in C++ // e.g. "cond ? val : throw std::runtime_error(...)" if (checkKeyword("throw")) { CppLexerToken t = advance(); if (checkPunct(";") || checkPunct(")") || checkPunct(":") || checkPunct(",")) { // bare throw (rethrow) with no operand return new UnaryExpr("throw ", new Literal(Literal.Kind.INT, "0", t.line(), t.col(), List.of()), t.line(), t.col(), List.of()); } Expr val = parseTernary(); return new UnaryExpr("throw ", val, t.line(), t.col(), List.of()); } Expr cond = parseLogicalOr(); if (checkPunct("?")) { CppLexerToken t = advance(); Expr thenExpr = parseExpr(); expectPunct(":"); Expr elseExpr = parseTernary(); return new TernaryExpr(cond, thenExpr, elseExpr, t.line(), t.col(), List.of()); } return cond; } private Expr parseLogicalOr() { Expr left = parseLogicalAnd(); while (checkOp("||")) { CppLexerToken t = advance(); Expr right = parseLogicalAnd(); left = new BinaryExpr("||", left, right, t.line(), t.col(), List.of()); } return left; } private Expr parseLogicalAnd() { Expr left = parseBitOr(); while (checkOp("&&")) { CppLexerToken t = advance(); Expr right = parseBitOr(); left = new BinaryExpr("&&", left, right, t.line(), t.col(), List.of()); } return left; } private Expr parseBitOr() { Expr left = parseBitXor(); while (checkOp("|")) { CppLexerToken t = advance(); Expr right = parseBitXor(); left = new BinaryExpr("|", left, right, t.line(), t.col(), List.of()); } return left; } private Expr parseBitXor() { Expr left = parseBitAnd(); while (checkOp("^")) { CppLexerToken t = advance(); Expr right = parseBitAnd(); left = new BinaryExpr("^", left, right, t.line(), t.col(), List.of()); } return left; } private Expr parseBitAnd() { Expr left = parseEquality(); while (checkOp("&")) { CppLexerToken t = advance(); Expr right = parseEquality(); left = new BinaryExpr("&", left, right, t.line(), t.col(), List.of()); } return left; } private Expr parseEquality() { Expr left = parseRelational(); while (checkOp("==") || checkOp("!=")) { CppLexerToken t = advance(); Expr right = parseRelational(); left = new BinaryExpr(t.text(), left, right, t.line(), t.col(), List.of()); } return left; } private Expr parseRelational() { Expr left = parseShift(); while (checkOp("<") || checkOp(">") || checkOp("<=") || checkOp(">=")) { // Spaceship operator <=> is lexed as <= then > -- detect and merge if (checkOp("<=") && pos + 1 < tokens.size() && tokens.get(pos + 1).isOp(">")) { CppLexerToken t = advance(); advance(); // consume <= and > Expr right = parseShift(); left = new BinaryExpr("<=>", left, right, t.line(), t.col(), List.of()); continue; } CppLexerToken t = advance(); Expr right = parseShift(); left = new BinaryExpr(t.text(), left, right, t.line(), t.col(), List.of()); } return left; } private Expr parseShift() { Expr left = parseAdditive(); while (checkOp("<<") || checkOp(">>")) { CppLexerToken t = advance(); Expr right = parseAdditive(); left = new BinaryExpr(t.text(), left, right, t.line(), t.col(), List.of()); } return left; } private Expr parseAdditive() { Expr left = parseMultiplicative(); while (checkOp("+") || checkOp("-")) { CppLexerToken t = advance(); Expr right = parseMultiplicative(); left = new BinaryExpr(t.text(), left, right, t.line(), t.col(), List.of()); } return left; } private Expr parseMultiplicative() { Expr left = parseUnary(); while (checkOp("*") || checkOp("/") || checkOp("%")) { CppLexerToken t = advance(); Expr right = parseUnary(); left = new BinaryExpr(t.text(), left, right, t.line(), t.col(), List.of()); } return left; } private static final java.util.Set UNARY_OPS = java.util.Set.of("-", "!", "&", "~", "+", "*"); private Expr parseUnary() { if (checkOp("++") || checkOp("--")) { CppLexerToken t = advance(); Expr operand = parseUnary(); return new UnaryExpr(t.text(), operand, t.line(), t.col(), List.of()); } if (peek().type() == CppLexerTokenType.OPERATOR && UNARY_OPS.contains(peek().text())) { CppLexerToken t = advance(); Expr operand = parseUnary(); return new UnaryExpr(t.text(), operand, t.line(), t.col(), List.of()); } if (checkKeyword("new")) { return parseNew(); } if (checkKeyword("delete")) { // delete is modeled as a statement (DeleteStatement) per the AST // design notes -- it should never be reached from expression // context. If it is, that's a real grammar gap, not something to // silently paper over. throw error("'delete' is only valid as a statement, not inside an expression"); } // C-style cast: "(" TypeName ")" expr -- only recognized when the // parenthesized content is unambiguously a type (an identifier/keyword // optionally followed by '*'/'&'/template-args, then immediately ')'), // and is followed by something that can start an expression. This // disambiguates from a plain parenthesized expression like "(a + b)". if (checkPunct("(") && looksLikeCast()) { return parseCast(); } return parsePostfix(); } /** * Lookahead-only check for whether the upcoming "(...)" is a C-style cast * rather than a parenthesized expression. We tentatively try parsing a * TypeRef starting just after "(" and require it to be immediately * followed by ")" and then something that looks like the start of a unary * expression (identifier, literal, "(", unary operator). This is a * backtracking lookahead -- cheap here since types are short and this * only runs when we've already seen "(". */ private boolean looksLikeCast() { int save = pos; try { if (!matchPunct("(")) return false; // Must look like a type: starts with an identifier or type-keyword. if (!(check(CppLexerTokenType.IDENTIFIER) || check(CppLexerTokenType.KEYWORD))) return false; TypeRef ignored; try { ignored = parseTypeRef(); } catch (ParseException e) { return false; } if (!checkPunct(")")) return false; advance(); // consume ')' // What follows a real cast must be able to start a unary expression. return canStartExpression(peek()); } finally { pos = save; } } private boolean canStartExpression(CppLexerToken t) { if (t.type() == CppLexerTokenType.IDENTIFIER || t.type() == CppLexerTokenType.INT_LITERAL || t.type() == CppLexerTokenType.FLOAT_LITERAL || t.type() == CppLexerTokenType.STRING_LITERAL || t.type() == CppLexerTokenType.CHAR_LITERAL || t.type() == CppLexerTokenType.BOOL_LITERAL) { return true; } if (t.isPunct("(")) return true; if (t.isOp("-") || t.isOp("!") || t.isOp("&") || t.isOp("~") || t.isOp("++") || t.isOp("--")) return true; if (t.isKeyword("new")) return true; return false; } private Expr parseCast() { CppLexerToken start = peek(); expectPunct("("); TypeRef targetType = parseTypeRef(); expectPunct(")"); Expr expr = parseUnary(); return new CastExpr(targetType, expr, start.line(), start.col(), List.of()); } private Expr parseNew() { CppLexerToken start = expectKeyword("new"); // Placement new: "new (buf) Type(args)" -- consume placement args first if (checkPunct("(")) { advance(); int _pd=1; while (!isAtEnd() && _pd > 0) { if (checkPunct("(")) _pd++; else if (checkPunct(")")) _pd--; advance(); } } TypeRef type = parseTypeRef(); if (matchPunct("[")) { Expr sizeExpr = parseExpr(); expectPunct("]"); // Consume optional value-initializer on array-new: "new float[n]()" // This is valid C++ (zero-initializes the array) and must be consumed // here or the "()" is left in the stream, breaking the enclosing // constructor init-list parser. Args inside are allowed but rare. if (checkPunct("(")) { advance(); int _d = 1; while (!isAtEnd() && _d > 0) { if (checkPunct("(")) _d++; else if (checkPunct(")")) _d--; advance(); } } return new ArrayNewExpr(type, sizeExpr, start.line(), start.col(), List.of()); } List args = new ArrayList<>(); if (matchPunct("(")) { if (!checkPunct(")")) { Expr a0 = parseExpr(); if (matchPunct("...")) a0 = new PostfixExpr("...", a0, a0.line(), a0.col(), List.of()); args.add(a0); while (matchPunct(",")) { Expr ai = parseExpr(); if (matchPunct("...")) ai = new PostfixExpr("...", ai, ai.line(), ai.col(), List.of()); args.add(ai); } } expectPunct(")"); } return new NewExpr(type, args, start.line(), start.col(), List.of()); } private Expr parsePostfix() { Expr expr = parsePrimary(); if (checkPunct("...") && pos + 1 < tokens.size() && tokens.get(pos + 1).isPunct("[")) { advance(); // pack indexing: args...[0] } while (true) { if (checkPunct("::")) { // Static member / scope resolution in expression context: // "std::is_pointer::value", "MyClass::staticMethod()", etc. // Fold the "::" and member name into the existing identifier string. advance(); // '::' String member = check(CppLexerTokenType.IDENTIFIER) || check(CppLexerTokenType.KEYWORD) ? advance().text() : ""; if (expr instanceof Identifier id) { expr = new Identifier(id.name() + "::" + member, id.line(), id.col(), List.of()); } else { expr = new Identifier("::" + member, peek().line(), peek().col(), List.of()); } // Template args in expression: "AutoParam<42>::value" } else if (checkOp("<") && expr instanceof Identifier eid && looksLikeTemplateArgList()) { int argStart = pos; advance(); // consume < int depth = 1, pd = 0, bd = 0; while (!isAtEnd() && depth > 0) { if (checkPunct("(") || checkPunct("[")) pd++; else if (checkPunct(")") || checkPunct("]")) pd--; else if (checkPunct("{")) bd++; else if (checkPunct("}")) bd--; else if (pd == 0 && bd == 0) { if (checkOp("<")) depth++; else if (checkOp(">")) { depth--; if (depth == 0) { advance(); break; } } else if (checkOp(">>")) { depth -= 2; splitTrailingShiftIntoTwoCloseAngles(); if (depth <= 0) { advance(); break; } } } if (depth > 0) advance(); } StringBuilder targs = new StringBuilder(eid.name()).append("<"); for (int i = argStart + 1; i < pos - 1; i++) targs.append(tokens.get(i).text()); targs.append(">"); expr = new Identifier(targs.toString(), eid.line(), eid.col(), List.of()); } else if (checkOp("->*")) { // "->*" lexed as a single token CppLexerToken t = advance(); Expr rhs = parseUnary(); expr = new BinaryExpr("->*", expr, rhs, t.line(), t.col(), List.of()); continue; } else if (checkPunct(".") || checkPunct("->")) { boolean isArrow = checkPunct("->"); CppLexerToken t = advance(); // Pointer-to-member dereference: "obj.*ptr" or "obj->*ptr" // After consuming "." or "->", if "*" follows, this is ".*" / "->*" if (checkOp("*")) { advance(); // consume the "*" Expr rhs = parseUnary(); // the member pointer expression expr = new BinaryExpr(isArrow ? "->*" : ".*", expr, rhs, t.line(), t.col(), List.of()); continue; } // Explicit destructor call: "p->~PoolObj()" String member; if (checkOp("~")) { advance(); // consume ~ member = "~" + expectIdentifier().text(); } else { member = expectIdentifierOrKeywordName(); } // Sibling fix to the std::vector(...) bug found via // RealHeaderStressTest: a member name can ALSO be // followed by explicit template arguments before the // call parens ("obj.method(5)" -- calling a // template member function with an explicit type // argument, since the argument types alone aren't always // enough for the compiler to deduce it). Without this // check, "obj.method(5)" misparses the same way // "std::vector(rows)" did before that fix: as // "obj.method" followed by an unrelated // "< int > (5)" comparison chain. Folded the member name // and its template args into one combined member-name // string, consistent with how a templated callee // Identifier already folds its name and args together // elsewhere in this parser. if (checkOp("<") && looksLikeTemplatedConstructionCallee()) { List templateArgs = parseTemplateArgList(); member = member + "<" + renderTemplateArgs(templateArgs) + ">"; } expr = new MemberAccessExpr(expr, member, isArrow, t.line(), t.col(), List.of()); } else if (checkPunct("(")) { CppLexerToken t = peek(); List args = parseArgList(); expr = new CallExpr(expr, args, t.line(), t.col(), List.of()); } else if (checkPunct("{") && expr instanceof Identifier id) { // Brace-init of a named type: "Inner{1, 2}", "Rect{x, y}" // Any identifier (with or without template args) can be // brace-initialized this way in C++. The guard "expr instanceof // Identifier" prevents misfiring on expressions like "x{..." which // are never valid C++ (x would be a function call target, not a type). CppLexerToken t = peek(); InitializerListExpr init = (InitializerListExpr) parseInitializerList(); expr = new CallExpr(expr, init.elements(), true, t.line(), t.col(), List.of()); } else if (checkPunct("[")) { CppLexerToken t = advance(); Expr index = parseExpr(); while (matchPunct(",")) parseExpr(); // multi-dim subscript C++23 expectPunct("]"); expr = new IndexExpr(expr, index, t.line(), t.col(), List.of()); } else if (checkOp("++") || checkOp("--")) { CppLexerToken t = advance(); expr = new PostfixExpr(t.text(), expr, t.line(), t.col(), List.of()); } else if (checkPunct("{") && (expr instanceof Identifier || expr instanceof ScopedName)) { // Brace-init after identifier: "std::pair{1, 1.0f}" or "Type{args}" int _bl = expr.line(); int _bc = expr.col(); Expr braceInit = parseInitializerList(); Identifier _callee = new Identifier( expr instanceof Identifier _eid ? _eid.name() : String.join("::", ((ScopedName)expr).parts()), _bl, _bc, List.of()); expr = new CallExpr(_callee, ((InitializerListExpr)braceInit).elements(), true, _bl, _bc, List.of()); } else { break; } } return expr; } /** * Member names are usually identifiers but may be a few keyword-shaped * tokens too in practice (e.g. nothing in-scope right now requires this, * but kept permissive since C++ member names are never actual keywords * in valid code -- this mainly just defers to expectIdentifier). */ private String expectIdentifierOrKeywordName() { return expectIdentifier().text(); } private List parseArgList() { expectPunct("("); List args = new ArrayList<>(); if (!checkPunct(")")) { Expr a0 = parseExpr(); if (matchPunct("...")) a0 = new PostfixExpr("...", a0, a0.line(), a0.col(), List.of()); args.add(a0); while (matchPunct(",")) { if (checkPunct(")")) break; Expr ai = parseExpr(); if (matchPunct("...")) ai = new PostfixExpr("...", ai, ai.line(), ai.col(), List.of()); args.add(ai); } } expectPunct(")"); return args; } private Expr parsePrimary() { CppLexerToken t = peek(); switch (t.type()) { case INT_LITERAL -> { String txt = advance().text(); if (txt.equals("0") && !isAtEnd() && check(CppLexerTokenType.IDENTIFIER)) { String nx=peek().text(); if(nx.startsWith("b")||nx.startsWith("B")||nx.startsWith("x")||nx.startsWith("X")) txt+=advance().text(); } while (!isAtEnd() && check(CppLexerTokenType.CHAR_LITERAL)) { String chunk=peek().text(); if(!chunk.startsWith("'"))break; String inner=chunk.substring(1); boolean wf=inner.endsWith("'"); if(wf)inner=inner.substring(0,inner.length()-1); if(!inner.matches("[0-9a-fA-F]+"))break; advance();txt+=inner; if(wf&&!isAtEnd()&&check(CppLexerTokenType.INT_LITERAL))txt+=advance().text(); if(!wf)break; } txt += consumeUdlSuffix(); return new Literal(Literal.Kind.INT, txt, t.line(), t.col(), List.of()); } case FLOAT_LITERAL -> { String txt = advance().text(); while (!isAtEnd() && check(CppLexerTokenType.CHAR_LITERAL)) { String chunk=peek().text(); if(!chunk.startsWith("'"))break; String inner=chunk.substring(1); boolean wf=inner.endsWith("'"); if(wf)inner=inner.substring(0,inner.length()-1); if(!inner.matches("[0-9a-fA-F]+"))break; advance();txt+=inner; if(wf&&!isAtEnd()&&(check(CppLexerTokenType.INT_LITERAL)||check(CppLexerTokenType.FLOAT_LITERAL)))txt+=advance().text(); if(!wf)break; } txt += consumeUdlSuffix(); return new Literal(Literal.Kind.FLOAT, txt, t.line(), t.col(), List.of()); } case STRING_LITERAL -> { String txt = advance().text(); txt += consumeUdlSuffix(); // Adjacent string literal concatenation: "hello" " " "world" while (check(CppLexerTokenType.STRING_LITERAL)) { txt += advance().text(); txt += consumeUdlSuffix(); } return new Literal(Literal.Kind.STRING, txt, t.line(), t.col(), List.of()); } case CHAR_LITERAL -> { advance(); return new Literal(Literal.Kind.CHAR, t.text(), t.line(), t.col(), List.of()); } case BOOL_LITERAL -> { advance(); return new Literal(Literal.Kind.BOOL, t.text(), t.line(), t.col(), List.of()); } default -> { /* fall through below */ } } if (checkPunct("[")) { // lambda capture-list start return parseLambda(); } if (checkPunct("(")) { advance(); Expr inner = parseExpr(); // Fold expression or comma operator inside parens. // Binary fold: "(init op ... op pack)" -- op and ... already in inner as BinaryExpr // Unary fold: "(pack op ...)" -- already in inner // Comma fold: "((void)(expr), ...)" -- comma then ... before ")" if (!checkPunct(")")) { int depth = 0; boolean seenCommaFold = false; while (!isAtEnd()) { if (checkPunct("(")) { depth++; advance(); } else if (checkPunct(")")) { if (depth == 0) break; depth--; advance(); } else if (checkPunct(",") && depth == 0) { advance(); if (checkPunct("...")) { advance(); seenCommaFold = true; break; } // Regular comma -- consume the next expression parseExpr(); } else advance(); } if (seenCommaFold) { // Comma fold: "(expr, ...)" -- wrap inner in BinaryExpr("," , ...) CppLexerToken fakeToken = peek(); inner = new BinaryExpr(",", inner, new Identifier("...", fakeToken.line(), fakeToken.col(), List.of()), fakeToken.line(), fakeToken.col(), List.of()); } } expectPunct(")"); return inner; } if (checkPunct("{")) { return parseInitializerList(); } // Global scope resolution: "::operator new(...)" or "::SomeFunc()" if (checkPunct("::")) { advance(); // consume :: // Parse what follows as an identifier/operator name if (checkKeyword("operator")) { String opName = "::" + parseFunctionOrVariableName(); return new Identifier(opName, t.line(), t.col(), List.of()); } if (check(CppLexerTokenType.IDENTIFIER)) { String name = advance().text(); return new Identifier("::" + name, t.line(), t.col(), List.of()); } } // Wide/unicode string prefix: L"..." u"..." U"..." u8"..." if (t.type() == CppLexerTokenType.IDENTIFIER && (t.text().equals("L") || t.text().equals("u") || t.text().equals("U") || t.text().equals("u8")) && pos + 1 < tokens.size() && tokens.get(pos + 1).type() == CppLexerTokenType.STRING_LITERAL) { String prefix = advance().text(); String txt = prefix + advance().text(); while (check(CppLexerTokenType.STRING_LITERAL)) txt += advance().text(); return new Literal(Literal.Kind.STRING, txt, t.line(), t.col(), List.of()); } // sizeof expression: "sizeof(T)" or "sizeof expr" (no parens) if (checkKeyword("sizeof")) { int sizeofStart = pos; advance(); // consume "sizeof" StringBuilder sizeofText = new StringBuilder("sizeof"); if (checkPunct("...")) { advance(); sizeofText.append("..."); } // sizeof... if (checkPunct("(")) { sizeofText.append("("); advance(); int d=1; while(!isAtEnd()&&d>0){ if(checkPunct("(")){ d++; sizeofText.append("("); advance(); } else if(checkPunct(")")){ d--; if(d>0)sizeofText.append(")"); advance(); } else { // Add space before identifier/keyword tokens to avoid merging String _st = peek().text(); if (sizeofText.length() > 0) { char _last = sizeofText.charAt(sizeofText.length()-1); if (Character.isLetterOrDigit(_last) || _last == '_') sizeofText.append(" "); } sizeofText.append(_st); advance(); } } sizeofText.append(")"); } else { // sizeof without parens: consume one unary expression sizeofText.append(" "); int before = pos; parseUnary(); for (int si = before; si < pos; si++) { if (si > before) sizeofText.append(" "); sizeofText.append(tokens.get(si).text()); } } return new Identifier(sizeofText.toString(), t.line(), t.col(), List.of()); } // alignof expression if (checkKeyword("alignof") || (t.type() == CppLexerTokenType.IDENTIFIER && t.text().equals("alignof"))) { advance(); expectPunct("("); int d=1; while(!isAtEnd()&&d>0){if(checkPunct("("))d++;else if(checkPunct(")"))d--;advance();} return new Literal(Literal.Kind.INT, "alignof(...)", t.line(), t.col(), List.of()); } if (t.type() == CppLexerTokenType.IDENTIFIER || t.type() == CppLexerTokenType.KEYWORD) { // "typename T::member" in expression context -- fold typename into the following qualified name if (t.text().equals("typename")) { advance(); // consume typename // Parse the following type and prepend "typename " CppLexerToken next = peek(); if (next.type() == CppLexerTokenType.IDENTIFIER || next.type() == CppLexerTokenType.KEYWORD) { String typeName = advance().text(); // Consume :: qualified parts while (checkPunct("::")) { advance(); if (check(CppLexerTokenType.IDENTIFIER) || check(CppLexerTokenType.KEYWORD)) typeName += "::" + advance().text(); } // Consume template args if present if (checkOp("<") && looksLikeTemplateArgList()) { int _as = pos; advance(); int _d=1; while (!isAtEnd() && _d > 0) { if (checkOp("<")) { _d++; advance(); } else if (checkOp(">")) { _d--; advance(); } else if (checkOp(">>")) { _d-=2; splitTrailingShiftIntoTwoCloseAngles(); advance(); } else advance(); } StringBuilder _ta = new StringBuilder(typeName).append("<"); for (int _i=_as+1; _i"); typeName = _ta.toString(); } // Consume trailing :: member if (checkPunct("::")) { advance(); if (check(CppLexerTokenType.IDENTIFIER)||check(CppLexerTokenType.KEYWORD)) typeName += "::" + advance().text(); } return new Identifier("typename " + typeName, t.line(), t.col(), List.of()); } return new Identifier("typename", t.line(), t.col(), List.of()); } if (t.text().equals("requires")) { advance(); if(checkPunct("(")){ advance();int d=1;while(!isAtEnd()&&d>0){if(checkPunct("("))d++;else if(checkPunct(")"))d--;advance();}} if(checkPunct("{")){ advance();int d=1;while(!isAtEnd()&&d>0){if(checkPunct("{"))d++;else if(checkPunct("}"))d--;advance();}} return new Literal(Literal.Kind.BOOL,"true",t.line(),t.col(),List.of()); } if (t.text().equals("R") && !isAtEnd() && check(CppLexerTokenType.STRING_LITERAL)) { advance(); String raw=advance().text(); return new Literal(Literal.Kind.STRING,"R"+raw,t.line(),t.col(),List.of()); } // Could be a plain identifier, the start of a "::"-qualified // ScopedName, or a bare templated-construction-call callee like // "ArrayList()" / "PenroseSnowflakeLSystem()" (the latter // with no template args at all). Confirmed by the corpus as the // no-"new" construction idiom (see CallExpr's design notes). // // The tricky case is the templated form: "ArrayList()" // is syntactically ambiguous with "a < b > ()" (comparisons // chained with an empty-paren expression, which isn't valid // anyway, but the grammar can't assume that). We resolve it with // a backtracking lookahead: if a '<' follows the identifier AND // the content up to a matching '>' parses as a template-arg-like // list AND is immediately followed by '(', treat the whole // "Name" as one callee identifier whose text includes the // template args verbatim -- semantic resolution (is this really // a constructor call) is deferred to a later pass either way, // consistent with how the untemplated bare-call case already // works. String first = t.text(); advance(); if (checkPunct("::")) { List parts = new ArrayList<>(); parts.add(first); while (matchPunct("::")) { parts.add(parseTypeNameSegment()); } // BUG FIX, found via RealHeaderStressTest against the real // engine headers (Game_Of_Life.pde's real // "cells.resize(cols, std::vector(rows));"): this // branch used to return immediately as a plain ScopedName // the moment it saw "::", WITHOUT ever checking for a // following "(" templated-construction-call suffix // -- unlike the bare-identifier case just below (line // ~1342 in this method), which already had this check. // This meant "std::vector(rows)" was parsed as // "std::vector" (a ScopedName) followed by a SEPARATE, // unrelated "< int > (rows)" comparison-chain expression // -- confirmed directly: g++ rejected the resulting // "((std::vector < int) > rows)" with "template argument // 1 is invalid". A bare bare "Foo(...)" (no "::") // already worked correctly; only the "::"-qualified form // (which is exactly what every "std::"-prefixed standard // container construction looks like) had this gap. String joined = String.join("::", parts); if (checkOp("<") && looksLikeTemplatedConstructionCallee()) { List templateArgs = parseTemplateArgList(); String rendered = joined + "<" + renderTemplateArgs(templateArgs) + ">"; return new Identifier(rendered, t.line(), t.col(), List.of()); } // Non-call template use: "std::is_arithmetic_v" (no "(" after ">") if (checkOp("<") && looksLikeTemplateArgList()) { int argStart = pos; advance(); int _d=1, _pd=0, _bd=0; while (!isAtEnd() && _d > 0) { if (checkPunct("(")||checkPunct("[")) _pd++; else if (checkPunct(")")||checkPunct("]")) _pd--; else if (checkPunct("{")) _bd++; else if (checkPunct("}")) _bd--; else if (_pd==0&&_bd==0) { if (checkOp("<")) _d++; else if (checkOp(">")) { _d--; if(_d==0){advance();break;} } else if (checkOp(">>")) { _d-=2; splitTrailingShiftIntoTwoCloseAngles(); if(_d<=0){advance();break;} } } if (_d>0) advance(); } StringBuilder _tb = new StringBuilder(joined).append("<"); for (int _i=argStart+1; _i"); return new Identifier(_tb.toString(), t.line(), t.col(), List.of()); } return new ScopedName(parts, t.line(), t.col(), List.of()); } if (checkOp("<") && looksLikeTemplatedConstructionCallee()) { List templateArgs = parseTemplateArgList(); String rendered = first + "<" + renderTemplateArgs(templateArgs) + ">"; return new Identifier(rendered, t.line(), t.col(), List.of()); } return new Identifier(first, t.line(), t.col(), List.of()); } // Fold expression pack expansion: "..." -- appears in "(args + ...)" and // "((void)(std::cout << ts), ...)" as the RHS of a fold operator. // Produce a placeholder identifier so the enclosing expression completes. if (checkPunct("...")) { CppLexerToken t2 = advance(); return new Identifier("...", t2.line(), t2.col(), List.of()); } throw error("unexpected token '" + t.text() + "' while parsing an expression"); } private String renderTemplateArgs(List args) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < args.size(); i++) { if (i > 0) sb.append(", "); sb.append(args.get(i).describe()); } return sb.toString(); } /** * Lookahead-only: from just before the '<' following an identifier, * determines whether this is a templated-construction-call callee * ("Name(" ) rather than a less-than comparison. Tries to parse a * template-arg list and checks that '(' immediately follows. Restores * position regardless. */ /** True if current position looks like a template arg list: '<' followed eventually by '>', * used to detect "Base" in base class context. Lighter than parseTemplateArgList. */ private boolean looksLikeTemplateArgList() { // Peek ahead to see if we can find a matching '>' without hitting ';' or '{' // Track () and {} depth so { inside decltype(...) doesn't abort int save = pos; try { if (!checkOp("<")) return false; advance(); int depth = 1; int parenDepth = 0; int braceDepth = 0; int steps = 0; while (!isAtEnd() && depth > 0 && steps < 120) { if (checkPunct("(")) parenDepth++; else if (checkPunct(")")) { parenDepth--; if (parenDepth < 0) return false; } else if (checkPunct("{")) braceDepth++; else if (checkPunct("}")) braceDepth--; else if (parenDepth == 0 && braceDepth == 0) { if (checkOp("<")) depth++; else if (checkOp(">")) depth--; else if (checkOp(">>")) depth -= 2; else if (checkPunct(";")) return false; // A bare "?" at depth 0 means ternary operator -- not a template arg list. // Template args never contain unparenthesized ternary operators. // e.g. "v < lo ? lo : v > hi" -- the "?" rules out template args. else if (checkPunct("?") && depth == 1) return false; // A bare "+" "-" "*" "/" "%" at depth 0 after a non-type token // strongly suggests arithmetic expression, not template args. // But these can appear in NTTPs so only reject "?" which is unambiguous. } advance(); steps++; } return depth <= 0; } finally { pos = save; } } /** Consumes a user-defined literal suffix (_deg, _kb, _v, etc.) and returns it (or empty string). */ private String consumeUdlSuffix() { if (check(CppLexerTokenType.IDENTIFIER) && peek().text().startsWith("_")) { return advance().text(); } return ""; } private void consumeAttributes() { while (checkPunct("[") && pos + 1 < tokens.size() && tokens.get(pos + 1).isPunct("[")) { advance(); advance(); int depth = 2; while (!isAtEnd() && depth > 0) { if (checkPunct("[")) depth++; else if (checkPunct("]")) depth--; advance(); } } } private boolean isStructuredBindingStart() { int i = pos; if (i < tokens.size() && tokens.get(i).isKeyword("const")) i++; if (i >= tokens.size() || !tokens.get(i).isKeyword("auto")) return false; i++; if (i < tokens.size() && (tokens.get(i).isOp("&") || tokens.get(i).isOp("&&"))) i++; return i < tokens.size() && tokens.get(i).isPunct("["); } private Statement parseStructuredBinding(List leadingComments) { CppLexerToken start = peek(); matchKeyword("const"); expectKeyword("auto"); matchOp("&"); matchOp("&&"); expectPunct("["); List names = new ArrayList<>(); names.add(expectIdentifier().text()); while (matchPunct(",")) names.add(expectIdentifier().text()); expectPunct("]"); expectOp("="); Expr initializer = parseExpr(); expectPunct(";"); String bindingName = "[" + String.join(", ", names) + "]"; return new DeclStatement(new NamedType("auto", List.of(), 0, false, false, false), bindingName, List.of(), initializer, false, false, start.line(), start.col(), leadingComments); } private boolean looksLikeTemplatedConstructionCallee() { int save = pos; try { try { parseTemplateArgList(); } catch (ParseException e) { return false; } // Paren-init / brace-init (construction call) if (checkPunct("(") || checkPunct("{")) return true; // Static member access: "std::is_pointer::value" if (checkPunct("::")) return true; // Variable template: "pi" followed by ; , ) = etc. if (checkPunct(";") || checkPunct(",") || checkPunct(")") || checkPunct("]") || checkOp("=") || isAtEnd()) return true; return false; } finally { pos = save; } } private Expr parseInitializerList() { CppLexerToken start = expectPunct("{"); List elements = new ArrayList<>(); if (!checkPunct("}")) { elements.add(parseInitializerElement()); while (matchPunct(",")) { if (checkPunct("}")) break; // trailing comma elements.add(parseInitializerElement()); } } expectPunct("}"); return new InitializerListExpr(elements, start.line(), start.col(), List.of()); } /** * One element of a brace-init list. Can be: * - A nested brace-init: { 1, 2 } * - A named brace-init: Inner{1, 2} (parsed as expr then brace-init) * - Any other expression: a + b, func(), etc. */ private Expr parseInitializerElement() { if (checkPunct("{")) { return parseInitializerList(); // anonymous nested brace-init } // Designated initializer: ".field = expr" (C++20) if (checkPunct(".") && pos + 1 < tokens.size() && tokens.get(pos + 1).type() == CppLexerTokenType.IDENTIFIER && pos + 2 < tokens.size() && tokens.get(pos + 2).isOp("=")) { advance(); // consume "." advance(); // consume field name advance(); // consume "=" } Expr e = parseExpr(); if (matchPunct("...")) // pack expansion in brace-init: "{args...}" e = new PostfixExpr("...", e, e.line(), e.col(), List.of()); return e; } /** * Parses a lambda: "[captures](params) [-> ReturnType] { body }". * Capture forms confirmed by the corpus: "[]" (none), "[x]" (by value), * "[&x]" (by reference). */ private Expr parseLambda() { CppLexerToken start = expectPunct("["); List captures = new ArrayList<>(); if (!checkPunct("]")) { captures.add(parseCapture()); while (matchPunct(",")) { captures.add(parseCapture()); } } expectPunct("]"); // C++20 template lambda: [](T val) { ... } if (checkOp("<")) { StringBuilder _tp = new StringBuilder(); int _td = 1; advance(); while (!isAtEnd() && _td > 0) { if (checkOp("<")) { _td++; _tp.append(peek().text()); advance(); } else if (checkOp(">>")) { _td-=2; if(_td<=0)break; _tp.append(peek().text()); advance(); } else if (checkOp(">")) { _td--; if (_td==0) break; _tp.append(peek().text()); advance(); } else { if(_tp.length()>0 && !_tp.toString().endsWith("<") && !peek().text().equals(">") && !peek().text().equals(",")) _tp.append(" "); _tp.append(peek().text()); advance(); } } if (!isAtEnd()) advance(); // consume final > captures.add(0, new Capture("__tmpl__<" + _tp.toString() + ">", false)); } List params = new ArrayList<>(); if (checkPunct("(")) { advance(); if (!checkPunct(")")) { params.add(parseParam()); while (matchPunct(",")) { params.add(parseParam()); } } expectPunct(")"); } // mutable, noexcept, attributes boolean isMutable = matchKeyword("mutable"); matchKeyword("constexpr"); matchKeyword("consteval"); if (checkKeyword("noexcept")) { advance(); if(checkPunct("(")){advance();int _d=1;while(!isAtEnd()&&_d>0){if(checkPunct("("))_d++;else if(checkPunct(")"))_d--;advance();}} } consumeAttributes(); TypeRef returnType = null; if (matchPunct("->")) { returnType = parseTypeRef(); } if (check(CppLexerTokenType.IDENTIFIER) && peek().text().equals("requires")) { advance(); int _rd = 0; while (!isAtEnd()) { if (checkPunct("(") || checkOp("<")) _rd++; else if (checkPunct(")") || checkOp(">")) _rd--; else if (checkOp(">>")) _rd -= 2; else if (checkPunct("{") && _rd == 0) break; advance(); } } Block body = parseBlock(); return new LambdaExpr(captures, params, returnType, isMutable, body, start.line(), start.col(), List.of()); } private Capture parseCapture() { // Capture-ALL forms ("[=]" -- everything by value, "[&]" -- // everything by reference) are real, common, idiomatic lambda // syntax that this method never recognized at all -- it // unconditionally called expectIdentifier(), assuming every // capture is a named identifier, with no check for a bare "=" // or a lone "&" (not followed by a name) first. // // Represented using the EXISTING Capture(name, byRef) shape, no // AST change needed: "[&]" is byRef=true with an empty name // (CodeGen's existing "if (byRef) append('&'); append(name);" // rendering already produces exactly "&" for this with zero // changes); "[=]" is byRef=false with name="=" (renders the // literal "=" token, matching real C++ syntax exactly, since // CodeGen always appends cap.name() verbatim). if (checkOp("=")) { advance(); return new Capture("=", false); } if (checkOp("&") && tokens.get(pos + 1).text().equals("]")) { advance(); return new Capture("", true); } boolean byRef = matchOp("&"); // "this" is a keyword capture: [this] or [&this] if (checkKeyword("this")) { advance(); return new Capture("this", byRef); } // C++20 pack init-capture: "[...vals = expr]" -- "..." precedes the name boolean packPrefix = checkPunct("..."); if (packPrefix) advance(); String name = expectIdentifier().text(); // Pack expansion in capture: "[args...]" if (checkPunct("...")) { advance(); name = name + "..."; } if (packPrefix) name = "..." + name; // Init-capture: "z = z * 2" or "w = x + y" -- encode into name string if (checkOp("=")) { advance(); StringBuilder _ie = new StringBuilder(); int _d = 0; while (!isAtEnd()) { if (checkPunct("(") || checkPunct("[")) { _d++; _ie.append(peek().text()); advance(); } else if (checkPunct(")") || checkPunct("]")) { if (_d == 0) break; _d--; _ie.append(peek().text()); advance(); } else if (checkPunct(",") && _d == 0) break; else { _ie.append(" ").append(peek().text()); advance(); } } name = name + " = " + _ie.toString().trim(); } return new Capture(name, byRef); } /** Parses a single "Type name [= default]" parameter, shared by function decls and lambdas. */ /** * Parses a single "Type name [= default]" parameter, shared by function * decls and lambdas. Also handles the C-style array-parameter form * ("int data[]"), confirmed real by Pie_Chart.pde's * "void pieChart(float diameter, int data[], int length)" -- per real * C++ semantics, an array parameter decays to a pointer, so this is * represented by bumping the TypeRef's pointerDepth by one rather than * adding a separate array-dims field to Param (which would need to mean * something different from -- and easily confusable with -- the fixed- * size array dims already tracked on VariableDecl/DeclStatement). */ private Param parseParam() { // C-style variadic: bare "..." as a parameter (e.g. "void f(int, ...)") if (checkPunct("...")) { advance(); return new Param(new NamedType("...", List.of(), 0, false, false, false), "...", null, List.of(), false); } if (checkKeyword("this")) advance(); // explicit object param (C++23) TypeRef type = parseTypeRef(); boolean isTypePackExpansion = matchPunct("..."); // pack expansion after type: "Rest... rest" // Member pointer parameter: "int Widget::* dp" or "int (Widget::* mp)(int) const" if (check(CppLexerTokenType.IDENTIFIER) && pos + 1 < tokens.size() && tokens.get(pos + 1).isPunct("::") && pos + 2 < tokens.size() && tokens.get(pos + 2).isOp("*")) { String className = advance().text(); // Class name advance(); // :: advance(); // * String pname = expectIdentifier().text(); // variable name // Member function pointer param: "int (Widget::* mp)(int) const" -- consume parens if (checkPunct("(")) { advance(); // ( int depth = 1; while (!isAtEnd() && depth > 0) { if (checkPunct("(")) depth++; else if (checkPunct(")")) depth--; advance(); } matchKeyword("const"); } return new Param(type, className + "::*" + pname, null, List.of(), false); } // Member function pointer param: "int (Widget::* mp)(int) const" // Detected by: ( IDENTIFIER :: * name ) ( params ) optional-const if (checkPunct("(") && pos + 1 < tokens.size() && tokens.get(pos + 1).type() == CppLexerTokenType.IDENTIFIER && pos + 2 < tokens.size() && tokens.get(pos + 2).isPunct("::") && pos + 3 < tokens.size() && tokens.get(pos + 3).isOp("*")) { advance(); // ( String className = advance().text(); // Widget advance(); // :: advance(); // * String pname = expectIdentifier().text(); // mp expectPunct(")"); // Capture (params) text for encoding StringBuilder paramsSig = new StringBuilder("("); if (checkPunct("(")) { advance(); int depth = 1; List paramTokens = new ArrayList<>(); while (!isAtEnd() && depth > 0) { if (checkPunct("(")) { depth++; paramTokens.add("("); advance(); } else if (checkPunct(")")) { depth--; if (depth > 0) { paramTokens.add(")"); advance(); } else advance(); } else { paramTokens.add(peek().text()); advance(); } } paramsSig.append(String.join(" ", paramTokens)).append(")"); } else { paramsSig.append(")"); } boolean isConstMethod = matchKeyword("const"); String encoded = className + "::*" + pname + paramsSig + (isConstMethod ? "__const__" : ""); return new Param(type, encoded, null, List.of(), false); } // Reference-to-array, pointer-to-array, or function-pointer parameter: // "int (&arr)[10]" -- ref-to-array // "int (*arr)[10]" -- ptr-to-array // "float (*fn)(float)" -- function pointer // "float (*fn)(float) = nullptr" -- function pointer with default // Discriminator: after consuming (*name), if "(" follows it is a function // pointer param list; if "[" follows it is an array dimension. if (checkPunct("(") && pos + 1 < tokens.size() && (tokens.get(pos + 1).isOp("&") || tokens.get(pos + 1).isOp("*")) && pos + 2 < tokens.size() && tokens.get(pos + 2).type() == CppLexerTokenType.IDENTIFIER) { advance(); // consume "(" boolean isRef = matchOp("&"); if (!isRef) matchOp("*"); String pname = expectIdentifier().text(); expectPunct(")"); // Function pointer param: "float (*fn)(float) = nullptr" // "(" immediately after ")" means this is a fn-ptr, not an array. if (checkPunct("(")) { // Capture the param list tokens verbatim for CodeGen encoding. // CodeGen detects "name__fnptr__(sig)" and emits "rettype (*name)(sig)". int sigStart = pos; advance(); int d = 1; List sigToks = new ArrayList<>(); while (!isAtEnd() && d > 0) { if (checkPunct("(")) { d++; sigToks.add("("); advance(); } else if (checkPunct(")")) { d--; if (d > 0) { sigToks.add(")"); advance(); } else advance(); } else { sigToks.add(peek().text()); advance(); } } String paramSig = "(" + String.join(", ", sigToks) + ")"; // Consume optional trailing qualifiers matchKeyword("const"); matchKeyword("noexcept"); // Consume optional default value: "= nullptr", "= identity", etc. Expr defaultValue = null; if (matchOp("=")) defaultValue = parseExpr(); // Encode as "name__fnptr__(sig)" -- CodeGen decodes this in emitParamList. // Type carries the return type; pointerDepth bump marks it as fn-ptr. if (type instanceof NamedType nt) { type = new NamedType(nt.baseName(), nt.templateArgs(), nt.pointerDepth() + 1, false, nt.isConst(), false); } return new Param(type, pname + "__fnptr__" + paramSig, defaultValue, List.of(), false); } // Ref-to-array or ptr-to-array: encode dims into name. // CodeGen uses the "&"/"*" prefix to emit "int (&arr)[10]". StringBuilder dimEnc = new StringBuilder(); while (checkPunct("[")) { int _ds = pos; advance(); dimEnc.append("["); if (!checkPunct("]")) { int _de = pos; parseExpr(); // consume dimension expression for (int _di = _de; _di < pos; _di++) dimEnc.append(tokens.get(_di).text()); } expectPunct("]"); dimEnc.append("]"); } return new Param(type, (isRef ? "&" : "*") + pname + dimEnc.toString(), null, List.of(), false); } // Variadic pack expansion after the type: "Args... args" or "Ts&&... ts" boolean isVariadic = isTypePackExpansion; if (checkPunct("...")) { advance(); isVariadic = true; } String name; if (check(CppLexerTokenType.IDENTIFIER) || (check(CppLexerTokenType.KEYWORD) && PSEUDO_TYPE_KEYWORDS_USABLE_AS_NAMES.contains(peek().text()))) { name = advance().text(); if (checkPunct("...")) { advance(); isVariadic = true; } } else { name = ""; } List innerDims = new ArrayList<>(); if (checkPunct("[")) { boolean first = true; while (checkPunct("[")) { advance(); Integer dimSize = null; if (!checkPunct("]")) { if (peek().type() == CppLexerTokenType.INT_LITERAL) { try { dimSize = Integer.parseInt(peek().text()); } catch (NumberFormatException ignored) {} } parseExpr(); } expectPunct("]"); if (!first) innerDims.add(dimSize != null ? dimSize : 0); first = false; } type = bumpPointerDepth(type); } // Function type parameter: "Widget()" or "int(*)(int)" -- function pointer param // If name is empty and "(" follows, this is a function-type or function-pointer param. // Consume the "(params)" signature and treat as pointer-to-function type. if (name.isEmpty() && checkPunct("(")) { advance(); // consume "(" // Consume optional * for explicit function pointer: "int(*)(int)" matchOp("*"); // Consume optional name inside parens: "int(*name)(int)" if (check(CppLexerTokenType.IDENTIFIER)) advance(); expectPunct(")"); // Consume the actual param list of the function type: "(int, float)" if (checkPunct("(")) { advance(); int d = 1; while (!isAtEnd() && d > 0) { if (checkPunct("(")) d++; else if (checkPunct(")")) d--; advance(); } } // Represent as a pointer type if (type instanceof NamedType nt) { type = new NamedType(nt.baseName(), nt.templateArgs(), nt.pointerDepth() + 1, false, nt.isConst(), false); } } Expr defaultValue = null; if (matchOp("=")) { defaultValue = parseExpr(); } return new Param(type, name, defaultValue, innerDims, isVariadic); } private TypeRef bumpPointerDepth(TypeRef type) { if (type instanceof NamedType nt) { return new NamedType(nt.baseName(), nt.templateArgs(), nt.pointerDepth() + 1, nt.isReference(), nt.isConst(), false); } // Function-pointer/function-signature types can't sensibly gain a // C-style array-parameter pointer bump this way; not encountered by // any corpus fixture in this position, so left as-is rather than // guessing at a shape nothing has confirmed yet. return type; } // ----- Statements ------------------------------------------------------- /** * Parses a brace-delimited block. Handles multi-declarator desugaring * directly here (e.g. "int boxx, boxy;" inside a method body, confirmed * present in the Handles fixture): a single source statement with N * comma-separated declarators becomes N consecutive DeclStatement nodes * in this block's statement list. This is the right place to do the * splicing since Block is the only structure that owns a List * it can freely expand -- parseStatement itself can only return one node. */ private Block parseBlock() { CppLexerToken start = expectPunct("{"); List statements = new ArrayList<>(); while (!checkPunct("}")) { if (isAtEnd()) { throw error("unexpected end of input while looking for closing '}'"); } List comments = consumeLeadingComments(); if (checkPunct("}")) break; statements.addAll(parseStatementOrMultiDecl(comments)); } expectPunct("}"); return new Block(statements, start.line(), start.col(), List.of()); } /** * Returns one or more statements: more than one only when this was a * multi-declarator local declaration ("int x, y;"), which desugars into * one DeclStatement per declarator, all sharing the same TypeRef. */ private List parseStatementOrMultiDecl(List leadingComments) { if (isStructuredBindingStart()) return List.of(parseStructuredBinding(leadingComments)); // Local struct/class definition: "struct Point { float x, y; }; Point p{...}" // Also handle forward decl: "struct LocalFwd;" -- consume and skip if ((checkKeyword("struct") || checkKeyword("class")) && pos + 1 < tokens.size() && (tokens.get(pos + 1).type() == CppLexerTokenType.IDENTIFIER || tokens.get(pos + 1).isPunct("{"))) { // Forward declaration: "struct Foo;" -- just consume and emit empty if (tokens.get(pos + 1).type() == CppLexerTokenType.IDENTIFIER && pos + 2 < tokens.size() && tokens.get(pos + 2).isPunct(";")) { CppLexerToken sk = peek(); advance(); advance(); advance(); // struct Name ; return List.of(new ExprStatement( new Identifier("", sk.line(), sk.col(), List.of()), sk.line(), sk.col(), leadingComments)); } TypeDef td = parseTypeDef(leadingComments, List.of()); matchPunct(";"); // Emit struct as verbatim text via CodeGen String structCode = processing.mode.cpp.CodeGen.generateNode(td, 0).stripTrailing(); ExprStatement structStmt = new ExprStatement( new Identifier(structCode, td.line(), td.col(), List.of()), td.line(), td.col(), leadingComments); // If a variable declaration follows, parse it too if (looksLikeDeclaration()) { List result = new java.util.ArrayList<>(); result.add(structStmt); result.addAll(parseDeclStatementsDesugared(List.of())); return result; } return List.of(structStmt); } // "using T = Type;" local alias -- consume verbatim as ExprStatement if (checkKeyword("using") && pos + 1 < tokens.size() && tokens.get(pos + 1).type() == CppLexerTokenType.IDENTIFIER && pos + 2 < tokens.size() && tokens.get(pos + 2).isOp("=")) { int _us = pos; while (!isAtEnd() && !checkPunct(";")) advance(); matchPunct(";"); StringBuilder _ur = new StringBuilder(); for (int _i = _us; _i < pos - 1; _i++) { if (_i > _us) _ur.append(" "); _ur.append(tokens.get(_i).text()); } _ur.append(";"); return List.of(new ExprStatement(new Identifier(_ur.toString(), tokens.get(_us).line(), tokens.get(_us).col(), List.of()), tokens.get(_us).line(), tokens.get(_us).col(), leadingComments)); } if (looksLikeDeclaration()) { return new ArrayList(parseDeclStatementsDesugared(leadingComments)); } return List.of(parseStatement(leadingComments)); } /** * Dispatches on lookahead to the correct single-statement parser. * Declarations are handled by the caller (parseStatementOrMultiDecl) * before this is reached in the common case, but this method still * handles the declaration form too for callers that only ever expect * exactly one declarator (if/for/while bodies that are a single * statement with no braces, e.g. "if (x) int y = 1;" -- unusual but * grammatically legal, and not distinguishable from the multi-decl case * until we've already looked). */ private Statement parseStatement(List leadingComments) { consumeAttributes(); if (checkPunct(";")) { CppLexerToken t = advance(); return new ExprStatement(new Literal(Literal.Kind.INT, "0", t.line(), t.col(), List.of()), t.line(), t.col(), leadingComments); } if (checkPunct("{")) { Block b = parseBlock(); return new Block(b.statements(), b.line(), b.col(), leadingComments); } // "if constexpr" -- consume constexpr before dispatching to parseIf if (checkKeyword("constexpr") && pos + 1 < tokens.size() && tokens.get(pos + 1).isKeyword("if")) { advance(); // consume constexpr, leave "if" for parseIf } // "typename T::member;" inside requires body -- type validity assertion if (checkKeyword("typename")) { CppLexerToken _t0 = peek(); advance(); try { parseTypeRef(); } catch (ParseException _e) {} matchPunct(";"); return new ExprStatement(new Identifier("typename", _t0.line(), _t0.col(), List.of()), _t0.line(), _t0.col(), leadingComments); } // static_assert: consume entirely and emit as-is if (checkKeyword("static_assert")) { int startPos = pos; advance(); // consume static_assert expectPunct("("); int d=1; while (!isAtEnd() && d > 0) { if (checkPunct("(")) d++; else if (checkPunct(")")) d--; advance(); } matchPunct(";"); StringBuilder raw = new StringBuilder(); for (int i = startPos; i < pos; i++) { if (i > startPos) raw.append(" "); raw.append(tokens.get(i).text()); } if (!raw.toString().endsWith(";")) raw.append(";"); CppLexerToken t0 = tokens.get(startPos); return new ExprStatement(new Identifier(raw.toString(), t0.line(), t0.col(), List.of()), t0.line(), t0.col(), leadingComments); } // throw statement: "throw expr;" or bare "throw;" if (checkKeyword("throw")) { CppLexerToken t0 = advance(); if (checkPunct(";")) { advance(); return new ReturnStatement(null, t0.line(), t0.col(), leadingComments); } Expr val = parseExpr(); expectPunct(";"); // Encode as ReturnStatement with a UnaryExpr("throw", val) -- CodeGen emits via renderExpr return new ExprStatement(new UnaryExpr("throw ", val, t0.line(), t0.col(), List.of()), t0.line(), t0.col(), leadingComments); } // Local struct/class definition inside a function body if ((checkKeyword("struct") || checkKeyword("class")) && pos + 1 < tokens.size() && (tokens.get(pos + 1).type() == CppLexerTokenType.IDENTIFIER || tokens.get(pos + 1).isPunct("{"))) { TypeDef td = parseTypeDef(leadingComments, List.of()); matchPunct(";"); return new ExprStatement(new Identifier("", td.line(), td.col(), List.of()), td.line(), td.col(), leadingComments); } if (checkKeyword("if")) return parseIf(leadingComments); if (checkKeyword("for")) return parseForOrRangeFor(leadingComments); if (checkKeyword("while")) return parseWhile(leadingComments); if (checkKeyword("do")) return parseDoWhile(leadingComments); if (checkKeyword("switch")) return parseSwitch(leadingComments); if (checkKeyword("using") && pos + 1 < tokens.size() && tokens.get(pos + 1).isKeyword("enum")) { int startPos = pos; while (!isAtEnd() && !checkPunct(";")) advance(); matchPunct(";"); StringBuilder raw = new StringBuilder(); for (int i = startPos; i < pos - 1; i++) { if (i > startPos) raw.append(" "); raw.append(tokens.get(i).text()); } raw.append(";"); return new ExprStatement(new Identifier(raw.toString(), tokens.get(startPos).line(), tokens.get(startPos).col(), List.of()), tokens.get(startPos).line(), tokens.get(startPos).col(), leadingComments); } if (checkKeyword("return")) return parseReturn(leadingComments); if (checkKeyword("break")) return parseBreak(leadingComments); if (checkKeyword("continue")) return parseContinue(leadingComments); if (checkKeyword("try")) return parseTry(leadingComments); if (checkKeyword("delete")) return parseDelete(leadingComments); if (looksLikeDeclaration()) { List decls = parseDeclStatementsDesugared(leadingComments); if (decls.size() > 1) { throw error("multiple comma-separated declarators are not supported in a single-statement context (e.g. directly inside an 'if'/'for'/'while' with no braces)"); } return decls.get(0); } return parseExprStatement(leadingComments); } private Statement parseIf(List leadingComments) { CppLexerToken start = expectKeyword("if"); boolean isConstexpr = matchKeyword("constexpr"); // C++17 if constexpr if (checkKeyword("consteval") || (checkOp("!") && pos + 1 < tokens.size() && tokens.get(pos+1).isKeyword("consteval"))) { if (checkOp("!")) advance(); advance(); // consume consteval Statement thenBr = parseStatement(consumeLeadingComments()); Statement elseBr = null; consumeLeadingComments(); if (checkKeyword("else")) { advance(); elseBr = parseStatement(consumeLeadingComments()); } return new IfStatement(new Identifier("true", start.line(), start.col(), List.of()), thenBr, elseBr, false, start.line(), start.col(), leadingComments); } expectPunct("("); // C++17 if-init-statement: if (init; cond) -- collect init decls, // wrap the IfStatement in a Block so the init vars are in scope. java.util.List initStmts = new ArrayList<>(); Expr cond; { int scan=pos,depth=0; boolean hasInit=false; while(scan leadingComments) { CppLexerToken start = expectKeyword("for"); expectPunct("("); // Structured binding range-for: "for (const auto& [k, v] : m)" if (isStructuredBindingStart()) { matchKeyword("const"); expectKeyword("auto"); boolean isRef = matchOp("&") || matchOp("&&"); expectPunct("["); List sbNames = new ArrayList<>(); sbNames.add(expectIdentifier().text()); while (matchPunct(",")) sbNames.add(expectIdentifier().text()); expectPunct("]"); expectPunct(":"); Expr sbIterable = parseExpr(); expectPunct(")"); Statement sbBody = parseStatement(consumeLeadingComments()); String sbName = "[" + String.join(", ", sbNames) + "]"; return new RangeForStatement(new NamedType("auto",List.of(),0,false,false, false), sbName, isRef, sbIterable, sbBody, start.line(), start.col(), leadingComments); } if (looksLikeRangeFor()) { TypeRef declType = parseTypeRef(); boolean isReference = false; if (declType instanceof NamedType nt && nt.isReference()) { isReference = true; declType = new NamedType(nt.baseName(), nt.templateArgs(), nt.pointerDepth(), false, nt.isConst(), false); } String declName = expectIdentifier().text(); expectPunct(":"); Expr iterable = parseExpr(); expectPunct(")"); Statement body = parseStatement(consumeLeadingComments()); return new RangeForStatement(declType, declName, isReference, iterable, body, start.line(), start.col(), leadingComments); } Statement init = null; if (!checkPunct(";")) { if (looksLikeDeclaration()) { List decls = parseDeclStatementsDesugared(List.of()); // Wrap multiple declarators ("int i=0, j=10") in a block for the init slot if (decls.size() == 1) { init = decls.get(0); } else { // Emit as a sequence of decl statements wrapped in a synthetic block List stmts = new ArrayList<>(decls); init = new Block(stmts, decls.get(0).line(), decls.get(0).col(), List.of()); } } else { CppLexerToken t = peek(); Expr e = parseExpr(); expectPunct(";"); init = new ExprStatement(e, t.line(), t.col(), List.of()); } } else { advance(); // consume the bare ';' } Expr cond = checkPunct(";") ? null : parseExpr(); expectPunct(";"); // For-loop update may be comma-separated: "i++, j--" Expr update = null; if (!checkPunct(")")) { update = parseExpr(); while (matchPunct(",")) { CppLexerToken ct = peek(); Expr next = parseExpr(); update = new BinaryExpr(",", update, next, ct.line(), ct.col(), List.of()); } } expectPunct(")"); Statement body = parseStatement(consumeLeadingComments()); return new ForStatement(init, cond, update, body, start.line(), start.col(), leadingComments); } /** * Lookahead-only: from just after "for (", determine whether this is a * range-for by tentatively parsing a TypeRef + identifier and checking * whether ':' (not ';') follows. Restores position regardless. */ private boolean looksLikeRangeFor() { int save = pos; try { if (!(check(CppLexerTokenType.IDENTIFIER) || check(CppLexerTokenType.KEYWORD))) return false; try { parseTypeRef(); } catch (ParseException e) { return false; } if (!check(CppLexerTokenType.IDENTIFIER) && !(check(CppLexerTokenType.KEYWORD) && PSEUDO_TYPE_KEYWORDS_USABLE_AS_NAMES.contains(peek().text()))) { return false; } advance(); return checkPunct(":"); } finally { pos = save; } } private Statement parseWhile(List leadingComments) { CppLexerToken start = expectKeyword("while"); expectPunct("("); // C++17 while with condition declaration: "while (auto val = expr)" Expr cond; if (looksLikeDeclaration()) { // Scan to find the = and parse the initializer expression while (!isAtEnd() && !checkOp("=") && !checkPunct(")")) advance(); if (matchOp("=")) cond = parseExpr(); else cond = new Identifier("true", start.line(), start.col(), List.of()); } else { cond = parseExpr(); } expectPunct(")"); Statement body = parseStatement(consumeLeadingComments()); return new WhileStatement(cond, body, start.line(), start.col(), leadingComments); } private Statement parseDoWhile(List leadingComments) { CppLexerToken start = expectKeyword("do"); Statement body = parseStatement(consumeLeadingComments()); expectKeyword("while"); expectPunct("("); Expr cond = parseExpr(); expectPunct(")"); expectPunct(";"); return new DoWhileStatement(body, cond, start.line(), start.col(), leadingComments); } private Statement parseSwitch(List leadingComments) { CppLexerToken start = expectKeyword("switch"); expectPunct("("); java.util.List swInitStmts = new ArrayList<>(); {int scan=pos,depth=0;boolean hasInit=false;while(scan cases = new ArrayList<>(); while (!checkPunct("}")) { consumeLeadingComments(); if (checkPunct("}")) break; Expr matchValue; if (matchKeyword("case")) { matchValue = parseExpr(); } else { expectKeyword("default"); matchValue = null; } expectPunct(":"); List body = new ArrayList<>(); while (!checkKeyword("case") && !checkKeyword("default") && !checkPunct("}")) { List comments = consumeLeadingComments(); if (checkKeyword("case") || checkKeyword("default") || checkPunct("}")) break; body.addAll(parseStatementOrMultiDecl(comments)); } cases.add(new SwitchCase(matchValue, body)); } expectPunct("}"); Statement swStmt = new SwitchStatement(subject, cases, start.line(), start.col(), leadingComments); if (!swInitStmts.isEmpty()) { swInitStmts.add(swStmt); return new Block(swInitStmts, start.line(), start.col(), leadingComments); } return swStmt; } private Statement parseReturn(List leadingComments) { CppLexerToken start = expectKeyword("return"); Expr value = checkPunct(";") ? null : parseExpr(); expectPunct(";"); return new ReturnStatement(value, start.line(), start.col(), leadingComments); } private Statement parseBreak(List leadingComments) { CppLexerToken start = expectKeyword("break"); expectPunct(";"); return new BreakStatement(start.line(), start.col(), leadingComments); } private Statement parseContinue(List leadingComments) { CppLexerToken start = expectKeyword("continue"); expectPunct(";"); return new ContinueStatement(start.line(), start.col(), leadingComments); } private Statement parseTry(List leadingComments) { CppLexerToken start = expectKeyword("try"); Block tryBlock = parseBlock(); List clauses = new ArrayList<>(); while (matchKeyword("catch")) { expectPunct("("); CatchClause clause; if (matchPunct("...")) { expectPunct(")"); Block body = parseBlock(); clause = new CatchClause(null, null, body, true); } else { TypeRef exType = parseTypeRef(); String varName = check(CppLexerTokenType.IDENTIFIER) ? expectIdentifier().text() : null; expectPunct(")"); Block body = parseBlock(); clause = new CatchClause(exType, varName, body, false); } clauses.add(clause); } return new TryStatement(tryBlock, clauses, start.line(), start.col(), leadingComments); } private Statement parseDelete(List leadingComments) { CppLexerToken start = expectKeyword("delete"); boolean isArray = false; if (checkPunct("[")) { advance(); expectPunct("]"); isArray = true; } Expr target = parseExpr(); expectPunct(";"); return new DeleteStatement(target, isArray, start.line(), start.col(), leadingComments); } private Statement parseExprStatement(List leadingComments) { CppLexerToken start = peek(); Expr expr = parseExpr(); expectPunct(";"); return new ExprStatement(expr, start.line(), start.col(), leadingComments); } /** * Lookahead-only: determines whether the upcoming tokens form a variable * declaration ("Type name ...") rather than an expression-statement. * Tries to parse a TypeRef followed by an identifier; if that succeeds * and is followed by '=', ';', '[', ',', or '(' (direct-init), it's * treated as a declaration. Restores position regardless of outcome. * * This disambiguation is necessary because both "int x = 5;" * (declaration) and "x = 5;" (plain assignment expression statement) * start with an identifier-shaped token; the grammar must decide which * without a symbol table -- same category of ambiguity documented on * CallExpr for bare construction calls. */ /** Keywords that introduce a statement form and can NEVER be the start * of a type name. Checked first and unconditionally in * looksLikeDeclaration, because parseTypeRef/parseQualifiedTypeName * have no concept of "which keywords are valid type names" -- they * accept ANY KEYWORD-or-IDENTIFIER token as a type name segment (this * is intentional and correct for real type keywords like "int", * "auto", "color", etc., see parseTypeNameSegment's notes -- the bug * is that nothing ever excluded the OTHER kind of keyword, the ones * that start an entirely different statement form). * * Found via PipelineCompositionTest's real g++ check, not by the * parser's own corpus sweep: "return foo(1, 2);" was being parsed as * a DeclStatement (treating "return" as a bogus type name, "foo" as * the declared name, and "(1, 2)" as a direct-init argument list), * not a ReturnStatement -- producing a syntactically-different-but- * still-valid-shaped AST that round-tripped through codegen as * "return foo{1, 2};" with no parse exception anywhere, so the * existing "did this fail to parse" corpus sweep never caught it. * Confirmed this affected real corpus files too (Handles.pde, * Scrollbar.pde) once specifically checked for, not just the * synthetic case that surfaced it. */ private static final java.util.Set STATEMENT_KEYWORDS = java.util.Set.of( "return", "if", "for", "while", "do", "switch", "break", "continue", "try", "delete", "case", "default", "else", "throw" ); /** Keyword type names that are valid cast-operator targets ("operator int()", * "operator bool()", etc.). Kept explicit and narrow to prevent arbitrary * keywords (like Processing's "color", or "auto", "return", etc.) from * being misidentified as cast-operator type names in parseFunctionOrVariableName. */ private static final java.util.Set CAST_OPERATOR_TYPE_KEYWORDS = java.util.Set.of( "int", "float", "double", "bool", "char", "long", "short", "unsigned", "signed", "void", "size_t" ); private boolean looksLikeDeclaration() { int save = pos; try { if (check(CppLexerTokenType.KEYWORD) && STATEMENT_KEYWORDS.contains(peek().text())) return false; // Tolerate a leading "static"/"const" (in either order) before // attempting to parse a type -- mirrors the actual parse path // in parseDeclStatementsDesugared, which consumes these same // qualifiers before calling parseTypeRef(). Without this, // "static int x = 5;" as a LOCAL variable was rejected right // here (parseTypeRef has no concept of consuming a leading // qualifier itself, so it choked on the literal token // "static"), before ever reaching parseDeclStatementsDesugared's // own (correct, but unreachable without this fix) handling of // that same qualifier. if (checkKeyword("static")) advance(); if (checkKeyword("volatile")) advance(); if (checkKeyword("const")) advance(); if (checkKeyword("volatile")) advance(); // vol // alignas specifier: "alignas(T) type name" if (checkKeyword("alignas") && pos + 1 < tokens.size() && tokens.get(pos + 1).isPunct("(")) { advance(); advance(); int _ad=1; while (!isAtEnd() && _ad > 0) { if (checkPunct("(")) _ad++; else if (checkPunct(")")) _ad--; advance(); } } // volatile after const if (checkKeyword("constexpr")) advance(); if (checkKeyword("inline")) advance(); if (checkKeyword("static")) advance(); // tolerate either order if (!(check(CppLexerTokenType.IDENTIFIER) || check(CppLexerTokenType.KEYWORD))) return false; // Function-pointer-variable form: "Type (*name)(Params) = init;" // Must be checked before the ordinary TypeRef+identifier path, // since after the return type this shape has '(' where an // ordinary declarator would have an identifier. if (looksLikeFunctionPointerVariable()) return true; TypeRef type; try { type = parseTypeRef(); } catch (ParseException e) { return false; } if (!check(CppLexerTokenType.IDENTIFIER) && !(check(CppLexerTokenType.KEYWORD) && PSEUDO_TYPE_KEYWORDS_USABLE_AS_NAMES.contains(peek().text()))) { return false; } advance(); // tentatively consume the name // Member data pointer: "int Point::* dp" -- ::* follows the class name if (checkPunct("::") && pos + 1 < tokens.size() && tokens.get(pos + 1).isOp("*")) return true; return checkOp("=") || checkPunct(";") || checkPunct("[") || checkPunct(",") || checkPunct("(") || checkPunct("{"); } finally { pos = save; } } /** * Parses "Type name1 [=init1|dims1], name2 [=init2|dims2], ...;" and * desugars it into one DeclStatement per declarator, all sharing the * same TypeRef instance. This is the single implementation backing * every declaration-statement call site (block-level, for-init, * single-statement contexts) -- callers that require exactly one * declarator check decls.size() themselves and raise a clear error, * rather than this method silently dropping extras. */ private List parseDeclStatementsDesugared(List leadingComments) { CppLexerToken start = peek(); // BUG FIX, found via adversarial stress-case probing: "static" // (and "const") were never consumed here at all -- only the // TOP-LEVEL declaration path consumed them. "static int x = 5;" // as a LOCAL variable failed to parse outright ("expected ';' // but found 'int'", since "static" was left sitting in the // token stream as if it were the start of an unrelated // statement, then "int" showed up where a ";" was expected). // Tolerate either order ("static const" or "const static"), // mirroring the top-level path's own tolerance. // alignas specifier: "alignas(T) type name" if (checkKeyword("alignas") && pos + 1 < tokens.size() && tokens.get(pos + 1).isPunct("(")) { advance(); advance(); int _ad=1; while (!isAtEnd() && _ad > 0) { if (checkPunct("(")) _ad++; else if (checkPunct(")")) _ad--; advance(); } } boolean isStatic = matchKeyword("static"); matchKeyword("volatile"); // consume volatile qualifier at statement scope boolean isConst = matchKeyword("const"); if (matchKeyword("constexpr")) isConst = true; // constexpr implies const if (!isStatic) isStatic = matchKeyword("static"); if (!isConst) isConst = matchKeyword("const"); matchKeyword("volatile"); // volatile may follow const if (looksLikeFunctionPointerVariable()) { TypeRef returnType = parseTypeRef(); NameAndFunctionPointerType decl = parseFunctionPointerDeclaratorTail(returnType); Expr initializer = null; if (matchOp("=")) { initializer = parseExpr(); } expectPunct(";"); return List.of(new DeclStatement(decl.type(), decl.name(), List.of(), initializer, isStatic, isConst, start.line(), start.col(), leadingComments)); } TypeRef type = parseTypeRef(); List result = new ArrayList<>(); result.add(parseOneDeclarator(type, isStatic, isConst, start, leadingComments)); while (matchPunct(",")) { // Same gap, same fix, as parseOneTopLevelDeclarator's comma- // continuation loop -- see its notes for the full rationale, // including why each declarator's type must be built from // the base type's name/templateArgs/const ONLY, never // reusing "type"'s own pointerDepth/isReference (which // belongs exclusively to the first declarator). int extraPointerDepth = 0; boolean extraIsReference = false; while (checkOp("*") || checkOp("&")) { if (matchOp("*")) extraPointerDepth++; else { matchOp("&"); extraIsReference = true; } } TypeRef declaratorType = type; if (type instanceof NamedType nt) { declaratorType = new NamedType(nt.baseName(), nt.templateArgs(), extraPointerDepth, extraIsReference, nt.isConst(), false); } result.add(parseOneDeclarator(declaratorType, isStatic, isConst, start, List.of())); } expectPunct(";"); return result; } private DeclStatement parseOneDeclarator(TypeRef type, boolean isStatic, boolean isConst, CppLexerToken start, List leadingComments) { String name = expectIdentifier().text(); if (checkPunct("::") && pos + 1 < tokens.size() && tokens.get(pos + 1).isOp("*")) { advance(); advance(); name = name + "::* " + expectIdentifier().text(); } List dims = parseOptionalArrayDims(); Expr initializer = null; if (matchOp("=")) { if (checkPunct("{")) { initializer = parseInitializerList(); } else { initializer = parseExpr(); } } else if (checkPunct("(")) { initializer = parseDirectInitAsCall(name); } else if (checkPunct("{")) { // Brace-direct-init at statement scope, same rationale as the // top-level declarator case -- see parseOneTopLevelDeclarator's // notes. initializer = parseDirectInitAsCall(name); } return new DeclStatement(type, name, dims, initializer, isStatic, isConst, start.line(), start.col(), leadingComments); } /** * Parses zero or more "[expr]" / "[]" array-dimension suffixes after * a declarator name. * * BUG FIX, found via RealHeaderStressTest against the real engine * headers (Pie_Chart.pde's real "int angles[] = { 30, 10, 45, ... };"): * an EMPTY bracket pair (no size expression, size inferred from the * initializer -- valid, common C++/Java-array syntax) must still be * recorded as a real dimension, just with a null size expression -- * NOT silently dropped as if no brackets were present at all. The * previous version of this method only ever called dims.add(...) * inside the "if (!checkPunct(\"]\"))" branch, meaning an empty "[]" * added NOTHING to the dims list at all, making * "int angles[] = {...}" structurally indistinguishable from * "int angles = {...}" (a scalar) once parsed -- confirmed directly: * vd.arrayDims() was an empty list, not a list containing one null * entry, for "int angles[] = { 30, 10, 45 };". This silently turned * a real top-level array declaration into a scalar declaration with * a brace-init initializer, which g++ correctly rejects ("cannot * convert '' to 'int'"). * * CodeGen.emitArrayDims already handled a null dim entry correctly * (rendering bare "[]" when dim is null) -- this representation was * already anticipated and supported on the rendering side; the * parser simply never produced it. */ private List parseOptionalArrayDims() { List dims = new ArrayList<>(); while (checkPunct("[")) { advance(); if (!checkPunct("]")) { dims.add(parseExpr()); } else { dims.add(null); // empty "[]" -- still a real dimension, size inferred from initializer } expectPunct("]"); } return dims; } /** * Handles C++'s "most vexing parse" direct-init form, "Type name(args);", * confirmed real by CppBuild.java's existing OBJECT_DIRECT_INIT handling * in classifyTopLevelDecls. Represented as a CallExpr initializer whose * callee is the variable's own name, consistent with how bare- * construction-call ("Type()") is already represented elsewhere. * * Handles both "Type name(args);" (paren-direct-init) and * "Type name{args};" (brace-direct-init) -- the latter confirmed * necessary so the parser can read back CodeGen's own brace-rendered * output (see CodeGen.emitDeclaratorTail's most-vexing-parse notes). * Both forms produce the identical CallExpr shape; which punctuation * was used in the source is not preserved on the AST, since nothing * downstream has needed that distinction so far -- if it ever does, * this is the place to add it. */ private Expr parseDirectInitAsCall(String varName) { CppLexerToken t = peek(); boolean isBrace = checkPunct("{"); List args = isBrace ? parseBraceArgList() : parseArgList(); return new CallExpr(new Identifier(varName, t.line(), t.col(), List.of()), args, isBrace, t.line(), t.col(), List.of()); } private List parseBraceArgList() { expectPunct("{"); List args = new ArrayList<>(); if (!checkPunct("}")) { args.add(parseExpr()); while (matchPunct(",")) { args.add(parseExpr()); } } expectPunct("}"); return args; } }