Skip to content
Open
3 changes: 2 additions & 1 deletion core/src/main/java/org/jruby/ast/LineStubVisitor.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import org.jruby.Ruby;
import org.jruby.RubyArray;
import org.jruby.RubyFixnum;
import org.jruby.ast.util.LineEvents;
import org.jruby.ast.visitor.AbstractNodeVisitor;

public class LineStubVisitor extends AbstractNodeVisitor {
Expand All @@ -15,7 +16,7 @@ public LineStubVisitor(Ruby runtime, RubyArray lines) {
}
@Override
protected Object defaultVisit(Node node) {
if (node.isNewline()) lines.set(node.getLine() + 1, RubyFixnum.newFixnum(runtime, 0));
if (node.isNewline()) lines.set(LineEvents.lineOf(node) + 1, RubyFixnum.newFixnum(runtime, 0));

for (Node child: node.childNodes()) {
if (child != null) defaultVisit(child);
Expand Down
154 changes: 154 additions & 0 deletions core/src/main/java/org/jruby/ast/util/LineEvents.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
package org.jruby.ast.util;

import org.jruby.ast.*;
import org.jruby.ir.builder.StringStyle;
import org.jruby.util.KeyValuePair;

import java.util.List;

/**
* Where MRI reports a statement's line event, and so counts its line coverage: on the line of the statement's first
* instruction. That is a later line whenever evaluating the statement starts with a part of it on one, as in
* <code>x =\n y.size</code> (line 2) or an assignment of a heredoc whose body starts with an interpolation.
* The parser marks this line as coverable (as does Coverage.line_stub) and the IR builder counts the statement on it.
*/
public final class LineEvents {
private LineEvents() {}

public static int lineOf(Node statement) {
return firstInstruction(statement).getLine();
}

/**
* The node whose instructions come first in statement's, on the line of the statement's line event. A statement
* inside another with the same first instruction shares its line event.
*/
public static Node firstInstruction(Node statement) {
Node node = statement;

for (Node first = firstEvaluated(node); first != null && first.getLine() >= 0; first = firstEvaluated(node)) {
node = first;
}

return node;
}

/**
* The part of node evaluated first, when its instructions start with that part's instead of one of its own.
*/
private static Node firstEvaluated(Node node) {
return switch (node) {
case BeginNode begin -> begin.getBodyNode();
case BlockNode block -> block.size() > 0 ? block.get(0) : null;
case RescueNode rescue -> rescue.getBodyNode();
case EnsureNode ensure -> ensure.getBodyNode();
case ConstDeclNode decl -> decl.getConstNode() instanceof Colon2Node path && path.getLeftNode() != null ?
path.getLeftNode() : decl.getValueNode();
case MultipleAsgnNode masgn -> firstReceiver(masgn.getPre(), masgn.getValueNode());
case AssignableNode asgn -> asgn.getValueNode();
case AttrAssignNode asgn -> asgn.getReceiverNode();
case OpAsgnNode asgn -> asgn.getReceiverNode();
case OpElementAsgnNode asgn -> asgn.getReceiverNode();
case OpAsgnOrNode asgn -> asgn.getFirstNode();
case OpAsgnAndNode asgn -> asgn.getFirstNode();
case OpAsgnConstDeclNode asgn -> asgn.getFirstNode();
case CallNode call -> call.getReceiverNode();
case Match2Node match -> match.getReceiverNode();
case Match3Node match -> match.getReceiverNode();
case AndNode and -> and.getFirstNode();
case OrNode or -> or.getFirstNode();
case IfNode ifNode -> ifNode.getCondition();
case CaseNode caseNode -> caseNode.getCaseNode() != null ? caseNode.getCaseNode() : firstWhen(caseNode.getCases());
case Colon2Node path -> path.getLeftNode();
case DotNode dot -> dot.getBeginNode();
case SplatNode splat -> splat.getValue();
case ArgsCatNode cat -> cat.getFirstNode();
case ArgsPushNode push -> push.getFirstNode();
case BreakNode breakNode -> breakNode.getValueNode();
case NextNode next -> next.getValueNode();
case ReturnNode ret -> ret.getValueNode();
case EvStrNode str -> str.getBody();
case ArrayNode array -> array.size() > 0 && !isStaticArray(array) ? array.get(0) : null;
case HashNode hash -> firstKeyUnlessStatic(hash.getPairs());
case DXStrNode ignored -> null; // the receiver of ` is self
case DRegexpNode regexp -> regexp.getOnce() ? null : firstInterpolation(regexp);
case DNode str -> firstInterpolation(str);
default -> null;
};
}

// Targets with a receiver have it evaluated before the value.
private static Node firstReceiver(ListNode targets, Node value) {
if (targets != null) {
for (int i = 0; i < targets.size(); i++) {
if (targets.get(i) instanceof AttrAssignNode asgn) return asgn.getReceiverNode();
}
}

return value;
}

private static Node firstWhen(ListNode cases) {
return cases != null && cases.size() > 0 && cases.get(0) instanceof WhenNode when ? when.getExpressionNodes() : null;
}

// An interpolated string starts with its first interpolation's instructions, unless a literal part comes first
// or the interpolation is the only part (then MRI starts by pushing an empty string).
private static Node firstInterpolation(DNode str) {
Node first = null;
int parts = 0;

for (int i = 0; i < str.size(); i++) {
Node part = str.get(i);
if (part instanceof StrNode literal && literal.getValue().realSize() == 0) continue;
if (parts++ == 0) first = part;
}

// MRI folds an interpolated literal string into the parts around it
boolean interpolation = first instanceof EvStrNode evStr && !(evStr.getBody() instanceof StrNode);

return parts > 1 && interpolation ? first : null;
}

// MRI builds an array of only simple literals (with frozen string literals, strings too) as one object, on the
// array's line.
private static boolean isStaticArray(ArrayNode array) {
for (int i = 0; i < array.size(); i++) {
if (!isStaticLiteral(array.get(i)) && !isFrozenString(array.get(i))) return false;
}

return true;
}

// Likewise a hash starting with a pair of simple literals (a string key is frozen, so it counts as one).
private static Node firstKeyUnlessStatic(List<KeyValuePair<Node, Node>> pairs) {
if (pairs.isEmpty()) return null;

Node key = pairs.get(0).getKey();
if (key == null) return null; // **splat

Node value = pairs.get(0).getValue();

return (isStaticLiteral(key) || key instanceof StrNode) && (isStaticLiteral(value) || isFrozenString(value)) ? null : key;
}

private static boolean isFrozenString(Node node) {
return node instanceof StrNode str && str.getStringStyle() == StringStyle.Frozen;
}

private static boolean isStaticLiteral(Node node) {
return node instanceof FixnumNode || node instanceof BignumNode || node instanceof FloatNode ||
node instanceof RationalNode || node instanceof ComplexNode || node instanceof SymbolNode ||
node instanceof NilNode || node instanceof TrueNode || node instanceof FalseNode ||
node instanceof RegexpNode || node instanceof DSymbolNode symbol && !isInterpolated(symbol);
}

// A DSymbolNode need not interpolate ("label": 1).
private static boolean isInterpolated(DNode str) {
for (int i = 0; i < str.size(); i++) {
if (!(str.get(i) instanceof StrNode)) return true;
}

return false;
}
}
2 changes: 1 addition & 1 deletion core/src/main/java/org/jruby/ext/ripper/RipperParser.java
Original file line number Diff line number Diff line change
Expand Up @@ -5708,7 +5708,7 @@ public Object yyparse (RubyLexer yyLex) throws java.io.IOException {
return yyVal;
};
states[635] = (RipperParser p, Object yyVal, ProductionState[] yyVals, int yyTop, int count, int yychar) -> {
yyVal = p.new_regexp(yyVals[yyTop - count + 2].start(), ((IRubyObject)yyVals[-1+yyTop].value), ((IRubyObject)yyVals[0+yyTop].value));
yyVal = p.new_regexp(yyVals[yyTop - count + 1].start(), ((IRubyObject)yyVals[-1+yyTop].value), ((IRubyObject)yyVals[0+yyTop].value));
return yyVal;
};
states[636] = (RipperParser p, Object yyVal, ProductionState[] yyVals, int yyTop, int count, int yychar) -> {
Expand Down
87 changes: 73 additions & 14 deletions core/src/main/java/org/jruby/ir/builder/IRBuilder.java
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,16 @@ enum LineInfo {
}
LineInfo needsLineNumInfo = null;

// How deep the builder is in the tree it builds, and how deep the statement whose line number is pending is.
int buildDepth = 0;
int needsLineNumInfoDepth = 0;
// The line coverage counts the pending statement on. Calls built before its first instruction move
// lastProcessedLineNum, the line backtraces report, but not this one.
int needsLineNumInfoLine = -1;
// With coverage on, the line the last statement that was a line event starts on, and its first instruction's node.
int lastLineEventLine = -1;
U lastLineEventFirstInstruction = null;

// SSS FIXME: Currently only used for retries -- we should be able to eliminate this
// Stack of nested rescue blocks -- this just tracks the start label of the blocks
final Deque<RescueBlockInfo> activeRescueBlockStack = new ArrayDeque<>(4);
Expand Down Expand Up @@ -200,7 +210,12 @@ protected Operand buildEnsureInternal(U body, U elseNode, U[] exceptions, U resc
if (isRescue) ebi.savedGlobalException = savedGlobalException;

// Record body of ensure and push to ensure body stack if there is an actual ensure body.
// It comes after the protected body, so its line events must not be the last ones the protected body sees.
int savedLineEventLine = lastLineEventLine;
U savedLineEventFirstInstruction = lastLineEventFirstInstruction;
Operand ensureRetVal = processEnsureBody(ensureNode, ebi);
lastLineEventLine = savedLineEventLine;
lastLineEventFirstInstruction = savedLineEventFirstInstruction;

// ------------ Build the protected region ------------
activeEnsureBlockStack.push(ebi);
Expand Down Expand Up @@ -370,21 +385,26 @@ protected IRLoop getCurrentLoop() {
return loopStack.peek();
}

public void addInstr(Instr instr) {
if (needsLineNumInfo != null) {
LineInfo type = needsLineNumInfo;
needsLineNumInfo = null;
private void addLineNumInfo() {
LineInfo type = needsLineNumInfo;
needsLineNumInfo = null;

if (type == LineInfo.Coverage) {
addInstr(new LineNumberInstr(lastProcessedLineNum, coverageMode));
} else {
addInstr(manager.newLineNumber(lastProcessedLineNum));
}
if (type == LineInfo.Coverage && coverageMode != 0) {
addInstr(new LineNumberInstr(needsLineNumInfoLine, coverageMode));
if (needsLineNumInfoLine != lastProcessedLineNum) addInstr(manager.newLineNumber(lastProcessedLineNum));
} else if (type == LineInfo.Coverage) {
addInstr(new LineNumberInstr(lastProcessedLineNum, coverageMode));
} else {
addInstr(manager.newLineNumber(lastProcessedLineNum));
}

if (RubyInstanceConfig.FULL_TRACE_ENABLED) {
addInstr(new TraceInstr(RubyEvent.LINE, getCurrentModuleVariable(), methodNameFor(), getFileName(), lastProcessedLineNum + 1));
}
if (RubyInstanceConfig.FULL_TRACE_ENABLED) {
addInstr(new TraceInstr(RubyEvent.LINE, getCurrentModuleVariable(), methodNameFor(), getFileName(), lastProcessedLineNum + 1));
}
}

public void addInstr(Instr instr) {
if (needsLineNumInfo != null) addLineNumInfo();

// If we are building an ensure body, stash the instruction
// in the ensure body's list. If not, add it to the scope directly.
Expand Down Expand Up @@ -2803,7 +2823,7 @@ protected Operand buildSuper(Variable aResult, U iterNode, U argsNode, int line,
int[] flags = new int[] { 0 };
Operand[] args = setupCallArgs(argsNode, flags);

determineIfWeNeedLineNumber(line, isNewline, false, false); // backtrace needs line of call in case of exception.
determineIfWeNeedLineNumberForCall(line, isNewline); // backtrace needs line of call in case of exception.
if ((flags[0] & CALL_KEYWORD_REST) != 0) { // {**k}, {**{}, **k}, etc...
Variable test = addResultInstr(new RuntimeHelperCall(temp(), IS_HASH_EMPTY, new Operand[] { args[args.length - 1] }));
if_else(test, tru(),
Expand Down Expand Up @@ -3154,7 +3174,7 @@ protected Variable createCall(Variable result, Operand receiver, CallType callTy
// check for refinement calls before building any closure
if (callType == FUNCTIONAL) determineIfMaybeRefined(name, args);
Operand block = setupCallClosure(argsNode, iter);
determineIfWeNeedLineNumber(line, isNewline, false, false); // backtrace needs line of call in case of exception.
determineIfWeNeedLineNumberForCall(line, isNewline); // backtrace needs line of call in case of exception.
if ((flags[0] & CALL_KEYWORD_REST) != 0) { // {**k}, {**{}, **k}, etc...
Variable test = addResultInstr(new RuntimeHelperCall(temp(), IS_HASH_EMPTY, new Operand[] { args[args.length - 1] }));
if_else(test, tru(),
Expand All @@ -3172,6 +3192,20 @@ protected Variable createCall(Variable result, Operand receiver, CallType callTy
return result;
}

/**
* A call that is a statement of its own already had its line (and coverage) event emitted when the
* statement started; if building its receiver or arguments moved the current line elsewhere, restore the
* call's line for backtraces without counting the statement a second time.
*/
protected void determineIfWeNeedLineNumberForCall(int line, boolean isNewline) {
if (line != lastProcessedLineNum) {
// A pending coverage event also restores this line for backtraces when it is emitted
if (isNewline && needsLineNumInfo == null) needsLineNumInfo = LineInfo.Backtrace;

lastProcessedLineNum = line;
}
}

protected void determineIfWeNeedLineNumber(int line, boolean isNewline, boolean implicitNil, boolean def) {
if (line != lastProcessedLineNum && !implicitNil) {
LineInfo needsCoverage = isNewline ? LineInfo.Coverage : null;
Expand All @@ -3185,6 +3219,31 @@ protected void determineIfWeNeedLineNumber(int line, boolean isNewline, boolean
}
}

/**
* With coverage on: as in MRI, a statement is a line event unless the last one started on the same line (calls
* built in between do not matter), and coverage counts it on the line of its first instruction, which comes from
* firstInstruction (see LineEvents).
*/
protected void determineIfWeNeedCoverageLine(int line, U firstInstruction) {
// A statement inside the last line event's statement, starting with the same instruction, shares its event
if (line != lastLineEventLine && firstInstruction != lastLineEventFirstInstruction) {
// A statement inside one whose event is still pending would replace that event, which coverage would
// then never count: emit it first.
if (needsLineNumInfo == LineInfo.Coverage && needsLineNumInfoDepth < buildDepth) addLineNumInfo();

needsLineNumInfo = LineInfo.Coverage;
needsLineNumInfoDepth = buildDepth;
needsLineNumInfoLine = getLine(firstInstruction);
lastLineEventLine = line;
lastLineEventFirstInstruction = firstInstruction;
} else {
lastLineEventLine = line;
if (line != lastProcessedLineNum && needsLineNumInfo == null) needsLineNumInfo = LineInfo.Backtrace;
}

lastProcessedLineNum = line;
}

// FIXME: This needs to be called on super/zsuper too
protected void determineIfMaybeRefined(RubySymbol methodName, Operand[] args) {
IRScope outerScope = scope.getNearestTopLocalVariableScope();
Expand Down
11 changes: 10 additions & 1 deletion core/src/main/java/org/jruby/ir/builder/IRBuilderAST.java
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import org.jruby.ast.*;
import org.jruby.ast.types.ILiteralNode;
import org.jruby.ast.types.INameNode;
import org.jruby.ast.util.LineEvents;
import org.jruby.common.IRubyWarnings;
import org.jruby.compiler.NotCompilableException;
import org.jruby.ir.IRClosure;
Expand Down Expand Up @@ -170,7 +171,13 @@ private NotCompilableException notCompilable(String message, Node node) {
}

private Operand buildOperand(Variable result, Node node) throws NotCompilableException {
if (node.isNewline()) determineIfWeNeedLineNumber(node.getLine(), true, node instanceof NilImplicitNode, node instanceof DefNode);
if (node.isNewline()) {
if (coverageMode != 0 && !(node instanceof NilImplicitNode)) {
determineIfWeNeedCoverageLine(node.getLine(), LineEvents.firstInstruction(node));
} else {
determineIfWeNeedLineNumber(node.getLine(), true, node instanceof NilImplicitNode, node instanceof DefNode);
}
}

switch (node.getNodeType()) {
case ALIASNODE: return buildAlias((AliasNode) node);
Expand Down Expand Up @@ -288,6 +295,7 @@ public Operand build(Variable result, Node node) {
if (node == null) return null;

boolean savedExecuteOnce = executesOnce;
buildDepth++;
try {
if (executesOnce) executesOnce = node.executesOnce();

Expand All @@ -299,6 +307,7 @@ public Operand build(Variable result, Node node) {

return operand;
} finally {
buildDepth--;
executesOnce = savedExecuteOnce;
}
}
Expand Down
8 changes: 8 additions & 0 deletions core/src/main/java/org/jruby/lexer/LexingCommon.java
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,14 @@ public int lineno() {
return ruby_sourceline + src.getLineOffset();
}

/**
* The last line read so far. Unlike lineno(), this is past a heredoc that ends the source: lineno() is still on
* the line the heredoc started on, until the lexer reads the next line.
*/
public int lastLineno() {
return Math.max(ruby_sourceline, heredoc_end) + src.getLineOffset();
}

protected void magicCommentEncoding(ByteList encoding) {
if (!comment_at_top()) return;

Expand Down
16 changes: 14 additions & 2 deletions core/src/main/java/org/jruby/lexer/yacc/RubyLexer.java
Original file line number Diff line number Diff line change
Expand Up @@ -368,10 +368,22 @@ public int nextc() {
public void heredoc_dedent(Node root) {
int indent = heredoc_indent;

if (indent <= 0) return;
if (indent > 0) {
heredoc_indent = 0;
dedent(root, indent);
}

heredoc_indent = 0;
// The newline flag only marked where the heredoc's lines start for dedenting; they are not statements.
if (root instanceof StrNode) {
root.unsetNewline();
} else if (root instanceof ListNode list) {
for (int i = 0; i < list.size(); i++) {
if (list.get(i) instanceof StrNode child) child.unsetNewline();
}
}
}

private void dedent(Node root, int indent) {
if (root == null) return;

if (root instanceof StrNode) {
Expand Down
Loading
Loading