Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions core/src/main/java/org/jruby/RubyArgsFile.java
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,7 @@ void setArgs(RubyArray argv) {
inited = false;
this.argv = argv;
this.inPlace = runtime.getFalse();
this.binmode = false; // MRI: a new instance is not in binmode, whatever ARGF.binmode did before
}

public boolean next_argv(ThreadContext context) {
Expand Down
14 changes: 10 additions & 4 deletions core/src/main/java/org/jruby/RubyIO.java
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ public static RubyIO prepStdio(Ruby runtime, InputStream f, Channel c, int fmode
}
}

prepStdioEcflags(fptr, fmode);
setDefaultTextModeEcflags(fptr, fmode);
fptr.stdio_file = f;

// We checkTTY again here because we're using stdout/stdin to indicate this is stdio
Expand All @@ -263,7 +263,7 @@ public static RubyIO prepStdio(Ruby runtime, OutputStream f, Channel c, int fmod
}
}

prepStdioEcflags(fptr, fmode);
setDefaultTextModeEcflags(fptr, fmode);
fptr.stdio_file = f;

return recheckTTY(runtime, fptr, io);
Expand All @@ -276,8 +276,8 @@ private static RubyIO recheckTTY(Ruby runtime, OpenFile fptr, RubyIO io) {
return io;
}

// MRI: part of prep_stdio
private static void prepStdioEcflags(OpenFile fptr, int fmode) {
// MRI: the newline decorators prep_stdio and pipe_open give an IO in the default text mode
private static void setDefaultTextModeEcflags(OpenFile fptr, int fmode) {
boolean locked = fptr.lock();
try {
fptr.encs.ecflags |= EncodingUtils.ECONV_DEFAULT_NEWLINE_DECORATOR;
Expand Down Expand Up @@ -4594,6 +4594,12 @@ private void setupPopen(ThreadContext context, ModeFlags modes, POpenProcess pro
openFile.setMode(modes.getOpenFileFlags() | OpenFile.SYNC);
openFile.setProcess(process);

// MRI: pipe_open; a popen pipe is in the platform's default text mode unless opened in binmode
if (EncodingUtils.DEFAULT_TEXTMODE != 0 && !openFile.isBinmode()) {
openFile.setTextMode();
setDefaultTextModeEcflags(openFile, openFile.getMode());
}

if (openFile.isReadable()) {
Channel inChannel;
if (process.getInput() != null) {
Expand Down
101 changes: 87 additions & 14 deletions core/src/main/java/org/jruby/util/io/OpenFile.java
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
package org.jruby.util.io;

import java.io.Closeable;
import java.io.Console;
import java.io.IOException;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
import java.nio.ByteBuffer;
import java.nio.channels.Channel;
import java.nio.channels.FileChannel;
Expand Down Expand Up @@ -977,20 +981,52 @@ public void finalizeFlush(ThreadContext context, boolean noraise) {

// MRI: NEED_READCONV
public boolean needsReadConversion() {
return Platform.IS_WINDOWS ?
(encs.enc2 != null || (encs.ecflags & ~EConvFlags.CRLF_NEWLINE_DECORATOR) != 0) || isTextMode()
return needsReadConversion(Platform.IS_WINDOWS, encs.enc2, mode, encs.ecflags);
}

// MRI: NEED_READCONV with the platform passed in; crlfEnvironment is MRI's RUBY_CRLF_ENVIRONMENT (Windows)
static boolean needsReadConversion(boolean crlfEnvironment, Encoding enc2, int mode, int ecflags) {
return crlfEnvironment ?
(enc2 != null || (ecflags & ~EConvFlags.CRLF_NEWLINE_DECORATOR) != 0) || (mode & TEXTMODE) != 0
:
(encs.enc2 != null || NEED_NEWLINE_DECORATOR_ON_READ());
(enc2 != null || (mode & TEXTMODE) != 0);
}

// MRI: the ecflags make_readconv opens the read converter with. MRI leaves the default text-mode
// newline conversion (the CRLF marker in ecflags) to the C runtime's O_TEXT descriptors; the JDK has
// no text mode, so the read converter does it here with the universal newline decorator.
static int readConversionFlags(boolean crlfEnvironment, int mode, int ecflags) {
int readFlags = ecflags & ~EConvFlags.NEWLINE_DECORATOR_WRITE_MASK;
if (crlfEnvironment && (mode & TEXTMODE) != 0 && (ecflags & EConvFlags.CRLF_NEWLINE_DECORATOR) != 0) {
readFlags |= EConvFlags.UNIVERSAL_NEWLINE_DECORATOR;
}
return readFlags;
}

// MRI: NEED_WRITECONV
public boolean needsWriteConversion(ThreadContext context) {
Encoding ascii8bit = encodingService(context).getAscii8bitEncoding();

return Platform.IS_WINDOWS ?
((encs.enc != null && encs.enc != ascii8bit) || (encs.ecflags & ((EConvFlags.DECORATOR_MASK & ~EConvFlags.CRLF_NEWLINE_DECORATOR)|EConvFlags.STATEFUL_DECORATOR_MASK)) != 0)
:
((encs.enc != null && encs.enc != ascii8bit) || NEED_NEWLINE_DECORATOR_ON_WRITE() || (encs.ecflags & (EConvFlags.DECORATOR_MASK|EConvFlags.STATEFUL_DECORATOR_MASK)) != 0);
return needsWriteConversion(Platform.IS_WINDOWS, crtTranslatesWrites(), encs.enc, ascii8bit, mode, encs.ecflags);
}

// MRI: rb_w32_write lets the C runtime insert the CRs only on a file or on the process's own stdout
// and stderr; a pipe is written raw even in text mode, so the default CRLF marker alone must not
// select the write converter there.
boolean crtTranslatesWrites() {
return fd != null && (fd.chFile != null || isStdio());
}

// MRI: NEED_WRITECONV with the platform passed in. MRI leaves the CRLF decorator out of its Windows
// mask because the C runtime's O_TEXT descriptors write CRLF; the JDK has no text mode, so the write
// converter must apply it where the C runtime would (crtTranslatesWrites).
static boolean needsWriteConversion(boolean crlfEnvironment, boolean crtTranslatesWrites, Encoding enc, Encoding ascii8bit, int mode, int ecflags) {
if (crlfEnvironment) {
int decorators = crtTranslatesWrites ?
EConvFlags.DECORATOR_MASK : EConvFlags.DECORATOR_MASK & ~EConvFlags.CRLF_NEWLINE_DECORATOR;
return (enc != null && enc != ascii8bit) || (ecflags & (decorators|EConvFlags.STATEFUL_DECORATOR_MASK)) != 0;
}
return (enc != null && enc != ascii8bit) || (mode & TEXTMODE) != 0 || (ecflags & (EConvFlags.DECORATOR_MASK|EConvFlags.STATEFUL_DECORATOR_MASK)) != 0;
}

// MRI: make_readconv
Expand All @@ -999,7 +1035,7 @@ public void makeReadConversion(ThreadContext context, int size) {
int ecflags;
IRubyObject ecopts;
byte[] sname, dname;
ecflags = encs.ecflags & ~EConvFlags.NEWLINE_DECORATOR_WRITE_MASK;
ecflags = readConversionFlags(Platform.IS_WINDOWS, mode, encs.ecflags);
ecopts = encs.ecopts;
if (encs.enc2 != null) {
sname = encs.enc2.getName();
Expand Down Expand Up @@ -2301,23 +2337,36 @@ private void unreadWindows(ThreadContext context) {

// MRI: io_fwrite
public long fwrite(ThreadContext context, RubyString str, boolean nosync) {
// The System.console null check is our poor-man's isatty for Windows. See jruby/jruby#3292
if (Platform.IS_WINDOWS && isStdio() && System.console() != null) {
if (Platform.IS_WINDOWS && isStdio() && stdioIsConsole()) {
return rbW32WriteConsole(str);
}

int requested = str.getByteList().length();
boolean crtNewlines = crtNewlinesOnly(context);
str = doWriteconv(context, str);
ByteList strByteList = str.getByteList();
return binwriteInt(context, strByteList.unsafeBytes(), strByteList.begin(), strByteList.length(), nosync);
long n = binwriteInt(context, strByteList.unsafeBytes(), strByteList.begin(), strByteList.length(), nosync);
// MRI: _write reports the caller's byte count, not the CRs the C runtime inserted
return crtNewlines && n == strByteList.length() ? requested : n;
}

// Windows text mode where the default CRLF marker is the only reason for a write converter: MRI
// leaves those CRs to the C runtime, whose _write does not count them in its return value.
private boolean crtNewlinesOnly(ThreadContext context) {
if (!Platform.IS_WINDOWS || !crtTranslatesWrites()) return false;
Encoding ascii8bit = encodingService(context).getAscii8bitEncoding();
return needsWriteConversion(true, true, encs.enc, ascii8bit, mode, encs.ecflags)
&& !needsWriteConversion(true, false, encs.enc, ascii8bit, mode, encs.ecflags);
}

// MRI: io_fwrite with source bytes
public int fwrite(ThreadContext context, byte[] bytes, int start, int length, Encoding encoding, boolean nosync) {
// The System.console null check is our poor-man's isatty for Windows. See jruby/jruby#3292
if (Platform.IS_WINDOWS && isStdio() && System.console() != null) {
if (Platform.IS_WINDOWS && isStdio() && stdioIsConsole()) {
return rbW32WriteConsole(bytes, start, length, encoding);
}

int requested = length;
boolean crtNewlines = crtNewlinesOnly(context);
ByteList str = doWriteconv(context, bytes, start, length, encoding);

if (str != null) {
Expand All @@ -2326,7 +2375,31 @@ public int fwrite(ThreadContext context, byte[] bytes, int start, int length, En
length = str.realSize();
}

return binwriteInt(context, bytes, start, length, nosync);
int n = binwriteInt(context, bytes, start, length, nosync);
return crtNewlines && n == length ? requested : n;
}

// Poor man's isatty for stdio on Windows (jruby/jruby#3292). System.console() is non-null on JDK 22-24
// even when the standard streams are redirected; Console#isTerminal (JDK 22+) tells the cases apart.
static boolean stdioIsConsole() {
Console console = System.console();
if (console == null) return false;
if (CONSOLE_IS_TERMINAL == null) return true;
try {
return (boolean) CONSOLE_IS_TERMINAL.invoke(console);
} catch (Throwable t) {
return true;
}
}

private static final MethodHandle CONSOLE_IS_TERMINAL = lookupConsoleIsTerminal();

private static MethodHandle lookupConsoleIsTerminal() {
try {
return MethodHandles.publicLookup().findVirtual(Console.class, "isTerminal", MethodType.methodType(boolean.class));
} catch (NoSuchMethodException | IllegalAccessException e) {
return null;
}
}

// MRI: rb_w32_write_console
Expand Down
88 changes: 88 additions & 0 deletions core/src/test/java/org/jruby/util/io/OpenFileTextModeTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package org.jruby.util.io;

import org.jcodings.Encoding;
import org.jcodings.specific.ASCIIEncoding;
import org.jcodings.specific.UTF8Encoding;
import org.jcodings.transcode.EConvFlags;
import org.junit.Test;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

/**
* Newline decorator selection for text-mode IO, with the platform passed in so the
* Windows (RUBY_CRLF_ENVIRONMENT) path is covered on every platform.
*/
public class OpenFileTextModeTest {
private static final Encoding BINARY = ASCIIEncoding.INSTANCE;
private static final Encoding UTF8 = UTF8Encoding.INSTANCE;

// "r" on Windows: fmode TEXTMODE, ecflags carry the default CRLF marker only
private static final int WINDOWS_TEXT_READ = OpenFile.READABLE | OpenFile.TEXTMODE;
private static final int WINDOWS_TEXT_WRITE = OpenFile.WRITABLE | OpenFile.TEXTMODE;
private static final int CRLF = EConvFlags.CRLF_NEWLINE_DECORATOR;
private static final int UNIVERSAL = EConvFlags.UNIVERSAL_NEWLINE_DECORATOR;

@Test
public void textModeReadOnWindowsUsesTheConverter() {
assertTrue(OpenFile.needsReadConversion(true, null, WINDOWS_TEXT_READ, CRLF));
}

@Test
public void textModeReadOnWindowsConvertsNewlines() {
assertEquals(UNIVERSAL, OpenFile.readConversionFlags(true, WINDOWS_TEXT_READ, CRLF));
}

@Test
public void textModeReadOnWindowsKeepsAnExplicitUniversalNewline() {
assertEquals(UNIVERSAL, OpenFile.readConversionFlags(true, WINDOWS_TEXT_READ, UNIVERSAL));
}

@Test
public void binmodeReadOnWindowsDoesNotConvertNewlines() {
int mode = OpenFile.READABLE | OpenFile.BINMODE;
assertFalse(OpenFile.needsReadConversion(true, null, mode, 0));
assertEquals(0, OpenFile.readConversionFlags(true, mode, 0));
}

@Test
public void textModeReadOffWindowsIsUnchanged() {
assertFalse(OpenFile.needsReadConversion(false, null, OpenFile.READABLE, 0));
assertEquals(0, OpenFile.readConversionFlags(false, OpenFile.READABLE | OpenFile.TEXTMODE, CRLF));
assertEquals(UNIVERSAL, OpenFile.readConversionFlags(false, OpenFile.READABLE | OpenFile.TEXTMODE, UNIVERSAL));
}

@Test
public void textModeWriteOnWindowsUsesTheConverter() {
assertTrue(OpenFile.needsWriteConversion(true, true, null, BINARY, WINDOWS_TEXT_WRITE, CRLF));
}

@Test
public void textModeWriteOnWindowsWithAnEncodingUsesTheConverter() {
assertTrue(OpenFile.needsWriteConversion(true, true, UTF8, BINARY, WINDOWS_TEXT_WRITE, CRLF));
}

@Test
public void textModeWriteToAWindowsPipeDoesNotUseTheConverter() {
assertFalse(OpenFile.needsWriteConversion(true, false, null, BINARY, WINDOWS_TEXT_WRITE, CRLF));
}

@Test
public void explicitDecoratorOrEncodingOnAWindowsPipeUsesTheConverter() {
assertTrue(OpenFile.needsWriteConversion(true, false, null, BINARY, WINDOWS_TEXT_WRITE, EConvFlags.CR_NEWLINE_DECORATOR));
assertTrue(OpenFile.needsWriteConversion(true, false, UTF8, BINARY, WINDOWS_TEXT_WRITE, CRLF));
}

@Test
public void binmodeWriteOnWindowsDoesNotUseTheConverter() {
assertFalse(OpenFile.needsWriteConversion(true, true, BINARY, BINARY, OpenFile.WRITABLE | OpenFile.BINMODE, 0));
}

@Test
public void writeOffWindowsIsUnchanged() {
assertFalse(OpenFile.needsWriteConversion(false, true, null, BINARY, OpenFile.WRITABLE, 0));
assertTrue(OpenFile.needsWriteConversion(false, true, null, BINARY, OpenFile.WRITABLE | OpenFile.TEXTMODE, CRLF));
assertTrue(OpenFile.needsWriteConversion(false, false, UTF8, BINARY, OpenFile.WRITABLE, 0));
}
}
63 changes: 63 additions & 0 deletions spec/ruby/core/file/open_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -711,3 +711,66 @@
it_behaves_like :open_directory, :open
end
end

platform_is :windows do
describe "File.open on Windows" do
before :each do
@fname = tmp("file_open_text_mode.txt")
end

after :each do
rm_r @fname
end

def written_with(*args, **kwargs)
rm_r @fname
File.open(@fname, *args, **kwargs) { |f| f.write "a\nb\n" }
File.binread(@fname)
end

def read_with(*args, **kwargs)
File.binwrite(@fname, "a\r\nb\r\n")
File.open(@fname, *args, **kwargs) { |f| f.read }
end

it "writes CRLF for LF in text mode" do
written_with("w").should == "a\r\nb\r\n"
written_with("wt").should == "a\r\nb\r\n"
written_with("w", textmode: true).should == "a\r\nb\r\n"
written_with("a").should == "a\r\nb\r\n"
end

it "writes LF for LF in binary mode" do
written_with("wb").should == "a\nb\n"
written_with("w", binmode: true).should == "a\nb\n"
written_with(File::WRONLY | File::CREAT | File::TRUNC | File::BINARY).should == "a\nb\n"
end

it "reads LF for CRLF in text mode" do
read_with("r").should == "a\nb\n"
read_with("rt").should == "a\nb\n"
read_with("r", textmode: true).should == "a\nb\n"
read_with("r", newline: :universal).should == "a\nb\n"
read_with("r:UTF-8").should == "a\nb\n"
read_with("r", encoding: "UTF-8").should == "a\nb\n"
end

it "reads CRLF for CRLF in binary mode" do
read_with("rb").should == "a\r\nb\r\n"
read_with("r", binmode: true).should == "a\r\nb\r\n"
read_with(File::RDONLY | File::BINARY).should == "a\r\nb\r\n"
read_with("rb:UTF-8").should == "a\r\nb\r\n"
end

it "reads and writes without normalizing after #binmode" do
File.open(@fname, "w") { |f| f.binmode; f.write "a\nb\n" }
File.binread(@fname).should == "a\nb\n"
File.binwrite(@fname, "a\r\nb\r\n")
File.open(@fname, "r") { |f| f.binmode; f.read }.should == "a\r\nb\r\n"
end

it "raises ArgumentError for a newline option in binary mode" do
-> { File.open(@fname, "wb", newline: :crlf) {} }.should raise_error(ArgumentError)
end
end
end
21 changes: 21 additions & 0 deletions spec/ruby/core/io/binread_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -55,3 +55,24 @@
end
end
end

platform_is :windows do
describe "IO.binread on Windows" do
before :each do
@fname = tmp("io_binread.txt")
touch(@fname, "wb") { |f| f.write "a\r\nb\r\nc" }
end

after :each do
rm_r @fname
end

it "does not normalize line endings" do
IO.binread(@fname).should == "a\r\nb\r\nc"
end

it "does not normalize line endings when a length is given" do
IO.binread(@fname, 3).should == "a\r\n"
end
end
end
Loading
Loading