unit EncodingFixToolCore;

System.Character, System.Classes, System.Generics.Collections, System.IOUtils, System.StrUtils,

System.SyncObjs, System.SysUtils, System.Threading;

TEncodingFixTool = class

public type

TEndOfLineMode = (elmPreserve, elmCrlf);

TOutputFormat = (ofText, ofJson);

TScopeMode = (smAll, smGitChanged);

TOptions = record

DryRun: boolean;

Silent: boolean;

Verbose: boolean; // not compatible with Silent

Path: string;

SingleFilePath: string;

Recursive: boolean; // default y

Exts: TArray<string>; // normalized: ".pas", ".dpr", etc.

Utf8Bom: boolean; // default y

EolMode: TEndOfLineMode; // default preserve

OutputFormat: TOutputFormat; // default text

ScopeMode: TScopeMode; // default all files

BackupDir: string; // if <> '', create backups preserving relative path

end;

private

type

TSafeConsole = class

strict private

class var FCrit: TCriticalSection;

public

class constructor Create;

class destructor Destroy;

class procedure WriteLine(const aMsg: string);

end;

private

fWantedExts: TStringList;

fEnc1250: TEncoding;

fEnc1252: TEncoding;

function ParseCommandLine(out aOptions: TOptions): integer;

function ShowHelp: integer;

function NormalizeExtList(const aCSV: string): TArray<string>;

function CollectFiles(const aOptions: TOptions): TArray<string>;

function CollectGitChangedFiles(const aOptions: TOptions): TArray<string>;

function IsWantedExt(const aFile: string): boolean;

function IsPathWithin(const aRootPath, aFileName: string): boolean;

function QuoteShellArg(const aValue: string): string;

function RunCommand(const aCommandLine, aWorkingDirectory: string): integer;

// Encoding helpers

function HasUtf8Bom(const aBytes: TBytes): boolean;

function HasUtf16BEBom(const aBytes: TBytes): boolean;

function HasUtf16LEBom(const aBytes: TBytes): boolean;

function IsUtf8Strict(const aBytes: TBytes): boolean;

function GetWithoutUtf8Bom(const aBytes: TBytes): TBytes;

function SplitLinesByBytes(const aBytes: TBytes): TArray<TBytes>;

function IsAsciiBytes(const aBytes: TBytes): boolean;

function IsBinaryDfmFile(const aFile: string; const aBytes: TBytes): boolean;

function NormalizeTextEolToCrlf(const aText: string; out aChanged: boolean): string;

function DecodeBestPerLine(const aLineBytes: TBytes; out aEncName: string): string;

function ScoreDecoded(const aText: string): integer;

function ContainsSpecials(const aText: string): integer;

function FixFile(const aFile: string; const aOptions: TOptions; out aChanged: boolean; out aReason: string): boolean;

function SaveTextUTF8(const aFile: string; const aText: string; aWithBOM: boolean; out aError: string): boolean;

function MakeBackupPath(const aOptions: TOptions; const aRootPath, aFile: string): string;

function MakeRelativeTo(const aRootPath, aFile: string): string;

procedure PrepareExtIndex(const aExts: TArray<string>);

public

constructor Create;

destructor Destroy; override;

class function RunFromCommandLine: integer; static;

end;

System.Diagnostics, System.JSON,

System.WideStrUtils,

Winapi.Windows,

AutoFree;

TPosixTimespec = record

Seconds: Int64;

Nanoseconds: Int64;

end;

cLibC = 'libc.so.6';

cPosixInterrupted = 4;

cPosixNoHang = 1;

cPosixSignalKill = 9;

function PosixClockGetTime(aClockId: integer; aTime: Pointer): integer; cdecl;

external cLibC name 'clock_gettime';

function PosixErrorLocation: PInteger; cdecl; external cLibC name '__errno_location';

function PosixExecv(aPath: PAnsiChar; aArgv: Pointer): integer; cdecl; external cLibC name 'execv';

procedure PosixExit(aStatus: integer); cdecl; external cLibC name '_exit';

function PosixFork: integer; cdecl; external cLibC name 'fork';

function PosixKill(aPid, aSignal: integer): integer; cdecl; external cLibC name 'kill';

function PosixSetProcessGroup(aPid, aProcessGroup: integer): integer; cdecl; external cLibC name 'setpgid';

function PosixUSleep(aMicroseconds: Cardinal): integer; cdecl; external cLibC name 'usleep';

function PosixWaitPid(aPid: integer; aStatus: PInteger; aOptions: integer): integer; cdecl;

external cLibC name 'waitpid';

function PosixLastError: integer;

Result := PosixErrorLocation^;

end;

function PosixMonotonicMilliseconds: Int64;

lTime: TPosixTimespec;

if PosixClockGetTime(1, @lTime) <> 0 then

begin

raise Exception.Create('clock_gettime failed');

end;

Result := (lTime.Seconds * 1000) + (lTime.Nanoseconds div 1000000);

end;

function PosixWaitForChild(aPid: integer; out aStatus: integer): boolean;

lWaitPid: integer;

repeat

lWaitPid := PosixWaitPid(aPid, @aStatus, 0);

if lWaitPid = aPid then

begin

Exit(True);

end;

if (lWaitPid < 0) and (PosixLastError <> cPosixInterrupted) then

begin

Exit(False);

end;

until False;

end;

cCommandPollMs = 10;

cCommandTimeoutMs = 30000;

cUtf8Bom0 = Byte($EF);

cUtf8Bom1 = Byte($BB);

cUtf8Bom2 = Byte($BF);

cUtf16LEBom0 = Byte($FF);

cUtf16LEBom1 = Byte($FE);

cUtf16BEBom0 = Byte($FE);

cUtf16BEBom1 = Byte($FF);

class constructor TEncodingFixTool.TSafeConsole.Create;

FCrit := TCriticalSection.Create;

end;

class destructor TEncodingFixTool.TSafeConsole.Destroy;

FCrit.Free;

end;

class procedure TEncodingFixTool.TSafeConsole.WriteLine(const aMsg: string);

FCrit.Acquire;

try

writeln(aMsg);

finally

FCrit.release;

end;

end;

constructor TEncodingFixTool.Create;

inherited Create;

try

fWantedExts := TStringList.Create;

fWantedExts.Sorted := True;

fWantedExts.Duplicates := dupIgnore;

fWantedExts.CaseSensitive := False;

fEnc1250 := TEncoding.GetEncoding(1250);

fEnc1252 := TEncoding.GetEncoding(1252);

except

fEnc1252.Free;

fEnc1252 := nil;

fEnc1250.Free;

fEnc1250 := nil;

fWantedExts.Free;

fWantedExts := nil;

raise;

end;

end;

destructor TEncodingFixTool.Destroy;

fWantedExts.Free;

fEnc1252.Free;

fEnc1250.Free;

inherited;

end;

procedure TEncodingFixTool.PrepareExtIndex(const aExts: TArray<string>);

s: string;

fWantedExts.BeginUpdate;

try

fWantedExts.Clear;

for s in aExts do

begin

if s <> '' then

begin

fWantedExts.Add(LowerCase(s));

end;

end;

finally

fWantedExts.EndUpdate;

end;

end;

function TEncodingFixTool.NormalizeExtList(const aCSV: string): TArray<string>;

lParts: TArray<string>;

i: integer;

s: string;

lParts := aCSV.Split([',', ';', ' '], TStringSplitOptions.ExcludeEmpty);

SetLength(Result, length(lParts));

for i := 0 to High(lParts) do

begin

s := lParts[i].ToLower

.Trim([' ', '"', '''']);

if s.StartsWith('*.') then

begin

s := s.Substring(1); // "*.pas" -> ".pas"

end else if (s <> '') and (s[1] <> '.') then

begin

s := '.' + s; // "pas" -> ".pas"

end;

Result[i] := s;

end;

end;

function TEncodingFixTool.CollectFiles(const aOptions: TOptions): TArray<string>;

g: TGarbos;

lFiles: TList<string>;

lSearchOpt: TSearchOption;

lExt, lPattern: string;

if aOptions.SingleFilePath <> '' then

begin

if IsWantedExt(aOptions.SingleFilePath) then

begin

Result := [aOptions.SingleFilePath];

end else begin

Result := [];

end;

Exit;

end;

if aOptions.ScopeMode = TScopeMode.smGitChanged then

begin

Exit(CollectGitChangedFiles(aOptions));

end;

GC(lFiles, TList<string>.Create, g);

if aOptions.Recursive then

lSearchOpt := TSearchOption.soAllDirectories

else

lSearchOpt := TSearchOption.soTopDirectoryOnly;

for lExt in fWantedExts do

begin

lPattern := '*' + lExt; // e.g. ".pas" -> "*.pas"

lFiles.AddRange(TDirectory.GetFiles(aOptions.Path, lPattern, lSearchOpt));

end;

Result := lFiles.ToArray;

end;

function TEncodingFixTool.CollectGitChangedFiles(const aOptions: TOptions): TArray<string>;

g: TGarbos;

lCommand: string;

lEntries: TArray<string>;

lEntryIndex: integer;

lExitCode: integer;

lFile: string;

lFiles: TList<string>;

lLine: string;

lOutputFileName: string;

lPath: string;

lRootPath: string;

lScanRootPath: string;

lStatus: string;

lWorkTreeRootFileName: string;

GC(lFiles, TList<string>.Create, g);

lOutputFileName := TPath.Combine(TPath.GetTempPath, 'EncodingFixTool-git-' + TGuid.NewGuid.ToString + '.txt');

lWorkTreeRootFileName := TPath.Combine(TPath.GetTempPath, 'EncodingFixTool-git-root-' + TGuid.NewGuid.ToString + '.txt');

try

lCommand := 'cmd.exe /C git -C ' + QuoteShellArg(aOptions.Path) + ' rev-parse --show-toplevel > ' +

QuoteShellArg(lWorkTreeRootFileName);

lCommand := 'git -C ' + QuoteShellArg(aOptions.Path) + ' rev-parse --show-toplevel > ' +

QuoteShellArg(lWorkTreeRootFileName);

lExitCode := RunCommand(lCommand, aOptions.Path);

if lExitCode <> 0 then

begin

raise Exception.Create('git rev-parse failed');

end;

lRootPath := IncludeTrailingPathDelimiter(TFile.ReadAllText(lWorkTreeRootFileName, TEncoding.UTF8).Trim);

lScanRootPath := IncludeTrailingPathDelimiter(ExpandFileName(aOptions.Path));

lCommand := 'cmd.exe /C git -C ' + QuoteShellArg(aOptions.Path) +

' status --porcelain=v1 -z --untracked-files=all > ' + QuoteShellArg(lOutputFileName);

lCommand := 'git -C ' + QuoteShellArg(aOptions.Path) +

' status --porcelain=v1 -z --untracked-files=all > ' + QuoteShellArg(lOutputFileName);

lExitCode := RunCommand(lCommand, aOptions.Path);

if lExitCode <> 0 then

begin

raise Exception.Create('git status failed');

end;

lEntries := TEncoding.UTF8.GetString(TFile.ReadAllBytes(lOutputFileName))

.Split([#0], TStringSplitOptions.ExcludeEmpty);

lEntryIndex := 0;

while lEntryIndex <= High(lEntries) do

begin

lLine := lEntries[lEntryIndex];

Inc(lEntryIndex);

if Length(lLine) < 4 then

begin

Continue;

end;

lStatus := Copy(lLine, 1, 2);

if (lStatus = ' D') or (lStatus = 'D ') or (lStatus = 'DD') then

begin

Continue;

end;

lPath := Copy(lLine, 4, MaxInt);

if (Pos('R', lStatus) > 0) or (Pos('C', lStatus) > 0) then

begin

Inc(lEntryIndex); // -z appends the rename/copy source as a second entry.

end;

lPath := StringReplace(lPath, '/', TPath.DirectorySeparatorChar, [rfReplaceAll]);

lFile := TPath.GetFullPath(TPath.Combine(lRootPath, lPath));

if TFile.Exists(lFile) and IsPathWithin(lScanRootPath, lFile) and IsWantedExt(lFile) then

begin

lFiles.Add(lFile);

end;

end;

finally

if TFile.Exists(lOutputFileName) then

begin

TFile.Delete(lOutputFileName);

end;

if TFile.Exists(lWorkTreeRootFileName) then

begin

TFile.Delete(lWorkTreeRootFileName);

end;

end;

Result := lFiles.ToArray;

end;

function TEncodingFixTool.IsWantedExt(const aFile: string): boolean;

Result := fWantedExts.IndexOf(LowerCase(ExtractFileExt(aFile))) >= 0;

end;

function TEncodingFixTool.IsPathWithin(const aRootPath, aFileName: string): boolean;

Result := StartsText(aRootPath, aFileName);

Result := StartsStr(aRootPath, aFileName);

end;

function TEncodingFixTool.QuoteShellArg(const aValue: string): string;

Result := '"' + StringReplace(aValue, '"', '\"', [rfReplaceAll]) + '"';

Result := #39 + StringReplace(aValue, #39, #39 + '"' + #39 + '"' + #39, [rfReplaceAll]) + #39;

end;

function TEncodingFixTool.RunCommand(const aCommandLine, aWorkingDirectory: string): integer;

lExitCode: DWORD;

lCommandLineChars: TArray<Char>;

lProcessInformation: TProcessInformation;

lStartupInfo: TStartupInfo;

lWaitResult: Cardinal;

lArguments: array[0..3] of PAnsiChar;

lCommandLine: UTF8String;

lPid: integer;

lReaped: boolean;

lShellArg: UTF8String;

lShellName: UTF8String;

lShellPath: UTF8String;

lStartTime: Int64;

lStatus: integer;

lWaitPid: integer;

lCommandLineChars := aCommandLine.ToCharArray;

SetLength(lCommandLineChars, Length(lCommandLineChars) + 1);

ZeroMemory(@lProcessInformation, SizeOf(lProcessInformation));

ZeroMemory(@lStartupInfo, SizeOf(lStartupInfo));

lStartupInfo.cb := SizeOf(lStartupInfo);

if not CreateProcess(nil, PChar(lCommandLineChars), nil, nil, False, CREATE_NO_WINDOW, nil, PChar(aWorkingDirectory),

lStartupInfo, lProcessInformation) then

begin

RaiseLastOSError;

end;

try

lWaitResult := WaitForSingleObject(lProcessInformation.hProcess, cCommandTimeoutMs);

if lWaitResult <> WAIT_OBJECT_0 then

begin

TerminateProcess(lProcessInformation.hProcess, 1);

raise Exception.Create('process timed out');

end;

if not GetExitCodeProcess(lProcessInformation.hProcess, lExitCode) then

begin

RaiseLastOSError;

end;

Result := Integer(lExitCode);

finally

CloseHandle(lProcessInformation.hThread);

CloseHandle(lProcessInformation.hProcess);

end;

lCommandLine := UTF8String('cd ' + QuoteShellArg(aWorkingDirectory) + ' && ' + aCommandLine);

lShellArg := UTF8String('-c');

lShellName := UTF8String('sh');

lShellPath := UTF8String('/bin/sh');

lArguments[0] := PAnsiChar(lShellName);

lArguments[1] := PAnsiChar(lShellArg);

lArguments[2] := PAnsiChar(lCommandLine);

lArguments[3] := nil;

lPid := PosixFork;

if lPid < 0 then

begin

raise Exception.Create('fork failed');

end;

if lPid = 0 then

begin

if PosixSetProcessGroup(0, 0) <> 0 then

begin

PosixExit(126);

end;

PosixExecv(PAnsiChar(lShellPath), @lArguments[0]);

PosixExit(127);

end;

lReaped := False;

try

PosixSetProcessGroup(lPid, lPid);

lStartTime := PosixMonotonicMilliseconds;

repeat

lWaitPid := PosixWaitPid(lPid, @lStatus, cPosixNoHang);

if lWaitPid = lPid then

begin

lReaped := True;

Break;

end;

if lWaitPid < 0 then

begin

if PosixLastError = cPosixInterrupted then

begin

Continue;

end;

raise Exception.Create('waitpid failed');

end;

if PosixMonotonicMilliseconds - lStartTime >= cCommandTimeoutMs then

begin

if PosixKill(-lPid, cPosixSignalKill) <> 0 then

begin

PosixKill(lPid, cPosixSignalKill);

end;

if not PosixWaitForChild(lPid, lStatus) then

begin

raise Exception.Create('waitpid failed after timeout');

end;

lReaped := True;

raise Exception.Create('process timed out');

end;

PosixUSleep(cCommandPollMs * 1000);

until False;

if (lStatus and $7F) = 0 then

Result := (lStatus shr 8) and $FF

else

Result := 128 + (lStatus and $7F);

finally

if not lReaped then

begin

PosixKill(-lPid, cPosixSignalKill);

PosixKill(lPid, cPosixSignalKill);

if not PosixWaitForChild(lPid, lStatus) then

begin

raise Exception.Create('waitpid cleanup failed');

end;

end;

end;

end;

function TEncodingFixTool.IsUtf8Strict(const aBytes: TBytes): boolean;

lBytesNoBom: TBytes;

lBytesNoBom := GetWithoutUtf8Bom(aBytes);

if length(lBytesNoBom) = 0 then

exit(True);

Result := TEncoding.Utf8.IsBufferValid(lBytesNoBom);

end;

function TEncodingFixTool.HasUtf8Bom(const aBytes: TBytes): boolean;

Result := (length(aBytes) >= 3) and (aBytes[0] = cUtf8Bom0) and (aBytes[1] = cUtf8Bom1) and

(aBytes[2] = cUtf8Bom2);

end;

function TEncodingFixTool.HasUtf16BEBom(const aBytes: TBytes): boolean;

Result := (length(aBytes) >= 2) and (aBytes[0] = cUtf16BEBom0) and (aBytes[1] = cUtf16BEBom1);

end;

function TEncodingFixTool.HasUtf16LEBom(const aBytes: TBytes): boolean;

Result := (length(aBytes) >= 2) and (aBytes[0] = cUtf16LEBom0) and (aBytes[1] = cUtf16LEBom1);

end;

function TEncodingFixTool.SplitLinesByBytes(const aBytes: TBytes): TArray<TBytes>;

g: TGarbos;

i, lStart: integer;

b: BYTE;

lLine: TBytes;

lList: TList<TBytes>;

GC(lList, TList<TBytes>.Create, g);

lStart := 0;

i := 0;

while i < length(aBytes) do

begin

b := aBytes[i];

if b = $0A then

begin

// LF: line ends before this char

SetLength(lLine, i - lStart);

if length(lLine) > 0 then

move(aBytes[lStart], lLine[0], length(lLine));

lList.Add(lLine);

Inc(i);

lStart := i;

end else if b = $0D then

begin

// CR: line ends here; handle CRLF or standalone CR

SetLength(lLine, i - lStart);

if length(lLine) > 0 then

move(aBytes[lStart], lLine[0], length(lLine));

lList.Add(lLine);

Inc(i);

if (i < length(aBytes)) and (aBytes[i] = $0A) then

begin

Inc(i); // consume LF

end;

lStart := i;

end else begin

Inc(i);

end;

end;

// last line (no trailing EOL)

if lStart <= length(aBytes) - 1 then

begin

SetLength(lLine, length(aBytes) - lStart);

if length(lLine) > 0 then

move(aBytes[lStart], lLine[0], length(lLine));

lList.Add(lLine);

end;

Result := lList.ToArray;

end;

function TEncodingFixTool.ContainsSpecials(const aText: string): integer;

// Polish + German diacritics; count good matches positively.

POL = 'ąćęłńóśźżĄĆĘŁŃÓŚŹŻ';

GER = 'äöüÄÖÜßẞ';

lScore: integer;

ch: char;

lScore := 0;

for ch in aText do

begin

if (pos(ch, POL) > 0) or (pos(ch, GER) > 0) then

begin

Inc(lScore);

end;

end;

Result := lScore;

end;

function TEncodingFixTool.ScoreDecoded(const aText: string): integer;

lScore: integer;

ch: char;

// Base heuristic:

// +2 for known PL/DE diacritics,

// +1 for typical source-text ASCII range,

// -2 for control chars (excluding tab),

// -1 for � replacement if it appears (just in case)

lScore := 0;

Inc(lScore, 2 * ContainsSpecials(aText));

for ch in aText do

begin

// U+FFFD REPLACEMENT CHARACTER: indicates decoding replacement for invalid/unknown bytes; penalize it.

if ch = #$FFFD then

begin

Dec(lScore, 1);

end else if (ch = #9) or (ch = #10) or (ch = #13) then

begin

// ignore tabs/newlines here

end else if ch.IsControl then

begin

Dec(lScore, 2);

end else if ch.IsLetterOrDigit or ch.IsWhiteSpace or CharInSet(ch, ['.', ',', ';', ':', '-', '_', '(', ')', '[', ']', '{', '}', '''', '"', '/', '\', '+', '*', '=', '<', '>', '!', '?', '@', '#', '$', '%', '^', '&', '|']) then

begin

Inc(lScore, 1);

end;

end;

Result := lScore;

end;

function TEncodingFixTool.IsAsciiBytes(const aBytes: TBytes): boolean;

b: Byte;

for b in aBytes do

begin

if b >= $80 then

Exit(False);

end;

Exit(True);

end;

function TEncodingFixTool.IsBinaryDfmFile(const aFile: string; const aBytes: TBytes): boolean;

Result := SameText(ExtractFileExt(aFile), '.dfm') and (Length(aBytes) >= 4) and

(aBytes[0] = Ord('T')) and (aBytes[1] = Ord('P')) and (aBytes[2] = Ord('F')) and (aBytes[3] = Ord('0'));

end;

function TEncodingFixTool.NormalizeTextEolToCrlf(const aText: string; out aChanged: boolean): string;

g: TGarbos;

i: integer;

c: Char;

lBuilder: TStringBuilder;

aChanged := False;

GC(lBuilder, TStringBuilder.Create(Length(aText) + 16), g);

i := 1;

while i <= Length(aText) do

begin

c := aText[i];

if c = #13 then

begin

lBuilder.Append(#13#10);

if (i < Length(aText)) and (aText[i + 1] = #10) then

begin

Inc(i, 2);

end else begin

aChanged := True;

Inc(i);

end;

end else if c = #10 then

begin

lBuilder.Append(#13#10);

aChanged := True;

Inc(i);

end else begin

lBuilder.Append(c);

Inc(i);

end;

end;

if aChanged then

Result := lBuilder.ToString

else

Result := aText;

end;

function TEncodingFixTool.DecodeBestPerLine(const aLineBytes: TBytes; out aEncName: string): string;

lANSI: TEncoding;

sANSI: string;

s1250, s1252: string;

bestS: string;

bestScore, sc: integer;

// 0) Pure ASCII? Treat as ASCII explicitly (safe UTF-8 subset)

if IsAsciiBytes(aLineBytes) then

begin

aEncName := 'ASCII';

Result := TEncoding.Utf8.GetString(aLineBytes);

exit;

end;

// 1) Try strict UTF-8 first

if IsUtf8Strict(aLineBytes) then

begin

aEncName := 'UTF-8';

Result := TEncoding.Utf8.GetString(aLineBytes);

exit;

end;

// 2) Try single-byte candidates; they never fail, so score them.

s1250 := fEnc1250.GetString(aLineBytes);

s1252 := fEnc1252.GetString(aLineBytes);

lANSI := TEncoding.ANSI;

sANSI := lANSI.GetString(aLineBytes);

bestS := s1250;

aEncName := 'Windows-1250';

bestScore := ScoreDecoded(s1250);

sc := ScoreDecoded(s1252);

if sc > bestScore then

begin

bestScore := sc;

bestS := s1252;

aEncName := 'Windows-1252';

end;

sc := ScoreDecoded(sANSI);

if sc > bestScore then

begin

bestS := sANSI;

aEncName := 'ANSI';

end;

Result := bestS;

end;

function TEncodingFixTool.MakeBackupPath(const aOptions: TOptions; const aRootPath, aFile: string): string;

lRel: string;

lRootPath: string;

lRel := aFile;

if aRootPath <> '' then

begin

lRootPath := IncludeTrailingPathDelimiter(aRootPath);

if IsPathWithin(lRootPath, aFile) then

begin

lRel := copy(aFile, length(lRootPath) + 1, MaxInt);

end;

end;

Result := TPath.Combine(IncludeTrailingPathDelimiter(aOptions.BackupDir), lRel);

end;

function TEncodingFixTool.MakeRelativeTo(const aRootPath, aFile: string): string;

sRoot: string;

Result := aFile;

if aRootPath <> '' then

begin

sRoot := IncludeTrailingPathDelimiter(aRootPath);

if IsPathWithin(sRoot, aFile) then

Result := copy(aFile, length(sRoot) + 1, MaxInt);

end;

end;

function TEncodingFixTool.SaveTextUTF8(const aFile: string; const aText: string; aWithBOM: boolean; out aError: string): boolean;

lContentBytes: TBytes;

lBytes: TBytes;

lTempFile: string;

Result := False;

aError := '';

lTempFile := '';

try

try

lContentBytes := TEncoding.UTF8.GetBytes(aText);

if aWithBOM then

begin

SetLength(lBytes, length(lContentBytes) + 3);

lBytes[0] := cUtf8Bom0;

lBytes[1] := cUtf8Bom1;

lBytes[2] := cUtf8Bom2;

if length(lContentBytes) > 0 then

begin

Move(lContentBytes[0], lBytes[3], length(lContentBytes));

end;

end else begin

lBytes := lContentBytes;

end;

lTempFile := TPath.Combine(ExtractFileDir(aFile), TPath.GetRandomFileName);

TFile.WriteAllBytes(lTempFile, lBytes);

TFile.Copy(lTempFile, aFile, True);

Result := True;

except

on e: Exception do

begin

aError := e.ClassName + ': ' + e.Message;

end;

end;

finally

if (lTempFile <> '') and TFile.Exists(lTempFile) then

begin

TFile.Delete(lTempFile);

end;

end;

end;

function TEncodingFixTool.FixFile(const aFile: string; const aOptions: TOptions; out aChanged: boolean; out aReason: string): boolean;

lBytes: TBytes;

lBytesNoBom: TBytes;

lLinesBytes: TArray<TBytes>;

lLine: TBytes;

lFixedLines: TStringBuilder;

lFirst: boolean;

lRoot: string;

lBackupPath: string;

lHasBOM: boolean;

lText: string;

lCRLF, lLF, lCR: integer;

i: integer;

lEOL: string;

lHadTrailingEOL: boolean;

rb: RawByteString;

lEncType: TEncodeType;

lCntUtf8, lCnt1250, lCnt1252: Integer;

lCntAnsi: Integer;

lEncName: string;

lReasonEnc: string;

lEolChanged: boolean;

lKinds: Integer;

lSaveError: string;

g: TGarbos;

aChanged := False;

aReason := '';

lBytes := TFile.ReadAllBytes(aFile);

if IsBinaryDfmFile(aFile, lBytes) then

begin

aChanged := False;

aReason := 'Skipped binary DFM';

Result := True;

exit;

end;

if HasUtf16LEBom(lBytes) or HasUtf16BEBom(lBytes) then

begin

if HasUtf16LEBom(lBytes) then

begin

lText := TEncoding.Unicode.GetString(copy(lBytes, 2, length(lBytes) - 2));

lReasonEnc := 'detected UTF-16 LE';

end else begin

lText := TEncoding.BigEndianUnicode.GetString(copy(lBytes, 2, length(lBytes) - 2));

lReasonEnc := 'detected UTF-16 BE';

end;

if aOptions.DryRun then

begin

aChanged := True;

aReason := Format('%s; would save UTF-8 (BOM=%s) (dry-run)',

[lReasonEnc, IfThen(aOptions.Utf8Bom, 'Y', 'N')]);

Result := True;

exit;

end;

if aOptions.BackupDir <> '' then

begin

lRoot := IncludeTrailingPathDelimiter(ExpandFileName(aOptions.Path));

lBackupPath := MakeBackupPath(aOptions, lRoot, aFile);

TDirectory.CreateDirectory(ExtractFileDir(lBackupPath));

TFile.copy(aFile, lBackupPath, True);

end;

if SaveTextUTF8(aFile, lText, aOptions.Utf8Bom, lSaveError) then

begin

aChanged := True;

aReason := Format('%s; saved UTF-8 (BOM=%s)', [lReasonEnc, IfThen(aOptions.Utf8Bom, 'Y', 'N')]);

Result := True;

end else begin

aChanged := False;

aReason := 'Save failed: ' + lSaveError;

Result := False;

end;

exit;

end;

// Detect encoding using System.WideStrUtils.DetectUTF8Encoding

SetLength(rb, length(lBytes));

if length(lBytes) > 0 then

move(lBytes[0], pAnsiChar(rb)^, length(lBytes));

lEncType := DetectUTF8Encoding(rb);

if lEncType = etUSAscii then

begin

if aOptions.EolMode = TEndOfLineMode.elmCrlf then

begin

lText := TEncoding.ASCII.GetString(lBytes);

lText := NormalizeTextEolToCrlf(lText, lEolChanged);

if lEolChanged then

begin

if aOptions.DryRun then

begin

aChanged := True;

aReason := 'Would normalize EOL to CRLF (dry-run)';

Result := True;

exit;

end;

if aOptions.BackupDir <> '' then

begin

lRoot := IncludeTrailingPathDelimiter(ExpandFileName(aOptions.Path));

lBackupPath := MakeBackupPath(aOptions, lRoot, aFile);

TDirectory.CreateDirectory(ExtractFileDir(lBackupPath));

TFile.copy(aFile, lBackupPath, True);

end;

if SaveTextUTF8(aFile, lText, False, lSaveError) then