unit uIterSort;

uses uIterator, uSyntax, uTransaction, uStmt, uAlgebra, uTuple,

uFile, uTempTape, uGlobal;

nTempFiles=7; //number of temporary files (tapes) //todo make flexible?

//Note: need to extend FNAME template if max digits changes

nNodes=20; //number of nodes for selection tree //todo make flexible?

FNAME='sort%4.4d_%4.4d_%1.1d'; //temporary filename template (stmt,plan-node,file#) //todo make unique to server etc.

MaxKeyCol=MaxCol; //maximum number of sort columns

{Temporary file (tape) wrappers}

TtFile=class //hide? - but need to keep in sort class to assist multi-user memory handling

fp:TTempTape; //temporary tape file

fpBuf:array [0..MaxRecSize-1] of char; //read buffer area

fpBufLen:integer; //read buffer length of current record

dummy:integer; //number of dummy runs D[]

fib:integer; //ideal fibonacci number A[]

eof:boolean; //end of file flag

eor:boolean; //end of run flag

valid:boolean; //true if tuple is valid

end; {TtFile}

{Selection tree nodes}

TiNode=class; //forward

TeNode=class //external node (4+4+4+4+1=17 bytes = 20 bytes)

parent:TiNode; //parent of external node

rec:PChar; //pointer to dynamic tuple record data buffer

recLen:integer;

run:integer; //run number

valid:boolean; //input tuple is valid

end; {TeNode}

TiNode=class //internal node (4+4=8 bytes)

parent:TiNode; //parent of internal node

loser:TeNode; //external loser

end; {TiNode}

TNode=class //(4+4=8 bytes) e.g. 1000 nodes = 8000 + 8000 + 20000 + rec buffer = 36000+ (e.g. 76000 for 40 char rec buffers)

i:TiNode; //internal node

e:TeNode; //external node

end; {TNode}

TIterSort=class(TIterator)

private

tFile:array [0..nTempFiles-1] of TtFile;

level:integer; //level of runs

sorted:boolean; //first call = sort & materialise

node:array [0..nNodes] of TNode; //array of selection tree nodes

win:TeNode; //new winner

eof:boolean; //end of file, input

maxrun:integer; //maximum run number

currun:integer; //current run number

lastKeyValid:boolean; //true if last key is valid

lastKey:TTuple; //buffer to store last key comparison for readTuple

noMoreData:boolean; //input tuple noMore flag (needed to be static by IterNestedLoop)

tempTuple1,tempTuple2:TTuple;

distinct:boolean; //remove duplicates?

function CompareTupleKeysLT(tl,tr:TTuple;var res:boolean):integer;

function CompareTupleKeysEQ(tl,tr:TTuple;var res:boolean):integer;

function CompareTupleKeysGT(tl,tr:TTuple;var res:boolean):integer;

function initTempFiles:integer;

function deleteTempFiles:integer;

function termTempFiles:integer;

function rewindFile(f:integer):integer;

function readTuple(var noMore:boolean):integer;

function makeRuns:integer;

function doMergeSort:integer;

function mergeSort:integer;

public

//TODO: use keyColMap:array [0..MaxKeyCol] of TKeyColMap;

//todo use same array/structure as TindexFile...

//Note: made public so parent iterSet can override them

keyCol:array [0..MaxKeyCol-1] of record col:colRef; direction:TsortDirection; end; //array of sort columns

keyColCount:integer;

function description:string; override;

function status:string; override;

constructor create(S:TStmt;itemExprRef:TAlgebraNodePtr;distinctFlag:boolean);

destructor destroy; override;

function prePlan(outerRef:TIterator):integer; override;

function optimise(var SARGlist:TSyntaxNodePtr;var newChildParent:TIterator):integer; override;

function start:integer; override; //begin

function stop:integer ;override; //end

function next(var noMore:boolean):integer; override; //loop

function GetPosition:cardinal;

function FindPosition(p:cardinal):integer;

end; {TIterSort}

uLog,

{$ENDIF}

sysUtils, uEvalCondExpr, uHeapFile, uMarshalGlobal;

where='uIterSort';

fileT=nTempFiles-1; //last file

fileP=fileT-1; //next to last file (P-way merging)

constructor TIterSort.create(S:TStmt;itemExprRef:TAlgebraNodePtr;distinctFlag:boolean);

const routine=':create';

i:integer;

inherited create(s);

aNodeRef:=itemExprRef;

distinct:=distinctFlag;

sorted:=false;

for i:=0 to nTempFiles-1 do

begin

tFile[i]:=TtFile.Create; //todo: faster if use records & new(TtFilePtr)?

tFile[i].fp:=TTempTape.Create;

end;

for i:=0 to nNodes-1 do //todo check for memory full...

begin

node[i]:=TNode.Create;

node[i].i:=TiNode.Create;

node[i].e:=TeNode.Create;

end;

lastKey:=TTuple.Create(nil);

tempTuple1:=TTuple.Create(nil);

tempTuple2:=TTuple.Create(nil);

end; {create}

destructor TIterSort.destroy;

const routine=':destroy';

var i:integer;

tempTuple2.free;

tempTuple1.free;

lastKey.free;

for i:=nNodes-1 downto 0 do

begin

if node[i].e.rec<>nil then

begin

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('buffer for external node %d was not released',[i]),vAssertion);

{$ENDIF}

//continue //abort?

end;

node[i].e.free;

node[i].i.free;

node[i].free;

end;

for i:=nTempFiles-1 downto 0 do

begin

tFile[i].fp.Free;

tFile[i].free;

end;

inherited destroy;

end; {destroy}

function TIterSort.description:string;

result:=inherited description;

if distinct then

result:=result+' (distinct)';

end; {description}

function TIterSort.status:string;

{$IFDEF DEBUG_LOG}

if distinct then

result:=format('TIterSort %d (distinct)',[keyColCount])

else

result:=format('TIterSort %d',[keyColCount]);

if anodeRef<>nil then result:=result+' '+anodeRef.rangeName; //not set for merge-join children

{$ELSE}

result:='';

{$ENDIF}

end; {status}

function TIterSort.prePlan(outerRef:TIterator):integer;

const routine=':prePlan';

nhead:TSyntaxNodePtr;

i:colRef;

colName:string;

cTuple:TTuple; //todo make global?

cId:TColId;

cRef:ColRef;

result:=inherited prePlan(outerRef);

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('%s preplanning',[self.status]),vDebugLow);

{$ENDIF}

if assigned(leftChild) then

begin

result:=leftChild.prePlan(outer); //recurse down tree

correlated:=correlated OR leftChild.correlated;

end;

if result<>ok then exit; //aborted by child

{Define this ituple from leftChild.ituple}

iTuple.CopyTupleDef(leftChild.iTuple);

{Define the temporary tuples to be identical to the input tuple}

lastKey.CopyTupleDef(leftChild.iTuple);

tempTuple1.CopyTupleDef(leftChild.iTuple);

tempTuple2.CopyTupleDef(leftChild.iTuple);

{Set up the sort key from the column list passed}

//in future may need to handle expressions here...?

// - (only for GROUP-BY sort?) although the standard grammar I have only allows column-refs!!!!

// could do SELECT exp as E ... GROUP BY E = legal

keyColCount:=0;

if anodeRef=nil then

nhead:=nil //e.g. sort for joinMerge - no syntax nodes passed at this stage...

else

begin

nhead:=anodeRef.nodeRef;

//todo remove the following line: overkill, since iterSet will push down the sort-order afterwards... no harm? better to do it this way? = one set of code...

if (nhead<>nil) and (nhead.nType in [ntCorresponding,ntCorrespondingBy]) then nhead:=nhead.leftChild;

end;

i:=0;

while nhead<>nil do

begin

//todo impossible, but double check nhead not nil!

//todo use new code that relies on complete* routines... although overkill here maybe?

{todo: take name from column name (pass up?)

//Note: the expression calculators will re-define these if needed anyway - ok?

colName:=intToStr(i); //default column name

case nhead.nType of

ntColumnRef: //todo note: this is from group-by! = no ASC/DESC

begin

//todo: if n.leftChild.ntype=ntTable then check if its leftChild is a schema & if so restrict find...

//assumes we have a right child! -assert!

result:=leftchild.iTuple.FindCol(nhead,nhead.rightChild.idval,'',leftchild.outer,cTuple,cRef,cid);

if result<>ok then

begin

if result=-2 then

begin

stmt.addError(seSyntaxAmbiguousColumn,format(seSyntaxAmbiguousColumnText,[nhead.rightChild.idval]));

end

else

begin

stmt.addError(seSyntaxLookupFailed,format(seSyntaxLookupFailedText,['column '+nhead.rightChild.idval]));

end;

exit; //abort if child aborts

end;

if cid=InvalidColId then

begin

//shouldn't this have been caught before now!?

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('Unknown column reference (%s)',[nhead.rightChild.idVal]),vError);

{$ENDIF}

stmt.addError(seSyntaxUnknownColumn,format(seSyntaxUnknownColumnText,[nhead.rightChild.idVal]));

result:=Fail;

exit; //abort, no point continuing?

end;

if cTuple<>leftChild.iTuple then //todo relax later when using expressions? if so remember to pull up correlated from any Complete... call

begin

//shouldn't this have been caught before now!?

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('Column reference (%s) must be in this sub-select',[nhead.rightChild.idVal]),vError);

{$ENDIF}

result:=Fail;

exit; //abort, no point continuing?

end;

if keyColCount>=MaxKeyCol-1 then

begin

//shouldn't this have been caught before now!?

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('Too many sort columns %d',[keyColCount]),vError);

{$ENDIF}

result:=Fail;

exit; //abort, no point continuing?

end;

inc(keyColCount);

keyCol[keyColCount-1].col:=cRef;

keyCol[keyColCount-1].direction:=sdASC;

inc(i); //single column added

end; {ntColumnRef}

ntOrderItem:

begin

//assumes we have a left child! -assert!

if (nhead.leftChild.idval='') and (nhead.leftChild.dtype=ctNumeric) then //assume integer //todo check dtype as well/instead?

begin

{Note: this is deprecated in SQL-92}

cRef:=trunc(nhead.leftChild.numVal); //user subscript

if (cRef<1) or (cRef>iTuple.ColCount) then

begin

//shouldn't this have been caught before now!?

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('Unknown column subscript (%d)',[trunc(nhead.leftChild.numVal)]),vError);

{$ENDIF}

stmt.addError(seSyntaxUnknownColumn,format(seSyntaxUnknownColumnText,[intToStr(cRef)]));

result:=Fail;

exit; //abort, no point continuing?

end;

cRef:=cRef-1; //todo check ok to map directly to subscript..

end

else

begin //column name

//assumes we have a right child! -assert!

result:=leftchild.iTuple.FindCol(nhead.leftChild,nhead.leftChild.rightChild.idval,'',outer,cTuple,cRef,cid);

if result<>ok then

begin

if result=-2 then

begin

stmt.addError(seSyntaxAmbiguousColumn,format(seSyntaxAmbiguousColumnText,[nhead.leftChild.rightChild.idval]));

end

else

begin

stmt.addError(seSyntaxLookupFailed,format(seSyntaxLookupFailedText,['column '+nhead.leftChild.rightChild.idval]));

end;

exit; //abort if child aborts

end;

if cid=InvalidColId then

begin

//shouldn't this have been caught before now!?

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('Unknown column (%s)',[nhead.leftChild.idVal]),vError);

{$ENDIF}

stmt.addError(seSyntaxUnknownColumn,format(seSyntaxUnknownColumnText,[nhead.leftChild.idVal]));

result:=Fail;

exit; //abort, no point continuing?

end;

if cTuple<>leftChild.iTuple then

begin

//shouldn't this have been caught before now!?

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('Column (%s) must be in this sub-select',[nhead.leftChild.rightChild.idVal]),vError);

{$ENDIF}

result:=Fail;

exit; //abort, no point continuing?

end;

if keyColCount>=MaxKeyCol-1 then

begin

//shouldn't this have been caught before now!?

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('Too many sort columns %d',[keyColCount]),vError);

{$ENDIF}

result:=Fail;

exit; //abort, no point continuing?

end;

end;

inc(keyColCount);

keyCol[keyColCount-1].col:=cRef;

keyCol[keyColCount-1].direction:=sdASC; //default

if nhead.rightChild<>nil then //Retrieve ASC/DESC from nhead.rightChild

case nhead.rightChild.nType of

ntASC: keyCol[keyColCount-1].direction:=sdASC;

ntDESC: keyCol[keyColCount-1].direction:=sdDESC;

else

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('Unknown column sort direction',[nil]),vError);

{$ENDIF}

//ignore it! ok?

end; {case}

inc(i); //single column added

end; {ntOrderItem}

else

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('Only columns/column references allowed in sort',[nil]),vError);

{$ENDIF}

//ignore it! ok?

end; {case}

nhead:=nhead.NextNode;

end; {while}

if keyColCount=0 then

begin

if distinct then //duplicate removal, so sort on all columns

begin

for i:=0 to iTuple.ColCount-1 do

begin

inc(keyColCount);

keyCol[keyColCount-1].col:=i;

keyCol[keyColCount-1].direction:=sdASC;

end;

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('Sorting on all %d columns (for distinct)',[keyColCount]),vDebugLow);

{$ENDIF}

end

else

begin //assume sort on all columns anyway (was needed for initial pre-iterSet - now pushed down)

for i:=0 to iTuple.ColCount-1 do

begin

inc(keyColCount);

keyCol[keyColCount-1].col:=i;

keyCol[keyColCount-1].direction:=sdASC;

end;

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('Sorting on all %d columns',[keyColCount]),vDebugLow);

{$ENDIF}

end;

end //todo instead assume all columns?

else

begin

{$IFDEF DEBUGDETAIL2}

{$IFDEF DEBUG_LOG}

for i:=0 to keyColCount-1 do

log.add(stmt.who,where+routine,format('Sorting on column %d',[keyCol[i].col]),vDebugLow);

{$ENDIF}

{$ENDIF}

end;

end; {prePlan}

function TIterSort.optimise(var SARGlist:TSyntaxNodePtr;var newChildParent:TIterator):integer;

const routine=':optimise';

result:=inherited optimise(SARGlist,newChildParent);

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('%s optimising',[self.status]),vDebugLow);

{$ENDIF}

//todo: optimise sort

// ensure projections have been pushed down below here at least

// if small result set (expected) then quicksort in memory! - use buffers or keep separate? how to limit...

if assigned(leftChild) then

begin

result:=leftChild.optimise(SARGlist,newChildParent); //recurse down tree

end;

if result<>ok then exit; //aborted by child

if newChildParent<>nil then

begin

{Child has inserted an intermediate node - re-link to new child for execution calls}

{$IFDEF DEBUGDETAIL2}

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('linking to new leftChild: %s',[newChildParent.Status]),vDebugLow);

{$ENDIF}

{$ENDIF}

leftChild:=newChildParent;

newChildParent:=nil; //don't continue passing up!

end;

//todo: same for rightChild if we could have one

//check if any of our SARGs are now marked 'pushed' & remove them from ourself if so

end; {optimise}

function TIterSort.start:integer;

const routine=':start';

i:integer;

result:=inherited start;

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('%s starting',[self.status]),vDebugLow);

{$ENDIF}

if assigned(leftChild) then result:=leftChild.start; //recurse down tree

if result<>ok then exit; //aborted by child

{Clear - mainly needed to point scratch data buffer pointer for all columns}

iTuple.clear(stmt);

lastKey.clear(stmt);

tempTuple1.clear(stmt);

tempTuple2.clear(stmt);

sorted:=false; //13/05/00 fix: nested iteration of groups called re-start but wasn't re-sorting on 1st call to next!

{18/11/02 fix: re-execute prepared caused win top errors (moved here from create)}

for i:=0 to nNodes-1 do

begin

node[i].i.loser:=node[i].e;

node[i].i.parent:=node[i div 2].i;

node[i].e.parent:=node[(nNodes+i) div 2].i;

node[i].e.run:=0;

node[i].e.valid:=False;

if node[i].e.rec<>nil then

begin

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('buffer for external node %d was not released',[i]),vAssertion);

{$ENDIF}

freemem(node[i].e.rec,node[i].e.recLen); //Note: we don't have to specify the length

node[i].e.rec:=nil;

node[i].e.recLen:=0; //todo remove - no need?

end;

end;

{29/03/03 fix: re-execute prepared caused empty results set (not resetting valid)}

for i:=0 to nTempFiles-1 do

begin

tFile[i].dummy:=0;

tFile[i].fib:=0;

tFile[i].eof:=false;

tFile[i].eor:=false;

tFile[i].valid:=false;

end;

win:=node[0].e;

eof:=False;

maxrun:=0; //29/03/03 fix: re-execute prepared caused empty results set

currun:=0; //29/03/03 fix: re-execute prepared caused empty results set

lastKeyValid:=False;

noMoreData:=false; //29/03/03 fix: re-execute prepared caused empty results set

end; {start}

function TIterSort.stop:integer;

const routine=':stop';

result:=inherited stop;

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('%s stopping',[self.status]),vDebugLow);

{$ENDIF}

if assigned(leftChild) then result:=leftChild.stop; //recurse down tree

//todo ok/better to continue?.... if result<>ok then exit; //aborted by child

result:=tFile[fileT].fp.close; //stop materialised scan

result:=tFile[fileT].fp.delete; //delete materialised scan

end; {stop}

function TIterSort.next(var noMore:boolean):integer;

const routine=':next';

result:=ok;

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('%s next',[self.status]),vDebugLow);

{$ENDIF}

if stmt.status=ssCancelled then

begin

result:=Cancelled;

exit;

end;

if not sorted then

begin //first call = sort input & materialise

sorted:=True;

//todo: when we can determine that the child iterator is already in sorted order, then skip this step & just shallow read child-tuple! -speed!

result:=mergeSort; //todo need to allow to be interrupted by caller somehow...

if result<>ok then exit; //abort

end;

if not tFile[fileT].fp.noMore then

begin

{already sorted, so get next from materialised relation}

result:=tFile[fileT].fp.readRecord(tFile[fileT].fpBuf,tFile[fileT].fpBufLen);

if result<>ok then exit; //abort

result:=iTuple.CopyBufferToData(tFile[fileT].fpBuf,tFile[fileT].fpBufLen);

noMore:=False; //todo: bug? we should not need this. Similar problem in materialise.next was because .stop was called but should have delayed closing children until 'really' done

{$IFDEF DEBUGDETAIL}

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('%s',[iTuple.Show(stmt)]),vDebugLow);

{$ENDIF}

{$ENDIF}

end

else //end of tape

noMore:=True;

end; {next}

function TIterSort.GetPosition:cardinal;

if not sorted then

result:=0

else

result:=tFile[fileT].fp.GetPosition;

end; {GetPosition}

function TIterSort.FindPosition(p:cardinal):integer;

if not sorted then

result:=fail

else

begin

result:=tFile[fileT].fp.FindPosition(p);

end;

end; {FindPosition}

function TIterSort.CompareTupleKeysLT(tl,tr:TTuple;var res:boolean):integer;

const routine=':compareTupleKeysLT';

cl:colRef;

resComp:shortint;

resNull:boolean;

result:=ok;

res:=False;

cl:=0;

//todo make into an assertion: if tl.ColCount<>tr.ColCount then res:=isFalse; //done! mismatch column valency

//todo speed logic

resComp:=0;

while (resComp=0) and (cl<keyColCount) do

begin

result:=tl.CompareCol(stmt,keyCol[cl].col,keyCol[cl].col,tr,resComp,resNull);

if result<>ok then exit; //abort if compare fails

if resNull then

begin

result:=tl.ColIsNull(keyCol[cl].col,resNull);

if result<>ok then exit; //abort

if resNull then

begin

result:=tr.ColIsNull(keyCol[cl].col,resNull);

if result<>ok then exit; //abort

if resNull then

resComp:=0 //both are null so treat as equal (SQL anomaly!)

else

resComp:=NullSortOthers;

end

else

resComp:=-NullSortOthers;

end;

if keyCol[cl].direction=sdDESC then resComp:=-resComp; //reverse direction

inc(cl);

end;

if resComp<0 then res:=True else res:=False;

end; {CompareTupleKeysLT}

function TIterSort.CompareTupleKeysEQ(tl,tr:TTuple;var res:boolean):integer;

const routine=':compareTupleKeysEQ';

cl:colRef;

resComp:shortint;

resNull:boolean;

result:=ok;

res:=False;

cl:=0;

//todo make into an assertion: if tl.ColCount<>tr.ColCount then res:=isFalse; //done! mismatch column valency

//todo speed logic

resComp:=0;

while (resComp=0) and (cl<keyColCount) do

begin

result:=tl.CompareCol(stmt,keyCol[cl].col,keyCol[cl].col,tr,resComp,resNull);

if result<>ok then exit; //abort if compare fails

if resNull then

begin

result:=tl.ColIsNull(keyCol[cl].col,resNull);

if result<>ok then exit; //abort

if resNull then

begin

result:=tr.ColIsNull(keyCol[cl].col,resNull);

if result<>ok then exit; //abort

if resNull then

resComp:=0 //both are null so treat as equal (SQL anomaly!)

else

resComp:=NullSortOthers;

end

else

resComp:=-NullSortOthers;

end;

if keyCol[cl].direction=sdDESC then resComp:=-resComp; //reverse direction

inc(cl);

end;

if resComp=0 then res:=True else res:=False;

end; {CompareTupleKeysEQ}

function TIterSort.CompareTupleKeysGT(tl,tr:TTuple;var res:boolean):integer;

const routine=':compareTupleKeysGT';

cl:colRef;

resComp:shortint;

resNull:boolean;

result:=ok;

res:=False;

cl:=0;

//todo make into an assertion: if tl.ColCount<>tr.ColCount then res:=isFalse; //done! mismatch column valency

//todo speed logic

resComp:=0;

while (resComp=0) and (cl<keyColCount) do

begin

result:=tl.CompareCol(stmt,keyCol[cl].col,keyCol[cl].col,tr,resComp,resNull);

if result<>ok then exit; //abort if compare fails

if resNull then

begin

result:=tl.ColIsNull(keyCol[cl].col,resNull);

if result<>ok then exit; //abort

if resNull then

begin

result:=tr.ColIsNull(keyCol[cl].col,resNull);

if result<>ok then exit; //abort

if resNull then

resComp:=0 //both are null so treat as equal (SQL anomaly!)

else

resComp:=NullSortOthers;

end

else

resComp:=-NullSortOthers;

end;

if keyCol[cl].direction=sdDESC then resComp:=-resComp; //reverse direction

inc(cl);

end;

if resComp>0 then res:=True else res:=False;

end; {CompareTupleKeysGT}

function TIterSort.initTempFiles:integer;

const routine=':initTempFiles';

i:integer;

r:integer;

result:=ok;

{$IFDEF DEBUGDETAIL}

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,'initialising temp files',vDebugLow);

{$ENDIF}

{$ENDIF}

if nTempFiles<3 then

begin

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('must have more than 3 temp files (%d)',[nTempFiles]),vAssertion);

{$ENDIF}

result:=fail;

exit; //abort

end;

for i:=0 to nTempFiles-1 do

begin

{Open each new relation ready for writing to}

r:=trunc(random(9999)); //todo DEBUG ONLY - REMOVE!! need to make file unique to trans+node! i.e. SYS-getNextFilename!

result:=tFile[i].fp.CreateNew(format(FNAME,[stmt.Rt.tranId,r,i]));

{$IFDEF DEBUGDETAIL}

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,'initialising temp file: '+format(FNAME,[stmt.Rt.tranId,r,i]),vDebugLow);

{$ENDIF}

{$ENDIF}

if result<>ok then exit; //abort

end;

end; {initTempFiles}

function TIterSort.deleteTempFiles:integer;

const routine=':deleteTempFiles';

i:integer;

result:=ok;

{$IFDEF DEBUGDETAIL}

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,'deleting temp files:',vDebugLow);

{$ENDIF}

{$ENDIF}

for i:=0 to fileT-1 do

begin

{$IFDEF DEBUGDETAIL}

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format(' %s',[tFile[i].fp.filename]),vDebugLow);

{$ENDIF}

{$ENDIF}

result:=tFile[i].fp.close;

result:=tFile[i].fp.delete;

//we don't clean up allocations until Iter is finished

end;

end; {deleteTempFiles}

function TIterSort.termTempFiles:integer;

const routine=':termTempFiles';

result:=ok;

{$IFDEF DEBUGDETAIL}

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,'finalising temp files',vDebugLow);

{$ENDIF}

{$ENDIF}

{file[T] contains results}

{$IFDEF DEBUGDETAIL}

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('re-opening results file %d',[fileT]),vDebugLow);

{$ENDIF}

{$ENDIF}

result:=tFile[fileT].fp.rewind;

//todo check result...

result:=deleteTempFiles;

end; {termTempFiles}

function TIterSort.rewindFile(f:integer):integer;

const routine=':rewindFile';

result:=ok;

{$IFDEF DEBUGDETAIL}

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('rewinding file %d',[f]),vDebugLow);

{$ENDIF}

{$ENDIF}

tFile[f].eor:=False;

tFile[f].eof:=False;

tFile[f].fp.rewind;

//todo check result...

if tFile[f].fp.noMore then

begin

if result<>ok then exit; //abort

tFile[f].eor:=True;

tFile[f].eof:=True;

{$IFDEF DEBUGDETAIL}

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('initial record read = eof from file %d',[f]),vDebugLow);

{$ENDIF}

{$ENDIF}

end

else

begin

result:=tFile[f].fp.readRecord(tFile[f].fpBuf,tFile[f].fpBufLen);

{$IFDEF DEBUGDETAIL}

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format('initial record read from file %d: %s',[f,tFile[f].fpBuf]),vDebugLow);

{$ENDIF}

{$ENDIF}

end;

end; {rewindFile}

function TIterSort.readTuple(var noMore:boolean):integer;

const routine=':readTuple';

p:TiNode; //pointer to internal nodes

t:TeNode; //pointer for swapping

swap:boolean;

res:boolean;

result:=ok;

while True do

begin

{replace previous winner with new tuple}

if not eof then

begin

if assigned(leftChild) then

begin

result:=leftChild.next(noMoreData); //recurse down tree (this input will be buffered)

if result<>ok then exit; //abort

{$IFDEF DEBUGDETAIL}

{$IFDEF DEBUG_LOG}

log.add(stmt.who,where+routine,format(' reading %s',[leftChild.iTuple.show(stmt)]),vDebugLow);

{$ENDIF}

{$ENDIF}

if not noMoreData then

begin

{copy leftChild.iTuple data to new win.rec buffer}

//todo: note this copy routine is probably relatively slow... (we need to account for multiple versions of buffers from child)

result:=leftChild.iTuple.CopyDataToBuffer(win.rec,win.recLen);

{$IFDEF DEBUGDETAIL}

{$IFDEF DEBUG_LOG}

log.quick('win.rec ['+intToStr(win.recLen)+']='+win.rec+']');

{$ENDIF}

{$ENDIF}

if lastKeyValid then

result:=CompareTupleKeysLT(leftChild.iTuple,lastKey,res)

else

res:=False; //speed?

if result<>ok then exit;

if not(lastKeyValid) or res then

begin

inc(win.run);

if win.run>maxRun then maxRun:=win.run;

end;

win.valid:=True;

end

else

begin

//todo: maybe leftChild.stop now? - may as well!

eof:=true;

win.valid:=False;

win.run:=maxRun+1;

end;

end;

end