//-------------------------------------------------------------------------------------------------------
// Copyright (C) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
//-------------------------------------------------------------------------------------------------------
#include "BackEnd.h"
///----------------------------------------------------------------------------
///
/// Encoder::Encode
///
/// Main entrypoint of encoder. Encode each IR instruction into the
/// appropriate machine encoding.
///
///----------------------------------------------------------------------------
void
Encoder::Encode()
{
NoRecoverMemoryArenaAllocator localAlloc(L"BE-Encoder", m_func->m_alloc->GetPageAllocator(), Js::Throw::OutOfMemory);
m_tempAlloc = &localAlloc;
uint32 instrCount = m_func->GetInstrCount();
size_t totalJmpTableSizeInBytes = 0;
JmpTableList * jumpTableListForSwitchStatement = nullptr;
m_encoderMD.Init(this);
m_encodeBufferSize = UInt32Math::Mul(instrCount, MachMaxInstrSize);
m_encodeBufferSize += m_func->m_totalJumpTableSizeInBytesForSwitchStatements;
m_encodeBuffer = AnewArray(m_tempAlloc, BYTE, m_encodeBufferSize);
#if DBG_DUMP
m_instrNumber = 0;
m_offsetBuffer = AnewArray(m_tempAlloc, uint, instrCount);
#endif
m_pragmaInstrToRecordMap = Anew(m_tempAlloc, PragmaInstrList, m_tempAlloc);
if (DoTrackAllStatementBoundary())
{
// Create a new list, if we are tracking all statement boundaries.
m_pragmaInstrToRecordOffset = Anew(m_tempAlloc, PragmaInstrList, m_tempAlloc);
}
else
{
// Set the list to the same as the throw map list, so that processing of the list
// of pragma are done on those only.
m_pragmaInstrToRecordOffset = m_pragmaInstrToRecordMap;
}
#if defined(_M_IX86) || defined(_M_X64)
// for BR shortening
m_inlineeFrameRecords = Anew(m_tempAlloc, InlineeFrameRecords, m_tempAlloc);
#endif
m_pc = m_encodeBuffer;
m_inlineeFrameMap = Anew(m_tempAlloc, InlineeFrameMap, m_tempAlloc);
m_bailoutRecordMap = Anew(m_tempAlloc, BailoutRecordMap, m_tempAlloc);
CodeGenWorkItem* workItem = m_func->m_workItem;
uint loopNum = Js::LoopHeader::NoLoop;
if (workItem->Type() == JsLoopBodyWorkItemType)
{
loopNum = ((JsLoopBodyCodeGen*)workItem)->GetLoopNumber();
}
Js::SmallSpanSequenceIter iter;
IR::PragmaInstr* pragmaInstr = nullptr;
uint32 pragmaOffsetInBuffer = 0;
#ifdef _M_X64
bool inProlog = false;
#endif
bool isCallInstr = false;
FOREACH_INSTR_IN_FUNC(instr, m_func)
{
Assert(Lowerer::ValidOpcodeAfterLower(instr, m_func));
if (GetCurrentOffset() + MachMaxInstrSize < m_encodeBufferSize)
{
ptrdiff_t count;
#if DBG_DUMP
AssertMsg(m_instrNumber < instrCount, "Bad instr count?");
__analysis_assume(m_instrNumber < instrCount);
m_offsetBuffer[m_instrNumber++] = GetCurrentOffset();
#endif
if (instr->IsPragmaInstr())
{
switch(instr->m_opcode)
{
#ifdef _M_X64
case Js::OpCode::PrologStart:
inProlog = true;
continue;
case Js::OpCode::PrologEnd:
inProlog = false;
continue;
#endif
case Js::OpCode::StatementBoundary:
pragmaOffsetInBuffer = GetCurrentOffset();
pragmaInstr = instr->AsPragmaInstr();
pragmaInstr->m_offsetInBuffer = pragmaOffsetInBuffer;
// will record after BR shortening with adjusted offsets
if (DoTrackAllStatementBoundary())
{
m_pragmaInstrToRecordOffset->Add(pragmaInstr);
}
break;
default:
continue;
}
}
else if (instr->IsBranchInstr() && instr->AsBranchInstr()->IsMultiBranch())
{
Assert(instr->GetSrc1() && instr->GetSrc1()->IsRegOpnd());
IR::MultiBranchInstr * multiBranchInstr = instr->AsBranchInstr()->AsMultiBrInstr();
if (multiBranchInstr->m_isSwitchBr &&
(multiBranchInstr->m_kind == IR::MultiBranchInstr::IntJumpTable || multiBranchInstr->m_kind == IR::MultiBranchInstr::SingleCharStrJumpTable))
{
BranchJumpTableWrapper * branchJumpTableWrapper = multiBranchInstr->GetBranchJumpTable();
if (jumpTableListForSwitchStatement == nullptr)
{
jumpTableListForSwitchStatement = Anew(m_tempAlloc, JmpTableList, m_tempAlloc);
}
jumpTableListForSwitchStatement->Add(branchJumpTableWrapper);
totalJmpTableSizeInBytes += (branchJumpTableWrapper->tableSize * sizeof(void*));
}
else
{
//Reloc Records
EncoderMD * encoderMD = &(this->m_encoderMD);
multiBranchInstr->MapMultiBrTargetByAddress([=](void ** offset) -> void
{
#if defined(_M_ARM32_OR_ARM64)
encoderMD->AddLabelReloc((byte*) offset);
#else
encoderMD->AppendRelocEntry(RelocTypeLabelUse, (void*) (offset));
#endif
});
}
}
else
{
isCallInstr = LowererMD::IsCall(instr);
if (pragmaInstr && (instr->isInlineeEntryInstr || isCallInstr))
{
// will record throw map after BR shortening with adjusted offsets
m_pragmaInstrToRecordMap->Add(pragmaInstr);
pragmaInstr = nullptr; // Only once per pragma instr -- do we need to make this record?
}
if (instr->HasBailOutInfo())
{
Assert(this->m_func->hasBailout);
Assert(LowererMD::IsCall(instr));
instr->GetBailOutInfo()->FinalizeBailOutRecord(this->m_func);
}
if (instr->isInlineeEntryInstr)
{
m_encoderMD.EncodeInlineeCallInfo(instr, GetCurrentOffset());
}
if (instr->m_opcode == Js::OpCode::InlineeStart)
{
Func* inlinee = instr->m_func;
if (inlinee->frameInfo && inlinee->frameInfo->record)
{
inlinee->frameInfo->record->Finalize(inlinee, GetCurrentOffset());
#if defined(_M_IX86) || defined(_M_X64)
// Store all records to be adjusted for BR shortening
m_inlineeFrameRecords->Add(inlinee->frameInfo->record);
#endif
}
continue;
}
}
count = m_encoderMD.Encode(instr, m_pc, m_encodeBuffer);
#if DBG_DUMP
if (PHASE_TRACE(Js::EncoderPhase, this->m_func))
{
instr->Dump((IRDumpFlags)(IRDumpFlags_SimpleForm | IRDumpFlags_SkipEndLine | IRDumpFlags_SkipByteCodeOffset));
Output::SkipToColumn(80);
for (BYTE * current = m_pc; current < m_pc + count; current++)
{
Output::Print(L"%02X ", *current);
}
Output::Print(L"\n");
Output::Flush();
}
#endif
#ifdef _M_X64
if (inProlog)
m_func->m_prologEncoder.EncodeInstr(instr, count & 0xFF);
#endif
m_pc += count;
#if defined(_M_IX86) || defined(_M_X64)
// for BR shortening.
if (instr->isInlineeEntryInstr)
m_encoderMD.AppendRelocEntry(RelocType::RelocTypeInlineeEntryOffset, (void*) (m_pc - MachPtr));
#endif
if (isCallInstr)
{
isCallInstr = false;
this->RecordInlineeFrame(instr->m_func, GetCurrentOffset());
}
if (instr->HasBailOutInfo() && Lowerer::DoLazyBailout(this->m_func))
{
this->RecordBailout(instr, (uint32)(m_pc - m_encodeBuffer));
}
}
else
{
Fatal();
}
} NEXT_INSTR_IN_FUNC;
ptrdiff_t codeSize = m_pc - m_encodeBuffer + totalJmpTableSizeInBytes;
#if defined(_M_IX86) || defined(_M_X64)
BOOL isSuccessBrShortAndLoopAlign = false;
// Shorten branches. ON by default
if (!PHASE_OFF(Js::BrShortenPhase, m_func))
{
isSuccessBrShortAndLoopAlign = ShortenBranchesAndLabelAlign(&m_encodeBuffer, &codeSize);
}
#endif
#if DBG_DUMP | defined(VTUNE_PROFILING)
if (this->m_func->DoRecordNativeMap())
{
// Record PragmaInstr offsets and throw maps
for (int32 i = 0; i < m_pragmaInstrToRecordOffset->Count(); i++)
{
IR::PragmaInstr *inst = m_pragmaInstrToRecordOffset->Item(i);
inst->Record(inst->m_offsetInBuffer);
}
}
#endif
for (int32 i = 0; i < m_pragmaInstrToRecordMap->Count(); i ++)
{
IR::PragmaInstr *inst = m_pragmaInstrToRecordMap->Item(i);
inst->RecordThrowMap(iter, inst->m_offsetInBuffer);
}
BEGIN_CODEGEN_PHASE(m_func, Js::EmitterPhase);
// Copy to permanent buffer.
Assert(Math::FitsInDWord(codeSize));
ushort xdataSize;
ushort pdataCount;
#ifdef _M_X64
pdataCount = 1;
xdataSize = (ushort)m_func->m_prologEncoder.SizeOfUnwindInfo();
#elif _M_ARM
pdataCount = (ushort)m_func->m_unwindInfo.GetPDataCount(codeSize);
xdataSize = (UnwindInfoManager::MaxXdataBytes + 3) * pdataCount;
#else
xdataSize = 0;
pdataCount = 0;
#endif
OUTPUT_VERBOSE_TRACE(Js::EmitterPhase, L"PDATA count:%u\n", pdataCount);
OUTPUT_VERBOSE_TRACE(Js::EmitterPhase, L"Size of XDATA:%u\n", xdataSize);
OUTPUT_VERBOSE_TRACE(Js::EmitterPhase, L"Size of code:%u\n", codeSize);
TryCopyAndAddRelocRecordsForSwitchJumpTableEntries(m_encodeBuffer, codeSize, jumpTableListForSwitchStatement, totalJmpTableSizeInBytes);
workItem->RecordNativeCodeSize(m_func, (DWORD)codeSize, pdataCount, xdataSize);
this->m_bailoutRecordMap->MapAddress([=](int index, LazyBailOutRecord* record)
{
this->m_encoderMD.AddLabelReloc((BYTE*)&record->instructionPointer);
});
// Relocs
m_encoderMD.ApplyRelocs((size_t) workItem->GetCodeAddress());
workItem->RecordNativeCode(m_func, m_encodeBuffer);
m_func->GetScriptContext()->GetThreadContext()->SetValidCallTargetForCFG((PVOID) workItem->GetCodeAddress());
#ifdef _M_X64
m_func->m_prologEncoder.FinalizeUnwindInfo();
workItem->RecordUnwindInfo(0, m_func->m_prologEncoder.GetUnwindInfo(), m_func->m_prologEncoder.SizeOfUnwindInfo());
#elif _M_ARM
m_func->m_unwindInfo.EmitUnwindInfo(workItem);
workItem->SetCodeAddress(workItem->GetCodeAddress() | 0x1); // Set thumb mode
#endif
Js::EntryPointInfo* entryPointInfo = this->m_func->m_workItem->GetEntryPoint();
const bool isSimpleJit = m_func->IsSimpleJit();
Assert(
isSimpleJit ||
entryPointInfo->GetJitTransferData() != nullptr && !entryPointInfo->GetJitTransferData()->GetIsReady());
if (this->m_inlineeFrameMap->Count() > 0 &&
!(this->m_inlineeFrameMap->Count() == 1 && this->m_inlineeFrameMap->Item(0).record == nullptr))
{
entryPointInfo->RecordInlineeFrameMap(m_inlineeFrameMap);
}
if (this->m_bailoutRecordMap->Count() > 0)
{
entryPointInfo->RecordBailOutMap(m_bailoutRecordMap);
}
if (this->m_func->pinnedTypeRefs != nullptr)
{
Assert(!isSimpleJit);
Func::TypeRefSet* pinnedTypeRefs = this->m_func->pinnedTypeRefs;
int pinnedTypeRefCount = pinnedTypeRefs->Count();
void** compactPinnedTypeRefs = HeapNewArrayZ(void*, pinnedTypeRefCount);
int index = 0;
pinnedTypeRefs->Map([compactPinnedTypeRefs, &index](void* typeRef) -> void
{
compactPinnedTypeRefs[index++] = typeRef;
});
if (PHASE_TRACE(Js::TracePinnedTypesPhase, this->m_func))
{
wchar_t debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
Output::Print(L"PinnedTypes: function %s(%s) pinned %d types.\n",
this->m_func->GetJnFunction()->GetDisplayName(), this->m_func->GetJnFunction()->GetDebugNumberSet(debugStringBuffer), pinnedTypeRefCount);
Output::Flush();
}
entryPointInfo->GetJitTransferData()->SetRuntimeTypeRefs(compactPinnedTypeRefs, pinnedTypeRefCount);
}
// Save all equivalent type guards in a fixed size array on the JIT transfer data
if (this->m_func->equivalentTypeGuards != nullptr)
{
AssertMsg(!PHASE_OFF(Js::EquivObjTypeSpecPhase, this->m_func), "Why do we have equivalent type guards if we don't do equivalent object type spec?");
int count = this->m_func->equivalentTypeGuards->Count();
Js::JitEquivalentTypeGuard** guards = HeapNewArrayZ(Js::JitEquivalentTypeGuard*, count);
Js::JitEquivalentTypeGuard** dstGuard = guards;
this->m_func->equivalentTypeGuards->Map([&dstGuard](Js::JitEquivalentTypeGuard* srcGuard) -> void
{
*dstGuard++ = srcGuard;
});
entryPointInfo->GetJitTransferData()->SetEquivalentTypeGuards(guards, count);
}
if (this->m_func->lazyBailoutProperties.Count() > 0)
{
int count = this->m_func->lazyBailoutProperties.Count();
Js::PropertyId* lazyBailoutProperties = HeapNewArrayZ(Js::PropertyId, count);
Js::PropertyId* dstProperties = lazyBailoutProperties;
this->m_func->lazyBailoutProperties.Map([&](Js::PropertyId propertyId)
{
*dstProperties++ = propertyId;
});
entryPointInfo->GetJitTransferData()->SetLazyBailoutProperties(lazyBailoutProperties, count);
}
// Save all property guards on the JIT transfer data in a map keyed by property ID. We will use this map when installing the entry
// point to register each guard for invalidation.
if (this->m_func->propertyGuardsByPropertyId != nullptr)
{
Assert(!isSimpleJit);
AssertMsg(!(PHASE_OFF(Js::ObjTypeSpecPhase, this->m_func) && PHASE_OFF(Js::FixedMethodsPhase, this->m_func)),
"Why do we have type guards if we don't do object type spec or fixed methods?");
int propertyCount = this->m_func->propertyGuardsByPropertyId->Count();
Assert(propertyCount > 0);
#if DBG
int totalGuardCount = (this->m_func->singleTypeGuards != nullptr ? this->m_func->singleTypeGuards->Count() : 0)
+ (this->m_func->equivalentTypeGuards != nullptr ? this->m_func->equivalentTypeGuards->Count() : 0);
Assert(totalGuardCount > 0);
Assert(totalGuardCount == this->m_func->indexedPropertyGuardCount);
#endif
int guardSlotCount = 0;
this->m_func->propertyGuardsByPropertyId->Map([&guardSlotCount](Js::PropertyId propertyId, Func::IndexedPropertyGuardSet* set) -> void
{
guardSlotCount += set->Count();
});
size_t typeGuardTransferSize = // Reserve enough room for:
propertyCount * sizeof(Js::TypeGuardTransferEntry) + // each propertyId,
propertyCount * sizeof(Js::JitIndexedPropertyGuard*) + // terminating nullptr guard for each propertyId,
guardSlotCount * sizeof(Js::JitIndexedPropertyGuard*); // a pointer for each guard we counted above.
// The extra room for sizeof(Js::TypePropertyGuardEntry) allocated by HeapNewPlus will be used for the terminating invalid propertyId.
// Review (jedmiad): Skip zeroing? This is heap allocated so there shouldn't be any false recycler references.
Js::TypeGuardTransferEntry* typeGuardTransferRecord = HeapNewPlusZ(typeGuardTransferSize, Js::TypeGuardTransferEntry);
Func* func = this->m_func;
Js::TypeGuardTransferEntry* dstEntry = typeGuardTransferRecord;
this->m_func->propertyGuardsByPropertyId->Map([func, &dstEntry](Js::PropertyId propertyId, Func::IndexedPropertyGuardSet* srcSet) -> void
{
dstEntry->propertyId = propertyId;
int guardIndex = 0;
srcSet->Map([dstEntry, &guardIndex](Js::JitIndexedPropertyGuard* guard) -> void
{
dstEntry->guards[guardIndex++] = guard;
});
dstEntry->guards[guardIndex++] = nullptr;
dstEntry = reinterpret_cast<:typeguardtransferentry>(&dstEntry->guards[guardIndex]);
});
dstEntry->propertyId = Js::Constants::NoProperty;
dstEntry++;
Assert(reinterpret_cast