You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2637 lines
75 KiB
2637 lines
75 KiB
/*++
|
|
|
|
Copyright (c) 1996-2000 Microsoft Corporation
|
|
|
|
Module Name:
|
|
|
|
patchfn.c
|
|
|
|
Abstract:
|
|
|
|
This module contains generic functions to patch fragments. Structures
|
|
that describe the fragments to be patched. The structures live in
|
|
the processor specific directory.
|
|
|
|
Author:
|
|
|
|
Dave Hastings (daveh) creation-date 24-Jun-1995
|
|
|
|
Revision History:
|
|
|
|
Barry Bond (barrybo) 1-Apr-1995
|
|
Switch the PPC build to the AXP model of patching
|
|
24-Aug-1999 [askhalid] copied from 32-bit wx86 directory and make work for 64bit.
|
|
20-Sept-1999[barrybo] added FRAG2REF(LockCmpXchg8bFrag32, ULONGLONG)
|
|
|
|
|
|
Notes:
|
|
|
|
--*/
|
|
|
|
#include <nt.h>
|
|
#include <ntrtl.h>
|
|
#include <nturtl.h>
|
|
#include <windows.h>
|
|
|
|
#define _WX86CPUAPI_
|
|
#include "wx86.h"
|
|
#include "wx86nt.h"
|
|
#include "wx86cpu.h"
|
|
#include "cpuassrt.h"
|
|
#include "instr.h"
|
|
#include "config.h"
|
|
#include "fragp.h"
|
|
#include "entrypt.h"
|
|
#include "compiler.h"
|
|
#include "ctrltrns.h"
|
|
#include "threadst.h"
|
|
#include "instr.h"
|
|
#include "frag.h"
|
|
#include "ptchstrc.h"
|
|
#include "mrsw.h"
|
|
#include "tc.h"
|
|
#include "codeseq.h"
|
|
#include "codesize.h"
|
|
#include "opt.h"
|
|
#if _ALPHA_
|
|
#define _codegen_
|
|
#include "soalpha.h"
|
|
#undef fTCUnlocked // this is a field in CPUCONTEXT
|
|
ULONG
|
|
GetCurrentECU(
|
|
PULONG CodeLocation
|
|
);
|
|
#endif
|
|
|
|
ASSERTNAME;
|
|
|
|
extern CHAR CallJxxHelper[];
|
|
extern CHAR CallJmpDirectHelper[];
|
|
extern CHAR IndirectControlTransferHelper[];
|
|
extern CHAR IndirectControlTransferFarHelper[];
|
|
extern CHAR CallDirectHelper[];
|
|
extern CHAR CallDirectHelper2[];
|
|
extern CHAR CallIndirectHelper[];
|
|
extern CHAR JumpToNextCompilationUnitHelper[];
|
|
|
|
#define OFFSET(type, field) ((LONG)(ULONGLONG)(&((type *)0)->field))
|
|
|
|
ULONG RegisterOffset[] = {
|
|
OFFSET(THREADSTATE, GpRegs[GP_EAX].i4), // EAX
|
|
OFFSET(THREADSTATE, GpRegs[GP_ECX].i4), // ECX
|
|
OFFSET(THREADSTATE, GpRegs[GP_EDX].i4), // EDX
|
|
OFFSET(THREADSTATE, GpRegs[GP_EBX].i4), // EBX
|
|
OFFSET(THREADSTATE, GpRegs[GP_ESP].i4), // ESP
|
|
OFFSET(THREADSTATE, GpRegs[GP_EBP].i4), // EBP
|
|
OFFSET(THREADSTATE, GpRegs[GP_ESI].i4), // ESI
|
|
OFFSET(THREADSTATE, GpRegs[GP_EDI].i4), // EDI
|
|
OFFSET(THREADSTATE, GpRegs[REG_ES]), // ES
|
|
OFFSET(THREADSTATE, GpRegs[REG_CS]), // CS
|
|
OFFSET(THREADSTATE, GpRegs[REG_SS]), // SS
|
|
OFFSET(THREADSTATE, GpRegs[REG_DS]), // DS
|
|
OFFSET(THREADSTATE, GpRegs[REG_FS]), // FS
|
|
OFFSET(THREADSTATE, GpRegs[REG_GS]), // GS
|
|
OFFSET(THREADSTATE, GpRegs[GP_EAX].i2), // AX
|
|
OFFSET(THREADSTATE, GpRegs[GP_ECX].i2), // CX
|
|
OFFSET(THREADSTATE, GpRegs[GP_EDX].i2), // DX
|
|
OFFSET(THREADSTATE, GpRegs[GP_EBX].i2), // BX
|
|
OFFSET(THREADSTATE, GpRegs[GP_ESP].i2), // SP
|
|
OFFSET(THREADSTATE, GpRegs[GP_EBP].i2), // BP
|
|
OFFSET(THREADSTATE, GpRegs[GP_ESI].i2), // SI
|
|
OFFSET(THREADSTATE, GpRegs[GP_EDI].i2), // DI
|
|
OFFSET(THREADSTATE, GpRegs[GP_EAX].i1), // AL
|
|
OFFSET(THREADSTATE, GpRegs[GP_ECX].i1), // CL
|
|
OFFSET(THREADSTATE, GpRegs[GP_EDX].i1), // DL
|
|
OFFSET(THREADSTATE, GpRegs[GP_EBX].i1), // BL
|
|
OFFSET(THREADSTATE, GpRegs[GP_EAX].hb), // AH
|
|
OFFSET(THREADSTATE, GpRegs[GP_ECX].hb), // CH
|
|
OFFSET(THREADSTATE, GpRegs[GP_EDX].hb), // DH
|
|
OFFSET(THREADSTATE, GpRegs[GP_EBX].hb) // BH
|
|
};
|
|
|
|
|
|
ULONG
|
|
PatchJumpToNextCompilationUnit(
|
|
IN PULONG PatchAddr,
|
|
IN ULONG IntelDest
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine ends basic blocks when the native address of the next
|
|
basic block is known.
|
|
|
|
Arguments:
|
|
|
|
PatchAddr -- address of JumpToNextCompilationUnit code in the TC
|
|
IntelDest -- intel address of the next basic block
|
|
|
|
Return Value:
|
|
|
|
Native address to jump to to resume execution.
|
|
|
|
--*/
|
|
{
|
|
DWORD TCTimestamp;
|
|
ULONG NativeDest;
|
|
ULONG NativeSize;
|
|
DECLARE_CPU;
|
|
|
|
|
|
if (cpu->flag_tf) {
|
|
return (ULONG)(ULONGLONG)&EndTranslatedCode;
|
|
}
|
|
//
|
|
// Switch from being a TC reader to a writer
|
|
//
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
MrswReaderExit(&MrswTC);
|
|
|
|
//
|
|
// Compile the next basic block, and get the TC write lock
|
|
//
|
|
NativeDest = (ULONG)(ULONGLONG)NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
PULONG CodeLocation = PatchAddr - JumpToNextCompilationUnit_SIZE;
|
|
//
|
|
// The Translation Cache wasn't flushed - replace the
|
|
// JumpToNextCompilationUnit fragment by JumpToNextCompilationUnit2
|
|
//
|
|
NativeSize=GenJumpToNextCompilationUnit2(CodeLocation,
|
|
FALSE, // patching, not compiling
|
|
#if _ALPHA_
|
|
GetCurrentECU(CodeLocation),
|
|
#endif
|
|
NativeDest,
|
|
0);
|
|
|
|
NtFlushInstructionCache(
|
|
NtCurrentProcess(),
|
|
CodeLocation,
|
|
NativeSize
|
|
);
|
|
|
|
} else {
|
|
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
|
|
if (cpu->CSTimestamp != TCTimestamp) {
|
|
//
|
|
// The cache was flushed by another thread in the small window
|
|
// between mrsw calls in this thread, we plan on jumping directly
|
|
// to NativeDest, so the CPU callstack needs to be flushed.
|
|
// Normally this would be done in the CpuSimulate() loop as a
|
|
// result of jumping to EndTranslatedCode.
|
|
//
|
|
FlushCallstack(cpu);
|
|
}
|
|
}
|
|
|
|
//
|
|
// Switch back to being a TC reader
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
//
|
|
// TC was not flushed - NativeDest is valid
|
|
//
|
|
return NativeDest;
|
|
} else {
|
|
//
|
|
// TC was flushed while becomming a TC reader again - NativeDest
|
|
// is now bogus.
|
|
//
|
|
return (ULONG)(ULONGLONG)&EndTranslatedCode;
|
|
}
|
|
}
|
|
|
|
ULONG
|
|
PlaceJxx(
|
|
IN PULONG CodeLocation,
|
|
#if _ALPHA_
|
|
IN ULONG CurrentECU,
|
|
#endif
|
|
IN PINSTRUCTION Instruction
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine copies the fragment into place, and modifies the
|
|
instructions that load the destination into the register
|
|
|
|
Arguments:
|
|
|
|
Instruction - Supplies a description of the instruction the fragment
|
|
represents
|
|
CodeLocation - Supplies the address the code for the fragment has been
|
|
copied to
|
|
|
|
Return Value:
|
|
|
|
Size of code placed at CodeLocation
|
|
|
|
--*/
|
|
{
|
|
DWORD IntelDest;
|
|
PPLACEOPERATIONFN pfn;
|
|
PENTRYPOINT EP;
|
|
ULONG NativeSize;
|
|
|
|
//
|
|
// Generate the code to determine if the branch is taken or not
|
|
//
|
|
pfn = (PPLACEOPERATIONFN)FragmentArray[Instruction->Operation];
|
|
NativeSize = (*pfn)(CodeLocation,
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
Instruction);
|
|
CodeLocation += NativeSize/sizeof(ULONG);
|
|
|
|
// Let's see if we can place the patched version immediately
|
|
|
|
// ASSUME: The first argument is always a NOCODEGEN
|
|
CPUASSERT( Instruction->Operand1.Type == OPND_NOCODEGEN );
|
|
IntelDest = Instruction->Operand1.Immed;
|
|
|
|
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
|
|
|
|
if (EP == NULL) {
|
|
//
|
|
// knowing NativeDest requires compilation. Just place the unpatched
|
|
// version to be patched later.
|
|
//
|
|
NativeSize += GenJxxBody(CodeLocation,
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
Instruction);
|
|
|
|
} else {
|
|
//
|
|
// We can place the patched version right away!
|
|
//
|
|
NativeSize += GenJxxBody2(CodeLocation,
|
|
TRUE, // compiling, not patching
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
(ULONG)IntelDest,
|
|
(ULONG)(ULONGLONG)EP);
|
|
}
|
|
return NativeSize;
|
|
}
|
|
|
|
|
|
ULONG
|
|
PlaceJxxSlow(
|
|
IN PULONG CodeLocation,
|
|
#if _ALPHA_
|
|
IN ULONG CurrentECU,
|
|
#endif
|
|
IN PINSTRUCTION Instruction
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine copies the fragment into place, and modifies the
|
|
instructions that load the destination into the register
|
|
|
|
Arguments:
|
|
|
|
Instruction - Supplies a description of the instruction the fragment
|
|
represents
|
|
CodeLocation - Supplies the address the code for the fragment has been
|
|
copied to
|
|
|
|
Return Value:
|
|
|
|
Size of code placed at CodeLocation
|
|
|
|
--*/
|
|
{
|
|
DWORD IntelDest;
|
|
PPLACEOPERATIONFN pfn;
|
|
PENTRYPOINT EP;
|
|
ULONG NativeSize;
|
|
|
|
//
|
|
// Generate the code to load RegEip with the branch-not-taken value
|
|
//
|
|
NativeSize = GenJxxStartSlow(CodeLocation,
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
Instruction);
|
|
|
|
//
|
|
// Generate the code to determine if the branch is taken or not
|
|
//
|
|
pfn = (PPLACEOPERATIONFN)FragmentArray[Instruction->Operation];
|
|
NativeSize += (*pfn)(CodeLocation+NativeSize/sizeof(ULONG),
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
Instruction);
|
|
CodeLocation += NativeSize/sizeof(ULONG);
|
|
|
|
// Let's see if we can place the patched version immediately
|
|
|
|
// ASSUME: The first argument is always a NOCODEGEN
|
|
CPUASSERT( Instruction->Operand1.Type == OPND_NOCODEGEN );
|
|
IntelDest = Instruction->Operand1.Immed;
|
|
|
|
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
|
|
|
|
if (EP == NULL) {
|
|
//
|
|
// knowing NativeDest requires compilation. Just place the unpatched
|
|
// version to be patched later.
|
|
//
|
|
NativeSize += GenJxxBodySlow(CodeLocation,
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
Instruction);
|
|
|
|
} else {
|
|
//
|
|
// We can place the patched version right away!
|
|
//
|
|
NativeSize += GenJxxBodySlow2(CodeLocation,
|
|
TRUE, // compiling, not patching
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
(ULONG)IntelDest,
|
|
(ULONG)(ULONGLONG)EP);
|
|
}
|
|
return NativeSize;
|
|
}
|
|
|
|
|
|
ULONG
|
|
PlaceJxxFwd(
|
|
IN PULONG CodeLocation,
|
|
#if _ALPHA_
|
|
IN ULONG CurrentECU,
|
|
#endif
|
|
IN PINSTRUCTION Instruction
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine copies the fragment into place, and modifies the
|
|
instructions that load the destination into the register
|
|
|
|
Arguments:
|
|
|
|
Instruction - Supplies a description of the instruction the fragment
|
|
represents
|
|
CodeLocation - Supplies the address the code for the fragment has been
|
|
copied to
|
|
|
|
Return Value:
|
|
|
|
Size of code placed at CodeLocation
|
|
|
|
--*/
|
|
{
|
|
DWORD IntelDest;
|
|
PPLACEOPERATIONFN pfn;
|
|
PENTRYPOINT EP;
|
|
ULONG NativeSize;
|
|
|
|
//
|
|
// Generate the code to determine if the branch is taken or not
|
|
//
|
|
pfn = (PPLACEOPERATIONFN)FragmentArray[Instruction->Operation];
|
|
NativeSize = (*pfn)(CodeLocation,
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
Instruction);
|
|
CodeLocation += NativeSize/sizeof(ULONG);
|
|
|
|
|
|
// Let's see if we can place the patched version immediately
|
|
|
|
// ASSUME: The first argument is always a NOCODEGEN
|
|
CPUASSERT( Instruction->Operand1.Type == OPND_NOCODEGEN );
|
|
IntelDest = Instruction->Operand1.Immed;
|
|
|
|
// Assert that the branch is going forward.
|
|
CPUASSERT(IntelDest > Instruction->IntelAddress);
|
|
|
|
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
|
|
|
|
if (EP == NULL) {
|
|
//
|
|
// knowing NativeDest requires compilation. Just place the unpatched
|
|
// version to be patched later.
|
|
NativeSize += GenJxxBodyFwd(CodeLocation,
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
Instruction);
|
|
|
|
} else {
|
|
//
|
|
// We can place the patched version right away!
|
|
//
|
|
NativeSize += GenJxxBodyFwd2(CodeLocation,
|
|
TRUE, // compiling, not patching
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
(ULONG)(ULONGLONG)EP,
|
|
0);
|
|
}
|
|
return NativeSize;
|
|
}
|
|
|
|
|
|
ULONG
|
|
PatchJxx(
|
|
IN ULONG IntelDest,
|
|
IN PULONG PatchAddr
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine replaces a JXXSTRUC by a JXXSTRC2 at runtime. It is called
|
|
when the conditional branch is taken, and the native address of the
|
|
destination is not yet known.
|
|
|
|
Arguments:
|
|
|
|
inteldest -- Intel destination address if the branch is taken
|
|
patchaddr -- address of the JXXSTRUC in the Translation Cache
|
|
|
|
Return Value:
|
|
|
|
Native address to jump to in order to resume execution.
|
|
|
|
--*/
|
|
{
|
|
ULONG NativeDest; // branch-taken address
|
|
PULONG fragaddr; // address of START of the fragment
|
|
DWORD TCTimestamp; // old timestamp of the Translation Cache
|
|
ULONG NativeSize;
|
|
DECLARE_CPU;
|
|
|
|
|
|
if (cpu->flag_tf) {
|
|
return (ULONG)(ULONGLONG)&EndTranslatedCode;
|
|
}
|
|
|
|
//
|
|
// Switch over to being a TC writer
|
|
//
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
MrswReaderExit(&MrswTC);
|
|
|
|
//
|
|
// Get the native destination address of the branch and get the TC
|
|
// write lock
|
|
//
|
|
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
PULONG CodeLocation = PatchAddr - CallJxx_PATCHRA_OFFSET;
|
|
//
|
|
// The Translation Cache was not flushed while switching to a TC
|
|
// writer or by the compilation. Replace JxxBody by the faster
|
|
// JxxBody2
|
|
//
|
|
NativeSize=GenJxxBody2(CodeLocation,
|
|
FALSE, // patching, not compiling
|
|
#if _ALPHA_
|
|
GetCurrentECU(CodeLocation),
|
|
#endif
|
|
(DWORD)IntelDest,
|
|
(DWORD)NativeDest);
|
|
|
|
NtFlushInstructionCache(
|
|
NtCurrentProcess(),
|
|
CodeLocation,
|
|
NativeSize
|
|
);
|
|
|
|
} else {
|
|
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
|
|
if (cpu->CSTimestamp != TCTimestamp) {
|
|
//
|
|
// The cache was flushed by another thread in the small window
|
|
// between mrsw calls in this thread, we plan on jumping directly
|
|
// to NativeDest, so the CPU callstack needs to be flushed.
|
|
// Normally this would be done in the CpuSimulate() loop as a
|
|
// result of jumping to EndTranslatedCode.
|
|
//
|
|
FlushCallstack(cpu);
|
|
}
|
|
}
|
|
|
|
//
|
|
// Switch back to being a TC reader again
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
//
|
|
// TC was not flushed while becomming a reader again
|
|
//
|
|
return NativeDest;
|
|
} else {
|
|
//
|
|
// TC was flushed while becomming a reader. nativedest is invalid
|
|
// so do an EndTranslatedCode instead.
|
|
//
|
|
return (ULONG)(ULONGLONG) &EndTranslatedCode;
|
|
}
|
|
}
|
|
|
|
|
|
ULONG
|
|
PatchJxxSlow(
|
|
IN ULONG IntelDest,
|
|
IN PULONG PatchAddr
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine replaces a JXXSTRUC by a JXXSTRC2 at runtime. It is called
|
|
when the conditional branch is taken, and the native address of the
|
|
destination is not yet known.
|
|
|
|
Arguments:
|
|
|
|
inteldest -- Intel destination address if the branch is taken
|
|
patchaddr -- address of the JXXSTRUC in the Translation Cache
|
|
|
|
Return Value:
|
|
|
|
Native address to jump to in order to resume execution.
|
|
|
|
--*/
|
|
{
|
|
ULONG NativeDest; // branch-taken address
|
|
PULONG fragaddr; // address of START of the fragment
|
|
DWORD TCTimestamp; // old timestamp of the Translation Cache
|
|
ULONG NativeSize;
|
|
DECLARE_CPU;
|
|
|
|
|
|
if (cpu->flag_tf) {
|
|
return (ULONG)(ULONGLONG)&EndTranslatedCode;
|
|
}
|
|
|
|
//
|
|
// Switch over to being a TC writer
|
|
//
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
MrswReaderExit(&MrswTC);
|
|
|
|
//
|
|
// Get the native destination address of the branch and get the TC
|
|
// write lock
|
|
//
|
|
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
PULONG CodeLocation = PatchAddr - CallJxxSlow_PATCHRA_OFFSET;
|
|
//
|
|
// The Translation Cache was not flushed while switching to a TC
|
|
// writer or by the compilation. Replace JxxBody by the faster
|
|
// JxxBody2
|
|
//
|
|
NativeSize = GenJxxBodySlow2(CodeLocation,
|
|
FALSE, // patching, not compiling
|
|
#if _ALPHA_
|
|
GetCurrentECU(CodeLocation),
|
|
#endif
|
|
(DWORD)IntelDest,
|
|
(DWORD)NativeDest);
|
|
|
|
NtFlushInstructionCache(
|
|
NtCurrentProcess(),
|
|
CodeLocation,
|
|
NativeSize
|
|
);
|
|
|
|
} else {
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
|
|
if (cpu->CSTimestamp != TCTimestamp) {
|
|
//
|
|
// The cache was flushed by another thread in the small window
|
|
// between mrsw calls in this thread, we plan on jumping directly
|
|
// to NativeDest, so the CPU callstack needs to be flushed.
|
|
// Normally this would be done in the CpuSimulate() loop as a
|
|
// result of jumping to EndTranslatedCode.
|
|
//
|
|
FlushCallstack(cpu);
|
|
}
|
|
}
|
|
|
|
//
|
|
// Switch back to being a TC reader again
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
//
|
|
// TC was not flushed while becomming a reader again
|
|
//
|
|
return NativeDest;
|
|
} else {
|
|
//
|
|
// TC was flushed while becomming a reader. nativedest is invalid
|
|
// so do an EndTranslatedCode instead.
|
|
//
|
|
return (ULONG)(ULONGLONG) &EndTranslatedCode;
|
|
}
|
|
}
|
|
|
|
|
|
ULONG
|
|
PatchJxxFwd(
|
|
IN ULONG IntelDest,
|
|
IN PULONG PatchAddr
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine replaces a JXXBODYFWD by a JXXBODYFWD2 at runtime. It is
|
|
called when the conditional branch is taken, and the native address of the
|
|
destination is not yet known.
|
|
|
|
Arguments:
|
|
|
|
inteldest -- Intel destination address if the branch is taken
|
|
patchaddr -- address of the JXXSTRUCFWD in the Translation Cache
|
|
|
|
Return Value:
|
|
|
|
Native address to jump to in order to resume execution.
|
|
|
|
--*/
|
|
{
|
|
ULONG NativeDest; // branch-taken address
|
|
PULONG fragaddr; // address of START of the fragment
|
|
DWORD TCTimestamp; // old timestamp of the Translation Cache
|
|
ULONG NativeSize;
|
|
DECLARE_CPU;
|
|
|
|
if (cpu->flag_tf) {
|
|
return (ULONG)(ULONGLONG)&EndTranslatedCode;
|
|
}
|
|
|
|
//
|
|
// Switch over to being a TC writer
|
|
//
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
MrswReaderExit(&MrswTC);
|
|
|
|
//
|
|
// Get the native destination address of the branch and get the TC write lock
|
|
//
|
|
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
PULONG CodeLocation = PatchAddr - CallJxxFwd_PATCHRA_OFFSET;
|
|
//
|
|
// The Translation Cache was not flushed while switching to a TC
|
|
// writer or by the compilation. Replace JxxBody by the faster
|
|
// JxxBodyFwd2
|
|
//
|
|
NativeSize = GenJxxBodyFwd2(CodeLocation,
|
|
FALSE, // patching, not compiling
|
|
#if _ALPHA_
|
|
GetCurrentECU(CodeLocation),
|
|
#endif
|
|
(DWORD)NativeDest,
|
|
0);
|
|
|
|
NtFlushInstructionCache(
|
|
NtCurrentProcess(),
|
|
CodeLocation,
|
|
NativeSize
|
|
);
|
|
|
|
} else {
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
|
|
if (cpu->CSTimestamp != TCTimestamp) {
|
|
//
|
|
// The cache was flushed by another thread in the small window
|
|
// between mrsw calls in this thread, we plan on jumping directly
|
|
// to NativeDest, so the CPU callstack needs to be flushed.
|
|
// Normally this would be done in the CpuSimulate() loop as a
|
|
// result of jumping to EndTranslatedCode.
|
|
//
|
|
FlushCallstack(cpu);
|
|
}
|
|
}
|
|
|
|
//
|
|
// Switch back to being a TC reader again
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
//
|
|
// TC was not flushed while becomming a reader again
|
|
//
|
|
return NativeDest;
|
|
} else {
|
|
//
|
|
// TC was flushed while becomming a reader. nativedest is invalid
|
|
// so do an EndTranslatedCode instead.
|
|
//
|
|
return (ULONG)(ULONGLONG) &EndTranslatedCode;
|
|
}
|
|
}
|
|
|
|
|
|
ULONG
|
|
PlaceJmpDirect(
|
|
IN PULONG CodeLocation,
|
|
#if _ALPHA_
|
|
IN ULONG CurrentECU,
|
|
#endif
|
|
IN PINSTRUCTION Instruction
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine copies the unconditional jump fragment into place and patches
|
|
the instructions that jump to EndTranslatedCode
|
|
|
|
Arguments:
|
|
|
|
Instruction - Supplies a description of the instruction the fragment
|
|
represents
|
|
CodeLocation - Supplies the address the code for the fragment has been
|
|
copied to
|
|
|
|
Return Value:
|
|
|
|
Size of code placed at CodeLocation
|
|
|
|
--*/
|
|
{
|
|
DWORD IntelDest;
|
|
PENTRYPOINT EP;
|
|
ULONG NativeSize;
|
|
|
|
// ASSUME: The first argument is always an immediate
|
|
CPUASSERT( Instruction->Operand1.Type == OPND_NOCODEGEN );
|
|
|
|
IntelDest = Instruction->Operand1.Immed;
|
|
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
|
|
|
|
if (EP == NULL) {
|
|
//
|
|
// Knowing NativeDest requires compilation. Just place the unpatched version for
|
|
// now and patch it later if necessary
|
|
//
|
|
NativeSize = GenCallJmpDirect(CodeLocation,
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
Instruction);
|
|
|
|
} else {
|
|
//
|
|
// We can place the patched version right away!
|
|
//
|
|
NativeSize = GenCallJmpDirect2(CodeLocation,
|
|
TRUE, // compiling, not patching
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
(ULONG)(ULONGLONG) EP,
|
|
IntelDest);
|
|
}
|
|
|
|
return NativeSize;
|
|
}
|
|
|
|
|
|
ULONG
|
|
PatchJmpDirect(
|
|
IN PULONG PatchAddr,
|
|
IN ULONG IntelDest
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine patches a JMPDIRECT to a JMPDIRECT2. It is called when
|
|
the native destination address of a jmp instruction is not yet known.
|
|
It patches the jmp to jump directly to the corresponding native code.
|
|
|
|
Arguments:
|
|
|
|
PatchAddr -- address of the JMPDIRECT in the Translation Cache
|
|
IntelDest -- intel address of the destination of the jmp
|
|
|
|
Return Value:
|
|
|
|
Native address to jump to in order to resume execution
|
|
|
|
--*/
|
|
{
|
|
DWORD TCTimestamp;
|
|
ULONG NativeDest;
|
|
ULONG NativeSize;
|
|
DECLARE_CPU;
|
|
|
|
if (cpu->flag_tf) {
|
|
return (ULONG)(ULONGLONG)&EndTranslatedCode;
|
|
}
|
|
|
|
//
|
|
// Switch from being a TC reader to a writer
|
|
//
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
MrswReaderExit(&MrswTC);
|
|
|
|
//
|
|
// Compile the destination of the jmp and get the TC write lock
|
|
//
|
|
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
PULONG CodeLocation = PatchAddr - CallJmpDirect_PATCHRA_OFFSET;
|
|
//
|
|
// The Translation Cache wasn't flushed - replace the JMPDIRECT
|
|
// fragment by JMPDIRECT2
|
|
//
|
|
NativeSize = GenCallJmpDirect2(CodeLocation,
|
|
FALSE, // patching, not compiling
|
|
#if _ALPHA_
|
|
GetCurrentECU(CodeLocation),
|
|
#endif
|
|
(ULONG)NativeDest,
|
|
IntelDest);
|
|
NtFlushInstructionCache(
|
|
NtCurrentProcess(),
|
|
(PVOID)CodeLocation,
|
|
NativeSize
|
|
);
|
|
} else {
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
|
|
if (cpu->CSTimestamp != TCTimestamp) {
|
|
//
|
|
// The cache was flushed by another thread in the small window
|
|
// between mrsw calls in this thread, we plan on jumping directly
|
|
// to NativeDest, so the CPU callstack needs to be flushed.
|
|
// Normally this would be done in the CpuSimulate() loop as a
|
|
// result of jumping to EndTranslatedCode.
|
|
//
|
|
FlushCallstack(cpu);
|
|
}
|
|
}
|
|
|
|
//
|
|
// Switch back to being a TC reader
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
//
|
|
// TC was not flushed - nativedest is valid
|
|
//
|
|
return NativeDest;
|
|
} else {
|
|
//
|
|
// TC was flushed while becomming a TC reader again - nativedest
|
|
// is now bogus.
|
|
//
|
|
return (ULONG)(ULONGLONG) &EndTranslatedCode;
|
|
}
|
|
}
|
|
|
|
ULONG
|
|
PlaceJmpDirectSlow(
|
|
IN PULONG CodeLocation,
|
|
#if _ALPHA_
|
|
IN ULONG CurrentECU,
|
|
#endif
|
|
IN PINSTRUCTION Instruction
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine copies the unconditional jump fragment into place and patches
|
|
the instructions that jump to EndTranslatedCode
|
|
|
|
Arguments:
|
|
|
|
Instruction - Supplies a description of the instruction the fragment
|
|
represents
|
|
CodeLocation - Supplies the address the code for the fragment has been
|
|
copied to
|
|
|
|
Return Value:
|
|
|
|
Size of code placed at CodeLocation
|
|
|
|
--*/
|
|
{
|
|
DWORD IntelDest;
|
|
PENTRYPOINT EP;
|
|
ULONG NativeSize;
|
|
|
|
// ASSUME: The first argument is always an immediate
|
|
CPUASSERT( Instruction->Operand1.Type == OPND_NOCODEGEN );
|
|
|
|
IntelDest = Instruction->Operand1.Immed;
|
|
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
|
|
|
|
if (EP == NULL) {
|
|
//
|
|
// Knowing NativeDest requires compilation. Just place the unpatched version for
|
|
// now and patch it later if necessary
|
|
//
|
|
NativeSize = GenCallJmpDirectSlow(CodeLocation,
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
Instruction);
|
|
|
|
} else {
|
|
//
|
|
// We can place the patched version right away!
|
|
//
|
|
NativeSize = GenCallJmpDirectSlow2(CodeLocation,
|
|
TRUE, // compiling, not patching
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
(ULONG)(ULONGLONG) EP,
|
|
IntelDest);
|
|
}
|
|
|
|
return NativeSize;
|
|
}
|
|
|
|
|
|
ULONG
|
|
PatchJmpDirectSlow(
|
|
IN PULONG PatchAddr,
|
|
IN ULONG IntelDest
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine patches a JMPDIRECT to a JMPDIRECT2. It is called when
|
|
the native destination address of a jmp instruction is not yet known.
|
|
It patches the jmp to jump directly to the corresponding native code.
|
|
|
|
Arguments:
|
|
|
|
PatchAddr -- address of the JMPDIRECT in the Translation Cache
|
|
IntelDest -- intel address of the destination of the jmp
|
|
|
|
Return Value:
|
|
|
|
Native address to jump to in order to resume execution
|
|
|
|
--*/
|
|
{
|
|
DWORD TCTimestamp;
|
|
ULONG NativeDest;
|
|
ULONG NativeSize;
|
|
DECLARE_CPU;
|
|
|
|
|
|
if (cpu->flag_tf) {
|
|
return (ULONG)(ULONGLONG)&EndTranslatedCode;
|
|
}
|
|
|
|
//
|
|
// Switch from being a TC reader to a writer
|
|
//
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
MrswReaderExit(&MrswTC);
|
|
|
|
//
|
|
// Compile the destination of the jmp and get the TC write lock
|
|
//
|
|
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
PULONG CodeLocation = PatchAddr - CallJmpDirectSlow_PATCHRA_OFFSET;
|
|
//
|
|
// The Translation Cache wasn't flushed - replace the JMPDIRECT
|
|
// fragment by JMPDIRECT2
|
|
//
|
|
NativeSize = GenCallJmpDirectSlow2(CodeLocation,
|
|
FALSE, // patching, not compiling
|
|
#if _ALPHA_
|
|
GetCurrentECU(CodeLocation),
|
|
#endif
|
|
(ULONG)NativeDest,
|
|
IntelDest);
|
|
NtFlushInstructionCache(
|
|
NtCurrentProcess(),
|
|
(PVOID)CodeLocation,
|
|
NativeSize
|
|
);
|
|
} else {
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
|
|
if (cpu->CSTimestamp != TCTimestamp) {
|
|
//
|
|
// The cache was flushed by another thread in the small window
|
|
// between mrsw calls in this thread, we plan on jumping directly
|
|
// to NativeDest, so the CPU callstack needs to be flushed.
|
|
// Normally this would be done in the CpuSimulate() loop as a
|
|
// result of jumping to EndTranslatedCode.
|
|
//
|
|
FlushCallstack(cpu);
|
|
}
|
|
}
|
|
|
|
//
|
|
// Switch back to being a TC reader
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
//
|
|
// TC was not flushed - nativedest is valid
|
|
//
|
|
return NativeDest;
|
|
} else {
|
|
//
|
|
// TC was flushed while becomming a TC reader again - nativedest
|
|
// is now bogus.
|
|
//
|
|
return (ULONG)(ULONGLONG) &EndTranslatedCode;
|
|
}
|
|
}
|
|
|
|
ULONG
|
|
PlaceJmpFwdDirect(
|
|
IN PULONG CodeLocation,
|
|
#if _ALPHA_
|
|
IN ULONG CurrentECU,
|
|
#endif
|
|
IN PINSTRUCTION Instruction
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine copies the unconditional jump fragment into place and patches
|
|
the instructions that jump to EndTranslatedCode
|
|
|
|
Arguments:
|
|
|
|
Instruction - Supplies a description of the instruction the fragment
|
|
represents
|
|
CodeLocation - Supplies the address the code for the fragment has been
|
|
copied to
|
|
|
|
Return Value:
|
|
|
|
Size of code placed at CodeLocation
|
|
|
|
--*/
|
|
{
|
|
DWORD IntelDest;
|
|
PENTRYPOINT EP;
|
|
ULONG NativeSize;
|
|
|
|
// ASSUME: The first argument is always an immediate
|
|
CPUASSERT( Instruction->Operand1.Type == OPND_NOCODEGEN );
|
|
|
|
IntelDest = Instruction->Operand1.Immed;
|
|
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
|
|
|
|
if (EP == NULL) {
|
|
//
|
|
// Knowing NativeDest requires compilation. Just place the unpatched version for
|
|
// now and patch it later if necessary
|
|
//
|
|
NativeSize = GenCallJmpFwdDirect(CodeLocation,
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
Instruction);
|
|
|
|
} else {
|
|
//
|
|
// We can place the patched version right away!
|
|
//
|
|
NativeSize = GenCallJmpFwdDirect2(CodeLocation,
|
|
TRUE, // compiling, not patching
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
(ULONG)(ULONGLONG) EP,
|
|
0);
|
|
}
|
|
|
|
return NativeSize;
|
|
}
|
|
|
|
|
|
ULONG
|
|
PatchJmpFwdDirect(
|
|
IN PULONG PatchAddr,
|
|
IN ULONG IntelDest
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine patches a JMPDIRECT to a JMPDIRECT2. It is called when
|
|
the native destination address of a jmp instruction is not yet known.
|
|
It patches the jmp to jump directly to the corresponding native code.
|
|
|
|
Arguments:
|
|
|
|
PatchAddr -- address of the JMPDIRECT in the Translation Cache
|
|
IntelDest -- intel address of the destination of the jmp
|
|
|
|
Return Value:
|
|
|
|
Native address to jump to in order to resume execution
|
|
|
|
--*/
|
|
{
|
|
DWORD TCTimestamp;
|
|
ULONG NativeDest;
|
|
ULONG NativeSize;
|
|
DECLARE_CPU;
|
|
|
|
if (cpu->flag_tf) {
|
|
return (ULONG)(ULONGLONG)&EndTranslatedCode;
|
|
}
|
|
|
|
//
|
|
// Switch from being a TC reader to a writer
|
|
//
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
MrswReaderExit(&MrswTC);
|
|
|
|
//
|
|
// Compile the destination of the jmp and get the TC write lock
|
|
//
|
|
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
PULONG CodeLocation = PatchAddr - CallJmpFwdDirect_PATCHRA_OFFSET;
|
|
//
|
|
// The Translation Cache wasn't flushed - replace the JMPDIRECT
|
|
// fragment by JMPDIRECT2
|
|
//
|
|
NativeSize = GenCallJmpFwdDirect2(CodeLocation,
|
|
FALSE, // patching, not compiling
|
|
#if _ALPHA_
|
|
GetCurrentECU(CodeLocation),
|
|
#endif
|
|
(ULONG)NativeDest,
|
|
0);
|
|
NtFlushInstructionCache(
|
|
NtCurrentProcess(),
|
|
(PVOID)CodeLocation,
|
|
NativeSize
|
|
);
|
|
} else {
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
|
|
if (cpu->CSTimestamp != TCTimestamp) {
|
|
//
|
|
// The cache was flushed by another thread in the small window
|
|
// between mrsw calls in this thread, we plan on jumping directly
|
|
// to NativeDest, so the CPU callstack needs to be flushed.
|
|
// Normally this would be done in the CpuSimulate() loop as a
|
|
// result of jumping to EndTranslatedCode.
|
|
//
|
|
FlushCallstack(cpu);
|
|
}
|
|
}
|
|
|
|
//
|
|
// Switch back to being a TC reader
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
//
|
|
// TC was not flushed - nativedest is valid
|
|
//
|
|
return NativeDest;
|
|
} else {
|
|
//
|
|
// TC was flushed while becomming a TC reader again - nativedest
|
|
// is now bogus.
|
|
//
|
|
return (ULONG)(ULONGLONG) &EndTranslatedCode;
|
|
}
|
|
}
|
|
|
|
ULONG
|
|
PlaceJmpfDirect(
|
|
IN PULONG CodeLocation,
|
|
#if _ALPHA_
|
|
IN ULONG CurrentECU,
|
|
#endif
|
|
IN PINSTRUCTION Instruction
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine copies the unconditional jump fragment into place and patches
|
|
the instructions that jump to EndTranslatedCode
|
|
|
|
Arguments:
|
|
|
|
Instruction - Supplies a description of the instruction the fragment
|
|
represents
|
|
CodeLocation - Supplies the address the code for the fragment has been
|
|
copied to
|
|
|
|
Return Value:
|
|
|
|
Size of code placed at CodeLocation
|
|
|
|
--*/
|
|
{
|
|
DWORD IntelDest;
|
|
PENTRYPOINT EP;
|
|
ULONG NativeSize;
|
|
|
|
// ASSUME: The first argument is always an IMM, pointing at the address
|
|
CPUASSERT( Instruction->Operand1.Type == OPND_IMM );
|
|
IntelDest = Instruction->Operand1.Immed;
|
|
|
|
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)*(UNALIGNED DWORD *)IntelDest);
|
|
|
|
if (EP == NULL) {
|
|
//
|
|
// Knowing NativeDest requires compilation. Just place the unpatched
|
|
// version for now and patch it later if necessary
|
|
//
|
|
NativeSize = GenCallJmpfDirect(CodeLocation,
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
Instruction);
|
|
|
|
} else {
|
|
//
|
|
// We can place the patched version right away!
|
|
//
|
|
NativeSize = GenCallJmpfDirect2(CodeLocation,
|
|
TRUE, // compiling, not patching
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
(ULONG)(ULONGLONG) EP,
|
|
0);
|
|
}
|
|
|
|
return NativeSize;
|
|
}
|
|
|
|
|
|
|
|
ULONG
|
|
PatchJmpfDirect(
|
|
PTHREADSTATE cpu,
|
|
IN PULONG pIntelDest,
|
|
IN PULONG PatchAddr
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine patches a JMPFDIRECT to a JMPFDIRECT2. It is called when
|
|
the native destination address of a jmp instruction is not yet known.
|
|
It patches the jmp to jump directly to the corresponding native code.
|
|
|
|
Arguments:
|
|
|
|
PatchAddr -- address of the JMPDIRECT in the Translation Cache
|
|
pIntelDest -- intel address of the destination of the jmp
|
|
|
|
Return Value:
|
|
|
|
Native address to jump to in order to resume execution
|
|
|
|
--*/
|
|
{
|
|
DWORD TCTimestamp;
|
|
ULONG NativeDest;
|
|
PVOID IntelDest;
|
|
USHORT Sel;
|
|
ULONG NativeSize;
|
|
|
|
if (cpu->flag_tf) {
|
|
return (ULONG)(ULONGLONG)&EndTranslatedCode;
|
|
}
|
|
|
|
//
|
|
// Switch from being a TC reader to a writer
|
|
//
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
MrswReaderExit(&MrswTC);
|
|
|
|
//
|
|
// Compile the destination of the jmp and get the TC write lock
|
|
//
|
|
IntelDest = (PVOID)*(UNALIGNED DWORD *)pIntelDest;
|
|
Sel = *(UNALIGNED PUSHORT)(pIntelDest+1);
|
|
eip = (ULONG)(ULONGLONG) IntelDest;
|
|
CS = Sel;
|
|
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip(IntelDest, TRUE)->nativeStart;
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
PULONG CodeLocation = PatchAddr - CallJmpfDirect_PATCHRA_OFFSET;
|
|
//
|
|
// The Translation Cache wasn't flushed - replace the JMPFDIRECT
|
|
// fragment by JMPFDIRECT2
|
|
//
|
|
NativeSize = GenCallJmpfDirect2(CodeLocation,
|
|
FALSE, // patching, not compiling
|
|
#if _ALPHA_
|
|
GetCurrentECU(CodeLocation),
|
|
#endif
|
|
(ULONG)NativeDest,
|
|
0);
|
|
NtFlushInstructionCache(
|
|
NtCurrentProcess(),
|
|
(PVOID)CodeLocation,
|
|
NativeSize
|
|
);
|
|
} else {
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
|
|
if (cpu->CSTimestamp != TCTimestamp) {
|
|
//
|
|
// The cache was flushed by another thread in the small window
|
|
// between mrsw calls in this thread, we plan on jumping directly
|
|
// to NativeDest, so the CPU callstack needs to be flushed.
|
|
// Normally this would be done in the CpuSimulate() loop as a
|
|
// result of jumping to EndTranslatedCode.
|
|
//
|
|
FlushCallstack(cpu);
|
|
}
|
|
}
|
|
|
|
//
|
|
// Switch back to being a TC reader
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
//
|
|
// TC was not flushed - nativedest is valid
|
|
//
|
|
return NativeDest;
|
|
} else {
|
|
//
|
|
// TC was flushed while becomming a TC reader again - nativedest
|
|
// is now bogus.
|
|
//
|
|
return (ULONG)(ULONGLONG) &EndTranslatedCode;
|
|
}
|
|
}
|
|
|
|
ULONG
|
|
PlaceCallDirect(
|
|
IN PULONG CodeLocation,
|
|
#if _ALPHA_
|
|
IN ULONG CurrentECU,
|
|
#endif
|
|
IN PINSTRUCTION Instruction
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine copies the unconditional call fragment into place.
|
|
|
|
Arguments:
|
|
|
|
Instruction - Supplies a description of the instruction the fragment
|
|
represents
|
|
CodeLocation - Supplies the address the code for the fragment has been
|
|
copied to
|
|
|
|
Return Value:
|
|
|
|
Size of code placed at CodeLocation
|
|
|
|
--*/
|
|
{
|
|
DWORD IntelDest;
|
|
DWORD IntelNext;
|
|
PENTRYPOINT EP;
|
|
|
|
// ASSUME: The first argument is always an immediate
|
|
CPUASSERT( Instruction->Operand1.Type == OPND_IMM );
|
|
|
|
IntelDest = Instruction->Operand1.Immed;
|
|
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
|
|
|
|
if (EP == NULL) {
|
|
//
|
|
// Knowing NativeDest requires compilation. Just place the unpatched
|
|
// version for now and patch it later if necessary
|
|
//
|
|
GenCallDirect(CodeLocation,
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
Instruction);
|
|
} else {
|
|
//
|
|
// We can place the patched version right away! Deterime if the
|
|
// NativeNext address is known.
|
|
//
|
|
IntelNext = Instruction->Operand2.Immed;
|
|
|
|
//
|
|
// If the current instruction is not the last one compiled, then
|
|
// NativeNext is CodeLocation+CallDirect_SIZE and CallDirect3 can
|
|
// be placed right away.
|
|
//
|
|
if (Instruction != &InstructionStream[NumberOfInstructions-1]) {
|
|
GenCallDirect3(CodeLocation,
|
|
TRUE, // compiling, not patching
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
(ULONG)(ULONGLONG) EP,
|
|
(ULONG)(ULONGLONG) (CodeLocation+CallDirect_SIZE));
|
|
} else {
|
|
GenCallDirect2(CodeLocation,
|
|
TRUE, // compiling, not patching
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
(ULONG)(ULONGLONG) EP,
|
|
0);
|
|
}
|
|
}
|
|
return CallDirect_SIZE * sizeof(ULONG);
|
|
}
|
|
|
|
|
|
ULONG
|
|
PlaceCallfDirect(
|
|
IN PULONG CodeLocation,
|
|
#if _ALPHA_
|
|
IN ULONG CurrentECU,
|
|
#endif
|
|
IN PINSTRUCTION Instruction
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine copies the unconditional FAR call fragment into place.
|
|
|
|
Arguments:
|
|
|
|
Instruction - Supplies a description of the instruction the fragment
|
|
represents
|
|
CodeLocation - Supplies the address the code for the fragment has been
|
|
copied to
|
|
|
|
Return Value:
|
|
|
|
Size of code placed at CodeLocation
|
|
|
|
--*/
|
|
{
|
|
DWORD IntelDest;
|
|
DWORD IntelAddr;
|
|
DWORD IntelNext;
|
|
PVOID NativeNext;
|
|
PENTRYPOINT EP;
|
|
|
|
// ASSUME: The first two arguments are pIntelDest and IntelNext, stored
|
|
// as immediates.
|
|
CPUASSERT( Instruction->Operand1.Type == OPND_IMM );
|
|
CPUASSERT( Instruction->Operand2.Type == OPND_IMM );
|
|
|
|
IntelAddr = Instruction->Operand1.Immed;
|
|
|
|
// Get the offset portion of the address (skipping the selector)
|
|
IntelDest = *(UNALIGNED DWORD *)(IntelAddr+2);
|
|
EP = NativeAddressFromEipNoCompileEPWrite((PVOID)IntelDest);
|
|
|
|
if (EP == NULL) {
|
|
//
|
|
// Knowing NativeDest requires compilation. Just place the unpatched
|
|
// version for now and patch it later if necessary
|
|
//
|
|
GenCallfDirect(CodeLocation,
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
Instruction);
|
|
} else {
|
|
//
|
|
// We can place the patched version right away! Deterime if the
|
|
// NativeNext address is known.
|
|
//
|
|
IntelNext = Instruction->Operand2.Immed;
|
|
|
|
//
|
|
// If the current instruction is not the last one compiled, then
|
|
// NativeNext is CodeLocation+CallfDirect_SIZE and CallDirect3 can
|
|
// be placed right away.
|
|
//
|
|
if (Instruction != &InstructionStream[NumberOfInstructions-1]) {
|
|
GenCallfDirect3(CodeLocation,
|
|
TRUE, // compiling, not patching
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
(ULONG)(ULONGLONG) EP,
|
|
(ULONG)(ULONGLONG) (CodeLocation+CallfDirect_SIZE));
|
|
} else {
|
|
GenCallfDirect2(CodeLocation,
|
|
TRUE, // compiling, not patching
|
|
#if _ALPHA_
|
|
CurrentECU,
|
|
#endif
|
|
(ULONG)(ULONGLONG) EP,
|
|
0);
|
|
}
|
|
}
|
|
return CallfDirect_SIZE * sizeof(ULONG);
|
|
}
|
|
|
|
DWORD
|
|
PatchCallDirectExceptionFilter(
|
|
PTHREADSTATE cpu
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
Called if CTRL_CallFrag() throws an exception from within
|
|
PatchCallDirect(). If this happens, the Translation Cache is in fact
|
|
unlocked, although cpu->fTCUnlocked == FALSE. Need to fix this up before
|
|
CpuResetToConsistentState() gets run and unlocks the cache a second time.
|
|
|
|
Arguments:
|
|
|
|
cpu
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
{
|
|
//
|
|
// Indicate the TC read lock is not held.
|
|
//
|
|
cpu->fTCUnlocked = TRUE;
|
|
|
|
//
|
|
// Continue unwinding the stack
|
|
//
|
|
return EXCEPTION_CONTINUE_SEARCH;
|
|
}
|
|
|
|
|
|
ULONG
|
|
PatchCallDirect(
|
|
IN PTHREADSTATE Cpu,
|
|
IN ULONG IntelDest,
|
|
IN ULONG IntelNext,
|
|
IN PULONG PatchAddr
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine patches a CALLDIRECT to a CALLDIRECT2. It is called when
|
|
the native destination address of a call instruction is not yet known.
|
|
It patches the call to jump directly to the corresponding native code.
|
|
|
|
Arguments:
|
|
|
|
Cpu -- per-thread info
|
|
IntelDest -- intel address of the destination of the call
|
|
IntelNext -- intel address of the instruction following the call
|
|
PatchAddr -- address of the CALLDIRECT in the Translation Cache
|
|
|
|
Return Value:
|
|
|
|
Native address to jump to in order to resume execution
|
|
|
|
--*/
|
|
{
|
|
DWORD TCTimestamp;
|
|
ULONG NativeDest;
|
|
ULONG NativeSize;
|
|
|
|
//
|
|
// Switch from being a TC reader to a writer
|
|
//
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
MrswReaderExit(&MrswTC);
|
|
|
|
//
|
|
// Compile the code at the destination of the call and get the TC write lock
|
|
//
|
|
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip((PVOID)IntelDest, TRUE)->nativeStart;
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
PULONG CodeLocation = PatchAddr - CallDirect_PATCHRA_OFFSET;
|
|
//
|
|
// The Translation Cache wasn't flushed - replace the CALLDIRECT
|
|
// fragment by CALLDIRECT2
|
|
//
|
|
NativeSize = GenCallDirect2(CodeLocation,
|
|
FALSE, // patching, not compiling
|
|
#if _ALPHA_
|
|
GetCurrentECU(CodeLocation),
|
|
#endif
|
|
NativeDest,
|
|
0);
|
|
NtFlushInstructionCache(
|
|
NtCurrentProcess(),
|
|
CodeLocation,
|
|
NativeSize
|
|
);
|
|
} else {
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
|
|
if (Cpu->CSTimestamp != TCTimestamp) {
|
|
//
|
|
// The cache was flushed by another thread in the small window
|
|
// between mrsw calls in this thread, we plan on jumping directly
|
|
// to NativeDest, so the CPU callstack needs to be flushed.
|
|
// Normally this would be done in the CpuSimulate() loop as a
|
|
// result of jumping to EndTranslatedCode.
|
|
//
|
|
FlushCallstack(Cpu);
|
|
}
|
|
}
|
|
|
|
//
|
|
// Push IntelNext on the stack and update the stack optimization code.
|
|
// This must be done while still in TC write mode. If it isn't, then
|
|
// the transition back to TC reader may allow a TC cache flush, invalidating
|
|
// nativedest before it is written into the stack optimization. (The
|
|
// stack optimization is blown away whenever the TC is flushed, so if
|
|
// it is written in BEFORE the flush, it will just get blown away.
|
|
//
|
|
try {
|
|
CTRL_CallFrag(Cpu, IntelDest, IntelNext, 0 /* nativenext is unknown */);
|
|
} _except(PatchCallDirectExceptionFilter(Cpu)) {
|
|
// nothing to do - the exception filter does everything
|
|
}
|
|
|
|
//
|
|
// Become a TC reader again.
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
Cpu->fTCUnlocked = FALSE;
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
//
|
|
// TC was not flushed while becomming a reader again.
|
|
//
|
|
return NativeDest;
|
|
} else {
|
|
//
|
|
// TC was flushed - nativedest is invalid. The callstack optimization
|
|
// was deleted when the TC flush occurred, so do an EndTranslatedCode
|
|
// instead.
|
|
return (ULONG)(ULONGLONG) &EndTranslatedCode;
|
|
}
|
|
}
|
|
|
|
|
|
ULONG
|
|
PatchCallfDirect(
|
|
IN PTHREADSTATE cpu,
|
|
IN PUSHORT pIntelDest,
|
|
IN ULONG IntelNext,
|
|
IN PULONG PatchAddr
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine patches a CALLFDIRECT to a CALLFDIRECT2. It is called when
|
|
the native destination address of a call instruction is not yet known.
|
|
It patches the call to jump directly to the corresponding native code.
|
|
|
|
Arguments:
|
|
|
|
cpu -- per-thread info
|
|
pIntelDest-- ptr to SEL:OFFSET intel address of the destination of the call
|
|
IntelNext -- intel address of the instruction following the call
|
|
PatchAddr -- address of the CALLDIRECT in the Translation Cache
|
|
|
|
Return Value:
|
|
|
|
Native address to jump to in order to resume execution
|
|
|
|
--*/
|
|
{
|
|
DWORD TCTimestamp;
|
|
ULONG NativeDest;
|
|
PVOID IntelDest;
|
|
ULONG NativeSize;
|
|
|
|
|
|
//
|
|
// Switch from being a TC reader to a writer
|
|
//
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
MrswReaderExit(&MrswTC);
|
|
|
|
//
|
|
// Compile the code at the destination of the call and get the TC write lock
|
|
//
|
|
IntelDest = (PVOID)*(UNALIGNED DWORD *)pIntelDest;
|
|
NativeDest = (ULONG)(ULONGLONG) NativeAddressFromEip(IntelDest, TRUE)->nativeStart;
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
PULONG CodeLocation = PatchAddr - CallfDirect_PATCHRA_OFFSET;
|
|
//
|
|
// The Translation Cache wasn't flushed - replace the CALLDIRECT
|
|
// fragment by CALLDIRECT2
|
|
//
|
|
NativeSize = GenCallfDirect2(CodeLocation,
|
|
FALSE, // patching, not compiling
|
|
#if _ALPHA_
|
|
GetCurrentECU(CodeLocation),
|
|
#endif
|
|
NativeDest,
|
|
0);
|
|
NtFlushInstructionCache(
|
|
NtCurrentProcess(),
|
|
CodeLocation,
|
|
NativeSize
|
|
);
|
|
} else {
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
|
|
if (cpu->CSTimestamp != TCTimestamp) {
|
|
//
|
|
// The cache was flushed by another thread in the small window
|
|
// between mrsw calls in this thread, we plan on jumping directly
|
|
// to NativeDest, so the CPU callstack needs to be flushed.
|
|
// Normally this would be done in the CpuSimulate() loop as a
|
|
// result of jumping to EndTranslatedCode.
|
|
//
|
|
FlushCallstack(cpu);
|
|
}
|
|
}
|
|
|
|
//
|
|
// Push IntelNext on the stack and update the stack optimization code.
|
|
// This must be done while still in TC write mode. If it isn't, then
|
|
// the transition back to TC reader may allow a TC cache flush, invalidating
|
|
// nativedest before it is written into the stack optimization. (The
|
|
// stack optimization is blown away whenever the TC is flushed, so if
|
|
// it is written in BEFORE the flush, it will just get blown away.
|
|
//
|
|
try {
|
|
CTRL_CallfFrag(cpu, pIntelDest, IntelNext, 0 /* nativenext is unknown */);
|
|
} _except(PatchCallDirectExceptionFilter(cpu)) {
|
|
// nothing to do - the exception filter does everything
|
|
}
|
|
|
|
//
|
|
// Become a TC reader again.
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
cpu->fTCUnlocked = FALSE;
|
|
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
//
|
|
// TC was not flushed while becomming a reader again.
|
|
//
|
|
return NativeDest;
|
|
} else {
|
|
//
|
|
// TC was flushed - nativedest is invalid. The callstack optimization
|
|
// was deleted when the TC flush occurred, so do an EndTranslatedCode
|
|
// instead.
|
|
return (ULONG)(ULONGLONG) &EndTranslatedCode;
|
|
}
|
|
}
|
|
|
|
|
|
ULONG
|
|
PatchCallDirect2(
|
|
IN PTHREADSTATE Cpu,
|
|
IN ULONG IntelDest,
|
|
IN ULONG IntelNext,
|
|
IN ULONG NativeDest,
|
|
IN PULONG PatchAddr
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine patches a CALLDIRECT2 to a CALLDIRECT3. It is called when
|
|
the native destination address of the instruction after the call is not yet
|
|
known. It patches the fragment to place the native address of the
|
|
instruction after the call on the optimized callstack.
|
|
|
|
Arguments:
|
|
|
|
Cpu -- per-thread info
|
|
IntelDest -- intel address of the destination of the call
|
|
IntelNext -- intel address of the instruction following the call
|
|
NativeDest -- native address of the destination of the call
|
|
PatchAddr -- address of the CALLDIRECT2 in the Translation Cache
|
|
|
|
Return Value:
|
|
|
|
Native address to jump to in order to resume execution
|
|
|
|
--*/
|
|
{
|
|
DWORD TCTimestamp;
|
|
ULONG NativeNext;
|
|
ULONG NativeSize;
|
|
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
MrswReaderExit(&MrswTC);
|
|
NativeNext = (ULONG)(ULONGLONG) NativeAddressFromEipNoCompile((PVOID)IntelNext);
|
|
|
|
if (NativeNext) {
|
|
PULONG CodeLocation;
|
|
|
|
//
|
|
// The code at the return address from the call has already been
|
|
// compiled. Replace CALLDIRECT2 by CALLDIRECT3. TC is locked
|
|
// for write.
|
|
//
|
|
|
|
if (TCTimestamp != TranslationCacheTimestamp) {
|
|
//
|
|
// The TC was flushed while switching from reader to writer.
|
|
// Become a TC reader again
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
// The CALLDIRECT2 code is now gone, so set up for the call and
|
|
// then go to EndTranslatedCode to make the control transfer.
|
|
//
|
|
CTRL_CallFrag(Cpu,
|
|
IntelDest,
|
|
IntelNext,
|
|
0 // nativenext is also unknown
|
|
);
|
|
|
|
return (ULONG)(ULONGLONG) &EndTranslatedCode;
|
|
}
|
|
|
|
CodeLocation = PatchAddr - CallDirect2_PATCHRA_OFFSET;
|
|
//
|
|
// Else the TC was not flushed, and nativenext is now known. Patch
|
|
// CALLDIRECT2 to be CALLDIRECT3
|
|
//
|
|
NativeSize = GenCallDirect3(CodeLocation,
|
|
FALSE, // patching, not compiling
|
|
#if _ALPHA_
|
|
GetCurrentECU(CodeLocation),
|
|
#endif
|
|
NativeDest,
|
|
NativeNext);
|
|
NtFlushInstructionCache(
|
|
NtCurrentProcess(),
|
|
CodeLocation,
|
|
NativeSize
|
|
);
|
|
|
|
//
|
|
// Push IntelNext on the stack and update the stack optimization code.
|
|
// This must be done while still in TC write mode. If it isn't, then
|
|
// the transition back to TC reader may allow a TC cache flush, invalidating
|
|
// nativedest before it is written into the stack optimization. (The
|
|
// stack optimization is blown away whenever the TC is flushed, so if
|
|
// it is written in BEFORE the flush, it will just get blown away.
|
|
//
|
|
CTRL_CallFrag(Cpu, IntelDest, IntelNext, NativeNext);
|
|
|
|
//
|
|
// Switch back to being a TC reader
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
if (TCTimestamp != TranslationCacheTimestamp) {
|
|
//
|
|
// TC was flushed while we were becomming a reader again.
|
|
// nativedest and nativenext are invalid, but stack optimization
|
|
// code was flushed.
|
|
//
|
|
return (ULONG)(ULONGLONG) &EndTranslatedCode;
|
|
}
|
|
} else { // NativeNext == NULL, TC locked for Read
|
|
CTRL_CallFrag(Cpu, IntelDest, IntelNext, 0);
|
|
|
|
if (TCTimestamp != TranslationCacheTimestamp) {
|
|
//
|
|
// Cache was flushed by another thread. NativeDest is invalid.
|
|
//
|
|
return (ULONG)(ULONGLONG) &EndTranslatedCode;
|
|
}
|
|
}
|
|
|
|
return NativeDest;
|
|
}
|
|
|
|
|
|
ULONG
|
|
PatchCallfDirect2(
|
|
IN PTHREADSTATE Cpu,
|
|
IN PUSHORT pIntelDest,
|
|
IN ULONG IntelNext,
|
|
IN ULONG NativeDest,
|
|
IN PULONG PatchAddr
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine patches a CALLFDIRECT2 to a CALLFDIRECT3. It is called when
|
|
the native destination address of the instruction after the call is not yet
|
|
known. It patches the fragment to place the native address of the
|
|
instruction after the call on the optimized callstack.
|
|
|
|
Arguments:
|
|
|
|
Cpu -- per-thread info
|
|
pIntelDest-- ptr to SEL:OFFSET intel address of the destination of the call
|
|
IntelNext -- intel address of the instruction following the call
|
|
NativeDest -- native address of the destination of the call
|
|
PatchAddr -- address of the CALLDIRECT2 in the Translation Cache
|
|
|
|
Return Value:
|
|
|
|
Native address to jump to in order to resume execution
|
|
|
|
--*/
|
|
{
|
|
DWORD TCTimestamp;
|
|
ULONG NativeNext;
|
|
ULONG NativeSize;
|
|
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
MrswReaderExit(&MrswTC);
|
|
NativeNext = (ULONG)(ULONGLONG) NativeAddressFromEipNoCompile((PVOID)IntelNext);
|
|
|
|
if (NativeNext) {
|
|
PULONG CodeLocation;
|
|
|
|
//
|
|
// The code at the return address from the call has already been
|
|
// compiled. Replace CALLDIRECT2 by CALLDIRECT3. TC is locked
|
|
// for write.
|
|
//
|
|
|
|
if (TCTimestamp != TranslationCacheTimestamp) {
|
|
//
|
|
// The TC was flushed while switching from reader to writer.
|
|
// Become a TC reader again
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
// The CALLFDIRECT2 code is now gone, so set up for the call and
|
|
// then go to EndTranslatedCode to make the control transfer.
|
|
//
|
|
CTRL_CallfFrag(Cpu,
|
|
pIntelDest,
|
|
IntelNext,
|
|
0 // nativenext is also unknown
|
|
);
|
|
|
|
return (ULONG)(ULONGLONG) &EndTranslatedCode;
|
|
}
|
|
|
|
CodeLocation = PatchAddr - CallfDirect2_PATCHRA_OFFSET;
|
|
//
|
|
// Else the TC was not flushed, and nativenext is now known. Patch
|
|
// CALLFDIRECT2 to be CALLFDIRECT3
|
|
//
|
|
NativeSize = GenCallfDirect3(CodeLocation,
|
|
FALSE, // patching, not compiling
|
|
#if _ALPHA_
|
|
GetCurrentECU(CodeLocation),
|
|
#endif
|
|
NativeDest,
|
|
NativeNext);
|
|
NtFlushInstructionCache(
|
|
NtCurrentProcess(),
|
|
CodeLocation,
|
|
NativeSize
|
|
);
|
|
|
|
//
|
|
// Push IntelNext on the stack and update the stack optimization code.
|
|
// This must be done while still in TC write mode. If it isn't, then
|
|
// the transition back to TC reader may allow a TC cache flush, invalidating
|
|
// nativedest before it is written into the stack optimization. (The
|
|
// stack optimization is blown away whenever the TC is flushed, so if
|
|
// it is written in BEFORE the flush, it will just get blown away.
|
|
//
|
|
CTRL_CallfFrag(Cpu, pIntelDest, IntelNext, NativeNext);
|
|
|
|
//
|
|
// Switch back to being a TC reader
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
if (TCTimestamp != TranslationCacheTimestamp) {
|
|
//
|
|
// TC was flushed while we were becomming a reader again.
|
|
// nativedest and nativenext are invalid, but stack optimization
|
|
// code was flushed.
|
|
//
|
|
return (ULONG)(ULONGLONG) &EndTranslatedCode;
|
|
}
|
|
} else { // NativeNext == NULL, TC locked for Read
|
|
CTRL_CallfFrag(Cpu, pIntelDest, IntelNext, 0);
|
|
|
|
if (TCTimestamp != TranslationCacheTimestamp) {
|
|
//
|
|
// Cache was flushed by another thread. NativeDest is invalid.
|
|
//
|
|
return (ULONG)(ULONGLONG) &EndTranslatedCode;
|
|
}
|
|
}
|
|
|
|
return NativeDest;
|
|
}
|
|
|
|
|
|
DWORD
|
|
PatchCallIndirectExceptionFilter(
|
|
PTHREADSTATE cpu
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
Called if CTRL_CallFrag() throws an exception from within
|
|
PatchCallIndirect(). If this happens, the Translation Cache Write lock
|
|
is being held. This must be released before the exception can be
|
|
allowed to continue.
|
|
|
|
Arguments:
|
|
|
|
cpu
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
{
|
|
//
|
|
// Release the TC write lock.
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
|
|
//
|
|
// Indicate the TC read lock is not held, either
|
|
//
|
|
cpu->fTCUnlocked = TRUE;
|
|
|
|
//
|
|
// Continue unwinding the stack
|
|
//
|
|
return EXCEPTION_CONTINUE_SEARCH;
|
|
}
|
|
|
|
VOID
|
|
PatchCallIndirect(
|
|
IN PTHREADSTATE Cpu,
|
|
IN ULONG IntelDest,
|
|
IN ULONG IntelNext,
|
|
IN PULONG PatchAddr
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine patches a CALLINDIRECT to a CALLINDIRECT2. It is called when
|
|
the native destination address of the instruction after the call is not yet
|
|
known. It patches the fragment to place the native address of the
|
|
instruction after the call on the optimized callstack.
|
|
|
|
Arguments:
|
|
|
|
Cpu -- per-thread info
|
|
IntelDest -- intel address of the destination of the call
|
|
IntelNext -- intel address of the instruction following the call
|
|
PatchAddr -- address of the CALLDIRECT2 in the Translation Cache
|
|
|
|
Return Value:
|
|
|
|
None. cpu->Eip updated to be IntelDest.
|
|
|
|
--*/
|
|
{
|
|
DWORD TCTimestamp;
|
|
ULONG NativeNext;
|
|
ULONG NativeSize;
|
|
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
MrswReaderExit(&MrswTC);
|
|
|
|
NativeNext = (ULONG)(ULONGLONG) NativeAddressFromEipNoCompile((PVOID)IntelNext);
|
|
|
|
if (NativeNext) {
|
|
PULONG CodeLocation;
|
|
|
|
//
|
|
// The code at the return address from the call has already been
|
|
// compiled. Replace CALLINDIRECT by CALLINDIRECT2. TC is locked
|
|
// for write.
|
|
//
|
|
|
|
if (TCTimestamp != TranslationCacheTimestamp) {
|
|
//
|
|
// The TC was flushed while switching from reader to writer.
|
|
// Become a TC reader again
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
|
|
// The CALLINDIRECT code is now gone, so set up for the call
|
|
// without patching anything
|
|
//
|
|
CTRL_CallFrag(
|
|
Cpu,
|
|
IntelDest,
|
|
IntelNext,
|
|
0 // nativenext is unknown
|
|
);
|
|
|
|
return;
|
|
}
|
|
|
|
//
|
|
// Else the TC was not flushed, and nativenext is now known. Patch
|
|
// CALLINDIRECT to be CALLINDIRECT2
|
|
//
|
|
CodeLocation = PatchAddr - CallIndirect_PATCHRA_OFFSET;
|
|
NativeSize = GenCallIndirect2(CodeLocation,
|
|
FALSE, // patching, not compiling
|
|
#if _ALPHA_
|
|
GetCurrentECU(CodeLocation),
|
|
#endif
|
|
NativeNext,
|
|
getUniqueIndex());
|
|
NtFlushInstructionCache(
|
|
NtCurrentProcess(),
|
|
CodeLocation,
|
|
NativeSize
|
|
);
|
|
|
|
//
|
|
// Push IntelNext on the stack and update the stack optimization code.
|
|
// This must be done while still in TC write mode. If it isn't, then
|
|
// the transition back to TC reader may allow a TC cache flush, invalidating
|
|
// nativedest before it is written into the stack optimization. (The
|
|
// stack optimization is blown away whenever the TC is flushed, so if
|
|
// it is written in BEFORE the flush, it will just get blown away.
|
|
//
|
|
_try {
|
|
CTRL_CallFrag(Cpu, IntelDest, IntelNext, NativeNext);
|
|
} _except(PatchCallIndirectExceptionFilter(Cpu)) {
|
|
// nothing to do - the exception filter does everything
|
|
}
|
|
|
|
//
|
|
// Switch back to being a TC reader. TC flushes during the switch
|
|
// are OK and require no extra work.
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
} else { // NativeNext == NULL, TC locked for read.
|
|
|
|
CTRL_CallFrag(Cpu, IntelDest, IntelNext, 0);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
PatchCallfIndirect(
|
|
IN PTHREADSTATE Cpu,
|
|
IN PUSHORT pIntelDest,
|
|
IN ULONG IntelNext,
|
|
IN PULONG PatchAddr
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine patches a CALLFINDIRECT to a CALLFINDIRECT2. It is called when
|
|
the native destination address of the instruction after the call is not yet
|
|
known. It patches the fragment to place the native address of the
|
|
instruction after the call on the optimized callstack.
|
|
|
|
Arguments:
|
|
|
|
Cpu -- per-thread info
|
|
pIntelDest-- ptr to SEL:OFFSET intel address of the destination of the call
|
|
IntelNext -- intel address of the instruction following the call
|
|
PatchAddr -- address of the CALLDIRECT2 in the Translation Cache
|
|
|
|
Return Value:
|
|
|
|
None. cpu->eip updated to IntelDest
|
|
|
|
--*/
|
|
{
|
|
DWORD TCTimestamp;
|
|
ULONG NativeNext;
|
|
ULONG IntelDest;
|
|
ULONG NativeSize;
|
|
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
MrswReaderExit(&MrswTC);
|
|
|
|
IntelDest = *(UNALIGNED DWORD *)pIntelDest;
|
|
NativeNext = (ULONG)(ULONGLONG) NativeAddressFromEipNoCompile((PVOID)IntelNext);
|
|
|
|
if (NativeNext) {
|
|
PULONG CodeLocation;
|
|
|
|
//
|
|
// The code at the return address from the call has already been
|
|
// compiled. Replace CALLINDIRECT by CALLINDIRECT2. TC is locked
|
|
// for write.
|
|
//
|
|
|
|
if (TCTimestamp != TranslationCacheTimestamp) {
|
|
//
|
|
// The TC was flushed while switching from reader to writer.
|
|
// Become a TC reader again
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
|
|
// The CALLFINDIRECT code is now gone, so set up for the call
|
|
// without patching anything
|
|
//
|
|
CTRL_CallfFrag(
|
|
Cpu,
|
|
pIntelDest,
|
|
IntelNext,
|
|
0 // nativenext is unknown
|
|
);
|
|
|
|
return;
|
|
}
|
|
|
|
//
|
|
// Else the TC was not flushed, and nativenext is now known. Patch
|
|
// CALLFINDIRECT to be CALLFINDIRECT2
|
|
//
|
|
CodeLocation = PatchAddr - CallfIndirect_PATCHRA_OFFSET;
|
|
NativeSize = GenCallfIndirect2(CodeLocation,
|
|
FALSE, // patching, not compiling
|
|
#if _ALPHA_
|
|
GetCurrentECU(CodeLocation),
|
|
#endif
|
|
NativeNext,
|
|
getUniqueIndex());
|
|
NtFlushInstructionCache(
|
|
NtCurrentProcess(),
|
|
CodeLocation,
|
|
NativeSize
|
|
);
|
|
|
|
//
|
|
// Push IntelNext on the stack and update the stack optimization code.
|
|
// This must be done while still in TC write mode. If it isn't, then
|
|
// the transition back to TC reader may allow a TC cache flush, invalidating
|
|
// nativedest before it is written into the stack optimization. (The
|
|
// stack optimization is blown away whenever the TC is flushed, so if
|
|
// it is written in BEFORE the flush, it will just get blown away.
|
|
//
|
|
_try {
|
|
CTRL_CallfFrag(Cpu, pIntelDest, IntelNext, NativeNext);
|
|
} _except(PatchCallIndirectExceptionFilter(Cpu)) {
|
|
// nothing to do - the exception filter does everything
|
|
}
|
|
|
|
//
|
|
// Switch back to being a TC reader. TC flushes during the switch
|
|
// are OK and require no extra work.
|
|
//
|
|
MrswWriterExit(&MrswTC);
|
|
MrswReaderEnter(&MrswTC);
|
|
|
|
} else { // NativeNext == NULL, TC locked for read.
|
|
|
|
CTRL_CallfFrag(Cpu, pIntelDest, IntelNext, 0);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
|
|
|
|
//*********************************************************************************
|
|
// Below are functions for the Indirect Control Transfer Table
|
|
//*********************************************************************************
|
|
|
|
// This number must be below 0xffff, because we want to be able to load it with just
|
|
// one instruction (now we use ori). It should also be a factor of two to get good
|
|
// code generation for % (so that we won't have to use a division instruction).
|
|
#define MAX_TABLE_ENTRIES 0x1000
|
|
|
|
typedef struct _IndirControlTransferTable {
|
|
ULONG intelAddr;
|
|
ULONG nativeAddr;
|
|
} INDIRCONTROLTRANSFERTABLE, *PINDIRCONTROLTRANSFERTABLE;
|
|
|
|
INDIRCONTROLTRANSFERTABLE IndirControlTransferTable[MAX_TABLE_ENTRIES];
|
|
|
|
// The last used index in the table
|
|
ULONG lastTableIndex;
|
|
|
|
ULONG
|
|
getUniqueIndex(
|
|
VOID
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function returns the next free index to the indirect control
|
|
transfer table. If it reaches the end of the table, it wraps around.
|
|
NOTE: we need not worry about synchronization here, because we have
|
|
an Entry Point write lock whenever we are called.
|
|
|
|
Arguments:
|
|
|
|
none
|
|
|
|
Return Value:
|
|
|
|
An index into the table
|
|
|
|
--*/
|
|
{
|
|
return (lastTableIndex = ((lastTableIndex + 1) % MAX_TABLE_ENTRIES));
|
|
}
|
|
|
|
VOID
|
|
FlushIndirControlTransferTable(
|
|
VOID
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine flushes the Indirect Control Transfer Table
|
|
NOTE: we need not worry about synchronizations here, because the routine
|
|
which calls us (FlushTranslationCache) has a Translation Cache write lock.
|
|
|
|
Arguments:
|
|
|
|
none
|
|
|
|
Return Value:
|
|
|
|
none
|
|
|
|
--*/
|
|
{
|
|
RtlZeroMemory (IndirControlTransferTable, sizeof(INDIRCONTROLTRANSFERTABLE)*MAX_TABLE_ENTRIES);
|
|
lastTableIndex = 0;
|
|
}
|
|
|
|
ULONG
|
|
IndirectControlTransfer(
|
|
IN ULONG tableEntry,
|
|
IN ULONG intelAddr,
|
|
IN PTHREADSTATE cpu
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine is used by an indirect control transfer operation to try
|
|
and save a call to the Entry Point Manager.
|
|
|
|
Arguments:
|
|
|
|
tableEntry -- The index of the table entry where information might be
|
|
available about intelAddr
|
|
|
|
intelAddr -- The intel address we want to go to
|
|
|
|
cpu -- pointer to per-thread CPU data
|
|
|
|
Return Value:
|
|
|
|
The native address we want to go to
|
|
|
|
--*/
|
|
{
|
|
ULONG nativeAddr;
|
|
DWORD TCTimestamp;
|
|
|
|
//
|
|
// Detect apps which do the following:
|
|
// call foo
|
|
// where
|
|
// foo: mov eax, [esp]
|
|
// ...
|
|
// jmp eax ; this is really a 'ret' instruction
|
|
//
|
|
// This is the way _alloca() works - you call it with eax==number of bytes
|
|
// to allocate, and it jumps back to its caller with esp munged.
|
|
//
|
|
// What happens is the callstack cache gets out-of-sync. If the app
|
|
// is trying to do an indirect jump to the address already on the
|
|
// callstack cache, we will pop the callstack cache.
|
|
//
|
|
if (ISTOPOF_CALLSTACK(intelAddr)) {
|
|
POP_CALLSTACK(intelAddr, nativeAddr);
|
|
if (nativeAddr) {
|
|
return nativeAddr;
|
|
}
|
|
}
|
|
|
|
// First see if the table is filled in correctly already
|
|
MrswReaderEnter(&MrswIndirTable);
|
|
if (IndirControlTransferTable[tableEntry].intelAddr == intelAddr){
|
|
nativeAddr = IndirControlTransferTable[tableEntry].nativeAddr;
|
|
if (nativeAddr) {
|
|
MrswReaderExit(&MrswIndirTable);
|
|
return nativeAddr;
|
|
}
|
|
}
|
|
MrswReaderExit(&MrswIndirTable);
|
|
|
|
// Give up the translation cache reading lock so that we can call NativeAddressFromEip
|
|
TCTimestamp = TranslationCacheTimestamp;
|
|
MrswReaderExit(&MrswTC);
|
|
nativeAddr = (ULONG) (ULONGLONG)NativeAddressFromEip((PVOID)intelAddr, FALSE)->nativeStart;
|
|
|
|
// Note: we now have a TC read lock obtained by NativeAddressFromEip.
|
|
if (TCTimestamp == TranslationCacheTimestamp) {
|
|
// We haven't flushed the cache. Save the native address in the table.
|
|
MrswWriterEnter(&MrswIndirTable);
|
|
IndirControlTransferTable[tableEntry].intelAddr = intelAddr;
|
|
IndirControlTransferTable[tableEntry].nativeAddr = nativeAddr;
|
|
MrswWriterExit(&MrswIndirTable);
|
|
} else {
|
|
//
|
|
// Translation cache was flushed, possibly by another thread.
|
|
// Flush our callstack before resuming execution of RISC code
|
|
// in the Translation Cache.
|
|
//
|
|
FlushCallstack(cpu);
|
|
}
|
|
// Return the native address to IndirectControlTransferHelper which will go there.
|
|
return nativeAddr;
|
|
}
|
|
|
|
ULONG
|
|
IndirectControlTransferFar(
|
|
IN PTHREADSTATE cpu,
|
|
IN PUSHORT pintelAddr,
|
|
IN ULONG tableEntry
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine is used by a FAR indirect control transfer operation to try
|
|
and save a call to the Entry Point Manager.
|
|
|
|
Arguments:
|
|
|
|
tableEntry -- The index of the table entry where information might be
|
|
available about intelAddr
|
|
|
|
pintelAddr -- Pointer to SEL:OFFSET intel address we want to go to
|
|
|
|
Return Value:
|
|
|
|
The native address we want to go to
|
|
|
|
--*/
|
|
{
|
|
USHORT Sel;
|
|
ULONG Offset;
|
|
|
|
Offset = *(UNALIGNED PULONG)pintelAddr;
|
|
Sel = *(UNALIGNED PUSHORT)(pintelAddr+2);
|
|
|
|
CS = Sel;
|
|
eip = Offset;
|
|
|
|
return IndirectControlTransfer(tableEntry, Offset, cpu);
|
|
}
|
|
|
|
|
|
|
|
ULONG
|
|
PlaceNop(
|
|
IN PULONG CodeLocation,
|
|
#if _ALPHA_
|
|
IN ULONG CurrentECU,
|
|
#endif
|
|
IN PINSTRUCTION Instruction
|
|
)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
|
|
#if _ALPHA_
|
|
ULONG
|
|
GetCurrentECU(
|
|
PULONG CodeLocation
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine returns the correct ECU. CurrentECU is the target for
|
|
branch instructions when a fragment wants to jump to EndTranslatedCode.
|
|
|
|
N.B. This routine cannot change the global CurrentECU. This is set in
|
|
Compile(), and is the only way to locate the ECU at the end of
|
|
the translation cache if the exception info hasn't been placed
|
|
yet.
|
|
|
|
Arguments:
|
|
|
|
CodeLocation -- The code location which will be patched.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
{
|
|
//
|
|
// Find an EndCompilationUnit fragment by searching the Translation Cache
|
|
// for the next EXCEPTIONDATA_SIGNATURE. The code immediately before it
|
|
// is an EndCompilationUnit fragment.
|
|
//
|
|
while (*CodeLocation != EXCEPTIONDATA_SIGNATURE) {
|
|
CodeLocation++;
|
|
}
|
|
return (ULONG)(ULONGLONG)(CodeLocation-EndCompilationUnit_SIZE);
|
|
}
|
|
#endif // _ALPHA_-only
|