diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 860efc6bc..d377fd09e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1172,6 +1172,12 @@ set (PCH_SOURCES scripting/vm/vmexec.cpp scripting/vm/vmframe.cpp scripting/vm/jit.cpp + scripting/vm/jit_call.cpp + scripting/vm/jit_flow.cpp + scripting/vm/jit_load.cpp + scripting/vm/jit_math.cpp + scripting/vm/jit_move.cpp + scripting/vm/jit_store.cpp scripting/zscript/ast.cpp scripting/zscript/zcc_compile.cpp scripting/zscript/zcc_parser.cpp diff --git a/src/scripting/backend/vmbuilder.cpp b/src/scripting/backend/vmbuilder.cpp index 1af5b46ba..a144a30cc 100644 --- a/src/scripting/backend/vmbuilder.cpp +++ b/src/scripting/backend/vmbuilder.cpp @@ -42,7 +42,7 @@ struct VMRemap }; -#define xx(op, name, mode, alt, kreg, ktype) {OP_##alt, kreg, ktype } +#define xx(op, name, mode, alt, kreg, ktype) {OP_##alt, kreg, ktype }, VMRemap opRemap[NUM_OPS] = { #include "vmops.h" }; diff --git a/src/scripting/backend/vmdisasm.cpp b/src/scripting/backend/vmdisasm.cpp index be5390bba..f7e46ac38 100644 --- a/src/scripting/backend/vmdisasm.cpp +++ b/src/scripting/backend/vmdisasm.cpp @@ -149,7 +149,7 @@ const VMOpInfo OpInfo[NUM_OPS] = { -#define xx(op, name, mode, alt, kreg, ktype) { #name, mode } +#define xx(op, name, mode, alt, kreg, ktype) { #name, mode }, #include "vmops.h" }; diff --git a/src/scripting/vm/jit.cpp b/src/scripting/vm/jit.cpp index 6ec30edb3..14960d0f1 100644 --- a/src/scripting/vm/jit.cpp +++ b/src/scripting/vm/jit.cpp @@ -1,3525 +1,6 @@ #include "jit.h" -#include "i_system.h" -#include "types.h" -#include "stats.h" - -// To do: get cmake to define these.. -#define ASMJIT_BUILD_EMBED -#define ASMJIT_STATIC - -#include -#include -#include - -extern cycle_t VMCycles[10]; -extern int VMCalls[10]; - -#define A (pc[0].a) -#define B (pc[0].b) -#define C (pc[0].c) -#define Cs (pc[0].cs) -#define BC (pc[0].i16u) -#define BCs (pc[0].i16) -#define ABCs (pc[0].i24) -#define JMPOFS(x) ((x)->i24) - -static const char *OpNames[NUM_OPS] = -{ -#define xx(op, name, mode, alt, kreg, ktype) #op -#include "vmops.h" -}; - -class JitCompiler -{ -public: - JitCompiler(asmjit::CodeHolder *code, VMScriptFunction *sfunc) : cc(code), sfunc(sfunc) { } - - void Codegen() - { - using namespace asmjit; - - Setup(); - - pc = sfunc->Code; - auto end = pc + sfunc->CodeSize; - while (pc != end) - { - int i = (int)(ptrdiff_t)(pc - sfunc->Code); - op = pc->op; - - cc.bind(labels[i]); - - FString lineinfo; - lineinfo.Format("; %s(line %d): %02x%02x%02x%02x %s", sfunc->PrintableName.GetChars(), sfunc->PCToLine(pc), pc->op, pc->a, pc->b, pc->c, OpNames[op]); - cc.comment(lineinfo.GetChars(), lineinfo.Len()); - - EmitFuncPtr opcodeFunc = GetOpcodeEmitFunc(op); - if (!opcodeFunc) - I_FatalError("JIT error: Unknown VM opcode %d\n", op); - (this->*opcodeFunc)(); - - pc++; - } - - cc.endFunc(); - cc.finalize(); - } - - static bool CanJit(VMScriptFunction *sfunc) - { - int size = sfunc->CodeSize; - for (int i = 0; i < size; i++) - { - auto opcodeFunc = GetOpcodeEmitFunc(sfunc->Code[i].op); - if (!opcodeFunc) - return false; - - auto pc = sfunc->Code + i; - if (sfunc->Code[i].op == OP_RET) - { - if (B != REGT_NIL) - { - int regtype = B; - if ((regtype & REGT_TYPE) == REGT_STRING) - return false; - } - } - else if (sfunc->Code[i].op == OP_CAST) - { - switch (C) - { - case CAST_I2F: - case CAST_U2F: - case CAST_F2I: - case CAST_F2U: - break; - default: - return false; - } - } - else if (sfunc->Code[i].op == OP_CASTB) - { - if (C == CASTB_S) - return false; - } - else if (sfunc->Code[i].op == OP_PARAM) - { - if (!!(B & REGT_MULTIREG3) || !!(B & REGT_MULTIREG2)) - return false; - - switch (B) - { - case REGT_STRING: - case REGT_STRING | REGT_ADDROF: - case REGT_STRING | REGT_KONST: - return false; - default: - break; - } - } - else if (sfunc->Code[i].op == OP_RESULT) - { - if (!!(B & REGT_MULTIREG3) || !!(B & REGT_MULTIREG2)) - return false; - - if ((B & REGT_TYPE) == REGT_STRING) - return false; - } - } - return true; - } - -private: - typedef void(JitCompiler::*EmitFuncPtr)(); - - static EmitFuncPtr GetOpcodeEmitFunc(VM_UBYTE op) - { - #define EMIT_OP(x) case OP_##x: return &JitCompiler::Emit##x - - switch (op) - { - default: return nullptr; - EMIT_OP(NOP); - EMIT_OP(LI); - EMIT_OP(LK); - EMIT_OP(LKF); - EMIT_OP(LKS); - EMIT_OP(LKP); - EMIT_OP(LK_R); - EMIT_OP(LKF_R); - //EMIT_OP(LKS_R); - EMIT_OP(LKP_R); - // EMIT_OP(LFP); - EMIT_OP(META); - EMIT_OP(CLSS); - EMIT_OP(LB); - EMIT_OP(LB_R); - EMIT_OP(LH); - EMIT_OP(LH_R); - EMIT_OP(LW); - EMIT_OP(LW_R); - EMIT_OP(LBU); - EMIT_OP(LBU_R); - EMIT_OP(LHU); - EMIT_OP(LHU_R); - EMIT_OP(LSP); - EMIT_OP(LSP_R); - EMIT_OP(LDP); - EMIT_OP(LDP_R); - EMIT_OP(LS); - EMIT_OP(LS_R); - EMIT_OP(LO); - EMIT_OP(LO_R); - EMIT_OP(LP); - EMIT_OP(LP_R); - EMIT_OP(LV2); - EMIT_OP(LV2_R); - EMIT_OP(LV3); - EMIT_OP(LV3_R); - EMIT_OP(LCS); - //EMIT_OP(LCS_R); - EMIT_OP(LBIT); - EMIT_OP(SB); - EMIT_OP(SB_R); - EMIT_OP(SH); - EMIT_OP(SH_R); - EMIT_OP(SW); - EMIT_OP(SW_R); - EMIT_OP(SSP); - EMIT_OP(SSP_R); - EMIT_OP(SDP); - EMIT_OP(SDP_R); - EMIT_OP(SS); - //EMIT_OP(SS_R); - EMIT_OP(SO); - EMIT_OP(SO_R); - EMIT_OP(SP); - EMIT_OP(SP_R); - EMIT_OP(SV2); - EMIT_OP(SV2_R); - EMIT_OP(SV3); - EMIT_OP(SV3_R); - EMIT_OP(SBIT); - EMIT_OP(MOVE); - EMIT_OP(MOVEF); - // EMIT_OP(MOVES); - EMIT_OP(MOVEA); - EMIT_OP(MOVEV2); - EMIT_OP(MOVEV3); - EMIT_OP(CAST); - EMIT_OP(CASTB); - EMIT_OP(DYNCAST_R); - EMIT_OP(DYNCAST_K); - EMIT_OP(DYNCASTC_R); - EMIT_OP(DYNCASTC_K); - EMIT_OP(TEST); - EMIT_OP(TESTN); - EMIT_OP(JMP); - //EMIT_OP(IJMP); - EMIT_OP(PARAM); - EMIT_OP(PARAMI); - //EMIT_OP(CALL); - EMIT_OP(CALL_K); - EMIT_OP(VTBL); - EMIT_OP(SCOPE); - //EMIT_OP(TAIL); - //EMIT_OP(TAIL_K); - EMIT_OP(RESULT); - EMIT_OP(RET); - EMIT_OP(RETI); - EMIT_OP(NEW); - EMIT_OP(NEW_K); - EMIT_OP(THROW); - EMIT_OP(BOUND); - EMIT_OP(BOUND_K); - EMIT_OP(BOUND_R); - //EMIT_OP(CONCAT); - //EMIT_OP(LENS); - //EMIT_OP(CMPS); - EMIT_OP(SLL_RR); - EMIT_OP(SLL_RI); - EMIT_OP(SLL_KR); - EMIT_OP(SRL_RR); - EMIT_OP(SRL_RI); - EMIT_OP(SRL_KR); - EMIT_OP(SRA_RR); - EMIT_OP(SRA_RI); - EMIT_OP(SRA_KR); - EMIT_OP(ADD_RR); - EMIT_OP(ADD_RK); - EMIT_OP(ADDI); - EMIT_OP(SUB_RR); - EMIT_OP(SUB_RK); - EMIT_OP(SUB_KR); - EMIT_OP(MUL_RR); - EMIT_OP(MUL_RK); - EMIT_OP(DIV_RR); - EMIT_OP(DIV_RK); - EMIT_OP(DIV_KR); - EMIT_OP(DIVU_RR); - EMIT_OP(DIVU_RK); - EMIT_OP(DIVU_KR); - EMIT_OP(MOD_RR); - EMIT_OP(MOD_RK); - EMIT_OP(MOD_KR); - EMIT_OP(MODU_RR); - EMIT_OP(MODU_RK); - EMIT_OP(MODU_KR); - EMIT_OP(AND_RR); - EMIT_OP(AND_RK); - EMIT_OP(OR_RR); - EMIT_OP(OR_RK); - EMIT_OP(XOR_RR); - EMIT_OP(XOR_RK); - EMIT_OP(MIN_RR); - EMIT_OP(MIN_RK); - EMIT_OP(MAX_RR); - EMIT_OP(MAX_RK); - EMIT_OP(ABS); - EMIT_OP(NEG); - EMIT_OP(NOT); - EMIT_OP(EQ_R); - EMIT_OP(EQ_K); - EMIT_OP(LT_RR); - EMIT_OP(LT_RK); - EMIT_OP(LT_KR); - EMIT_OP(LE_RR); - EMIT_OP(LE_RK); - EMIT_OP(LE_KR); - EMIT_OP(LTU_RR); - EMIT_OP(LTU_RK); - EMIT_OP(LTU_KR); - EMIT_OP(LEU_RR); - EMIT_OP(LEU_RK); - EMIT_OP(LEU_KR); - EMIT_OP(ADDF_RR); - EMIT_OP(ADDF_RK); - EMIT_OP(SUBF_RR); - EMIT_OP(SUBF_RK); - EMIT_OP(SUBF_KR); - EMIT_OP(MULF_RR); - EMIT_OP(MULF_RK); - EMIT_OP(DIVF_RR); - EMIT_OP(DIVF_RK); - EMIT_OP(DIVF_KR); - EMIT_OP(MODF_RR); - EMIT_OP(MODF_RK); - EMIT_OP(MODF_KR); - EMIT_OP(POWF_RR); - EMIT_OP(POWF_RK); - EMIT_OP(POWF_KR); - EMIT_OP(MINF_RR); - EMIT_OP(MINF_RK); - EMIT_OP(MAXF_RR); - EMIT_OP(MAXF_RK); - EMIT_OP(ATAN2); - EMIT_OP(FLOP); - EMIT_OP(EQF_R); - EMIT_OP(EQF_K); - EMIT_OP(LTF_RR); - EMIT_OP(LTF_RK); - EMIT_OP(LTF_KR); - EMIT_OP(LEF_RR); - EMIT_OP(LEF_RK); - EMIT_OP(LEF_KR); - EMIT_OP(NEGV2); - EMIT_OP(ADDV2_RR); - EMIT_OP(SUBV2_RR); - EMIT_OP(DOTV2_RR); - EMIT_OP(MULVF2_RR); - EMIT_OP(MULVF2_RK); - EMIT_OP(DIVVF2_RR); - EMIT_OP(DIVVF2_RK); - EMIT_OP(LENV2); - EMIT_OP(EQV2_R); - EMIT_OP(EQV2_K); - EMIT_OP(NEGV3); - EMIT_OP(ADDV3_RR); - EMIT_OP(SUBV3_RR); - EMIT_OP(DOTV3_RR); - EMIT_OP(CROSSV_RR); - EMIT_OP(MULVF3_RR); - EMIT_OP(MULVF3_RK); - EMIT_OP(DIVVF3_RR); - EMIT_OP(DIVVF3_RK); - EMIT_OP(LENV3); - EMIT_OP(EQV3_R); - EMIT_OP(EQV3_K); - EMIT_OP(ADDA_RR); - EMIT_OP(ADDA_RK); - EMIT_OP(SUBA); - EMIT_OP(EQA_R); - EMIT_OP(EQA_K); - } - } - - void EmitNOP() - { - cc.nop(); - } - - // Load constants. - - void EmitLI() - { - cc.mov(regD[A], BCs); - } - - void EmitLK() - { - cc.mov(regD[A], konstd[BC]); - } - - void EmitLKF() - { - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(konstf + BC)); - cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); - } - - void EmitLKS() { - auto loadLambda = [] (FString* to, FString* from) -> void { - *to = *from; - }; - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(loadLambda))), - asmjit::FuncSignature2(asmjit::CallConv::kIdHostCDecl)); - call->setArg(0, regS[A]); - call->setArg(1, asmjit::imm(ToMemAddress(konsts + BC))); - } - - void EmitLKP() - { - cc.mov(regA[A], (int64_t)konsta[BC].v); - } - - void EmitLK_R() - { - cc.mov(regD[A], asmjit::x86::ptr(ToMemAddress(konstd), regD[B], 2, C * sizeof(int32_t))); - } - - void EmitLKF_R() - { - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(konstf + BC)); - cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp, regD[B], 3, C * sizeof(double))); - } - - //void EmitLKS_R() { } - - void EmitLKP_R() - { - cc.mov(regA[A], asmjit::x86::ptr(ToMemAddress(konsta), regD[B], 2, C * sizeof(void*))); - } - - //void EmitLFP() { } // load frame pointer - - void EmitMETA() - { - typedef void*(*FuncPtr)(void*); - - auto label = cc.newLabel(); - cc.test(regA[B], regA[B]); - cc.jne(label); - EmitThrowException(X_READ_NIL); - cc.bind(label); - - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](void *o) -> void* - { - return static_cast(o)->GetClass()->Meta; - }))), asmjit::FuncSignature1()); - call->setRet(0, regA[A]); - call->setArg(0, regA[B]); - } - - void EmitCLSS() - { - typedef void*(*FuncPtr)(void*); - - auto label = cc.newLabel(); - cc.test(regA[B], regA[B]); - cc.jne(label); - EmitThrowException(X_READ_NIL); - cc.bind(label); - - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](void *o) -> void* - { - return static_cast(o)->GetClass(); - }))), asmjit::FuncSignature1()); - call->setRet(0, regA[A]); - call->setArg(0, regA[B]); - } - - // Load from memory. rA = *(rB + rkC) - - void EmitLB() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.movsx(regD[A], asmjit::x86::byte_ptr(regA[B], konstd[C])); - } - - void EmitLB_R() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.movsx(regD[A], asmjit::x86::byte_ptr(regA[B], regD[C])); - } - - void EmitLH() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.movsx(regD[A], asmjit::x86::word_ptr(regA[B], konstd[C])); - } - - void EmitLH_R() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.movsx(regD[A], asmjit::x86::word_ptr(regA[B], regD[C])); - } - - void EmitLW() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.mov(regD[A], asmjit::x86::dword_ptr(regA[B], konstd[C])); - } - - void EmitLW_R() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.mov(regD[A], asmjit::x86::dword_ptr(regA[B], regD[C])); - } - - void EmitLBU() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.movzx(regD[A], asmjit::x86::byte_ptr(regA[B], konstd[C])); - } - - void EmitLBU_R() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.movzx(regD[A].r8Lo(), asmjit::x86::byte_ptr(regA[B], regD[C])); - } - - void EmitLHU() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.movzx(regD[A].r16(), asmjit::x86::word_ptr(regA[B], konstd[C])); - } - - void EmitLHU_R() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.movzx(regD[A].r16(), asmjit::x86::word_ptr(regA[B], regD[C])); - } - - void EmitLSP() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.movss(regF[A], asmjit::x86::dword_ptr(regA[B], konstd[C])); - } - - void EmitLSP_R() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.movss(regF[A], asmjit::x86::dword_ptr(regA[B], regD[C])); - } - - void EmitLDP() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.movsd(regF[A], asmjit::x86::qword_ptr(regA[B], konstd[C])); - } - - void EmitLDP_R() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.movsd(regF[A], asmjit::x86::qword_ptr(regA[B], regD[C])); - } - - void EmitLS() { - EmitNullPointerThrow(B, X_READ_NIL); - auto ptr = cc.newIntPtr(); - cc.mov(ptr, regA[B]); - cc.add(ptr, konstd[C]); - auto loadLambda = [](FString* to, FString* from) -> void { - *to = *from; - }; - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(loadLambda))), - asmjit::FuncSignature2(asmjit::CallConv::kIdHostCDecl)); - call->setArg(0, regS[A]); - call->setArg(1, ptr); - } - - void EmitLS_R() { - EmitNullPointerThrow(B, X_READ_NIL); - auto ptr = cc.newIntPtr(); - cc.mov(ptr, regA[B]); - auto tmp = cc.newIntPtr(); - cc.mov(tmp, regD[C]); - cc.add(ptr, tmp); - auto loadLambda = [](FString* to, FString* from) -> void { - *to = *from; - }; - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(loadLambda))), - asmjit::FuncSignature2(asmjit::CallConv::kIdHostCDecl)); - call->setArg(0, regS[A]); - call->setArg(1, ptr); - } - - void EmitLO() - { - EmitNullPointerThrow(B, X_READ_NIL); - - auto ptr = cc.newIntPtr(); - cc.mov(ptr, asmjit::x86::ptr(regA[B], konstd[C])); - - typedef void*(*FuncPtr)(void*); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](void *ptr) -> void* - { - DObject *p = static_cast(ptr); - return GC::ReadBarrier(p); - }))), asmjit::FuncSignature1()); - call->setRet(0, regA[A]); - call->setArg(0, ptr); - } - - void EmitLO_R() - { - EmitNullPointerThrow(B, X_READ_NIL); - - auto ptr = cc.newIntPtr(); - cc.mov(ptr, asmjit::x86::ptr(regA[B], regD[C])); - - typedef void*(*FuncPtr)(void*); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](void *ptr) -> void* - { - DObject *p = static_cast(ptr); - return GC::ReadBarrier(p); - }))), asmjit::FuncSignature1()); - call->setRet(0, regA[A]); - call->setArg(0, ptr); - } - - void EmitLP() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.mov(regA[A], asmjit::x86::ptr(regA[B], konstd[C])); - } - - void EmitLP_R() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.mov(regA[A], asmjit::x86::ptr(regA[B], regD[C])); - } - - void EmitLV2() - { - EmitNullPointerThrow(B, X_READ_NIL); - auto tmp = cc.newIntPtr(); - cc.mov(tmp, regA[B]); - cc.add(tmp, konstd[C]); - cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); - cc.movsd(regF[A + 1], asmjit::x86::qword_ptr(tmp, 8)); - } - - void EmitLV2_R() - { - EmitNullPointerThrow(B, X_READ_NIL); - auto tmp = cc.newIntPtr(); - cc.mov(tmp, regA[B]); - cc.add(tmp, regD[C]); - cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); - cc.movsd(regF[A + 1], asmjit::x86::qword_ptr(tmp, 8)); - } - - void EmitLV3() - { - EmitNullPointerThrow(B, X_READ_NIL); - auto tmp = cc.newIntPtr(); - cc.mov(tmp, regA[B]); - cc.add(tmp, konstd[C]); - cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); - cc.movsd(regF[A + 1], asmjit::x86::qword_ptr(tmp, 8)); - cc.movsd(regF[A + 2], asmjit::x86::qword_ptr(tmp, 16)); - } - - void EmitLV3_R() - { - EmitNullPointerThrow(B, X_READ_NIL); - auto tmp = cc.newIntPtr(); - cc.mov(tmp, regA[B]); - cc.add(tmp, regD[C]); - cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); - cc.movsd(regF[A + 1], asmjit::x86::qword_ptr(tmp, 8)); - cc.movsd(regF[A + 2], asmjit::x86::qword_ptr(tmp, 16)); - } - - void EmitLCS() { - EmitNullPointerThrow(B, X_READ_NIL); - auto ptr = cc.newIntPtr(); - cc.mov(ptr, regA[B]); - cc.add(ptr, konstd[C]); - auto loadLambda = [](FString* to, char** from) -> void { - *to = *from; - }; - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(loadLambda))), - asmjit::FuncSignature2(asmjit::CallConv::kIdHostCDecl)); - call->setArg(0, regS[A]); - call->setArg(1, ptr); - } - - //void EmitLCS_R() { } - - void EmitLBIT() - { - EmitNullPointerThrow(B, X_READ_NIL); - cc.movsx(regD[A], asmjit::x86::byte_ptr(regA[B])); - cc.and_(regD[A], C); - cc.cmp(regD[A], 0); - cc.setne(regD[A]); - } - - // Store instructions. *(rA + rkC) = rB - - void EmitSB() - { - EmitNullPointerThrow(A, X_WRITE_NIL); - cc.mov(asmjit::x86::byte_ptr(regA[A], konstd[C]), regD[B].r8Lo()); - } - - void EmitSB_R() - { - EmitNullPointerThrow(A, X_WRITE_NIL); - cc.mov(asmjit::x86::byte_ptr(regA[A], regD[C]), regD[B].r8Lo()); - } - - void EmitSH() - { - EmitNullPointerThrow(A, X_WRITE_NIL); - cc.mov(asmjit::x86::word_ptr(regA[A], konstd[C]), regD[B].r16()); - } - - void EmitSH_R() - { - EmitNullPointerThrow(A, X_WRITE_NIL); - cc.mov(asmjit::x86::word_ptr(regA[A], regD[C]), regD[B].r16()); - } - - void EmitSW() - { - EmitNullPointerThrow(A, X_WRITE_NIL); - cc.mov(asmjit::x86::dword_ptr(regA[A], konstd[C]), regD[B]); - } - - void EmitSW_R() - { - EmitNullPointerThrow(A, X_WRITE_NIL); - cc.mov(asmjit::x86::dword_ptr(regA[A], regD[C]), regD[B]); - } - - void EmitSSP() - { - EmitNullPointerThrow(A, X_WRITE_NIL); - cc.movss(asmjit::x86::dword_ptr(regA[A], konstd[C]), regF[B]); - } - - void EmitSSP_R() - { - EmitNullPointerThrow(A, X_WRITE_NIL); - cc.movss(asmjit::x86::dword_ptr(regA[A], regD[C]), regF[B]); - } - - void EmitSDP() - { - EmitNullPointerThrow(A, X_WRITE_NIL); - cc.movsd(asmjit::x86::qword_ptr(regA[A], konstd[C]), regF[B]); - } - - void EmitSDP_R() - { - EmitNullPointerThrow(A, X_WRITE_NIL); - cc.movsd(asmjit::x86::qword_ptr(regA[A], regD[C]), regF[B]); - } - - void EmitSS() - { - EmitNullPointerThrow(B, X_WRITE_NIL); - auto ptr = cc.newIntPtr(); - cc.mov(ptr, regA[A]); - cc.add(ptr, konstd[C]); - auto loadLambda = [](FString* to, FString* from) -> void { - *to = *from; - }; - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(loadLambda))), - asmjit::FuncSignature2(asmjit::CallConv::kIdHostCDecl)); - call->setArg(0, ptr); - call->setArg(1, regS[B]); - } - - void EmitSS_R() - { - EmitNullPointerThrow(B, X_WRITE_NIL); - auto ptr = cc.newIntPtr(); - cc.mov(ptr, regA[A]); - auto tmp = cc.newIntPtr(); - cc.mov(tmp, regD[C]); - cc.add(ptr, tmp); - auto loadLambda = [](FString* to, FString* from) -> void { - *to = *from; - }; - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(loadLambda))), - asmjit::FuncSignature2(asmjit::CallConv::kIdHostCDecl)); - call->setArg(0, ptr); - call->setArg(1, regS[B]); - } - - void EmitSO() - { - cc.mov(asmjit::x86::ptr(regA[A], konstd[C]), regA[B]); - - typedef void(*FuncPtr)(DObject*); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(GC::WriteBarrier))), asmjit::FuncSignature1()); - call->setArg(0, regA[B]); - } - - void EmitSO_R() - { - cc.mov(asmjit::x86::ptr(regA[A], regD[C]), regA[B]); - - typedef void(*FuncPtr)(DObject*); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(GC::WriteBarrier))), asmjit::FuncSignature1()); - call->setArg(0, regA[B]); - } - - void EmitSP() - { - cc.mov(asmjit::x86::ptr(regA[A], konstd[C]), regA[B]); - } - - void EmitSP_R() - { - cc.mov(asmjit::x86::ptr(regA[A], regD[C]), regA[B]); - } - - void EmitSV2() - { - EmitNullPointerThrow(B, X_WRITE_NIL); - auto tmp = cc.newIntPtr(); - cc.mov(tmp, regA[B]); - cc.add(tmp, konstd[C]); - cc.movsd(asmjit::x86::qword_ptr(tmp), regF[B]); - cc.movsd(asmjit::x86::qword_ptr(tmp, 8), regF[B + 1]); - } - - void EmitSV2_R() - { - EmitNullPointerThrow(B, X_WRITE_NIL); - auto tmp = cc.newIntPtr(); - cc.mov(tmp, regA[B]); - cc.add(tmp, regD[C]); - cc.movsd(asmjit::x86::qword_ptr(tmp), regF[B]); - cc.movsd(asmjit::x86::qword_ptr(tmp, 8), regF[B + 1]); - } - - void EmitSV3() - { - EmitNullPointerThrow(B, X_WRITE_NIL); - auto tmp = cc.newIntPtr(); - cc.mov(tmp, regA[B]); - cc.add(tmp, konstd[C]); - cc.movsd(asmjit::x86::qword_ptr(tmp), regF[B]); - cc.movsd(asmjit::x86::qword_ptr(tmp, 8), regF[B + 1]); - cc.movsd(asmjit::x86::qword_ptr(tmp, 16), regF[B + 2]); - } - - void EmitSV3_R() - { - EmitNullPointerThrow(B, X_WRITE_NIL); - auto tmp = cc.newIntPtr(); - cc.mov(tmp, regA[B]); - cc.add(tmp, regD[C]); - cc.movsd(asmjit::x86::qword_ptr(tmp), regF[B]); - cc.movsd(asmjit::x86::qword_ptr(tmp, 8), regF[B + 1]); - cc.movsd(asmjit::x86::qword_ptr(tmp, 16), regF[B + 2]); - } - - void EmitSBIT() - { - EmitNullPointerThrow(B, X_WRITE_NIL); - auto tmp1 = cc.newInt32(); - auto tmp2 = cc.newInt32(); - cc.mov(tmp1, asmjit::x86::byte_ptr(regA[A])); - cc.mov(tmp2, tmp1); - cc.or_(tmp1, (int)C); - cc.and_(tmp2, ~(int)C); - cc.test(regD[B], regD[B]); - cc.cmove(tmp1, tmp2); - cc.mov(asmjit::x86::byte_ptr(regA[A]), tmp1); - } - - // Move instructions. - - void EmitMOVE() - { - cc.mov(regD[A], regD[B]); - } - void EmitMOVEF() - { - cc.movsd(regF[A], regF[B]); - } - - // void EmitMOVES() {} // sA = sB - - void EmitMOVEA() - { - cc.mov(regA[A], regA[B]); - } - - void EmitMOVEV2() - { - cc.movsd(regF[A], regF[B]); - cc.movsd(regF[A + 1], regF[B + 1]); - } - - void EmitMOVEV3() - { - cc.movsd(regF[A], regF[B]); - cc.movsd(regF[A + 1], regF[B + 1]); - cc.movsd(regF[A + 2], regF[B + 2]); - } - - void EmitCAST() - { - switch (C) - { - case CAST_I2F: - cc.cvtsi2sd(regF[A], regD[B]); - break; - case CAST_U2F: - { - auto tmp = cc.newInt64(); - cc.xor_(tmp, tmp); - cc.mov(tmp.r32(), regD[B]); - cc.cvtsi2sd(regF[A], tmp); - break; - } - case CAST_F2I: - cc.cvttsd2si(regD[A], regF[B]); - break; - case CAST_F2U: - { - auto tmp = cc.newInt64(); - cc.cvttsd2si(tmp, regF[B]); - cc.mov(regD[A], tmp.r32()); - break; - } - /*case CAST_I2S: - reg.s[A].Format("%d", reg.d[B]); - break; - case CAST_U2S: - reg.s[A].Format("%u", reg.d[B]); - break; - case CAST_F2S: - reg.s[A].Format("%.5f", reg.f[B]); // keep this small. For more precise conversion there should be a conversion function. - break; - case CAST_V22S: - reg.s[A].Format("(%.5f, %.5f)", reg.f[B], reg.f[b + 1]); - break; - case CAST_V32S: - reg.s[A].Format("(%.5f, %.5f, %.5f)", reg.f[B], reg.f[b + 1], reg.f[b + 2]); - break; - case CAST_P2S: - { - if (reg.a[B] == nullptr) reg.s[A] = "null"; - else reg.s[A].Format("%p", reg.a[B]); - break; - } - case CAST_S2I: - reg.d[A] = (VM_SWORD)reg.s[B].ToLong(); - break; - case CAST_S2F: - reg.f[A] = reg.s[B].ToDouble(); - break; - case CAST_S2N: - reg.d[A] = reg.s[B].Len() == 0 ? FName(NAME_None) : FName(reg.s[B]); - break; - case CAST_N2S: - { - FName name = FName(ENamedName(reg.d[B])); - reg.s[A] = name.IsValidName() ? name.GetChars() : ""; - break; - } - case CAST_S2Co: - reg.d[A] = V_GetColor(NULL, reg.s[B]); - break; - case CAST_Co2S: - reg.s[A].Format("%02x %02x %02x", PalEntry(reg.d[B]).r, PalEntry(reg.d[B]).g, PalEntry(reg.d[B]).b); - break; - case CAST_S2So: - reg.d[A] = FSoundID(reg.s[B]); - break; - case CAST_So2S: - reg.s[A] = S_sfx[reg.d[B]].name; - break; - case CAST_SID2S: - reg.s[A] = unsigned(reg.d[B]) >= sprites.Size() ? "TNT1" : sprites[reg.d[B]].name; - break; - case CAST_TID2S: - { - auto tex = TexMan[*(FTextureID*)&(reg.d[B])]; - reg.s[A] = tex == nullptr ? "(null)" : tex->Name.GetChars(); - break; - }*/ - default: - assert(0); - } - } - - void EmitCASTB() - { - if (C == CASTB_I) - { - cc.mov(regD[A], regD[B]); - cc.shr(regD[A], 31); - } - else if (C == CASTB_F) - { - auto zero = cc.newXmm(); - auto one = cc.newInt32(); - cc.xorpd(zero, zero); - cc.mov(one, 1); - cc.ucomisd(regF[A], zero); - cc.setp(regD[A]); - cc.cmovne(regD[A], one); - } - else if (C == CASTB_A) - { - cc.test(regA[A], regA[A]); - cc.setne(regD[A]); - } - else - { - //reg.d[A] = reg.s[B].Len() > 0; - } - } - - void EmitDYNCAST_R() - { - using namespace asmjit; - typedef DObject*(*FuncPtr)(DObject*, PClass*); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](DObject *obj, PClass *cls) -> DObject* { - return (obj && obj->IsKindOf(cls)) ? obj : nullptr; - }))), FuncSignature2()); - call->setRet(0, regA[A]); - call->setArg(0, regA[B]); - call->setArg(1, regA[C]); - } - - void EmitDYNCAST_K() - { - using namespace asmjit; - auto c = cc.newIntPtr(); - cc.mov(c, ToMemAddress(konsta[C].o)); - typedef DObject*(*FuncPtr)(DObject*, PClass*); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](DObject *obj, PClass *cls) -> DObject* { - return (obj && obj->IsKindOf(cls)) ? obj : nullptr; - }))), FuncSignature2()); - call->setRet(0, regA[A]); - call->setArg(0, regA[B]); - call->setArg(1, c); - } - - void EmitDYNCASTC_R() - { - using namespace asmjit; - typedef PClass*(*FuncPtr)(PClass*, PClass*); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](PClass *cls1, PClass *cls2) -> PClass* { - return (cls1 && cls1->IsDescendantOf(cls2)) ? cls1 : nullptr; - }))), FuncSignature2()); - call->setRet(0, regA[A]); - call->setArg(0, regA[B]); - call->setArg(1, regA[C]); - } - - void EmitDYNCASTC_K() - { - using namespace asmjit; - auto c = cc.newIntPtr(); - cc.mov(c, ToMemAddress(konsta[C].o)); - typedef PClass*(*FuncPtr)(PClass*, PClass*); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](PClass *cls1, PClass *cls2) -> PClass* { - return (cls1 && cls1->IsDescendantOf(cls2)) ? cls1 : nullptr; - }))), FuncSignature2()); - call->setRet(0, regA[A]); - call->setArg(0, regA[B]); - call->setArg(1, c); - } - - // Control flow. - - void EmitTEST() - { - int i = (int)(ptrdiff_t)(pc - sfunc->Code); - cc.cmp(regD[A], BC); - cc.jne(labels[i + 2]); - } - - void EmitTESTN() - { - int bc = BC; - int i = (int)(ptrdiff_t)(pc - sfunc->Code); - cc.cmp(regD[A], -bc); - cc.jne(labels[i + 2]); - } - - void EmitJMP() - { - auto dest = pc + JMPOFS(pc) + 1; - int i = (int)(ptrdiff_t)(dest - sfunc->Code); - cc.jmp(labels[i]); - } - - //void EmitIJMP() {} // pc += dA + BC -- BC is a signed offset. The target instruction must be a JMP. - - void EmitPARAM() - { - using namespace asmjit; - - int index = NumParam++; - ParamOpcodes.Push(pc); - - X86Gp stackPtr, tmp; - X86Xmm tmp2; - - switch (B) - { - case REGT_NIL: - cc.mov(x86::ptr(params, index * sizeof(VMValue) + offsetof(VMValue, a)), (int64_t)0); - cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_NIL); - break; - case REGT_INT: - cc.mov(x86::dword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, i)), regD[C]); - cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_INT); - break; - case REGT_INT | REGT_ADDROF: - stackPtr = cc.newIntPtr(); - cc.mov(stackPtr, frameD); - cc.add(stackPtr, C * sizeof(int32_t)); - cc.mov(x86::ptr(params, index * sizeof(VMValue) + offsetof(VMValue, a)), stackPtr); - cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_POINTER); - break; - case REGT_INT | REGT_KONST: - cc.mov(x86::dword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, i)), konstd[C]); - cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_INT); - break; - //case REGT_STRING: - //case REGT_STRING | REGT_ADDROF: - //case REGT_STRING | REGT_KONST: - case REGT_POINTER: - cc.mov(x86::ptr(params, index * sizeof(VMValue) + offsetof(VMValue, a)), regA[C]); - cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_POINTER); - break; - case REGT_POINTER | REGT_ADDROF: - stackPtr = cc.newIntPtr(); - cc.mov(stackPtr, frameA); - cc.add(stackPtr, C * sizeof(void*)); - cc.mov(x86::ptr(params, index * sizeof(VMValue) + offsetof(VMValue, a)), stackPtr); - cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_POINTER); - break; - case REGT_POINTER | REGT_KONST: - tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(konsta[C].v)); - cc.mov(x86::ptr(params, index * sizeof(VMValue) + offsetof(VMValue, a)), tmp); - cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_POINTER); - break; - case REGT_FLOAT: - cc.movsd(x86::qword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, f)), regF[C]); - cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_FLOAT); - break; - case REGT_FLOAT | REGT_MULTIREG2: - cc.movsd(x86::qword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, f)), regF[C]); - cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_FLOAT); - index = NumParam++; - ParamOpcodes.Push(pc); - cc.movsd(x86::qword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, f)), regF[C + 1]); - cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_FLOAT); - break; - case REGT_FLOAT | REGT_MULTIREG3: - cc.movsd(x86::qword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, f)), regF[C]); - cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_FLOAT); - index = NumParam++; - ParamOpcodes.Push(pc); - cc.movsd(x86::qword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, f)), regF[C + 1]); - cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_FLOAT); - index = NumParam++; - ParamOpcodes.Push(pc); - cc.movsd(x86::qword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, f)), regF[C + 2]); - cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_FLOAT); - break; - case REGT_FLOAT | REGT_ADDROF: - stackPtr = cc.newIntPtr(); - cc.mov(stackPtr, frameF); - cc.add(stackPtr, C * sizeof(double)); - cc.mov(x86::ptr(params, index * sizeof(VMValue) + offsetof(VMValue, a)), stackPtr); - cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_POINTER); - break; - case REGT_FLOAT | REGT_KONST: - tmp = cc.newIntPtr(); - tmp2 = cc.newXmmSd(); - cc.mov(tmp, ToMemAddress(konstf + C)); - cc.movsd(tmp2, asmjit::x86::qword_ptr(tmp)); - cc.movsd(x86::qword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, f)), tmp2); - cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_FLOAT); - break; - default: - I_FatalError("Unknown REGT value passed to EmitPARAM\n"); - break; - } - - } - - void EmitPARAMI() - { - int index = NumParam++; - ParamOpcodes.Push(pc); - cc.mov(asmjit::x86::dword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, i)), (int)ABCs); - cc.mov(asmjit::x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_INT); - } - - void EmitCALL() - { - EmitDoCall(regA[A]); - } - - void EmitCALL_K() - { - auto ptr = cc.newIntPtr(); - cc.mov(ptr, ToMemAddress(konsta[A].o)); - EmitDoCall(ptr); - } - - void EmitDoCall(asmjit::X86Gp ptr) - { - using namespace asmjit; - - if (NumParam < B) - I_FatalError("OP_CALL parameter count does not match the number of preceding OP_PARAM instructions"); - - StoreInOuts(B); - FillReturns(pc + 1, C); - - X86Gp paramsptr; - if (B != NumParam) - { - paramsptr = cc.newIntPtr(); - cc.mov(paramsptr, params); - cc.add(paramsptr, (NumParam - B) * sizeof(VMValue)); - } - else - { - paramsptr = params; - } - - auto result = cc.newInt32(); - auto call = cc.call(ToMemAddress(&JitCompiler::DoCall), FuncSignature7()); - call->setRet(0, result); - call->setArg(0, stack); - call->setArg(1, ptr); - call->setArg(2, asmjit::Imm(B)); - call->setArg(3, asmjit::Imm(C)); - call->setArg(4, paramsptr); - call->setArg(5, callReturns); - call->setArg(6, exceptInfo); - - auto noexception = cc.newLabel(); - auto exceptResult = cc.newInt32(); - cc.mov(exceptResult, x86::dword_ptr(exceptInfo, 0 * 4)); - cc.cmp(exceptResult, (int)-1); - cc.je(noexception); - X86Gp vReg = cc.newInt32(); - cc.mov(vReg, 0); - cc.ret(vReg); - cc.bind(noexception); - - LoadReturns(pc - B, B, true); - LoadReturns(pc + 1, C, false); - - NumParam -= B; - ParamOpcodes.Resize(ParamOpcodes.Size() - B); - } - - void StoreInOuts(int b) - { - using namespace asmjit; - - for (unsigned int i = ParamOpcodes.Size() - b; i < ParamOpcodes.Size(); i++) - { - asmjit::X86Gp stackPtr; - switch (ParamOpcodes[i]->b) - { - case REGT_INT | REGT_ADDROF: - stackPtr = cc.newIntPtr(); - cc.mov(stackPtr, frameD); - cc.add(stackPtr, C * sizeof(int32_t)); - cc.mov(x86::dword_ptr(stackPtr), regD[C]); - break; - //case REGT_STRING | REGT_ADDROF: - // break; - case REGT_POINTER | REGT_ADDROF: - stackPtr = cc.newIntPtr(); - cc.mov(stackPtr, frameA); - cc.add(stackPtr, C * sizeof(void*)); - cc.mov(x86::ptr(stackPtr), regA[C]); - break; - case REGT_FLOAT | REGT_ADDROF: - stackPtr = cc.newIntPtr(); - cc.mov(stackPtr, frameF); - cc.add(stackPtr, C * sizeof(double)); - cc.movsd(x86::qword_ptr(stackPtr), regF[C]); - break; - default: - break; - } - } - } - - void LoadReturns(const VMOP *retval, int numret, bool inout) - { - for (int i = 0; i < numret; ++i) - { - if (!inout && retval[i].op != OP_RESULT) - I_FatalError("Expected OP_RESULT to follow OP_CALL\n"); - else if (inout && retval[i].op != OP_PARAMI) - continue; - else if (inout && retval[i].op != OP_PARAM) - I_FatalError("Expected OP_PARAM to precede OP_CALL\n"); - - int type = retval[i].b; - int regnum = retval[i].c; - - if (inout && !(type & REGT_ADDROF)) - continue; - - switch (type & REGT_TYPE) - { - case REGT_INT: - cc.mov(regD[regnum], asmjit::x86::dword_ptr(frameD, regnum * sizeof(int32_t))); - break; - case REGT_FLOAT: - cc.movsd(regF[regnum], asmjit::x86::qword_ptr(frameF, regnum * sizeof(double))); - break; - /*case REGT_STRING: - break;*/ - case REGT_POINTER: - cc.mov(regA[regnum], asmjit::x86::ptr(frameA, regnum * sizeof(void*))); - break; - default: - I_FatalError("Unknown OP_RESULT type encountered in LoadReturns\n"); - break; - } - } - } - - void FillReturns(const VMOP *retval, int numret) - { - using namespace asmjit; - - for (int i = 0; i < numret; ++i) - { - if (retval[i].op != OP_RESULT) - { - I_FatalError("Expected OP_RESULT to follow OP_CALL\n"); - } - - int type = retval[i].b; - int regnum = retval[i].c; - - if (type & REGT_KONST) - { - I_FatalError("OP_RESULT with REGT_KONST is not allowed\n"); - } - - auto regPtr = cc.newIntPtr(); - - switch (type & REGT_TYPE) - { - case REGT_INT: - cc.mov(regPtr, frameD); - cc.add(regPtr, regnum * sizeof(int32_t)); - break; - case REGT_FLOAT: - cc.mov(regPtr, frameF); - cc.add(regPtr, regnum * sizeof(double)); - break; - /*case REGT_STRING: - cc.mov(regPtr, frameS); - cc.add(regPtr, regnum * sizeof(FString)); - break;*/ - case REGT_POINTER: - cc.mov(regPtr, frameA); - cc.add(regPtr, regnum * sizeof(void*)); - break; - default: - I_FatalError("Unknown OP_RESULT type encountered in FillReturns\n"); - break; - } - - cc.mov(x86::ptr(callReturns, i * sizeof(VMReturn) + offsetof(VMReturn, Location)), regPtr); - cc.mov(x86::byte_ptr(callReturns, i * sizeof(VMReturn) + offsetof(VMReturn, RegType)), type); - } - } - - static int DoCall(VMFrameStack *stack, VMFunction *call, int b, int c, VMValue *param, VMReturn *returns, JitExceptionInfo *exceptinfo) - { - try - { - int numret; - if (call->VarFlags & VARF_Native) - { - try - { - VMCycles[0].Unclock(); - numret = static_cast(call)->NativeCall(param, call->DefaultArgs, b, returns, c); - VMCycles[0].Clock(); - } - catch (CVMAbortException &err) - { - err.MaybePrintMessage(); - err.stacktrace.AppendFormat("Called from %s\n", call->PrintableName.GetChars()); - throw; - } - } - else - { - VMCalls[0]++; - VMScriptFunction *script = static_cast(call); - VMFrame *newf = stack->AllocFrame(script); - VMFillParams(param, newf, b); - try - { - numret = VMExec(stack, script->Code, returns, c); - } - catch (...) - { - stack->PopFrame(); - throw; - } - stack->PopFrame(); - } - - return numret; - } - catch (...) - { - // To do: store full exception in exceptinfo - exceptinfo->reason = X_OTHER; - return 0; - } - } - - void EmitVTBL() - { - auto notnull = cc.newLabel(); - cc.test(regA[B], regA[B]); - cc.jnz(notnull); - EmitThrowException(X_READ_NIL); - cc.bind(notnull); - - auto result = cc.newInt32(); - typedef VMFunction*(*FuncPtr)(DObject*, int); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](DObject *o, int c) -> VMFunction* { - auto p = o->GetClass(); - assert(c < (int)p->Virtuals.Size()); - return p->Virtuals[c]; - }))), asmjit::FuncSignature2()); - call->setRet(0, result); - call->setArg(0, regA[A]); - call->setArg(1, asmjit::Imm(C)); - } - - void EmitSCOPE() - { - auto notnull = cc.newLabel(); - cc.test(regA[A], regA[A]); - cc.jnz(notnull); - EmitThrowException(X_READ_NIL); - cc.bind(notnull); - - auto f = cc.newIntPtr(); - cc.mov(f, ToMemAddress(konsta[C].v)); - - auto result = cc.newInt32(); - typedef int(*FuncPtr)(DObject*, VMFunction*, int); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](DObject *o, VMFunction *f, int b) -> int { - try - { - FScopeBarrier::ValidateCall(o->GetClass(), f, b - 1); - return 1; - } - catch (const CVMAbortException &) - { - // To do: pass along the exception info - return 0; - } - }))), asmjit::FuncSignature3()); - call->setRet(0, result); - call->setArg(0, regA[A]); - call->setArg(1, f); - call->setArg(2, asmjit::Imm(B)); - - auto notzero = cc.newLabel(); - cc.test(result, result); - cc.jnz(notzero); - EmitThrowException(X_OTHER); - cc.bind(notzero); - } - - //void EmitTAIL() {} // Call+Ret in a single instruction - //void EmitTAIL_K() {} - - void EmitRESULT() - { - // This instruction is just a placeholder to indicate where a return - // value should be stored. It does nothing on its own and should not - // be executed. - } - - void EmitRET() - { - using namespace asmjit; - if (B == REGT_NIL) - { - X86Gp vReg = cc.newInt32(); - cc.mov(vReg, 0); - cc.ret(vReg); - } - else - { - int a = A; - int retnum = a & ~RET_FINAL; - - X86Gp reg_retnum = cc.newInt32(); - X86Gp location = cc.newIntPtr(); - Label L_endif = cc.newLabel(); - - cc.mov(reg_retnum, retnum); - cc.test(reg_retnum, numret); - cc.jg(L_endif); - - cc.mov(location, x86::ptr(ret, retnum * sizeof(VMReturn))); - - int regtype = B; - int regnum = C; - switch (regtype & REGT_TYPE) - { - case REGT_INT: - if (regtype & REGT_KONST) - cc.mov(x86::dword_ptr(location), konstd[regnum]); - else - cc.mov(x86::dword_ptr(location), regD[regnum]); - break; - case REGT_FLOAT: - if (regtype & REGT_KONST) - { - auto tmp = cc.newInt64(); - if (regtype & REGT_MULTIREG3) - { - cc.mov(tmp, (((int64_t *)konstf)[regnum])); - cc.mov(x86::qword_ptr(location), tmp); - - cc.mov(tmp, (((int64_t *)konstf)[regnum + 1])); - cc.mov(x86::qword_ptr(location, 8), tmp); - - cc.mov(tmp, (((int64_t *)konstf)[regnum + 2])); - cc.mov(x86::qword_ptr(location, 16), tmp); - } - else if (regtype & REGT_MULTIREG2) - { - cc.mov(tmp, (((int64_t *)konstf)[regnum])); - cc.mov(x86::qword_ptr(location), tmp); - - cc.mov(tmp, (((int64_t *)konstf)[regnum + 1])); - cc.mov(x86::qword_ptr(location, 8), tmp); - } - else - { - cc.mov(tmp, (((int64_t *)konstf)[regnum])); - cc.mov(x86::qword_ptr(location), tmp); - } - } - else - { - if (regtype & REGT_MULTIREG3) - { - cc.movsd(x86::qword_ptr(location), regF[regnum]); - cc.movsd(x86::qword_ptr(location, 8), regF[regnum + 1]); - cc.movsd(x86::qword_ptr(location, 16), regF[regnum + 2]); - } - else if (regtype & REGT_MULTIREG2) - { - cc.movsd(x86::qword_ptr(location), regF[regnum]); - cc.movsd(x86::qword_ptr(location, 8), regF[regnum + 1]); - } - else - { - cc.movsd(x86::qword_ptr(location), regF[regnum]); - } - } - break; - case REGT_STRING: - break; - case REGT_POINTER: - #ifdef ASMJIT_ARCH_X64 - if (regtype & REGT_KONST) - cc.mov(x86::qword_ptr(location), ToMemAddress(konsta[regnum].v)); - else - cc.mov(x86::qword_ptr(location), regA[regnum]); - #else - if (regtype & REGT_KONST) - cc.mov(x86::dword_ptr(location), ToMemAddress(konsta[regnum].v)); - else - cc.mov(x86::dword_ptr(location), regA[regnum]); - #endif - break; - } - - if (a & RET_FINAL) - { - cc.add(reg_retnum, 1); - cc.ret(reg_retnum); - } - - cc.bind(L_endif); - if (a & RET_FINAL) - cc.ret(numret); - } - } - - void EmitRETI() - { - using namespace asmjit; - - int a = A; - int retnum = a & ~RET_FINAL; - - X86Gp reg_retnum = cc.newInt32(); - X86Gp location = cc.newIntPtr(); - Label L_endif = cc.newLabel(); - - cc.mov(reg_retnum, retnum); - cc.test(reg_retnum, numret); - cc.jg(L_endif); - - cc.mov(location, x86::ptr(ret, retnum * sizeof(VMReturn))); - cc.mov(x86::dword_ptr(location), BCs); - - if (a & RET_FINAL) - { - cc.add(reg_retnum, 1); - cc.ret(reg_retnum); - } - - cc.bind(L_endif); - if (a & RET_FINAL) - cc.ret(numret); - } - - void EmitNEW() - { - auto result = cc.newIntPtr(); - typedef DObject*(*FuncPtr)(PClass*, int); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](PClass *cls, int c) -> DObject* { - try - { - if (!cls->ConstructNative) - { - ThrowAbortException(X_OTHER, "Class %s requires native construction", cls->TypeName.GetChars()); - } - else if (cls->bAbstract) - { - ThrowAbortException(X_OTHER, "Cannot instantiate abstract class %s", cls->TypeName.GetChars()); - } - else if (cls->IsDescendantOf(NAME_Actor)) // Creating actors here must be outright prohibited - { - ThrowAbortException(X_OTHER, "Cannot create actors with 'new'"); - } - - // [ZZ] validate readonly and between scope construction - if (c) FScopeBarrier::ValidateNew(cls, c - 1); - return cls->CreateNew(); - } - catch (const CVMAbortException &) - { - // To do: pass along the exception info - return nullptr; - } - }))), asmjit::FuncSignature2()); - call->setRet(0, result); - call->setArg(0, regA[B]); - call->setArg(1, asmjit::Imm(C)); - - auto notnull = cc.newLabel(); - cc.test(result, result); - cc.jnz(notnull); - EmitThrowException(X_OTHER); - cc.bind(notnull); - cc.mov(regA[A], result); - } - - void EmitNEW_K() - { - PClass *cls = (PClass*)konsta[B].v; - if (!cls->ConstructNative) - { - EmitThrowException(X_OTHER); // "Class %s requires native construction", cls->TypeName.GetChars() - } - else if (cls->bAbstract) - { - EmitThrowException(X_OTHER); // "Cannot instantiate abstract class %s", cls->TypeName.GetChars() - } - else if (cls->IsDescendantOf(NAME_Actor)) // Creating actors here must be outright prohibited - { - EmitThrowException(X_OTHER); // "Cannot create actors with 'new'" - } - else - { - auto result = cc.newIntPtr(); - auto regcls = cc.newIntPtr(); - cc.mov(regcls, ToMemAddress(konsta[B].v)); - typedef DObject*(*FuncPtr)(PClass*, int); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](PClass *cls, int c) -> DObject* { - try - { - if (c) FScopeBarrier::ValidateNew(cls, c - 1); - return cls->CreateNew(); - } - catch (const CVMAbortException &) - { - // To do: pass along the exception info - return nullptr; - } - }))), asmjit::FuncSignature2()); - call->setRet(0, result); - call->setArg(0, regcls); - call->setArg(1, asmjit::Imm(C)); - - auto notnull = cc.newLabel(); - cc.test(result, result); - cc.jnz(notnull); - EmitThrowException(X_OTHER); - cc.bind(notnull); - cc.mov(regA[A], result); - } - } - - void EmitTHROW() - { - EmitThrowException(EVMAbortException(BC)); - } - - void EmitBOUND() - { - auto label1 = cc.newLabel(); - auto label2 = cc.newLabel(); - cc.cmp(regD[A], (int)BC); - cc.jl(label1); - EmitThrowException(X_ARRAY_OUT_OF_BOUNDS); // "Max.index = %u, current index = %u\n", BC, reg.d[A] - cc.bind(label1); - cc.cmp(regD[A], (int)0); - cc.jge(label2); - EmitThrowException(X_ARRAY_OUT_OF_BOUNDS); // "Negative current index = %i\n", reg.d[A] - cc.bind(label2); - } - - void EmitBOUND_K() - { - auto label1 = cc.newLabel(); - auto label2 = cc.newLabel(); - cc.cmp(regD[A], (int)konstd[BC]); - cc.jl(label1); - EmitThrowException(X_ARRAY_OUT_OF_BOUNDS); // "Max.index = %u, current index = %u\n", konstd[BC], reg.d[A] - cc.bind(label1); - cc.cmp(regD[A], (int)0); - cc.jge(label2); - EmitThrowException(X_ARRAY_OUT_OF_BOUNDS); // "Negative current index = %i\n", reg.d[A] - cc.bind(label2); - } - - void EmitBOUND_R() - { - auto label1 = cc.newLabel(); - auto label2 = cc.newLabel(); - cc.cmp(regD[A], regD[B]); - cc.jl(label1); - EmitThrowException(X_ARRAY_OUT_OF_BOUNDS); // "Max.index = %u, current index = %u\n", reg.d[B], reg.d[A] - cc.bind(label1); - cc.cmp(regD[A], (int)0); - cc.jge(label2); - EmitThrowException(X_ARRAY_OUT_OF_BOUNDS); // "Negative current index = %i\n", reg.d[A] - cc.bind(label2); - } - - // String instructions. - - //void EmitCONCAT() {} // sA = sB..sC - //void EmitLENS() {} // dA = sB.Length - //void EmitCMPS() {} // if ((skB op skC) != (A & 1)) then pc++ - - // Integer math. - - void EmitSLL_RR() - { - auto rc = CheckRegD(C, A); - if (A != B) - cc.mov(regD[A], regD[B]); - cc.shl(regD[A], rc); - } - - void EmitSLL_RI() - { - if (A != B) - cc.mov(regD[A], regD[B]); - cc.shl(regD[A], C); - } - - void EmitSLL_KR() - { - auto rc = CheckRegD(C, A); - cc.mov(regD[A], konstd[B]); - cc.shl(regD[A], rc); - } - - void EmitSRL_RR() - { - auto rc = CheckRegD(C, A); - if (A != B) - cc.mov(regD[A], regD[B]); - cc.shr(regD[A], rc); - } - - void EmitSRL_RI() - { - if (A != B) - cc.mov(regD[A], regD[B]); - cc.shr(regD[A], C); - } - - void EmitSRL_KR() - { - auto rc = CheckRegD(C, A); - cc.mov(regD[A], konstd[B]); - cc.shr(regD[A], rc); - } - - void EmitSRA_RR() - { - auto rc = CheckRegD(C, A); - if (A != B) - cc.mov(regD[A], regD[B]); - cc.sar(regD[A], rc); - } - - void EmitSRA_RI() - { - if (A != B) - cc.mov(regD[A], regD[B]); - cc.sar(regD[A], C); - } - - void EmitSRA_KR() - { - auto rc = CheckRegD(C, A); - cc.mov(regD[A], konstd[B]); - cc.sar(regD[A], rc); - } - - void EmitADD_RR() - { - auto rc = CheckRegD(C, A); - if (A != B) - cc.mov(regD[A], regD[B]); - cc.add(regD[A], rc); - } - - void EmitADD_RK() - { - if (A != B) - cc.mov(regD[A], regD[B]); - cc.add(regD[A], konstd[C]); - } - - void EmitADDI() - { - if (A != B) - cc.mov(regD[A], regD[B]); - cc.add(regD[A], Cs); - } - - void EmitSUB_RR() - { - auto rc = CheckRegD(C, A); - if (A != B) - cc.mov(regD[A], regD[B]); - cc.sub(regD[A], rc); - } - - void EmitSUB_RK() - { - if (A != B) - cc.mov(regD[A], regD[B]); - cc.sub(regD[A], konstd[C]); - } - - void EmitSUB_KR() - { - auto rc = CheckRegD(C, A); - cc.mov(regD[A], konstd[B]); - cc.sub(regD[A], rc); - } - - void EmitMUL_RR() - { - auto rc = CheckRegD(C, A); - if (A != B) - cc.mov(regD[A], regD[B]); - cc.imul(regD[A], rc); - } - - void EmitMUL_RK() - { - if (A != B) - cc.mov(regD[A], regD[B]); - cc.imul(regD[A], konstd[C]); - } - - void EmitDIV_RR() - { - auto tmp0 = cc.newInt32(); - auto tmp1 = cc.newInt32(); - auto label = cc.newLabel(); - - cc.test(regD[C], regD[C]); - cc.jne(label); - - EmitThrowException(X_DIVISION_BY_ZERO); - - cc.bind(label); - cc.mov(tmp0, regD[B]); - cc.cdq(tmp1, tmp0); - cc.idiv(tmp1, tmp0, regD[C]); - cc.mov(regD[A], tmp0); - } - - void EmitDIV_RK() - { - if (konstd[C] != 0) - { - auto tmp0 = cc.newInt32(); - auto tmp1 = cc.newInt32(); - auto konstTmp = cc.newIntPtr(); - cc.mov(tmp0, regD[B]); - cc.cdq(tmp1, tmp0); - cc.mov(konstTmp, ToMemAddress(&konstd[C])); - cc.idiv(tmp1, tmp0, asmjit::x86::ptr(konstTmp)); - cc.mov(regD[A], tmp0); - } - else EmitThrowException(X_DIVISION_BY_ZERO); - } - - void EmitDIV_KR() - { - auto tmp0 = cc.newInt32(); - auto tmp1 = cc.newInt32(); - auto label = cc.newLabel(); - - cc.test(regD[C], regD[C]); - cc.jne(label); - - EmitThrowException(X_DIVISION_BY_ZERO); - - cc.bind(label); - cc.mov(tmp0, konstd[B]); - cc.cdq(tmp1, tmp0); - cc.idiv(tmp1, tmp0, regD[C]); - cc.mov(regD[A], tmp0); - } - - void EmitDIVU_RR() - { - auto tmp0 = cc.newInt32(); - auto tmp1 = cc.newInt32(); - auto label = cc.newLabel(); - - cc.test(regD[C], regD[C]); - cc.jne(label); - - EmitThrowException(X_DIVISION_BY_ZERO); - - cc.bind(label); - cc.mov(tmp0, regD[B]); - cc.mov(tmp1, 0); - cc.div(tmp1, tmp0, regD[C]); - cc.mov(regD[A], tmp0); - } - - void EmitDIVU_RK() - { - if (konstd[C] != 0) - { - auto tmp0 = cc.newInt32(); - auto tmp1 = cc.newInt32(); - auto konstTmp = cc.newIntPtr(); - cc.mov(tmp0, regD[B]); - cc.mov(tmp1, 0); - cc.mov(konstTmp, ToMemAddress(&konstd[C])); - cc.div(tmp1, tmp0, asmjit::x86::ptr(konstTmp)); - cc.mov(regD[A], tmp0); - } - else EmitThrowException(X_DIVISION_BY_ZERO); - } - - void EmitDIVU_KR() - { - auto tmp0 = cc.newInt32(); - auto tmp1 = cc.newInt32(); - auto label = cc.newLabel(); - - cc.test(regD[C], regD[C]); - cc.jne(label); - - EmitThrowException(X_DIVISION_BY_ZERO); - - cc.bind(label); - cc.mov(tmp0, konstd[B]); - cc.mov(tmp1, 0); - cc.div(tmp1, tmp0, regD[C]); - cc.mov(regD[A], tmp0); - } - - void EmitMOD_RR() - { - auto tmp0 = cc.newInt32(); - auto tmp1 = cc.newInt32(); - auto label = cc.newLabel(); - - cc.test(regD[C], regD[C]); - cc.jne(label); - - EmitThrowException(X_DIVISION_BY_ZERO); - - cc.bind(label); - cc.mov(tmp0, regD[B]); - cc.cdq(tmp1, tmp0); - cc.idiv(tmp1, tmp0, regD[C]); - cc.mov(regD[A], tmp1); - } - - void EmitMOD_RK() - { - if (konstd[C] != 0) - { - auto tmp0 = cc.newInt32(); - auto tmp1 = cc.newInt32(); - auto konstTmp = cc.newIntPtr(); - cc.mov(tmp0, regD[B]); - cc.cdq(tmp1, tmp0); - cc.mov(konstTmp, ToMemAddress(&konstd[C])); - cc.idiv(tmp1, tmp0, asmjit::x86::ptr(konstTmp)); - cc.mov(regD[A], tmp1); - } - else EmitThrowException(X_DIVISION_BY_ZERO); - } - - void EmitMOD_KR() - { - auto tmp0 = cc.newInt32(); - auto tmp1 = cc.newInt32(); - auto label = cc.newLabel(); - - cc.test(regD[C], regD[C]); - cc.jne(label); - - EmitThrowException(X_DIVISION_BY_ZERO); - - cc.bind(label); - cc.mov(tmp0, konstd[B]); - cc.cdq(tmp1, tmp0); - cc.idiv(tmp1, tmp0, regD[C]); - cc.mov(regD[A], tmp1); - } - - void EmitMODU_RR() - { - auto tmp0 = cc.newInt32(); - auto tmp1 = cc.newInt32(); - auto label = cc.newLabel(); - - cc.test(regD[C], regD[C]); - cc.jne(label); - - EmitThrowException(X_DIVISION_BY_ZERO); - - cc.bind(label); - cc.mov(tmp0, regD[B]); - cc.mov(tmp1, 0); - cc.div(tmp1, tmp0, regD[C]); - cc.mov(regD[A], tmp1); - } - - void EmitMODU_RK() - { - if (konstd[C] != 0) - { - auto tmp0 = cc.newInt32(); - auto tmp1 = cc.newInt32(); - auto konstTmp = cc.newIntPtr(); - cc.mov(tmp0, regD[B]); - cc.mov(tmp1, 0); - cc.mov(konstTmp, ToMemAddress(&konstd[C])); - cc.div(tmp1, tmp0, asmjit::x86::ptr(konstTmp)); - cc.mov(regD[A], tmp1); - } - else EmitThrowException(X_DIVISION_BY_ZERO); - } - - void EmitMODU_KR() - { - auto tmp0 = cc.newInt32(); - auto tmp1 = cc.newInt32(); - auto label = cc.newLabel(); - - cc.test(regD[C], regD[C]); - cc.jne(label); - - EmitThrowException(X_DIVISION_BY_ZERO); - - cc.bind(label); - cc.mov(tmp0, konstd[B]); - cc.mov(tmp1, 0); - cc.div(tmp1, tmp0, regD[C]); - cc.mov(regD[A], tmp1); - } - - void EmitAND_RR() - { - auto rc = CheckRegD(C, A); - if (A != B) - cc.mov(regD[A], regD[B]); - cc.and_(regD[A], rc); - } - - void EmitAND_RK() - { - if (A != B) - cc.mov(regD[A], regD[B]); - cc.and_(regD[A], konstd[C]); - } - - void EmitOR_RR() - { - auto rc = CheckRegD(C, A); - if (A != B) - cc.mov(regD[A], regD[B]); - cc.or_(regD[A], rc); - } - - void EmitOR_RK() - { - if (A != B) - cc.mov(regD[A], regD[B]); - cc.or_(regD[A], konstd[C]); - } - - void EmitXOR_RR() - { - auto rc = CheckRegD(C, A); - if (A != B) - cc.mov(regD[A], regD[B]); - cc.xor_(regD[A], rc); - } - - void EmitXOR_RK() - { - if (A != B) - cc.mov(regD[A], regD[B]); - cc.xor_(regD[A], konstd[C]); - } - - void EmitMIN_RR() - { - auto tmp0 = cc.newXmmSs(); - auto tmp1 = cc.newXmmSs(); - cc.movd(tmp0, regD[B]); - cc.movd(tmp1, regD[C]); - cc.pminsd(tmp0, tmp1); - cc.movd(regD[A], tmp0); - } - - void EmitMIN_RK() - { - auto tmp0 = cc.newXmmSs(); - auto tmp1 = cc.newXmmSs(); - auto konstTmp = cc.newIntPtr(); - cc.mov(konstTmp, ToMemAddress(&konstd[C])); - cc.movd(tmp0, regD[B]); - cc.movss(tmp1, asmjit::x86::dword_ptr(konstTmp)); - cc.pminsd(tmp0, tmp1); - cc.movd(regD[A], tmp0); - } - - void EmitMAX_RR() - { - auto tmp0 = cc.newXmmSs(); - auto tmp1 = cc.newXmmSs(); - cc.movd(tmp0, regD[B]); - cc.movd(tmp1, regD[C]); - cc.pmaxsd(tmp0, tmp1); - cc.movd(regD[A], tmp0); - } - - void EmitMAX_RK() - { - auto tmp0 = cc.newXmmSs(); - auto tmp1 = cc.newXmmSs(); - auto konstTmp = cc.newIntPtr(); - cc.mov(konstTmp, ToMemAddress(&konstd[C])); - cc.movd(tmp0, regD[B]); - cc.movss(tmp1, asmjit::x86::dword_ptr(konstTmp)); - cc.pmaxsd(tmp0, tmp1); - cc.movd(regD[A], tmp0); - } - - void EmitABS() - { - auto srcB = CheckRegD(B, A); - auto tmp = cc.newInt32(); - cc.mov(tmp, regD[B]); - cc.sar(tmp, 31); - cc.mov(regD[A], tmp); - cc.xor_(regD[A], srcB); - cc.sub(regD[A], tmp); - } - - void EmitNEG() - { - auto srcB = CheckRegD(B, A); - cc.xor_(regD[A], regD[A]); - cc.sub(regD[A], srcB); - } - - void EmitNOT() - { - if (A != B) - cc.mov(regD[A], regD[B]); - cc.not_(regD[A]); - } - - void EmitEQ_R() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - cc.cmp(regD[B], regD[C]); - if (check) cc.je(fail); - else cc.jne(fail); - }); - } - - void EmitEQ_K() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - cc.cmp(regD[B], konstd[C]); - if (check) cc.je(fail); - else cc.jne(fail); - }); - } - - void EmitLT_RR() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - cc.cmp(regD[B], regD[C]); - if (check) cc.jl(fail); - else cc.jnl(fail); - }); - } - - void EmitLT_RK() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - cc.cmp(regD[B], konstd[C]); - if (check) cc.jl(fail); - else cc.jnl(fail); - }); - } - - void EmitLT_KR() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(&konstd[B])); - cc.cmp(asmjit::x86::ptr(tmp), regD[C]); - if (check) cc.jl(fail); - else cc.jnl(fail); - }); - } - - void EmitLE_RR() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - cc.cmp(regD[B], regD[C]); - if (check) cc.jle(fail); - else cc.jnle(fail); - }); - } - - void EmitLE_RK() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - cc.cmp(regD[B], konstd[C]); - if (check) cc.jle(fail); - else cc.jnle(fail); - }); - } - - void EmitLE_KR() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(&konstd[B])); - cc.cmp(asmjit::x86::ptr(tmp), regD[C]); - if (check) cc.jle(fail); - else cc.jnle(fail); - }); - } - - void EmitLTU_RR() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - cc.cmp(regD[B], regD[C]); - if (check) cc.jb(fail); - else cc.jnb(fail); - }); - } - - void EmitLTU_RK() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - cc.cmp(regD[B], konstd[C]); - if (check) cc.jb(fail); - else cc.jnb(fail); - }); - } - - void EmitLTU_KR() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(&konstd[B])); - cc.cmp(asmjit::x86::ptr(tmp), regD[C]); - if (check) cc.jb(fail); - else cc.jnb(fail); - }); - } - - void EmitLEU_RR() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - cc.cmp(regD[B], regD[C]); - if (check) cc.jbe(fail); - else cc.jnbe(fail); - }); - } - - void EmitLEU_RK() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - cc.cmp(regD[B], konstd[C]); - if (check) cc.jbe(fail); - else cc.jnbe(fail); - }); - } - - void EmitLEU_KR() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(&konstd[B])); - cc.cmp(asmjit::x86::ptr(tmp), regD[C]); - if (check) cc.jbe(fail); - else cc.jnbe(fail); - }); - } - - // Double-precision floating point math. - - void EmitADDF_RR() - { - auto rc = CheckRegF(C, A); - if (A != B) - cc.movsd(regF[A], regF[B]); - cc.addsd(regF[A], rc); - } - - void EmitADDF_RK() - { - auto tmp = cc.newIntPtr(); - if (A != B) - cc.movsd(regF[A], regF[B]); - cc.mov(tmp, ToMemAddress(&konstf[C])); - cc.addsd(regF[A], asmjit::x86::qword_ptr(tmp)); - } - - void EmitSUBF_RR() - { - auto rc = CheckRegF(C, A); - if (A != B) - cc.movsd(regF[A], regF[B]); - cc.subsd(regF[A], rc); - } - - void EmitSUBF_RK() - { - auto tmp = cc.newIntPtr(); - if (A != B) - cc.movsd(regF[A], regF[B]); - cc.mov(tmp, ToMemAddress(&konstf[C])); - cc.subsd(regF[A], asmjit::x86::qword_ptr(tmp)); - } - - void EmitSUBF_KR() - { - auto rc = CheckRegF(C, A); - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(&konstf[B])); - cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); - cc.subsd(regF[A], rc); - } - - void EmitMULF_RR() - { - auto rc = CheckRegF(C, A); - if (A != B) - cc.movsd(regF[A], regF[B]); - cc.mulsd(regF[A], rc); - } - - void EmitMULF_RK() - { - auto tmp = cc.newIntPtr(); - if (A != B) - cc.movsd(regF[A], regF[B]); - cc.mov(tmp, ToMemAddress(&konstf[C])); - cc.mulsd(regF[A], asmjit::x86::qword_ptr(tmp)); - } - - void EmitDIVF_RR() - { - auto label = cc.newLabel(); - cc.ptest(regF[C], regF[C]); - cc.jne(label); - EmitThrowException(X_DIVISION_BY_ZERO); - cc.bind(label); - auto rc = CheckRegF(C, A); - cc.movsd(regF[A], regF[B]); - cc.divsd(regF[A], rc); - } - - void EmitDIVF_RK() - { - if (konstf[C] == 0.) - { - EmitThrowException(X_DIVISION_BY_ZERO); - } - else - { - auto tmp = cc.newIntPtr(); - cc.movsd(regF[A], regF[B]); - cc.mov(tmp, ToMemAddress(&konstf[C])); - cc.divsd(regF[A], asmjit::x86::qword_ptr(tmp)); - } - } - - void EmitDIVF_KR() - { - auto rc = CheckRegF(C, A); - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(&konstf[B])); - cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); - cc.divsd(regF[A], rc); - } - - void EmitMODF_RR() - { - using namespace asmjit; - typedef double(*FuncPtr)(double, double); - - auto label = cc.newLabel(); - cc.ptest(regF[C], regF[C]); - cc.jne(label); - EmitThrowException(X_DIVISION_BY_ZERO); - cc.bind(label); - - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](double a, double b) -> double - { - return a - floor(a / b) * b; - }))), FuncSignature2()); - call->setRet(0, regF[A]); - call->setArg(0, regF[B]); - call->setArg(1, regF[C]); - } - - void EmitMODF_RK() - { - using namespace asmjit; - typedef double(*FuncPtr)(double, double); - - auto label = cc.newLabel(); - cc.ptest(regF[C], regF[C]); - cc.jne(label); - EmitThrowException(X_DIVISION_BY_ZERO); - cc.bind(label); - - auto tmp = cc.newXmm(); - cc.movsd(tmp, x86::ptr(ToMemAddress(&konstf[C]))); - - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](double a, double b) -> double { - return a - floor(a / b) * b; - }))), FuncSignature2()); - call->setRet(0, regF[A]); - call->setArg(0, regF[B]); - call->setArg(1, tmp); - } - - void EmitMODF_KR() - { - using namespace asmjit; - typedef double(*FuncPtr)(double, double); - - auto label = cc.newLabel(); - cc.ptest(regF[C], regF[C]); - cc.jne(label); - EmitThrowException(X_DIVISION_BY_ZERO); - cc.bind(label); - - auto tmp = cc.newXmm(); - cc.movsd(tmp, x86::ptr(ToMemAddress(&konstf[B]))); - - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](double a, double b) -> double { - return a - floor(a / b) * b; - }))), FuncSignature2()); - call->setRet(0, regF[A]); - call->setArg(0, tmp); - call->setArg(1, regF[C]); - } - - void EmitPOWF_RR() - { - using namespace asmjit; - typedef double(*FuncPtr)(double, double); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(g_pow))), FuncSignature2()); - call->setRet(0, regF[A]); - call->setArg(0, regF[B]); - call->setArg(1, regF[C]); - } - - void EmitPOWF_RK() - { - auto tmp = cc.newIntPtr(); - auto tmp2 = cc.newXmm(); - cc.mov(tmp, ToMemAddress(&konstf[C])); - cc.movsd(tmp2, asmjit::x86::qword_ptr(tmp)); - - using namespace asmjit; - typedef double(*FuncPtr)(double, double); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(g_pow))), FuncSignature2()); - call->setRet(0, regF[A]); - call->setArg(0, regF[B]); - call->setArg(1, tmp2); - } - - void EmitPOWF_KR() - { - auto tmp = cc.newIntPtr(); - auto tmp2 = cc.newXmm(); - cc.mov(tmp, ToMemAddress(&konstf[B])); - cc.movsd(tmp2, asmjit::x86::qword_ptr(tmp)); - - using namespace asmjit; - typedef double(*FuncPtr)(double, double); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(g_pow))), FuncSignature2()); - call->setRet(0, regF[A]); - call->setArg(0, tmp2); - call->setArg(1, regF[C]); - } - - void EmitMINF_RR() - { - auto rc = CheckRegF(C, A); - if (A != B) - cc.movsd(regF[A], regF[B]); - cc.minsd(regF[A], rc); - } - - void EmitMINF_RK() - { - auto rb = CheckRegF(B, A); - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(&konstf[C])); - cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); - cc.minsd(regF[A], rb); - } - - void EmitMAXF_RR() - { - auto rc = CheckRegF(C, A); - if (A != B) - cc.movsd(regF[A], regF[B]); - cc.maxsd(regF[A], rc); - } - - void EmitMAXF_RK() - { - auto rb = CheckRegF(B, A); - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(&konstf[C])); - cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); - cc.maxsd(regF[A], rb); - } - - void EmitATAN2() - { - using namespace asmjit; - typedef double(*FuncPtr)(double, double); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(g_atan2))), FuncSignature2()); - call->setRet(0, regF[A]); - call->setArg(0, regF[B]); - call->setArg(1, regF[C]); - - static const double constant = 180 / M_PI; - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(&constant)); - cc.mulsd(regF[A], asmjit::x86::qword_ptr(tmp)); - } - - void EmitFLOP() - { - if (C == FLOP_NEG) - { - auto mask = cc.newDoubleConst(asmjit::kConstScopeLocal, -0.0); - auto maskXmm = cc.newXmmSd(); - cc.movsd(maskXmm, mask); - if (A != B) - cc.movsd(regF[A], regF[B]); - cc.xorpd(regF[A], maskXmm); - } - else - { - auto v = cc.newXmm(); - cc.movsd(v, regF[B]); - - if (C == FLOP_TAN_DEG) - { - static const double constant = M_PI / 180; - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(&constant)); - cc.mulsd(v, asmjit::x86::qword_ptr(tmp)); - } - - typedef double(*FuncPtr)(double); - FuncPtr func = nullptr; - switch (C) - { - default: I_FatalError("Unknown OP_FLOP subfunction"); - case FLOP_ABS: func = fabs; break; - case FLOP_EXP: func = g_exp; break; - case FLOP_LOG: func = g_log; break; - case FLOP_LOG10: func = g_log10; break; - case FLOP_SQRT: func = g_sqrt; break; - case FLOP_CEIL: func = ceil; break; - case FLOP_FLOOR: func = floor; break; - case FLOP_ACOS: func = g_acos; break; - case FLOP_ASIN: func = g_asin; break; - case FLOP_ATAN: func = g_atan; break; - case FLOP_COS: func = g_cos; break; - case FLOP_SIN: func = g_sin; break; - case FLOP_TAN: func = g_tan; break; - case FLOP_ACOS_DEG: func = g_acos; break; - case FLOP_ASIN_DEG: func = g_asin; break; - case FLOP_ATAN_DEG: func = g_atan; break; - case FLOP_COS_DEG: func = g_cosdeg; break; - case FLOP_SIN_DEG: func = g_sindeg; break; - case FLOP_TAN_DEG: func = g_tan; break; - case FLOP_COSH: func = g_cosh; break; - case FLOP_SINH: func = g_sinh; break; - case FLOP_TANH: func = g_tanh; break; - } - - auto call = cc.call(ToMemAddress(reinterpret_cast(func)), asmjit::FuncSignature1()); - call->setRet(0, regF[A]); - call->setArg(0, v); - - if (C == FLOP_ACOS_DEG || C == FLOP_ASIN_DEG || C == FLOP_ATAN_DEG) - { - static const double constant = 180 / M_PI; - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(&constant)); - cc.mulsd(regF[A], asmjit::x86::qword_ptr(tmp)); - } - } - } - - void EmitEQF_R() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - bool approx = static_cast(A & CMP_APPROX); - if (!approx) - { - cc.ucomisd(regF[B], regF[C]); - if (check) { - cc.jp(success); - cc.je(fail); - } - else { - cc.jp(fail); - cc.jne(fail); - } - } - else - { - auto tmp = cc.newXmmSd(); - - const int64_t absMaskInt = 0x7FFFFFFFFFFFFFFF; - auto absMask = cc.newDoubleConst(asmjit::kConstScopeLocal, reinterpret_cast(absMaskInt)); - auto absMaskXmm = cc.newXmmPd(); - - auto epsilon = cc.newDoubleConst(asmjit::kConstScopeLocal, VM_EPSILON); - auto epsilonXmm = cc.newXmmSd(); - - cc.movsd(tmp, regF[B]); - cc.subsd(tmp, regF[C]); - cc.movsd(absMaskXmm, absMask); - cc.andpd(tmp, absMaskXmm); - cc.movsd(epsilonXmm, epsilon); - cc.ucomisd(epsilonXmm, tmp); - - if (check) cc.ja(fail); - else cc.jna(fail); - } - }); - } - - void EmitEQF_K() - { - using namespace asmjit; - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - bool approx = static_cast(A & CMP_APPROX); - if (!approx) { - auto konstTmp = cc.newIntPtr(); - cc.mov(konstTmp, ToMemAddress(&konstf[C])); - cc.ucomisd(regF[B], x86::qword_ptr(konstTmp)); - if (check) { - cc.jp(success); - cc.je(fail); - } - else { - cc.jp(fail); - cc.jne(fail); - } - } - else { - auto konstTmp = cc.newIntPtr(); - auto subTmp = cc.newXmmSd(); - - const int64_t absMaskInt = 0x7FFFFFFFFFFFFFFF; - auto absMask = cc.newDoubleConst(kConstScopeLocal, reinterpret_cast(absMaskInt)); - auto absMaskXmm = cc.newXmmPd(); - - auto epsilon = cc.newDoubleConst(kConstScopeLocal, VM_EPSILON); - auto epsilonXmm = cc.newXmmSd(); - - cc.mov(konstTmp, ToMemAddress(&konstf[C])); - - cc.movsd(subTmp, regF[B]); - cc.subsd(subTmp, x86::qword_ptr(konstTmp)); - cc.movsd(absMaskXmm, absMask); - cc.andpd(subTmp, absMaskXmm); - cc.movsd(epsilonXmm, epsilon); - cc.ucomisd(epsilonXmm, subTmp); - - if (check) cc.ja(fail); - else cc.jna(fail); - } - }); - } - - void EmitLTF_RR() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for LTF_RR.\n"); - - cc.ucomisd(regF[C], regF[B]); - if (check) cc.ja(fail); - else cc.jna(fail); - }); - } - - void EmitLTF_RK() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for LTF_RK.\n"); - - auto constTmp = cc.newIntPtr(); - auto xmmTmp = cc.newXmmSd(); - cc.mov(constTmp, ToMemAddress(&konstf[C])); - cc.movsd(xmmTmp, asmjit::x86::qword_ptr(constTmp)); - - cc.ucomisd(xmmTmp, regF[B]); - if (check) cc.ja(fail); - else cc.jna(fail); - }); - } - - void EmitLTF_KR() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for LTF_KR.\n"); - - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(&konstf[B])); - - cc.ucomisd(regF[C], asmjit::x86::qword_ptr(tmp)); - if (check) cc.ja(fail); - else cc.jna(fail); - }); - } - - void EmitLEF_RR() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for LEF_RR.\n"); - - cc.ucomisd(regF[C], regF[B]); - if (check) cc.jae(fail); - else cc.jnae(fail); - }); - } - - void EmitLEF_RK() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for LEF_RK.\n"); - - auto constTmp = cc.newIntPtr(); - auto xmmTmp = cc.newXmmSd(); - cc.mov(constTmp, ToMemAddress(&konstf[C])); - cc.movsd(xmmTmp, asmjit::x86::qword_ptr(constTmp)); - - cc.ucomisd(xmmTmp, regF[B]); - if (check) cc.jae(fail); - else cc.jnae(fail); - }); - } - - void EmitLEF_KR() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for LEF_KR.\n"); - - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(&konstf[B])); - - cc.ucomisd(regF[C], asmjit::x86::qword_ptr(tmp)); - if (check) cc.jae(fail); - else cc.jnae(fail); - }); - } - - // Vector math. (2D) - - void EmitNEGV2() - { - auto mask = cc.newDoubleConst(asmjit::kConstScopeLocal, -0.0); - auto maskXmm = cc.newXmmSd(); - cc.movsd(maskXmm, mask); - cc.movsd(regF[A], regF[B]); - cc.xorpd(regF[A], maskXmm); - cc.movsd(regF[A + 1], regF[B + 1]); - cc.xorpd(regF[A + 1], maskXmm); - } - - void EmitADDV2_RR() - { - auto rc0 = CheckRegF(C, A); - auto rc1 = CheckRegF(C + 1, A + 1); - cc.movsd(regF[A], regF[B]); - cc.movsd(regF[A + 1], regF[B + 1]); - cc.addsd(regF[A], rc0); - cc.addsd(regF[A + 1], rc1); - } - - void EmitSUBV2_RR() - { - auto rc0 = CheckRegF(C, A); - auto rc1 = CheckRegF(C + 1, A + 1); - cc.movsd(regF[A], regF[B]); - cc.movsd(regF[A + 1], regF[B + 1]); - cc.subsd(regF[A], rc0); - cc.subsd(regF[A + 1], rc1); - } - - void EmitDOTV2_RR() - { - auto rc0 = CheckRegF(C, A); - auto rc1 = CheckRegF(C + 1, A); - auto tmp = cc.newXmmSd(); - cc.movsd(regF[A], regF[B]); - cc.mulsd(regF[A], rc0); - cc.movsd(tmp, regF[B + 1]); - cc.mulsd(tmp, rc1); - cc.addsd(regF[A], tmp); - } - - void EmitMULVF2_RR() - { - auto rc = CheckRegF(C, A, A + 1); - cc.movsd(regF[A], regF[B]); - cc.movsd(regF[A + 1], regF[B + 1]); - cc.mulsd(regF[A], rc); - cc.mulsd(regF[A + 1], rc); - } - - void EmitMULVF2_RK() - { - auto tmp = cc.newIntPtr(); - cc.movsd(regF[A], regF[B]); - cc.movsd(regF[A + 1], regF[B + 1]); - cc.mov(tmp, ToMemAddress(&konstf[C])); - cc.mulsd(regF[A], asmjit::x86::qword_ptr(tmp)); - cc.mulsd(regF[A + 1], asmjit::x86::qword_ptr(tmp)); - } - - void EmitDIVVF2_RR() - { - auto rc = CheckRegF(C, A, A + 1); - cc.movsd(regF[A], regF[B]); - cc.movsd(regF[A + 1], regF[B + 1]); - cc.divsd(regF[A], rc); - cc.divsd(regF[A + 1], rc); - } - - void EmitDIVVF2_RK() - { - auto tmp = cc.newIntPtr(); - cc.movsd(regF[A], regF[B]); - cc.movsd(regF[A + 1], regF[B + 1]); - cc.mov(tmp, ToMemAddress(&konstf[C])); - cc.divsd(regF[A], asmjit::x86::qword_ptr(tmp)); - cc.divsd(regF[A + 1], asmjit::x86::qword_ptr(tmp)); - } - - void EmitLENV2() - { - auto rb0 = CheckRegF(B, A); - auto rb1 = CheckRegF(B + 1, A); - auto tmp = cc.newXmmSd(); - cc.movsd(regF[A], regF[B]); - cc.mulsd(regF[A], rb0); - cc.movsd(tmp, rb1); - cc.mulsd(tmp, rb1); - cc.addsd(regF[A], tmp); - CallSqrt(regF[A], regF[A]); - } - - void EmitEQV2_R() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for EQV2_R.\n"); - - cc.ucomisd(regF[B], regF[C]); - if (check) { - cc.jp(success); - cc.jne(success); - } - else { - cc.jp(fail); - cc.jne(fail); - } - - cc.ucomisd(regF[B + 1], regF[C + 1]); - if (check) { - cc.jp(success); - cc.je(fail); - } - else { - cc.jp(fail); - cc.jne(fail); - } - }); - } - - void EmitEQV2_K() - { - I_FatalError("EQV2_K is not used."); - } - - // Vector math. (3D) - - void EmitNEGV3() - { - auto mask = cc.newDoubleConst(asmjit::kConstScopeLocal, -0.0); - auto maskXmm = cc.newXmmSd(); - cc.movsd(maskXmm, mask); - cc.movsd(regF[A], regF[B]); - cc.xorpd(regF[A], maskXmm); - cc.movsd(regF[A + 1], regF[B + 1]); - cc.xorpd(regF[A + 1], maskXmm); - cc.movsd(regF[A + 2], regF[B + 2]); - cc.xorpd(regF[A + 2], maskXmm); - } - - void EmitADDV3_RR() - { - auto rc0 = CheckRegF(C, A); - auto rc1 = CheckRegF(C + 1, A + 1); - auto rc2 = CheckRegF(C + 2, A + 2); - cc.movsd(regF[A], regF[B]); - cc.movsd(regF[A + 1], regF[B + 1]); - cc.movsd(regF[A + 2], regF[B + 2]); - cc.addsd(regF[A], rc0); - cc.addsd(regF[A + 1], rc1); - cc.addsd(regF[A + 2], rc2); - } - - void EmitSUBV3_RR() - { - auto rc0 = CheckRegF(C, A); - auto rc1 = CheckRegF(C + 1, A + 1); - auto rc2 = CheckRegF(C + 2, A + 2); - cc.movsd(regF[A], regF[B]); - cc.movsd(regF[A + 1], regF[B + 1]); - cc.movsd(regF[A + 2], regF[B + 2]); - cc.subsd(regF[A], rc0); - cc.subsd(regF[A + 1], rc1); - cc.subsd(regF[A + 2], rc2); - } - - void EmitDOTV3_RR() - { - auto rb1 = CheckRegF(B + 1, A); - auto rb2 = CheckRegF(B + 2, A); - auto rc0 = CheckRegF(C, A); - auto rc1 = CheckRegF(C + 1, A); - auto rc2 = CheckRegF(C + 2, A); - auto tmp = cc.newXmmSd(); - cc.movsd(regF[A], regF[B]); - cc.mulsd(regF[A], rc0); - cc.movsd(tmp, rb1); - cc.mulsd(tmp, rc1); - cc.addsd(regF[A], tmp); - cc.movsd(tmp, rb2); - cc.mulsd(tmp, rc2); - cc.addsd(regF[A], tmp); - } - - void EmitCROSSV_RR() - { - auto tmp = cc.newXmmSd(); - - auto a0 = CheckRegF(B, A); - auto a1 = CheckRegF(B + 1, A + 1); - auto a2 = CheckRegF(B + 2, A + 2); - auto b0 = CheckRegF(C, A); - auto b1 = CheckRegF(C + 1, A + 1); - auto b2 = CheckRegF(C + 2, A + 2); - - // r0 = a1b2 - a2b1 - cc.movsd(regF[A], a1); - cc.mulsd(regF[A], b2); - cc.movsd(tmp, a2); - cc.mulsd(tmp, b1); - cc.subsd(regF[A], tmp); - - // r1 = a2b0 - a0b2 - cc.movsd(regF[A + 1], a2); - cc.mulsd(regF[A + 1], b0); - cc.movsd(tmp, a0); - cc.mulsd(tmp, b2); - cc.subsd(regF[A + 1], tmp); - - // r2 = a0b1 - a1b0 - cc.movsd(regF[A + 2], a0); - cc.mulsd(regF[A + 2], b1); - cc.movsd(tmp, a1); - cc.mulsd(tmp, b0); - cc.subsd(regF[A + 2], tmp); - } - - void EmitMULVF3_RR() - { - auto rc = CheckRegF(C, A, A + 1, A + 2); - cc.movsd(regF[A], regF[B]); - cc.movsd(regF[A + 1], regF[B + 1]); - cc.movsd(regF[A + 2], regF[B + 2]); - cc.mulsd(regF[A], rc); - cc.mulsd(regF[A + 1], rc); - cc.mulsd(regF[A + 2], rc); - } - - void EmitMULVF3_RK() - { - auto tmp = cc.newIntPtr(); - cc.movsd(regF[A], regF[B]); - cc.movsd(regF[A + 1], regF[B + 1]); - cc.movsd(regF[A + 2], regF[B + 2]); - cc.mov(tmp, ToMemAddress(&konstf[C])); - cc.mulsd(regF[A], asmjit::x86::qword_ptr(tmp)); - cc.mulsd(regF[A + 1], asmjit::x86::qword_ptr(tmp)); - cc.mulsd(regF[A + 2], asmjit::x86::qword_ptr(tmp)); - } - - void EmitDIVVF3_RR() - { - auto rc = CheckRegF(C, A, A + 1, A + 2); - cc.movsd(regF[A], regF[B]); - cc.movsd(regF[A + 1], regF[B + 1]); - cc.movsd(regF[A + 2], regF[B + 2]); - cc.divsd(regF[A], rc); - cc.divsd(regF[A + 1], rc); - cc.divsd(regF[A + 2], rc); - } - - void EmitDIVVF3_RK() - { - auto tmp = cc.newIntPtr(); - cc.movsd(regF[A], regF[B]); - cc.movsd(regF[A + 1], regF[B + 1]); - cc.movsd(regF[A + 2], regF[B + 2]); - cc.mov(tmp, ToMemAddress(&konstf[C])); - cc.divsd(regF[A], asmjit::x86::qword_ptr(tmp)); - cc.divsd(regF[A + 1], asmjit::x86::qword_ptr(tmp)); - cc.divsd(regF[A + 2], asmjit::x86::qword_ptr(tmp)); - } - - void EmitLENV3() - { - auto rb1 = CheckRegF(B + 1, A); - auto rb2 = CheckRegF(B + 2, A); - auto tmp = cc.newXmmSd(); - cc.movsd(regF[A], regF[B]); - cc.mulsd(regF[A], regF[B]); - cc.movsd(tmp, rb1); - cc.mulsd(tmp, rb1); - cc.addsd(regF[A], tmp); - cc.movsd(tmp, rb2); - cc.mulsd(tmp, rb2); - cc.addsd(regF[A], tmp); - CallSqrt(regF[A], regF[A]); - } - - void EmitEQV3_R() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for EQV3_R.\n"); - - cc.ucomisd(regF[B], regF[C]); - if (check) { - cc.jp(success); - cc.jne(success); - } - else { - cc.jp(fail); - cc.jne(fail); - } - - cc.ucomisd(regF[B + 1], regF[C + 1]); - if (check) { - cc.jp(success); - cc.jne(success); - } - else { - cc.jp(fail); - cc.jne(fail); - } - - cc.ucomisd(regF[B + 2], regF[C + 2]); - if (check) { - cc.jp(success); - cc.je(fail); - } - else { - cc.jp(fail); - cc.jne(fail); - } - }); - } - - void EmitEQV3_K() - { - I_FatalError("EQV3_K is not used."); - } - - // Pointer math. - - void EmitADDA_RR() - { - auto tmp = cc.newIntPtr(); - auto label = cc.newLabel(); - - cc.mov(tmp, regA[B]); - - // Check if zero, the first operand is zero, if it is, don't add. - cc.cmp(tmp, 0); - cc.je(label); - - auto tmpptr = cc.newIntPtr(); - cc.mov(tmpptr, regD[C]); - cc.add(tmp, tmpptr); - - cc.bind(label); - cc.mov(regA[A], tmp); - } - - void EmitADDA_RK() - { - auto tmp = cc.newIntPtr(); - auto label = cc.newLabel(); - - cc.mov(tmp, regA[B]); - - // Check if zero, the first operand is zero, if it is, don't add. - cc.cmp(tmp, 0); - cc.je(label); - - cc.add(tmp, konstd[C]); - - cc.bind(label); - cc.mov(regA[A], tmp); - } - - void EmitSUBA() - { - auto tmp = cc.newIntPtr(); - cc.mov(tmp, regA[B]); - cc.sub(tmp, regD[C]); - cc.mov(regA[A], tmp); - } - - void EmitEQA_R() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - cc.cmp(regA[B], regA[C]); - if (check) cc.je(fail); - else cc.jne(fail); - }); - } - - void EmitEQA_K() - { - EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { - auto tmp = cc.newIntPtr(); - cc.mov(tmp, ToMemAddress(konsta[C].v)); - cc.cmp(regA[B], tmp); - if (check) cc.je(fail); - else cc.jne(fail); - }); - } - - void Setup() - { - using namespace asmjit; - - FString funcname; - funcname.Format("Function: %s", sfunc->PrintableName.GetChars()); - cc.comment(funcname.GetChars(), funcname.Len()); - - stack = cc.newIntPtr("stack"); // VMFrameStack *stack - vmregs = cc.newIntPtr("vmregs"); // VMRegisters *vmregs - ret = cc.newIntPtr("ret"); // VMReturn *ret - numret = cc.newInt32("numret"); // int numret - exceptInfo = cc.newIntPtr("exceptinfo"); // JitExceptionInfo *exceptInfo - - cc.addFunc(FuncSignature5()); - cc.setArg(0, stack); - cc.setArg(1, vmregs); - cc.setArg(2, ret); - cc.setArg(3, numret); - cc.setArg(4, exceptInfo); - - auto stack = cc.newStack(sizeof(VMReturn) * MAX_RETURNS, alignof(VMReturn)); - callReturns = cc.newIntPtr("callReturns"); - cc.lea(callReturns, stack); - - konstd = sfunc->KonstD; - konstf = sfunc->KonstF; - konsts = sfunc->KonstS; - konsta = sfunc->KonstA; - - regD.Resize(sfunc->NumRegD); - regF.Resize(sfunc->NumRegF); - regA.Resize(sfunc->NumRegA); - regS.Resize(sfunc->NumRegS); - - frameD = cc.newIntPtr(); - frameF = cc.newIntPtr(); - frameS = cc.newIntPtr(); - frameA = cc.newIntPtr(); - params = cc.newIntPtr(); - cc.mov(frameD, x86::ptr(vmregs, offsetof(VMRegisters, d))); - cc.mov(frameF, x86::ptr(vmregs, offsetof(VMRegisters, f))); - cc.mov(frameS, x86::ptr(vmregs, offsetof(VMRegisters, s))); - cc.mov(frameA, x86::ptr(vmregs, offsetof(VMRegisters, a))); - cc.mov(params, x86::ptr(vmregs, offsetof(VMRegisters, param))); - - if (sfunc->NumRegD > 0) - { - for (int i = 0; i < sfunc->NumRegD; i++) - { - FString regname; - regname.Format("regD%d", i); - regD[i] = cc.newInt32(regname.GetChars()); - cc.mov(regD[i], x86::dword_ptr(frameD, i * sizeof(int32_t))); - } - } - if (sfunc->NumRegF > 0) - { - for (int i = 0; i < sfunc->NumRegF; i++) - { - FString regname; - regname.Format("regF%d", i); - regF[i] = cc.newXmmSd(regname.GetChars()); - cc.movsd(regF[i], x86::qword_ptr(frameF, i * sizeof(double))); - } - } - if (sfunc->NumRegS > 0) - { - for (int i = 0; i < sfunc->NumRegS; i++) - { - FString regname; - regname.Format("regS%d", i); - regS[i] = cc.newIntPtr(regname.GetChars()); - cc.mov(regS[i], frameS); - if (i * sizeof(FString) != 0) cc.add(regS[i], i * sizeof(FString)); - } - } - if (sfunc->NumRegA > 0) - { - for (int i = 0; i < sfunc->NumRegA; i++) - { - FString regname; - regname.Format("regA%d", i); - regA[i] = cc.newIntPtr(regname.GetChars()); - cc.mov(regA[i], x86::ptr(frameA, i * sizeof(void*))); - } - } - - int size = sfunc->CodeSize; - labels.Resize(size); - for (int i = 0; i < size; i++) labels[i] = cc.newLabel(); - } - - template - void EmitComparisonOpcode(Func jmpFunc) - { - using namespace asmjit; - - int i = (int)(ptrdiff_t)(pc - sfunc->Code); - - auto successLabel = cc.newLabel(); - - auto failLabel = labels[i + 2 + JMPOFS(pc + 1)]; - - jmpFunc(static_cast(A & CMP_CHECK), failLabel, successLabel); - - cc.bind(successLabel); - pc++; // This instruction uses two instruction slots - skip the next one - } - - static int64_t ToMemAddress(const void *d) - { - return (int64_t)(ptrdiff_t)d; - } - - void CallSqrt(const asmjit::X86Xmm &a, const asmjit::X86Xmm &b) - { - using namespace asmjit; - typedef double(*FuncPtr)(double); - auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(g_sqrt))), FuncSignature1()); - call->setRet(0, a); - call->setArg(0, b); - } - - void EmitNullPointerThrow(int index, EVMAbortException reason) - { - auto label = cc.newLabel(); - cc.test(regA[index], regA[index]); - cc.jne(label); - EmitThrowException(reason); - cc.bind(label); - } - - void EmitThrowException(EVMAbortException reason) - { - using namespace asmjit; - - // Update JitExceptionInfo struct - cc.mov(x86::dword_ptr(exceptInfo, 0 * 4), (int32_t)reason); -#ifdef ASMJIT_ARCH_X64 - cc.mov(x86::qword_ptr(exceptInfo, 4 * 4), ToMemAddress(pc)); -#else - cc.mov(x86::dword_ptr(exceptInfo, 4 * 4), ToMemAddress(pc)); -#endif - - // Return from function - X86Gp vReg = cc.newInt32(); - cc.mov(vReg, 0); - cc.ret(vReg); - } - - void EmitThrowException(EVMAbortException reason, asmjit::X86Gp arg1) - { - using namespace asmjit; - - // Update JitExceptionInfo struct - cc.mov(x86::dword_ptr(exceptInfo, 0 * 4), (int32_t)reason); - cc.mov(x86::dword_ptr(exceptInfo, 1 * 4), arg1); -#ifdef ASMJIT_ARCH_X64 - cc.mov(x86::qword_ptr(exceptInfo, 4 * 4), ToMemAddress(pc)); -#else - cc.mov(x86::dword_ptr(exceptInfo, 4 * 4), ToMemAddress(pc)); -#endif - - // Return from function - X86Gp vReg = cc.newInt32(); - cc.mov(vReg, 0); - cc.ret(vReg); - } - - asmjit::X86Gp CheckRegD(int r0, int r1) - { - if (r0 != r1) - { - return regD[r0]; - } - else - { - auto copy = cc.newInt32(); - cc.mov(copy, regD[r0]); - return copy; - } - } - - asmjit::X86Xmm CheckRegF(int r0, int r1) - { - if (r0 != r1) - { - return regF[r0]; - } - else - { - auto copy = cc.newXmm(); - cc.movsd(copy, regF[r0]); - return copy; - } - } - - asmjit::X86Xmm CheckRegF(int r0, int r1, int r2) - { - if (r0 != r1 && r0 != r2) - { - return regF[r0]; - } - else - { - auto copy = cc.newXmm(); - cc.movsd(copy, regF[r0]); - return copy; - } - } - - asmjit::X86Xmm CheckRegF(int r0, int r1, int r2, int r3) - { - if (r0 != r1 && r0 != r2 && r0 != r3) - { - return regF[r0]; - } - else - { - auto copy = cc.newXmm(); - cc.movsd(copy, regF[r0]); - return copy; - } - } - - asmjit::X86Gp CheckRegA(int r0, int r1) - { - if (r0 != r1) - { - return regA[r0]; - } - else - { - auto copy = cc.newIntPtr(); - cc.mov(copy, regA[r0]); - return copy; - } - } - - asmjit::X86Compiler cc; - VMScriptFunction *sfunc; - - asmjit::X86Gp stack; - asmjit::X86Gp vmregs; - asmjit::X86Gp ret; - asmjit::X86Gp numret; - asmjit::X86Gp exceptInfo; - - asmjit::X86Gp frameD; - asmjit::X86Gp frameF; - asmjit::X86Gp frameS; - asmjit::X86Gp frameA; - asmjit::X86Gp params; - int NumParam = 0; // Actually part of vmframe (f->NumParam), but nobody seems to read that? - TArray ParamOpcodes; - asmjit::X86Gp callReturns; - - const int *konstd; - const double *konstf; - const FString *konsts; - const FVoidObj *konsta; - - TArray regD; - TArray regF; - TArray regA; - TArray regS; - - TArray labels; - - const VMOP *pc; - VM_UBYTE op; -}; - -class AsmJitException : public std::exception -{ -public: - AsmJitException(asmjit::Error error, const char *message) noexcept : error(error), message(message) - { - } - - const char* what() const noexcept override - { - return message.GetChars(); - } - - asmjit::Error error; - FString message; -}; - -class ThrowingErrorHandler : public asmjit::ErrorHandler -{ -public: - bool handleError(asmjit::Error err, const char *message, asmjit::CodeEmitter *origin) override - { - throw AsmJitException(err, message); - } -}; +#include "jitintern.h" static asmjit::JitRuntime *jit; static int jitRefCount = 0; @@ -3608,3 +89,344 @@ JitFuncPtr JitCompile(VMScriptFunction *sfunc) return nullptr; } } + +///////////////////////////////////////////////////////////////////////////// + +void JitCompiler::Codegen() +{ + using namespace asmjit; + + Setup(); + + pc = sfunc->Code; + auto end = pc + sfunc->CodeSize; + while (pc != end) + { + int i = (int)(ptrdiff_t)(pc - sfunc->Code); + op = pc->op; + + cc.bind(labels[i]); + + FString lineinfo; + lineinfo.Format("; %s(line %d): %02x%02x%02x%02x %s", sfunc->PrintableName.GetChars(), sfunc->PCToLine(pc), pc->op, pc->a, pc->b, pc->c, OpNames[op]); + cc.comment(lineinfo.GetChars(), lineinfo.Len()); + + EmitOpcode(); + + pc++; + } + + cc.endFunc(); + cc.finalize(); +} + +void JitCompiler::EmitOpcode() +{ + switch (op) + { + #define xx(op, name, mode, alt, kreg, ktype) case OP_##op: Emit##op(); break; + #include "vmops.h" + #undef xx + + default: + I_FatalError("JIT error: Unknown VM opcode %d\n", op); + break; + } +} + +bool JitCompiler::CanJit(VMScriptFunction *sfunc) +{ + int size = sfunc->CodeSize; + for (int i = 0; i < size; i++) + { + // Functions not implemented at all yet: + + switch (sfunc->Code[i].op) + { + default: + break; + case OP_LKS_R: + case OP_LFP: + case OP_LCS_R: + case OP_SS_R: + case OP_MOVES: + case OP_IJMP: + case OP_CALL: // this one is implemented but crashes currently + case OP_TAIL: + case OP_TAIL_K: + case OP_CONCAT: + case OP_LENS: + case OP_CMPS: + return false; + } + + // Partially implemented functions: + + auto pc = sfunc->Code + i; + if (sfunc->Code[i].op == OP_RET) + { + if (B != REGT_NIL) + { + int regtype = B; + if ((regtype & REGT_TYPE) == REGT_STRING) + return false; + } + } + else if (sfunc->Code[i].op == OP_CAST) + { + switch (C) + { + case CAST_I2F: + case CAST_U2F: + case CAST_F2I: + case CAST_F2U: + break; + default: + return false; + } + } + else if (sfunc->Code[i].op == OP_CASTB) + { + if (C == CASTB_S) + return false; + } + else if (sfunc->Code[i].op == OP_PARAM) + { + if (!!(B & REGT_MULTIREG3) || !!(B & REGT_MULTIREG2)) + return false; + + switch (B) + { + case REGT_STRING: + case REGT_STRING | REGT_ADDROF: + case REGT_STRING | REGT_KONST: + return false; + default: + break; + } + } + else if (sfunc->Code[i].op == OP_RESULT) + { + if (!!(B & REGT_MULTIREG3) || !!(B & REGT_MULTIREG2)) + return false; + + if ((B & REGT_TYPE) == REGT_STRING) + return false; + } + } + return true; +} + +void JitCompiler::Setup() +{ + using namespace asmjit; + + FString funcname; + funcname.Format("Function: %s", sfunc->PrintableName.GetChars()); + cc.comment(funcname.GetChars(), funcname.Len()); + + stack = cc.newIntPtr("stack"); // VMFrameStack *stack + vmregs = cc.newIntPtr("vmregs"); // VMRegisters *vmregs + ret = cc.newIntPtr("ret"); // VMReturn *ret + numret = cc.newInt32("numret"); // int numret + exceptInfo = cc.newIntPtr("exceptinfo"); // JitExceptionInfo *exceptInfo + + cc.addFunc(FuncSignature5()); + cc.setArg(0, stack); + cc.setArg(1, vmregs); + cc.setArg(2, ret); + cc.setArg(3, numret); + cc.setArg(4, exceptInfo); + + auto stack = cc.newStack(sizeof(VMReturn) * MAX_RETURNS, alignof(VMReturn)); + callReturns = cc.newIntPtr("callReturns"); + cc.lea(callReturns, stack); + + konstd = sfunc->KonstD; + konstf = sfunc->KonstF; + konsts = sfunc->KonstS; + konsta = sfunc->KonstA; + + regD.Resize(sfunc->NumRegD); + regF.Resize(sfunc->NumRegF); + regA.Resize(sfunc->NumRegA); + regS.Resize(sfunc->NumRegS); + + frameD = cc.newIntPtr(); + frameF = cc.newIntPtr(); + frameS = cc.newIntPtr(); + frameA = cc.newIntPtr(); + params = cc.newIntPtr(); + cc.mov(frameD, x86::ptr(vmregs, offsetof(VMRegisters, d))); + cc.mov(frameF, x86::ptr(vmregs, offsetof(VMRegisters, f))); + cc.mov(frameS, x86::ptr(vmregs, offsetof(VMRegisters, s))); + cc.mov(frameA, x86::ptr(vmregs, offsetof(VMRegisters, a))); + cc.mov(params, x86::ptr(vmregs, offsetof(VMRegisters, param))); + + if (sfunc->NumRegD > 0) + { + for (int i = 0; i < sfunc->NumRegD; i++) + { + FString regname; + regname.Format("regD%d", i); + regD[i] = cc.newInt32(regname.GetChars()); + cc.mov(regD[i], x86::dword_ptr(frameD, i * sizeof(int32_t))); + } + } + if (sfunc->NumRegF > 0) + { + for (int i = 0; i < sfunc->NumRegF; i++) + { + FString regname; + regname.Format("regF%d", i); + regF[i] = cc.newXmmSd(regname.GetChars()); + cc.movsd(regF[i], x86::qword_ptr(frameF, i * sizeof(double))); + } + } + if (sfunc->NumRegS > 0) + { + for (int i = 0; i < sfunc->NumRegS; i++) + { + FString regname; + regname.Format("regS%d", i); + regS[i] = cc.newIntPtr(regname.GetChars()); + cc.mov(regS[i], frameS); + if (i * sizeof(FString) != 0) cc.add(regS[i], i * sizeof(FString)); + } + } + if (sfunc->NumRegA > 0) + { + for (int i = 0; i < sfunc->NumRegA; i++) + { + FString regname; + regname.Format("regA%d", i); + regA[i] = cc.newIntPtr(regname.GetChars()); + cc.mov(regA[i], x86::ptr(frameA, i * sizeof(void*))); + } + } + + int size = sfunc->CodeSize; + labels.Resize(size); + for (int i = 0; i < size; i++) labels[i] = cc.newLabel(); +} + +void JitCompiler::EmitNullPointerThrow(int index, EVMAbortException reason) +{ + auto label = cc.newLabel(); + cc.test(regA[index], regA[index]); + cc.jne(label); + EmitThrowException(reason); + cc.bind(label); +} + +void JitCompiler::EmitThrowException(EVMAbortException reason) +{ + using namespace asmjit; + + // Update JitExceptionInfo struct + cc.mov(x86::dword_ptr(exceptInfo, 0 * 4), (int32_t)reason); +#ifdef ASMJIT_ARCH_X64 + cc.mov(x86::qword_ptr(exceptInfo, 4 * 4), ToMemAddress(pc)); +#else + cc.mov(x86::dword_ptr(exceptInfo, 4 * 4), ToMemAddress(pc)); +#endif + + // Return from function + X86Gp vReg = cc.newInt32(); + cc.mov(vReg, 0); + cc.ret(vReg); +} + +void JitCompiler::EmitThrowException(EVMAbortException reason, asmjit::X86Gp arg1) +{ + using namespace asmjit; + + // Update JitExceptionInfo struct + cc.mov(x86::dword_ptr(exceptInfo, 0 * 4), (int32_t)reason); + cc.mov(x86::dword_ptr(exceptInfo, 1 * 4), arg1); +#ifdef ASMJIT_ARCH_X64 + cc.mov(x86::qword_ptr(exceptInfo, 4 * 4), ToMemAddress(pc)); +#else + cc.mov(x86::dword_ptr(exceptInfo, 4 * 4), ToMemAddress(pc)); +#endif + + // Return from function + X86Gp vReg = cc.newInt32(); + cc.mov(vReg, 0); + cc.ret(vReg); +} + +asmjit::X86Gp JitCompiler::CheckRegD(int r0, int r1) +{ + if (r0 != r1) + { + return regD[r0]; + } + else + { + auto copy = cc.newInt32(); + cc.mov(copy, regD[r0]); + return copy; + } +} + +asmjit::X86Xmm JitCompiler::CheckRegF(int r0, int r1) +{ + if (r0 != r1) + { + return regF[r0]; + } + else + { + auto copy = cc.newXmm(); + cc.movsd(copy, regF[r0]); + return copy; + } +} + +asmjit::X86Xmm JitCompiler::CheckRegF(int r0, int r1, int r2) +{ + if (r0 != r1 && r0 != r2) + { + return regF[r0]; + } + else + { + auto copy = cc.newXmm(); + cc.movsd(copy, regF[r0]); + return copy; + } +} + +asmjit::X86Xmm JitCompiler::CheckRegF(int r0, int r1, int r2, int r3) +{ + if (r0 != r1 && r0 != r2 && r0 != r3) + { + return regF[r0]; + } + else + { + auto copy = cc.newXmm(); + cc.movsd(copy, regF[r0]); + return copy; + } +} + +asmjit::X86Gp JitCompiler::CheckRegA(int r0, int r1) +{ + if (r0 != r1) + { + return regA[r0]; + } + else + { + auto copy = cc.newIntPtr(); + cc.mov(copy, regA[r0]); + return copy; + } +} + +void JitCompiler::EmitNOP() +{ + cc.nop(); +} diff --git a/src/scripting/vm/jit_call.cpp b/src/scripting/vm/jit_call.cpp new file mode 100644 index 000000000..79409ccd4 --- /dev/null +++ b/src/scripting/vm/jit_call.cpp @@ -0,0 +1,348 @@ + +#include "jitintern.h" + +void JitCompiler::EmitPARAM() +{ + using namespace asmjit; + + int index = NumParam++; + ParamOpcodes.Push(pc); + + X86Gp stackPtr, tmp; + X86Xmm tmp2; + + switch (B) + { + case REGT_NIL: + cc.mov(x86::ptr(params, index * sizeof(VMValue) + offsetof(VMValue, a)), (int64_t)0); + cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_NIL); + break; + case REGT_INT: + cc.mov(x86::dword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, i)), regD[C]); + cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_INT); + break; + case REGT_INT | REGT_ADDROF: + stackPtr = cc.newIntPtr(); + cc.mov(stackPtr, frameD); + cc.add(stackPtr, C * sizeof(int32_t)); + cc.mov(x86::ptr(params, index * sizeof(VMValue) + offsetof(VMValue, a)), stackPtr); + cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_POINTER); + break; + case REGT_INT | REGT_KONST: + cc.mov(x86::dword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, i)), konstd[C]); + cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_INT); + break; + //case REGT_STRING: + //case REGT_STRING | REGT_ADDROF: + //case REGT_STRING | REGT_KONST: + case REGT_POINTER: + cc.mov(x86::ptr(params, index * sizeof(VMValue) + offsetof(VMValue, a)), regA[C]); + cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_POINTER); + break; + case REGT_POINTER | REGT_ADDROF: + stackPtr = cc.newIntPtr(); + cc.mov(stackPtr, frameA); + cc.add(stackPtr, C * sizeof(void*)); + cc.mov(x86::ptr(params, index * sizeof(VMValue) + offsetof(VMValue, a)), stackPtr); + cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_POINTER); + break; + case REGT_POINTER | REGT_KONST: + tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(konsta[C].v)); + cc.mov(x86::ptr(params, index * sizeof(VMValue) + offsetof(VMValue, a)), tmp); + cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_POINTER); + break; + case REGT_FLOAT: + cc.movsd(x86::qword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, f)), regF[C]); + cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_FLOAT); + break; + case REGT_FLOAT | REGT_MULTIREG2: + cc.movsd(x86::qword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, f)), regF[C]); + cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_FLOAT); + index = NumParam++; + ParamOpcodes.Push(pc); + cc.movsd(x86::qword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, f)), regF[C + 1]); + cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_FLOAT); + break; + case REGT_FLOAT | REGT_MULTIREG3: + cc.movsd(x86::qword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, f)), regF[C]); + cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_FLOAT); + index = NumParam++; + ParamOpcodes.Push(pc); + cc.movsd(x86::qword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, f)), regF[C + 1]); + cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_FLOAT); + index = NumParam++; + ParamOpcodes.Push(pc); + cc.movsd(x86::qword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, f)), regF[C + 2]); + cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_FLOAT); + break; + case REGT_FLOAT | REGT_ADDROF: + stackPtr = cc.newIntPtr(); + cc.mov(stackPtr, frameF); + cc.add(stackPtr, C * sizeof(double)); + cc.mov(x86::ptr(params, index * sizeof(VMValue) + offsetof(VMValue, a)), stackPtr); + cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_POINTER); + break; + case REGT_FLOAT | REGT_KONST: + tmp = cc.newIntPtr(); + tmp2 = cc.newXmmSd(); + cc.mov(tmp, ToMemAddress(konstf + C)); + cc.movsd(tmp2, asmjit::x86::qword_ptr(tmp)); + cc.movsd(x86::qword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, f)), tmp2); + cc.mov(x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_FLOAT); + break; + default: + I_FatalError("Unknown REGT value passed to EmitPARAM\n"); + break; + } + +} + +void JitCompiler::EmitPARAMI() +{ + int index = NumParam++; + ParamOpcodes.Push(pc); + cc.mov(asmjit::x86::dword_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, i)), (int)ABCs); + cc.mov(asmjit::x86::byte_ptr(params, index * sizeof(VMValue) + offsetof(VMValue, Type)), (int)REGT_INT); +} + +void JitCompiler::EmitCALL() +{ + EmitDoCall(regA[A]); +} + +void JitCompiler::EmitCALL_K() +{ + auto ptr = cc.newIntPtr(); + cc.mov(ptr, ToMemAddress(konsta[A].o)); + EmitDoCall(ptr); +} + +void JitCompiler::EmitTAIL() +{ + I_FatalError("EmitTAIL not implemented\n"); +} + +void JitCompiler::EmitTAIL_K() +{ + I_FatalError("EmitTAIL_K not implemented\n"); +} + +void JitCompiler::EmitDoCall(asmjit::X86Gp ptr) +{ + using namespace asmjit; + + if (NumParam < B) + I_FatalError("OP_CALL parameter count does not match the number of preceding OP_PARAM instructions"); + + StoreInOuts(B); + FillReturns(pc + 1, C); + + X86Gp paramsptr; + if (B != NumParam) + { + paramsptr = cc.newIntPtr(); + cc.mov(paramsptr, params); + cc.add(paramsptr, (NumParam - B) * sizeof(VMValue)); + } + else + { + paramsptr = params; + } + + auto result = cc.newInt32(); + auto call = cc.call(ToMemAddress(&JitCompiler::DoCall), FuncSignature7()); + call->setRet(0, result); + call->setArg(0, stack); + call->setArg(1, ptr); + call->setArg(2, asmjit::Imm(B)); + call->setArg(3, asmjit::Imm(C)); + call->setArg(4, paramsptr); + call->setArg(5, callReturns); + call->setArg(6, exceptInfo); + + auto noexception = cc.newLabel(); + auto exceptResult = cc.newInt32(); + cc.mov(exceptResult, x86::dword_ptr(exceptInfo, 0 * 4)); + cc.cmp(exceptResult, (int)-1); + cc.je(noexception); + X86Gp vReg = cc.newInt32(); + cc.mov(vReg, 0); + cc.ret(vReg); + cc.bind(noexception); + + LoadReturns(pc - B, B, true); + LoadReturns(pc + 1, C, false); + + NumParam -= B; + ParamOpcodes.Resize(ParamOpcodes.Size() - B); +} + +void JitCompiler::StoreInOuts(int b) +{ + using namespace asmjit; + + for (unsigned int i = ParamOpcodes.Size() - b; i < ParamOpcodes.Size(); i++) + { + asmjit::X86Gp stackPtr; + switch (ParamOpcodes[i]->b) + { + case REGT_INT | REGT_ADDROF: + stackPtr = cc.newIntPtr(); + cc.mov(stackPtr, frameD); + cc.add(stackPtr, C * sizeof(int32_t)); + cc.mov(x86::dword_ptr(stackPtr), regD[C]); + break; + //case REGT_STRING | REGT_ADDROF: + // break; + case REGT_POINTER | REGT_ADDROF: + stackPtr = cc.newIntPtr(); + cc.mov(stackPtr, frameA); + cc.add(stackPtr, C * sizeof(void*)); + cc.mov(x86::ptr(stackPtr), regA[C]); + break; + case REGT_FLOAT | REGT_ADDROF: + stackPtr = cc.newIntPtr(); + cc.mov(stackPtr, frameF); + cc.add(stackPtr, C * sizeof(double)); + cc.movsd(x86::qword_ptr(stackPtr), regF[C]); + break; + default: + break; + } + } +} + +void JitCompiler::LoadReturns(const VMOP *retval, int numret, bool inout) +{ + for (int i = 0; i < numret; ++i) + { + if (!inout && retval[i].op != OP_RESULT) + I_FatalError("Expected OP_RESULT to follow OP_CALL\n"); + else if (inout && retval[i].op != OP_PARAMI) + continue; + else if (inout && retval[i].op != OP_PARAM) + I_FatalError("Expected OP_PARAM to precede OP_CALL\n"); + + int type = retval[i].b; + int regnum = retval[i].c; + + if (inout && !(type & REGT_ADDROF)) + continue; + + switch (type & REGT_TYPE) + { + case REGT_INT: + cc.mov(regD[regnum], asmjit::x86::dword_ptr(frameD, regnum * sizeof(int32_t))); + break; + case REGT_FLOAT: + cc.movsd(regF[regnum], asmjit::x86::qword_ptr(frameF, regnum * sizeof(double))); + break; + /*case REGT_STRING: + break;*/ + case REGT_POINTER: + cc.mov(regA[regnum], asmjit::x86::ptr(frameA, regnum * sizeof(void*))); + break; + default: + I_FatalError("Unknown OP_RESULT type encountered in LoadReturns\n"); + break; + } + } +} + +void JitCompiler::FillReturns(const VMOP *retval, int numret) +{ + using namespace asmjit; + + for (int i = 0; i < numret; ++i) + { + if (retval[i].op != OP_RESULT) + { + I_FatalError("Expected OP_RESULT to follow OP_CALL\n"); + } + + int type = retval[i].b; + int regnum = retval[i].c; + + if (type & REGT_KONST) + { + I_FatalError("OP_RESULT with REGT_KONST is not allowed\n"); + } + + auto regPtr = cc.newIntPtr(); + + switch (type & REGT_TYPE) + { + case REGT_INT: + cc.mov(regPtr, frameD); + cc.add(regPtr, regnum * sizeof(int32_t)); + break; + case REGT_FLOAT: + cc.mov(regPtr, frameF); + cc.add(regPtr, regnum * sizeof(double)); + break; + /*case REGT_STRING: + cc.mov(regPtr, frameS); + cc.add(regPtr, regnum * sizeof(FString)); + break;*/ + case REGT_POINTER: + cc.mov(regPtr, frameA); + cc.add(regPtr, regnum * sizeof(void*)); + break; + default: + I_FatalError("Unknown OP_RESULT type encountered in FillReturns\n"); + break; + } + + cc.mov(x86::ptr(callReturns, i * sizeof(VMReturn) + offsetof(VMReturn, Location)), regPtr); + cc.mov(x86::byte_ptr(callReturns, i * sizeof(VMReturn) + offsetof(VMReturn, RegType)), type); + } +} + +int JitCompiler::DoCall(VMFrameStack *stack, VMFunction *call, int b, int c, VMValue *param, VMReturn *returns, JitExceptionInfo *exceptinfo) +{ + try + { + int numret; + if (call->VarFlags & VARF_Native) + { + try + { + VMCycles[0].Unclock(); + numret = static_cast(call)->NativeCall(param, call->DefaultArgs, b, returns, c); + VMCycles[0].Clock(); + } + catch (CVMAbortException &err) + { + err.MaybePrintMessage(); + err.stacktrace.AppendFormat("Called from %s\n", call->PrintableName.GetChars()); + throw; + } + } + else + { + VMCalls[0]++; + VMScriptFunction *script = static_cast(call); + VMFrame *newf = stack->AllocFrame(script); + VMFillParams(param, newf, b); + try + { + numret = VMExec(stack, script->Code, returns, c); + } + catch (...) + { + stack->PopFrame(); + throw; + } + stack->PopFrame(); + } + + return numret; + } + catch (...) + { + // To do: store full exception in exceptinfo + exceptinfo->reason = X_OTHER; + return 0; + } +} diff --git a/src/scripting/vm/jit_flow.cpp b/src/scripting/vm/jit_flow.cpp new file mode 100644 index 000000000..b99940988 --- /dev/null +++ b/src/scripting/vm/jit_flow.cpp @@ -0,0 +1,368 @@ + +#include "jitintern.h" + +void JitCompiler::EmitTEST() +{ + int i = (int)(ptrdiff_t)(pc - sfunc->Code); + cc.cmp(regD[A], BC); + cc.jne(labels[i + 2]); +} + +void JitCompiler::EmitTESTN() +{ + int bc = BC; + int i = (int)(ptrdiff_t)(pc - sfunc->Code); + cc.cmp(regD[A], -bc); + cc.jne(labels[i + 2]); +} + +void JitCompiler::EmitJMP() +{ + auto dest = pc + JMPOFS(pc) + 1; + int i = (int)(ptrdiff_t)(dest - sfunc->Code); + cc.jmp(labels[i]); +} + +void JitCompiler::EmitIJMP() +{ + I_FatalError("EmitIJMP not implemented\n"); +} + +void JitCompiler::EmitVTBL() +{ + auto notnull = cc.newLabel(); + cc.test(regA[B], regA[B]); + cc.jnz(notnull); + EmitThrowException(X_READ_NIL); + cc.bind(notnull); + + auto result = cc.newInt32(); + typedef VMFunction*(*FuncPtr)(DObject*, int); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](DObject *o, int c) -> VMFunction* { + auto p = o->GetClass(); + assert(c < (int)p->Virtuals.Size()); + return p->Virtuals[c]; + }))), asmjit::FuncSignature2()); + call->setRet(0, result); + call->setArg(0, regA[A]); + call->setArg(1, asmjit::Imm(C)); +} + +void JitCompiler::EmitSCOPE() +{ + auto notnull = cc.newLabel(); + cc.test(regA[A], regA[A]); + cc.jnz(notnull); + EmitThrowException(X_READ_NIL); + cc.bind(notnull); + + auto f = cc.newIntPtr(); + cc.mov(f, ToMemAddress(konsta[C].v)); + + auto result = cc.newInt32(); + typedef int(*FuncPtr)(DObject*, VMFunction*, int); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](DObject *o, VMFunction *f, int b) -> int { + try + { + FScopeBarrier::ValidateCall(o->GetClass(), f, b - 1); + return 1; + } + catch (const CVMAbortException &) + { + // To do: pass along the exception info + return 0; + } + }))), asmjit::FuncSignature3()); + call->setRet(0, result); + call->setArg(0, regA[A]); + call->setArg(1, f); + call->setArg(2, asmjit::Imm(B)); + + auto notzero = cc.newLabel(); + cc.test(result, result); + cc.jnz(notzero); + EmitThrowException(X_OTHER); + cc.bind(notzero); +} + +void JitCompiler::EmitRESULT() +{ + // This instruction is just a placeholder to indicate where a return + // value should be stored. It does nothing on its own and should not + // be executed. +} + +void JitCompiler::EmitRET() +{ + using namespace asmjit; + if (B == REGT_NIL) + { + X86Gp vReg = cc.newInt32(); + cc.mov(vReg, 0); + cc.ret(vReg); + } + else + { + int a = A; + int retnum = a & ~RET_FINAL; + + X86Gp reg_retnum = cc.newInt32(); + X86Gp location = cc.newIntPtr(); + Label L_endif = cc.newLabel(); + + cc.mov(reg_retnum, retnum); + cc.test(reg_retnum, numret); + cc.jg(L_endif); + + cc.mov(location, x86::ptr(ret, retnum * sizeof(VMReturn))); + + int regtype = B; + int regnum = C; + switch (regtype & REGT_TYPE) + { + case REGT_INT: + if (regtype & REGT_KONST) + cc.mov(x86::dword_ptr(location), konstd[regnum]); + else + cc.mov(x86::dword_ptr(location), regD[regnum]); + break; + case REGT_FLOAT: + if (regtype & REGT_KONST) + { + auto tmp = cc.newInt64(); + if (regtype & REGT_MULTIREG3) + { + cc.mov(tmp, (((int64_t *)konstf)[regnum])); + cc.mov(x86::qword_ptr(location), tmp); + + cc.mov(tmp, (((int64_t *)konstf)[regnum + 1])); + cc.mov(x86::qword_ptr(location, 8), tmp); + + cc.mov(tmp, (((int64_t *)konstf)[regnum + 2])); + cc.mov(x86::qword_ptr(location, 16), tmp); + } + else if (regtype & REGT_MULTIREG2) + { + cc.mov(tmp, (((int64_t *)konstf)[regnum])); + cc.mov(x86::qword_ptr(location), tmp); + + cc.mov(tmp, (((int64_t *)konstf)[regnum + 1])); + cc.mov(x86::qword_ptr(location, 8), tmp); + } + else + { + cc.mov(tmp, (((int64_t *)konstf)[regnum])); + cc.mov(x86::qword_ptr(location), tmp); + } + } + else + { + if (regtype & REGT_MULTIREG3) + { + cc.movsd(x86::qword_ptr(location), regF[regnum]); + cc.movsd(x86::qword_ptr(location, 8), regF[regnum + 1]); + cc.movsd(x86::qword_ptr(location, 16), regF[regnum + 2]); + } + else if (regtype & REGT_MULTIREG2) + { + cc.movsd(x86::qword_ptr(location), regF[regnum]); + cc.movsd(x86::qword_ptr(location, 8), regF[regnum + 1]); + } + else + { + cc.movsd(x86::qword_ptr(location), regF[regnum]); + } + } + break; + case REGT_STRING: + break; + case REGT_POINTER: + #ifdef ASMJIT_ARCH_X64 + if (regtype & REGT_KONST) + cc.mov(x86::qword_ptr(location), ToMemAddress(konsta[regnum].v)); + else + cc.mov(x86::qword_ptr(location), regA[regnum]); + #else + if (regtype & REGT_KONST) + cc.mov(x86::dword_ptr(location), ToMemAddress(konsta[regnum].v)); + else + cc.mov(x86::dword_ptr(location), regA[regnum]); + #endif + break; + } + + if (a & RET_FINAL) + { + cc.add(reg_retnum, 1); + cc.ret(reg_retnum); + } + + cc.bind(L_endif); + if (a & RET_FINAL) + cc.ret(numret); + } +} + +void JitCompiler::EmitRETI() +{ + using namespace asmjit; + + int a = A; + int retnum = a & ~RET_FINAL; + + X86Gp reg_retnum = cc.newInt32(); + X86Gp location = cc.newIntPtr(); + Label L_endif = cc.newLabel(); + + cc.mov(reg_retnum, retnum); + cc.test(reg_retnum, numret); + cc.jg(L_endif); + + cc.mov(location, x86::ptr(ret, retnum * sizeof(VMReturn))); + cc.mov(x86::dword_ptr(location), BCs); + + if (a & RET_FINAL) + { + cc.add(reg_retnum, 1); + cc.ret(reg_retnum); + } + + cc.bind(L_endif); + if (a & RET_FINAL) + cc.ret(numret); +} + +void JitCompiler::EmitNEW() +{ + auto result = cc.newIntPtr(); + typedef DObject*(*FuncPtr)(PClass*, int); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](PClass *cls, int c) -> DObject* { + try + { + if (!cls->ConstructNative) + { + ThrowAbortException(X_OTHER, "Class %s requires native construction", cls->TypeName.GetChars()); + } + else if (cls->bAbstract) + { + ThrowAbortException(X_OTHER, "Cannot instantiate abstract class %s", cls->TypeName.GetChars()); + } + else if (cls->IsDescendantOf(NAME_Actor)) // Creating actors here must be outright prohibited + { + ThrowAbortException(X_OTHER, "Cannot create actors with 'new'"); + } + + // [ZZ] validate readonly and between scope construction + if (c) FScopeBarrier::ValidateNew(cls, c - 1); + return cls->CreateNew(); + } + catch (const CVMAbortException &) + { + // To do: pass along the exception info + return nullptr; + } + }))), asmjit::FuncSignature2()); + call->setRet(0, result); + call->setArg(0, regA[B]); + call->setArg(1, asmjit::Imm(C)); + + auto notnull = cc.newLabel(); + cc.test(result, result); + cc.jnz(notnull); + EmitThrowException(X_OTHER); + cc.bind(notnull); + cc.mov(regA[A], result); +} + +void JitCompiler::EmitNEW_K() +{ + PClass *cls = (PClass*)konsta[B].v; + if (!cls->ConstructNative) + { + EmitThrowException(X_OTHER); // "Class %s requires native construction", cls->TypeName.GetChars() + } + else if (cls->bAbstract) + { + EmitThrowException(X_OTHER); // "Cannot instantiate abstract class %s", cls->TypeName.GetChars() + } + else if (cls->IsDescendantOf(NAME_Actor)) // Creating actors here must be outright prohibited + { + EmitThrowException(X_OTHER); // "Cannot create actors with 'new'" + } + else + { + auto result = cc.newIntPtr(); + auto regcls = cc.newIntPtr(); + cc.mov(regcls, ToMemAddress(konsta[B].v)); + typedef DObject*(*FuncPtr)(PClass*, int); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](PClass *cls, int c) -> DObject* { + try + { + if (c) FScopeBarrier::ValidateNew(cls, c - 1); + return cls->CreateNew(); + } + catch (const CVMAbortException &) + { + // To do: pass along the exception info + return nullptr; + } + }))), asmjit::FuncSignature2()); + call->setRet(0, result); + call->setArg(0, regcls); + call->setArg(1, asmjit::Imm(C)); + + auto notnull = cc.newLabel(); + cc.test(result, result); + cc.jnz(notnull); + EmitThrowException(X_OTHER); + cc.bind(notnull); + cc.mov(regA[A], result); + } +} + +void JitCompiler::EmitTHROW() +{ + EmitThrowException(EVMAbortException(BC)); +} + +void JitCompiler::EmitBOUND() +{ + auto label1 = cc.newLabel(); + auto label2 = cc.newLabel(); + cc.cmp(regD[A], (int)BC); + cc.jl(label1); + EmitThrowException(X_ARRAY_OUT_OF_BOUNDS); // "Max.index = %u, current index = %u\n", BC, reg.d[A] + cc.bind(label1); + cc.cmp(regD[A], (int)0); + cc.jge(label2); + EmitThrowException(X_ARRAY_OUT_OF_BOUNDS); // "Negative current index = %i\n", reg.d[A] + cc.bind(label2); +} + +void JitCompiler::EmitBOUND_K() +{ + auto label1 = cc.newLabel(); + auto label2 = cc.newLabel(); + cc.cmp(regD[A], (int)konstd[BC]); + cc.jl(label1); + EmitThrowException(X_ARRAY_OUT_OF_BOUNDS); // "Max.index = %u, current index = %u\n", konstd[BC], reg.d[A] + cc.bind(label1); + cc.cmp(regD[A], (int)0); + cc.jge(label2); + EmitThrowException(X_ARRAY_OUT_OF_BOUNDS); // "Negative current index = %i\n", reg.d[A] + cc.bind(label2); +} + +void JitCompiler::EmitBOUND_R() +{ + auto label1 = cc.newLabel(); + auto label2 = cc.newLabel(); + cc.cmp(regD[A], regD[B]); + cc.jl(label1); + EmitThrowException(X_ARRAY_OUT_OF_BOUNDS); // "Max.index = %u, current index = %u\n", reg.d[B], reg.d[A] + cc.bind(label1); + cc.cmp(regD[A], (int)0); + cc.jge(label2); + EmitThrowException(X_ARRAY_OUT_OF_BOUNDS); // "Negative current index = %i\n", reg.d[A] + cc.bind(label2); +} diff --git a/src/scripting/vm/jit_load.cpp b/src/scripting/vm/jit_load.cpp new file mode 100644 index 000000000..718f511c2 --- /dev/null +++ b/src/scripting/vm/jit_load.cpp @@ -0,0 +1,337 @@ + +#include "jitintern.h" + +///////////////////////////////////////////////////////////////////////////// +// Load constants. + +void JitCompiler::EmitLI() +{ + cc.mov(regD[A], BCs); +} + +void JitCompiler::EmitLK() +{ + cc.mov(regD[A], konstd[BC]); +} + +void JitCompiler::EmitLKF() +{ + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(konstf + BC)); + cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); +} + +void JitCompiler::EmitLKS() +{ + auto loadLambda = [] (FString* to, FString* from) -> void { + *to = *from; + }; + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(loadLambda))), + asmjit::FuncSignature2(asmjit::CallConv::kIdHostCDecl)); + call->setArg(0, regS[A]); + call->setArg(1, asmjit::imm(ToMemAddress(konsts + BC))); +} + +void JitCompiler::EmitLKP() +{ + cc.mov(regA[A], (int64_t)konsta[BC].v); +} + +void JitCompiler::EmitLK_R() +{ + cc.mov(regD[A], asmjit::x86::ptr(ToMemAddress(konstd), regD[B], 2, C * sizeof(int32_t))); +} + +void JitCompiler::EmitLKF_R() +{ + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(konstf + BC)); + cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp, regD[B], 3, C * sizeof(double))); +} + +void JitCompiler::EmitLKS_R() +{ + I_FatalError("EmitLKS_R not implemented\n"); +} + +void JitCompiler::EmitLKP_R() +{ + cc.mov(regA[A], asmjit::x86::ptr(ToMemAddress(konsta), regD[B], 2, C * sizeof(void*))); +} + +void JitCompiler::EmitLFP() +{ + I_FatalError("EmitLFP not implemented\n"); +} + +void JitCompiler::EmitMETA() +{ + typedef void*(*FuncPtr)(void*); + + auto label = cc.newLabel(); + cc.test(regA[B], regA[B]); + cc.jne(label); + EmitThrowException(X_READ_NIL); + cc.bind(label); + + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](void *o) -> void* + { + return static_cast(o)->GetClass()->Meta; + }))), asmjit::FuncSignature1()); + call->setRet(0, regA[A]); + call->setArg(0, regA[B]); +} + +void JitCompiler::EmitCLSS() +{ + typedef void*(*FuncPtr)(void*); + + auto label = cc.newLabel(); + cc.test(regA[B], regA[B]); + cc.jne(label); + EmitThrowException(X_READ_NIL); + cc.bind(label); + + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](void *o) -> void* + { + return static_cast(o)->GetClass(); + }))), asmjit::FuncSignature1()); + call->setRet(0, regA[A]); + call->setArg(0, regA[B]); +} + +///////////////////////////////////////////////////////////////////////////// +// Load from memory. rA = *(rB + rkC) + +void JitCompiler::EmitLB() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.movsx(regD[A], asmjit::x86::byte_ptr(regA[B], konstd[C])); +} + +void JitCompiler::EmitLB_R() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.movsx(regD[A], asmjit::x86::byte_ptr(regA[B], regD[C])); +} + +void JitCompiler::EmitLH() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.movsx(regD[A], asmjit::x86::word_ptr(regA[B], konstd[C])); +} + +void JitCompiler::EmitLH_R() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.movsx(regD[A], asmjit::x86::word_ptr(regA[B], regD[C])); +} + +void JitCompiler::EmitLW() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.mov(regD[A], asmjit::x86::dword_ptr(regA[B], konstd[C])); +} + +void JitCompiler::EmitLW_R() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.mov(regD[A], asmjit::x86::dword_ptr(regA[B], regD[C])); +} + +void JitCompiler::EmitLBU() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.movzx(regD[A], asmjit::x86::byte_ptr(regA[B], konstd[C])); +} + +void JitCompiler::EmitLBU_R() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.movzx(regD[A].r8Lo(), asmjit::x86::byte_ptr(regA[B], regD[C])); +} + +void JitCompiler::EmitLHU() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.movzx(regD[A].r16(), asmjit::x86::word_ptr(regA[B], konstd[C])); +} + +void JitCompiler::EmitLHU_R() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.movzx(regD[A].r16(), asmjit::x86::word_ptr(regA[B], regD[C])); +} + +void JitCompiler::EmitLSP() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.movss(regF[A], asmjit::x86::dword_ptr(regA[B], konstd[C])); +} + +void JitCompiler::EmitLSP_R() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.movss(regF[A], asmjit::x86::dword_ptr(regA[B], regD[C])); +} + +void JitCompiler::EmitLDP() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.movsd(regF[A], asmjit::x86::qword_ptr(regA[B], konstd[C])); +} + +void JitCompiler::EmitLDP_R() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.movsd(regF[A], asmjit::x86::qword_ptr(regA[B], regD[C])); +} + +void JitCompiler::EmitLS() +{ + EmitNullPointerThrow(B, X_READ_NIL); + auto ptr = cc.newIntPtr(); + cc.mov(ptr, regA[B]); + cc.add(ptr, konstd[C]); + auto loadLambda = [](FString* to, FString* from) -> void { + *to = *from; + }; + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(loadLambda))), + asmjit::FuncSignature2(asmjit::CallConv::kIdHostCDecl)); + call->setArg(0, regS[A]); + call->setArg(1, ptr); +} + +void JitCompiler::EmitLS_R() +{ + EmitNullPointerThrow(B, X_READ_NIL); + auto ptr = cc.newIntPtr(); + cc.mov(ptr, regA[B]); + auto tmp = cc.newIntPtr(); + cc.mov(tmp, regD[C]); + cc.add(ptr, tmp); + auto loadLambda = [](FString* to, FString* from) -> void { + *to = *from; + }; + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(loadLambda))), + asmjit::FuncSignature2(asmjit::CallConv::kIdHostCDecl)); + call->setArg(0, regS[A]); + call->setArg(1, ptr); +} + +void JitCompiler::EmitLO() +{ + EmitNullPointerThrow(B, X_READ_NIL); + + auto ptr = cc.newIntPtr(); + cc.mov(ptr, asmjit::x86::ptr(regA[B], konstd[C])); + + typedef void*(*FuncPtr)(void*); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](void *ptr) -> void* + { + DObject *p = static_cast(ptr); + return GC::ReadBarrier(p); + }))), asmjit::FuncSignature1()); + call->setRet(0, regA[A]); + call->setArg(0, ptr); +} + +void JitCompiler::EmitLO_R() +{ + EmitNullPointerThrow(B, X_READ_NIL); + + auto ptr = cc.newIntPtr(); + cc.mov(ptr, asmjit::x86::ptr(regA[B], regD[C])); + + typedef void*(*FuncPtr)(void*); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](void *ptr) -> void* + { + DObject *p = static_cast(ptr); + return GC::ReadBarrier(p); + }))), asmjit::FuncSignature1()); + call->setRet(0, regA[A]); + call->setArg(0, ptr); +} + +void JitCompiler::EmitLP() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.mov(regA[A], asmjit::x86::ptr(regA[B], konstd[C])); +} + +void JitCompiler::EmitLP_R() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.mov(regA[A], asmjit::x86::ptr(regA[B], regD[C])); +} + +void JitCompiler::EmitLV2() +{ + EmitNullPointerThrow(B, X_READ_NIL); + auto tmp = cc.newIntPtr(); + cc.mov(tmp, regA[B]); + cc.add(tmp, konstd[C]); + cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); + cc.movsd(regF[A + 1], asmjit::x86::qword_ptr(tmp, 8)); +} + +void JitCompiler::EmitLV2_R() +{ + EmitNullPointerThrow(B, X_READ_NIL); + auto tmp = cc.newIntPtr(); + cc.mov(tmp, regA[B]); + cc.add(tmp, regD[C]); + cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); + cc.movsd(regF[A + 1], asmjit::x86::qword_ptr(tmp, 8)); +} + +void JitCompiler::EmitLV3() +{ + EmitNullPointerThrow(B, X_READ_NIL); + auto tmp = cc.newIntPtr(); + cc.mov(tmp, regA[B]); + cc.add(tmp, konstd[C]); + cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); + cc.movsd(regF[A + 1], asmjit::x86::qword_ptr(tmp, 8)); + cc.movsd(regF[A + 2], asmjit::x86::qword_ptr(tmp, 16)); +} + +void JitCompiler::EmitLV3_R() +{ + EmitNullPointerThrow(B, X_READ_NIL); + auto tmp = cc.newIntPtr(); + cc.mov(tmp, regA[B]); + cc.add(tmp, regD[C]); + cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); + cc.movsd(regF[A + 1], asmjit::x86::qword_ptr(tmp, 8)); + cc.movsd(regF[A + 2], asmjit::x86::qword_ptr(tmp, 16)); +} + +void JitCompiler::EmitLCS() +{ + EmitNullPointerThrow(B, X_READ_NIL); + auto ptr = cc.newIntPtr(); + cc.mov(ptr, regA[B]); + cc.add(ptr, konstd[C]); + auto loadLambda = [](FString* to, char** from) -> void { + *to = *from; + }; + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(loadLambda))), + asmjit::FuncSignature2(asmjit::CallConv::kIdHostCDecl)); + call->setArg(0, regS[A]); + call->setArg(1, ptr); +} + +void JitCompiler::EmitLCS_R() +{ + I_FatalError("EmitLCS_R not implemented\n"); +} + +void JitCompiler::EmitLBIT() +{ + EmitNullPointerThrow(B, X_READ_NIL); + cc.movsx(regD[A], asmjit::x86::byte_ptr(regA[B])); + cc.and_(regD[A], C); + cc.cmp(regD[A], 0); + cc.setne(regD[A]); +} diff --git a/src/scripting/vm/jit_math.cpp b/src/scripting/vm/jit_math.cpp new file mode 100644 index 000000000..47413c087 --- /dev/null +++ b/src/scripting/vm/jit_math.cpp @@ -0,0 +1,1491 @@ + +#include "jitintern.h" + +///////////////////////////////////////////////////////////////////////////// +// String instructions. + +void JitCompiler::EmitCONCAT() +{ + I_FatalError("EmitCONCAT not implemented\n"); +} + +void JitCompiler::EmitLENS() +{ + I_FatalError("EmitLENS not implemented\n"); +} + +void JitCompiler::EmitCMPS() +{ + I_FatalError("EmitCMPS not implemented\n"); +} + +///////////////////////////////////////////////////////////////////////////// +// Integer math. + +void JitCompiler::EmitSLL_RR() +{ + auto rc = CheckRegD(C, A); + if (A != B) + cc.mov(regD[A], regD[B]); + cc.shl(regD[A], rc); +} + +void JitCompiler::EmitSLL_RI() +{ + if (A != B) + cc.mov(regD[A], regD[B]); + cc.shl(regD[A], C); +} + +void JitCompiler::EmitSLL_KR() +{ + auto rc = CheckRegD(C, A); + cc.mov(regD[A], konstd[B]); + cc.shl(regD[A], rc); +} + +void JitCompiler::EmitSRL_RR() +{ + auto rc = CheckRegD(C, A); + if (A != B) + cc.mov(regD[A], regD[B]); + cc.shr(regD[A], rc); +} + +void JitCompiler::EmitSRL_RI() +{ + if (A != B) + cc.mov(regD[A], regD[B]); + cc.shr(regD[A], C); +} + +void JitCompiler::EmitSRL_KR() +{ + auto rc = CheckRegD(C, A); + cc.mov(regD[A], konstd[B]); + cc.shr(regD[A], rc); +} + +void JitCompiler::EmitSRA_RR() +{ + auto rc = CheckRegD(C, A); + if (A != B) + cc.mov(regD[A], regD[B]); + cc.sar(regD[A], rc); +} + +void JitCompiler::EmitSRA_RI() +{ + if (A != B) + cc.mov(regD[A], regD[B]); + cc.sar(regD[A], C); +} + +void JitCompiler::EmitSRA_KR() +{ + auto rc = CheckRegD(C, A); + cc.mov(regD[A], konstd[B]); + cc.sar(regD[A], rc); +} + +void JitCompiler::EmitADD_RR() +{ + auto rc = CheckRegD(C, A); + if (A != B) + cc.mov(regD[A], regD[B]); + cc.add(regD[A], rc); +} + +void JitCompiler::EmitADD_RK() +{ + if (A != B) + cc.mov(regD[A], regD[B]); + cc.add(regD[A], konstd[C]); +} + +void JitCompiler::EmitADDI() +{ + if (A != B) + cc.mov(regD[A], regD[B]); + cc.add(regD[A], Cs); +} + +void JitCompiler::EmitSUB_RR() +{ + auto rc = CheckRegD(C, A); + if (A != B) + cc.mov(regD[A], regD[B]); + cc.sub(regD[A], rc); +} + +void JitCompiler::EmitSUB_RK() +{ + if (A != B) + cc.mov(regD[A], regD[B]); + cc.sub(regD[A], konstd[C]); +} + +void JitCompiler::EmitSUB_KR() +{ + auto rc = CheckRegD(C, A); + cc.mov(regD[A], konstd[B]); + cc.sub(regD[A], rc); +} + +void JitCompiler::EmitMUL_RR() +{ + auto rc = CheckRegD(C, A); + if (A != B) + cc.mov(regD[A], regD[B]); + cc.imul(regD[A], rc); +} + +void JitCompiler::EmitMUL_RK() +{ + if (A != B) + cc.mov(regD[A], regD[B]); + cc.imul(regD[A], konstd[C]); +} + +void JitCompiler::EmitDIV_RR() +{ + auto tmp0 = cc.newInt32(); + auto tmp1 = cc.newInt32(); + auto label = cc.newLabel(); + + cc.test(regD[C], regD[C]); + cc.jne(label); + + EmitThrowException(X_DIVISION_BY_ZERO); + + cc.bind(label); + cc.mov(tmp0, regD[B]); + cc.cdq(tmp1, tmp0); + cc.idiv(tmp1, tmp0, regD[C]); + cc.mov(regD[A], tmp0); +} + +void JitCompiler::EmitDIV_RK() +{ + if (konstd[C] != 0) + { + auto tmp0 = cc.newInt32(); + auto tmp1 = cc.newInt32(); + auto konstTmp = cc.newIntPtr(); + cc.mov(tmp0, regD[B]); + cc.cdq(tmp1, tmp0); + cc.mov(konstTmp, ToMemAddress(&konstd[C])); + cc.idiv(tmp1, tmp0, asmjit::x86::ptr(konstTmp)); + cc.mov(regD[A], tmp0); + } + else EmitThrowException(X_DIVISION_BY_ZERO); +} + +void JitCompiler::EmitDIV_KR() +{ + auto tmp0 = cc.newInt32(); + auto tmp1 = cc.newInt32(); + auto label = cc.newLabel(); + + cc.test(regD[C], regD[C]); + cc.jne(label); + + EmitThrowException(X_DIVISION_BY_ZERO); + + cc.bind(label); + cc.mov(tmp0, konstd[B]); + cc.cdq(tmp1, tmp0); + cc.idiv(tmp1, tmp0, regD[C]); + cc.mov(regD[A], tmp0); +} + +void JitCompiler::EmitDIVU_RR() +{ + auto tmp0 = cc.newInt32(); + auto tmp1 = cc.newInt32(); + auto label = cc.newLabel(); + + cc.test(regD[C], regD[C]); + cc.jne(label); + + EmitThrowException(X_DIVISION_BY_ZERO); + + cc.bind(label); + cc.mov(tmp0, regD[B]); + cc.mov(tmp1, 0); + cc.div(tmp1, tmp0, regD[C]); + cc.mov(regD[A], tmp0); +} + +void JitCompiler::EmitDIVU_RK() +{ + if (konstd[C] != 0) + { + auto tmp0 = cc.newInt32(); + auto tmp1 = cc.newInt32(); + auto konstTmp = cc.newIntPtr(); + cc.mov(tmp0, regD[B]); + cc.mov(tmp1, 0); + cc.mov(konstTmp, ToMemAddress(&konstd[C])); + cc.div(tmp1, tmp0, asmjit::x86::ptr(konstTmp)); + cc.mov(regD[A], tmp0); + } + else EmitThrowException(X_DIVISION_BY_ZERO); +} + +void JitCompiler::EmitDIVU_KR() +{ + auto tmp0 = cc.newInt32(); + auto tmp1 = cc.newInt32(); + auto label = cc.newLabel(); + + cc.test(regD[C], regD[C]); + cc.jne(label); + + EmitThrowException(X_DIVISION_BY_ZERO); + + cc.bind(label); + cc.mov(tmp0, konstd[B]); + cc.mov(tmp1, 0); + cc.div(tmp1, tmp0, regD[C]); + cc.mov(regD[A], tmp0); +} + +void JitCompiler::EmitMOD_RR() +{ + auto tmp0 = cc.newInt32(); + auto tmp1 = cc.newInt32(); + auto label = cc.newLabel(); + + cc.test(regD[C], regD[C]); + cc.jne(label); + + EmitThrowException(X_DIVISION_BY_ZERO); + + cc.bind(label); + cc.mov(tmp0, regD[B]); + cc.cdq(tmp1, tmp0); + cc.idiv(tmp1, tmp0, regD[C]); + cc.mov(regD[A], tmp1); +} + +void JitCompiler::EmitMOD_RK() +{ + if (konstd[C] != 0) + { + auto tmp0 = cc.newInt32(); + auto tmp1 = cc.newInt32(); + auto konstTmp = cc.newIntPtr(); + cc.mov(tmp0, regD[B]); + cc.cdq(tmp1, tmp0); + cc.mov(konstTmp, ToMemAddress(&konstd[C])); + cc.idiv(tmp1, tmp0, asmjit::x86::ptr(konstTmp)); + cc.mov(regD[A], tmp1); + } + else EmitThrowException(X_DIVISION_BY_ZERO); +} + +void JitCompiler::EmitMOD_KR() +{ + auto tmp0 = cc.newInt32(); + auto tmp1 = cc.newInt32(); + auto label = cc.newLabel(); + + cc.test(regD[C], regD[C]); + cc.jne(label); + + EmitThrowException(X_DIVISION_BY_ZERO); + + cc.bind(label); + cc.mov(tmp0, konstd[B]); + cc.cdq(tmp1, tmp0); + cc.idiv(tmp1, tmp0, regD[C]); + cc.mov(regD[A], tmp1); +} + +void JitCompiler::EmitMODU_RR() +{ + auto tmp0 = cc.newInt32(); + auto tmp1 = cc.newInt32(); + auto label = cc.newLabel(); + + cc.test(regD[C], regD[C]); + cc.jne(label); + + EmitThrowException(X_DIVISION_BY_ZERO); + + cc.bind(label); + cc.mov(tmp0, regD[B]); + cc.mov(tmp1, 0); + cc.div(tmp1, tmp0, regD[C]); + cc.mov(regD[A], tmp1); +} + +void JitCompiler::EmitMODU_RK() +{ + if (konstd[C] != 0) + { + auto tmp0 = cc.newInt32(); + auto tmp1 = cc.newInt32(); + auto konstTmp = cc.newIntPtr(); + cc.mov(tmp0, regD[B]); + cc.mov(tmp1, 0); + cc.mov(konstTmp, ToMemAddress(&konstd[C])); + cc.div(tmp1, tmp0, asmjit::x86::ptr(konstTmp)); + cc.mov(regD[A], tmp1); + } + else EmitThrowException(X_DIVISION_BY_ZERO); +} + +void JitCompiler::EmitMODU_KR() +{ + auto tmp0 = cc.newInt32(); + auto tmp1 = cc.newInt32(); + auto label = cc.newLabel(); + + cc.test(regD[C], regD[C]); + cc.jne(label); + + EmitThrowException(X_DIVISION_BY_ZERO); + + cc.bind(label); + cc.mov(tmp0, konstd[B]); + cc.mov(tmp1, 0); + cc.div(tmp1, tmp0, regD[C]); + cc.mov(regD[A], tmp1); +} + +void JitCompiler::EmitAND_RR() +{ + auto rc = CheckRegD(C, A); + if (A != B) + cc.mov(regD[A], regD[B]); + cc.and_(regD[A], rc); +} + +void JitCompiler::EmitAND_RK() +{ + if (A != B) + cc.mov(regD[A], regD[B]); + cc.and_(regD[A], konstd[C]); +} + +void JitCompiler::EmitOR_RR() +{ + auto rc = CheckRegD(C, A); + if (A != B) + cc.mov(regD[A], regD[B]); + cc.or_(regD[A], rc); +} + +void JitCompiler::EmitOR_RK() +{ + if (A != B) + cc.mov(regD[A], regD[B]); + cc.or_(regD[A], konstd[C]); +} + +void JitCompiler::EmitXOR_RR() +{ + auto rc = CheckRegD(C, A); + if (A != B) + cc.mov(regD[A], regD[B]); + cc.xor_(regD[A], rc); +} + +void JitCompiler::EmitXOR_RK() +{ + if (A != B) + cc.mov(regD[A], regD[B]); + cc.xor_(regD[A], konstd[C]); +} + +void JitCompiler::EmitMIN_RR() +{ + auto tmp0 = cc.newXmmSs(); + auto tmp1 = cc.newXmmSs(); + cc.movd(tmp0, regD[B]); + cc.movd(tmp1, regD[C]); + cc.pminsd(tmp0, tmp1); + cc.movd(regD[A], tmp0); +} + +void JitCompiler::EmitMIN_RK() +{ + auto tmp0 = cc.newXmmSs(); + auto tmp1 = cc.newXmmSs(); + auto konstTmp = cc.newIntPtr(); + cc.mov(konstTmp, ToMemAddress(&konstd[C])); + cc.movd(tmp0, regD[B]); + cc.movss(tmp1, asmjit::x86::dword_ptr(konstTmp)); + cc.pminsd(tmp0, tmp1); + cc.movd(regD[A], tmp0); +} + +void JitCompiler::EmitMAX_RR() +{ + auto tmp0 = cc.newXmmSs(); + auto tmp1 = cc.newXmmSs(); + cc.movd(tmp0, regD[B]); + cc.movd(tmp1, regD[C]); + cc.pmaxsd(tmp0, tmp1); + cc.movd(regD[A], tmp0); +} + +void JitCompiler::EmitMAX_RK() +{ + auto tmp0 = cc.newXmmSs(); + auto tmp1 = cc.newXmmSs(); + auto konstTmp = cc.newIntPtr(); + cc.mov(konstTmp, ToMemAddress(&konstd[C])); + cc.movd(tmp0, regD[B]); + cc.movss(tmp1, asmjit::x86::dword_ptr(konstTmp)); + cc.pmaxsd(tmp0, tmp1); + cc.movd(regD[A], tmp0); +} + +void JitCompiler::EmitABS() +{ + auto srcB = CheckRegD(B, A); + auto tmp = cc.newInt32(); + cc.mov(tmp, regD[B]); + cc.sar(tmp, 31); + cc.mov(regD[A], tmp); + cc.xor_(regD[A], srcB); + cc.sub(regD[A], tmp); +} + +void JitCompiler::EmitNEG() +{ + auto srcB = CheckRegD(B, A); + cc.xor_(regD[A], regD[A]); + cc.sub(regD[A], srcB); +} + +void JitCompiler::EmitNOT() +{ + if (A != B) + cc.mov(regD[A], regD[B]); + cc.not_(regD[A]); +} + +void JitCompiler::EmitEQ_R() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + cc.cmp(regD[B], regD[C]); + if (check) cc.je(fail); + else cc.jne(fail); + }); +} + +void JitCompiler::EmitEQ_K() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + cc.cmp(regD[B], konstd[C]); + if (check) cc.je(fail); + else cc.jne(fail); + }); +} + +void JitCompiler::EmitLT_RR() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + cc.cmp(regD[B], regD[C]); + if (check) cc.jl(fail); + else cc.jnl(fail); + }); +} + +void JitCompiler::EmitLT_RK() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + cc.cmp(regD[B], konstd[C]); + if (check) cc.jl(fail); + else cc.jnl(fail); + }); +} + +void JitCompiler::EmitLT_KR() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(&konstd[B])); + cc.cmp(asmjit::x86::ptr(tmp), regD[C]); + if (check) cc.jl(fail); + else cc.jnl(fail); + }); +} + +void JitCompiler::EmitLE_RR() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + cc.cmp(regD[B], regD[C]); + if (check) cc.jle(fail); + else cc.jnle(fail); + }); +} + +void JitCompiler::EmitLE_RK() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + cc.cmp(regD[B], konstd[C]); + if (check) cc.jle(fail); + else cc.jnle(fail); + }); +} + +void JitCompiler::EmitLE_KR() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(&konstd[B])); + cc.cmp(asmjit::x86::ptr(tmp), regD[C]); + if (check) cc.jle(fail); + else cc.jnle(fail); + }); +} + +void JitCompiler::EmitLTU_RR() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + cc.cmp(regD[B], regD[C]); + if (check) cc.jb(fail); + else cc.jnb(fail); + }); +} + +void JitCompiler::EmitLTU_RK() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + cc.cmp(regD[B], konstd[C]); + if (check) cc.jb(fail); + else cc.jnb(fail); + }); +} + +void JitCompiler::EmitLTU_KR() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(&konstd[B])); + cc.cmp(asmjit::x86::ptr(tmp), regD[C]); + if (check) cc.jb(fail); + else cc.jnb(fail); + }); +} + +void JitCompiler::EmitLEU_RR() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + cc.cmp(regD[B], regD[C]); + if (check) cc.jbe(fail); + else cc.jnbe(fail); + }); +} + +void JitCompiler::EmitLEU_RK() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + cc.cmp(regD[B], konstd[C]); + if (check) cc.jbe(fail); + else cc.jnbe(fail); + }); +} + +void JitCompiler::EmitLEU_KR() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(&konstd[B])); + cc.cmp(asmjit::x86::ptr(tmp), regD[C]); + if (check) cc.jbe(fail); + else cc.jnbe(fail); + }); +} + +///////////////////////////////////////////////////////////////////////////// +// Double-precision floating point math. + +void JitCompiler::EmitADDF_RR() +{ + auto rc = CheckRegF(C, A); + if (A != B) + cc.movsd(regF[A], regF[B]); + cc.addsd(regF[A], rc); +} + +void JitCompiler::EmitADDF_RK() +{ + auto tmp = cc.newIntPtr(); + if (A != B) + cc.movsd(regF[A], regF[B]); + cc.mov(tmp, ToMemAddress(&konstf[C])); + cc.addsd(regF[A], asmjit::x86::qword_ptr(tmp)); +} + +void JitCompiler::EmitSUBF_RR() +{ + auto rc = CheckRegF(C, A); + if (A != B) + cc.movsd(regF[A], regF[B]); + cc.subsd(regF[A], rc); +} + +void JitCompiler::EmitSUBF_RK() +{ + auto tmp = cc.newIntPtr(); + if (A != B) + cc.movsd(regF[A], regF[B]); + cc.mov(tmp, ToMemAddress(&konstf[C])); + cc.subsd(regF[A], asmjit::x86::qword_ptr(tmp)); +} + +void JitCompiler::EmitSUBF_KR() +{ + auto rc = CheckRegF(C, A); + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(&konstf[B])); + cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); + cc.subsd(regF[A], rc); +} + +void JitCompiler::EmitMULF_RR() +{ + auto rc = CheckRegF(C, A); + if (A != B) + cc.movsd(regF[A], regF[B]); + cc.mulsd(regF[A], rc); +} + +void JitCompiler::EmitMULF_RK() +{ + auto tmp = cc.newIntPtr(); + if (A != B) + cc.movsd(regF[A], regF[B]); + cc.mov(tmp, ToMemAddress(&konstf[C])); + cc.mulsd(regF[A], asmjit::x86::qword_ptr(tmp)); +} + +void JitCompiler::EmitDIVF_RR() +{ + auto label = cc.newLabel(); + cc.ptest(regF[C], regF[C]); + cc.jne(label); + EmitThrowException(X_DIVISION_BY_ZERO); + cc.bind(label); + auto rc = CheckRegF(C, A); + cc.movsd(regF[A], regF[B]); + cc.divsd(regF[A], rc); +} + +void JitCompiler::EmitDIVF_RK() +{ + if (konstf[C] == 0.) + { + EmitThrowException(X_DIVISION_BY_ZERO); + } + else + { + auto tmp = cc.newIntPtr(); + cc.movsd(regF[A], regF[B]); + cc.mov(tmp, ToMemAddress(&konstf[C])); + cc.divsd(regF[A], asmjit::x86::qword_ptr(tmp)); + } +} + +void JitCompiler::EmitDIVF_KR() +{ + auto rc = CheckRegF(C, A); + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(&konstf[B])); + cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); + cc.divsd(regF[A], rc); +} + +void JitCompiler::EmitMODF_RR() +{ + using namespace asmjit; + typedef double(*FuncPtr)(double, double); + + auto label = cc.newLabel(); + cc.ptest(regF[C], regF[C]); + cc.jne(label); + EmitThrowException(X_DIVISION_BY_ZERO); + cc.bind(label); + + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](double a, double b) -> double + { + return a - floor(a / b) * b; + }))), FuncSignature2()); + call->setRet(0, regF[A]); + call->setArg(0, regF[B]); + call->setArg(1, regF[C]); +} + +void JitCompiler::EmitMODF_RK() +{ + using namespace asmjit; + typedef double(*FuncPtr)(double, double); + + auto label = cc.newLabel(); + cc.ptest(regF[C], regF[C]); + cc.jne(label); + EmitThrowException(X_DIVISION_BY_ZERO); + cc.bind(label); + + auto tmp = cc.newXmm(); + cc.movsd(tmp, x86::ptr(ToMemAddress(&konstf[C]))); + + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](double a, double b) -> double { + return a - floor(a / b) * b; + }))), FuncSignature2()); + call->setRet(0, regF[A]); + call->setArg(0, regF[B]); + call->setArg(1, tmp); +} + +void JitCompiler::EmitMODF_KR() +{ + using namespace asmjit; + typedef double(*FuncPtr)(double, double); + + auto label = cc.newLabel(); + cc.ptest(regF[C], regF[C]); + cc.jne(label); + EmitThrowException(X_DIVISION_BY_ZERO); + cc.bind(label); + + auto tmp = cc.newXmm(); + cc.movsd(tmp, x86::ptr(ToMemAddress(&konstf[B]))); + + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](double a, double b) -> double { + return a - floor(a / b) * b; + }))), FuncSignature2()); + call->setRet(0, regF[A]); + call->setArg(0, tmp); + call->setArg(1, regF[C]); +} + +void JitCompiler::EmitPOWF_RR() +{ + using namespace asmjit; + typedef double(*FuncPtr)(double, double); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(g_pow))), FuncSignature2()); + call->setRet(0, regF[A]); + call->setArg(0, regF[B]); + call->setArg(1, regF[C]); +} + +void JitCompiler::EmitPOWF_RK() +{ + auto tmp = cc.newIntPtr(); + auto tmp2 = cc.newXmm(); + cc.mov(tmp, ToMemAddress(&konstf[C])); + cc.movsd(tmp2, asmjit::x86::qword_ptr(tmp)); + + using namespace asmjit; + typedef double(*FuncPtr)(double, double); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(g_pow))), FuncSignature2()); + call->setRet(0, regF[A]); + call->setArg(0, regF[B]); + call->setArg(1, tmp2); +} + +void JitCompiler::EmitPOWF_KR() +{ + auto tmp = cc.newIntPtr(); + auto tmp2 = cc.newXmm(); + cc.mov(tmp, ToMemAddress(&konstf[B])); + cc.movsd(tmp2, asmjit::x86::qword_ptr(tmp)); + + using namespace asmjit; + typedef double(*FuncPtr)(double, double); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(g_pow))), FuncSignature2()); + call->setRet(0, regF[A]); + call->setArg(0, tmp2); + call->setArg(1, regF[C]); +} + +void JitCompiler::EmitMINF_RR() +{ + auto rc = CheckRegF(C, A); + if (A != B) + cc.movsd(regF[A], regF[B]); + cc.minsd(regF[A], rc); +} + +void JitCompiler::EmitMINF_RK() +{ + auto rb = CheckRegF(B, A); + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(&konstf[C])); + cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); + cc.minsd(regF[A], rb); +} + +void JitCompiler::EmitMAXF_RR() +{ + auto rc = CheckRegF(C, A); + if (A != B) + cc.movsd(regF[A], regF[B]); + cc.maxsd(regF[A], rc); +} + +void JitCompiler::EmitMAXF_RK() +{ + auto rb = CheckRegF(B, A); + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(&konstf[C])); + cc.movsd(regF[A], asmjit::x86::qword_ptr(tmp)); + cc.maxsd(regF[A], rb); +} + +void JitCompiler::EmitATAN2() +{ + using namespace asmjit; + typedef double(*FuncPtr)(double, double); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(g_atan2))), FuncSignature2()); + call->setRet(0, regF[A]); + call->setArg(0, regF[B]); + call->setArg(1, regF[C]); + + static const double constant = 180 / M_PI; + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(&constant)); + cc.mulsd(regF[A], asmjit::x86::qword_ptr(tmp)); +} + +void JitCompiler::EmitFLOP() +{ + if (C == FLOP_NEG) + { + auto mask = cc.newDoubleConst(asmjit::kConstScopeLocal, -0.0); + auto maskXmm = cc.newXmmSd(); + cc.movsd(maskXmm, mask); + if (A != B) + cc.movsd(regF[A], regF[B]); + cc.xorpd(regF[A], maskXmm); + } + else + { + auto v = cc.newXmm(); + cc.movsd(v, regF[B]); + + if (C == FLOP_TAN_DEG) + { + static const double constant = M_PI / 180; + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(&constant)); + cc.mulsd(v, asmjit::x86::qword_ptr(tmp)); + } + + typedef double(*FuncPtr)(double); + FuncPtr func = nullptr; + switch (C) + { + default: I_FatalError("Unknown OP_FLOP subfunction"); + case FLOP_ABS: func = fabs; break; + case FLOP_EXP: func = g_exp; break; + case FLOP_LOG: func = g_log; break; + case FLOP_LOG10: func = g_log10; break; + case FLOP_SQRT: func = g_sqrt; break; + case FLOP_CEIL: func = ceil; break; + case FLOP_FLOOR: func = floor; break; + case FLOP_ACOS: func = g_acos; break; + case FLOP_ASIN: func = g_asin; break; + case FLOP_ATAN: func = g_atan; break; + case FLOP_COS: func = g_cos; break; + case FLOP_SIN: func = g_sin; break; + case FLOP_TAN: func = g_tan; break; + case FLOP_ACOS_DEG: func = g_acos; break; + case FLOP_ASIN_DEG: func = g_asin; break; + case FLOP_ATAN_DEG: func = g_atan; break; + case FLOP_COS_DEG: func = g_cosdeg; break; + case FLOP_SIN_DEG: func = g_sindeg; break; + case FLOP_TAN_DEG: func = g_tan; break; + case FLOP_COSH: func = g_cosh; break; + case FLOP_SINH: func = g_sinh; break; + case FLOP_TANH: func = g_tanh; break; + } + + auto call = cc.call(ToMemAddress(reinterpret_cast(func)), asmjit::FuncSignature1()); + call->setRet(0, regF[A]); + call->setArg(0, v); + + if (C == FLOP_ACOS_DEG || C == FLOP_ASIN_DEG || C == FLOP_ATAN_DEG) + { + static const double constant = 180 / M_PI; + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(&constant)); + cc.mulsd(regF[A], asmjit::x86::qword_ptr(tmp)); + } + } +} + +void JitCompiler::EmitEQF_R() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + bool approx = static_cast(A & CMP_APPROX); + if (!approx) + { + cc.ucomisd(regF[B], regF[C]); + if (check) { + cc.jp(success); + cc.je(fail); + } + else { + cc.jp(fail); + cc.jne(fail); + } + } + else + { + auto tmp = cc.newXmmSd(); + + const int64_t absMaskInt = 0x7FFFFFFFFFFFFFFF; + auto absMask = cc.newDoubleConst(asmjit::kConstScopeLocal, reinterpret_cast(absMaskInt)); + auto absMaskXmm = cc.newXmmPd(); + + auto epsilon = cc.newDoubleConst(asmjit::kConstScopeLocal, VM_EPSILON); + auto epsilonXmm = cc.newXmmSd(); + + cc.movsd(tmp, regF[B]); + cc.subsd(tmp, regF[C]); + cc.movsd(absMaskXmm, absMask); + cc.andpd(tmp, absMaskXmm); + cc.movsd(epsilonXmm, epsilon); + cc.ucomisd(epsilonXmm, tmp); + + if (check) cc.ja(fail); + else cc.jna(fail); + } + }); +} + +void JitCompiler::EmitEQF_K() +{ + using namespace asmjit; + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + bool approx = static_cast(A & CMP_APPROX); + if (!approx) { + auto konstTmp = cc.newIntPtr(); + cc.mov(konstTmp, ToMemAddress(&konstf[C])); + cc.ucomisd(regF[B], x86::qword_ptr(konstTmp)); + if (check) { + cc.jp(success); + cc.je(fail); + } + else { + cc.jp(fail); + cc.jne(fail); + } + } + else { + auto konstTmp = cc.newIntPtr(); + auto subTmp = cc.newXmmSd(); + + const int64_t absMaskInt = 0x7FFFFFFFFFFFFFFF; + auto absMask = cc.newDoubleConst(kConstScopeLocal, reinterpret_cast(absMaskInt)); + auto absMaskXmm = cc.newXmmPd(); + + auto epsilon = cc.newDoubleConst(kConstScopeLocal, VM_EPSILON); + auto epsilonXmm = cc.newXmmSd(); + + cc.mov(konstTmp, ToMemAddress(&konstf[C])); + + cc.movsd(subTmp, regF[B]); + cc.subsd(subTmp, x86::qword_ptr(konstTmp)); + cc.movsd(absMaskXmm, absMask); + cc.andpd(subTmp, absMaskXmm); + cc.movsd(epsilonXmm, epsilon); + cc.ucomisd(epsilonXmm, subTmp); + + if (check) cc.ja(fail); + else cc.jna(fail); + } + }); +} + +void JitCompiler::EmitLTF_RR() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for LTF_RR.\n"); + + cc.ucomisd(regF[C], regF[B]); + if (check) cc.ja(fail); + else cc.jna(fail); + }); +} + +void JitCompiler::EmitLTF_RK() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for LTF_RK.\n"); + + auto constTmp = cc.newIntPtr(); + auto xmmTmp = cc.newXmmSd(); + cc.mov(constTmp, ToMemAddress(&konstf[C])); + cc.movsd(xmmTmp, asmjit::x86::qword_ptr(constTmp)); + + cc.ucomisd(xmmTmp, regF[B]); + if (check) cc.ja(fail); + else cc.jna(fail); + }); +} + +void JitCompiler::EmitLTF_KR() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for LTF_KR.\n"); + + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(&konstf[B])); + + cc.ucomisd(regF[C], asmjit::x86::qword_ptr(tmp)); + if (check) cc.ja(fail); + else cc.jna(fail); + }); +} + +void JitCompiler::EmitLEF_RR() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for LEF_RR.\n"); + + cc.ucomisd(regF[C], regF[B]); + if (check) cc.jae(fail); + else cc.jnae(fail); + }); +} + +void JitCompiler::EmitLEF_RK() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for LEF_RK.\n"); + + auto constTmp = cc.newIntPtr(); + auto xmmTmp = cc.newXmmSd(); + cc.mov(constTmp, ToMemAddress(&konstf[C])); + cc.movsd(xmmTmp, asmjit::x86::qword_ptr(constTmp)); + + cc.ucomisd(xmmTmp, regF[B]); + if (check) cc.jae(fail); + else cc.jnae(fail); + }); +} + +void JitCompiler::EmitLEF_KR() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for LEF_KR.\n"); + + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(&konstf[B])); + + cc.ucomisd(regF[C], asmjit::x86::qword_ptr(tmp)); + if (check) cc.jae(fail); + else cc.jnae(fail); + }); +} + +///////////////////////////////////////////////////////////////////////////// +// Vector math. (2D) + +void JitCompiler::EmitNEGV2() +{ + auto mask = cc.newDoubleConst(asmjit::kConstScopeLocal, -0.0); + auto maskXmm = cc.newXmmSd(); + cc.movsd(maskXmm, mask); + cc.movsd(regF[A], regF[B]); + cc.xorpd(regF[A], maskXmm); + cc.movsd(regF[A + 1], regF[B + 1]); + cc.xorpd(regF[A + 1], maskXmm); +} + +void JitCompiler::EmitADDV2_RR() +{ + auto rc0 = CheckRegF(C, A); + auto rc1 = CheckRegF(C + 1, A + 1); + cc.movsd(regF[A], regF[B]); + cc.movsd(regF[A + 1], regF[B + 1]); + cc.addsd(regF[A], rc0); + cc.addsd(regF[A + 1], rc1); +} + +void JitCompiler::EmitSUBV2_RR() +{ + auto rc0 = CheckRegF(C, A); + auto rc1 = CheckRegF(C + 1, A + 1); + cc.movsd(regF[A], regF[B]); + cc.movsd(regF[A + 1], regF[B + 1]); + cc.subsd(regF[A], rc0); + cc.subsd(regF[A + 1], rc1); +} + +void JitCompiler::EmitDOTV2_RR() +{ + auto rc0 = CheckRegF(C, A); + auto rc1 = CheckRegF(C + 1, A); + auto tmp = cc.newXmmSd(); + cc.movsd(regF[A], regF[B]); + cc.mulsd(regF[A], rc0); + cc.movsd(tmp, regF[B + 1]); + cc.mulsd(tmp, rc1); + cc.addsd(regF[A], tmp); +} + +void JitCompiler::EmitMULVF2_RR() +{ + auto rc = CheckRegF(C, A, A + 1); + cc.movsd(regF[A], regF[B]); + cc.movsd(regF[A + 1], regF[B + 1]); + cc.mulsd(regF[A], rc); + cc.mulsd(regF[A + 1], rc); +} + +void JitCompiler::EmitMULVF2_RK() +{ + auto tmp = cc.newIntPtr(); + cc.movsd(regF[A], regF[B]); + cc.movsd(regF[A + 1], regF[B + 1]); + cc.mov(tmp, ToMemAddress(&konstf[C])); + cc.mulsd(regF[A], asmjit::x86::qword_ptr(tmp)); + cc.mulsd(regF[A + 1], asmjit::x86::qword_ptr(tmp)); +} + +void JitCompiler::EmitDIVVF2_RR() +{ + auto rc = CheckRegF(C, A, A + 1); + cc.movsd(regF[A], regF[B]); + cc.movsd(regF[A + 1], regF[B + 1]); + cc.divsd(regF[A], rc); + cc.divsd(regF[A + 1], rc); +} + +void JitCompiler::EmitDIVVF2_RK() +{ + auto tmp = cc.newIntPtr(); + cc.movsd(regF[A], regF[B]); + cc.movsd(regF[A + 1], regF[B + 1]); + cc.mov(tmp, ToMemAddress(&konstf[C])); + cc.divsd(regF[A], asmjit::x86::qword_ptr(tmp)); + cc.divsd(regF[A + 1], asmjit::x86::qword_ptr(tmp)); +} + +void JitCompiler::EmitLENV2() +{ + auto rb0 = CheckRegF(B, A); + auto rb1 = CheckRegF(B + 1, A); + auto tmp = cc.newXmmSd(); + cc.movsd(regF[A], regF[B]); + cc.mulsd(regF[A], rb0); + cc.movsd(tmp, rb1); + cc.mulsd(tmp, rb1); + cc.addsd(regF[A], tmp); + CallSqrt(regF[A], regF[A]); +} + +void JitCompiler::EmitEQV2_R() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for EQV2_R.\n"); + + cc.ucomisd(regF[B], regF[C]); + if (check) { + cc.jp(success); + cc.jne(success); + } + else { + cc.jp(fail); + cc.jne(fail); + } + + cc.ucomisd(regF[B + 1], regF[C + 1]); + if (check) { + cc.jp(success); + cc.je(fail); + } + else { + cc.jp(fail); + cc.jne(fail); + } + }); +} + +void JitCompiler::EmitEQV2_K() +{ + I_FatalError("EQV2_K is not used."); +} + +///////////////////////////////////////////////////////////////////////////// +// Vector math. (3D) + +void JitCompiler::EmitNEGV3() +{ + auto mask = cc.newDoubleConst(asmjit::kConstScopeLocal, -0.0); + auto maskXmm = cc.newXmmSd(); + cc.movsd(maskXmm, mask); + cc.movsd(regF[A], regF[B]); + cc.xorpd(regF[A], maskXmm); + cc.movsd(regF[A + 1], regF[B + 1]); + cc.xorpd(regF[A + 1], maskXmm); + cc.movsd(regF[A + 2], regF[B + 2]); + cc.xorpd(regF[A + 2], maskXmm); +} + +void JitCompiler::EmitADDV3_RR() +{ + auto rc0 = CheckRegF(C, A); + auto rc1 = CheckRegF(C + 1, A + 1); + auto rc2 = CheckRegF(C + 2, A + 2); + cc.movsd(regF[A], regF[B]); + cc.movsd(regF[A + 1], regF[B + 1]); + cc.movsd(regF[A + 2], regF[B + 2]); + cc.addsd(regF[A], rc0); + cc.addsd(regF[A + 1], rc1); + cc.addsd(regF[A + 2], rc2); +} + +void JitCompiler::EmitSUBV3_RR() +{ + auto rc0 = CheckRegF(C, A); + auto rc1 = CheckRegF(C + 1, A + 1); + auto rc2 = CheckRegF(C + 2, A + 2); + cc.movsd(regF[A], regF[B]); + cc.movsd(regF[A + 1], regF[B + 1]); + cc.movsd(regF[A + 2], regF[B + 2]); + cc.subsd(regF[A], rc0); + cc.subsd(regF[A + 1], rc1); + cc.subsd(regF[A + 2], rc2); +} + +void JitCompiler::EmitDOTV3_RR() +{ + auto rb1 = CheckRegF(B + 1, A); + auto rb2 = CheckRegF(B + 2, A); + auto rc0 = CheckRegF(C, A); + auto rc1 = CheckRegF(C + 1, A); + auto rc2 = CheckRegF(C + 2, A); + auto tmp = cc.newXmmSd(); + cc.movsd(regF[A], regF[B]); + cc.mulsd(regF[A], rc0); + cc.movsd(tmp, rb1); + cc.mulsd(tmp, rc1); + cc.addsd(regF[A], tmp); + cc.movsd(tmp, rb2); + cc.mulsd(tmp, rc2); + cc.addsd(regF[A], tmp); +} + +void JitCompiler::EmitCROSSV_RR() +{ + auto tmp = cc.newXmmSd(); + + auto a0 = CheckRegF(B, A); + auto a1 = CheckRegF(B + 1, A + 1); + auto a2 = CheckRegF(B + 2, A + 2); + auto b0 = CheckRegF(C, A); + auto b1 = CheckRegF(C + 1, A + 1); + auto b2 = CheckRegF(C + 2, A + 2); + + // r0 = a1b2 - a2b1 + cc.movsd(regF[A], a1); + cc.mulsd(regF[A], b2); + cc.movsd(tmp, a2); + cc.mulsd(tmp, b1); + cc.subsd(regF[A], tmp); + + // r1 = a2b0 - a0b2 + cc.movsd(regF[A + 1], a2); + cc.mulsd(regF[A + 1], b0); + cc.movsd(tmp, a0); + cc.mulsd(tmp, b2); + cc.subsd(regF[A + 1], tmp); + + // r2 = a0b1 - a1b0 + cc.movsd(regF[A + 2], a0); + cc.mulsd(regF[A + 2], b1); + cc.movsd(tmp, a1); + cc.mulsd(tmp, b0); + cc.subsd(regF[A + 2], tmp); +} + +void JitCompiler::EmitMULVF3_RR() +{ + auto rc = CheckRegF(C, A, A + 1, A + 2); + cc.movsd(regF[A], regF[B]); + cc.movsd(regF[A + 1], regF[B + 1]); + cc.movsd(regF[A + 2], regF[B + 2]); + cc.mulsd(regF[A], rc); + cc.mulsd(regF[A + 1], rc); + cc.mulsd(regF[A + 2], rc); +} + +void JitCompiler::EmitMULVF3_RK() +{ + auto tmp = cc.newIntPtr(); + cc.movsd(regF[A], regF[B]); + cc.movsd(regF[A + 1], regF[B + 1]); + cc.movsd(regF[A + 2], regF[B + 2]); + cc.mov(tmp, ToMemAddress(&konstf[C])); + cc.mulsd(regF[A], asmjit::x86::qword_ptr(tmp)); + cc.mulsd(regF[A + 1], asmjit::x86::qword_ptr(tmp)); + cc.mulsd(regF[A + 2], asmjit::x86::qword_ptr(tmp)); +} + +void JitCompiler::EmitDIVVF3_RR() +{ + auto rc = CheckRegF(C, A, A + 1, A + 2); + cc.movsd(regF[A], regF[B]); + cc.movsd(regF[A + 1], regF[B + 1]); + cc.movsd(regF[A + 2], regF[B + 2]); + cc.divsd(regF[A], rc); + cc.divsd(regF[A + 1], rc); + cc.divsd(regF[A + 2], rc); +} + +void JitCompiler::EmitDIVVF3_RK() +{ + auto tmp = cc.newIntPtr(); + cc.movsd(regF[A], regF[B]); + cc.movsd(regF[A + 1], regF[B + 1]); + cc.movsd(regF[A + 2], regF[B + 2]); + cc.mov(tmp, ToMemAddress(&konstf[C])); + cc.divsd(regF[A], asmjit::x86::qword_ptr(tmp)); + cc.divsd(regF[A + 1], asmjit::x86::qword_ptr(tmp)); + cc.divsd(regF[A + 2], asmjit::x86::qword_ptr(tmp)); +} + +void JitCompiler::EmitLENV3() +{ + auto rb1 = CheckRegF(B + 1, A); + auto rb2 = CheckRegF(B + 2, A); + auto tmp = cc.newXmmSd(); + cc.movsd(regF[A], regF[B]); + cc.mulsd(regF[A], regF[B]); + cc.movsd(tmp, rb1); + cc.mulsd(tmp, rb1); + cc.addsd(regF[A], tmp); + cc.movsd(tmp, rb2); + cc.mulsd(tmp, rb2); + cc.addsd(regF[A], tmp); + CallSqrt(regF[A], regF[A]); +} + +void JitCompiler::EmitEQV3_R() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + if (static_cast(A & CMP_APPROX)) I_FatalError("CMP_APPROX not implemented for EQV3_R.\n"); + + cc.ucomisd(regF[B], regF[C]); + if (check) { + cc.jp(success); + cc.jne(success); + } + else { + cc.jp(fail); + cc.jne(fail); + } + + cc.ucomisd(regF[B + 1], regF[C + 1]); + if (check) { + cc.jp(success); + cc.jne(success); + } + else { + cc.jp(fail); + cc.jne(fail); + } + + cc.ucomisd(regF[B + 2], regF[C + 2]); + if (check) { + cc.jp(success); + cc.je(fail); + } + else { + cc.jp(fail); + cc.jne(fail); + } + }); +} + +void JitCompiler::EmitEQV3_K() +{ + I_FatalError("EQV3_K is not used."); +} + +///////////////////////////////////////////////////////////////////////////// +// Pointer math. + +void JitCompiler::EmitADDA_RR() +{ + auto tmp = cc.newIntPtr(); + auto label = cc.newLabel(); + + cc.mov(tmp, regA[B]); + + // Check if zero, the first operand is zero, if it is, don't add. + cc.cmp(tmp, 0); + cc.je(label); + + auto tmpptr = cc.newIntPtr(); + cc.mov(tmpptr, regD[C]); + cc.add(tmp, tmpptr); + + cc.bind(label); + cc.mov(regA[A], tmp); +} + +void JitCompiler::EmitADDA_RK() +{ + auto tmp = cc.newIntPtr(); + auto label = cc.newLabel(); + + cc.mov(tmp, regA[B]); + + // Check if zero, the first operand is zero, if it is, don't add. + cc.cmp(tmp, 0); + cc.je(label); + + cc.add(tmp, konstd[C]); + + cc.bind(label); + cc.mov(regA[A], tmp); +} + +void JitCompiler::EmitSUBA() +{ + auto tmp = cc.newIntPtr(); + cc.mov(tmp, regA[B]); + cc.sub(tmp, regD[C]); + cc.mov(regA[A], tmp); +} + +void JitCompiler::EmitEQA_R() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + cc.cmp(regA[B], regA[C]); + if (check) cc.je(fail); + else cc.jne(fail); + }); +} + +void JitCompiler::EmitEQA_K() +{ + EmitComparisonOpcode([&](bool check, asmjit::Label& fail, asmjit::Label& success) { + auto tmp = cc.newIntPtr(); + cc.mov(tmp, ToMemAddress(konsta[C].v)); + cc.cmp(regA[B], tmp); + if (check) cc.je(fail); + else cc.jne(fail); + }); +} + +void JitCompiler::CallSqrt(const asmjit::X86Xmm &a, const asmjit::X86Xmm &b) +{ + using namespace asmjit; + typedef double(*FuncPtr)(double); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(g_sqrt))), FuncSignature1()); + call->setRet(0, a); + call->setArg(0, b); +} diff --git a/src/scripting/vm/jit_move.cpp b/src/scripting/vm/jit_move.cpp new file mode 100644 index 000000000..8d5e8cc3a --- /dev/null +++ b/src/scripting/vm/jit_move.cpp @@ -0,0 +1,201 @@ + +#include "jitintern.h" + +void JitCompiler::EmitMOVE() +{ + cc.mov(regD[A], regD[B]); +} +void JitCompiler::EmitMOVEF() +{ + cc.movsd(regF[A], regF[B]); +} + +void JitCompiler::EmitMOVES() +{ + I_FatalError("EmitMOVES not implemented\n"); +} + +void JitCompiler::EmitMOVEA() +{ + cc.mov(regA[A], regA[B]); +} + +void JitCompiler::EmitMOVEV2() +{ + cc.movsd(regF[A], regF[B]); + cc.movsd(regF[A + 1], regF[B + 1]); +} + +void JitCompiler::EmitMOVEV3() +{ + cc.movsd(regF[A], regF[B]); + cc.movsd(regF[A + 1], regF[B + 1]); + cc.movsd(regF[A + 2], regF[B + 2]); +} + +void JitCompiler::EmitCAST() +{ + switch (C) + { + case CAST_I2F: + cc.cvtsi2sd(regF[A], regD[B]); + break; + case CAST_U2F: + { + auto tmp = cc.newInt64(); + cc.xor_(tmp, tmp); + cc.mov(tmp.r32(), regD[B]); + cc.cvtsi2sd(regF[A], tmp); + break; + } + case CAST_F2I: + cc.cvttsd2si(regD[A], regF[B]); + break; + case CAST_F2U: + { + auto tmp = cc.newInt64(); + cc.cvttsd2si(tmp, regF[B]); + cc.mov(regD[A], tmp.r32()); + break; + } + /*case CAST_I2S: + reg.s[A].Format("%d", reg.d[B]); + break; + case CAST_U2S: + reg.s[A].Format("%u", reg.d[B]); + break; + case CAST_F2S: + reg.s[A].Format("%.5f", reg.f[B]); // keep this small. For more precise conversion there should be a conversion function. + break; + case CAST_V22S: + reg.s[A].Format("(%.5f, %.5f)", reg.f[B], reg.f[b + 1]); + break; + case CAST_V32S: + reg.s[A].Format("(%.5f, %.5f, %.5f)", reg.f[B], reg.f[b + 1], reg.f[b + 2]); + break; + case CAST_P2S: + { + if (reg.a[B] == nullptr) reg.s[A] = "null"; + else reg.s[A].Format("%p", reg.a[B]); + break; + } + case CAST_S2I: + reg.d[A] = (VM_SWORD)reg.s[B].ToLong(); + break; + case CAST_S2F: + reg.f[A] = reg.s[B].ToDouble(); + break; + case CAST_S2N: + reg.d[A] = reg.s[B].Len() == 0 ? FName(NAME_None) : FName(reg.s[B]); + break; + case CAST_N2S: + { + FName name = FName(ENamedName(reg.d[B])); + reg.s[A] = name.IsValidName() ? name.GetChars() : ""; + break; + } + case CAST_S2Co: + reg.d[A] = V_GetColor(NULL, reg.s[B]); + break; + case CAST_Co2S: + reg.s[A].Format("%02x %02x %02x", PalEntry(reg.d[B]).r, PalEntry(reg.d[B]).g, PalEntry(reg.d[B]).b); + break; + case CAST_S2So: + reg.d[A] = FSoundID(reg.s[B]); + break; + case CAST_So2S: + reg.s[A] = S_sfx[reg.d[B]].name; + break; + case CAST_SID2S: + reg.s[A] = unsigned(reg.d[B]) >= sprites.Size() ? "TNT1" : sprites[reg.d[B]].name; + break; + case CAST_TID2S: + { + auto tex = TexMan[*(FTextureID*)&(reg.d[B])]; + reg.s[A] = tex == nullptr ? "(null)" : tex->Name.GetChars(); + break; + }*/ + default: + assert(0); + } +} + +void JitCompiler::EmitCASTB() +{ + if (C == CASTB_I) + { + cc.mov(regD[A], regD[B]); + cc.shr(regD[A], 31); + } + else if (C == CASTB_F) + { + auto zero = cc.newXmm(); + auto one = cc.newInt32(); + cc.xorpd(zero, zero); + cc.mov(one, 1); + cc.ucomisd(regF[A], zero); + cc.setp(regD[A]); + cc.cmovne(regD[A], one); + } + else if (C == CASTB_A) + { + cc.test(regA[A], regA[A]); + cc.setne(regD[A]); + } + else + { + //reg.d[A] = reg.s[B].Len() > 0; + } +} + +void JitCompiler::EmitDYNCAST_R() +{ + using namespace asmjit; + typedef DObject*(*FuncPtr)(DObject*, PClass*); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](DObject *obj, PClass *cls) -> DObject* { + return (obj && obj->IsKindOf(cls)) ? obj : nullptr; + }))), FuncSignature2()); + call->setRet(0, regA[A]); + call->setArg(0, regA[B]); + call->setArg(1, regA[C]); +} + +void JitCompiler::EmitDYNCAST_K() +{ + using namespace asmjit; + auto c = cc.newIntPtr(); + cc.mov(c, ToMemAddress(konsta[C].o)); + typedef DObject*(*FuncPtr)(DObject*, PClass*); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](DObject *obj, PClass *cls) -> DObject* { + return (obj && obj->IsKindOf(cls)) ? obj : nullptr; + }))), FuncSignature2()); + call->setRet(0, regA[A]); + call->setArg(0, regA[B]); + call->setArg(1, c); +} + +void JitCompiler::EmitDYNCASTC_R() +{ + using namespace asmjit; + typedef PClass*(*FuncPtr)(PClass*, PClass*); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](PClass *cls1, PClass *cls2) -> PClass* { + return (cls1 && cls1->IsDescendantOf(cls2)) ? cls1 : nullptr; + }))), FuncSignature2()); + call->setRet(0, regA[A]); + call->setArg(0, regA[B]); + call->setArg(1, regA[C]); +} + +void JitCompiler::EmitDYNCASTC_K() +{ + using namespace asmjit; + auto c = cc.newIntPtr(); + cc.mov(c, ToMemAddress(konsta[C].o)); + typedef PClass*(*FuncPtr)(PClass*, PClass*); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast([](PClass *cls1, PClass *cls2) -> PClass* { + return (cls1 && cls1->IsDescendantOf(cls2)) ? cls1 : nullptr; + }))), FuncSignature2()); + call->setRet(0, regA[A]); + call->setArg(0, regA[B]); + call->setArg(1, c); +} diff --git a/src/scripting/vm/jit_store.cpp b/src/scripting/vm/jit_store.cpp new file mode 100644 index 000000000..4a3aa0e2a --- /dev/null +++ b/src/scripting/vm/jit_store.cpp @@ -0,0 +1,178 @@ + +#include "jitintern.h" + +void JitCompiler::EmitSB() +{ + EmitNullPointerThrow(A, X_WRITE_NIL); + cc.mov(asmjit::x86::byte_ptr(regA[A], konstd[C]), regD[B].r8Lo()); +} + +void JitCompiler::EmitSB_R() +{ + EmitNullPointerThrow(A, X_WRITE_NIL); + cc.mov(asmjit::x86::byte_ptr(regA[A], regD[C]), regD[B].r8Lo()); +} + +void JitCompiler::EmitSH() +{ + EmitNullPointerThrow(A, X_WRITE_NIL); + cc.mov(asmjit::x86::word_ptr(regA[A], konstd[C]), regD[B].r16()); +} + +void JitCompiler::EmitSH_R() +{ + EmitNullPointerThrow(A, X_WRITE_NIL); + cc.mov(asmjit::x86::word_ptr(regA[A], regD[C]), regD[B].r16()); +} + +void JitCompiler::EmitSW() +{ + EmitNullPointerThrow(A, X_WRITE_NIL); + cc.mov(asmjit::x86::dword_ptr(regA[A], konstd[C]), regD[B]); +} + +void JitCompiler::EmitSW_R() +{ + EmitNullPointerThrow(A, X_WRITE_NIL); + cc.mov(asmjit::x86::dword_ptr(regA[A], regD[C]), regD[B]); +} + +void JitCompiler::EmitSSP() +{ + EmitNullPointerThrow(A, X_WRITE_NIL); + cc.movss(asmjit::x86::dword_ptr(regA[A], konstd[C]), regF[B]); +} + +void JitCompiler::EmitSSP_R() +{ + EmitNullPointerThrow(A, X_WRITE_NIL); + cc.movss(asmjit::x86::dword_ptr(regA[A], regD[C]), regF[B]); +} + +void JitCompiler::EmitSDP() +{ + EmitNullPointerThrow(A, X_WRITE_NIL); + cc.movsd(asmjit::x86::qword_ptr(regA[A], konstd[C]), regF[B]); +} + +void JitCompiler::EmitSDP_R() +{ + EmitNullPointerThrow(A, X_WRITE_NIL); + cc.movsd(asmjit::x86::qword_ptr(regA[A], regD[C]), regF[B]); +} + +void JitCompiler::EmitSS() +{ + EmitNullPointerThrow(B, X_WRITE_NIL); + auto ptr = cc.newIntPtr(); + cc.mov(ptr, regA[A]); + cc.add(ptr, konstd[C]); + auto loadLambda = [](FString* to, FString* from) -> void { + *to = *from; + }; + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(loadLambda))), + asmjit::FuncSignature2(asmjit::CallConv::kIdHostCDecl)); + call->setArg(0, ptr); + call->setArg(1, regS[B]); +} + +void JitCompiler::EmitSS_R() +{ + EmitNullPointerThrow(B, X_WRITE_NIL); + auto ptr = cc.newIntPtr(); + cc.mov(ptr, regA[A]); + auto tmp = cc.newIntPtr(); + cc.mov(tmp, regD[C]); + cc.add(ptr, tmp); + auto loadLambda = [](FString* to, FString* from) -> void { + *to = *from; + }; + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(loadLambda))), + asmjit::FuncSignature2(asmjit::CallConv::kIdHostCDecl)); + call->setArg(0, ptr); + call->setArg(1, regS[B]); +} + +void JitCompiler::EmitSO() +{ + cc.mov(asmjit::x86::ptr(regA[A], konstd[C]), regA[B]); + + typedef void(*FuncPtr)(DObject*); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(GC::WriteBarrier))), asmjit::FuncSignature1()); + call->setArg(0, regA[B]); +} + +void JitCompiler::EmitSO_R() +{ + cc.mov(asmjit::x86::ptr(regA[A], regD[C]), regA[B]); + + typedef void(*FuncPtr)(DObject*); + auto call = cc.call(ToMemAddress(reinterpret_cast(static_cast(GC::WriteBarrier))), asmjit::FuncSignature1()); + call->setArg(0, regA[B]); +} + +void JitCompiler::EmitSP() +{ + cc.mov(asmjit::x86::ptr(regA[A], konstd[C]), regA[B]); +} + +void JitCompiler::EmitSP_R() +{ + cc.mov(asmjit::x86::ptr(regA[A], regD[C]), regA[B]); +} + +void JitCompiler::EmitSV2() +{ + EmitNullPointerThrow(B, X_WRITE_NIL); + auto tmp = cc.newIntPtr(); + cc.mov(tmp, regA[B]); + cc.add(tmp, konstd[C]); + cc.movsd(asmjit::x86::qword_ptr(tmp), regF[B]); + cc.movsd(asmjit::x86::qword_ptr(tmp, 8), regF[B + 1]); +} + +void JitCompiler::EmitSV2_R() +{ + EmitNullPointerThrow(B, X_WRITE_NIL); + auto tmp = cc.newIntPtr(); + cc.mov(tmp, regA[B]); + cc.add(tmp, regD[C]); + cc.movsd(asmjit::x86::qword_ptr(tmp), regF[B]); + cc.movsd(asmjit::x86::qword_ptr(tmp, 8), regF[B + 1]); +} + +void JitCompiler::EmitSV3() +{ + EmitNullPointerThrow(B, X_WRITE_NIL); + auto tmp = cc.newIntPtr(); + cc.mov(tmp, regA[B]); + cc.add(tmp, konstd[C]); + cc.movsd(asmjit::x86::qword_ptr(tmp), regF[B]); + cc.movsd(asmjit::x86::qword_ptr(tmp, 8), regF[B + 1]); + cc.movsd(asmjit::x86::qword_ptr(tmp, 16), regF[B + 2]); +} + +void JitCompiler::EmitSV3_R() +{ + EmitNullPointerThrow(B, X_WRITE_NIL); + auto tmp = cc.newIntPtr(); + cc.mov(tmp, regA[B]); + cc.add(tmp, regD[C]); + cc.movsd(asmjit::x86::qword_ptr(tmp), regF[B]); + cc.movsd(asmjit::x86::qword_ptr(tmp, 8), regF[B + 1]); + cc.movsd(asmjit::x86::qword_ptr(tmp, 16), regF[B + 2]); +} + +void JitCompiler::EmitSBIT() +{ + EmitNullPointerThrow(B, X_WRITE_NIL); + auto tmp1 = cc.newInt32(); + auto tmp2 = cc.newInt32(); + cc.mov(tmp1, asmjit::x86::byte_ptr(regA[A])); + cc.mov(tmp2, tmp1); + cc.or_(tmp1, (int)C); + cc.and_(tmp2, ~(int)C); + cc.test(regD[B], regD[B]); + cc.cmove(tmp1, tmp2); + cc.mov(asmjit::x86::byte_ptr(regA[A]), tmp1); +} diff --git a/src/scripting/vm/jitintern.h b/src/scripting/vm/jitintern.h new file mode 100644 index 000000000..b716843ef --- /dev/null +++ b/src/scripting/vm/jitintern.h @@ -0,0 +1,149 @@ + +#include "jit.h" +#include "i_system.h" +#include "types.h" +#include "stats.h" + +// To do: get cmake to define these.. +#define ASMJIT_BUILD_EMBED +#define ASMJIT_STATIC + +#include +#include +#include + +extern cycle_t VMCycles[10]; +extern int VMCalls[10]; + +#define A (pc[0].a) +#define B (pc[0].b) +#define C (pc[0].c) +#define Cs (pc[0].cs) +#define BC (pc[0].i16u) +#define BCs (pc[0].i16) +#define ABCs (pc[0].i24) +#define JMPOFS(x) ((x)->i24) + +static const char *OpNames[NUM_OPS] = +{ +#define xx(op, name, mode, alt, kreg, ktype) #op +#include "vmops.h" +#undef xx +}; + +class JitCompiler +{ +public: + JitCompiler(asmjit::CodeHolder *code, VMScriptFunction *sfunc) : cc(code), sfunc(sfunc) { } + + void Codegen(); + + static bool CanJit(VMScriptFunction *sfunc); + +private: + // Declare EmitXX functions for the opcodes: + #define xx(op, name, mode, alt, kreg, ktype) void Emit##op(); + #include "vmops.h" + #undef xx + + void EmitOpcode(); + void EmitDoCall(asmjit::X86Gp ptr); + void StoreInOuts(int b); + void LoadReturns(const VMOP *retval, int numret, bool inout); + void FillReturns(const VMOP *retval, int numret); + static int DoCall(VMFrameStack *stack, VMFunction *call, int b, int c, VMValue *param, VMReturn *returns, JitExceptionInfo *exceptinfo); + + void Setup(); + + template + void EmitComparisonOpcode(Func jmpFunc) + { + using namespace asmjit; + + int i = (int)(ptrdiff_t)(pc - sfunc->Code); + + auto successLabel = cc.newLabel(); + + auto failLabel = labels[i + 2 + JMPOFS(pc + 1)]; + + jmpFunc(static_cast(A & CMP_CHECK), failLabel, successLabel); + + cc.bind(successLabel); + pc++; // This instruction uses two instruction slots - skip the next one + } + + static int64_t ToMemAddress(const void *d) + { + return (int64_t)(ptrdiff_t)d; + } + + void CallSqrt(const asmjit::X86Xmm &a, const asmjit::X86Xmm &b); + + void EmitNullPointerThrow(int index, EVMAbortException reason); + void EmitThrowException(EVMAbortException reason); + void EmitThrowException(EVMAbortException reason, asmjit::X86Gp arg1); + + asmjit::X86Gp CheckRegD(int r0, int r1); + asmjit::X86Xmm CheckRegF(int r0, int r1); + asmjit::X86Xmm CheckRegF(int r0, int r1, int r2); + asmjit::X86Xmm CheckRegF(int r0, int r1, int r2, int r3); + asmjit::X86Gp CheckRegA(int r0, int r1); + + asmjit::X86Compiler cc; + VMScriptFunction *sfunc; + + asmjit::X86Gp stack; + asmjit::X86Gp vmregs; + asmjit::X86Gp ret; + asmjit::X86Gp numret; + asmjit::X86Gp exceptInfo; + + asmjit::X86Gp frameD; + asmjit::X86Gp frameF; + asmjit::X86Gp frameS; + asmjit::X86Gp frameA; + asmjit::X86Gp params; + int NumParam = 0; // Actually part of vmframe (f->NumParam), but nobody seems to read that? + TArray ParamOpcodes; + asmjit::X86Gp callReturns; + + const int *konstd; + const double *konstf; + const FString *konsts; + const FVoidObj *konsta; + + TArray regD; + TArray regF; + TArray regA; + TArray regS; + + TArray labels; + + const VMOP *pc; + VM_UBYTE op; +}; + +class AsmJitException : public std::exception +{ +public: + AsmJitException(asmjit::Error error, const char *message) noexcept : error(error), message(message) + { + } + + const char* what() const noexcept override + { + return message.GetChars(); + } + + asmjit::Error error; + FString message; +}; + +class ThrowingErrorHandler : public asmjit::ErrorHandler +{ +public: + bool handleError(asmjit::Error err, const char *message, asmjit::CodeEmitter *origin) override + { + throw AsmJitException(err, message); + } +}; diff --git a/src/scripting/vm/vmexec.h b/src/scripting/vm/vmexec.h index e2ec3ead3..1b6c49480 100644 --- a/src/scripting/vm/vmexec.h +++ b/src/scripting/vm/vmexec.h @@ -42,7 +42,7 @@ static int Exec(VMFrameStack *stack, const VMOP *pc, VMReturn *ret, int numret) #if COMPGOTO static const void * const ops[256] = { -#define xx(op,sym,mode,alt,kreg,ktype) &&op +#define xx(op,sym,mode,alt,kreg,ktype) &&op, #include "vmops.h" }; #endif diff --git a/src/scripting/vm/vmops.h b/src/scripting/vm/vmops.h index cfdf3fce2..afa0dbfa9 100644 --- a/src/scripting/vm/vmops.h +++ b/src/scripting/vm/vmops.h @@ -1,5 +1,5 @@ #ifndef xx -#define xx(op, name, mode, alt, kreg, ktype) OP_##op +#define xx(op, name, mode, alt, kreg, ktype) OP_##op, #endif // first row is the opcode @@ -10,249 +10,249 @@ // sixth row is the constant register type. // OP_PARAM and OP_CMPS need special treatment because they encode this information in the instruction. -xx(NOP, nop, NOP, NOP, 0, 0), // no operation +xx(NOP, nop, NOP, NOP, 0, 0) // no operation // Load constants. -xx(LI, li, LI, NOP, 0, 0), // load immediate signed 16-bit constant -xx(LK, lk, LKI, NOP, 0, 0), // load integer constant -xx(LKF, lk, LKF, NOP, 0, 0), // load float constant -xx(LKS, lk, LKS, NOP, 0, 0), // load string constant -xx(LKP, lk, LKP, NOP, 0, 0), // load pointer constant -xx(LK_R, lk, RIRII8, NOP, 0, 0), // load integer constant indexed -xx(LKF_R, lk, RFRII8, NOP, 0, 0), // load float constant indexed -xx(LKS_R, lk, RSRII8, NOP, 0, 0), // load string constant indexed -xx(LKP_R, lk, RPRII8, NOP, 0, 0), // load pointer constant indexed -xx(LFP, lf, LFP, NOP, 0, 0), // load frame pointer -xx(META, meta, RPRP, NOP, 0, 0), // load a class's meta data address -xx(CLSS, clss, RPRP, NOP, 0, 0), // load a class's descriptor address +xx(LI, li, LI, NOP, 0, 0) // load immediate signed 16-bit constant +xx(LK, lk, LKI, NOP, 0, 0) // load integer constant +xx(LKF, lk, LKF, NOP, 0, 0) // load float constant +xx(LKS, lk, LKS, NOP, 0, 0) // load string constant +xx(LKP, lk, LKP, NOP, 0, 0) // load pointer constant +xx(LK_R, lk, RIRII8, NOP, 0, 0) // load integer constant indexed +xx(LKF_R, lk, RFRII8, NOP, 0, 0) // load float constant indexed +xx(LKS_R, lk, RSRII8, NOP, 0, 0) // load string constant indexed +xx(LKP_R, lk, RPRII8, NOP, 0, 0) // load pointer constant indexed +xx(LFP, lf, LFP, NOP, 0, 0) // load frame pointer +xx(META, meta, RPRP, NOP, 0, 0) // load a class's meta data address +xx(CLSS, clss, RPRP, NOP, 0, 0) // load a class's descriptor address // Load from memory. rA = *(rB + rkC) -xx(LB, lb, RIRPKI, LB_R, 4, REGT_INT), // load byte -xx(LB_R, lb, RIRPRI, NOP, 0, 0), -xx(LH, lh, RIRPKI, LH_R, 4, REGT_INT), // load halfword -xx(LH_R, lh, RIRPRI, NOP, 0, 0), -xx(LW, lw, RIRPKI, LW_R, 4, REGT_INT), // load word -xx(LW_R, lw, RIRPRI, NOP, 0, 0), -xx(LBU, lbu, RIRPKI, LBU_R, 4, REGT_INT), // load byte unsigned -xx(LBU_R, lbu, RIRPRI, NOP, 0, 0), -xx(LHU, lhu, RIRPKI, LHU_R, 4, REGT_INT), // load halfword unsigned -xx(LHU_R, lhu, RIRPRI, NOP, 0, 0), -xx(LSP, lsp, RFRPKI, LSP_R, 4, REGT_INT), // load single-precision fp -xx(LSP_R, lsp, RFRPRI, NOP, 0, 0), -xx(LDP, ldp, RFRPKI, LDP_R, 4, REGT_INT), // load double-precision fp -xx(LDP_R, ldp, RFRPRI, NOP, 0, 0), -xx(LS, ls, RSRPKI, LS_R, 4, REGT_INT), // load string -xx(LS_R, ls, RSRPRI, NOP, 0, 0), -xx(LO, lo, RPRPKI, LO_R, 4, REGT_INT), // load object -xx(LO_R, lo, RPRPRI, NOP, 0, 0), -xx(LP, lp, RPRPKI, LP_R, 4, REGT_INT), // load pointer -xx(LP_R, lp, RPRPRI, NOP, 0, 0), -xx(LV2, lv2, RVRPKI, LV2_R, 4, REGT_INT), // load vector2 -xx(LV2_R, lv2, RVRPRI, NOP, 0, 0), -xx(LV3, lv3, RVRPKI, LV3_R, 4, REGT_INT), // load vector3 -xx(LV3_R, lv3, RVRPRI, NOP, 0, 0), -xx(LCS, lcs, RSRPKI, LCS_R, 4, REGT_INT), // load string from char ptr. -xx(LCS_R, lcs, RSRPRI, NOP, 0, 0), +xx(LB, lb, RIRPKI, LB_R, 4, REGT_INT) // load byte +xx(LB_R, lb, RIRPRI, NOP, 0, 0) +xx(LH, lh, RIRPKI, LH_R, 4, REGT_INT) // load halfword +xx(LH_R, lh, RIRPRI, NOP, 0, 0) +xx(LW, lw, RIRPKI, LW_R, 4, REGT_INT) // load word +xx(LW_R, lw, RIRPRI, NOP, 0, 0) +xx(LBU, lbu, RIRPKI, LBU_R, 4, REGT_INT) // load byte unsigned +xx(LBU_R, lbu, RIRPRI, NOP, 0, 0) +xx(LHU, lhu, RIRPKI, LHU_R, 4, REGT_INT) // load halfword unsigned +xx(LHU_R, lhu, RIRPRI, NOP, 0, 0) +xx(LSP, lsp, RFRPKI, LSP_R, 4, REGT_INT) // load single-precision fp +xx(LSP_R, lsp, RFRPRI, NOP, 0, 0) +xx(LDP, ldp, RFRPKI, LDP_R, 4, REGT_INT) // load double-precision fp +xx(LDP_R, ldp, RFRPRI, NOP, 0, 0) +xx(LS, ls, RSRPKI, LS_R, 4, REGT_INT) // load string +xx(LS_R, ls, RSRPRI, NOP, 0, 0) +xx(LO, lo, RPRPKI, LO_R, 4, REGT_INT) // load object +xx(LO_R, lo, RPRPRI, NOP, 0, 0) +xx(LP, lp, RPRPKI, LP_R, 4, REGT_INT) // load pointer +xx(LP_R, lp, RPRPRI, NOP, 0, 0) +xx(LV2, lv2, RVRPKI, LV2_R, 4, REGT_INT) // load vector2 +xx(LV2_R, lv2, RVRPRI, NOP, 0, 0) +xx(LV3, lv3, RVRPKI, LV3_R, 4, REGT_INT) // load vector3 +xx(LV3_R, lv3, RVRPRI, NOP, 0, 0) +xx(LCS, lcs, RSRPKI, LCS_R, 4, REGT_INT) // load string from char ptr. +xx(LCS_R, lcs, RSRPRI, NOP, 0, 0) -xx(LBIT, lbit, RIRPI8, NOP, 0, 0), // rA = !!(*rB & C) -- *rB is a byte +xx(LBIT, lbit, RIRPI8, NOP, 0, 0) // rA = !!(*rB & C) -- *rB is a byte // Store instructions. *(rA + rkC) = rB -xx(SB, sb, RPRIKI, SB_R, 4, REGT_INT), // store byte -xx(SB_R, sb, RPRIRI, NOP, 0, 0), -xx(SH, sh, RPRIKI, SH_R, 4, REGT_INT), // store halfword -xx(SH_R, sh, RPRIRI, NOP, 0, 0), -xx(SW, sw, RPRIKI, SW_R, 4, REGT_INT), // store word -xx(SW_R, sw, RPRIRI, NOP, 0, 0), -xx(SSP, ssp, RPRFKI, SSP_R, 4, REGT_INT), // store single-precision fp -xx(SSP_R, ssp, RPRFRI, NOP, 0, 0), -xx(SDP, sdp, RPRFKI, SDP_R, 4, REGT_INT), // store double-precision fp -xx(SDP_R, sdp, RPRFRI, NOP, 0, 0), -xx(SS, ss, RPRSKI, SS_R, 4, REGT_INT), // store string -xx(SS_R, ss, RPRSRI, NOP, 0, 0), -xx(SP, sp, RPRPKI, SP_R, 4, REGT_INT), // store pointer -xx(SP_R, sp, RPRPRI, NOP, 0, 0), -xx(SO, so, RPRPKI, SO_R, 4, REGT_INT), // store object pointer with write barrier (only needed for non thinkers and non types) -xx(SO_R, so, RPRPRI, NOP, 0, 0), -xx(SV2, sv2, RPRVKI, SV2_R, 4, REGT_INT), // store vector2 -xx(SV2_R, sv2, RPRVRI, NOP, 0, 0), -xx(SV3, sv3, RPRVKI, SV3_R, 4, REGT_INT), // store vector3 -xx(SV3_R, sv3, RPRVRI, NOP, 0, 0), +xx(SB, sb, RPRIKI, SB_R, 4, REGT_INT) // store byte +xx(SB_R, sb, RPRIRI, NOP, 0, 0) +xx(SH, sh, RPRIKI, SH_R, 4, REGT_INT) // store halfword +xx(SH_R, sh, RPRIRI, NOP, 0, 0) +xx(SW, sw, RPRIKI, SW_R, 4, REGT_INT) // store word +xx(SW_R, sw, RPRIRI, NOP, 0, 0) +xx(SSP, ssp, RPRFKI, SSP_R, 4, REGT_INT) // store single-precision fp +xx(SSP_R, ssp, RPRFRI, NOP, 0, 0) +xx(SDP, sdp, RPRFKI, SDP_R, 4, REGT_INT) // store double-precision fp +xx(SDP_R, sdp, RPRFRI, NOP, 0, 0) +xx(SS, ss, RPRSKI, SS_R, 4, REGT_INT) // store string +xx(SS_R, ss, RPRSRI, NOP, 0, 0) +xx(SP, sp, RPRPKI, SP_R, 4, REGT_INT) // store pointer +xx(SP_R, sp, RPRPRI, NOP, 0, 0) +xx(SO, so, RPRPKI, SO_R, 4, REGT_INT) // store object pointer with write barrier (only needed for non thinkers and non types) +xx(SO_R, so, RPRPRI, NOP, 0, 0) +xx(SV2, sv2, RPRVKI, SV2_R, 4, REGT_INT) // store vector2 +xx(SV2_R, sv2, RPRVRI, NOP, 0, 0) +xx(SV3, sv3, RPRVKI, SV3_R, 4, REGT_INT) // store vector3 +xx(SV3_R, sv3, RPRVRI, NOP, 0, 0) -xx(SBIT, sbit, RPRII8, NOP, 0, 0), // *rA |= C if rB is true, *rA &= ~C otherwise +xx(SBIT, sbit, RPRII8, NOP, 0, 0) // *rA |= C if rB is true, *rA &= ~C otherwise // Move instructions. -xx(MOVE, mov, RIRI, NOP, 0, 0), // dA = dB -xx(MOVEF, mov, RFRF, NOP, 0, 0), // fA = fB -xx(MOVES, mov, RSRS, NOP, 0, 0), // sA = sB -xx(MOVEA, mov, RPRP, NOP, 0, 0), // aA = aB -xx(MOVEV2, mov2, RFRF, NOP, 0, 0), // fA = fB (2 elements) -xx(MOVEV3, mov3, RFRF, NOP, 0, 0), // fA = fB (3 elements) -xx(CAST, cast, CAST, NOP, 0, 0), // xA = xB, conversion specified by C -xx(CASTB, castb, CAST, NOP, 0, 0), // xA = !!xB, type specified by C -xx(DYNCAST_R, dyncast, RPRPRP, NOP, 0, 0), // aA = dyn_cast(aB); -xx(DYNCAST_K, dyncast, RPRPKP, NOP, 0, 0), // aA = dyn_cast(aB); -xx(DYNCASTC_R, dyncastc, RPRPRP, NOP, 0, 0), // aA = dyn_cast(aB); for class types -xx(DYNCASTC_K, dyncastc, RPRPKP, NOP, 0, 0), // aA = dyn_cast(aB); +xx(MOVE, mov, RIRI, NOP, 0, 0) // dA = dB +xx(MOVEF, mov, RFRF, NOP, 0, 0) // fA = fB +xx(MOVES, mov, RSRS, NOP, 0, 0) // sA = sB +xx(MOVEA, mov, RPRP, NOP, 0, 0) // aA = aB +xx(MOVEV2, mov2, RFRF, NOP, 0, 0) // fA = fB (2 elements) +xx(MOVEV3, mov3, RFRF, NOP, 0, 0) // fA = fB (3 elements) +xx(CAST, cast, CAST, NOP, 0, 0) // xA = xB, conversion specified by C +xx(CASTB, castb, CAST, NOP, 0, 0) // xA = !!xB, type specified by C +xx(DYNCAST_R, dyncast, RPRPRP, NOP, 0, 0) // aA = dyn_cast(aB); +xx(DYNCAST_K, dyncast, RPRPKP, NOP, 0, 0) // aA = dyn_cast(aB); +xx(DYNCASTC_R, dyncastc, RPRPRP, NOP, 0, 0) // aA = dyn_cast(aB); for class types +xx(DYNCASTC_K, dyncastc, RPRPKP, NOP, 0, 0) // aA = dyn_cast(aB); // Control flow. -xx(TEST, test, RII16, NOP, 0, 0), // if (dA != BC) then pc++ -xx(TESTN, testn, RII16, NOP, 0, 0), // if (dA != -BC) then pc++ -xx(JMP, jmp, I24, NOP, 0, 0), // pc += ABC -- The ABC fields contain a signed 24-bit offset. -xx(IJMP, ijmp, RII16, NOP, 0, 0), // pc += dA + BC -- BC is a signed offset. The target instruction must be a JMP. -xx(PARAM, param, __BCP, NOP, 0, 0), // push parameter encoded in BC for function call (B=regtype, C=regnum) -xx(PARAMI, parami, I24, NOP, 0, 0), // push immediate, signed integer for function call -xx(CALL, call, RPI8I8, NOP, 0, 0), // Call function pkA with parameter count B and expected result count C -xx(CALL_K, call, KPI8I8, CALL, 1, REGT_POINTER), -xx(VTBL, vtbl, RPRPI8, NOP, 0, 0), // dereferences a virtual method table. -xx(SCOPE, scope, RPI8, NOP, 0, 0), // Scope check at runtime. -xx(TAIL, tail, RPI8, NOP, 0, 0), // Call+Ret in a single instruction -xx(TAIL_K, tail, KPI8, TAIL, 1, REGT_POINTER), -xx(RESULT, result, __BCP, NOP, 0, 0), // Result should go in register encoded in BC (in caller, after CALL) -xx(RET, ret, I8BCP, NOP, 0, 0), // Copy value from register encoded in BC to return value A, possibly returning -xx(RETI, reti, I8I16, NOP, 0, 0), // Copy immediate from BC to return value A, possibly returning -xx(NEW, new, RPRPI8, NOP, 0, 0), -xx(NEW_K, new, RPKP, NOP, 0, 0), -//xx(TRY, try, I24, NOP, 0, 0), // When an exception is thrown, start searching for a handler at pc + ABC -//xx(UNTRY, untry, I8, NOP, 0, 0), // Pop A entries off the exception stack -xx(THROW, throw, THROW, NOP, 0, 0), // A == 0: Throw exception object pB +xx(TEST, test, RII16, NOP, 0, 0) // if (dA != BC) then pc++ +xx(TESTN, testn, RII16, NOP, 0, 0) // if (dA != -BC) then pc++ +xx(JMP, jmp, I24, NOP, 0, 0) // pc += ABC -- The ABC fields contain a signed 24-bit offset. +xx(IJMP, ijmp, RII16, NOP, 0, 0) // pc += dA + BC -- BC is a signed offset. The target instruction must be a JMP. +xx(PARAM, param, __BCP, NOP, 0, 0) // push parameter encoded in BC for function call (B=regtype, C=regnum) +xx(PARAMI, parami, I24, NOP, 0, 0) // push immediate, signed integer for function call +xx(CALL, call, RPI8I8, NOP, 0, 0) // Call function pkA with parameter count B and expected result count C +xx(CALL_K, call, KPI8I8, CALL, 1, REGT_POINTER) +xx(VTBL, vtbl, RPRPI8, NOP, 0, 0) // dereferences a virtual method table. +xx(SCOPE, scope, RPI8, NOP, 0, 0) // Scope check at runtime. +xx(TAIL, tail, RPI8, NOP, 0, 0) // Call+Ret in a single instruction +xx(TAIL_K, tail, KPI8, TAIL, 1, REGT_POINTER) +xx(RESULT, result, __BCP, NOP, 0, 0) // Result should go in register encoded in BC (in caller, after CALL) +xx(RET, ret, I8BCP, NOP, 0, 0) // Copy value from register encoded in BC to return value A, possibly returning +xx(RETI, reti, I8I16, NOP, 0, 0) // Copy immediate from BC to return value A, possibly returning +xx(NEW, new, RPRPI8, NOP, 0, 0) +xx(NEW_K, new, RPKP, NOP, 0, 0) +//xx(TRY, try, I24, NOP, 0, 0) // When an exception is thrown, start searching for a handler at pc + ABC +//xx(UNTRY, untry, I8, NOP, 0, 0) // Pop A entries off the exception stack +xx(THROW, throw, THROW, NOP, 0, 0) // A == 0: Throw exception object pB // A == 1: Throw exception object pkB // A >= 2: Throw VM exception of type BC -//xx(CATCH, catch, CATCH, NOP, 0, 0), // A == 0: continue search on next try +//xx(CATCH, catch, CATCH, NOP, 0, 0) // A == 0: continue search on next try // A == 1: continue execution at instruction immediately following CATCH (catches any exception) // A == 2: (pB == ) then pc++ ; next instruction must JMP to another CATCH // A == 3: (pkB == ) then pc++ ; next instruction must JMP to another CATCH // for A > 0, exception is stored in pC -xx(BOUND, bound, RII16, NOP, 0, 0), // if rA < 0 or rA >= BC, throw exception -xx(BOUND_K, bound, LKI, NOP, 0, 0), // if rA < 0 or rA >= const[BC], throw exception -xx(BOUND_R, bound, RIRI, NOP, 0, 0), // if rA < 0 or rA >= rB, throw exception +xx(BOUND, bound, RII16, NOP, 0, 0) // if rA < 0 or rA >= BC, throw exception +xx(BOUND_K, bound, LKI, NOP, 0, 0) // if rA < 0 or rA >= const[BC], throw exception +xx(BOUND_R, bound, RIRI, NOP, 0, 0) // if rA < 0 or rA >= rB, throw exception // String instructions. -xx(CONCAT, concat, RSRSRS, NOP, 0, 0), // sA = sB..sC -xx(LENS, lens, RIRS, NOP, 0, 0), // dA = sB.Length -xx(CMPS, cmps, I8RXRX, NOP, 0, 0), // if ((skB op skC) != (A & 1)) then pc++ +xx(CONCAT, concat, RSRSRS, NOP, 0, 0) // sA = sB..sC +xx(LENS, lens, RIRS, NOP, 0, 0) // dA = sB.Length +xx(CMPS, cmps, I8RXRX, NOP, 0, 0) // if ((skB op skC) != (A & 1)) then pc++ // Integer math. -xx(SLL_RR, sll, RIRIRI, NOP, 0, 0), // dA = dkB << diC -xx(SLL_RI, sll, RIRII8, NOP, 0, 0), -xx(SLL_KR, sll, RIKIRI, SLL_RR, 2, REGT_INT), -xx(SRL_RR, srl, RIRIRI, NOP, 0, 0), // dA = dkB >> diC -- unsigned -xx(SRL_RI, srl, RIRII8, NOP, 0, 0), -xx(SRL_KR, srl, RIKIRI, SRL_RR, 2, REGT_INT), -xx(SRA_RR, sra, RIRIRI, NOP, 0, 0), // dA = dkB >> diC -- signed -xx(SRA_RI, sra, RIRII8, NOP, 0, 0), -xx(SRA_KR, sra, RIKIRI, SRA_RR, 2, REGT_INT), -xx(ADD_RR, add, RIRIRI, NOP, 0, 0), // dA = dB + dkC -xx(ADD_RK, add, RIRIKI, ADD_RR, 4, REGT_INT), -xx(ADDI, addi, RIRIIs, NOP, 0, 0), // dA = dB + C -- C is a signed 8-bit constant -xx(SUB_RR, sub, RIRIRI, NOP, 0, 0), // dA = dkB - dkC -xx(SUB_RK, sub, RIRIKI, SUB_RR, 4, REGT_INT), -xx(SUB_KR, sub, RIKIRI, SUB_RR, 2, REGT_INT), -xx(MUL_RR, mul, RIRIRI, NOP, 0, 0), // dA = dB * dkC -xx(MUL_RK, mul, RIRIKI, MUL_RR, 4, REGT_INT), -xx(DIV_RR, div, RIRIRI, NOP, 0, 0), // dA = dkB / dkC (signed) -xx(DIV_RK, div, RIRIKI, DIV_RR, 4, REGT_INT), -xx(DIV_KR, div, RIKIRI, DIV_RR, 2, REGT_INT), -xx(DIVU_RR, divu, RIRIRI, NOP, 0, 0), // dA = dkB / dkC (unsigned) -xx(DIVU_RK, divu, RIRIKI, DIVU_RR,4, REGT_INT), -xx(DIVU_KR, divu, RIKIRI, DIVU_RR,2, REGT_INT), -xx(MOD_RR, mod, RIRIRI, NOP, 0, 0), // dA = dkB % dkC (signed) -xx(MOD_RK, mod, RIRIKI, MOD_RR, 4, REGT_INT), -xx(MOD_KR, mod, RIKIRI, MOD_RR, 2, REGT_INT), -xx(MODU_RR, modu, RIRIRI, NOP, 0, 0), // dA = dkB % dkC (unsigned) -xx(MODU_RK, modu, RIRIKI, MODU_RR,4, REGT_INT), -xx(MODU_KR, modu, RIKIRI, MODU_RR,2, REGT_INT), -xx(AND_RR, and, RIRIRI, NOP, 0, 0), // dA = dB & dkC -xx(AND_RK, and, RIRIKI, AND_RR, 4, REGT_INT), -xx(OR_RR, or, RIRIRI, NOP, 0, 0), // dA = dB | dkC -xx(OR_RK, or, RIRIKI, OR_RR, 4, REGT_INT), -xx(XOR_RR, xor, RIRIRI, NOP, 0, 0), // dA = dB ^ dkC -xx(XOR_RK, xor, RIRIKI, XOR_RR, 4, REGT_INT), -xx(MIN_RR, min, RIRIRI, NOP, 0, 0), // dA = min(dB,dkC) -xx(MIN_RK, min, RIRIKI, MIN_RR, 4, REGT_INT), -xx(MAX_RR, max, RIRIRI, NOP, 0, 0), // dA = max(dB,dkC) -xx(MAX_RK, max, RIRIKI, MAX_RR, 4, REGT_INT), -xx(ABS, abs, RIRI, NOP, 0, 0), // dA = abs(dB) -xx(NEG, neg, RIRI, NOP, 0, 0), // dA = -dB -xx(NOT, not, RIRI, NOP, 0, 0), // dA = ~dB -xx(EQ_R, beq, CIRR, NOP, 0, 0), // if ((dB == dkC) != A) then pc++ -xx(EQ_K, beq, CIRK, EQ_R, 4, REGT_INT), -xx(LT_RR, blt, CIRR, NOP, 0, 0), // if ((dkB < dkC) != A) then pc++ -xx(LT_RK, blt, CIRK, LT_RR, 4, REGT_INT), -xx(LT_KR, blt, CIKR, LT_RR, 2, REGT_INT), -xx(LE_RR, ble, CIRR, NOP, 0, 0), // if ((dkB <= dkC) != A) then pc++ -xx(LE_RK, ble, CIRK, LE_RR, 4, REGT_INT), -xx(LE_KR, ble, CIKR, LE_RR, 2, REGT_INT), -xx(LTU_RR, bltu, CIRR, NOP, 0, 0), // if ((dkB < dkC) != A) then pc++ -- unsigned -xx(LTU_RK, bltu, CIRK, LTU_RR, 4, REGT_INT), -xx(LTU_KR, bltu, CIKR, LTU_RR, 2, REGT_INT), -xx(LEU_RR, bleu, CIRR, NOP, 0, 0), // if ((dkB <= dkC) != A) then pc++ -- unsigned -xx(LEU_RK, bleu, CIRK, LEU_RR, 4, REGT_INT), -xx(LEU_KR, bleu, CIKR, LEU_RR, 2, REGT_INT), +xx(SLL_RR, sll, RIRIRI, NOP, 0, 0) // dA = dkB << diC +xx(SLL_RI, sll, RIRII8, NOP, 0, 0) +xx(SLL_KR, sll, RIKIRI, SLL_RR, 2, REGT_INT) +xx(SRL_RR, srl, RIRIRI, NOP, 0, 0) // dA = dkB >> diC -- unsigned +xx(SRL_RI, srl, RIRII8, NOP, 0, 0) +xx(SRL_KR, srl, RIKIRI, SRL_RR, 2, REGT_INT) +xx(SRA_RR, sra, RIRIRI, NOP, 0, 0) // dA = dkB >> diC -- signed +xx(SRA_RI, sra, RIRII8, NOP, 0, 0) +xx(SRA_KR, sra, RIKIRI, SRA_RR, 2, REGT_INT) +xx(ADD_RR, add, RIRIRI, NOP, 0, 0) // dA = dB + dkC +xx(ADD_RK, add, RIRIKI, ADD_RR, 4, REGT_INT) +xx(ADDI, addi, RIRIIs, NOP, 0, 0) // dA = dB + C -- C is a signed 8-bit constant +xx(SUB_RR, sub, RIRIRI, NOP, 0, 0) // dA = dkB - dkC +xx(SUB_RK, sub, RIRIKI, SUB_RR, 4, REGT_INT) +xx(SUB_KR, sub, RIKIRI, SUB_RR, 2, REGT_INT) +xx(MUL_RR, mul, RIRIRI, NOP, 0, 0) // dA = dB * dkC +xx(MUL_RK, mul, RIRIKI, MUL_RR, 4, REGT_INT) +xx(DIV_RR, div, RIRIRI, NOP, 0, 0) // dA = dkB / dkC (signed) +xx(DIV_RK, div, RIRIKI, DIV_RR, 4, REGT_INT) +xx(DIV_KR, div, RIKIRI, DIV_RR, 2, REGT_INT) +xx(DIVU_RR, divu, RIRIRI, NOP, 0, 0) // dA = dkB / dkC (unsigned) +xx(DIVU_RK, divu, RIRIKI, DIVU_RR,4, REGT_INT) +xx(DIVU_KR, divu, RIKIRI, DIVU_RR,2, REGT_INT) +xx(MOD_RR, mod, RIRIRI, NOP, 0, 0) // dA = dkB % dkC (signed) +xx(MOD_RK, mod, RIRIKI, MOD_RR, 4, REGT_INT) +xx(MOD_KR, mod, RIKIRI, MOD_RR, 2, REGT_INT) +xx(MODU_RR, modu, RIRIRI, NOP, 0, 0) // dA = dkB % dkC (unsigned) +xx(MODU_RK, modu, RIRIKI, MODU_RR,4, REGT_INT) +xx(MODU_KR, modu, RIKIRI, MODU_RR,2, REGT_INT) +xx(AND_RR, and, RIRIRI, NOP, 0, 0) // dA = dB & dkC +xx(AND_RK, and, RIRIKI, AND_RR, 4, REGT_INT) +xx(OR_RR, or, RIRIRI, NOP, 0, 0) // dA = dB | dkC +xx(OR_RK, or, RIRIKI, OR_RR, 4, REGT_INT) +xx(XOR_RR, xor, RIRIRI, NOP, 0, 0) // dA = dB ^ dkC +xx(XOR_RK, xor, RIRIKI, XOR_RR, 4, REGT_INT) +xx(MIN_RR, min, RIRIRI, NOP, 0, 0) // dA = min(dB,dkC) +xx(MIN_RK, min, RIRIKI, MIN_RR, 4, REGT_INT) +xx(MAX_RR, max, RIRIRI, NOP, 0, 0) // dA = max(dB,dkC) +xx(MAX_RK, max, RIRIKI, MAX_RR, 4, REGT_INT) +xx(ABS, abs, RIRI, NOP, 0, 0) // dA = abs(dB) +xx(NEG, neg, RIRI, NOP, 0, 0) // dA = -dB +xx(NOT, not, RIRI, NOP, 0, 0) // dA = ~dB +xx(EQ_R, beq, CIRR, NOP, 0, 0) // if ((dB == dkC) != A) then pc++ +xx(EQ_K, beq, CIRK, EQ_R, 4, REGT_INT) +xx(LT_RR, blt, CIRR, NOP, 0, 0) // if ((dkB < dkC) != A) then pc++ +xx(LT_RK, blt, CIRK, LT_RR, 4, REGT_INT) +xx(LT_KR, blt, CIKR, LT_RR, 2, REGT_INT) +xx(LE_RR, ble, CIRR, NOP, 0, 0) // if ((dkB <= dkC) != A) then pc++ +xx(LE_RK, ble, CIRK, LE_RR, 4, REGT_INT) +xx(LE_KR, ble, CIKR, LE_RR, 2, REGT_INT) +xx(LTU_RR, bltu, CIRR, NOP, 0, 0) // if ((dkB < dkC) != A) then pc++ -- unsigned +xx(LTU_RK, bltu, CIRK, LTU_RR, 4, REGT_INT) +xx(LTU_KR, bltu, CIKR, LTU_RR, 2, REGT_INT) +xx(LEU_RR, bleu, CIRR, NOP, 0, 0) // if ((dkB <= dkC) != A) then pc++ -- unsigned +xx(LEU_RK, bleu, CIRK, LEU_RR, 4, REGT_INT) +xx(LEU_KR, bleu, CIKR, LEU_RR, 2, REGT_INT) // Double-precision floating point math. -xx(ADDF_RR, add, RFRFRF, NOP, 0, 0), // fA = fB + fkC -xx(ADDF_RK, add, RFRFKF, ADDF_RR,4, REGT_FLOAT), -xx(SUBF_RR, sub, RFRFRF, NOP, 0, 0), // fA = fkB - fkC -xx(SUBF_RK, sub, RFRFKF, SUBF_RR,4, REGT_FLOAT), -xx(SUBF_KR, sub, RFKFRF, SUBF_RR,2, REGT_FLOAT), -xx(MULF_RR, mul, RFRFRF, NOP, 0, 0), // fA = fB * fkC -xx(MULF_RK, mul, RFRFKF, MULF_RR,4, REGT_FLOAT), -xx(DIVF_RR, div, RFRFRF, NOP, 0, 0), // fA = fkB / fkC -xx(DIVF_RK, div, RFRFKF, DIVF_RR,4, REGT_FLOAT), -xx(DIVF_KR, div, RFKFRF, DIVF_RR,2, REGT_FLOAT), -xx(MODF_RR, mod, RFRFRF, NOP, 0, 0), // fA = fkB % fkC -xx(MODF_RK, mod, RFRFKF, MODF_RR,4, REGT_FLOAT), -xx(MODF_KR, mod, RFKFRF, MODF_RR,4, REGT_FLOAT), -xx(POWF_RR, pow, RFRFRF, NOP, 0, 0), // fA = fkB ** fkC -xx(POWF_RK, pow, RFRFKF, POWF_RR,4, REGT_FLOAT), -xx(POWF_KR, pow, RFKFRF, POWF_RR,2, REGT_FLOAT), -xx(MINF_RR, min, RFRFRF, NOP, 0, 0), // fA = min(fB),fkC) -xx(MINF_RK, min, RFRFKF, MINF_RR,4, REGT_FLOAT), -xx(MAXF_RR, max, RFRFRF, NOP, 0, 0), // fA = max(fB),fkC) -xx(MAXF_RK, max, RFRFKF, MAXF_RR,4, REGT_FLOAT), -xx(ATAN2, atan2, RFRFRF, NOP, 0, 0), // fA = atan2(fB,fC), result is in degrees -xx(FLOP, flop, RFRFI8, NOP, 0, 0), // fA = f(fB), where function is selected by C -xx(EQF_R, beq, CFRR, NOP, 0, 0), // if ((fB == fkC) != (A & 1)) then pc++ -xx(EQF_K, beq, CFRK, EQF_R, 4, REGT_FLOAT), -xx(LTF_RR, blt, CFRR, NOP, 0, 0), // if ((fkB < fkC) != (A & 1)) then pc++ -xx(LTF_RK, blt, CFRK, LTF_RR, 4, REGT_FLOAT), -xx(LTF_KR, blt, CFKR, LTF_RR, 2, REGT_FLOAT), -xx(LEF_RR, ble, CFRR, NOP, 0, 0), // if ((fkb <= fkC) != (A & 1)) then pc++ -xx(LEF_RK, ble, CFRK, LEF_RR, 4, REGT_FLOAT), -xx(LEF_KR, ble, CFKR, LEF_RR, 2, REGT_FLOAT), +xx(ADDF_RR, add, RFRFRF, NOP, 0, 0) // fA = fB + fkC +xx(ADDF_RK, add, RFRFKF, ADDF_RR,4, REGT_FLOAT) +xx(SUBF_RR, sub, RFRFRF, NOP, 0, 0) // fA = fkB - fkC +xx(SUBF_RK, sub, RFRFKF, SUBF_RR,4, REGT_FLOAT) +xx(SUBF_KR, sub, RFKFRF, SUBF_RR,2, REGT_FLOAT) +xx(MULF_RR, mul, RFRFRF, NOP, 0, 0) // fA = fB * fkC +xx(MULF_RK, mul, RFRFKF, MULF_RR,4, REGT_FLOAT) +xx(DIVF_RR, div, RFRFRF, NOP, 0, 0) // fA = fkB / fkC +xx(DIVF_RK, div, RFRFKF, DIVF_RR,4, REGT_FLOAT) +xx(DIVF_KR, div, RFKFRF, DIVF_RR,2, REGT_FLOAT) +xx(MODF_RR, mod, RFRFRF, NOP, 0, 0) // fA = fkB % fkC +xx(MODF_RK, mod, RFRFKF, MODF_RR,4, REGT_FLOAT) +xx(MODF_KR, mod, RFKFRF, MODF_RR,4, REGT_FLOAT) +xx(POWF_RR, pow, RFRFRF, NOP, 0, 0) // fA = fkB ** fkC +xx(POWF_RK, pow, RFRFKF, POWF_RR,4, REGT_FLOAT) +xx(POWF_KR, pow, RFKFRF, POWF_RR,2, REGT_FLOAT) +xx(MINF_RR, min, RFRFRF, NOP, 0, 0) // fA = min(fB)fkC) +xx(MINF_RK, min, RFRFKF, MINF_RR,4, REGT_FLOAT) +xx(MAXF_RR, max, RFRFRF, NOP, 0, 0) // fA = max(fB)fkC) +xx(MAXF_RK, max, RFRFKF, MAXF_RR,4, REGT_FLOAT) +xx(ATAN2, atan2, RFRFRF, NOP, 0, 0) // fA = atan2(fB,fC) result is in degrees +xx(FLOP, flop, RFRFI8, NOP, 0, 0) // fA = f(fB) where function is selected by C +xx(EQF_R, beq, CFRR, NOP, 0, 0) // if ((fB == fkC) != (A & 1)) then pc++ +xx(EQF_K, beq, CFRK, EQF_R, 4, REGT_FLOAT) +xx(LTF_RR, blt, CFRR, NOP, 0, 0) // if ((fkB < fkC) != (A & 1)) then pc++ +xx(LTF_RK, blt, CFRK, LTF_RR, 4, REGT_FLOAT) +xx(LTF_KR, blt, CFKR, LTF_RR, 2, REGT_FLOAT) +xx(LEF_RR, ble, CFRR, NOP, 0, 0) // if ((fkb <= fkC) != (A & 1)) then pc++ +xx(LEF_RK, ble, CFRK, LEF_RR, 4, REGT_FLOAT) +xx(LEF_KR, ble, CFKR, LEF_RR, 2, REGT_FLOAT) // Vector math. (2D) -xx(NEGV2, negv2, RVRV, NOP, 0, 0), // vA = -vB -xx(ADDV2_RR, addv2, RVRVRV, NOP, 0, 0), // vA = vB + vkC -xx(SUBV2_RR, subv2, RVRVRV, NOP, 0, 0), // vA = vkB - vkC -xx(DOTV2_RR, dotv2, RVRVRV, NOP, 0, 0), // va = vB dot vkC -xx(MULVF2_RR, mulv2, RVRVRF, NOP, 0, 0), // vA = vkB * fkC -xx(MULVF2_RK, mulv2, RVRVKF, MULVF2_RR,4, REGT_FLOAT), -xx(DIVVF2_RR, divv2, RVRVRF, NOP, 0, 0), // vA = vkB / fkC -xx(DIVVF2_RK, divv2, RVRVKF, DIVVF2_RR,4, REGT_FLOAT), -xx(LENV2, lenv2, RFRV, NOP, 0, 0), // fA = vB.Length -xx(EQV2_R, beqv2, CVRR, NOP, 0, 0), // if ((vB == vkC) != A) then pc++ (inexact if A & 32) -xx(EQV2_K, beqv2, CVRK, NOP, 0, 0), // this will never be used. +xx(NEGV2, negv2, RVRV, NOP, 0, 0) // vA = -vB +xx(ADDV2_RR, addv2, RVRVRV, NOP, 0, 0) // vA = vB + vkC +xx(SUBV2_RR, subv2, RVRVRV, NOP, 0, 0) // vA = vkB - vkC +xx(DOTV2_RR, dotv2, RVRVRV, NOP, 0, 0) // va = vB dot vkC +xx(MULVF2_RR, mulv2, RVRVRF, NOP, 0, 0) // vA = vkB * fkC +xx(MULVF2_RK, mulv2, RVRVKF, MULVF2_RR,4, REGT_FLOAT) +xx(DIVVF2_RR, divv2, RVRVRF, NOP, 0, 0) // vA = vkB / fkC +xx(DIVVF2_RK, divv2, RVRVKF, DIVVF2_RR,4, REGT_FLOAT) +xx(LENV2, lenv2, RFRV, NOP, 0, 0) // fA = vB.Length +xx(EQV2_R, beqv2, CVRR, NOP, 0, 0) // if ((vB == vkC) != A) then pc++ (inexact if A & 32) +xx(EQV2_K, beqv2, CVRK, NOP, 0, 0) // this will never be used. // Vector math (3D) -xx(NEGV3, negv3, RVRV, NOP, 0, 0), // vA = -vB -xx(ADDV3_RR, addv3, RVRVRV, NOP, 0, 0), // vA = vB + vkC -xx(SUBV3_RR, subv3, RVRVRV, NOP, 0, 0), // vA = vkB - vkC -xx(DOTV3_RR, dotv3, RVRVRV, NOP, 0, 0), // va = vB dot vkC -xx(CROSSV_RR, crossv, RVRVRV, NOP, 0, 0), // vA = vkB cross vkC -xx(MULVF3_RR, mulv3, RVRVRF, NOP, 0, 0), // vA = vkB * fkC -xx(MULVF3_RK, mulv3, RVRVKF, MULVF3_RR,4, REGT_FLOAT), -xx(DIVVF3_RR, divv3, RVRVRF, NOP, 0, 0), // vA = vkB / fkC -xx(DIVVF3_RK, divv3, RVRVKF, DIVVF3_RR,4, REGT_FLOAT), -xx(LENV3, lenv3, RFRV, NOP, 0, 0), // fA = vB.Length -xx(EQV3_R, beqv3, CVRR, NOP, 0, 0), // if ((vB == vkC) != A) then pc++ (inexact if A & 33) -xx(EQV3_K, beqv3, CVRK, NOP, 0, 0), // this will never be used. +xx(NEGV3, negv3, RVRV, NOP, 0, 0) // vA = -vB +xx(ADDV3_RR, addv3, RVRVRV, NOP, 0, 0) // vA = vB + vkC +xx(SUBV3_RR, subv3, RVRVRV, NOP, 0, 0) // vA = vkB - vkC +xx(DOTV3_RR, dotv3, RVRVRV, NOP, 0, 0) // va = vB dot vkC +xx(CROSSV_RR, crossv, RVRVRV, NOP, 0, 0) // vA = vkB cross vkC +xx(MULVF3_RR, mulv3, RVRVRF, NOP, 0, 0) // vA = vkB * fkC +xx(MULVF3_RK, mulv3, RVRVKF, MULVF3_RR,4, REGT_FLOAT) +xx(DIVVF3_RR, divv3, RVRVRF, NOP, 0, 0) // vA = vkB / fkC +xx(DIVVF3_RK, divv3, RVRVKF, DIVVF3_RR,4, REGT_FLOAT) +xx(LENV3, lenv3, RFRV, NOP, 0, 0) // fA = vB.Length +xx(EQV3_R, beqv3, CVRR, NOP, 0, 0) // if ((vB == vkC) != A) then pc++ (inexact if A & 33) +xx(EQV3_K, beqv3, CVRK, NOP, 0, 0) // this will never be used. // Pointer math. -xx(ADDA_RR, add, RPRPRI, NOP, 0, 0), // pA = pB + dkC -xx(ADDA_RK, add, RPRPKI, ADDA_RR,4, REGT_INT), -xx(SUBA, sub, RIRPRP, NOP, 0, 0), // dA = pB - pC -xx(EQA_R, beq, CPRR, NOP, 0, 0), // if ((pB == pkC) != A) then pc++ -xx(EQA_K, beq, CPRK, EQA_R, 4, REGT_POINTER), +xx(ADDA_RR, add, RPRPRI, NOP, 0, 0) // pA = pB + dkC +xx(ADDA_RK, add, RPRPKI, ADDA_RR,4, REGT_INT) +xx(SUBA, sub, RIRPRP, NOP, 0, 0) // dA = pB - pC +xx(EQA_R, beq, CPRR, NOP, 0, 0) // if ((pB == pkC) != A) then pc++ +xx(EQA_K, beq, CPRK, EQA_R, 4, REGT_POINTER) #undef xx