mirrors/dynarmic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

emit_x64_floating_point: Use deferred emits

Browse source
This commit is contained in:
Merry 2022-07-13 10:51:50 +01:00 committed by merry
parent 6c38ed8a89
commit 11ba75b7f0
1 changed files with 295 additions and 292 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

197
src/dynarmic/backend/x64/emit_x64_floating_point.cpp
View file

@ -152,18 +152,18 @@ void ForceToDefaultNaN(BlockOfCode& code, Xbyak::Xmm result) {
} }
template<size_t fsize> template<size_t fsize>
Xbyak::Label ProcessNaN(BlockOfCode& code, Xbyak::Xmm a) { SharedLabel ProcessNaN(BlockOfCode& code, EmitContext& ctx, Xbyak::Xmm a) {
Xbyak::Label nan, end; SharedLabel nan = GenSharedLabel(), end = GenSharedLabel();
FCODE(ucomis)(a, a); FCODE(ucomis)(a, a);
code.jp(nan, code.T_NEAR); code.jp(*nan, code.T_NEAR);
code.SwitchToFarCode();
code.L(nan);
ctx.deferred_emits.emplace_back([=, &code] {
code.L(*nan);
code.orps(a, code.XmmBConst<fsize>(xword, fsize == 32 ? 0x00400000 : 0x0008'0000'0000'0000)); code.orps(a, code.XmmBConst<fsize>(xword, fsize == 32 ? 0x00400000 : 0x0008'0000'0000'0000));
code.jmp(*end, code.T_NEAR);
});
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
return end; return end;
} }
@ -268,12 +268,12 @@ template<size_t fsize, typename Function>
void FPTwoOp(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst, Function fn) { void FPTwoOp(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst, Function fn) {
auto args = ctx.reg_alloc.GetArgumentInfo(inst); auto args = ctx.reg_alloc.GetArgumentInfo(inst);
Xbyak::Label end; SharedLabel end = GenSharedLabel();
Xbyak::Xmm result = ctx.reg_alloc.UseScratchXmm(args[0]); Xbyak::Xmm result = ctx.reg_alloc.UseScratchXmm(args[0]);
if (!ctx.FPCR().DN() && !ctx.HasOptimization(OptimizationFlag::Unsafe_InaccurateNaN)) { if (!ctx.FPCR().DN() && !ctx.HasOptimization(OptimizationFlag::Unsafe_InaccurateNaN)) {
end = ProcessNaN<fsize>(code, result); end = ProcessNaN<fsize>(code, ctx, result);
} }
if constexpr (std::is_member_function_pointer_v<Function>) { if constexpr (std::is_member_function_pointer_v<Function>) {
(code.*fn)(result, result); (code.*fn)(result, result);
@ -287,7 +287,7 @@ void FPTwoOp(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst, Function fn) {
} else { } else {
PostProcessNaN<fsize>(code, result, ctx.reg_alloc.ScratchXmm()); PostProcessNaN<fsize>(code, result, ctx.reg_alloc.ScratchXmm());
} }
code.L(end); code.L(*end);
ctx.reg_alloc.DefineValue(inst, result); ctx.reg_alloc.DefineValue(inst, result);
} }
@ -321,7 +321,7 @@ void FPThreeOp(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst, Function fn)
const Xbyak::Xmm result = ctx.reg_alloc.ScratchXmm(); const Xbyak::Xmm result = ctx.reg_alloc.ScratchXmm();
const Xbyak::Reg64 tmp = ctx.reg_alloc.ScratchGpr(); const Xbyak::Reg64 tmp = ctx.reg_alloc.ScratchGpr();
Xbyak::Label end, nan, op_are_nans; SharedLabel end = GenSharedLabel(), nan = GenSharedLabel();
code.movaps(result, op1); code.movaps(result, op1);
if constexpr (std::is_member_function_pointer_v<Function>) { if constexpr (std::is_member_function_pointer_v<Function>) {
@ -330,19 +330,21 @@ void FPThreeOp(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst, Function fn)
fn(result, op2); fn(result, op2);
} }
FCODE(ucomis)(result, result); FCODE(ucomis)(result, result);
code.jp(nan, code.T_NEAR); code.jp(*nan, code.T_NEAR);
code.L(end); code.L(*end);
code.SwitchToFarCode(); ctx.deferred_emits.emplace_back([=, &code] {
code.L(nan); Xbyak::Label op_are_nans;
code.L(*nan);
FCODE(ucomis)(op1, op2); FCODE(ucomis)(op1, op2);
code.jp(op_are_nans); code.jp(op_are_nans);
// Here we must return a positive NaN, because the indefinite value on x86 is a negative NaN! // Here we must return a positive NaN, because the indefinite value on x86 is a negative NaN!
code.movaps(result, code.XmmBConst<fsize>(xword, FP::FPInfo<FPT>::DefaultNaN())); code.movaps(result, code.XmmBConst<fsize>(xword, FP::FPInfo<FPT>::DefaultNaN()));
code.jmp(end, code.T_NEAR); code.jmp(*end, code.T_NEAR);
code.L(op_are_nans); code.L(op_are_nans);
EmitPostProcessNaNs<fsize>(code, result, op1, op2, tmp, end); EmitPostProcessNaNs<fsize>(code, result, op1, op2, tmp, *end);
code.SwitchToNearCode(); });
ctx.reg_alloc.DefineValue(inst, result); ctx.reg_alloc.DefineValue(inst, result);
} }
@ -428,39 +430,39 @@ static void EmitFPMinMax(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
DenormalsAreZero<fsize>(code, ctx, {result, operand}); DenormalsAreZero<fsize>(code, ctx, {result, operand});
Xbyak::Label equal, end, nan; SharedLabel equal = GenSharedLabel(), end = GenSharedLabel();
FCODE(ucomis)(result, operand); FCODE(ucomis)(result, operand);
code.jz(equal, code.T_NEAR); code.jz(*equal, code.T_NEAR);
if constexpr (is_max) { if constexpr (is_max) {
FCODE(maxs)(result, operand); FCODE(maxs)(result, operand);
} else { } else {
FCODE(mins)(result, operand); FCODE(mins)(result, operand);
} }
code.L(end); code.L(*end);
code.SwitchToFarCode(); ctx.deferred_emits.emplace_back([=, &code, &ctx] {
Xbyak::Label nan;
code.L(equal); code.L(*equal);
code.jp(nan); code.jp(nan);
if constexpr (is_max) { if constexpr (is_max) {
code.andps(result, operand); code.andps(result, operand);
} else { } else {
code.orps(result, operand); code.orps(result, operand);
} }
code.jmp(end); code.jmp(*end);
code.L(nan); code.L(nan);
if (ctx.FPCR().DN()) { if (ctx.FPCR().DN()) {
code.movaps(result, code.XmmBConst<fsize>(xword, fsize == 32 ? f32_nan : f64_nan)); code.movaps(result, code.XmmBConst<fsize>(xword, fsize == 32 ? f32_nan : f64_nan));
code.jmp(end); code.jmp(*end);
} else { } else {
code.movaps(tmp, result); code.movaps(tmp, result);
FCODE(adds)(result, operand); FCODE(adds)(result, operand);
EmitPostProcessNaNs<fsize>(code, result, tmp, operand, gpr_scratch, end); EmitPostProcessNaNs<fsize>(code, result, tmp, operand, gpr_scratch, *end);
} }
});
code.SwitchToNearCode();
ctx.reg_alloc.DefineValue(inst, result); ctx.reg_alloc.DefineValue(inst, result);
} }
@ -492,7 +494,7 @@ static void EmitFPMinMaxNumeric(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
Xbyak::Reg tmp = ctx.reg_alloc.ScratchGpr(); Xbyak::Reg tmp = ctx.reg_alloc.ScratchGpr();
tmp.setBit(fsize); tmp.setBit(fsize);
const auto move_to_tmp = [&](const Xbyak::Xmm& xmm) { const auto move_to_tmp = [=, &code](const Xbyak::Xmm& xmm) {
if constexpr (fsize == 32) { if constexpr (fsize == 32) {
code.movd(tmp.cvt32(), xmm); code.movd(tmp.cvt32(), xmm);
} else { } else {
@ -500,28 +502,28 @@ static void EmitFPMinMaxNumeric(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
} }
}; };
Xbyak::Label end, z, nan, op2_is_nan, snan, maybe_both_nan, normal; SharedLabel end = GenSharedLabel(), z = GenSharedLabel();
FCODE(ucomis)(op1, op2); FCODE(ucomis)(op1, op2);
code.jz(z, code.T_NEAR); code.jz(*z, code.T_NEAR);
code.L(normal);
if constexpr (is_max) { if constexpr (is_max) {
FCODE(maxs)(op2, op1); FCODE(maxs)(op2, op1);
} else { } else {
FCODE(mins)(op2, op1); FCODE(mins)(op2, op1);
} }
code.L(end); code.L(*end);
code.SwitchToFarCode(); ctx.deferred_emits.emplace_back([=, &code, &ctx] {
Xbyak::Label nan, op2_is_nan, snan, maybe_both_nan;
code.L(z); code.L(*z);
code.jp(nan); code.jp(nan);
if constexpr (is_max) { if constexpr (is_max) {
code.andps(op2, op1); code.andps(op2, op1);
} else { } else {
code.orps(op2, op1); code.orps(op2, op1);
} }
code.jmp(end); code.jmp(*end);
// NaN requirements: // NaN requirements:
// op1 op2 result // op1 op2 result
@ -542,17 +544,17 @@ static void EmitFPMinMaxNumeric(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
if (ctx.FPCR().DN()) { if (ctx.FPCR().DN()) {
code.L(snan); code.L(snan);
code.movaps(op2, code.XmmBConst<fsize>(xword, default_nan)); code.movaps(op2, code.XmmBConst<fsize>(xword, default_nan));
code.jmp(end); code.jmp(*end);
} else { } else {
code.movaps(op2, op1); code.movaps(op2, op1);
code.L(snan); code.L(snan);
code.orps(op2, code.XmmBConst<fsize>(xword, FP::FPInfo<FPT>::mantissa_msb)); code.orps(op2, code.XmmBConst<fsize>(xword, FP::FPInfo<FPT>::mantissa_msb));
code.jmp(end); code.jmp(*end);
} }
code.L(maybe_both_nan); code.L(maybe_both_nan);
FCODE(ucomis)(op2, op2); FCODE(ucomis)(op2, op2);
code.jnp(end, code.T_NEAR); code.jnp(*end, code.T_NEAR);
if (ctx.FPCR().DN()) { if (ctx.FPCR().DN()) {
code.jmp(snan); code.jmp(snan);
} else { } else {
@ -560,7 +562,7 @@ static void EmitFPMinMaxNumeric(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
code.bt(tmp.cvt64(), mantissa_msb_bit); code.bt(tmp.cvt64(), mantissa_msb_bit);
code.jnc(snan); code.jnc(snan);
code.movaps(op2, op1); code.movaps(op2, op1);
code.jmp(end); code.jmp(*end);
} }
// op2 is NaN // op2 is NaN
@ -569,9 +571,8 @@ static void EmitFPMinMaxNumeric(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
code.bt(tmp, mantissa_msb_bit); code.bt(tmp, mantissa_msb_bit);
code.jnc(snan); code.jnc(snan);
code.movaps(op2, op1); code.movaps(op2, op1);
code.jmp(end); code.jmp(*end);
});
code.SwitchToNearCode();
} }
ctx.reg_alloc.DefineValue(inst, op2); ctx.reg_alloc.DefineValue(inst, op2);
@ -636,7 +637,7 @@ static void EmitFPMulAdd(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
} }
if (code.HasHostFeature(HostFeature::FMA)) { if (code.HasHostFeature(HostFeature::FMA)) {
Xbyak::Label end, fallback; SharedLabel end = GenSharedLabel(), fallback = GenSharedLabel();
const Xbyak::Xmm operand1 = ctx.reg_alloc.UseXmm(args[0]); const Xbyak::Xmm operand1 = ctx.reg_alloc.UseXmm(args[0]);
const Xbyak::Xmm operand2 = ctx.reg_alloc.UseXmm(args[1]); const Xbyak::Xmm operand2 = ctx.reg_alloc.UseXmm(args[1]);
@ -650,11 +651,11 @@ static void EmitFPMulAdd(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
code.movaps(tmp, code.XmmBConst<fsize>(xword, fsize == 32 ? f32_non_sign_mask : f64_non_sign_mask)); code.movaps(tmp, code.XmmBConst<fsize>(xword, fsize == 32 ? f32_non_sign_mask : f64_non_sign_mask));
code.andps(tmp, result); code.andps(tmp, result);
FCODE(ucomis)(tmp, code.XmmBConst<fsize>(xword, fsize == 32 ? f32_smallest_normal : f64_smallest_normal)); FCODE(ucomis)(tmp, code.XmmBConst<fsize>(xword, fsize == 32 ? f32_smallest_normal : f64_smallest_normal));
code.jz(fallback, code.T_NEAR); code.jz(*fallback, code.T_NEAR);
code.L(end); code.L(*end);
code.SwitchToFarCode(); ctx.deferred_emits.emplace_back([=, &code, &ctx] {
code.L(fallback); code.L(*fallback);
code.sub(rsp, 8); code.sub(rsp, 8);
ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx())); ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx()));
@ -676,8 +677,8 @@ static void EmitFPMulAdd(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx())); ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx()));
code.add(rsp, 8); code.add(rsp, 8);
code.jmp(end, code.T_NEAR); code.jmp(*end, code.T_NEAR);
code.SwitchToNearCode(); });
ctx.reg_alloc.DefineValue(inst, result); ctx.reg_alloc.DefineValue(inst, result);
return; return;
@ -735,7 +736,7 @@ static void EmitFPMulX(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
const Xbyak::Xmm result = ctx.reg_alloc.ScratchXmm(); const Xbyak::Xmm result = ctx.reg_alloc.ScratchXmm();
const Xbyak::Reg64 tmp = do_default_nan ? INVALID_REG : ctx.reg_alloc.ScratchGpr(); const Xbyak::Reg64 tmp = do_default_nan ? INVALID_REG : ctx.reg_alloc.ScratchGpr();
Xbyak::Label end, nan, op_are_nans; SharedLabel end = GenSharedLabel(), nan = GenSharedLabel();
if (code.HasHostFeature(HostFeature::AVX)) { if (code.HasHostFeature(HostFeature::AVX)) {
FCODE(vmuls)(result, op1, op2); FCODE(vmuls)(result, op1, op2);
@ -744,11 +745,13 @@ static void EmitFPMulX(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
FCODE(muls)(result, op2); FCODE(muls)(result, op2);
} }
FCODE(ucomis)(result, result); FCODE(ucomis)(result, result);
code.jp(nan, code.T_NEAR); code.jp(*nan, code.T_NEAR);
code.L(end); code.L(*end);
code.SwitchToFarCode(); ctx.deferred_emits.emplace_back([=, &code] {
code.L(nan); Xbyak::Label op_are_nans;
code.L(*nan);
FCODE(ucomis)(op1, op2); FCODE(ucomis)(op1, op2);
code.jp(op_are_nans); code.jp(op_are_nans);
if (code.HasHostFeature(HostFeature::AVX)) { if (code.HasHostFeature(HostFeature::AVX)) {
@ -759,15 +762,15 @@ static void EmitFPMulX(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
} }
code.andps(result, code.XmmBConst<fsize>(xword, FP::FPInfo<FPT>::sign_mask)); code.andps(result, code.XmmBConst<fsize>(xword, FP::FPInfo<FPT>::sign_mask));
code.orps(result, code.XmmBConst<fsize>(xword, FP::FPValue<FPT, false, 0, 2>())); code.orps(result, code.XmmBConst<fsize>(xword, FP::FPValue<FPT, false, 0, 2>()));
code.jmp(end, code.T_NEAR); code.jmp(*end, code.T_NEAR);
code.L(op_are_nans); code.L(op_are_nans);
if (do_default_nan) { if (do_default_nan) {
code.movaps(result, code.XmmBConst<fsize>(xword, FP::FPInfo<FPT>::DefaultNaN())); code.movaps(result, code.XmmBConst<fsize>(xword, FP::FPInfo<FPT>::DefaultNaN()));
code.jmp(end, code.T_NEAR); code.jmp(*end, code.T_NEAR);
} else { } else {
EmitPostProcessNaNs<fsize>(code, result, op1, op2, tmp, end); EmitPostProcessNaNs<fsize>(code, result, op1, op2, tmp, *end);
} }
code.SwitchToNearCode(); });
ctx.reg_alloc.DefineValue(inst, result); ctx.reg_alloc.DefineValue(inst, result);
} }
@ -871,7 +874,7 @@ static void EmitFPRecipStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*
} }
if (code.HasHostFeature(HostFeature::FMA)) { if (code.HasHostFeature(HostFeature::FMA)) {
Xbyak::Label end, fallback; SharedLabel end = GenSharedLabel(), fallback = GenSharedLabel();
const Xbyak::Xmm operand1 = ctx.reg_alloc.UseXmm(args[0]); const Xbyak::Xmm operand1 = ctx.reg_alloc.UseXmm(args[0]);
const Xbyak::Xmm operand2 = ctx.reg_alloc.UseXmm(args[1]); const Xbyak::Xmm operand2 = ctx.reg_alloc.UseXmm(args[1]);
@ -880,11 +883,11 @@ static void EmitFPRecipStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*
code.movaps(result, code.XmmBConst<fsize>(xword, FP::FPValue<FPT, false, 0, 2>())); code.movaps(result, code.XmmBConst<fsize>(xword, FP::FPValue<FPT, false, 0, 2>()));
FCODE(vfnmadd231s)(result, operand1, operand2); FCODE(vfnmadd231s)(result, operand1, operand2);
FCODE(ucomis)(result, result); FCODE(ucomis)(result, result);
code.jp(fallback, code.T_NEAR); code.jp(*fallback, code.T_NEAR);
code.L(end); code.L(*end);
code.SwitchToFarCode(); ctx.deferred_emits.emplace_back([=, &code, &ctx] {
code.L(fallback); code.L(*fallback);
code.sub(rsp, 8); code.sub(rsp, 8);
ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx())); ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx()));
@ -897,8 +900,8 @@ static void EmitFPRecipStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx())); ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx()));
code.add(rsp, 8); code.add(rsp, 8);
code.jmp(end, code.T_NEAR); code.jmp(*end, code.T_NEAR);
code.SwitchToNearCode(); });
ctx.reg_alloc.DefineValue(inst, result); ctx.reg_alloc.DefineValue(inst, result);
return; return;
@ -1034,8 +1037,7 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
const Xbyak::Xmm value = ctx.reg_alloc.ScratchXmm(); const Xbyak::Xmm value = ctx.reg_alloc.ScratchXmm();
[[maybe_unused]] const Xbyak::Reg32 tmp = ctx.reg_alloc.ScratchGpr().cvt32(); [[maybe_unused]] const Xbyak::Reg32 tmp = ctx.reg_alloc.ScratchGpr().cvt32();
Xbyak::Label fallback, bad_values, end, default_nan; SharedLabel bad_values = GenSharedLabel(), end = GenSharedLabel();
bool needs_fallback = false;
code.movaps(value, operand); code.movaps(value, operand);
@ -1045,7 +1047,7 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
// Detect NaNs, negatives, zeros, denormals and infinities // Detect NaNs, negatives, zeros, denormals and infinities
FCODE(ucomis)(value, code.XmmBConst<fsize>(xword, FPT(1) << FP::FPInfo<FPT>::explicit_mantissa_width)); FCODE(ucomis)(value, code.XmmBConst<fsize>(xword, FPT(1) << FP::FPInfo<FPT>::explicit_mantissa_width));
code.jna(bad_values, code.T_NEAR); code.jna(*bad_values, code.T_NEAR);
FCODE(sqrts)(value, value); FCODE(sqrts)(value, value);
ICODE(mov)(result, code.XmmBConst<fsize>(xword, FP::FPValue<FPT, false, 0, 1>())); ICODE(mov)(result, code.XmmBConst<fsize>(xword, FP::FPValue<FPT, false, 0, 1>()));
@ -1054,11 +1056,13 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
ICODE(padd)(result, code.XmmBConst<fsize>(xword, fsize == 32 ? 0x00004000 : 0x0000'0800'0000'0000)); ICODE(padd)(result, code.XmmBConst<fsize>(xword, fsize == 32 ? 0x00004000 : 0x0000'0800'0000'0000));
code.pand(result, xmm0); code.pand(result, xmm0);
code.L(end); code.L(*end);
code.SwitchToFarCode(); ctx.deferred_emits.emplace_back([=, &code, &ctx] {
Xbyak::Label fallback, default_nan;
bool needs_fallback = false;
code.L(bad_values); code.L(*bad_values);
if constexpr (fsize == 32) { if constexpr (fsize == 32) {
code.movd(tmp, operand); code.movd(tmp, operand);
@ -1080,18 +1084,18 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
if (ctx.FPCR().DN()) { if (ctx.FPCR().DN()) {
code.ucomiss(result, result); code.ucomiss(result, result);
code.jnp(end, code.T_NEAR); code.jnp(*end, code.T_NEAR);
} else { } else {
// FZ ? (a >= 0x80800000 && a <= 0xFF800000) : (a >= 0x80000001 && a <= 0xFF800000) // FZ ? (a >= 0x80800000 && a <= 0xFF800000) : (a >= 0x80000001 && a <= 0xFF800000)
// !FZ path takes into account the subtraction by one from the earlier block // !FZ path takes into account the subtraction by one from the earlier block
code.add(tmp, ctx.FPCR().FZ() ? 0x7F800000 : 0x80000000); code.add(tmp, ctx.FPCR().FZ() ? 0x7F800000 : 0x80000000);
code.cmp(tmp, ctx.FPCR().FZ() ? 0x7F000001 : 0x7F800000); code.cmp(tmp, ctx.FPCR().FZ() ? 0x7F000001 : 0x7F800000);
code.jnb(end, code.T_NEAR); code.jnb(*end, code.T_NEAR);
} }
code.L(default_nan); code.L(default_nan);
code.movd(result, code.XmmBConst<32>(xword, 0x7FC00000)); code.movd(result, code.XmmBConst<32>(xword, 0x7FC00000));
code.jmp(end, code.T_NEAR); code.jmp(*end, code.T_NEAR);
} else { } else {
Xbyak::Label nan, zero; Xbyak::Label nan, zero;
@ -1114,7 +1118,7 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
code.jmp(fallback); code.jmp(fallback);
} else { } else {
// result = 0 // result = 0
code.jmp(end, code.T_NEAR); code.jmp(*end, code.T_NEAR);
} }
code.L(zero); code.L(zero);
@ -1124,7 +1128,7 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
code.movaps(result, value); code.movaps(result, value);
code.por(result, code.XmmBConst<64>(xword, 0x7FF0'0000'0000'0000)); code.por(result, code.XmmBConst<64>(xword, 0x7FF0'0000'0000'0000));
} }
code.jmp(end, code.T_NEAR); code.jmp(*end, code.T_NEAR);
code.L(nan); code.L(nan);
if (!ctx.FPCR().DN()) { if (!ctx.FPCR().DN()) {
@ -1134,12 +1138,12 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
code.movaps(result, operand); code.movaps(result, operand);
code.por(result, code.XmmBConst<64>(xword, 0x0008'0000'0000'0000)); code.por(result, code.XmmBConst<64>(xword, 0x0008'0000'0000'0000));
} }
code.jmp(end, code.T_NEAR); code.jmp(*end, code.T_NEAR);
} }
code.L(default_nan); code.L(default_nan);
code.movq(result, code.XmmBConst<64>(xword, 0x7FF8'0000'0000'0000)); code.movq(result, code.XmmBConst<64>(xword, 0x7FF8'0000'0000'0000));
code.jmp(end, code.T_NEAR); code.jmp(*end, code.T_NEAR);
} }
code.L(fallback); code.L(fallback);
@ -1153,10 +1157,9 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
code.movq(result, rax); code.movq(result, rax);
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx())); ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx()));
code.add(rsp, 8); code.add(rsp, 8);
code.jmp(end, code.T_NEAR); code.jmp(*end, code.T_NEAR);
} }
});
code.SwitchToNearCode();
ctx.reg_alloc.DefineValue(inst, result); ctx.reg_alloc.DefineValue(inst, result);
} else { } else {
@ -1201,7 +1204,7 @@ static void EmitFPRSqrtStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*
} }
if (code.HasHostFeature(HostFeature::FMA | HostFeature::AVX)) { if (code.HasHostFeature(HostFeature::FMA | HostFeature::AVX)) {
Xbyak::Label end, fallback; SharedLabel end = GenSharedLabel(), fallback = GenSharedLabel();
const Xbyak::Xmm operand1 = ctx.reg_alloc.UseXmm(args[0]); const Xbyak::Xmm operand1 = ctx.reg_alloc.UseXmm(args[0]);
const Xbyak::Xmm operand2 = ctx.reg_alloc.UseXmm(args[1]); const Xbyak::Xmm operand2 = ctx.reg_alloc.UseXmm(args[1]);
@ -1220,13 +1223,13 @@ static void EmitFPRSqrtStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*
code.cmp(tmp.cvt16(), fsize == 32 ? 0x7f00 : 0x7fe0); code.cmp(tmp.cvt16(), fsize == 32 ? 0x7f00 : 0x7fe0);
ctx.reg_alloc.Release(tmp); ctx.reg_alloc.Release(tmp);
code.jae(fallback, code.T_NEAR); code.jae(*fallback, code.T_NEAR);
FCODE(vmuls)(result, result, code.XmmBConst<fsize>(xword, FP::FPValue<FPT, false, -1, 1>())); FCODE(vmuls)(result, result, code.XmmBConst<fsize>(xword, FP::FPValue<FPT, false, -1, 1>()));
code.L(end); code.L(*end);
code.SwitchToFarCode(); ctx.deferred_emits.emplace_back([=, &code, &ctx] {
code.L(fallback); code.L(*fallback);
code.sub(rsp, 8); code.sub(rsp, 8);
ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx())); ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx()));
@ -1239,8 +1242,8 @@ static void EmitFPRSqrtStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx())); ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx()));
code.add(rsp, 8); code.add(rsp, 8);
code.jmp(end, code.T_NEAR); code.jmp(*end, code.T_NEAR);
code.SwitchToNearCode(); });
ctx.reg_alloc.DefineValue(inst, result); ctx.reg_alloc.DefineValue(inst, result);
return; return;
@ -1528,22 +1531,22 @@ static void EmitFPToFixed(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
if constexpr (isize == 64) { if constexpr (isize == 64) {
const Xbyak::Xmm scratch = ctx.reg_alloc.ScratchXmm(); const Xbyak::Xmm scratch = ctx.reg_alloc.ScratchXmm();
Xbyak::Label saturate_max, end;
if (!unsigned_) { if (!unsigned_) {
SharedLabel saturate_max = GenSharedLabel(), end = GenSharedLabel();
ZeroIfNaN<64>(code, src, scratch); ZeroIfNaN<64>(code, src, scratch);
code.movsd(scratch, code.XmmBConst<64>(xword, f64_max_s64_lim)); code.movsd(scratch, code.XmmBConst<64>(xword, f64_max_s64_lim));
code.comisd(scratch, src); code.comisd(scratch, src);
code.jna(saturate_max, code.T_NEAR); code.jna(*saturate_max, code.T_NEAR);
code.cvttsd2si(result, src); // 64 bit gpr code.cvttsd2si(result, src); // 64 bit gpr
code.L(end); code.L(*end);
code.SwitchToFarCode(); ctx.deferred_emits.emplace_back([=, &code] {
code.L(saturate_max); code.L(*saturate_max);
code.mov(result, 0x7FFF'FFFF'FFFF'FFFF); code.mov(result, 0x7FFF'FFFF'FFFF'FFFF);
code.jmp(end, code.T_NEAR); code.jmp(*end, code.T_NEAR);
code.SwitchToNearCode(); });
} else { } else {
Xbyak::Label below_max; Xbyak::Label below_max;