mirrors/dynarmic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

emit_x64_floating_point: Use deferred emits

Browse source
This commit is contained in:
Merry 2022-07-13 10:51:50 +01:00 committed by merry
parent 6c38ed8a89
commit 11ba75b7f0
1 changed files with 295 additions and 292 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

197
src/dynarmic/backend/x64/emit_x64_floating_point.cpp
View file

@ -152,18 +152,18 @@ void ForceToDefaultNaN(BlockOfCode& code, Xbyak::Xmm result) {
}
template<size_t fsize>
Xbyak::Label ProcessNaN(BlockOfCode& code, Xbyak::Xmm a) {
Xbyak::Label nan, end;
SharedLabel ProcessNaN(BlockOfCode& code, EmitContext& ctx, Xbyak::Xmm a) {
SharedLabel nan = GenSharedLabel(), end = GenSharedLabel();
FCODE(ucomis)(a, a);
code.jp(nan, code.T_NEAR);
code.SwitchToFarCode();
code.L(nan);
code.jp(*nan, code.T_NEAR);
ctx.deferred_emits.emplace_back([=, &code] {
code.L(*nan);
code.orps(a, code.XmmBConst<fsize>(xword, fsize == 32 ? 0x00400000 : 0x0008'0000'0000'0000));
code.jmp(*end, code.T_NEAR);
});
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
return end;
}
@ -268,12 +268,12 @@ template<size_t fsize, typename Function>
void FPTwoOp(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst, Function fn) {
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
Xbyak::Label end;
SharedLabel end = GenSharedLabel();
Xbyak::Xmm result = ctx.reg_alloc.UseScratchXmm(args[0]);
if (!ctx.FPCR().DN() && !ctx.HasOptimization(OptimizationFlag::Unsafe_InaccurateNaN)) {
end = ProcessNaN<fsize>(code, result);
end = ProcessNaN<fsize>(code, ctx, result);
}
if constexpr (std::is_member_function_pointer_v<Function>) {
(code.*fn)(result, result);
@ -287,7 +287,7 @@ void FPTwoOp(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst, Function fn) {
} else {
PostProcessNaN<fsize>(code, result, ctx.reg_alloc.ScratchXmm());
}
code.L(end);
code.L(*end);
ctx.reg_alloc.DefineValue(inst, result);
}
@ -321,7 +321,7 @@ void FPThreeOp(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst, Function fn)
const Xbyak::Xmm result = ctx.reg_alloc.ScratchXmm();
const Xbyak::Reg64 tmp = ctx.reg_alloc.ScratchGpr();
Xbyak::Label end, nan, op_are_nans;
SharedLabel end = GenSharedLabel(), nan = GenSharedLabel();
code.movaps(result, op1);
if constexpr (std::is_member_function_pointer_v<Function>) {
@ -330,19 +330,21 @@ void FPThreeOp(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst, Function fn)
fn(result, op2);
}
FCODE(ucomis)(result, result);
code.jp(nan, code.T_NEAR);
code.L(end);
code.jp(*nan, code.T_NEAR);
code.L(*end);
code.SwitchToFarCode();
code.L(nan);
ctx.deferred_emits.emplace_back([=, &code] {
Xbyak::Label op_are_nans;
code.L(*nan);
FCODE(ucomis)(op1, op2);
code.jp(op_are_nans);
// Here we must return a positive NaN, because the indefinite value on x86 is a negative NaN!
code.movaps(result, code.XmmBConst<fsize>(xword, FP::FPInfo<FPT>::DefaultNaN()));
code.jmp(end, code.T_NEAR);
code.jmp(*end, code.T_NEAR);
code.L(op_are_nans);
EmitPostProcessNaNs<fsize>(code, result, op1, op2, tmp, end);
code.SwitchToNearCode();
EmitPostProcessNaNs<fsize>(code, result, op1, op2, tmp, *end);
});
ctx.reg_alloc.DefineValue(inst, result);
}
@ -428,39 +430,39 @@ static void EmitFPMinMax(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
DenormalsAreZero<fsize>(code, ctx, {result, operand});
Xbyak::Label equal, end, nan;
SharedLabel equal = GenSharedLabel(), end = GenSharedLabel();
FCODE(ucomis)(result, operand);
code.jz(equal, code.T_NEAR);
code.jz(*equal, code.T_NEAR);
if constexpr (is_max) {
FCODE(maxs)(result, operand);
} else {
FCODE(mins)(result, operand);
}
code.L(end);
code.L(*end);
code.SwitchToFarCode();
ctx.deferred_emits.emplace_back([=, &code, &ctx] {
Xbyak::Label nan;
code.L(equal);
code.L(*equal);
code.jp(nan);
if constexpr (is_max) {
code.andps(result, operand);
} else {
code.orps(result, operand);
}
code.jmp(end);
code.jmp(*end);
code.L(nan);
if (ctx.FPCR().DN()) {
code.movaps(result, code.XmmBConst<fsize>(xword, fsize == 32 ? f32_nan : f64_nan));
code.jmp(end);
code.jmp(*end);
} else {
code.movaps(tmp, result);
FCODE(adds)(result, operand);
EmitPostProcessNaNs<fsize>(code, result, tmp, operand, gpr_scratch, end);
EmitPostProcessNaNs<fsize>(code, result, tmp, operand, gpr_scratch, *end);
}
code.SwitchToNearCode();
});
ctx.reg_alloc.DefineValue(inst, result);
}
@ -492,7 +494,7 @@ static void EmitFPMinMaxNumeric(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
Xbyak::Reg tmp = ctx.reg_alloc.ScratchGpr();
tmp.setBit(fsize);
const auto move_to_tmp = [&](const Xbyak::Xmm& xmm) {
const auto move_to_tmp = [=, &code](const Xbyak::Xmm& xmm) {
if constexpr (fsize == 32) {
code.movd(tmp.cvt32(), xmm);
} else {
@ -500,28 +502,28 @@ static void EmitFPMinMaxNumeric(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
}
};
Xbyak::Label end, z, nan, op2_is_nan, snan, maybe_both_nan, normal;
SharedLabel end = GenSharedLabel(), z = GenSharedLabel();
FCODE(ucomis)(op1, op2);
code.jz(z, code.T_NEAR);
code.L(normal);
code.jz(*z, code.T_NEAR);
if constexpr (is_max) {
FCODE(maxs)(op2, op1);
} else {
FCODE(mins)(op2, op1);
}
code.L(end);
code.L(*end);
code.SwitchToFarCode();
ctx.deferred_emits.emplace_back([=, &code, &ctx] {
Xbyak::Label nan, op2_is_nan, snan, maybe_both_nan;
code.L(z);
code.L(*z);
code.jp(nan);
if constexpr (is_max) {
code.andps(op2, op1);
} else {
code.orps(op2, op1);
}
code.jmp(end);
code.jmp(*end);
// NaN requirements:
// op1 op2 result
@ -542,17 +544,17 @@ static void EmitFPMinMaxNumeric(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
if (ctx.FPCR().DN()) {
code.L(snan);
code.movaps(op2, code.XmmBConst<fsize>(xword, default_nan));
code.jmp(end);
code.jmp(*end);
} else {
code.movaps(op2, op1);
code.L(snan);
code.orps(op2, code.XmmBConst<fsize>(xword, FP::FPInfo<FPT>::mantissa_msb));
code.jmp(end);
code.jmp(*end);
}
code.L(maybe_both_nan);
FCODE(ucomis)(op2, op2);
code.jnp(end, code.T_NEAR);
code.jnp(*end, code.T_NEAR);
if (ctx.FPCR().DN()) {
code.jmp(snan);
} else {
@ -560,7 +562,7 @@ static void EmitFPMinMaxNumeric(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
code.bt(tmp.cvt64(), mantissa_msb_bit);
code.jnc(snan);
code.movaps(op2, op1);
code.jmp(end);
code.jmp(*end);
}
// op2 is NaN
@ -569,9 +571,8 @@ static void EmitFPMinMaxNumeric(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
code.bt(tmp, mantissa_msb_bit);
code.jnc(snan);
code.movaps(op2, op1);
code.jmp(end);
code.SwitchToNearCode();
code.jmp(*end);
});
}
ctx.reg_alloc.DefineValue(inst, op2);
@ -636,7 +637,7 @@ static void EmitFPMulAdd(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
}
if (code.HasHostFeature(HostFeature::FMA)) {
Xbyak::Label end, fallback;
SharedLabel end = GenSharedLabel(), fallback = GenSharedLabel();
const Xbyak::Xmm operand1 = ctx.reg_alloc.UseXmm(args[0]);
const Xbyak::Xmm operand2 = ctx.reg_alloc.UseXmm(args[1]);
@ -650,11 +651,11 @@ static void EmitFPMulAdd(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
code.movaps(tmp, code.XmmBConst<fsize>(xword, fsize == 32 ? f32_non_sign_mask : f64_non_sign_mask));
code.andps(tmp, result);
FCODE(ucomis)(tmp, code.XmmBConst<fsize>(xword, fsize == 32 ? f32_smallest_normal : f64_smallest_normal));
code.jz(fallback, code.T_NEAR);
code.L(end);
code.jz(*fallback, code.T_NEAR);
code.L(*end);
code.SwitchToFarCode();
code.L(fallback);
ctx.deferred_emits.emplace_back([=, &code, &ctx] {
code.L(*fallback);
code.sub(rsp, 8);
ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx()));
@ -676,8 +677,8 @@ static void EmitFPMulAdd(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx()));
code.add(rsp, 8);
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
code.jmp(*end, code.T_NEAR);
});
ctx.reg_alloc.DefineValue(inst, result);
return;
@ -735,7 +736,7 @@ static void EmitFPMulX(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
const Xbyak::Xmm result = ctx.reg_alloc.ScratchXmm();
const Xbyak::Reg64 tmp = do_default_nan ? INVALID_REG : ctx.reg_alloc.ScratchGpr();
Xbyak::Label end, nan, op_are_nans;
SharedLabel end = GenSharedLabel(), nan = GenSharedLabel();
if (code.HasHostFeature(HostFeature::AVX)) {
FCODE(vmuls)(result, op1, op2);
@ -744,11 +745,13 @@ static void EmitFPMulX(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
FCODE(muls)(result, op2);
}
FCODE(ucomis)(result, result);
code.jp(nan, code.T_NEAR);
code.L(end);
code.jp(*nan, code.T_NEAR);
code.L(*end);
code.SwitchToFarCode();
code.L(nan);
ctx.deferred_emits.emplace_back([=, &code] {
Xbyak::Label op_are_nans;
code.L(*nan);
FCODE(ucomis)(op1, op2);
code.jp(op_are_nans);
if (code.HasHostFeature(HostFeature::AVX)) {
@ -759,15 +762,15 @@ static void EmitFPMulX(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
}
code.andps(result, code.XmmBConst<fsize>(xword, FP::FPInfo<FPT>::sign_mask));
code.orps(result, code.XmmBConst<fsize>(xword, FP::FPValue<FPT, false, 0, 2>()));
code.jmp(end, code.T_NEAR);
code.jmp(*end, code.T_NEAR);
code.L(op_are_nans);
if (do_default_nan) {
code.movaps(result, code.XmmBConst<fsize>(xword, FP::FPInfo<FPT>::DefaultNaN()));
code.jmp(end, code.T_NEAR);
code.jmp(*end, code.T_NEAR);
} else {
EmitPostProcessNaNs<fsize>(code, result, op1, op2, tmp, end);
EmitPostProcessNaNs<fsize>(code, result, op1, op2, tmp, *end);
}
code.SwitchToNearCode();
});
ctx.reg_alloc.DefineValue(inst, result);
}
@ -871,7 +874,7 @@ static void EmitFPRecipStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*
}
if (code.HasHostFeature(HostFeature::FMA)) {
Xbyak::Label end, fallback;
SharedLabel end = GenSharedLabel(), fallback = GenSharedLabel();
const Xbyak::Xmm operand1 = ctx.reg_alloc.UseXmm(args[0]);
const Xbyak::Xmm operand2 = ctx.reg_alloc.UseXmm(args[1]);
@ -880,11 +883,11 @@ static void EmitFPRecipStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*
code.movaps(result, code.XmmBConst<fsize>(xword, FP::FPValue<FPT, false, 0, 2>()));
FCODE(vfnmadd231s)(result, operand1, operand2);
FCODE(ucomis)(result, result);
code.jp(fallback, code.T_NEAR);
code.L(end);
code.jp(*fallback, code.T_NEAR);
code.L(*end);
code.SwitchToFarCode();
code.L(fallback);
ctx.deferred_emits.emplace_back([=, &code, &ctx] {
code.L(*fallback);
code.sub(rsp, 8);
ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx()));
@ -897,8 +900,8 @@ static void EmitFPRecipStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx()));
code.add(rsp, 8);
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
code.jmp(*end, code.T_NEAR);
});
ctx.reg_alloc.DefineValue(inst, result);
return;
@ -1034,8 +1037,7 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
const Xbyak::Xmm value = ctx.reg_alloc.ScratchXmm();
[[maybe_unused]] const Xbyak::Reg32 tmp = ctx.reg_alloc.ScratchGpr().cvt32();
Xbyak::Label fallback, bad_values, end, default_nan;
bool needs_fallback = false;
SharedLabel bad_values = GenSharedLabel(), end = GenSharedLabel();
code.movaps(value, operand);
@ -1045,7 +1047,7 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
// Detect NaNs, negatives, zeros, denormals and infinities
FCODE(ucomis)(value, code.XmmBConst<fsize>(xword, FPT(1) << FP::FPInfo<FPT>::explicit_mantissa_width));
code.jna(bad_values, code.T_NEAR);
code.jna(*bad_values, code.T_NEAR);
FCODE(sqrts)(value, value);
ICODE(mov)(result, code.XmmBConst<fsize>(xword, FP::FPValue<FPT, false, 0, 1>()));
@ -1054,11 +1056,13 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
ICODE(padd)(result, code.XmmBConst<fsize>(xword, fsize == 32 ? 0x00004000 : 0x0000'0800'0000'0000));
code.pand(result, xmm0);
code.L(end);
code.L(*end);
code.SwitchToFarCode();
ctx.deferred_emits.emplace_back([=, &code, &ctx] {
Xbyak::Label fallback, default_nan;
bool needs_fallback = false;
code.L(bad_values);
code.L(*bad_values);
if constexpr (fsize == 32) {
code.movd(tmp, operand);
@ -1080,18 +1084,18 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
if (ctx.FPCR().DN()) {
code.ucomiss(result, result);
code.jnp(end, code.T_NEAR);
code.jnp(*end, code.T_NEAR);
} else {
// FZ ? (a >= 0x80800000 && a <= 0xFF800000) : (a >= 0x80000001 && a <= 0xFF800000)
// !FZ path takes into account the subtraction by one from the earlier block
code.add(tmp, ctx.FPCR().FZ() ? 0x7F800000 : 0x80000000);
code.cmp(tmp, ctx.FPCR().FZ() ? 0x7F000001 : 0x7F800000);
code.jnb(end, code.T_NEAR);
code.jnb(*end, code.T_NEAR);
}
code.L(default_nan);
code.movd(result, code.XmmBConst<32>(xword, 0x7FC00000));
code.jmp(end, code.T_NEAR);
code.jmp(*end, code.T_NEAR);
} else {
Xbyak::Label nan, zero;
@ -1114,7 +1118,7 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
code.jmp(fallback);
} else {
// result = 0
code.jmp(end, code.T_NEAR);
code.jmp(*end, code.T_NEAR);
}
code.L(zero);
@ -1124,7 +1128,7 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
code.movaps(result, value);
code.por(result, code.XmmBConst<64>(xword, 0x7FF0'0000'0000'0000));
}
code.jmp(end, code.T_NEAR);
code.jmp(*end, code.T_NEAR);
code.L(nan);
if (!ctx.FPCR().DN()) {
@ -1134,12 +1138,12 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
code.movaps(result, operand);
code.por(result, code.XmmBConst<64>(xword, 0x0008'0000'0000'0000));
}
code.jmp(end, code.T_NEAR);
code.jmp(*end, code.T_NEAR);
}
code.L(default_nan);
code.movq(result, code.XmmBConst<64>(xword, 0x7FF8'0000'0000'0000));
code.jmp(end, code.T_NEAR);
code.jmp(*end, code.T_NEAR);
}
code.L(fallback);
@ -1153,10 +1157,9 @@ static void EmitFPRSqrtEstimate(BlockOfCode& code, EmitContext& ctx, IR::Inst* i
code.movq(result, rax);
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx()));
code.add(rsp, 8);
code.jmp(end, code.T_NEAR);
code.jmp(*end, code.T_NEAR);
}
code.SwitchToNearCode();
});
ctx.reg_alloc.DefineValue(inst, result);
} else {
@ -1201,7 +1204,7 @@ static void EmitFPRSqrtStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*
}
if (code.HasHostFeature(HostFeature::FMA | HostFeature::AVX)) {
Xbyak::Label end, fallback;
SharedLabel end = GenSharedLabel(), fallback = GenSharedLabel();
const Xbyak::Xmm operand1 = ctx.reg_alloc.UseXmm(args[0]);
const Xbyak::Xmm operand2 = ctx.reg_alloc.UseXmm(args[1]);
@ -1220,13 +1223,13 @@ static void EmitFPRSqrtStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*
code.cmp(tmp.cvt16(), fsize == 32 ? 0x7f00 : 0x7fe0);
ctx.reg_alloc.Release(tmp);
code.jae(fallback, code.T_NEAR);
code.jae(*fallback, code.T_NEAR);
FCODE(vmuls)(result, result, code.XmmBConst<fsize>(xword, FP::FPValue<FPT, false, -1, 1>()));
code.L(end);
code.L(*end);
code.SwitchToFarCode();
code.L(fallback);
ctx.deferred_emits.emplace_back([=, &code, &ctx] {
code.L(*fallback);
code.sub(rsp, 8);
ABI_PushCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx()));
@ -1239,8 +1242,8 @@ static void EmitFPRSqrtStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*
ABI_PopCallerSaveRegistersAndAdjustStackExcept(code, HostLocXmmIdx(result.getIdx()));
code.add(rsp, 8);
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
code.jmp(*end, code.T_NEAR);
});
ctx.reg_alloc.DefineValue(inst, result);
return;
@ -1528,22 +1531,22 @@ static void EmitFPToFixed(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
if constexpr (isize == 64) {
const Xbyak::Xmm scratch = ctx.reg_alloc.ScratchXmm();
Xbyak::Label saturate_max, end;
if (!unsigned_) {
SharedLabel saturate_max = GenSharedLabel(), end = GenSharedLabel();
ZeroIfNaN<64>(code, src, scratch);
code.movsd(scratch, code.XmmBConst<64>(xword, f64_max_s64_lim));
code.comisd(scratch, src);
code.jna(saturate_max, code.T_NEAR);
code.jna(*saturate_max, code.T_NEAR);
code.cvttsd2si(result, src); // 64 bit gpr
code.L(end);
code.L(*end);
code.SwitchToFarCode();
code.L(saturate_max);
ctx.deferred_emits.emplace_back([=, &code] {
code.L(*saturate_max);
code.mov(result, 0x7FFF'FFFF'FFFF'FFFF);
code.jmp(end, code.T_NEAR);
code.SwitchToNearCode();
code.jmp(*end, code.T_NEAR);
});
} else {
Xbyak::Label below_max;