mirrors/dynarmic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

emit_x64_memory: Ensure 128-bit loads/stores are atomic

Browse source
This commit is contained in:
merry 2022-04-02 19:33:48 +01:00
parent e27733464b
commit 8bcd46b7e9
2 changed files with 61 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
src/dynarmic/backend/x64/emit_x64_memory.cpp.inc
View file

@ -75,6 +75,14 @@ void AxxEmitX64::EmitMemoryRead(AxxEmitContext& ctx, IR::Inst* inst) {
return; return;
} }
if (ordered && bitsize == 128) {
// Required for atomic 128-bit loads/stores
ctx.reg_alloc.ScratchGpr(HostLoc::RAX);
ctx.reg_alloc.ScratchGpr(HostLoc::RBX);
ctx.reg_alloc.ScratchGpr(HostLoc::RCX);
ctx.reg_alloc.ScratchGpr(HostLoc::RDX);
}
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]); const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const int value_idx = bitsize == 128 ? ctx.reg_alloc.ScratchXmm().getIdx() : ctx.reg_alloc.ScratchGpr().getIdx(); const int value_idx = bitsize == 128 ? ctx.reg_alloc.ScratchXmm().getIdx() : ctx.reg_alloc.ScratchGpr().getIdx();
@ -145,6 +153,14 @@ void AxxEmitX64::EmitMemoryWrite(AxxEmitContext& ctx, IR::Inst* inst) {
return; return;
} }
if (ordered && bitsize == 128) {
// Required for atomic 128-bit loads/stores
ctx.reg_alloc.ScratchGpr(HostLoc::RAX);
ctx.reg_alloc.ScratchGpr(HostLoc::RBX);
ctx.reg_alloc.ScratchGpr(HostLoc::RCX);
ctx.reg_alloc.ScratchGpr(HostLoc::RDX);
}
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]); const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const int value_idx = bitsize == 128 const int value_idx = bitsize == 128
? ctx.reg_alloc.UseXmm(args[1]).getIdx() ? ctx.reg_alloc.UseXmm(args[1]).getIdx()
@ -305,6 +321,14 @@ void AxxEmitX64::EmitExclusiveReadMemoryInline(AxxEmitContext& ctx, IR::Inst* in
auto args = ctx.reg_alloc.GetArgumentInfo(inst); auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const bool ordered = true; const bool ordered = true;
if (ordered && bitsize == 128) {
// Required for atomic 128-bit loads/stores
ctx.reg_alloc.ScratchGpr(HostLoc::RAX);
ctx.reg_alloc.ScratchGpr(HostLoc::RBX);
ctx.reg_alloc.ScratchGpr(HostLoc::RCX);
ctx.reg_alloc.ScratchGpr(HostLoc::RDX);
}
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]); const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const int value_idx = bitsize == 128 ? ctx.reg_alloc.ScratchXmm().getIdx() : ctx.reg_alloc.ScratchGpr().getIdx(); const int value_idx = bitsize == 128 ? ctx.reg_alloc.ScratchXmm().getIdx() : ctx.reg_alloc.ScratchGpr().getIdx();
const Xbyak::Reg64 tmp = ctx.reg_alloc.ScratchGpr(); const Xbyak::Reg64 tmp = ctx.reg_alloc.ScratchGpr();

45
src/dynarmic/backend/x64/emit_x64_memory.h
View file

@ -201,10 +201,13 @@ template<>
template<std::size_t bitsize> template<std::size_t bitsize>
const void* EmitReadMemoryMov(BlockOfCode& code, int value_idx, const Xbyak::RegExp& addr, bool ordered) { const void* EmitReadMemoryMov(BlockOfCode& code, int value_idx, const Xbyak::RegExp& addr, bool ordered) {
if (ordered) { if (ordered) {
if constexpr (bitsize == 128) { if constexpr (bitsize != 128) {
code.mfence();
} else {
code.xor_(Xbyak::Reg32{value_idx}, Xbyak::Reg32{value_idx}); code.xor_(Xbyak::Reg32{value_idx}, Xbyak::Reg32{value_idx});
} else {
code.xor_(eax, eax);
code.xor_(ebx, ebx);
code.xor_(ecx, ecx);
code.xor_(edx, edx);
} }
const void* fastmem_location = code.getCurr(); const void* fastmem_location = code.getCurr();
@ -226,8 +229,16 @@ const void* EmitReadMemoryMov(BlockOfCode& code, int value_idx, const Xbyak::Reg
code.xadd(qword[addr], Xbyak::Reg64{value_idx}); code.xadd(qword[addr], Xbyak::Reg64{value_idx});
break; break;
case 128: case 128:
// TODO (HACK): Detect CPUs where this load is not atomic code.lock();
code.movaps(Xbyak::Xmm{value_idx}, xword[addr]); code.cmpxchg16b(xword[addr]);
if (code.HasHostFeature(HostFeature::SSE41)) {
code.movq(Xbyak::Xmm{value_idx}, rax);
code.pinsrq(Xbyak::Xmm{value_idx}, rdx, 1);
} else {
code.movq(Xbyak::Xmm{value_idx}, rax);
code.movq(xmm0, rdx);
code.punpcklqdq(Xbyak::Xmm{value_idx}, xmm0);
}
break; break;
default: default:
ASSERT_FALSE("Invalid bitsize"); ASSERT_FALSE("Invalid bitsize");
@ -261,6 +272,20 @@ const void* EmitReadMemoryMov(BlockOfCode& code, int value_idx, const Xbyak::Reg
template<std::size_t bitsize> template<std::size_t bitsize>
const void* EmitWriteMemoryMov(BlockOfCode& code, const Xbyak::RegExp& addr, int value_idx, bool ordered) { const void* EmitWriteMemoryMov(BlockOfCode& code, const Xbyak::RegExp& addr, int value_idx, bool ordered) {
if (ordered) { if (ordered) {
if constexpr (bitsize == 128) {
code.xor_(eax, eax);
code.xor_(edx, edx);
if (code.HasHostFeature(HostFeature::SSE41)) {
code.movq(rbx, Xbyak::Xmm{value_idx});
code.pextrq(rcx, Xbyak::Xmm{value_idx}, 1);
} else {
code.movaps(xmm0, Xbyak::Xmm{value_idx});
code.movq(rbx, xmm0);
code.punpckhqdq(xmm0, xmm0);
code.movq(rcx, xmm0);
}
}
const void* fastmem_location = code.getCurr(); const void* fastmem_location = code.getCurr();
switch (bitsize) { switch (bitsize) {
case 8: case 8:
@ -275,10 +300,14 @@ const void* EmitWriteMemoryMov(BlockOfCode& code, const Xbyak::RegExp& addr, int
case 64: case 64:
code.xchg(qword[addr], Xbyak::Reg64{value_idx}); code.xchg(qword[addr], Xbyak::Reg64{value_idx});
break; break;
case 128: case 128: {
code.movaps(xword[addr], Xbyak::Xmm{value_idx}); Xbyak::Label loop;
code.mfence(); code.L(loop);
code.lock();
code.cmpxchg16b(xword[addr]);
code.jnz(loop);
break; break;
}
default: default:
ASSERT_FALSE("Invalid bitsize"); ASSERT_FALSE("Invalid bitsize");
} }