mirrors/dynarmic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Exclusive Monitor: Rework exclusive monitor interface.

Browse source
This commit is contained in:
Fernando Sahmkow 2020-03-10 18:07:44 -04:00 committed by MerryMage
parent b5d8b24a3c
commit 97b9d3e058
15 changed files with 278 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
include/dynarmic/A64/config.h
View file

@ -83,6 +83,13 @@ struct UserCallbacks {
virtual void MemoryWrite64(VAddr vaddr, std::uint64_t value) = 0; virtual void MemoryWrite64(VAddr vaddr, std::uint64_t value) = 0;
virtual void MemoryWrite128(VAddr vaddr, Vector value) = 0; virtual void MemoryWrite128(VAddr vaddr, Vector value) = 0;
// Writes through these callbacks may not be aligned.
virtual bool MemoryWriteExclusive8(VAddr vaddr, std::uint8_t value, std::uint8_t expected) = 0;
virtual bool MemoryWriteExclusive16(VAddr vaddr, std::uint16_t value, std::uint16_t expected) = 0;
virtual bool MemoryWriteExclusive32(VAddr vaddr, std::uint32_t value, std::uint32_t expected) = 0;
virtual bool MemoryWriteExclusive64(VAddr vaddr, std::uint64_t value, std::uint64_t expected) = 0;
virtual bool MemoryWriteExclusive128(VAddr vaddr, Vector value, Vector expected) = 0;
// If this callback returns true, the JIT will assume MemoryRead* callbacks will always // If this callback returns true, the JIT will assume MemoryRead* callbacks will always
// return the same value at any point in time for this vaddr. The JIT may use this information // return the same value at any point in time for this vaddr. The JIT may use this information
// in optimizations. // in optimizations.

34
include/dynarmic/A64/exclusive_monitor.h
View file

@ -6,14 +6,17 @@
#pragma once #pragma once
#include <atomic> #include <atomic>
#include <array>
#include <cstddef> #include <cstddef>
#include <cstdint> #include <cstdint>
#include <cstring>
#include <vector> #include <vector>
namespace Dynarmic { namespace Dynarmic {
namespace A64 { namespace A64 {
using VAddr = std::uint64_t; using VAddr = std::uint64_t;
using Vector = std::array<std::uint64_t, 2>;
class ExclusiveMonitor { class ExclusiveMonitor {
public: public:
@ -26,31 +29,45 @@ public:
/// Marks a region containing [address, address+size) to be exclusive to /// Marks a region containing [address, address+size) to be exclusive to
/// processor processor_id. /// processor processor_id.
void Mark(size_t processor_id, VAddr address, size_t size); template <typename T, typename Function>
T ReadAndMark(size_t processor_id, VAddr address, Function op) {
static_assert(std::is_trivially_copyable_v<T>);
const VAddr masked_address = address & RESERVATION_GRANULE_MASK;
Lock();
exclusive_addresses[processor_id] = masked_address;
const T value = op();
std::memcpy(exclusive_values[processor_id].data(), &value, sizeof(T));
Unlock();
return value;
}
/// Checks to see if processor processor_id has exclusive access to the /// Checks to see if processor processor_id has exclusive access to the
/// specified region. If it does, executes the operation then clears /// specified region. If it does, executes the operation then clears
/// the exclusive state for processors if their exclusive region(s) /// the exclusive state for processors if their exclusive region(s)
/// contain [address, address+size). /// contain [address, address+size).
template <typename Function> template <typename T, typename Function>
bool DoExclusiveOperation(size_t processor_id, VAddr address, size_t size, Function op) { bool DoExclusiveOperation(size_t processor_id, VAddr address, Function op) {
if (!CheckAndClear(processor_id, address, size)) { static_assert(std::is_trivially_copyable_v<T>);
if (!CheckAndClear(processor_id, address)) {
return false; return false;
} }
op(); T saved_value;
std::memcpy(&saved_value, exclusive_values[processor_id].data(), sizeof(T));
const bool result = op(saved_value);
Unlock(); Unlock();
return true; return result;
} }
/// Unmark everything. /// Unmark everything.
void Clear(); void Clear();
/// Unmark processor id /// Unmark processor id
void Clear(size_t processor_id); void ClearProcessor(size_t processor_id);
private: private:
bool CheckAndClear(size_t processor_id, VAddr address, size_t size); bool CheckAndClear(size_t processor_id, VAddr address);
void Lock(); void Lock();
void Unlock(); void Unlock();
@ -59,6 +76,7 @@ private:
static constexpr VAddr INVALID_EXCLUSIVE_ADDRESS = 0xDEAD'DEAD'DEAD'DEADull; static constexpr VAddr INVALID_EXCLUSIVE_ADDRESS = 0xDEAD'DEAD'DEAD'DEADull;
std::atomic_flag is_locked; std::atomic_flag is_locked;
std::vector<VAddr> exclusive_addresses; std::vector<VAddr> exclusive_addresses;
std::vector<Vector> exclusive_values;
}; };
} // namespace A64 } // namespace A64

266
src/backend/x64/a64_emit_x64.cpp
View file

@ -703,30 +703,6 @@ void A64EmitX64::EmitA64ClearExclusive(A64EmitContext&, IR::Inst*) {
code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(0)); code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(0));
} }
void A64EmitX64::EmitA64SetExclusive(A64EmitContext& ctx, IR::Inst* inst) {
if (conf.global_monitor) {
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
ctx.reg_alloc.HostCall(nullptr, {}, args[0], args[1]);
code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(1));
code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr, u8 size) {
conf.global_monitor->Mark(conf.processor_id, vaddr, size);
}
);
return;
}
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
ASSERT(args[1].IsImmediate());
const Xbyak::Reg64 address = ctx.reg_alloc.UseGpr(args[0]);
code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(1));
code.mov(qword[r15 + offsetof(A64JitState, exclusive_address)], address);
}
namespace { namespace {
constexpr size_t page_bits = 12; constexpr size_t page_bits = 12;
@ -951,6 +927,89 @@ void A64EmitX64::EmitA64ReadMemory128(A64EmitContext& ctx, IR::Inst* inst) {
ctx.reg_alloc.DefineValue(inst, xmm1); ctx.reg_alloc.DefineValue(inst, xmm1);
} }
void A64EmitX64::EmitA64ExclusiveReadMemory8(A64EmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor != nullptr);
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
ctx.reg_alloc.HostCall(inst, {}, args[0]);
code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(1));
code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr) -> u8 {
return conf.global_monitor->ReadAndMark<u8>(conf.processor_id, vaddr, [&]() -> u8 {
return conf.callbacks->MemoryRead8(vaddr);
});
}
);
}
void A64EmitX64::EmitA64ExclusiveReadMemory16(A64EmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor != nullptr);
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
ctx.reg_alloc.HostCall(inst, {}, args[0]);
code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(1));
code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr) -> u16 {
return conf.global_monitor->ReadAndMark<u16>(conf.processor_id, vaddr, [&]() -> u16 {
return conf.callbacks->MemoryRead16(vaddr);
});
}
);
}
void A64EmitX64::EmitA64ExclusiveReadMemory32(A64EmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor != nullptr);
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
ctx.reg_alloc.HostCall(inst, {}, args[0]);
code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(1));
code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr) -> u32 {
return conf.global_monitor->ReadAndMark<u32>(conf.processor_id, vaddr, [&]() -> u32 {
return conf.callbacks->MemoryRead32(vaddr);
});
}
);
}
void A64EmitX64::EmitA64ExclusiveReadMemory64(A64EmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor != nullptr);
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
ctx.reg_alloc.HostCall(inst, {}, args[0]);
code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(1));
code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr) -> u64 {
return conf.global_monitor->ReadAndMark<u64>(conf.processor_id, vaddr, [&]() -> u64 {
return conf.callbacks->MemoryRead64(vaddr);
});
}
);
}
void A64EmitX64::EmitA64ExclusiveReadMemory128(A64EmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor != nullptr);
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const Xbyak::Xmm result = ctx.reg_alloc.ScratchXmm();
ctx.reg_alloc.EndOfAllocScope();
ctx.reg_alloc.HostCall(nullptr, {}, args[0]);
code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(1));
code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
code.sub(rsp, 16 + ABI_SHADOW_SPACE);
code.lea(code.ABI_PARAM3, ptr[rsp + ABI_SHADOW_SPACE]);
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr, A64::Vector& ret) {
ret = conf.global_monitor->ReadAndMark<A64::Vector>(conf.processor_id, vaddr, [&]() -> A64::Vector {
return conf.callbacks->MemoryRead128(vaddr);
});
}
);
code.movups(result, xword[rsp + ABI_SHADOW_SPACE]);
code.add(rsp, 16 + ABI_SHADOW_SPACE);
ctx.reg_alloc.DefineValue(inst, result);
}
void A64EmitX64::EmitA64WriteMemory8(A64EmitContext& ctx, IR::Inst* inst) { void A64EmitX64::EmitA64WriteMemory8(A64EmitContext& ctx, IR::Inst* inst) {
if (conf.page_table) { if (conf.page_table) {
EmitDirectPageTableMemoryWrite(ctx, inst, 8); EmitDirectPageTableMemoryWrite(ctx, inst, 8);
@ -1024,105 +1083,84 @@ void A64EmitX64::EmitA64WriteMemory128(A64EmitContext& ctx, IR::Inst* inst) {
} }
void A64EmitX64::EmitExclusiveWrite(A64EmitContext& ctx, IR::Inst* inst, size_t bitsize) { void A64EmitX64::EmitExclusiveWrite(A64EmitContext& ctx, IR::Inst* inst, size_t bitsize) {
if (conf.global_monitor) { ASSERT(conf.global_monitor != nullptr);
auto args = ctx.reg_alloc.GetArgumentInfo(inst); auto args = ctx.reg_alloc.GetArgumentInfo(inst);
if (bitsize != 128) { if (bitsize != 128) {
ctx.reg_alloc.HostCall(inst, {}, args[0], args[1]); ctx.reg_alloc.HostCall(inst, {}, args[0], args[1]);
} else { } else {
ctx.reg_alloc.Use(args[0], ABI_PARAM2); ctx.reg_alloc.Use(args[0], ABI_PARAM2);
ctx.reg_alloc.Use(args[1], HostLoc::XMM1); ctx.reg_alloc.Use(args[1], HostLoc::XMM1);
ctx.reg_alloc.EndOfAllocScope(); ctx.reg_alloc.EndOfAllocScope();
ctx.reg_alloc.HostCall(inst); ctx.reg_alloc.HostCall(inst);
}
Xbyak::Label end;
code.mov(code.ABI_RETURN, u32(1));
code.cmp(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(0));
code.je(end);
code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
switch (bitsize) {
case 8:
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr, u8 value) -> u32 {
return conf.global_monitor->DoExclusiveOperation(conf.processor_id, vaddr, 1, [&]{
conf.callbacks->MemoryWrite8(vaddr, value);
}) ? 0 : 1;
}
);
break;
case 16:
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr, u16 value) -> u32 {
return conf.global_monitor->DoExclusiveOperation(conf.processor_id, vaddr, 2, [&]{
conf.callbacks->MemoryWrite16(vaddr, value);
}) ? 0 : 1;
}
);
break;
case 32:
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr, u32 value) -> u32 {
return conf.global_monitor->DoExclusiveOperation(conf.processor_id, vaddr, 4, [&]{
conf.callbacks->MemoryWrite32(vaddr, value);
}) ? 0 : 1;
}
);
break;
case 64:
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr, u64 value) -> u32 {
return conf.global_monitor->DoExclusiveOperation(conf.processor_id, vaddr, 8, [&]{
conf.callbacks->MemoryWrite64(vaddr, value);
}) ? 0 : 1;
}
);
break;
case 128:
code.sub(rsp, 16 + ABI_SHADOW_SPACE);
code.lea(code.ABI_PARAM3, ptr[rsp + ABI_SHADOW_SPACE]);
code.movaps(xword[code.ABI_PARAM3], xmm1);
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr, A64::Vector& value) -> u32 {
return conf.global_monitor->DoExclusiveOperation(conf.processor_id, vaddr, 16, [&]{
conf.callbacks->MemoryWrite128(vaddr, value);
}) ? 0 : 1;
}
);
code.add(rsp, 16 + ABI_SHADOW_SPACE);
break;
default:
UNREACHABLE();
}
code.L(end);
return;
} }
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
const Xbyak::Reg64 vaddr = ctx.reg_alloc.UseGpr(args[0]);
const int value_idx = bitsize != 128
? ctx.reg_alloc.UseGpr(args[1]).getIdx()
: ctx.reg_alloc.UseXmm(args[1]).getIdx();
Xbyak::Label end; Xbyak::Label end;
const Xbyak::Reg32 passed = ctx.reg_alloc.ScratchGpr().cvt32();
const Xbyak::Reg64 tmp = ctx.reg_alloc.ScratchGpr();
code.mov(passed, u32(1)); code.mov(code.ABI_RETURN, u32(1));
code.cmp(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(0)); code.cmp(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(0));
code.je(end); code.je(end);
code.mov(tmp, vaddr);
code.xor_(tmp, qword[r15 + offsetof(A64JitState, exclusive_address)]);
code.test(tmp, static_cast<u32>(A64JitState::RESERVATION_GRANULE_MASK & 0xFFFF'FFFF));
code.jne(end);
code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(0)); code.mov(code.byte[r15 + offsetof(A64JitState, exclusive_state)], u8(0));
code.call(write_fallbacks[std::make_tuple(bitsize, vaddr.getIdx(), value_idx)]); code.mov(code.ABI_PARAM1, reinterpret_cast<u64>(&conf));
code.xor_(passed, passed); switch (bitsize) {
case 8:
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr, u8 value) -> u32 {
return conf.global_monitor->DoExclusiveOperation<u8>(conf.processor_id, vaddr,
[&](u8 expected) -> bool {
return conf.callbacks->MemoryWriteExclusive8(vaddr, value, expected);
}) ? 0 : 1;
}
);
break;
case 16:
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr, u16 value) -> u32 {
return conf.global_monitor->DoExclusiveOperation<u16>(conf.processor_id, vaddr,
[&](u16 expected) -> bool {
return conf.callbacks->MemoryWriteExclusive16(vaddr, value, expected);
}) ? 0 : 1;
}
);
break;
case 32:
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr, u32 value) -> u32 {
return conf.global_monitor->DoExclusiveOperation<u32>(conf.processor_id, vaddr,
[&](u32 expected) -> bool {
return conf.callbacks->MemoryWriteExclusive32(vaddr, value, expected);
}) ? 0 : 1;
}
);
break;
case 64:
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr, u64 value) -> u32 {
return conf.global_monitor->DoExclusiveOperation<u64>(conf.processor_id, vaddr,
[&](u64 expected) -> bool {
return conf.callbacks->MemoryWriteExclusive64(vaddr, value, expected);
}) ? 0 : 1;
}
);
break;
case 128:
code.sub(rsp, 16 + ABI_SHADOW_SPACE);
code.lea(code.ABI_PARAM3, ptr[rsp + ABI_SHADOW_SPACE]);
code.movaps(xword[code.ABI_PARAM3], xmm1);
code.CallLambda(
[](A64::UserConfig& conf, u64 vaddr, A64::Vector& value) -> u32 {
return conf.global_monitor->DoExclusiveOperation<A64::Vector>(conf.processor_id, vaddr,
[&](A64::Vector expected) -> bool {
return conf.callbacks->MemoryWriteExclusive128(vaddr, value, expected);
}) ? 0 : 1;
}
);
code.add(rsp, 16 + ABI_SHADOW_SPACE);
break;
default:
UNREACHABLE();
}
code.L(end); code.L(end);
ctx.reg_alloc.DefineValue(inst, passed);
} }
void A64EmitX64::EmitA64ExclusiveWriteMemory8(A64EmitContext& ctx, IR::Inst* inst) { void A64EmitX64::EmitA64ExclusiveWriteMemory8(A64EmitContext& ctx, IR::Inst* inst) {

17
src/backend/x64/a64_exclusive_monitor.cpp
View file

@ -11,7 +11,8 @@
namespace Dynarmic { namespace Dynarmic {
namespace A64 { namespace A64 {
ExclusiveMonitor::ExclusiveMonitor(size_t processor_count) : exclusive_addresses(processor_count, INVALID_EXCLUSIVE_ADDRESS) { ExclusiveMonitor::ExclusiveMonitor(size_t processor_count) :
exclusive_addresses(processor_count, INVALID_EXCLUSIVE_ADDRESS), exclusive_values(processor_count) {
Unlock(); Unlock();
} }
@ -19,15 +20,6 @@ size_t ExclusiveMonitor::GetProcessorCount() const {
return exclusive_addresses.size(); return exclusive_addresses.size();
} }
void ExclusiveMonitor::Mark(size_t processor_id, VAddr address, size_t size) {
ASSERT(size <= 16);
const VAddr masked_address = address & RESERVATION_GRANULE_MASK;
Lock();
exclusive_addresses[processor_id] = masked_address;
Unlock();
}
void ExclusiveMonitor::Lock() { void ExclusiveMonitor::Lock() {
while (is_locked.test_and_set(std::memory_order_acquire)) {} while (is_locked.test_and_set(std::memory_order_acquire)) {}
} }
@ -36,8 +28,7 @@ void ExclusiveMonitor::Unlock() {
is_locked.clear(std::memory_order_release); is_locked.clear(std::memory_order_release);
} }
bool ExclusiveMonitor::CheckAndClear(size_t processor_id, VAddr address, size_t size) { bool ExclusiveMonitor::CheckAndClear(size_t processor_id, VAddr address) {
ASSERT(size <= 16);
const VAddr masked_address = address & RESERVATION_GRANULE_MASK; const VAddr masked_address = address & RESERVATION_GRANULE_MASK;
Lock(); Lock();
@ -60,7 +51,7 @@ void ExclusiveMonitor::Clear() {
Unlock(); Unlock();
} }
void ExclusiveMonitor::Clear(size_t processor_id) { void ExclusiveMonitor::ClearProcessor(size_t processor_id) {
Lock(); Lock();
exclusive_addresses[processor_id] = INVALID_EXCLUSIVE_ADDRESS; exclusive_addresses[processor_id] = INVALID_EXCLUSIVE_ADDRESS;
Unlock(); Unlock();

1
src/backend/x64/a64_jitstate.h
View file

@ -59,7 +59,6 @@ struct A64JitState {
// Exclusive state // Exclusive state
static constexpr u64 RESERVATION_GRANULE_MASK = 0xFFFF'FFFF'FFFF'FFF0ull; static constexpr u64 RESERVATION_GRANULE_MASK = 0xFFFF'FFFF'FFFF'FFF0ull;
u8 exclusive_state = 0; u8 exclusive_state = 0;
u64 exclusive_address = 0;
static constexpr size_t RSBSize = 8; // MUST be a power of 2. static constexpr size_t RSBSize = 8; // MUST be a power of 2.
static constexpr size_t RSBPtrMask = RSBSize - 1; static constexpr size_t RSBPtrMask = RSBSize - 1;

25
src/frontend/A64/ir_emitter.cpp
View file

@ -100,11 +100,6 @@ void IREmitter::ClearExclusive() {
Inst(Opcode::A64ClearExclusive); Inst(Opcode::A64ClearExclusive);
} }
void IREmitter::SetExclusive(const IR::U64& vaddr, size_t byte_size) {
ASSERT(byte_size == 1 || byte_size == 2 || byte_size == 4 || byte_size == 8 || byte_size == 16);
Inst(Opcode::A64SetExclusive, vaddr, Imm8(u8(byte_size)));
}
IR::U8 IREmitter::ReadMemory8(const IR::U64& vaddr) { IR::U8 IREmitter::ReadMemory8(const IR::U64& vaddr) {
return Inst<IR::U8>(Opcode::A64ReadMemory8, vaddr); return Inst<IR::U8>(Opcode::A64ReadMemory8, vaddr);
} }
@ -125,6 +120,26 @@ IR::U128 IREmitter::ReadMemory128(const IR::U64& vaddr) {
return Inst<IR::U128>(Opcode::A64ReadMemory128, vaddr); return Inst<IR::U128>(Opcode::A64ReadMemory128, vaddr);
} }
IR::U8 IREmitter::ExclusiveReadMemory8(const IR::U64& vaddr) {
return Inst<IR::U8>(Opcode::A64ExclusiveReadMemory8, vaddr);
}
IR::U16 IREmitter::ExclusiveReadMemory16(const IR::U64& vaddr) {
return Inst<IR::U16>(Opcode::A64ExclusiveReadMemory16, vaddr);
}
IR::U32 IREmitter::ExclusiveReadMemory32(const IR::U64& vaddr) {
return Inst<IR::U32>(Opcode::A64ExclusiveReadMemory32, vaddr);
}
IR::U64 IREmitter::ExclusiveReadMemory64(const IR::U64& vaddr) {
return Inst<IR::U64>(Opcode::A64ExclusiveReadMemory64, vaddr);
}
IR::U128 IREmitter::ExclusiveReadMemory128(const IR::U64& vaddr) {
return Inst<IR::U128>(Opcode::A64ExclusiveReadMemory128, vaddr);
}
void IREmitter::WriteMemory8(const IR::U64& vaddr, const IR::U8& value) { void IREmitter::WriteMemory8(const IR::U64& vaddr, const IR::U8& value) {
Inst(Opcode::A64WriteMemory8, vaddr, value); Inst(Opcode::A64WriteMemory8, vaddr, value);
} }

6
src/frontend/A64/ir_emitter.h
View file

@ -54,12 +54,16 @@ public:
void SetTPIDR(const IR::U64& value); void SetTPIDR(const IR::U64& value);
void ClearExclusive(); void ClearExclusive();
void SetExclusive(const IR::U64& vaddr, size_t byte_size);
IR::U8 ReadMemory8(const IR::U64& vaddr); IR::U8 ReadMemory8(const IR::U64& vaddr);
IR::U16 ReadMemory16(const IR::U64& vaddr); IR::U16 ReadMemory16(const IR::U64& vaddr);
IR::U32 ReadMemory32(const IR::U64& vaddr); IR::U32 ReadMemory32(const IR::U64& vaddr);
IR::U64 ReadMemory64(const IR::U64& vaddr); IR::U64 ReadMemory64(const IR::U64& vaddr);
IR::U128 ReadMemory128(const IR::U64& vaddr); IR::U128 ReadMemory128(const IR::U64& vaddr);
IR::U8 ExclusiveReadMemory8(const IR::U64& vaddr);
IR::U16 ExclusiveReadMemory16(const IR::U64& vaddr);
IR::U32 ExclusiveReadMemory32(const IR::U64& vaddr);
IR::U64 ExclusiveReadMemory64(const IR::U64& vaddr);
IR::U128 ExclusiveReadMemory128(const IR::U64& vaddr);
void WriteMemory8(const IR::U64& vaddr, const IR::U8& value); void WriteMemory8(const IR::U64& vaddr, const IR::U8& value);
void WriteMemory16(const IR::U64& vaddr, const IR::U16& value); void WriteMemory16(const IR::U64& vaddr, const IR::U16& value);
void WriteMemory32(const IR::U64& vaddr, const IR::U32& value); void WriteMemory32(const IR::U64& vaddr, const IR::U32& value);

19
src/frontend/A64/translate/impl/impl.cpp
View file

@ -308,7 +308,24 @@ void TranslatorVisitor::Mem(IR::U64 address, size_t bytesize, IR::AccType /*acc_
} }
} }
IR::U32 TranslatorVisitor::ExclusiveMem(IR::U64 address, size_t bytesize, IR::AccType /*acc_type*/, IR::UAnyU128 value) { IR::UAnyU128 TranslatorVisitor::ExclusiveMem(IR::U64 address, size_t bytesize, IR::AccType /*acctype*/) {
switch (bytesize) {
case 1:
return ir.ExclusiveReadMemory8(address);
case 2:
return ir.ExclusiveReadMemory16(address);
case 4:
return ir.ExclusiveReadMemory32(address);
case 8:
return ir.ExclusiveReadMemory64(address);
case 16:
return ir.ExclusiveReadMemory128(address);
default:
ASSERT_FALSE("Invalid bytesize parameter {}", bytesize);
}
}
IR::U32 TranslatorVisitor::ExclusiveMem(IR::U64 address, size_t bytesize, IR::AccType /*acctype*/, IR::UAnyU128 value) {
switch (bytesize) { switch (bytesize) {
case 1: case 1:
return ir.ExclusiveWriteMemory8(address, value); return ir.ExclusiveWriteMemory8(address, value);

1
src/frontend/A64/translate/impl/impl.h
View file

@ -57,6 +57,7 @@ struct TranslatorVisitor final {
IR::UAnyU128 Mem(IR::U64 address, size_t size, IR::AccType acctype); IR::UAnyU128 Mem(IR::U64 address, size_t size, IR::AccType acctype);
void Mem(IR::U64 address, size_t size, IR::AccType acctype, IR::UAnyU128 value); void Mem(IR::U64 address, size_t size, IR::AccType acctype, IR::UAnyU128 value);
IR::UAnyU128 ExclusiveMem(IR::U64 address, size_t size, IR::AccType acctype);
IR::U32 ExclusiveMem(IR::U64 address, size_t size, IR::AccType acctype, IR::UAnyU128 value); IR::U32 ExclusiveMem(IR::U64 address, size_t size, IR::AccType acctype, IR::UAnyU128 value);
IR::U32U64 SignExtend(IR::UAny value, size_t to_size); IR::U32U64 SignExtend(IR::UAny value, size_t to_size);

3
src/frontend/A64/translate/impl/load_store_exclusive.cpp
View file

@ -56,8 +56,7 @@ static bool ExclusiveSharedDecodeAndOperation(TranslatorVisitor& v, bool pair, s
break; break;
} }
case IR::MemOp::LOAD: { case IR::MemOp::LOAD: {
v.ir.SetExclusive(address, dbytes); const IR::UAnyU128 data = v.ExclusiveMem(address, dbytes, acctype);
const IR::UAnyU128 data = v.Mem(address, dbytes, acctype);
if (pair && elsize == 64) { if (pair && elsize == 64) {
v.X(64, Rt, v.ir.VectorGetElement(64, data, 0)); v.X(64, Rt, v.ir.VectorGetElement(64, data, 0));
v.X(64, *Rt2, v.ir.VectorGetElement(64, data, 1)); v.X(64, *Rt2, v.ir.VectorGetElement(64, data, 1));

18
src/frontend/ir/microinstruction.cpp
View file

@ -99,6 +99,20 @@ bool Inst::IsSharedMemoryReadOrWrite() const {
return IsSharedMemoryRead() || IsSharedMemoryWrite(); return IsSharedMemoryRead() || IsSharedMemoryWrite();
} }
bool Inst::IsExclusiveMemoryRead() const {
switch (op) {
case Opcode::A64ExclusiveReadMemory8:
case Opcode::A64ExclusiveReadMemory16:
case Opcode::A64ExclusiveReadMemory32:
case Opcode::A64ExclusiveReadMemory64:
case Opcode::A64ExclusiveReadMemory128:
return true;
default:
return false;
}
}
bool Inst::IsExclusiveMemoryWrite() const { bool Inst::IsExclusiveMemoryWrite() const {
switch (op) { switch (op) {
case Opcode::A32ExclusiveWriteMemory8: case Opcode::A32ExclusiveWriteMemory8:
@ -118,7 +132,7 @@ bool Inst::IsExclusiveMemoryWrite() const {
} }
bool Inst::IsMemoryRead() const { bool Inst::IsMemoryRead() const {
return IsSharedMemoryRead(); return IsSharedMemoryRead() || IsExclusiveMemoryRead();
} }
bool Inst::IsMemoryWrite() const { bool Inst::IsMemoryWrite() const {
@ -457,7 +471,7 @@ bool Inst::AltersExclusiveState() const {
return op == Opcode::A32ClearExclusive || return op == Opcode::A32ClearExclusive ||
op == Opcode::A32SetExclusive || op == Opcode::A32SetExclusive ||
op == Opcode::A64ClearExclusive || op == Opcode::A64ClearExclusive ||
op == Opcode::A64SetExclusive || IsExclusiveMemoryRead() ||
IsExclusiveMemoryWrite(); IsExclusiveMemoryWrite();
} }

2
src/frontend/ir/microinstruction.h
View file

@ -44,6 +44,8 @@ public:
bool IsSharedMemoryWrite() const; bool IsSharedMemoryWrite() const;
/// Determines whether or not this instruction performs a shared memory read or write. /// Determines whether or not this instruction performs a shared memory read or write.
bool IsSharedMemoryReadOrWrite() const; bool IsSharedMemoryReadOrWrite() const;
/// Determines whether or not this instruction performs an atomic memory read.
bool IsExclusiveMemoryRead() const;
/// Determines whether or not this instruction performs an atomic memory write. /// Determines whether or not this instruction performs an atomic memory write.
bool IsExclusiveMemoryWrite() const; bool IsExclusiveMemoryWrite() const;

6
src/frontend/ir/opcodes.inc
View file

@ -637,12 +637,16 @@ A32OPC(ExclusiveWriteMemory64, U32, U32,
// A64 Memory access // A64 Memory access
A64OPC(ClearExclusive, Void, ) A64OPC(ClearExclusive, Void, )
A64OPC(SetExclusive, Void, U64, U8 )
A64OPC(ReadMemory8, U8, U64 ) A64OPC(ReadMemory8, U8, U64 )
A64OPC(ReadMemory16, U16, U64 ) A64OPC(ReadMemory16, U16, U64 )
A64OPC(ReadMemory32, U32, U64 ) A64OPC(ReadMemory32, U32, U64 )
A64OPC(ReadMemory64, U64, U64 ) A64OPC(ReadMemory64, U64, U64 )
A64OPC(ReadMemory128, U128, U64 ) A64OPC(ReadMemory128, U128, U64 )
A64OPC(ExclusiveReadMemory8, U8, U64 )
A64OPC(ExclusiveReadMemory16, U16, U64 )
A64OPC(ExclusiveReadMemory32, U32, U64 )
A64OPC(ExclusiveReadMemory64, U64, U64 )
A64OPC(ExclusiveReadMemory128, U128, U64 )
A64OPC(WriteMemory8, Void, U64, U8 ) A64OPC(WriteMemory8, Void, U64, U8 )
A64OPC(WriteMemory16, Void, U64, U16 ) A64OPC(WriteMemory16, Void, U64, U16 )
A64OPC(WriteMemory32, Void, U64, U32 ) A64OPC(WriteMemory32, Void, U64, U32 )

3
tests/A64/a64.cpp
View file

@ -287,9 +287,6 @@ TEST_CASE("A64: 128-bit exclusive read/write", "[a64]") {
conf.callbacks = &env; conf.callbacks = &env;
conf.processor_id = 0; conf.processor_id = 0;
SECTION("Local Monitor Only") {
conf.global_monitor = nullptr;
}
SECTION("Global Monitor") { SECTION("Global Monitor") {
conf.global_monitor = &monitor; conf.global_monitor = &monitor;
} }

21
tests/A64/testenv.h
View file

@ -84,6 +84,27 @@ public:
MemoryWrite64(vaddr + 8, value[1]); MemoryWrite64(vaddr + 8, value[1]);
} }
bool MemoryWriteExclusive8(u64 vaddr, std::uint8_t value, [[maybe_unused]] std::uint8_t expected) override {
MemoryWrite8(vaddr, value);
return true;
}
bool MemoryWriteExclusive16(u64 vaddr, std::uint16_t value, [[maybe_unused]] std::uint16_t expected) override {
MemoryWrite16(vaddr, value);
return true;
}
bool MemoryWriteExclusive32(u64 vaddr, std::uint32_t value, [[maybe_unused]] std::uint32_t expected) override {
MemoryWrite32(vaddr, value);
return true;
}
bool MemoryWriteExclusive64(u64 vaddr, std::uint64_t value, [[maybe_unused]] std::uint64_t expected) override {
MemoryWrite64(vaddr, value);
return true;
}
bool MemoryWriteExclusive128(u64 vaddr, Vector value, [[maybe_unused]] Vector expected) override {
MemoryWrite128(vaddr, value);
return true;
}
void InterpreterFallback(u64 pc, size_t num_instructions) override { ASSERT_MSG(false, "InterpreterFallback({:016x}, {})", pc, num_instructions); } void InterpreterFallback(u64 pc, size_t num_instructions) override { ASSERT_MSG(false, "InterpreterFallback({:016x}, {})", pc, num_instructions); }
void CallSVC(std::uint32_t swi) override { ASSERT_MSG(false, "CallSVC({})", swi); } void CallSVC(std::uint32_t swi) override { ASSERT_MSG(false, "CallSVC({})", swi); }