Implement DC instructions

include/dynarmic/A64/config.h

@@ -30,6 +30,27 @@ enum class Exception {
     UnpredictableInstruction,
 };
 
+enum class DataCacheOperation {
+    /// DC CISW
+    CleanAndInvalidateBySetWay,
+    /// DC CIVAC
+    CleanAndInvalidateByVAToPoC,
+    /// DC CSW
+    CleanBySetWay,
+    /// DC CVAC
+    CleanByVAToPoC,
+    /// DC CVAU
+    CleanByVAToPoU,
+    /// DC CVAP
+    CleanByVAToPoP,
+    /// DC ISW
+    InvalidateBySetWay,
+    /// DC IVAC
+    InvaldiateByVAToPoC,
+    /// DC ZVA
+    ZeroByVA,
+};
+
 struct UserCallbacks {
     virtual ~UserCallbacks() = default;
 
@@ -64,6 +85,7 @@ struct UserCallbacks {
     virtual void CallSVC(std::uint32_t swi) = 0;
 
     virtual void ExceptionRaised(VAddr pc, Exception exception) = 0;
+    virtual void DataCacheOperationRaised(DataCacheOperation /*op*/, VAddr /*value*/) {}
 
     // Timing-related callbacks
     // ticks ticks have passed
@@ -75,6 +97,16 @@ struct UserCallbacks {
 struct UserConfig {
     UserCallbacks* callbacks;
 
+    /// When set to true, UserCallbacks::DataCacheOperationRaised will be called when any
+    /// data cache instruction is executed. Notably DC ZVA will not implicitly do anything.
+    /// When set to false, UserCallbacks::DataCacheOperationRaised will never be called.
+    /// Executing DC ZVA in this mode will result in zeros being written to memory.
+    bool hook_data_cache_operations = false;
+
+    /// DCZID_EL0<3:0> is log2 of the block size in words
+    /// DCZID_EL0<4> is 0 if the DC ZVA instruction is permitted.
+    std::uint32_t dczid_el0 = 4;
+
     // Determines whether AddTicks and GetTicksRemaining are called.
     // If false, execution will continue until soon after Jit::HaltExecution is called.
     // bool enable_ticks = true; // TODO

src/CMakeLists.txt

@@ -100,6 +100,7 @@ add_library(dynarmic
     frontend/A64/translate/impl/simd_shift_by_immediate.cpp
     frontend/A64/translate/impl/simd_three_same.cpp
     frontend/A64/translate/impl/simd_two_register_misc.cpp
+    frontend/A64/translate/impl/sys_dc.cpp
     frontend/A64/translate/impl/system.cpp
     frontend/A64/translate/translate.cpp
     frontend/A64/translate/translate.h
@@ -125,6 +126,7 @@ add_library(dynarmic
     frontend/ir/value.h
     ir_opt/a32_constant_memory_reads_pass.cpp
     ir_opt/a32_get_set_elimination_pass.cpp
+    ir_opt/a64_callback_config_pass.cpp
     ir_opt/a64_get_set_elimination_pass.cpp
     ir_opt/a64_merge_interpret_blocks.cpp
     ir_opt/constant_propagation_pass.cpp

src/backend_x64/a64_emit_x64.cpp

@@ -310,6 +310,12 @@ void A64EmitX64::EmitA64ExceptionRaised(A64EmitContext& ctx, IR::Inst* inst) {
     });
 }
 
+void A64EmitX64::EmitA64DataCacheOperationRaised(A64EmitContext& ctx, IR::Inst* inst) {
+    auto args = ctx.reg_alloc.GetArgumentInfo(inst);
+    ctx.reg_alloc.HostCall(nullptr, args[0], args[1]);
+    DEVIRT(conf.callbacks, &A64::UserCallbacks::DataCacheOperationRaised).EmitCall(code);
+}
+
 void A64EmitX64::EmitA64ReadMemory8(A64EmitContext& ctx, IR::Inst* inst) {
     auto args = ctx.reg_alloc.GetArgumentInfo(inst);
     ctx.reg_alloc.HostCall(inst, {}, args[0]);

src/backend_x64/a64_interface.cpp

@@ -186,6 +186,7 @@ private:
 
         // JIT Compile
         IR::Block ir_block = A64::Translate(A64::LocationDescriptor{current_location}, [this](u64 vaddr) { return conf.callbacks->MemoryReadCode(vaddr); });
+        Optimization::A64CallbackConfigPass(ir_block, conf);
         Optimization::A64GetSetElimination(ir_block);
         Optimization::DeadCodeElimination(ir_block);
         Optimization::A64MergeInterpretBlocksPass(ir_block, conf.callbacks);

src/frontend/A64/decoder/a64.inc

@@ -70,6 +70,17 @@ INST(SEVL,                   "SEVL",                                      "11010
 //INST(SYSL,                   "SYSL",                                      "1101010100101oooNNNNMMMMooottttt")
 //INST(MRS,                    "MRS",                                       "110101010011poooNNNNMMMMooottttt")
 
+// SYS: Data Cache
+INST(DC_IVAC,                "DC IVAC",                                   "110101010000100001110110001ttttt")
+INST(DC_ISW,                 "DC ISW",                                    "110101010000100001110110010ttttt")
+INST(DC_CSW,                 "DC CSW",                                    "110101010000100001111010010ttttt")
+INST(DC_CISW,                "DC CISW",                                   "110101010000100001111110010ttttt")
+INST(DC_ZVA,                 "DC ZVA",                                    "110101010000101101110100001ttttt")
+INST(DC_CVAC,                "DC CVAC",                                   "110101010000101101111010001ttttt")
+INST(DC_CVAU,                "DC CVAU",                                   "110101010000101101111011001ttttt")
+INST(DC_CVAP,                "DC CVAP",                                   "110101010000101101111100001ttttt")
+INST(DC_CIVAC,               "DC CIVAC",                                  "110101010000101101111110001ttttt")
+
 // Unconditonal branch (Register)
 INST(BLR,                    "BLR",                                       "1101011000111111000000nnnnn00000")
 INST(BR,                     "BR",                                        "1101011000011111000000nnnnn00000")

src/frontend/A64/ir_emitter.cpp

@@ -13,7 +13,7 @@ namespace Dynarmic::A64 {
 using Opcode = IR::Opcode;
 
 u64 IREmitter::PC() {
-    return current_location.PC();
+    return current_location->PC();
 }
 
 u64 IREmitter::AlignPC(size_t alignment) {
@@ -41,6 +41,10 @@ void IREmitter::ExceptionRaised(Exception exception) {
     Inst(Opcode::A64ExceptionRaised, Imm64(PC()), Imm64(static_cast<u64>(exception)));
 }
 
+void IREmitter::DataCacheOperationRaised(DataCacheOperation op, const IR::U64& value) {
+    Inst(Opcode::A64DataCacheOperationRaised, Imm64(static_cast<u64>(op)), value);
+}
+
 IR::U8 IREmitter::ReadMemory8(const IR::U64& vaddr) {
     return Inst<IR::U8>(Opcode::A64ReadMemory8, vaddr);
 }

src/frontend/A64/ir_emitter.h

@@ -8,6 +8,8 @@
 
 #include <initializer_list>
 
+#include <boost/optional.hpp>
+
 #include <dynarmic/A64/config.h>
 
 #include "common/common_types.h"
@@ -25,9 +27,10 @@ namespace Dynarmic::A64 {
  */
 class IREmitter : public IR::IREmitter {
 public:
+    explicit IREmitter(IR::Block& block) : IR::IREmitter(block) {}
     explicit IREmitter(IR::Block& block, LocationDescriptor descriptor) : IR::IREmitter(block), current_location(descriptor) {}
 
-    LocationDescriptor current_location;
+    boost::optional<LocationDescriptor> current_location;
 
     u64 PC();
     u64 AlignPC(size_t alignment);
@@ -38,6 +41,7 @@ public:
 
     void CallSupervisor(u32 imm);
     void ExceptionRaised(Exception exception);
+    void DataCacheOperationRaised(DataCacheOperation op, const IR::U64& value);
 
     IR::U8 ReadMemory8(const IR::U64& vaddr);
     IR::U16 ReadMemory16(const IR::U64& vaddr);

src/frontend/A64/translate/impl/branch.cpp

@@ -12,8 +12,8 @@ bool TranslatorVisitor::B_cond(Imm<19> imm19, Cond cond) {
     s64 offset = concatenate(imm19, Imm<2>{0}).SignExtend<s64>();
 
     u64 target = ir.PC() + offset;
-    auto cond_pass = IR::Term::LinkBlock{ir.current_location.SetPC(target)};
+    auto cond_pass = IR::Term::LinkBlock{ir.current_location->SetPC(target)};
-    auto cond_fail = IR::Term::LinkBlock{ir.current_location.AdvancePC(4)};
+    auto cond_fail = IR::Term::LinkBlock{ir.current_location->AdvancePC(4)};
     ir.SetTerm(IR::Term::If{cond, cond_pass, cond_fail});
     return false;
 }
@@ -22,7 +22,7 @@ bool TranslatorVisitor::B_uncond(Imm<26> imm26) {
     s64 offset = concatenate(imm26, Imm<2>{0}).SignExtend<s64>();
 
     u64 target = ir.PC() + offset;
-    ir.SetTerm(IR::Term::LinkBlock{ir.current_location.SetPC(target)});
+    ir.SetTerm(IR::Term::LinkBlock{ir.current_location->SetPC(target)});
     return false;
 }
 
@@ -30,10 +30,10 @@ bool TranslatorVisitor::BL(Imm<26> imm26) {
     s64 offset = concatenate(imm26, Imm<2>{0}).SignExtend<s64>();
 
     X(64, Reg::R30, ir.Imm64(ir.PC() + 4));
-    ir.PushRSB(ir.current_location.AdvancePC(4));
+    ir.PushRSB(ir.current_location->AdvancePC(4));
 
     u64 target = ir.PC() + offset;
-    ir.SetTerm(IR::Term::LinkBlock{ir.current_location.SetPC(target)});
+    ir.SetTerm(IR::Term::LinkBlock{ir.current_location->SetPC(target)});
     return false;
 }
 
@@ -41,7 +41,7 @@ bool TranslatorVisitor::BLR(Reg Rn) {
     auto target = X(64, Rn);
 
     X(64, Reg::R30, ir.Imm64(ir.PC() + 4));
-    ir.PushRSB(ir.current_location.AdvancePC(4));
+    ir.PushRSB(ir.current_location->AdvancePC(4));
 
     ir.SetPC(target);
     ir.SetTerm(IR::Term::ReturnToDispatch{});
@@ -73,8 +73,8 @@ bool TranslatorVisitor::CBZ(bool sf, Imm<19> imm19, Reg Rt) {
     ir.SetCheckBit(ir.IsZero(operand1));
 
     u64 target = ir.PC() + offset;
-    auto cond_pass = IR::Term::LinkBlock{ir.current_location.SetPC(target)};
+    auto cond_pass = IR::Term::LinkBlock{ir.current_location->SetPC(target)};
-    auto cond_fail = IR::Term::LinkBlock{ir.current_location.AdvancePC(4)};
+    auto cond_fail = IR::Term::LinkBlock{ir.current_location->AdvancePC(4)};
     ir.SetTerm(IR::Term::CheckBit{cond_pass, cond_fail});
     return false;
 }
@@ -88,8 +88,8 @@ bool TranslatorVisitor::CBNZ(bool sf, Imm<19> imm19, Reg Rt) {
     ir.SetCheckBit(ir.IsZero(operand1));
 
     u64 target = ir.PC() + offset;
-    auto cond_pass = IR::Term::LinkBlock{ir.current_location.AdvancePC(4)};
+    auto cond_pass = IR::Term::LinkBlock{ir.current_location->AdvancePC(4)};
-    auto cond_fail = IR::Term::LinkBlock{ir.current_location.SetPC(target)};
+    auto cond_fail = IR::Term::LinkBlock{ir.current_location->SetPC(target)};
     ir.SetTerm(IR::Term::CheckBit{cond_pass, cond_fail});
     return false;
 }
@@ -104,8 +104,8 @@ bool TranslatorVisitor::TBZ(Imm<1> b5, Imm<5> b40, Imm<14> imm14, Reg Rt) {
     ir.SetCheckBit(ir.TestBit(operand, ir.Imm8(bit_pos)));
 
     u64 target = ir.PC() + offset;
-    auto cond_1 = IR::Term::LinkBlock{ir.current_location.AdvancePC(4)};
+    auto cond_1 = IR::Term::LinkBlock{ir.current_location->AdvancePC(4)};
-    auto cond_0 = IR::Term::LinkBlock{ir.current_location.SetPC(target)};
+    auto cond_0 = IR::Term::LinkBlock{ir.current_location->SetPC(target)};
     ir.SetTerm(IR::Term::CheckBit{cond_1, cond_0});
     return false;
 }
@@ -120,8 +120,8 @@ bool TranslatorVisitor::TBNZ(Imm<1> b5, Imm<5> b40, Imm<14> imm14, Reg Rt) {
     ir.SetCheckBit(ir.TestBit(operand, ir.Imm8(bit_pos)));
 
     u64 target = ir.PC() + offset;
-    auto cond_1 = IR::Term::LinkBlock{ir.current_location.SetPC(target)};
+    auto cond_1 = IR::Term::LinkBlock{ir.current_location->SetPC(target)};
-    auto cond_0 = IR::Term::LinkBlock{ir.current_location.AdvancePC(4)};
+    auto cond_0 = IR::Term::LinkBlock{ir.current_location->AdvancePC(4)};
     ir.SetTerm(IR::Term::CheckBit{cond_1, cond_0});
     return false;
 }

src/frontend/A64/translate/impl/exception_generating.cpp

@@ -9,8 +9,8 @@
 namespace Dynarmic::A64 {
 
 bool TranslatorVisitor::SVC(Imm<16> imm16) {
-    ir.PushRSB(ir.current_location.AdvancePC(4));
+    ir.PushRSB(ir.current_location->AdvancePC(4));
-    ir.SetPC(ir.Imm64(ir.current_location.PC() + 4));
+    ir.SetPC(ir.Imm64(ir.current_location->PC() + 4));
     ir.CallSupervisor(imm16.ZeroExtend());
     ir.SetTerm(IR::Term::CheckHalt{IR::Term::PopRSBHint{}});
     return false;

src/frontend/A64/translate/impl/impl.cpp

@@ -11,7 +11,7 @@
 namespace Dynarmic::A64 {
 
 bool TranslatorVisitor::InterpretThisInstruction() {
-    ir.SetTerm(IR::Term::Interpret(ir.current_location));
+    ir.SetTerm(IR::Term::Interpret(*ir.current_location));
     return false;
 }
 

src/frontend/A64/translate/impl/impl.h

@@ -141,6 +141,17 @@ struct TranslatorVisitor final {
     bool SYSL(Imm<3> op1, Imm<4> CRn, Imm<4> CRm, Imm<3> op2, Reg Rt);
     bool MRS(bool o0, Imm<3> op1, Imm<4> CRn, Imm<4> CRm, Imm<3> op2, Reg Rt);
 
+    // SYS: Data Cache
+    bool DC_IVAC(Reg Rt);
+    bool DC_ISW(Reg Rt);
+    bool DC_CSW(Reg Rt);
+    bool DC_CISW(Reg Rt);
+    bool DC_ZVA(Reg Rt);
+    bool DC_CVAC(Reg Rt);
+    bool DC_CVAU(Reg Rt);
+    bool DC_CVAP(Reg Rt);
+    bool DC_CIVAC(Reg Rt);
+
     // Unconditonal branch (Register)
     bool BR(Reg Rn);
     bool BRA(bool Z, bool M, Reg Rn, Reg Rm);

src/frontend/A64/translate/impl/sys_dc.cpp

@@ -0,0 +1,52 @@
+/* This file is part of the dynarmic project.
+ * Copyright (c) 2018 MerryMage
+ * This software may be used and distributed according to the terms of the GNU
+ * General Public License version 2 or any later version.
+ */
+
+#include "frontend/A64/translate/impl/impl.h"
+
+namespace Dynarmic::A64 {
+
+static bool DataCacheInstruction(TranslatorVisitor& tv, IREmitter& ir, DataCacheOperation op, const Reg Rt) {
+    ir.DataCacheOperationRaised(op, tv.X(64, Rt));
+    return true;
+}
+
+bool TranslatorVisitor::DC_IVAC(Reg Rt) {
+    return DataCacheInstruction(*this, ir, DataCacheOperation::InvaldiateByVAToPoC, Rt);
+}
+
+bool TranslatorVisitor::DC_ISW(Reg Rt) {
+    return DataCacheInstruction(*this, ir, DataCacheOperation::InvalidateBySetWay, Rt);
+}
+
+bool TranslatorVisitor::DC_CSW(Reg Rt) {
+    return DataCacheInstruction(*this, ir, DataCacheOperation::CleanBySetWay, Rt);
+}
+
+bool TranslatorVisitor::DC_CISW(Reg Rt) {
+    return DataCacheInstruction(*this, ir, DataCacheOperation::CleanAndInvalidateBySetWay, Rt);
+}
+
+bool TranslatorVisitor::DC_ZVA(Reg Rt) {
+    return DataCacheInstruction(*this, ir, DataCacheOperation::ZeroByVA, Rt);
+}
+
+bool TranslatorVisitor::DC_CVAC(Reg Rt) {
+    return DataCacheInstruction(*this, ir, DataCacheOperation::CleanByVAToPoC, Rt);
+}
+
+bool TranslatorVisitor::DC_CVAU(Reg Rt) {
+    return DataCacheInstruction(*this, ir, DataCacheOperation::CleanByVAToPoU, Rt);
+}
+
+bool TranslatorVisitor::DC_CVAP(Reg Rt) {
+    return DataCacheInstruction(*this, ir, DataCacheOperation::CleanByVAToPoP, Rt);
+}
+
+bool TranslatorVisitor::DC_CIVAC(Reg Rt) {
+    return DataCacheInstruction(*this, ir, DataCacheOperation::CleanAndInvalidateByVAToPoC, Rt);
+}
+
+} // namespace Dynarmic::A64

src/frontend/A64/translate/translate.cpp

@@ -18,7 +18,7 @@ IR::Block Translate(LocationDescriptor descriptor, MemoryReadCodeFuncType memory
 
     bool should_continue = true;
     while (should_continue) {
-        const u64 pc = visitor.ir.current_location.PC();
+        const u64 pc = visitor.ir.current_location->PC();
         const u32 instruction = memory_read_code(pc);
 
         if (auto decoder = Decode<TranslatorVisitor>(instruction)) {
@@ -27,13 +27,13 @@ IR::Block Translate(LocationDescriptor descriptor, MemoryReadCodeFuncType memory
             should_continue = visitor.InterpretThisInstruction();
         }
 
-        visitor.ir.current_location = visitor.ir.current_location.AdvancePC(4);
+        visitor.ir.current_location = visitor.ir.current_location->AdvancePC(4);
         block.CycleCount()++;
     }
 
     ASSERT_MSG(block.HasTerminal(), "Terminal has not been set");
 
-    block.SetEndLocation(visitor.ir.current_location);
+    block.SetEndLocation(*visitor.ir.current_location);
 
     return block;
 }
@@ -48,10 +48,10 @@ bool TranslateSingleInstruction(IR::Block& block, LocationDescriptor descriptor,
         should_continue = visitor.InterpretThisInstruction();
     }
 
-    visitor.ir.current_location = visitor.ir.current_location.AdvancePC(4);
+    visitor.ir.current_location = visitor.ir.current_location->AdvancePC(4);
     block.CycleCount()++;
 
-    block.SetEndLocation(visitor.ir.current_location);
+    block.SetEndLocation(*visitor.ir.current_location);
 
     return should_continue;
 }

src/frontend/ir/microinstruction.cpp

@@ -269,14 +269,15 @@ bool Inst::IsCoprocessorInstruction() const {
 }
 
 bool Inst::MayHaveSideEffects() const {
-    return op == Opcode::PushRSB        ||
+    return op == Opcode::PushRSB                        ||
-           op == Opcode::A64SetCheckBit ||
+           op == Opcode::A64SetCheckBit                 ||
-           CausesCPUException()         ||
+           op == Opcode::A64DataCacheOperationRaised    ||
-           WritesToCoreRegister()       ||
+           CausesCPUException()                         ||
-           WritesToCPSR()               ||
+           WritesToCoreRegister()                       ||
-           WritesToFPSCR()              ||
+           WritesToCPSR()                               ||
-           AltersExclusiveState()       ||
+           WritesToFPSCR()                              ||
-           IsMemoryWrite()              ||
+           AltersExclusiveState()                       ||
+           IsMemoryWrite()                              ||
            IsCoprocessorInstruction();
 }
 

src/frontend/ir/opcodes.inc

@@ -58,6 +58,7 @@ A64OPC(SetSP,                       T::Void,        T::U64
 A64OPC(SetPC,                       T::Void,        T::U64                                          )
 A64OPC(CallSupervisor,              T::Void,        T::U32                                          )
 A64OPC(ExceptionRaised,             T::Void,        T::U64,         T::U64                          )
+A64OPC(DataCacheOperationRaised,    T::Void,        T::U64,         T::U64                          )
 
 // Hints
 OPCODE(PushRSB,                     T::Void,        T::U64                                          )

src/ir_opt/a64_callback_config_pass.cpp

@@ -0,0 +1,59 @@
+/* This file is part of the dynarmic project.
+ * Copyright (c) 2018 MerryMage
+ * This software may be used and distributed according to the terms of the GNU
+ * General Public License version 2 or any later version.
+ */
+
+#include <array>
+
+#include "dynarmic/A64/config.h"
+#include "frontend/A64/ir_emitter.h"
+#include "frontend/ir/basic_block.h"
+#include "frontend/ir/microinstruction.h"
+#include "frontend/ir/opcodes.h"
+#include "ir_opt/passes.h"
+
+namespace Dynarmic::Optimization {
+
+void A64CallbackConfigPass(IR::Block& block, const A64::UserConfig& conf) {
+    if (conf.hook_data_cache_operations) {
+        return;
+    }
+
+    for (auto& inst : block) {
+        if (inst.GetOpcode() != IR::Opcode::A64DataCacheOperationRaised) {
+            continue;
+        }
+
+        auto op = static_cast<A64::DataCacheOperation>(inst.GetArg(0).GetU64());
+        if (op == A64::DataCacheOperation::ZeroByVA) {
+            A64::IREmitter ir{block};
+            ir.SetInsertionPoint(&inst);
+
+            size_t bytes = 4 << static_cast<size_t>(conf.dczid_el0 & 0b1111);
+            IR::U64 addr{inst.GetArg(1)};
+
+            const IR::U128 zero_u128 = ir.ZeroExtendToQuad(ir.Imm64(0));
+            while (bytes >= 16) {
+                ir.WriteMemory128(addr, zero_u128);
+                addr = ir.Add(addr, ir.Imm64(16));
+                bytes -= 16;
+            }
+
+            while (bytes >= 8) {
+                ir.WriteMemory64(addr, ir.Imm64(0));
+                addr = ir.Add(addr, ir.Imm64(8));
+                bytes -= 8;
+            }
+
+            while (bytes >= 4) {
+                ir.WriteMemory32(addr, ir.Imm32(0));
+                addr = ir.Add(addr, ir.Imm64(4));
+                bytes -= 4;
+            }
+        }
+        inst.Invalidate();
+    }
+}
+
+} // namespace Dynarmic::Optimization

src/ir_opt/passes.h

@@ -17,6 +17,7 @@ namespace Dynarmic::Optimization {
 
 void A32GetSetElimination(IR::Block& block);
 void A32ConstantMemoryReads(IR::Block& block, A32::UserCallbacks* cb);
+void A64CallbackConfigPass(IR::Block& block, const A64::UserConfig& conf);
 void A64GetSetElimination(IR::Block& block);
 void A64MergeInterpretBlocksPass(IR::Block& block, A64::UserCallbacks* cb);
 void ConstantPropagation(IR::Block& block);

tests/A64/fuzz_with_unicorn.cpp

@@ -83,7 +83,7 @@ restart:
     if (auto terminal = block.GetTerminal(); boost::get<IR::Term::Interpret>(&terminal))
         goto restart;
     for (const auto& ir_inst : block)
-        if (ir_inst.GetOpcode() == IR::Opcode::A64ExceptionRaised || ir_inst.GetOpcode() == IR::Opcode::A64CallSupervisor)
+        if (ir_inst.GetOpcode() == IR::Opcode::A64ExceptionRaised || ir_inst.GetOpcode() == IR::Opcode::A64CallSupervisor || ir_inst.GetOpcode() == IR::Opcode::A64DataCacheOperationRaised)
             goto restart;
 
     return instruction;