Ranged cache invalidation

include/dynarmic/dynarmic.h

@@ -8,8 +8,8 @@
 
 #include <cstddef>
 #include <cstdint>
-#include <string>
 #include <memory>
+#include <string>
 
 #include <dynarmic/callbacks.h>
 
@@ -38,6 +38,13 @@ public:
      */
     void ClearCache();
 
+    /**
+     * Invalidate the code cache at a range of addresses.
+     * @param start_address The starting address of the range to invalidate.
+     * @param length The length (in bytes) of the range to invalidate.
+     */
+    void InvalidateCacheRange(std::uint32_t start_address, std::size_t length);
+
     /**
      * Reset CPU state to state at startup. Does not clear code cache.
      * Cannot be called from a callback.

src/CMakeLists.txt

@@ -39,6 +39,7 @@ set(HEADERS
     ../include/dynarmic/coprocessor_util.h
     ../include/dynarmic/disassembler.h
     ../include/dynarmic/dynarmic.h
+    common/address_range.h
     common/assert.h
     common/bit_util.h
     common/common_types.h

src/backend_x64/emit_x64.cpp

@@ -12,8 +12,10 @@
 #include "backend_x64/block_of_code.h"
 #include "backend_x64/emit_x64.h"
 #include "backend_x64/jitstate.h"
+#include "common/address_range.h"
 #include "common/assert.h"
 #include "common/bit_util.h"
+#include "common/common_types.h"
 #include "common/variant_util.h"
 #include "frontend/arm/types.h"
 #include "frontend/ir/basic_block.h"
@@ -74,8 +76,9 @@ EmitX64::EmitX64(BlockOfCode* code, UserCallbacks cb, Jit* jit_interface)
 
 EmitX64::BlockDescriptor EmitX64::Emit(IR::Block& block) {
     code->align();
-    const u8* const emitted_code_start_ptr = code->getCurr();
+    const u8* const entrypoint = code->getCurr();
 
+    // Start emitting.
     EmitCondPrelude(block);
 
     RegAlloc reg_alloc{code};
@@ -108,11 +111,12 @@ EmitX64::BlockDescriptor EmitX64::Emit(IR::Block& block) {
     code->int3();
 
     const IR::LocationDescriptor descriptor = block.Location();
-    Patch(descriptor, emitted_code_start_ptr);
+    Patch(descriptor, entrypoint);
+
+    const size_t size = static_cast<size_t>(code->getCurr() - entrypoint);
+    EmitX64::BlockDescriptor block_desc{entrypoint, size, block.Location(), block.EndLocation().PC()};
+    block_descriptors.emplace(descriptor.UniqueHash(), block_desc);
 
-    EmitX64::BlockDescriptor& block_desc = block_descriptors[descriptor.UniqueHash()];
-    size_t emitted_code_size = static_cast<size_t>(code->getCurr() - emitted_code_start_ptr);
-    block_desc = {emitted_code_start_ptr, emitted_code_size};
     return block_desc;
 }
 
@@ -459,7 +463,7 @@ void EmitX64::EmitPushRSB(RegAlloc& reg_alloc, IR::Block&, IR::Inst* inst) {
 
     auto iter = block_descriptors.find(unique_hash_of_target);
     CodePtr target_code_ptr = iter != block_descriptors.end()
-                            ? iter->second.code_ptr
+                            ? iter->second.entrypoint
                             : code->GetReturnFromRunCodeAddress();
 
     Xbyak::Reg64 code_ptr_reg = reg_alloc.ScratchGpr({HostLoc::RCX});
@@ -3345,7 +3349,7 @@ void EmitX64::EmitTerminal(IR::Term::LinkBlock terminal, IR::LocationDescriptor
 
     patch_information[terminal.next.UniqueHash()].jg.emplace_back(code->getCurr());
     if (auto next_bb = GetBasicBlock(terminal.next)) {
-        EmitPatchJg(next_bb->code_ptr);
+        EmitPatchJg(next_bb->entrypoint);
     } else {
         EmitPatchJg();
     }
@@ -3374,7 +3378,7 @@ void EmitX64::EmitTerminal(IR::Term::LinkBlockFast terminal, IR::LocationDescrip
 
     patch_information[terminal.next.UniqueHash()].jmp.emplace_back(code->getCurr());
     if (auto next_bb = GetBasicBlock(terminal.next)) {
-        EmitPatchJmp(terminal.next, next_bb->code_ptr);
+        EmitPatchJmp(terminal.next, next_bb->entrypoint);
     } else {
         EmitPatchJmp(terminal.next);
     }
@@ -3475,5 +3479,34 @@ void EmitX64::ClearCache() {
     patch_information.clear();
 }
 
+void EmitX64::InvalidateCacheRange(const Common::AddressRange& range) {
+    // Remove cached block descriptors and patch information overlapping with the given range.
+
+    switch (range.which()) {
+    case 0: // FullAddressRange
+        ClearCache();
+        break;
+
+    case 1: // AddressInterval
+        auto interval = boost::get<Common::AddressInterval>(range);
+        for (auto it = std::begin(block_descriptors); it != std::end(block_descriptors);) {
+            const IR::LocationDescriptor& descriptor = it->second.start_location;
+            u32 start = descriptor.PC();
+            u32 end = it->second.end_location_pc;
+            if (interval.Overlaps(start, end)) {
+                it = block_descriptors.erase(it);
+
+                auto patch_it = patch_information.find(descriptor.UniqueHash());
+                if (patch_it != patch_information.end()) {
+                    Unpatch(descriptor);
+                }
+            } else {
+                ++it;
+            }
+        }
+        break;
+    }
+}
+
 } // namespace BackendX64
 } // namespace Dynarmic

src/backend_x64/emit_x64.h

@@ -14,6 +14,7 @@
 #include <xbyak_util.h>
 
 #include "backend_x64/reg_alloc.h"
+#include "common/address_range.h"
 #include "dynarmic/callbacks.h"
 #include "frontend/ir/location_descriptor.h"
 #include "frontend/ir/terminal.h"
@@ -34,8 +35,11 @@ class BlockOfCode;
 class EmitX64 final {
 public:
     struct BlockDescriptor {
-        CodePtr code_ptr; ///< Entrypoint of emitted code
-        size_t size;      ///< Length in bytes of emitted code
+        CodePtr entrypoint;  // Entrypoint of emitted code
+        size_t size;         // Length in bytes of emitted code
+
+        IR::LocationDescriptor start_location;
+        u32 end_location_pc;
     };
 
     EmitX64(BlockOfCode* code, UserCallbacks cb, Jit* jit_interface);
@@ -49,9 +53,16 @@ public:
     /// Looks up an emitted host block in the cache.
     boost::optional<BlockDescriptor> GetBasicBlock(IR::LocationDescriptor descriptor) const;
 
-    /// Empties the cache.
+    /// Empties the entire cache.
     void ClearCache();
 
+    /**
+     * Invalidate the cache at a range of addresses.
+     * @param start_address The starting address of the range to invalidate.
+     * @param length The length (in bytes) of the range to invalidate.
+     */
+    void InvalidateCacheRange(const Common::AddressRange& range);
+
 private:
     // Microinstruction emitters
 #define OPCODE(name, type, ...) void Emit##name(RegAlloc& reg_alloc, IR::Block& block, IR::Inst* inst);

src/backend_x64/interface_x64.cpp

@@ -5,6 +5,7 @@
  */
 
 #include <memory>
+#include <queue>
 
 #include <fmt/format.h>
 
@@ -35,6 +36,7 @@ struct Jit::Impl {
             , jit_state()
             , emitter(&block_of_code, callbacks, jit)
             , callbacks(callbacks)
+            , jit_interface(jit)
     {}
 
     BlockOfCode block_of_code;
@@ -42,20 +44,21 @@ struct Jit::Impl {
     EmitX64 emitter;
     const UserCallbacks callbacks;
 
-    bool clear_cache_required = false;
+    // Requests made during execution to invalidate the cache are queued up here.
+    std::queue<Common::AddressRange> invalid_cache_ranges;
 
     size_t Execute(size_t cycle_count) {
         u32 pc = jit_state.Reg[15];
 
         IR::LocationDescriptor descriptor{pc, Arm::PSR{jit_state.Cpsr}, Arm::FPSCR{jit_state.FPSCR_mode}};
 
-        CodePtr code_ptr = GetBasicBlock(descriptor).code_ptr;
-        return block_of_code.RunCode(&jit_state, code_ptr, cycle_count);
+        CodePtr entrypoint = GetBasicBlock(descriptor).entrypoint;
+        return block_of_code.RunCode(&jit_state, entrypoint, cycle_count);
     }
 
     std::string Disassemble(const IR::LocationDescriptor& descriptor) {
         auto block = GetBasicBlock(descriptor);
-        std::string result = fmt::format("address: {}\nsize: {} bytes\n", block.code_ptr, block.size);
+        std::string result = fmt::format("address: {}\nsize: {} bytes\n", block.entrypoint, block.size);
 
 #ifdef DYNARMIC_USE_LLVM
         LLVMInitializeX86TargetInfo();
@@ -64,7 +67,7 @@ struct Jit::Impl {
         LLVMDisasmContextRef llvm_ctx = LLVMCreateDisasm("x86_64", nullptr, 0, nullptr, nullptr);
         LLVMSetDisasmOptions(llvm_ctx, LLVMDisassembler_Option_AsmPrinterVariant);
 
-        const u8* pos = static_cast<const u8*>(block.code_ptr);
+        const u8* pos = static_cast<const u8*>(block.entrypoint);
         const u8* end = pos + block.size;
         size_t remaining = block.size;
 
@@ -91,14 +94,31 @@ struct Jit::Impl {
         return result;
     }
 
-    void ClearCache() {
-        block_of_code.ClearCache();
-        emitter.ClearCache();
+    void PerformCacheInvalidation() {
+        if (invalid_cache_ranges.empty()) {
+            return;
+        }
+
         jit_state.ResetRSB();
-        clear_cache_required = false;
+        block_of_code.ClearCache();
+        while (!invalid_cache_ranges.empty()) {
+            emitter.InvalidateCacheRange(invalid_cache_ranges.front());
+            invalid_cache_ranges.pop();
+        }
+    }
+
+    void HandleNewCacheRange() {
+        if (jit_interface->is_executing) {
+            jit_state.halt_requested = true;
+            return;
+        }
+
+        PerformCacheInvalidation();
     }
 
 private:
+    Jit* jit_interface;
+
     EmitX64::BlockDescriptor GetBasicBlock(IR::LocationDescriptor descriptor) {
         auto block = emitter.GetBasicBlock(descriptor);
         if (block)
@@ -130,21 +150,19 @@ size_t Jit::Run(size_t cycle_count) {
         cycles_executed += impl->Execute(cycle_count - cycles_executed);
     }
 
-    if (impl->clear_cache_required) {
-        impl->ClearCache();
-    }
+    impl->PerformCacheInvalidation();
 
     return cycles_executed;
 }
 
 void Jit::ClearCache() {
-    if (is_executing) {
-        impl->jit_state.halt_requested = true;
-        impl->clear_cache_required = true;
-        return;
-    }
+    impl->invalid_cache_ranges.push(Common::FullAddressRange{});
+    impl->HandleNewCacheRange();
+}
 
-    impl->ClearCache();
+void Jit::InvalidateCacheRange(std::uint32_t start_address, std::size_t length) {
+    impl->invalid_cache_ranges.push(Common::AddressInterval{start_address, length});
+    impl->HandleNewCacheRange();
 }
 
 void Jit::Reset() {

src/common/address_range.h

@@ -0,0 +1,33 @@
+/* This file is part of the dynarmic project.
+ * Copyright (c) 2016 MerryMage
+ * This software may be used and distributed according to the terms of the GNU
+ * General Public License version 2 or any later version.
+ */
+
+#pragma once
+
+#include <cstddef>
+
+#include <boost/variant.hpp>
+
+#include "common/common_types.h"
+
+namespace Dynarmic {
+namespace Common {
+
+struct FullAddressRange {};
+
+struct AddressInterval {
+    u32 start_address;
+    std::size_t length;
+
+    // Does this interval overlap with [from, to)?
+    bool Overlaps(u32 from, u32 to) const {
+        return start_address <= to && from <= start_address + length;
+    }
+};
+
+using AddressRange = boost::variant<FullAddressRange, AddressInterval>;
+
+} // namespace Common
+} // namespace Dynarmic

src/frontend/ir/basic_block.cpp

@@ -35,6 +35,14 @@ LocationDescriptor Block::Location() const {
     return location;
 }
 
+LocationDescriptor Block::EndLocation() const {
+    return end_location;
+}
+
+void Block::SetEndLocation(const LocationDescriptor& descriptor) {
+    end_location = descriptor;
+}
+
 Arm::Cond Block::GetCondition() const {
     return cond;
 }

src/frontend/ir/basic_block.h

@@ -40,7 +40,8 @@ public:
     using reverse_iterator       = InstructionList::reverse_iterator;
     using const_reverse_iterator = InstructionList::const_reverse_iterator;
 
-    explicit Block(const LocationDescriptor& location) : location(location) {}
+    explicit Block(const LocationDescriptor& location)
+        : location(location), end_location(location) {}
 
     bool                   empty()   const { return instructions.empty();   }
     size_type              size()    const { return instructions.size();    }
@@ -78,6 +79,10 @@ public:
 
     /// Gets the starting location for this basic block.
     LocationDescriptor Location() const;
+    /// Gets the end location for this basic block.
+    LocationDescriptor EndLocation() const;
+    /// Sets the end location for this basic block.
+    void SetEndLocation(const LocationDescriptor& descriptor);
 
     /// Gets the condition required to pass in order to execute this block.
     Arm::Cond GetCondition() const;
@@ -116,6 +121,8 @@ public:
 private:
     /// Description of the starting location of this block
     LocationDescriptor location;
+    /// Description of the end location of this block
+    LocationDescriptor end_location;
     /// Conditional to pass in order to execute this block
     Arm::Cond cond = Arm::Cond::AL;
     /// Block to execute next if `cond` did not pass.

src/frontend/translate/translate_arm.cpp

@@ -60,6 +60,8 @@ IR::Block TranslateArm(IR::LocationDescriptor descriptor, MemoryReadCodeFuncType
 
     ASSERT_MSG(visitor.ir.block.HasTerminal(), "Terminal has not been set");
 
+    visitor.ir.block.SetEndLocation(visitor.ir.current_location);
+
     return std::move(visitor.ir.block);
 }
 

src/frontend/translate/translate_thumb.cpp

@@ -909,6 +909,8 @@ IR::Block TranslateThumb(IR::LocationDescriptor descriptor, MemoryReadCodeFuncTy
         visitor.ir.block.CycleCount()++;
     }
 
+    visitor.ir.block.SetEndLocation(visitor.ir.current_location);
+
     return std::move(visitor.ir.block);
 }
 

tests/arm/fuzz_arm.cpp

@@ -1166,3 +1166,38 @@ TEST_CASE("Fuzz ARM packing instructions", "[JitX64]") {
         });
     }
 }
+
+TEST_CASE("arm: Test InvalidateCacheRange", "[arm]") {
+    Dynarmic::Jit jit{GetUserCallbacks()};
+    code_mem.fill({});
+    code_mem[0] = 0xe3a00005; // mov r0, #5
+    code_mem[1] = 0xe3a0100D; // mov r1, #13
+    code_mem[2] = 0xe0812000; // add r2, r1, r0
+    code_mem[3] = 0xeafffffe; // b +#0 (infinite loop)
+
+    jit.Regs() = {};
+    jit.Cpsr() = 0x000001d0; // User-mode
+
+    jit.Run(4);
+
+    REQUIRE(jit.Regs()[0] == 5);
+    REQUIRE(jit.Regs()[1] == 13);
+    REQUIRE(jit.Regs()[2] == 18);
+    REQUIRE(jit.Regs()[15] == 0x0000000c);
+    REQUIRE(jit.Cpsr() == 0x000001d0);
+
+    // Change the code
+    code_mem[1] = 0xe3a01007; // mov r1, #7
+    jit.InvalidateCacheRange(/*start_memory_location = */ 4, /* length_in_bytes = */ 4);
+
+    // Reset position of PC
+    jit.Regs()[15] = 0;
+
+    jit.Run(4);
+
+    REQUIRE(jit.Regs()[0] == 5);
+    REQUIRE(jit.Regs()[1] == 7);
+    REQUIRE(jit.Regs()[2] == 12);
+    REQUIRE(jit.Regs()[15] == 0x0000000c);
+    REQUIRE(jit.Cpsr() == 0x000001d0);
+}