Android #161

ci: fix homebrew installation issue in actions runner images

.github/workflows/verify.yml

@@ -79,7 +79,8 @@ jobs:
           fetch-depth: 0
       - name: Install dependencies
         run: |
-          brew install autoconf automake boost@1.83 ccache ffmpeg fmt glslang hidapi libtool libusb lz4 ninja nlohmann-json openssl pkg-config qt@5 sdl2 speexdsp zlib zlib zstd
+          # workaround for https://github.com/actions/setup-python/issues/577
+          brew install autoconf automake boost@1.83 ccache ffmpeg fmt glslang hidapi libtool libusb lz4 ninja nlohmann-json openssl pkg-config qt@5 sdl2 speexdsp zlib zlib zstd || brew link --overwrite python@3.12
       - name: Build
         run: |
           mkdir build

README.md

@@ -2,8 +2,10 @@
 |----|----|----|----|----|
 | [12237](https://github.com/yuzu-emu/yuzu//pull/12237) | [`a05c24242`](https://github.com/yuzu-emu/yuzu//pull/12237/files) | nce: implement instruction emulation for misaligned memory accesses | [liamwhite](https://github.com/liamwhite/) | Yes |
 | [12335](https://github.com/yuzu-emu/yuzu//pull/12335) | [`86d26914a`](https://github.com/yuzu-emu/yuzu//pull/12335/files) | android: Game Properties | [t895](https://github.com/t895/) | Yes |
-| [12345](https://github.com/yuzu-emu/yuzu//pull/12345) | [`24a0bde93`](https://github.com/yuzu-emu/yuzu//pull/12345/files) | renderer_vulkan: cap async presentation frame count | [liamwhite](https://github.com/liamwhite/) | Yes |
+| [12344](https://github.com/yuzu-emu/yuzu//pull/12344) | [`beb61a0c8`](https://github.com/yuzu-emu/yuzu//pull/12344/files) | video_core: use interval map for page count tracking | [liamwhite](https://github.com/liamwhite/) | Yes |
+| [12345](https://github.com/yuzu-emu/yuzu//pull/12345) | [`b560ade66`](https://github.com/yuzu-emu/yuzu//pull/12345/files) | renderer_vulkan: cap async presentation frame count | [liamwhite](https://github.com/liamwhite/) | Yes |
 | [12349](https://github.com/yuzu-emu/yuzu//pull/12349) | [`8abdfcf8d`](https://github.com/yuzu-emu/yuzu//pull/12349/files) | Have GetActiveChannelCount return the system channels instead of host device channels | [Kelebek1](https://github.com/Kelebek1/) | Yes |
+| [12358](https://github.com/yuzu-emu/yuzu//pull/12358) | [`df41e81fd`](https://github.com/yuzu-emu/yuzu//pull/12358/files) | common: use memory holepunching when clearing memory | [liamwhite](https://github.com/liamwhite/) | Yes |
 
 
 End of merge log. You can find the original README.md below the break.

src/common/host_memory.cpp

@@ -11,10 +11,6 @@
 
 #elif defined(__linux__) || defined(__FreeBSD__) // ^^^ Windows ^^^ vvv Linux vvv
 
-#ifdef ANDROID
-#include <android/sharedmem.h>
-#endif
-
 #ifndef _GNU_SOURCE
 #define _GNU_SOURCE
 #endif
@@ -193,6 +189,11 @@ public:
         }
     }
 
+    bool ClearBackingRegion(size_t physical_offset, size_t length) {
+        // TODO: This does not seem to be possible on Windows.
+        return false;
+    }
+
     void EnableDirectMappedAddress() {
         // TODO
         UNREACHABLE();
@@ -442,9 +443,7 @@ public:
         }
 
         // Backing memory initialization
-#ifdef ANDROID
-        fd = ASharedMemory_create("HostMemory", backing_size);
-#elif defined(__FreeBSD__) && __FreeBSD__ < 13
+#if defined(__FreeBSD__) && __FreeBSD__ < 13
         // XXX Drop after FreeBSD 12.* reaches EOL on 2024-06-30
         fd = shm_open(SHM_ANON, O_RDWR, 0600);
 #else
@@ -455,7 +454,6 @@ public:
             throw std::bad_alloc{};
         }
 
-#ifndef ANDROID
         // Defined to extend the file with zeros
         int ret = ftruncate(fd, backing_size);
         if (ret != 0) {
@@ -463,7 +461,6 @@ public:
                          strerror(errno));
             throw std::bad_alloc{};
         }
-#endif
 
         backing_base = static_cast<u8*>(
             mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
@@ -552,6 +549,19 @@ public:
         ASSERT_MSG(ret == 0, "mprotect failed: {}", strerror(errno));
     }
 
+    bool ClearBackingRegion(size_t physical_offset, size_t length) {
+#ifdef __linux__
+        // Set MADV_REMOVE on backing map to destroy it instantly.
+        // This also deletes the area from the backing file.
+        int ret = madvise(backing_base + physical_offset, length, MADV_REMOVE);
+        ASSERT_MSG(ret == 0, "madvise failed: {}", strerror(errno));
+
+        return true;
+#else
+        return false;
+#endif
+    }
+
     void EnableDirectMappedAddress() {
         virtual_base = nullptr;
     }
@@ -623,6 +633,8 @@ public:
 
     void Protect(size_t virtual_offset, size_t length, bool read, bool write, bool execute) {}
 
+    void ClearBackingRegion(size_t physical_offset, size_t length) {}
+
     void EnableDirectMappedAddress() {}
 
     u8* backing_base{nullptr};
@@ -698,6 +710,12 @@ void HostMemory::Protect(size_t virtual_offset, size_t length, bool read, bool w
     impl->Protect(virtual_offset + virtual_base_offset, length, read, write, execute);
 }
 
+void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value) {
+    if (!impl || fill_value != 0 || !impl->ClearBackingRegion(physical_offset, length)) {
+        std::memset(backing_base + physical_offset, fill_value, length);
+    }
+}
+
 void HostMemory::EnableDirectMappedAddress() {
     if (impl) {
         impl->EnableDirectMappedAddress();

src/common/host_memory.h

@@ -48,6 +48,8 @@ public:
 
     void EnableDirectMappedAddress();
 
+    void ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value);
+
     [[nodiscard]] u8* BackingBasePointer() noexcept {
         return backing_base;
     }

src/core/hle/kernel/k_memory_manager.cpp

@@ -421,8 +421,9 @@ Result KMemoryManager::AllocateForProcess(KPageGroup* out, size_t num_pages, u32
     } else {
         // Set all the allocated memory.
         for (const auto& block : *out) {
-            std::memset(m_system.DeviceMemory().GetPointer<void>(block.GetAddress()), fill_pattern,
-                        block.GetSize());
+            m_system.DeviceMemory().buffer.ClearBackingRegion(GetInteger(block.GetAddress()) -
+                                                                  Core::DramMemoryMap::Base,
+                                                              block.GetSize(), fill_pattern);
         }
     }
 

src/core/hle/kernel/k_page_table_base.cpp

@@ -81,6 +81,11 @@ void InvalidateInstructionCache(KernelCore& kernel, AddressType addr, u64 size)
     }
 }
 
+void ClearBackingRegion(Core::System& system, KPhysicalAddress addr, u64 size, u32 fill_value) {
+    system.DeviceMemory().buffer.ClearBackingRegion(GetInteger(addr) - Core::DramMemoryMap::Base,
+                                                    size, fill_value);
+}
+
 template <typename AddressType>
 Result InvalidateDataCache(AddressType addr, u64 size) {
     R_SUCCEED();
@@ -1363,8 +1368,7 @@ Result KPageTableBase::MapInsecureMemory(KProcessAddress address, size_t size) {
 
     // Clear all the newly allocated pages.
     for (const auto& it : pg) {
-        std::memset(GetHeapVirtualPointer(m_kernel, it.GetAddress()),
-                    static_cast<u32>(m_heap_fill_value), it.GetSize());
+        ClearBackingRegion(m_system, it.GetAddress(), it.GetSize(), m_heap_fill_value);
     }
 
     // Lock the table.
@@ -1570,8 +1574,7 @@ Result KPageTableBase::AllocateAndMapPagesImpl(PageLinkedList* page_list, KProce
 
     // Clear all pages.
     for (const auto& it : pg) {
-        std::memset(GetHeapVirtualPointer(m_kernel, it.GetAddress()),
-                    static_cast<u32>(m_heap_fill_value), it.GetSize());
+        ClearBackingRegion(m_system, it.GetAddress(), it.GetSize(), m_heap_fill_value);
     }
 
     // Map the pages.
@@ -2159,8 +2162,7 @@ Result KPageTableBase::SetHeapSize(KProcessAddress* out, size_t size) {
 
     // Clear all the newly allocated pages.
     for (const auto& it : pg) {
-        std::memset(GetHeapVirtualPointer(m_kernel, it.GetAddress()), m_heap_fill_value,
-                    it.GetSize());
+        ClearBackingRegion(m_system, it.GetAddress(), it.GetSize(), m_heap_fill_value);
     }
 
     // Map the pages.

src/tests/video_core/memory_tracker.cpp

@@ -23,13 +23,13 @@ constexpr VAddr c = 16 * HIGH_PAGE_SIZE;
 
 class RasterizerInterface {
 public:
-    void UpdatePagesCachedCount(VAddr addr, u64 size, int delta) {
+    void UpdatePagesCachedCount(VAddr addr, u64 size, bool cache) {
         const u64 page_start{addr >> Core::Memory::YUZU_PAGEBITS};
         const u64 page_end{(addr + size + Core::Memory::YUZU_PAGESIZE - 1) >>
                            Core::Memory::YUZU_PAGEBITS};
         for (u64 page = page_start; page < page_end; ++page) {
             int& value = page_table[page];
-            value += delta;
+            value += (cache ? 1 : -1);
             if (value < 0) {
                 throw std::logic_error{"negative page"};
             }
@@ -546,4 +546,4 @@ TEST_CASE("MemoryTracker: Cached write downloads") {
     REQUIRE(!memory_track->IsRegionGpuModified(c + PAGE, PAGE));
     memory_track->MarkRegionAsCpuModified(c, WORD);
     REQUIRE(rasterizer.Count() == 0);
-}
+}

src/video_core/buffer_cache/word_manager.h

@@ -473,7 +473,7 @@ private:
         VAddr addr = cpu_addr + word_index * BYTES_PER_WORD;
         IteratePages(changed_bits, [&](size_t offset, size_t size) {
             rasterizer->UpdatePagesCachedCount(addr + offset * BYTES_PER_PAGE,
-                                               size * BYTES_PER_PAGE, add_to_rasterizer ? 1 : -1);
+                                               size * BYTES_PER_PAGE, add_to_rasterizer);
         });
     }
 

src/video_core/rasterizer_accelerated.cpp

@@ -3,6 +3,7 @@
 
 #include <atomic>
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/common_types.h"
 #include "common/div_ceil.h"
@@ -11,61 +12,63 @@
 
 namespace VideoCore {
 
+static constexpr u16 IdentityValue = 1;
+
 using namespace Core::Memory;
 
-RasterizerAccelerated::RasterizerAccelerated(Memory& cpu_memory_)
-    : cached_pages(std::make_unique<CachedPages>()), cpu_memory{cpu_memory_} {}
+RasterizerAccelerated::RasterizerAccelerated(Memory& cpu_memory_) : map{}, cpu_memory{cpu_memory_} {
+    // We are tracking CPU memory, which cannot map more than 39 bits.
+    const VAddr start_address = 0;
+    const VAddr end_address = (1ULL << 39);
+    const IntervalType address_space_interval(start_address, end_address);
+    const auto value = std::make_pair(address_space_interval, IdentityValue);
+
+    map.add(value);
+}
 
 RasterizerAccelerated::~RasterizerAccelerated() = default;
 
-void RasterizerAccelerated::UpdatePagesCachedCount(VAddr addr, u64 size, int delta) {
-    u64 uncache_begin = 0;
-    u64 cache_begin = 0;
-    u64 uncache_bytes = 0;
-    u64 cache_bytes = 0;
+void RasterizerAccelerated::UpdatePagesCachedCount(VAddr addr, u64 size, bool cache) {
+    // Align sizes.
+    addr = Common::AlignDown(addr, YUZU_PAGESIZE);
+    size = Common::AlignUp(size, YUZU_PAGESIZE);
 
-    std::atomic_thread_fence(std::memory_order_acquire);
-    const u64 page_end = Common::DivCeil(addr + size, YUZU_PAGESIZE);
-    for (u64 page = addr >> YUZU_PAGEBITS; page != page_end; ++page) {
-        std::atomic_uint16_t& count = cached_pages->at(page >> 2).Count(page);
+    // Declare the overall interval we are going to operate on.
+    const VAddr start_address = addr;
+    const VAddr end_address = addr + size;
+    const IntervalType modification_range(start_address, end_address);
 
-        if (delta > 0) {
-            ASSERT_MSG(count.load(std::memory_order::relaxed) < UINT16_MAX, "Count may overflow!");
-        } else if (delta < 0) {
-            ASSERT_MSG(count.load(std::memory_order::relaxed) > 0, "Count may underflow!");
-        } else {
-            ASSERT_MSG(false, "Delta must be non-zero!");
-        }
+    // Find the boundaries of where to iterate.
+    const auto lower = map.lower_bound(modification_range);
+    const auto upper = map.upper_bound(modification_range);
 
-        // Adds or subtracts 1, as count is a unsigned 8-bit value
-        count.fetch_add(static_cast<u16>(delta), std::memory_order_release);
+    // Iterate over the contained intervals.
+    for (auto it = lower; it != upper; it++) {
+        // Intersect interval range with modification range.
+        const auto current_range = modification_range & it->first;
 
-        // Assume delta is either -1 or 1
-        if (count.load(std::memory_order::relaxed) == 0) {
-            if (uncache_bytes == 0) {
-                uncache_begin = page;
-            }
-            uncache_bytes += YUZU_PAGESIZE;
-        } else if (uncache_bytes > 0) {
-            cpu_memory.RasterizerMarkRegionCached(uncache_begin << YUZU_PAGEBITS, uncache_bytes,
-                                                  false);
-            uncache_bytes = 0;
-        }
-        if (count.load(std::memory_order::relaxed) == 1 && delta > 0) {
-            if (cache_bytes == 0) {
-                cache_begin = page;
-            }
-            cache_bytes += YUZU_PAGESIZE;
-        } else if (cache_bytes > 0) {
-            cpu_memory.RasterizerMarkRegionCached(cache_begin << YUZU_PAGEBITS, cache_bytes, true);
-            cache_bytes = 0;
+        // Calculate the address and size to operate over.
+        const auto current_addr = current_range.lower();
+        const auto current_size = current_range.upper() - current_addr;
+
+        // Get the current value of the range.
+        const auto value = it->second;
+
+        if (cache && value == IdentityValue) {
+            // If we are going to cache, and the value is not yet referenced, then cache this range.
+            cpu_memory.RasterizerMarkRegionCached(current_addr, current_size, true);
+        } else if (!cache && value == IdentityValue + 1) {
+            // If we are going to uncache, and this is the last reference, then uncache this range.
+            cpu_memory.RasterizerMarkRegionCached(current_addr, current_size, false);
         }
     }
-    if (uncache_bytes > 0) {
-        cpu_memory.RasterizerMarkRegionCached(uncache_begin << YUZU_PAGEBITS, uncache_bytes, false);
-    }
-    if (cache_bytes > 0) {
-        cpu_memory.RasterizerMarkRegionCached(cache_begin << YUZU_PAGEBITS, cache_bytes, true);
+
+    // Update the set.
+    const auto value = std::make_pair(modification_range, IdentityValue);
+    if (cache) {
+        map.add(value);
+    } else {
+        map.subtract(value);
     }
 }
 

src/video_core/rasterizer_accelerated.h

@@ -3,8 +3,7 @@
 
 #pragma once
 
-#include <array>
-#include <atomic>
+#include <boost/icl/interval_map.hpp>
 
 #include "common/common_types.h"
 #include "video_core/rasterizer_interface.h"
@@ -21,28 +20,16 @@ public:
     explicit RasterizerAccelerated(Core::Memory::Memory& cpu_memory_);
     ~RasterizerAccelerated() override;
 
-    void UpdatePagesCachedCount(VAddr addr, u64 size, int delta) override;
+    void UpdatePagesCachedCount(VAddr addr, u64 size, bool cache) override;
 
 private:
-    class CacheEntry final {
-    public:
-        CacheEntry() = default;
+    using PageIndex = VAddr;
+    using PageReferenceCount = u16;
 
-        std::atomic_uint16_t& Count(std::size_t page) {
-            return values[page & 3];
-        }
+    using IntervalMap = boost::icl::interval_map<PageIndex, PageReferenceCount>;
+    using IntervalType = IntervalMap::interval_type;
 
-        const std::atomic_uint16_t& Count(std::size_t page) const {
-            return values[page & 3];
-        }
-
-    private:
-        std::array<std::atomic_uint16_t, 4> values{};
-    };
-    static_assert(sizeof(CacheEntry) == 8, "CacheEntry should be 8 bytes!");
-
-    using CachedPages = std::array<CacheEntry, 0x2000000>;
-    std::unique_ptr<CachedPages> cached_pages;
+    IntervalMap map;
     Core::Memory::Memory& cpu_memory;
 };
 

src/video_core/rasterizer_interface.h

@@ -162,7 +162,7 @@ public:
     }
 
     /// Increase/decrease the number of object in pages touching the specified region
-    virtual void UpdatePagesCachedCount(VAddr addr, u64 size, int delta) {}
+    virtual void UpdatePagesCachedCount(VAddr addr, u64 size, bool cache) {}
 
     /// Initialize disk cached resources for the game being emulated
     virtual void LoadDiskResources(u64 title_id, std::stop_token stop_loading,

src/video_core/renderer_vulkan/vk_present_manager.cpp

@@ -289,7 +289,7 @@ void PresentManager::PresentThread(std::stop_token token) {
 
 void PresentManager::RecreateSwapchain(Frame* frame) {
     swapchain.Create(*surface, frame->width, frame->height);
-    image_count = swapchain.GetImageCount();
+    SetImageCount();
 }
 
 void PresentManager::SetImageCount() {

src/video_core/shader_cache.cpp

@@ -132,7 +132,7 @@ void ShaderCache::Register(std::unique_ptr<ShaderInfo> data, VAddr addr, size_t
 
     storage.push_back(std::move(data));
 
-    rasterizer.UpdatePagesCachedCount(addr, size, 1);
+    rasterizer.UpdatePagesCachedCount(addr, size, true);
 }
 
 void ShaderCache::InvalidatePagesInRegion(VAddr addr, size_t size) {
@@ -209,7 +209,7 @@ void ShaderCache::UnmarkMemory(Entry* entry) {
 
     const VAddr addr = entry->addr_start;
     const size_t size = entry->addr_end - addr;
-    rasterizer.UpdatePagesCachedCount(addr, size, -1);
+    rasterizer.UpdatePagesCachedCount(addr, size, false);
 }
 
 void ShaderCache::RemoveShadersFromStorage(std::span<ShaderInfo*> removed_shaders) {

src/video_core/texture_cache/texture_cache.h

@@ -2080,7 +2080,7 @@ void TextureCache<P>::TrackImage(ImageBase& image, ImageId image_id) {
     ASSERT(False(image.flags & ImageFlagBits::Tracked));
     image.flags |= ImageFlagBits::Tracked;
     if (False(image.flags & ImageFlagBits::Sparse)) {
-        rasterizer.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, 1);
+        rasterizer.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, true);
         return;
     }
     if (True(image.flags & ImageFlagBits::Registered)) {
@@ -2091,13 +2091,13 @@ void TextureCache<P>::TrackImage(ImageBase& image, ImageId image_id) {
             const auto& map = slot_map_views[map_view_id];
             const VAddr cpu_addr = map.cpu_addr;
             const std::size_t size = map.size;
-            rasterizer.UpdatePagesCachedCount(cpu_addr, size, 1);
+            rasterizer.UpdatePagesCachedCount(cpu_addr, size, true);
         }
         return;
     }
     ForEachSparseSegment(image,
                          [this]([[maybe_unused]] GPUVAddr gpu_addr, VAddr cpu_addr, size_t size) {
-                             rasterizer.UpdatePagesCachedCount(cpu_addr, size, 1);
+                             rasterizer.UpdatePagesCachedCount(cpu_addr, size, true);
                          });
 }
 
@@ -2106,7 +2106,7 @@ void TextureCache<P>::UntrackImage(ImageBase& image, ImageId image_id) {
     ASSERT(True(image.flags & ImageFlagBits::Tracked));
     image.flags &= ~ImageFlagBits::Tracked;
     if (False(image.flags & ImageFlagBits::Sparse)) {
-        rasterizer.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, -1);
+        rasterizer.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, false);
         return;
     }
     ASSERT(True(image.flags & ImageFlagBits::Registered));
@@ -2117,7 +2117,7 @@ void TextureCache<P>::UntrackImage(ImageBase& image, ImageId image_id) {
         const auto& map = slot_map_views[map_view_id];
         const VAddr cpu_addr = map.cpu_addr;
         const std::size_t size = map.size;
-        rasterizer.UpdatePagesCachedCount(cpu_addr, size, -1);
+        rasterizer.UpdatePagesCachedCount(cpu_addr, size, false);
     }
 }