diff --git a/src/video_core/rasterizer_accelerated.cpp b/src/video_core/rasterizer_accelerated.cpp
index 9f689d152..ed6f37c76 100644
--- a/src/video_core/rasterizer_accelerated.cpp
+++ b/src/video_core/rasterizer_accelerated.cpp
@@ -71,4 +71,38 @@ void RasterizerAccelerated::UpdatePagesCachedCount(PAddr addr, u32 size, int del
     }
 }
 
+void RasterizerAccelerated::ClearAll(bool flush) {
+    // Force flush all surfaces from the cache
+    if (flush) {
+        FlushRegion(0x0, 0xFFFFFFFF);
+    }
+
+    u32 uncache_start_addr = 0;
+    u32 uncache_bytes = 0;
+
+    for (u32 page = 0; page != cached_pages.size(); page++) {
+        auto& count = cached_pages.at(page);
+
+        // Assume delta is either -1 or 1
+        if (count != 0) {
+            if (uncache_bytes == 0) {
+                uncache_start_addr = page << Memory::CITRA_PAGE_BITS;
+            }
+
+            uncache_bytes += Memory::CITRA_PAGE_SIZE;
+        } else if (uncache_bytes > 0) {
+            VideoCore::g_memory->RasterizerMarkRegionCached(uncache_start_addr, uncache_bytes,
+                                                            false);
+            uncache_bytes = 0;
+        }
+    }
+
+    if (uncache_bytes > 0) {
+        VideoCore::g_memory->RasterizerMarkRegionCached(uncache_start_addr, uncache_bytes,
+                                                        false);
+    }
+
+    cached_pages = {};
+}
+
 } // namespace VideoCore
diff --git a/src/video_core/rasterizer_accelerated.h b/src/video_core/rasterizer_accelerated.h
index b5bf1487d..ff238b623 100644
--- a/src/video_core/rasterizer_accelerated.h
+++ b/src/video_core/rasterizer_accelerated.h
@@ -13,6 +13,8 @@ public:
 
     void UpdatePagesCachedCount(PAddr addr, u32 size, int delta) override;
 
+    void ClearAll(bool flush) override;
+
 private:
     std::array<u16, 0x30000> cached_pages{};
 };
diff --git a/src/video_core/rasterizer_cache/rasterizer_cache.cpp b/src/video_core/rasterizer_cache/rasterizer_cache.cpp
index 1d103c6f7..0557ac169 100644
--- a/src/video_core/rasterizer_cache/rasterizer_cache.cpp
+++ b/src/video_core/rasterizer_cache/rasterizer_cache.cpp
@@ -920,30 +920,6 @@ bool RasterizerCache::ValidateByReinterpretation(const Surface& surface,
     return false;
 }
 
-void RasterizerCache::ClearAll(bool flush) {
-    const auto flush_interval = PageMap::interval_type::right_open(0x0, 0xFFFFFFFF);
-    // Force flush all surfaces from the cache
-    if (flush) {
-        FlushRegion(0x0, 0xFFFFFFFF);
-    }
-    // Unmark all of the marked pages
-    for (auto& pair : RangeFromInterval(cached_pages, flush_interval)) {
-        const auto interval = pair.first & flush_interval;
-
-        const PAddr interval_start_addr = boost::icl::first(interval) << Memory::CITRA_PAGE_BITS;
-        const PAddr interval_end_addr = boost::icl::last_next(interval) << Memory::CITRA_PAGE_BITS;
-        const u32 interval_size = interval_end_addr - interval_start_addr;
-
-        VideoCore::g_memory->RasterizerMarkRegionCached(interval_start_addr, interval_size, false);
-    }
-
-    // Remove the whole cache without really looking at it.
-    cached_pages -= flush_interval;
-    dirty_regions -= SurfaceInterval(0x0, 0xFFFFFFFF);
-    surface_cache -= SurfaceInterval(0x0, 0xFFFFFFFF);
-    remove_surfaces.clear();
-}
-
 void RasterizerCache::FlushRegion(PAddr addr, u32 size, Surface flush_surface) {
     std::lock_guard lock{mutex};
 
diff --git a/src/video_core/rasterizer_cache/rasterizer_cache.h b/src/video_core/rasterizer_cache/rasterizer_cache.h
index 583b59628..152f2529b 100644
--- a/src/video_core/rasterizer_cache/rasterizer_cache.h
+++ b/src/video_core/rasterizer_cache/rasterizer_cache.h
@@ -30,7 +30,6 @@ static_assert(std::is_same<SurfaceRegions::interval_type, SurfaceCache::interval
 
 using SurfaceRect_Tuple = std::tuple<Surface, Common::Rectangle<u32>>;
 using SurfaceSurfaceRect_Tuple = std::tuple<Surface, Surface, Common::Rectangle<u32>>;
-using PageMap = boost::icl::interval_map<u32, int>;
 
 enum class ScaleMatch {
     Exact,   // Only accept same res scale
@@ -91,9 +90,6 @@ public:
     /// Flush all cached resources tracked by this cache manager
     void FlushAll();
 
-    /// Clear all cached resources tracked by this cache manager
-    void ClearAll(bool flush);
-
     // Textures from destroyed surfaces are stored here to be recyled to reduce allocation overhead
     // in the driver
     // this must be placed above the surface_cache to ensure all cached surfaces are destroyed
@@ -130,7 +126,6 @@ private:
     VideoCore::RasterizerAccelerated& rasterizer;
     TextureRuntime runtime;
     SurfaceCache surface_cache;
-    PageMap cached_pages;
     SurfaceMap dirty_regions;
     SurfaceSet remove_surfaces;
 
diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp
index c95dbe2b4..8040ad29e 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer.cpp
+++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp
@@ -1359,10 +1359,6 @@ void RasterizerOpenGL::FlushAndInvalidateRegion(PAddr addr, u32 size) {
     res_cache.InvalidateRegion(addr, size, nullptr);
 }
 
-void RasterizerOpenGL::ClearAll(bool flush) {
-    res_cache.ClearAll(flush);
-}
-
 bool RasterizerOpenGL::AccelerateDisplayTransfer(const GPU::Regs::DisplayTransferConfig& config) {
     MICROPROFILE_SCOPE(OpenGL_Blits);
 
diff --git a/src/video_core/renderer_opengl/gl_rasterizer.h b/src/video_core/renderer_opengl/gl_rasterizer.h
index d86381740..1cb2489fc 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer.h
+++ b/src/video_core/renderer_opengl/gl_rasterizer.h
@@ -39,7 +39,6 @@ public:
     void FlushRegion(PAddr addr, u32 size) override;
     void InvalidateRegion(PAddr addr, u32 size) override;
     void FlushAndInvalidateRegion(PAddr addr, u32 size) override;
-    void ClearAll(bool flush) override;
     bool AccelerateDisplayTransfer(const GPU::Regs::DisplayTransferConfig& config) override;
     bool AccelerateTextureCopy(const GPU::Regs::DisplayTransferConfig& config) override;
     bool AccelerateFill(const GPU::Regs::MemoryFillConfig& config) override;