@@ -14,21 +14,113 @@ namespace rsx
1414 rsx::simple_array<MM_block> g_deferred_mprotect_queue;
1515 shared_mutex g_mprotect_queue_lock;
1616
17- void mm_flush_mprotect_queue_internal ()
17+ void mm_sanitize_queue_internal ()
1818 {
19- for (const auto & block : g_deferred_mprotect_queue)
19+ u32 w = 0 , r = 0 ;
20+ for (; r < g_deferred_mprotect_queue.size (); ++r)
2021 {
22+ auto & block = g_deferred_mprotect_queue[r];
23+ if (!block.range .valid ())
24+ {
25+ continue ;
26+ }
27+ g_deferred_mprotect_queue[w++] = block;
28+ }
29+ g_deferred_mprotect_queue.resize (w);
30+ }
31+
32+ void mm_flush_mprotect_queue_internal (u32 count)
33+ {
34+ AUDIT (count <= g_deferred_mprotect_queue.size ());
35+
36+ for (u32 i = 0 ; i < count; ++i)
37+ {
38+ auto & block = g_deferred_mprotect_queue[i];
39+ ensure (block.range .valid ());
40+
2141 utils::memory_protect (reinterpret_cast <void *>(block.range .start ), block.range .length (), block.prot );
42+ block.range .invalidate ();
43+ }
44+
45+ const u32 remaining = g_deferred_mprotect_queue.size () - count;
46+ if (!remaining)
47+ {
48+ g_deferred_mprotect_queue.clear ();
49+ return ;
50+ }
51+
52+ // Pop processed entries from the queue
53+ mm_sanitize_queue_internal ();
54+ }
55+
56+ // Reverse scan to find the latest overlapping MM conflict. The result is a count of prefix blocks.
57+ template <typename F>
58+ u32 mm_find_conflict_internal (F&& predicate)
59+ {
60+ for (u32 i = g_deferred_mprotect_queue.size (); i > 0 ; --i)
61+ {
62+ if (std::invoke (predicate, g_deferred_mprotect_queue[i - 1 ]))
63+ {
64+ return i;
65+ }
2266 }
2367
24- g_deferred_mprotect_queue. clear () ;
68+ return 0 ;
2569 }
2670
2771 void mm_defer_mprotect_internal (u64 start, u64 length, utils::protection prot)
2872 {
29- // We could stack and merge requests here, but that is more trouble than it is truly worth.
30- // A fresh call to memory_protect only takes a few nanoseconds of setup overhead, it is not worth the risk of hanging because of conflicts.
31- g_deferred_mprotect_queue.push_back ({ utils::address_range64::start_length (start, length), prot });
73+ const auto range = utils::address_range64::start_length (start, length);
74+ bool has_invalid = false ;
75+ bool is_merged = false ;
76+
77+ // Attempt a merge first. The queue length is short but the time taken to run mprotect is very high in comparison.
78+ for (auto it = g_deferred_mprotect_queue.rbegin ();
79+ it != g_deferred_mprotect_queue.rend ();
80+ ++it)
81+ {
82+ auto & block = *it;
83+ if (!block.touches (range))
84+ {
85+ continue ;
86+ }
87+
88+ if (block.prot != prot)
89+ {
90+ // Optimization. If our new range swallows the old one, replace it.
91+ if (block.range .inside (range))
92+ {
93+ block.range .invalidate ();
94+ has_invalid = true ;
95+ continue ;
96+ }
97+
98+ if (!block.overlaps (range))
99+ {
100+ // Adjacent. Skip.
101+ continue ;
102+ }
103+
104+ // Preserve ordering. Do not proceed with merge.
105+ break ;
106+ }
107+
108+ block.merge (range);
109+ is_merged = true ;
110+ break ;
111+ }
112+
113+ if (has_invalid)
114+ {
115+ mm_sanitize_queue_internal ();
116+ }
117+
118+ if (is_merged)
119+ {
120+ return ;
121+ }
122+
123+ g_deferred_mprotect_queue.push_back ({ range, prot });
32124 }
33125
34126 void mm_protect (void * ptr, u64 length, utils::protection prot)
@@ -47,13 +139,24 @@ namespace rsx
47139
48140 if (prot == utils::protection::rw || prot == utils::protection::wx)
49141 {
50- // Basically an unlock op. Flush if any overlap is detected
51- for ( const auto & block : g_deferred_mprotect_queue )
142+ // Basically an unlock op. Flush the conflicting prefix block if any overlap is detected.
143+ if ( u32 count = mm_find_conflict_internal ( FN (x. overlaps (range))) )
52144 {
53- if (block.overlaps (range))
145+ // Check for degenerate ranges that we'll be crushing
146+ for (auto pblock = &g_deferred_mprotect_queue[count - 1 ];
147+ count > 0 && pblock->range .inside (range);
148+ count--, pblock--)
54149 {
55- mm_flush_mprotect_queue_internal ();
56- break ;
150+ pblock->range .invalidate ();
151+ }
152+
153+ if (count)
154+ {
155+ mm_flush_mprotect_queue_internal (count);
156+ }
157+ else
158+ {
159+ mm_sanitize_queue_internal ();
57160 }
58161 }
59162
@@ -68,7 +171,7 @@ namespace rsx
68171 void mm_flush ()
69172 {
70173 std::lock_guard lock (g_mprotect_queue_lock);
71- mm_flush_mprotect_queue_internal ();
174+ mm_flush_mprotect_queue_internal (g_deferred_mprotect_queue. size () );
72175 }
73176
74177 void mm_flush (u32 vm_address)
@@ -80,31 +183,24 @@ namespace rsx
80183 }
81184
82185 const auto addr = reinterpret_cast <u64 >(vm::base (vm_address));
83- for (const auto & block : g_deferred_mprotect_queue )
186+ if (const u32 count = mm_find_conflict_internal ( FN (x. overlaps (addr))) )
84187 {
85- if (block.overlaps (addr))
86- {
87- mm_flush_mprotect_queue_internal ();
88- return ;
89- }
188+ mm_flush_mprotect_queue_internal (count);
90189 }
91190 }
92191
93192 void mm_flush (const rsx::simple_array<utils::address_range64>& ranges)
94193 {
95194 std::lock_guard lock (g_mprotect_queue_lock);
96- if (g_deferred_mprotect_queue.empty ())
195+ if (g_deferred_mprotect_queue.empty () || ranges. empty () )
97196 {
98197 return ;
99198 }
100199
101- for (const auto & block : g_deferred_mprotect_queue)
200+ const auto block_overlaps_ranges = [&](const MM_block& block) { return ranges.any (FN (block.overlaps (x))); };
201+ if (const u32 count = mm_find_conflict_internal (block_overlaps_ranges))
102202 {
103- if (ranges.any (FN (block.overlaps (x))))
104- {
105- mm_flush_mprotect_queue_internal ();
106- return ;
107- }
203+ mm_flush_mprotect_queue_internal (count);
108204 }
109205 }
110206
0 commit comments