Fix RCTInstance self-deadlock during invalidate

Summary:
`-[RCTInstance invalidate]` held `_invalidationMutex` while it synchronously called `[_surfacePresenter suspend]`. Stopping surfaces notifies mount observers on the calling thread, and any observer that reaches `callFunctionOnJSModule:` (e.g. `RCTLog` forwarding to JS in debug builds) blocks forever on the same non-recursive mutex.

Release the mutex after setting `_valid = false` and unregistering from the inspector, before `suspend`. Re-entrant calls now see `!_valid` and return.

Changelog:
[iOS][Fixed] - Fix deadlock in `RCTInstance` invalidation when a surface observer calls into JS

Reviewed By: javache

Differential Revision: D125706316

Author: zeyap

packages/react-native/ReactCommon/react/runtime/iostests/RCTInstanceTests.mm

@@ -11,6 +11,7 @@
 #import <React/RCTBridgeModule.h>
 #import <React/RCTBundleManager.h>
 #import <React/RCTJavaScriptLoader.h>
+#import <React/RCTSurfacePresenter.h>
 #import <ReactCommon/RCTHermesInstance.h>
 #import <ReactCommon/RCTInstance.h>
 #import <ReactCommon/RCTTurboModule.h>
@@ -241,4 +242,30 @@ - (void)testBundleLoadAwaitsMainQueueModuleSetup
       @"FakeEagerModule -init must have fully returned before the JS bundle finishes loading");
 }
 
+- (void)testInvalidateDoesNotDeadlockWhenSuspendCallsIntoJS
+{
+  OCMStub([_mockDelegate unstableModulesRequiringMainQueueSetup]).andReturn(@[]);
+
+  RCTInstance *instance = [self makeInstance];
+  __weak RCTInstance *weakInstance = instance;
+
+  // Stands in for a surface observer that calls into JS while suspend stops surfaces.
+  id presenterMock = OCMPartialMock(instance.surfacePresenter);
+  OCMStub([(RCTSurfacePresenter *)presenterMock suspend]).andDo(^(NSInvocation *invocation) {
+    [weakInstance callFunctionOnJSModule:@"RCTLog" method:@"logIfNoNativeHook" args:@[]];
+    BOOL result = YES;
+    [invocation setReturnValue:&result];
+  });
+
+  // Invalidate off-main so a deadlock fails the wait instead of hanging the test runner.
+  XCTestExpectation *invalidated = [self expectationWithDescription:@"invalidate returned"];
+  dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), ^{
+    [instance invalidate];
+    [invalidated fulfill];
+  });
+  [self waitForExpectations:@[ invalidated ] timeout:5.0];
+
+  [presenterMock stopMocking];
+}
+
 @end

packages/react-native/ReactCommon/react/runtime/platform/ios/ReactCommon/RCTInstance.mm

@@ -183,11 +183,15 @@ - (void)callFunctionOnJSModule:(NSString *)moduleName method:(NSString *)method
 
 - (void)invalidate
 {
-  std::lock_guard<std::mutex> lock(_invalidationMutex);
-  _valid = false;
-  if (_reactInstance) {
-    _reactInstance->unregisterFromInspector();
+  {
+    std::lock_guard<std::mutex> lock(_invalidationMutex);
+    _valid = false;
+    if (_reactInstance) {
+      _reactInstance->unregisterFromInspector();
+    }
   }
+  // suspend shouldn't be called under _invalidationMutex: it notifies surface observers,
+  // which can call callFunctionOnJSModule and deadlock on the same mutex.
   [_surfacePresenter suspend];
   [_jsThreadManager dispatchToJSThread:^{
     /**