Recognize Copilot "insufficient remaining quota" notice for review fallback (#501)

* Initial plan

* Recognize insufficient-quota Copilot notice for review fallback

Co-authored-by: trask <218610+trask@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: trask <218610+trask@users.noreply.github.com>

Author: Copilot

.github/scripts/pull-request-dashboard/copilot_review.py

@@ -22,10 +22,12 @@
 )
 
 
-COPILOT_QUOTA_FAILURE_BODY = (
+COPILOT_QUOTA_FAILURE_BODIES = frozenset({
     "Copilot was unable to review this pull request because the user who "
-    "requested the review has reached their quota limit."
-)
+    "requested the review has reached their quota limit.",
+    "Copilot was unable to review this pull request because the user who "
+    "requested the review has insufficient remaining quota.",
+})
 COPILOT_ERROR_BODY = (
     "Copilot encountered an error and was unable to review this pull request. "
     "You can try again by re-requesting a review."
@@ -51,7 +53,7 @@ def is_copilot_reviewer(
 def is_copilot_quota_failure(review: Review) -> bool:
     return (
         is_copilot_reviewer(review)
-        and review.body.strip() == COPILOT_QUOTA_FAILURE_BODY
+        and review.body.strip() in COPILOT_QUOTA_FAILURE_BODIES
     )
 
 

.github/scripts/pull-request-dashboard/test_copilot_review.py

@@ -52,6 +52,10 @@
     "Copilot was unable to review this pull request because the user who "
     "requested the review has reached their quota limit."
 )
+INSUFFICIENT_QUOTA_FAILURE_BODY = (
+    "Copilot was unable to review this pull request because the user who "
+    "requested the review has insufficient remaining quota."
+)
 ERROR_BODY = (
     "Copilot encountered an error and was unable to review this pull request. "
     "You can try again by re-requesting a review."
@@ -203,6 +207,17 @@ def test_quota_failure_requires_the_copilot_actor_and_exact_notice(self) -> None
         self.assertFalse(is_copilot_quota_failure(
             replace(failure, actor=actor("reviewer"))
         ))
+        insufficient = replace(failure, body=INSUFFICIENT_QUOTA_FAILURE_BODY)
+        self.assertTrue(is_copilot_quota_failure(insufficient))
+        self.assertTrue(is_copilot_quota_failure(
+            replace(insufficient, body=f"\n{INSUFFICIENT_QUOTA_FAILURE_BODY}\n")
+        ))
+        self.assertFalse(is_copilot_quota_failure(
+            replace(insufficient, actor=actor("reviewer"))
+        ))
+        self.assertFalse(is_copilot_quota_failure(
+            replace(insufficient, body=f"> {INSUFFICIENT_QUOTA_FAILURE_BODY}")
+        ))
         self.assertFalse(is_copilot_quota_failure(
             replace(failure, body=f"Handle this failure:\n> {QUOTA_FAILURE_BODY}")
         ))
@@ -226,6 +241,11 @@ def test_quota_failures_do_not_count_as_completed_reviews(self) -> None:
                 (replace(quota_failure, body=f"\n{QUOTA_FAILURE_BODY}\n"),),
                 (False, False),
             ),
+            (
+                "insufficient_quota",
+                (replace(quota_failure, body=INSUFFICIENT_QUOTA_FAILURE_BODY),),
+                (False, False),
+            ),
             (
                 "current_review_then_failure",
                 (completed_review, quota_failure),
@@ -418,12 +438,12 @@ def enqueue(self, *, quota_exhausted: bool = False) -> None:
             NOW,
         )
 
-    def quota_notice(self, review_id: int) -> dict:
+    def quota_notice(self, review_id: int, body: str = QUOTA_FAILURE_BODY) -> dict:
         return {
             "id": review_id,
             "user": {"login": "copilot"},
             "commit_id": "current-head",
-            "body": QUOTA_FAILURE_BODY,
+            "body": body,
         }
 
     def error_notice(self, review_id: int) -> dict:
@@ -438,6 +458,19 @@ def error_notice(self, review_id: int) -> dict:
     def deliver(self) -> list[str]:
         return deliver_copilot_review_requests("open-telemetry/example", NOW)
 
+    def test_insufficient_quota_notice_triggers_the_fallback_token(self) -> None:
+        self.enqueue()
+        self.reviews = [
+            self.quota_notice(10, INSUFFICIENT_QUOTA_FAILURE_BODY),
+            self.quota_notice(11),
+        ]
+        self.assertEqual([], self.deliver())
+        self.assertEqual(
+            CopilotReviewFallback((10, 11), format_ts(NOW)),
+            load_copilot_review_fallbacks()["7"],
+        )
+        self.request.assert_called_with("PR_node", token="request-token")
+
     def test_uses_app_until_a_live_quota_notice_arrives(self) -> None:
         self.assertEqual([], self.deliver())
         self.request.assert_called_once_with("PR_node")