added concurrency to fetches for faster audit sync

Author: spotta85

features/audit-scraping/background-controller.ts

@@ -22,9 +22,8 @@ export interface AuditBatchDependencies {
   scrapeAudit: (auditId: string, tabId: number) => Promise<CachedAuditData>;
   saveAudit: (auditId: string, audit: CachedAuditData) => Promise<void>;
   broadcast: (state: "started" | "complete") => Promise<void>;
-  delay?: (milliseconds: number) => Promise<void>;
   scrapeTimeoutMs?: number;
-  requestDelayMs?: number;
+  concurrency?: number;
 }
 
 export class AuditBatchController {
@@ -59,8 +58,17 @@ export class AuditBatchController {
     const result: AuditBatchResult = { succeeded: [], failed: [] };
     await this.dependencies.broadcast("started");
 
-    try {
-      for (const [index, auditId] of auditIds.entries()) {
+    // A few plain page fetches in flight at once — fewer than a normal page
+    // load opens against one host — keeps the "Syncing" window short.
+    const queue = [...auditIds];
+    let aborted = false;
+
+    const worker = async (): Promise<void> => {
+      for (
+        let auditId = queue.shift();
+        auditId !== undefined && !aborted;
+        auditId = queue.shift()
+      ) {
         try {
           const audit = await this.scrapeWithTimeout(auditId, tabId);
           await this.dependencies.saveAudit(auditId, audit);
@@ -71,20 +79,25 @@ export class AuditBatchController {
           // A dead session fails every remaining audit the same way; stop
           // instead of hammering the login redirect.
           if (error instanceof Error && error.message === "AUTH_REQUIRED") {
-            result.failed.push(...auditIds.slice(index + 1));
-            break;
-          }
-        }
-
-        if (index < auditIds.length - 1) {
-          const delayMs = this.dependencies.requestDelayMs ?? 150;
-          if (this.dependencies.delay) {
-            await this.dependencies.delay(delayMs);
-          } else {
-            await new Promise((resolve) => setTimeout(resolve, delayMs));
+            aborted = true;
+            result.failed.push(...queue.splice(0));
           }
         }
       }
+    };
+
+    try {
+      await Promise.all(
+        Array.from(
+          {
+            length: Math.min(
+              this.dependencies.concurrency ?? 3,
+              auditIds.length,
+            ),
+          },
+          () => worker(),
+        ),
+      );
     } finally {
       await this.dependencies.broadcast("complete");
       const summary = `Audit batch complete: ${result.succeeded.length} succeeded, ${result.failed.length} failed`;

features/popup/popup-app.tsx

@@ -102,9 +102,7 @@ export default function App() {
     });
   };
 
-  // The background owns the auth checks: it refuses on a known-dead session
-  // and the run's own form fetch is a live check, either way opening the
-  // login page — no need for a slow probe from here first.
+  // The background owns the auth checks: 
   const handleRerunAudit = async () => {
     setRunningAudit(true);
     try {

tests/audit-scraping/background-controller.test.ts

@@ -15,7 +15,6 @@ function createController(
     scrapeAudit: async () => audit,
     saveAudit: async () => {},
     broadcast: async () => {},
-    delay: async () => {},
     scrapeTimeoutMs: 20,
     ...overrides,
   });
@@ -69,6 +68,7 @@ describe("audit batch controller", () => {
         scrapes++;
         throw new Error("AUTH_REQUIRED");
       },
+      concurrency: 1,
     });
 
     controller.start(["1", "2", "3"], TAB_ID);
@@ -79,6 +79,26 @@ describe("audit batch controller", () => {
     expect(scrapes).toBe(1);
   });
 
+  test("fetches audits in parallel up to the concurrency limit", async () => {
+    let inFlight = 0;
+    let maxInFlight = 0;
+    const controller = createController({
+      scrapeAudit: async () => {
+        inFlight++;
+        maxInFlight = Math.max(maxInFlight, inFlight);
+        await new Promise((resolve) => setTimeout(resolve, 5));
+        inFlight--;
+        return audit;
+      },
+      concurrency: 2,
+    });
+
+    controller.start(["1", "2", "3", "4"], TAB_ID);
+    const result = await controller.waitForIdle();
+    expect(result?.succeeded.toSorted()).toEqual(["1", "2", "3", "4"]);
+    expect(maxInFlight).toBe(2);
+  });
+
   test("times out a scrape that never responds", async () => {
     const controller = createController({
       scrapeAudit: () => new Promise<never>(() => {}),