Improve IMAP scaling and refresh README

Author: lucaomul

README.md

@@ -10,15 +10,17 @@
 from pydantic import BaseModel, Field
 
 from inboxanchor.api.v1.routers.auth import router as auth_router
-from inboxanchor.api.v1.routers.frontend import (
-    mark_frontend_provider_dirty,
-)
-from inboxanchor.api.v1.routers.frontend import (
-    router as frontend_router,
-)
+from inboxanchor.api.v1.routers.frontend import mark_frontend_provider_dirty
+from inboxanchor.api.v1.routers.frontend import router as frontend_router
 from inboxanchor.api.v1.routers.oauth import router as oauth_router
 from inboxanchor.api.v1.routers.webhooks import router as webhook_router
 from inboxanchor.bootstrap import InboxAnchorService, list_provider_profiles
+from inboxanchor.connectors.imap_transport import (
+    IMAPAuthenticationError,
+    IMAPFolderError,
+    ImaplibTransport,
+    IMAPTransportError,
+)
 from inboxanchor.infra.auth import AuthService
 from inboxanchor.infra.database import session_scope
 from inboxanchor.infra.repository import InboxRepository
@@ -138,6 +140,79 @@ def _service_for_request(provider_name: Optional[str] = None) -> InboxAnchorServ
     return InboxAnchorService(provider_name=provider_name)
 
 
+def _imap_auth_failure_message(provider: str) -> str:
+    if provider == "yahoo":
+        return (
+            "Yahoo rejected the IMAP login. Reconnect with a Yahoo app password from "
+            "Yahoo Account Security; normal Yahoo passwords usually do not work for IMAP."
+        )
+    if provider == "outlook":
+        return (
+            "Outlook rejected the IMAP login. Reconnect with the mailbox username and an "
+            "Outlook app password or provider-specific IMAP credential."
+        )
+    return (
+        "InboxAnchor could not log into the IMAP mailbox. Reconnect with the mailbox "
+        "username and the correct IMAP or app password."
+    )
+
+
+def _validate_imap_connection(
+    provider: str,
+    *,
+    state: IMAPConnectionState,
+    password: str,
+) -> None:
+    if not state.host.strip():
+        raise HTTPException(status_code=400, detail="Enter the IMAP host before connecting.")
+    if not state.username.strip():
+        raise HTTPException(
+            status_code=400,
+            detail="Enter the mailbox username before connecting the IMAP provider.",
+        )
+    if not password.strip():
+        raise HTTPException(
+            status_code=400,
+            detail=(
+                "Enter the mailbox password or app password before connecting the IMAP provider."
+            ),
+        )
+
+    transport = ImaplibTransport(
+        state.host,
+        state.port,
+        state.username,
+        password,
+        use_ssl=state.use_ssl,
+        mailbox=state.mailbox,
+        provider_name=provider,
+        archive_mailbox=state.archive_mailbox or None,
+        trash_mailbox=state.trash_mailbox or None,
+    )
+    try:
+        transport._connect()
+    except IMAPAuthenticationError as exc:
+        raise HTTPException(status_code=400, detail=_imap_auth_failure_message(provider)) from exc
+    except IMAPFolderError as exc:
+        raise HTTPException(
+            status_code=400,
+            detail=(
+                f"InboxAnchor logged into the mailbox, but could not open '{state.mailbox}'. "
+                "Check the mailbox name and try again."
+            ),
+        ) from exc
+    except IMAPTransportError as exc:
+        raise HTTPException(
+            status_code=502,
+            detail=(
+                f"InboxAnchor could not reach the live {provider.upper()} IMAP server. "
+                "Check the host, port, and backend network access, then try again."
+            ),
+        ) from exc
+    finally:
+        transport.close()
+
+
 def _cors_origins() -> list[str]:
     configured = os.getenv("INBOXANCHOR_CORS_ORIGINS", "").strip()
     if configured:
@@ -272,16 +347,46 @@ def save_provider_connection(
     if provider in {"imap", "yahoo", "outlook"} and payload.imap is not None:
         with session_scope() as session:
             repository = InboxRepository(session)
+            existing_state = repository.get_provider_connection(provider)
             existing_secret = repository.get_provider_secret(provider)
+            previous_username = (
+                existing_state.imap.username.strip().lower()
+                if existing_state.imap is not None
+                else ""
+            )
+            next_username = payload.imap.username.strip().lower()
+            switching_mailbox = bool(
+                previous_username
+                and next_username
+                and previous_username != next_username
+            )
             next_password = str(existing_secret.get("password") or "")
             if payload.imap.clear_password:
                 next_password = ""
             elif payload.imap.password.strip():
                 next_password = payload.imap.password.strip()
+            elif switching_mailbox:
+                raise HTTPException(
+                    status_code=400,
+                    detail=(
+                        "You changed the IMAP mailbox username. Enter the new Yahoo or IMAP "
+                        "app password as well so InboxAnchor does not reuse the previous "
+                        "account's secret."
+                    ),
+                )
+            if payload.sync_enabled:
+                _validate_imap_connection(
+                    provider,
+                    state=imap_state,
+                    password=next_password,
+                )
             if next_password:
                 repository.save_provider_secret(provider, {"password": next_password})
             else:
                 repository.clear_provider_secret(provider)
+            disconnecting_mailbox = payload.imap.clear_password and not payload.sync_enabled
+            if switching_mailbox or disconnecting_mailbox:
+                repository.reset_provider_runtime_state(provider)
     saved = service.save_provider_connection(state)
     mark_frontend_provider_dirty(provider)
     return saved.model_dump(mode="json")

inboxanchor/api/main.py

@@ -222,11 +222,24 @@ def _mailbox_progress_target(
 
 
 def _use_industrial_unread_sync(provider: object, *, time_range: Optional[str] = None) -> bool:
-    if normalize_time_range(time_range) != ALL_TIME_RANGE:
-        return False
+    del time_range
     return callable(getattr(provider, "iter_all_unread_batches", None))
 
 
+def _unread_scan_batch_size(
+    provider: object | None,
+    *,
+    default_batch_size: int,
+    industrial_unread_mode: bool,
+) -> int:
+    if not industrial_unread_mode:
+        return min(default_batch_size, 250)
+    provider_name = str(getattr(provider, "provider_name", "") or "").lower()
+    if provider_name in {"imap", "yahoo", "outlook"}:
+        return min(default_batch_size, 250)
+    return min(default_batch_size, 100)
+
+
 def _extract_bearer_token(authorization: Optional[str]) -> Optional[str]:
     if not authorization:
         return None
@@ -445,6 +458,16 @@ def _provider_runtime_error_message(provider_name: str, exc: Exception) -> str:
     if provider_name == "gmail":
         return f"Gmail connected, but InboxAnchor could not fetch unread mail: {message}"
     if provider_name in {"imap", "yahoo", "outlook"}:
+        if "imap login failed" in lowered or "authenticationfailed" in lowered:
+            if provider_name == "yahoo":
+                return (
+                    "Yahoo rejected the IMAP login. Reconnect Yahoo with an app password from "
+                    "Yahoo Account Security; the normal Yahoo password usually will not work here."
+                )
+            return (
+                f"{provider_name.upper()} rejected the IMAP login. Reconnect the mailbox with "
+                "the correct username and IMAP or app password."
+            )
         return (
             f"{provider_name.upper()} connected, but InboxAnchor could not fetch unread "
             f"mail: {message}"
@@ -1356,9 +1379,10 @@ def _sync_unread_working_set(
         time_range=normalized_time_range,
     )
     scan_limit = None if industrial_unread_mode else (limit_override or settings.default_scan_limit)
-    scan_batch_size = min(
-        batch_size_override or settings.default_batch_size,
-        100 if industrial_unread_mode else 250,
+    scan_batch_size = _unread_scan_batch_size(
+        provider,
+        default_batch_size=batch_size_override or settings.default_batch_size,
+        industrial_unread_mode=industrial_unread_mode,
     )
     wait_job: Optional[FrontendRunJob] = None
     job = pre_registered_job
@@ -2646,6 +2670,10 @@ def _build_ops_overview(
         connection = load_provider_connection(provider_name)
     else:
         connection = ProviderConnectionState(provider=provider_name)
+    industrial_unread_mode = _use_industrial_unread_sync(
+        getattr(service, "provider", None),
+        time_range=normalized_time_range,
+    )
     cache_stats = _load_mailbox_cache_stats(provider_name, time_range=normalized_time_range)
     sync_state = _load_mailbox_sync_state(provider_name, time_range=normalized_time_range)
     processed_total = max(
@@ -2743,7 +2771,7 @@ def _build_ops_overview(
                         f"Scans the full unread working set in "
                         f"{time_range_label(normalized_time_range).lower()}."
                     )
-                    if normalized_time_range == ALL_TIME_RANGE and provider_name == "gmail"
+                    if industrial_unread_mode
                     else (
                         f"Scans up to {settings.default_scan_limit} unread emails in "
                         f"{time_range_label(normalized_time_range).lower()}."

inboxanchor/api/v1/routers/frontend.py

@@ -425,8 +425,13 @@ def build_provider(provider_name: Optional[str] = None, *, owner_email: Optional
                     provider_name,
                     error,
                 )
+        preview_seed_messages = (
+            []
+            if state.status in {"configured", "connected"} or state.sync_enabled
+            else demo_emails
+        )
         return IMAPEmailClient(
-            seed_messages=demo_emails,
+            seed_messages=preview_seed_messages,
             provider_name=provider_name,
         )
     return FakeEmailProvider(demo_emails, provider_name=provider_name)

inboxanchor/bootstrap.py

@@ -61,6 +61,21 @@ def iter_unread_batches(
         for start in range(0, len(emails), batch_size):
             yield emails[start : start + batch_size]
 
+    def iter_all_unread_batches(
+        self,
+        *,
+        batch_size: int = 100,
+        include_body: bool = True,
+        time_range: Optional[str] = None,
+    ):
+        emails = self.list_unread(
+            limit=0,
+            include_body=include_body,
+            time_range=time_range,
+        )
+        for start in range(0, len(emails), batch_size):
+            yield emails[start : start + batch_size]
+
     def iter_mailbox_batches(
         self,
         *,