Sort the responsive derivation server-side, oldest first

`$orderby` naming an aggregate alias is accepted by this feed - `$orderby=last_at`
against a `last_at` the aggregate produced returns correctly ordered rows,
verified against the live service. odata.py asserted the opposite, and that claim
is why the sort was done client-side.

Sorting here meant reading everything to find the head of a queue: 130408
responsive http hosts pulled to pick the 1101 refresh would take next, against a
100000-row page ceiling this feed cannot be paged past without silently
duplicating and dropping rows. So http and https could not be derived at all.

Ordering in the query turns $top from a completeness problem into a deliberate
truncation of the newest end - refresh consumes ~26000 hosts a day oldest-first,
so the ones it needs next are nowhere near the cut, and the list is rebuilt daily
regardless. The discovery blocklist loses the most recently seen ~23% of its
entries, which is worth about one re-discovered host an hour out of ~1100 found.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: joocer

src/ichnos/odata.py

@@ -208,6 +208,7 @@ def grouped_max(
     base_url: str = DEFAULT_ODATA_BASE,
     prefix: str = DEFAULT_ODATA_PREFIX,
     top: int = MAX_TOP,
+    order_by: Optional[str] = None,
     get: Optional[Callable[..., Any]] = None,
 ) -> List[Dict[str, Any]]:
     """One row per distinct `column`, carrying `max(max_column)` as `alias`.
@@ -222,14 +223,21 @@ def grouped_max(
     ever succeed" and "when did we last try it") instead of issuing a second query and
     joining the results here. Grouping is what the feed is for.
 
-    Note the feed will not `$orderby` an aggregate alias, so callers sort the result
-    themselves; at the sizes this returns that is trivial next to the transfer saved by
-    aggregating server-side.
+    `order_by` sorts server-side, and it may name the aggregate alias - `$orderby=last_at`
+    against a `last_at` produced by the aggregate is accepted and correct, verified
+    against the live feed. This module used to assert the opposite and sort client-side,
+    which is what forced whole-result reads: to find the thousand oldest hosts it pulled
+    all 130408 of them, and so needed a page size it did not have. Ordering here instead
+    turns `$top` from a completeness problem into a deliberate truncation of the end of
+    the queue nobody reaches.
     """
     columns = [column] if isinstance(column, str) else list(column)
     inner = f"groupby(({','.join(columns)}),aggregate({max_column} with max as {alias}))"
     apply_expr = f"filter({where})/{inner}" if where else inner
-    query = f"$apply={quote(apply_expr, safe='()/,')}&$top={top}"
+    query = f"$apply={quote(apply_expr, safe='()/,')}"
+    if order_by:
+        query += f"&$orderby={quote(order_by, safe='')}"
+    query += f"&$top={top}"
     return [
         row
         for row in iter_rows(path, query, token=token, base_url=base_url, prefix=prefix,

src/ichnos/responsive.py

@@ -137,7 +137,14 @@ def fetch_responsive_hosts(
     dataset = f"{workspace}/{collection}/{OBSERVATIONS_DATASET}"
 
     def newest_per_ip(status_clause: str) -> dict:
-        kwargs = {"where": f"{scoped} and {status_clause}", "token": token, "get": get}
+        # Oldest first, server-side, so a result too large for one page is truncated at
+        # the *newest* end - the end refresh would not have reached before tomorrow's
+        # derivation replaces the list anyway. Sorting here instead of client-side is
+        # what makes a single page sufficient: http has 130408 responsive hosts against
+        # a 100000-row ceiling, but refresh consumes ~26000 a day, so the thousand it
+        # needs next are never near the truncation.
+        kwargs = {"where": f"{scoped} and {status_clause}", "token": token, "get": get,
+                  "order_by": "last_at"}
         if base_url:
             kwargs["base_url"] = base_url
         rows = grouped_max(dataset, "ip", "observed_at", "last_at", **kwargs)

tests/test_responsive.py

@@ -314,3 +314,42 @@ def test_a_single_page_aggregate_read_is_returned_normally():
     )
     assert rows == [{"ip": "203.0.113.1", "response_status": "success",
                      "last_at": "2026-08-01T00:00:00Z"}]
+
+
+def test_the_derivation_asks_the_feed_to_sort_oldest_first():
+    """Sorting server-side is what makes a single page enough.
+
+    Sorting client-side meant the whole set had to be read to find the head of the
+    queue: 130408 responsive http hosts against a 100000-row page ceiling that cannot
+    be paged past without silently corrupting the result. Ordering in the query turns
+    `$top` from a completeness problem into a truncation of the *newest* end - and
+    refresh consumes ~26000 hosts a day oldest-first, so the ones it needs next are
+    nowhere near the cut, and tomorrow's derivation replaces the list regardless.
+
+    `$orderby` naming the aggregate alias is accepted by this feed. The module asserted
+    the opposite for a long time, which is how the client-side sort got there."""
+    calls = []
+    fetch_responsive_hosts(
+        "http", workspace="ichnos", collection="landing", token="t",
+        now=datetime(2026, 8, 4, tzinfo=timezone.utc),
+        get=_fake_derivation([], [], calls),
+    )
+    for url in calls:
+        assert "$orderby=last_at" in url
+        assert url.index("$orderby") < url.index("$top"), "orderby must precede top"
+
+
+def test_truncation_lands_on_the_newest_hosts_not_the_oldest():
+    """The property that makes the truncation safe. Whatever the feed returns, the
+    hosts refresh takes next are the least recently attempted - so a short read costs
+    coverage at the back of the queue, never at the front."""
+    hosts = fetch_responsive_hosts(
+        "http", workspace="ichnos", collection="landing", token="t",
+        now=datetime(2026, 8, 5, tzinfo=timezone.utc),
+        get=_fake_derivation([
+            _row("oldest", "2026-07-30T00:00:00Z"),
+            _row("middle", "2026-08-02T00:00:00Z"),
+            _row("newest", "2026-08-04T00:00:00Z"),
+        ]),
+    )
+    assert [ip for ip, _ in hosts] == ["oldest", "middle", "newest"]