Merge pull request #56 from NirrWorks/fix/hashtag-case-variants

Author: kshitijrajsharma

osmsg/catalog.py

@@ -198,7 +198,7 @@ def recent_user_hashtags(attach, uids: list[int], *, prefixes, frontier, start=N
     uid_list = ", ".join(str(int(u)) for u in uids)
     ranges = " OR ".join(f"(lower(h) >= {_pg_str(lo)} AND lower(h) < {_pg_str(hi)})" for lo, hi in prefixes)
     inner = (
-        f"SELECT DISTINCT s.uid AS uid, h AS hashtag "
+        f"SELECT DISTINCT s.uid AS uid, lower(h) AS hashtag "
         f"FROM changeset_stats s JOIN changesets c USING (changeset_id), unnest(c.hashtags) AS h "
         f"WHERE s.uid IN ({uid_list}) AND {_pg_user_window(frontier, start, end)} AND ({ranges})"
     )
@@ -211,7 +211,7 @@ def recent_user_cooccur_hashtags(attach, uids: list[int], *, prefixes, frontier,
     uid_list = ", ".join(str(int(u)) for u in uids)
     ranges = " OR ".join(f"(lower(x) >= {_pg_str(lo)} AND lower(x) < {_pg_str(hi)})" for lo, hi in prefixes)
     inner = (
-        f"SELECT DISTINCT s.uid AS uid, h AS hashtag "
+        f"SELECT DISTINCT s.uid AS uid, lower(h) AS hashtag "
         f"FROM changeset_stats s JOIN changesets c USING (changeset_id), unnest(c.hashtags) AS h "
         f"WHERE s.uid IN ({uid_list}) AND {_pg_user_window(frontier, start, end)} "
         f"AND EXISTS (SELECT 1 FROM unnest(c.hashtags) AS x WHERE {ranges})"

osmsg/query.py

@@ -338,7 +338,8 @@ def _attach_user_hashtags(
     else:
         recent_pred = " OR ".join("(lower(x) >= ? AND lower(x) < ?)" for _ in prefixes)
         recent = (
-            f"SELECT uid, h AS hashtag FROM (SELECT uid, UNNEST(hashtags) AS h FROM {s.recent_changesets_rel} "
+            f"SELECT uid, lower(h) AS hashtag FROM (SELECT uid, UNNEST(hashtags) AS h "
+            f"FROM {s.recent_changesets_rel} "
             f"WHERE created_at >= ?{window_sql} AND uid IN ({ph}) "
             f"AND EXISTS (SELECT 1 FROM UNNEST(hashtags) AS t(x) WHERE {recent_pred}))"
         )
@@ -592,7 +593,7 @@ def hashtags(
     else:
         recent_match = " OR ".join("(lower(x) >= ? AND lower(x) < ?)" for _ in prefixes)
         parts.append(
-            f"SELECT h AS hashtag, uid, mc FROM (SELECT c.uid, UNNEST(c.hashtags) AS h, cs.mc AS mc "
+            f"SELECT lower(h) AS hashtag, uid, mc FROM (SELECT c.uid, UNNEST(c.hashtags) AS h, cs.mc AS mc "
             f"FROM {s.recent_changesets_rel} c JOIN (SELECT changeset_id, {map_changes_sum(alias='mc')} "
             f"FROM {s.recent_stats_rel} GROUP BY changeset_id) cs USING (changeset_id) "
             f"WHERE c.created_at >= ?{window_sql} AND EXISTS "

tests/test_query.py

@@ -127,6 +127,31 @@ def test_cooccurring_hashtags(con, sources):
     ]
 
 
+def _tag_written_both_ways(con) -> None:
+    """The same hashtag as the two sides really store it: the rollup lowercases, while the base
+    `changesets` table keeps the case the mapper typed."""
+    con.execute("INSERT INTO history SELECT '#youthmappers', * EXCLUDE (hashtag) FROM history WHERE changeset_id = 1")
+    con.execute("UPDATE csets SET hashtags = list_append(hashtags, '#YouthMappers') WHERE changeset_id = 3")
+
+
+def test_cooccurring_hashtags_merge_case_variants(con, sources):
+    # Grouping is on the raw string, so a tag written both ways used to come back as two rows, each
+    # holding only part of the contributors. Both sides lowercase before grouping.
+    _tag_written_both_ways(con)
+    tags = [r["hashtag"] for r in query.hashtags(con, "hotosm", sources)]
+    assert "#YouthMappers" not in tags
+    assert tags.count("#youthmappers") == 1
+
+
+def test_leaderboard_cooccurring_hashtags_merge_case_variants(con, sources):
+    # Same split on the per-user chips: alice carries the tag from history and from the recent tail.
+    _tag_written_both_ways(con)
+    items = query.leaderboard(con, "hotosm", sources)["items"]
+    alice = next(r for r in items if r["name"] == "alice")
+    assert "#YouthMappers" not in alice["hashtags"]
+    assert alice["hashtags"].count("#youthmappers") == 1
+
+
 def test_tags_breakdown(con, sources):
     tg = query.tags(con, "hotosm", sources)
     building = next(r for r in tg if r["tag_key"] == "building")