Skip to content

Commit a2a9d8c

Browse files
committed
feat(user-sync): store items, redirects, orgs and verify
1 parent e0c54db commit a2a9d8c

6 files changed

Lines changed: 436 additions & 16 deletions

File tree

‎src/osw/tools/user_sync/build.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
"""Build pydantic User/Organization entities from proposals.
2+
3+
Must run within an open OSW connection (the models resolve linked entities on
4+
construction). Organization references are passed as a list of page-title IRIs,
5+
which oold stores in the entity's ``__iris__`` map; the reference attribute is
6+
never read back here, as that would trigger a network resolve.
7+
"""
8+
9+
from __future__ import annotations
10+
11+
from typing import Any, List, Set
12+
13+
from opensemantic.base.v1 import Organization, User
14+
15+
from .mapping import ProposedOrganization, ProposedUser
16+
17+
# The concrete DisplayName type used by the label field.
18+
_DISPLAY_NAME = User.__fields__["label"].type_
19+
20+
21+
def _labels(text: str) -> List[Any]:
22+
return [_DISPLAY_NAME(text=text)]
23+
24+
25+
def build_organization(proposed: ProposedOrganization) -> Organization:
26+
return Organization(uuid=proposed.uuid, label=_labels(proposed.name))
27+
28+
29+
def build_user(proposed: ProposedUser) -> User:
30+
"""Build a full User entity for creation."""
31+
args: dict = {
32+
"uuid": proposed.uuid,
33+
"username": proposed.username,
34+
"first_name": proposed.first_name,
35+
"surname": proposed.surname,
36+
"label": _labels(proposed.label),
37+
}
38+
if proposed.orcid:
39+
args["orcid"] = proposed.orcid
40+
if proposed.emails:
41+
args["email"] = set(proposed.emails)
42+
if proposed.websites:
43+
args["website"] = set(proposed.websites)
44+
if proposed.organizations:
45+
args["organization"] = list(proposed.organizations)
46+
return User(**args)
47+
48+
49+
def apply_update(entity: Any, proposed: ProposedUser, apply_fields: Set[str]) -> Any:
50+
"""Apply the accepted fields onto a loaded existing entity, in place."""
51+
if "label" in apply_fields:
52+
entity.label = _labels(proposed.label)
53+
if "first_name" in apply_fields:
54+
entity.first_name = proposed.first_name
55+
if "surname" in apply_fields:
56+
entity.surname = proposed.surname
57+
if "orcid" in apply_fields:
58+
entity.orcid = proposed.orcid
59+
if "emails" in apply_fields:
60+
entity.email = set(proposed.emails)
61+
if "websites" in apply_fields:
62+
entity.website = set(proposed.websites)
63+
if "organizations" in apply_fields:
64+
entity.__iris__["organization"] = list(proposed.organizations)
65+
return entity
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
"""Create ``User:<username>`` redirect pages pointing at the user item.
2+
3+
An existing user page with real content is never overwritten; only missing
4+
pages or pages that are already redirects are (re)written.
5+
"""
6+
7+
from __future__ import annotations
8+
9+
from typing import Any, Optional
10+
11+
from osw.wtsite import WtPage
12+
13+
14+
def redirect_wikitext(item_title: str) -> str:
15+
return f"#REDIRECT [[{item_title}]]"
16+
17+
18+
def is_redirect(content: str) -> bool:
19+
return content.lstrip().upper().startswith("#REDIRECT")
20+
21+
22+
def ensure_redirect(osw: Any, username: str, item_title: str) -> Optional[str]:
23+
"""Create or refresh ``User:<username>`` -> item redirect.
24+
25+
Returns the page title if written, or None if skipped (real page exists).
26+
"""
27+
title = f"User:{username}"
28+
page = WtPage(wtSite=osw.site, title=title)
29+
if getattr(page, "exists", False):
30+
content = page.get_content() or ""
31+
if content.strip() and not is_redirect(content):
32+
return None
33+
page.set_content(redirect_wikitext(item_title))
34+
page.edit(comment="user-sync: redirect to user item")
35+
return title

‎src/osw/tools/user_sync/sync.py‎

Lines changed: 158 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,30 @@
11
"""Orchestrator for the user-item sync tool.
22
3-
Later phases implement enumeration, ORCID enrichment, reconciliation, the
4-
interactive layer and the store step. This module currently defines the report
5-
type and the public entry point.
3+
Ties the sources, mapping, reconciliation, interactive and store steps into one
4+
run: enumerate MediaWiki users, enrich ORCID users, reconcile against existing
5+
items, preview and confirm, then store items, redirects and organizations and
6+
verify.
67
"""
78

89
from __future__ import annotations
910

1011
from dataclasses import dataclass, field
11-
from typing import Dict, List, Optional
12+
from typing import Any, Dict, List, Optional
1213

1314
from osw.core import OSW
1415

16+
from .build import apply_update, build_organization, build_user
1517
from .config import SyncConfig
18+
from .existing import load_existing_users
19+
from .interactive import Prompter, Resolution, resolve_plan
20+
from .mapping import ProposedOrganization, map_user
21+
from .reconcile import reconcile
22+
from .redirects import ensure_redirect
23+
from .sources import (
24+
OrcidRateLimitError,
25+
enumerate_mw_users,
26+
fetch_orcid_record,
27+
)
1628

1729

1830
@dataclass
@@ -25,22 +37,154 @@ class SyncReport:
2537
failed: Dict[str, str] = field(default_factory=dict)
2638
redirects_created: List[str] = field(default_factory=list)
2739
organizations: List[str] = field(default_factory=list)
40+
accepted: bool = False
2841

2942
def summary(self) -> str:
30-
"""One-line human-readable summary of the run."""
43+
state = "applied" if self.accepted else "no changes written"
3144
return (
32-
f"created={len(self.created)} updated={len(self.updated)} "
45+
f"{state}: created={len(self.created)} updated={len(self.updated)} "
3346
f"skipped={len(self.skipped)} failed={len(self.failed)} "
3447
f"redirects={len(self.redirects_created)} orgs={len(self.organizations)}"
3548
)
3649

3750

38-
def run_user_sync(config: SyncConfig, osw: Optional[OSW] = None) -> SyncReport:
39-
"""Create or update OSW User items from MediaWiki accounts and ORCID data.
51+
def _build_proposals(config: SyncConfig, osw: Any, session: Any):
52+
"""Enumerate MediaWiki users, enrich ORCID users and map to proposals."""
53+
excluded = tuple(config.excluded_groups) if config.exclude_bot_group else ()
54+
mw_users = enumerate_mw_users(
55+
osw.mw_site,
56+
excluded_groups=excluded,
57+
include_non_orcid=config.include_non_orcid,
58+
limit=config.limit,
59+
)
60+
cache: Dict[str, Any] = {}
61+
proposed_users = []
62+
org_map: Dict[Any, ProposedOrganization] = {}
63+
for mw_user in mw_users:
64+
profile = None
65+
if mw_user.is_orcid:
66+
try:
67+
profile = fetch_orcid_record(
68+
mw_user.orcid_id,
69+
session=session,
70+
base=config.orcid_api_base,
71+
cache=cache,
72+
)
73+
except OrcidRateLimitError:
74+
profile = None
75+
proposed, orgs = map_user(
76+
mw_user, profile, link_organizations=config.link_organizations
77+
)
78+
proposed_users.append(proposed)
79+
for org in orgs:
80+
org_map.setdefault(org.uuid, org)
81+
return proposed_users, org_map
82+
83+
84+
def _store_organizations(osw: Any, org_map, report: SyncReport) -> None:
85+
if not org_map:
86+
return
87+
entities = [build_organization(org) for org in org_map.values()]
88+
try:
89+
osw.store_entity(
90+
OSW.StoreEntityParam(entities=entities, edit_comment="user-sync: orgs")
91+
)
92+
report.organizations = [org.full_page_title for org in org_map.values()]
93+
except Exception as exc: # pragma: no cover - network failure path
94+
report.failed["organizations"] = str(exc)
95+
96+
97+
def _store_users(osw: Any, resolution: Resolution, report: SyncReport) -> None:
98+
entities = []
99+
for resolved in resolution.resolved:
100+
proposed = resolved.change.proposed
101+
if resolved.action == "create":
102+
entities.append(build_user(proposed))
103+
report.created.append(proposed.full_page_title)
104+
elif resolved.action == "update":
105+
entities.append(
106+
apply_update(resolved.change.existing, proposed, resolved.apply_fields)
107+
)
108+
report.updated.append(proposed.full_page_title)
109+
else:
110+
report.skipped.append(proposed.full_page_title)
111+
if entities:
112+
osw.store_entity(
113+
OSW.StoreEntityParam(
114+
entities=entities, overwrite=True, edit_comment="user-sync"
115+
)
116+
)
117+
118+
119+
def _create_redirects(osw: Any, resolution: Resolution, report: SyncReport) -> None:
120+
for resolved in resolution.resolved:
121+
if resolved.action not in ("create", "update"):
122+
continue
123+
proposed = resolved.change.proposed
124+
try:
125+
written = ensure_redirect(osw, proposed.username, proposed.full_page_title)
126+
if written:
127+
report.redirects_created.append(written)
128+
except Exception as exc: # pragma: no cover - network failure path
129+
report.failed[f"redirect:{proposed.username}"] = str(exc)
130+
131+
132+
def _verify(osw: Any, report: SyncReport) -> None:
133+
titles = report.created + report.updated
134+
if not titles:
135+
return
136+
try:
137+
loaded = osw.load_entity(titles)
138+
loaded_list = loaded if isinstance(loaded, list) else [loaded]
139+
found = {getattr(e, "username", None) for e in loaded_list}
140+
missing = [
141+
t
142+
for t, e in zip(titles, loaded_list)
143+
if getattr(e, "username", None) is None
144+
]
145+
if missing or None in found:
146+
report.failed["verify"] = f"missing username on {missing}"
147+
except Exception as exc: # pragma: no cover - network failure path
148+
report.failed["verify"] = str(exc)
149+
150+
151+
def run_user_sync(
152+
config: SyncConfig,
153+
osw: Optional[OSW] = None,
154+
prompter: Optional[Prompter] = None,
155+
session: Any = None,
156+
) -> SyncReport:
157+
"""Create or update OSW User items from MediaWiki accounts and ORCID data."""
158+
if osw is None:
159+
raise ValueError("run_user_sync requires an authenticated OSW connection")
160+
prompter = prompter or Prompter()
161+
162+
proposed_users, org_map = _build_proposals(config, osw, session)
163+
existing = load_existing_users(osw)
164+
plan = reconcile(proposed_users, existing)
165+
166+
summary_lines = []
167+
if config.link_organizations and org_map:
168+
summary_lines.append(f"Organizations to ensure: {len(org_map)}")
169+
if config.create_redirects:
170+
summary_lines.append("Redirects: User:<username> -> item for created/updated")
171+
172+
resolution = resolve_plan(
173+
plan,
174+
prompter,
175+
dry_run=config.dry_run,
176+
assume_yes=config.assume_yes,
177+
summary_lines=summary_lines,
178+
)
179+
180+
report = SyncReport(accepted=resolution.accepted)
181+
if not resolution.accepted:
182+
return report
40183

41-
Args:
42-
config: Runtime options for the run.
43-
osw: An authenticated OSW/OswExpress connection. Required for any wiki
44-
access; the example wrapper builds one from ``config``.
45-
"""
46-
raise NotImplementedError("Implemented incrementally in phases 1 to 6.")
184+
if config.link_organizations:
185+
_store_organizations(osw, org_map, report)
186+
_store_users(osw, resolution, report)
187+
if config.create_redirects:
188+
_create_redirects(osw, resolution, report)
189+
_verify(osw, report)
190+
return report

‎tests/tools/test_user_sync_config.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,8 @@ def test_report_summary():
4141
assert "updated=1" in report.summary()
4242

4343

44-
def test_run_user_sync_is_stub():
44+
def test_run_user_sync_requires_connection():
4545
import pytest
4646

47-
with pytest.raises(NotImplementedError):
47+
with pytest.raises(ValueError, match="connection"):
4848
run_user_sync(SyncConfig())
Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
"""Unit tests for redirect page handling."""
2+
3+
from osw.tools.user_sync import redirects
4+
from osw.tools.user_sync.redirects import (
5+
ensure_redirect,
6+
is_redirect,
7+
redirect_wikitext,
8+
)
9+
10+
11+
def test_redirect_wikitext():
12+
assert redirect_wikitext("Item:OSW123") == "#REDIRECT [[Item:OSW123]]"
13+
14+
15+
def test_is_redirect():
16+
assert is_redirect("#REDIRECT [[Item:OSW1]]")
17+
assert is_redirect(" #redirect [[x]]")
18+
assert not is_redirect("Some real user page content")
19+
20+
21+
class FakeWtPage:
22+
exists_map: dict = {}
23+
content_map: dict = {}
24+
25+
def __init__(self, wtSite=None, title=None, do_init=True):
26+
self.title = title
27+
self.exists = self.exists_map.get(title, False)
28+
self._content = self.content_map.get(title, "")
29+
self.written = None
30+
self.edited = False
31+
32+
def get_content(self):
33+
return self._content
34+
35+
def set_content(self, content):
36+
self.written = content
37+
38+
def edit(self, comment=None):
39+
self.edited = True
40+
41+
42+
class FakeOsw:
43+
site = object()
44+
45+
46+
def _patch(monkeypatch, exists_map, content_map):
47+
FakeWtPage.exists_map = exists_map
48+
FakeWtPage.content_map = content_map
49+
monkeypatch.setattr(redirects, "WtPage", FakeWtPage)
50+
51+
52+
def test_ensure_redirect_creates_missing(monkeypatch):
53+
_patch(monkeypatch, {}, {})
54+
written = ensure_redirect(FakeOsw(), "0000-0002-6374-9831", "Item:OSW1")
55+
assert written == "User:0000-0002-6374-9831"
56+
57+
58+
def test_ensure_redirect_overwrites_existing_redirect(monkeypatch):
59+
title = "User:alice"
60+
_patch(monkeypatch, {title: True}, {title: "#REDIRECT [[Item:OSWold]]"})
61+
written = ensure_redirect(FakeOsw(), "alice", "Item:OSWnew")
62+
assert written == title
63+
64+
65+
def test_ensure_redirect_skips_real_page(monkeypatch):
66+
title = "User:bob"
67+
_patch(monkeypatch, {title: True}, {title: "This is my real user page."})
68+
written = ensure_redirect(FakeOsw(), "bob", "Item:OSW1")
69+
assert written is None

0 commit comments

Comments
 (0)