feat(user-sync): opt-in extras, standard email, --auto-apply

Author: raederan

docs/tools/user-sync.md

@@ -22,31 +22,49 @@ it to write after confirming at the prompt.
 
 ### Options
 
+By default the tool syncs both ORCID and MediaWiki-native accounts and writes
+only core identity plus email.
+
 | Flag | Effect |
 | --- | --- |
 | `--domain` | Target OSL domain. |
 | `--cred-filepath` | Path to `accounts.pwd.yaml`. |
 | `--dry-run` | Preview only; never writes. |
-| `--yes` | Non-interactive: create and gap-fill, keep existing on conflicts. |
+| `--auto-apply` | Non-interactive: apply creates, gap-fills and removals; keep existing on conflicts. |
 | `--limit N` | Process at most N accounts (testing). |
-| `--orcid-only` | Only accounts whose username is an ORCID iD. |
+| `--orcid-only` | Only ORCID-username accounts (excludes `--mw-only`). |
+| `--mw-only` | Only non-ORCID (MediaWiki-native) accounts (excludes `--orcid-only`). |
 | `--include-system` | Do not skip MediaWiki reserved system accounts. |
 | `--no-redirects` | Do not create `User:` redirect pages. |
-| `--no-organizations` | Do not link ORCID affiliations to Organization items. |
+| `--with-websites` | Also store ORCID researcher URLs (opt-in). |
+| `--with-organizations` | Also create and link Organization items from ORCID affiliations (opt-in). |
+| `--with-extras` | Enable both websites and organizations. |
 
 ## What it does
 
 1. Enumerate MediaWiki accounts, skipping bots and privileged groups
    (`bot`, `sysop`, `bureaucrat`, `interface-admin`) and MediaWiki reserved
    system usernames.
-2. Enrich ORCID users from `https://pub.orcid.org/v3.0/<id>` (names, public
-   email, websites, employment or affiliation).
+2. Enrich ORCID users from `https://pub.orcid.org/v3.0/<id>`: names and email
+   always; researcher URLs and affiliations only when their flag is enabled.
 3. Map each account to a proposed `User` item with a deterministic id
    (uuid5 on the ORCID iD, else the username), so re-runs are idempotent.
-4. Reconcile against existing items by `username` into NEW, GAP_FILL, CONFLICT
-   or UNCHANGED, then preview and resolve conflicts interactively.
-5. Store items, create `User:<username>` redirects to each item, ensure linked
-   Organization items, and verify by reloading.
+4. Reconcile against existing items by `username` into NEW, GAP_FILL, CONFLICT,
+   REMOVE or UNCHANGED, then preview; gap-fills and removals apply
+   automatically, conflicts are resolved interactively.
+5. Store items, create `User:<username>` redirects to each item, ensure any
+   linked Organization items, verify by reloading, and warn about users with no
+   email.
+
+## Data written
+
+| Field | Policy |
+| --- | --- |
+| `username`, `first_name`, `surname`, `label`, `orcid` | Always (core identity). |
+| `email` | Standard: always attempted; missing email is warned, not fatal; never removed. |
+| `website` | Opt-in (`--with-websites`); removed from existing items when disabled. |
+| `organization` | Opt-in (`--with-organizations`); removed from existing items when disabled. |
+| `employment_contract_status` | Never written; removed from existing items (data protection). |
 
 ## Notes
 

src/osw/tools/user_sync/config.py

@@ -4,7 +4,7 @@
 
 import argparse
 from dataclasses import dataclass, field
-from typing import List, Optional
+from typing import List, Optional, Set
 
 # Category page titles of the target item types (see opensemantic.base.v1).
 USER_CATEGORY = "Category:OSWd9aa0bca9b0040d8af6f5c091bf9eec7"
@@ -17,6 +17,10 @@
 # All are standard MediaWiki groups, so this is not a per-instance skip-list.
 SYSTEM_GROUPS = ["bot", "sysop", "bureaucrat", "interface-admin"]
 
+# User fields the script must never write and must remove from existing items
+# (data protection).
+PROTECTED_FIELDS = ("employment_contract_status",)
+
 
 @dataclass
 class SyncConfig:
@@ -25,16 +29,30 @@ class SyncConfig:
     domain: Optional[str] = None
     cred_filepath: Optional[str] = None
     dry_run: bool = False
-    assume_yes: bool = False
+    auto_apply: bool = False
     limit: Optional[int] = None
+    include_orcid: bool = True
     include_non_orcid: bool = True
     exclude_bot_group: bool = True
     exclude_system_usernames: bool = True
     create_redirects: bool = True
-    link_organizations: bool = True
+    # Optional enrichment beyond core identity, opt-in (off by default).
+    # Email is not optional: it is always attempted, and missing emails are
+    # reported as warnings rather than failing the run.
+    include_websites: bool = False
+    link_organizations: bool = False
     orcid_api_base: str = ORCID_API_BASE_DEFAULT
     excluded_groups: List[str] = field(default_factory=lambda: list(SYSTEM_GROUPS))
 
+    def enabled_optional_fields(self) -> Set[str]:
+        """Reconcile field names for the enabled optional data."""
+        enabled: Set[str] = set()
+        if self.include_websites:
+            enabled.add("websites")
+        if self.link_organizations:
+            enabled.add("organizations")
+        return enabled
+
 
 def build_arg_parser() -> argparse.ArgumentParser:
     """Build the argument parser used by the example wrapper."""
@@ -53,10 +71,10 @@ def build_arg_parser() -> argparse.ArgumentParser:
         help="Show the preview only; do not write anything.",
     )
     parser.add_argument(
-        "--yes",
-        dest="assume_yes",
+        "--auto-apply",
+        dest="auto_apply",
         action="store_true",
-        help="Non-interactive: apply new items and gap-fills, keep existing on conflicts.",
+        help="Non-interactive: apply creates, gap-fills and removals; keep existing on conflicts.",
     )
     parser.add_argument(
         "--limit",
@@ -70,24 +88,36 @@ def build_arg_parser() -> argparse.ArgumentParser:
         help="Do not create User: redirect pages.",
     )
     parser.add_argument(
-        "--no-organizations",
-        dest="link_organizations",
-        action="store_false",
-        help="Do not resolve or link ORCID affiliations to Organization items.",
+        "--with-websites",
+        dest="include_websites",
+        action="store_true",
+        help="Also store ORCID researcher URLs on the user item (opt-in).",
     )
     parser.add_argument(
-        "--include-non-orcid",
-        dest="include_non_orcid",
+        "--with-organizations",
+        dest="link_organizations",
         action="store_true",
-        default=True,
-        help="Also sync non-bot accounts without an ORCID username (default).",
+        help="Also resolve ORCID affiliations to Organization items and link them.",
     )
     parser.add_argument(
+        "--with-extras",
+        dest="with_extras",
+        action="store_true",
+        help="Enable all optional data: websites and organizations.",
+    )
+    account_scope = parser.add_mutually_exclusive_group()
+    account_scope.add_argument(
         "--orcid-only",
         dest="include_non_orcid",
         action="store_false",
         help="Sync only accounts whose username is an ORCID iD.",
     )
+    account_scope.add_argument(
+        "--mw-only",
+        dest="include_orcid",
+        action="store_false",
+        help="Sync only non-ORCID (MediaWiki-native) accounts.",
+    )
     parser.add_argument(
         "--include-system",
         dest="exclude_system_usernames",
@@ -104,10 +134,12 @@ def config_from_args(argv: Optional[List[str]] = None) -> SyncConfig:
         domain=args.domain,
         cred_filepath=args.cred_filepath,
         dry_run=args.dry_run,
-        assume_yes=args.assume_yes,
+        auto_apply=args.auto_apply,
         limit=args.limit,
+        include_orcid=args.include_orcid,
         include_non_orcid=args.include_non_orcid,
         exclude_system_usernames=args.exclude_system_usernames,
         create_redirects=args.create_redirects,
-        link_organizations=args.link_organizations,
+        include_websites=args.include_websites or args.with_extras,
+        link_organizations=args.link_organizations or args.with_extras,
     )

src/osw/tools/user_sync/interactive.py

@@ -143,7 +143,7 @@ def resolve_plan(
     plan: ReconcilePlan,
     prompter: Prompter,
     dry_run: bool = False,
-    assume_yes: bool = False,
+    auto_apply: bool = False,
     summary_lines: Optional[Sequence[str]] = None,
 ) -> Resolution:
     """Preview the plan and resolve conflicts into a write decision."""
@@ -154,7 +154,7 @@ def resolve_plan(
 
     conflict_mode: Optional[str] = None
     if plan.conflicts and not dry_run:
-        if assume_yes:
+        if auto_apply:
             conflict_mode = "keep"
         else:
             choice = prompter.ask(
@@ -188,7 +188,7 @@ def resolve_plan(
 
     if dry_run:
         accepted_run = False
-    elif assume_yes:
+    elif auto_apply:
         accepted_run = True
     else:
         n_create = sum(1 for r in resolved if r.action == "create")

src/osw/tools/user_sync/sources.py

@@ -96,6 +96,7 @@ def enumerate_mw_users(
     site: Any,
     excluded_groups: Sequence[str] = ("bot",),
     excluded_usernames: Sequence[str] = (),
+    include_orcid: bool = True,
     include_non_orcid: bool = True,
     limit: Optional[int] = None,
     batch: Any = "max",
@@ -106,6 +107,7 @@ def enumerate_mw_users(
         site: An object exposing ``api("query", ...)`` (an mwclient Site).
         excluded_groups: Accounts in any of these groups are skipped (bots, admins).
         excluded_usernames: Exact usernames to skip (reserved system accounts).
+        include_orcid: If False, skip ORCID-username accounts.
         include_non_orcid: If False, keep only ORCID-username accounts.
         limit: Keep at most this many accounts after filtering.
         batch: ``aulimit`` value passed to the API.
@@ -121,6 +123,8 @@ def enumerate_mw_users(
             continue
         if not include_non_orcid and not user.is_orcid:
             continue
+        if not include_orcid and user.is_orcid:
+            continue
         users.append(user)
         if limit is not None and len(users) >= limit:
             break

src/osw/tools/user_sync/sync.py

@@ -57,6 +57,7 @@ def _build_proposals(config: SyncConfig, osw: Any, session: Any):
         osw.mw_site,
         excluded_groups=excluded,
         excluded_usernames=reserved,
+        include_orcid=config.include_orcid,
         include_non_orcid=config.include_non_orcid,
         limit=config.limit,
     )
@@ -184,7 +185,7 @@ def run_user_sync(
         plan,
         prompter,
         dry_run=config.dry_run,
-        assume_yes=config.assume_yes,
+        auto_apply=config.auto_apply,
         summary_lines=summary_lines,
     )
 

tests/tools/test_user_sync_config.py

@@ -14,7 +14,10 @@ def test_defaults():
     cfg = SyncConfig()
     assert cfg.include_non_orcid is True
     assert cfg.create_redirects is True
-    assert cfg.link_organizations is True
+    # Websites and organizations are opt-in (off by default); email is standard.
+    assert cfg.include_websites is False
+    assert cfg.link_organizations is False
+    assert cfg.enabled_optional_fields() == set()
     assert cfg.exclude_system_usernames is True
     assert "bot" in cfg.excluded_groups
     assert "sysop" in cfg.excluded_groups
@@ -27,6 +30,18 @@ def test_include_system_flag():
     assert cfg.exclude_system_usernames is False
 
 
+def test_optional_field_flags():
+    cfg = config_from_args(["--with-organizations", "--with-websites"])
+    assert cfg.link_organizations is True
+    assert cfg.include_websites is True
+    assert cfg.enabled_optional_fields() == {"organizations", "websites"}
+
+
+def test_with_extras_enables_all():
+    cfg = config_from_args(["--with-extras"])
+    assert cfg.enabled_optional_fields() == {"websites", "organizations"}
+
+
 def test_config_from_args_parses_flags():
     cfg = config_from_args([
         "--domain",
@@ -40,6 +55,16 @@ def test_config_from_args_parses_flags():
     assert cfg.dry_run is True
     assert cfg.limit == 5
     assert cfg.include_non_orcid is False
+    assert cfg.include_orcid is True
+
+
+def test_account_scope_defaults_and_mw_only():
+    cfg = SyncConfig()
+    assert cfg.include_orcid is True
+    assert cfg.include_non_orcid is True  # both are standard by default
+    mw_only = config_from_args(["--mw-only"])
+    assert mw_only.include_orcid is False
+    assert mw_only.include_non_orcid is True
 
 
 def test_report_summary():

tests/tools/test_user_sync_interactive.py

@@ -136,9 +136,9 @@ def test_resolve_dry_run_makes_no_writes_and_no_prompts():
     assert io.answers == []  # nothing was asked
 
 
-def test_resolve_assume_yes_keeps_conflicts():
+def test_resolve_auto_apply_keeps_conflicts():
     prompter, io = _prompter([])
-    res = resolve_plan(_mixed_plan(), prompter, assume_yes=True)
+    res = resolve_plan(_mixed_plan(), prompter, auto_apply=True)
     assert res.accepted is True
     updates = {r.change.proposed.username: r.apply_fields for r in res.updates()}
     assert updates == {"gap1": {"emails"}, "mix1": {"emails"}}

tests/tools/test_user_sync_sources.py

@@ -110,6 +110,17 @@ def test_enumerate_excludes_privileged_groups_and_reserved_names():
     assert [u.name for u in users] == ["0000-0002-6374-9831"]
 
 
+def test_enumerate_mw_only_skips_orcid():
+    site = FakeSite([
+        _page([
+            {"name": "0000-0002-6374-9831", "groups": ["user"]},
+            {"name": "Alice", "groups": ["user"]},
+        ])
+    ])
+    users = enumerate_mw_users(site, include_orcid=False)
+    assert [u.name for u in users] == ["Alice"]
+
+
 def test_partition_by_orcid():
     users = [
         MwUser(name="0000-0002-6374-9831"),