Skip to content

Commit f84a529

Browse files
committed
fix(core): verify that store_entity actually created the page
- add StoreEntityParam.verify_write, default True - query the edited titles after the upload, one API request per 50 titles - an absent page goes to StoreEntityResult.failed as PageNotCreatedError instead of being reported in StoreEntityResult.pages - the query bypasses the page cache, which would answer pre-write - correct the comment claiming created_pages only records successes Closes #175
1 parent 848615c commit f84a529

4 files changed

Lines changed: 235 additions & 1 deletion

File tree

‎src/osw/core.py‎

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1639,6 +1639,12 @@ class StoreEntityParam(OswBaseModel):
16391639
offline: Optional[bool] = False
16401640
"""If set to True, the processed entities are not upload but only returned as WtPages.
16411641
Can be used to create WtPage objects from entities without uploading them."""
1642+
verify_write: Optional[bool] = True
1643+
"""If set to True, the existence of every edited page is queried after the
1644+
upload. A page that does not exist afterwards is reported in
1645+
StoreEntityResult.failed instead of StoreEntityResult.pages. This costs one
1646+
additional API request per 50 edited pages. Has no effect if 'offline' is
1647+
True."""
16421648
_overwrite_per_class: Dict[str, Dict[str, OSW.OverwriteClassParam]] = (
16431649
PrivateAttr()
16441650
)
@@ -1717,6 +1723,50 @@ def __init__(self, result: OSW.StoreEntityResult):
17171723
f"entities: {failed_titles}"
17181724
)
17191725

1726+
class PageNotCreatedError(Exception):
1727+
"""Raised for a page that does not exist after store_entity() edited it.
1728+
1729+
The edit was sent and no exception was raised, but the page is absent when
1730+
the wiki is asked afterwards. A form or template driven creation step on
1731+
the category can reject the content server-side without reporting an error
1732+
to the API client.
1733+
"""
1734+
1735+
def __init__(self, title: str):
1736+
self.title = title
1737+
super().__init__(
1738+
f"Page '{title}' does not exist after the edit. The write was "
1739+
f"rejected by the wiki without an error response."
1740+
)
1741+
1742+
def _get_missing_page_titles(self, titles: List[str]) -> List[str]:
1743+
"""Returns those of the given page titles that do not exist on the wiki.
1744+
1745+
The query goes to the MediaWiki API directly and not through
1746+
WtSite.get_page(), because the page cache would answer with the state from
1747+
before the write.
1748+
1749+
Parameters
1750+
----------
1751+
titles:
1752+
Full page titles to check.
1753+
"""
1754+
missing = []
1755+
batch_size = 50 # maximum number of titles per API query
1756+
for start in range(0, len(titles), batch_size):
1757+
batch = titles[start : start + batch_size]
1758+
result = self.mw_site.api(
1759+
"query", titles="|".join(batch), prop="info", format="json"
1760+
)
1761+
query = result.get("query", {})
1762+
# the API normalizes titles, map them back to what was requested
1763+
normalized = {n["to"]: n["from"] for n in query.get("normalized", [])}
1764+
for page_info in query.get("pages", {}).values():
1765+
if "missing" in page_info:
1766+
title = page_info.get("title")
1767+
missing.append(normalized.get(title, title))
1768+
return missing
1769+
17201770
def store_entity(
17211771
self, param: Union[StoreEntityParam, OswBaseModel, List[OswBaseModel]]
17221772
) -> StoreEntityResult:
@@ -1739,6 +1789,8 @@ def store_entity(
17391789

17401790
max_index = len(param.entities)
17411791
created_pages = {}
1792+
edited_titles = set()
1793+
"""Titles of the pages an edit was sent for, to be verified below."""
17421794

17431795
meta_category_templates = {}
17441796
if param.namespace == "Category":
@@ -1863,6 +1915,7 @@ def store_entity_(
18631915
page.edit(
18641916
param.edit_comment, bot_edit=param.bot_edit
18651917
) # will set page.changed if the content of the page has changed
1918+
edited_titles.add(page.title)
18661919
if not param.offline and page.changed:
18671920
if index is None:
18681921
print(f"Entity stored at '{page.get_url()}'.")
@@ -1922,7 +1975,10 @@ class UploadObject(BaseModel):
19221975
def handle_upload_object_(upload_object: UploadObject) -> None:
19231976
# Let exceptions propagate: the caller collects them per entity below,
19241977
# so a single failure neither aborts the batch nor is silently
1925-
# swallowed (store_entity_ only records created_pages on success).
1978+
# swallowed. store_entity_ records a page in created_pages whenever it
1979+
# reaches its last statement, which only means that nothing raised.
1980+
# Whether the page exists afterwards is checked by the verification
1981+
# step below.
19261982
store_entity_(
19271983
upload_object.entity,
19281984
upload_object.namespace,
@@ -1969,6 +2025,19 @@ def failure_title_(upload_object: UploadObject) -> str:
19692025
_logger.error(f"Error storing entity '{title}': {result}")
19702026
failed[title] = result
19712027

2028+
if param.verify_write and not param.offline and edited_titles:
2029+
# An edit that raised no exception is not proof that the page exists:
2030+
# a form or template driven creation step can reject the content
2031+
# server-side. page.changed is no help either, it is True in that case.
2032+
titles_to_verify = [
2033+
title for title in edited_titles if title in created_pages
2034+
]
2035+
for title in self._get_missing_page_titles(titles_to_verify):
2036+
error = OSW.PageNotCreatedError(title)
2037+
_logger.error(f"Error storing entity '{title}': {error}")
2038+
failed[title] = error
2039+
del created_pages[title]
2040+
19722041
store_result = OSW.StoreEntityResult(
19732042
change_id=param.change_id, pages=created_pages, failed=failed
19742043
)

‎tests/test_overwrite_policy.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -355,6 +355,10 @@ def _stub_page_io(monkeypatch, exists: bool):
355355
)
356356
edited = []
357357
monkeypatch.setattr(WtPage, "edit", lambda self, *a, **k: edited.append(self.title))
358+
# the write verification of store_entity() would query the wiki. These tests
359+
# cover the overwrite policy, so report every edited page as existing. The
360+
# verification itself is covered by test_store_entity_verify.py
361+
monkeypatch.setattr(OSW, "_get_missing_page_titles", lambda self, titles: [])
358362
return edited
359363

360364

‎tests/test_store_entity_failure.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ def offline_osw(monkeypatch):
2626
monkeypatch.setattr(
2727
OSW, "_apply_overwrite_policy", staticmethod(lambda param: param.page)
2828
)
29+
# the write verification would query the wiki. These tests cover the reporting
30+
# of a failed edit, so report every edited page as existing. The verification
31+
# itself is covered by test_store_entity_verify.py
32+
monkeypatch.setattr(OSW, "_get_missing_page_titles", lambda self, titles: [])
2933
return OSW.construct(site=object())
3034

3135

‎tests/test_store_entity_verify.py‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
"""Unit tests for the write verification of store_entity().
2+
3+
Regression guard for #175: store_entity() returned normally and listed a page in
4+
created_pages when that page did not exist on the wiki afterwards. An edit that
5+
raises nothing is not proof that the page was created, and page.changed is True
6+
in the failing case as well. store_entity() must now query the edited pages and
7+
report an absent one in result.failed.
8+
9+
These run fully offline: WtPage.init, the overwrite policy and the existence
10+
query are stubbed, so no network is required.
11+
"""
12+
13+
import pytest
14+
15+
import osw.model.entity as model
16+
from osw.core import OSW
17+
from osw.utils.wiki import get_namespace, get_title
18+
from osw.wtsite import WtPage
19+
20+
21+
def _title(entity):
22+
return f"{get_namespace(entity)}:{get_title(entity)}"
23+
24+
25+
class _FakeMwSite:
26+
"""Records the API queries and answers them from a set of missing titles."""
27+
28+
def __init__(self, missing_titles=(), normalized=None):
29+
self.missing_titles = set(missing_titles)
30+
self.normalized = normalized or {}
31+
self.queries = []
32+
33+
def api(self, action, **kwargs):
34+
assert action == "query"
35+
titles = kwargs["titles"].split("|")
36+
self.queries.append(titles)
37+
pages = {}
38+
for i, title in enumerate(titles):
39+
reported = self.normalized.get(title, title)
40+
if title in self.missing_titles:
41+
pages[str(-(i + 1))] = {"title": reported, "missing": ""}
42+
else:
43+
pages[str(i + 1)] = {"title": reported, "pageid": i + 1}
44+
query = {"pages": pages}
45+
if self.normalized:
46+
query["normalized"] = [
47+
{"from": k, "to": v} for k, v in self.normalized.items()
48+
]
49+
return {"query": query}
50+
51+
52+
class _FakeSite:
53+
def __init__(self, mw_site):
54+
self.mw_site = mw_site
55+
56+
57+
@pytest.fixture
58+
def offline_osw(monkeypatch):
59+
"""An OSW that never touches the network, with a controllable existence query."""
60+
monkeypatch.setattr(WtPage, "init", lambda self: setattr(self, "exists", False))
61+
monkeypatch.setattr(
62+
OSW, "_apply_overwrite_policy", staticmethod(lambda param: param.page)
63+
)
64+
monkeypatch.setattr(WtPage, "edit", lambda self, *a, **kw: None)
65+
66+
def _make(missing_titles=(), normalized=None):
67+
mw_site = _FakeMwSite(missing_titles, normalized)
68+
return OSW.construct(site=_FakeSite(mw_site)), mw_site
69+
70+
return _make
71+
72+
73+
def test_absent_page_is_reported_as_failed(offline_osw):
74+
item = model.Item(label=[model.Label(text="Ghost")])
75+
title = _title(item)
76+
osw_obj, _mw_site = offline_osw(missing_titles=[title])
77+
78+
with pytest.raises(OSW.StoreEntityPartialError) as exc_info:
79+
osw_obj.store_entity(OSW.StoreEntityParam(entities=[item], parallel=False))
80+
81+
err = exc_info.value
82+
assert title in err.failed
83+
assert isinstance(err.failed[title], OSW.PageNotCreatedError)
84+
assert err.failed[title].title == title
85+
assert title not in err.result.pages
86+
assert err.stored == []
87+
88+
89+
def test_present_page_is_reported_as_stored(offline_osw):
90+
item = model.Item(label=[model.Label(text="Real")])
91+
title = _title(item)
92+
osw_obj, mw_site = offline_osw()
93+
94+
result = osw_obj.store_entity(OSW.StoreEntityParam(entities=[item], parallel=False))
95+
96+
assert set(result.pages.keys()) == {title}
97+
assert result.failed == {}
98+
assert mw_site.queries == [[title]]
99+
100+
101+
def test_only_the_absent_page_of_a_batch_is_reported(offline_osw):
102+
items = [model.Item(label=[model.Label(text=f"Batch{i}")]) for i in range(3)]
103+
titles = [_title(it) for it in items]
104+
osw_obj, _mw_site = offline_osw(missing_titles=[titles[1]])
105+
106+
with pytest.raises(OSW.StoreEntityPartialError) as exc_info:
107+
osw_obj.store_entity(OSW.StoreEntityParam(entities=items, parallel=True))
108+
109+
err = exc_info.value
110+
assert set(err.result.pages.keys()) == {titles[0], titles[2]}
111+
assert set(err.failed.keys()) == {titles[1]}
112+
113+
114+
def test_verification_is_skipped_when_disabled(offline_osw):
115+
item = model.Item(label=[model.Label(text="Unchecked")])
116+
title = _title(item)
117+
osw_obj, mw_site = offline_osw(missing_titles=[title])
118+
119+
result = osw_obj.store_entity(
120+
OSW.StoreEntityParam(entities=[item], parallel=False, verify_write=False)
121+
)
122+
123+
assert mw_site.queries == []
124+
assert set(result.pages.keys()) == {title}
125+
126+
127+
def test_verification_is_skipped_offline(offline_osw):
128+
item = model.Item(label=[model.Label(text="Offline")])
129+
title = _title(item)
130+
osw_obj, mw_site = offline_osw(missing_titles=[title])
131+
132+
result = osw_obj.store_entity(
133+
OSW.StoreEntityParam(entities=[item], parallel=False, offline=True)
134+
)
135+
136+
assert mw_site.queries == []
137+
assert set(result.pages.keys()) == {title}
138+
139+
140+
def test_titles_are_queried_in_batches_of_fifty(offline_osw):
141+
osw_obj, mw_site = offline_osw()
142+
titles = [f"Item:OSW{i:04d}" for i in range(120)]
143+
144+
missing = osw_obj._get_missing_page_titles(titles)
145+
146+
assert missing == []
147+
assert [len(batch) for batch in mw_site.queries] == [50, 50, 20]
148+
149+
150+
def test_missing_titles_are_mapped_back_to_the_requested_form(offline_osw):
151+
requested = "Item:OSW_with_underscores"
152+
osw_obj, _mw_site = offline_osw(
153+
missing_titles=[requested],
154+
normalized={requested: "Item:OSW with underscores"},
155+
)
156+
157+
assert osw_obj._get_missing_page_titles([requested]) == [requested]

0 commit comments

Comments
 (0)