Skip to content

Commit d2e2952

Browse files
committed
feat(wtsite): parametrize and parallelize modify_search_results
- wire up the previously unused ModifySearchResultsParam - add a parallel flag using the existing parallelize helper - widen param.query to accept a plain string query - keep the legacy positional and keyword call form working - closes #27
1 parent 3a4a0f2 commit d2e2952

2 files changed

Lines changed: 186 additions & 20 deletions

File tree

‎src/osw/wtsite.py‎

Lines changed: 40 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -573,25 +573,27 @@ def semantic_search(self, query: Union[str, SearchParam]):
573573
return wt.semantic_search(self._site, query)
574574

575575
class ModifySearchResultsParam(OswBaseModel):
576-
"""Todo: should become param of modify_search_results"""
576+
"""Parameter object for modify_search_results method."""
577577

578578
mode: str
579579
"""The search mode. Either 'prefix' or 'semantic'."""
580-
query: wt.SearchParam
580+
query: Union[str, wt.SearchParam]
581581
"""The search query."""
582582
comment: str = None
583583
"""The comment for the edit."""
584584
log: bool = False
585585
"""Whether to log changes."""
586586
dryrun: bool = False
587587
"""if True, no actual changes are made"""
588+
parallel: Optional[bool] = False
589+
"""If True, processes the search results in parallel."""
588590

589591
@try_and_renew_token
590592
def modify_search_results(
591593
self,
592-
mode: str,
593-
query: str,
594-
modify_page,
594+
param: Union[str, ModifySearchResultsParam],
595+
query: str = None,
596+
modify_page=None,
595597
limit=None,
596598
comment=None,
597599
log=False,
@@ -602,43 +604,61 @@ def modify_search_results(
602604
603605
Parameters
604606
----------
605-
mode
606-
The search mode. Either 'prefix' or 'semantic'.
607+
param
608+
ModifySearchResultsParam object. For backwards compatibility, the
609+
search mode ('prefix' or 'semantic') can be passed directly instead,
610+
together with the deprecated keyword arguments below.
607611
query
608-
The search query.
612+
Deprecated, use param.query instead. The search query.
609613
modify_page
610614
The callback that modifies the pages.
611615
limit
612616
query limit, by default None
613617
comment
614-
edit comment, by default None
618+
Deprecated, use param.comment instead. edit comment, by default None
615619
log
616-
log changes, by default False
620+
Deprecated, use param.log instead. log changes, by default False
617621
dryrun
618-
if True, no actual changes are made, by default False
622+
Deprecated, use param.dryrun instead. if True, no actual changes are
623+
made, by default False
619624
"""
625+
if not isinstance(param, WtSite.ModifySearchResultsParam):
626+
param = WtSite.ModifySearchResultsParam(
627+
mode=param,
628+
query=query,
629+
comment=comment,
630+
log=log,
631+
dryrun=dryrun,
632+
)
633+
620634
titles = []
621-
if mode == "prefix":
622-
titles = wt.prefix_search(self._site, query)
623-
elif mode == "semantic":
624-
titles = wt.semantic_search(self._site, query)
635+
if param.mode == "prefix":
636+
titles = wt.prefix_search(self._site, param.query)
637+
elif param.mode == "semantic":
638+
titles = wt.semantic_search(self._site, param.query)
625639
if limit:
626640
titles = titles[0:limit]
627-
if log:
641+
if param.log:
628642
print(f"Found: {titles}")
629-
for title in titles:
643+
644+
def modify_single_result(title: str):
630645
wtpage = self.get_page(WtSite.GetPageParam(titles=[title])).pages[0]
631646
modify_page(wtpage)
632-
if log:
647+
if param.log:
633648
print(f"\n======= {title} =======")
634649
for slot in wtpage._slots:
635650
content = wtpage.get_slot_content(slot)
636651
# if isinstance(content, dict): content = json.dumps(content)
637652
print(f" ==== {title}:{slot} ==== ")
638653
pprint(content)
639654
print("\n")
640-
if not dryrun:
641-
wtpage.edit(comment)
655+
if not param.dryrun:
656+
wtpage.edit(param.comment)
657+
658+
if param.parallel:
659+
_ = parallelize(modify_single_result, titles, flush_at_end=param.log)
660+
else:
661+
_ = [modify_single_result(title) for title in titles]
642662

643663
class UploadPageParam(OswBaseModel):
644664
"""Parameter object for upload_page method."""
Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
"""Unit tests for WtSite.modify_search_results().
2+
3+
Regression guard for #27: modify_search_results now accepts the previously
4+
unused ModifySearchResultsParam object, gains an optional parallel mode, and
5+
must keep the legacy keyword-argument call form working.
6+
"""
7+
8+
import threading
9+
10+
import osw.wtsite as wtsite_mod
11+
from osw.wtsite import WtPage, WtSite
12+
13+
14+
class _FakeSite:
15+
"""Stands in for mwclient.Site. No method on it should ever be touched."""
16+
17+
host = "example.org"
18+
19+
20+
def _make_fake_wtsite():
21+
"""A WtSite that performs no network calls."""
22+
ws = WtSite.__new__(WtSite)
23+
ws._session_lock = threading.RLock()
24+
ws._site = _FakeSite()
25+
return ws
26+
27+
28+
def _make_pages(wtsite, titles):
29+
return {
30+
title: WtPage(wtSite=wtsite, title=title, do_init=False) for title in titles
31+
}
32+
33+
34+
def _stub_search(monkeypatch, titles):
35+
"""Both prefix_search and semantic_search return the same fixed titles."""
36+
monkeypatch.setattr(wtsite_mod.wt, "prefix_search", lambda site, query: titles)
37+
monkeypatch.setattr(wtsite_mod.wt, "semantic_search", lambda site, query: titles)
38+
39+
40+
def _stub_get_page(monkeypatch, ws, pages_by_title):
41+
"""WtSite.get_page returns the canned page instead of hitting the network."""
42+
43+
def fake_get_page(param):
44+
title = param.titles[0]
45+
return WtSite.GetPageResult(pages=[pages_by_title[title]], errors=[])
46+
47+
monkeypatch.setattr(ws, "get_page", fake_get_page)
48+
49+
50+
def _stub_edits(monkeypatch, pages_by_title, comments, lock=None):
51+
"""Records (title, comment) for every page.edit() call instead of editing."""
52+
53+
def make_recorder(page):
54+
def _edit(comment=None):
55+
if lock is not None:
56+
with lock:
57+
comments.append((page.title, comment))
58+
else:
59+
comments.append((page.title, comment))
60+
61+
return _edit
62+
63+
for page in pages_by_title.values():
64+
monkeypatch.setattr(page, "edit", make_recorder(page))
65+
66+
67+
def test_modify_search_results_with_param_object_sequential(monkeypatch):
68+
"""The new param-object call form produces the same result as the old loop."""
69+
ws = _make_fake_wtsite()
70+
titles = ["Item:OSW1", "Item:OSW2", "Item:OSW3"]
71+
pages = _make_pages(ws, titles)
72+
comments = []
73+
74+
_stub_search(monkeypatch, titles)
75+
_stub_get_page(monkeypatch, ws, pages)
76+
_stub_edits(monkeypatch, pages, comments)
77+
78+
handled = []
79+
80+
def modify_page(wtpage):
81+
handled.append(wtpage.title)
82+
83+
param = WtSite.ModifySearchResultsParam(
84+
mode="prefix", query="Item:", comment="[bot] test", dryrun=False
85+
)
86+
ws.modify_search_results(param, modify_page=modify_page)
87+
88+
assert handled == titles
89+
assert comments == [(title, "[bot] test") for title in titles]
90+
91+
92+
def test_modify_search_results_parallel_processes_every_result(monkeypatch):
93+
"""Every search result is handled when parallel processing is enabled."""
94+
ws = _make_fake_wtsite()
95+
titles = [f"Item:OSW{i}" for i in range(8)]
96+
pages = _make_pages(ws, titles)
97+
comments = []
98+
lock = threading.Lock()
99+
100+
_stub_search(monkeypatch, titles)
101+
_stub_get_page(monkeypatch, ws, pages)
102+
_stub_edits(monkeypatch, pages, comments, lock=lock)
103+
104+
handled = []
105+
106+
def modify_page(wtpage):
107+
with lock:
108+
handled.append(wtpage.title)
109+
110+
param = WtSite.ModifySearchResultsParam(
111+
mode="prefix",
112+
query="Item:",
113+
comment="[bot] parallel",
114+
parallel=True,
115+
)
116+
ws.modify_search_results(param, modify_page=modify_page)
117+
118+
assert sorted(handled) == sorted(titles)
119+
assert sorted(comments) == sorted((title, "[bot] parallel") for title in titles)
120+
121+
122+
def test_modify_search_results_legacy_keyword_call_still_works(monkeypatch):
123+
"""The pre-existing positional/keyword call form is unchanged."""
124+
ws = _make_fake_wtsite()
125+
titles = ["Item:OSW1"]
126+
pages = _make_pages(ws, titles)
127+
comments = []
128+
129+
_stub_search(monkeypatch, titles)
130+
_stub_get_page(monkeypatch, ws, pages)
131+
_stub_edits(monkeypatch, pages, comments)
132+
133+
handled = []
134+
135+
ws.modify_search_results(
136+
"prefix",
137+
"Item:",
138+
lambda wtpage: handled.append(wtpage.title),
139+
limit=1,
140+
comment="[bot] legacy",
141+
log=False,
142+
dryrun=False,
143+
)
144+
145+
assert handled == titles
146+
assert comments == [("Item:OSW1", "[bot] legacy")]

0 commit comments

Comments
 (0)