Skip to content

Commit 92d68ed

Browse files
committed
feat: add offline mode to store_entity
Can be used to create WtPage objects from entities without uploading them.
1 parent ab29e4d commit 92d68ed

3 files changed

Lines changed: 76 additions & 12 deletions

File tree

‎src/osw/controller/page_package.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
from osw.model.page_package import NAMESPACE_CONST_TO_NAMESPACE_MAPPING
1616
from osw.model.static import OswBaseModel
1717
from osw.utils.regex import RegExPatternExtended
18-
from osw.wtsite import WtSite
18+
from osw.wtsite import WtPage, WtSite
1919

2020
# Definition of constants
2121
PATTERNS = {
@@ -369,6 +369,12 @@ class CreationConfig(OswBaseModel):
369369
ignore_titles: Optional[List[str]] = None
370370
"""List of page titles to ignore in auto-detected dependencies, e.g.,
371371
example files"""
372+
offline_pages: Optional[Dict[str, WtPage]] = None
373+
"""A dictionary of pages that are already loaded. Pages in this dictionary
374+
will not be fetched again."""
375+
376+
class Config:
377+
arbitrary_types_allowed = True
372378

373379
def create(
374380
self,
@@ -423,7 +429,12 @@ def create(
423429
skip_slot_suffix_for_main=creation_config.skip_slot_suffix_for_main,
424430
)
425431
# Create the page package in the working directory
426-
wtsite.create_page_package(WtSite.CreatePagePackageParam(config=config))
432+
wtsite.create_page_package(
433+
WtSite.CreatePagePackageParam(
434+
config=config,
435+
offline_pages=creation_config.offline_pages,
436+
)
437+
)
427438

428439
class GetRequiredPagesParams(OswBaseModel):
429440
creation_config: "PagePackageController.CreationConfig"

‎src/osw/core.py‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -914,6 +914,7 @@ class _ApplyOverwriteParam(OswBaseModel):
914914
remove_empty: Optional[bool] = True
915915
inplace: Optional[bool] = False
916916
debug: Optional[bool] = False
917+
offline: Optional[bool] = False
917918

918919
class Config:
919920
arbitrary_types_allowed = True
@@ -989,6 +990,7 @@ def set_content(content_to_set: dict) -> None:
989990
if (
990991
not page.exists
991992
or param.policy.overwrite == AddOverwriteClassOptions.replace_remote
993+
or param.offline is True
992994
):
993995
# Use pydantic serialization, skip none values:
994996
new_content["jsondata"] = json.loads(param.entity.json(exclude_none=True))
@@ -1123,6 +1125,9 @@ class StoreEntityParam(OswBaseModel):
11231125
store_entity() calls."""
11241126
meta_category_title: Optional[Union[str, List[str]]] = "Category:Category"
11251127
debug: Optional[bool] = False
1128+
offline: Optional[bool] = False
1129+
"""If set to True, the processed entities are not upload but only returned as WtPages.
1130+
Can be used to create WtPage objects from entities without uploading them."""
11261131
_overwrite_per_class: Dict[str, Dict[str, OSW.OverwriteClassParam]] = (
11271132
PrivateAttr()
11281133
)
@@ -1172,6 +1177,11 @@ class StoreEntityResult(OswBaseModel):
11721177

11731178
change_id: str
11741179
"""The ID of the change"""
1180+
pages: Dict[str, WtPage]
1181+
"""The pages that have been stored"""
1182+
1183+
class Config:
1184+
arbitrary_types_allowed = True
11751185

11761186
def store_entity(
11771187
self, param: Union[StoreEntityParam, OswBaseModel, List[OswBaseModel]]
@@ -1193,6 +1203,7 @@ def store_entity(
11931203
param: OSW.StoreEntityParam = param
11941204

11951205
max_index = len(param.entities)
1206+
created_pages = {}
11961207

11971208
meta_category_templates = {}
11981209
if param.namespace == "Category":
@@ -1240,12 +1251,15 @@ def store_entity_(
12401251
entity_title = namespace_ + ":" + title_
12411252
page = self._apply_overwrite_policy(
12421253
OSW._ApplyOverwriteParam(
1243-
page=WtPage(wtSite=self.site, title=entity_title),
1254+
page=WtPage(
1255+
wtSite=self.site, title=entity_title, do_init=not param.offline
1256+
),
12441257
entity=entity_,
12451258
namespace=namespace_,
12461259
policy=overwrite_class_param,
12471260
remove_empty=param.remove_empty,
12481261
debug=param.debug,
1262+
offline=param.offline,
12491263
)
12501264
)
12511265
if len(meta_category_templates.keys()) > 0:
@@ -1285,15 +1299,17 @@ def store_entity_(
12851299
)
12861300
).aggregated_schema
12871301
page.set_slot_content("jsonschema", new_schema)
1288-
page.edit() # will set page.changed if the content of the page has changed
1289-
if page.changed:
1302+
if param.offline is False:
1303+
page.edit() # will set page.changed if the content of the page has changed
1304+
if not param.offline and page.changed:
12901305
if index is None:
12911306
print(f"Entity stored at '{page.get_url()}'.")
12921307
else:
12931308
print(
12941309
f"({index + 1}/{max_index}) Entity stored at "
12951310
f"'{page.get_url()}'."
12961311
)
1312+
created_pages[page.title] = page
12971313

12981314
sorted_entities = OSW.sort_list_of_entities_by_class(param.entities)
12991315
print(
@@ -1358,7 +1374,7 @@ def handle_upload_object_(upload_object: UploadObject) -> None:
13581374
handle_upload_object_(upload_object)
13591375
for upload_object in upload_object_list
13601376
]
1361-
return OSW.StoreEntityResult(change_id=param.change_id)
1377+
return OSW.StoreEntityResult(change_id=param.change_id, pages=created_pages)
13621378

13631379
class DeleteEntityParam(OswBaseModel):
13641380
entities: Union[OswBaseModel, List[OswBaseModel]]

‎src/osw/wtsite.py‎

Lines changed: 43 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -690,6 +690,9 @@ class CreatePagePackageParam(OswBaseModel):
690690
"""If True, debug messages will be printed."""
691691
parallel: Optional[bool] = None
692692
"""If true, processes the pages in parallel."""
693+
offline_pages: Optional[Dict[str, "WtPage"]] = None
694+
"""A dictionary of pages that are already loaded. Pages in this dictionary
695+
will not be fetched again."""
693696

694697
class Config:
695698
arbitrary_types_allowed = True
@@ -736,10 +739,27 @@ def create_page_package(self, param: CreatePagePackageParam):
736739
bundle.packages[config.name].pages = []
737740
# remove duplicates while keeping order
738741
added_titles = list(dict.fromkeys(config.titles))
742+
titles_to_fetch = added_titles.copy()
739743

740-
pages = self.get_page(
741-
WtSite.GetPageParam(titles=added_titles, parallel=param.parallel)
742-
).pages
744+
pages = []
745+
if param.offline_pages is not None:
746+
pages.extend(
747+
(
748+
param.offline_pages[title]
749+
for title in added_titles
750+
if title in param.offline_pages
751+
)
752+
)
753+
titles_to_fetch = list(
754+
set(titles_to_fetch) - set(param.offline_pages.keys())
755+
)
756+
757+
if len(titles_to_fetch) > 0:
758+
pages.extend(
759+
self.get_page(
760+
WtSite.GetPageParam(titles=titles_to_fetch, parallel=param.parallel)
761+
).pages
762+
)
743763
added_file_titles = []
744764
page_dumps = {}
745765
page_files = {}
@@ -771,10 +791,27 @@ def create_page_package(self, param: CreatePagePackageParam):
771791
added_file_titles = list(set(added_file_titles + referenced_file_pages))
772792

773793
file_dumps = {}
794+
774795
if len(added_file_titles) > 0:
775-
file_pages = self.get_page(
776-
WtSite.GetPageParam(titles=added_file_titles, parallel=param.parallel)
777-
).pages
796+
file_titles_to_fetch = added_file_titles.copy()
797+
file_pages = []
798+
if param.offline_pages is not None:
799+
file_pages.extend(
800+
(
801+
param.offline_pages[title]
802+
for title in added_file_titles
803+
if title in param.offline_pages
804+
)
805+
)
806+
file_titles_to_fetch = list(
807+
set(added_file_titles) - set(param.offline_pages.keys())
808+
)
809+
if len(file_titles_to_fetch) > 0:
810+
file_pages = self.get_page(
811+
WtSite.GetPageParam(
812+
titles=file_titles_to_fetch, parallel=param.parallel
813+
)
814+
).pages
778815
for file_page in file_pages:
779816
if not file_page.exists:
780817
continue

0 commit comments

Comments
 (0)