Repository navigation
Expand file tree
/
Copy pathexpress.py
More file actions
995 lines (930 loc) · 42.6 KB
/
Copy pathexpress.py
File metadata and controls
995 lines (930 loc) · 42.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
"""
This module provides convenience functions for osw-python.
This module expects environment variables to be set and available, e.g. through
dotenv.load_dotenv()
- OSW_DOMAIN: domain of the OSL instance to connect to
- OSW_CRED_FILEPATH: filepath to the credential file, otherwise falls back to the
default file path, which is based on current working directory
- OSW_DOWNLOAD_DIR: directory to download files, otherwise falls back to the default
directory, which is based on the current working directory
"""
import importlib.util
import os
import re
from io import TextIOWrapper
from pathlib import Path
from typing import overload
from uuid import uuid4
from warnings import warn
import requests
from opensemantic.v1 import OswBaseModel
from pydantic.v1 import validator
from typing_extensions import (
IO,
Any,
AnyStr,
Buffer,
Dict,
List,
Optional,
TextIO,
Union,
)
import osw.model.entity as model
from osw.auth import CredentialManager
from osw.controller.file.base import FileController # depends on File
from osw.controller.file.local import LocalFileController # depends on LocalFile
from osw.controller.file.memory import InMemoryController # depends on LocalFile
from osw.controller.file.wiki import WikiFileController # depends on WikiFile
from osw.core import OSW, OVERWRITE_CLASS_OPTIONS, OverwriteOptions
from osw.defaults import params as default_params
from osw.defaults import paths as default_paths
from osw.utils.wiki import namespace_from_full_title, title_from_full_title
from osw.wtsite import WtSite
class OswExpress(OSW):
"""This class provides convenience functions for osw-python."""
domain: str
"""The domain of the OSL instance to connect to."""
cred_filepath: Optional[Union[str, Path]]
"""The filepath to the credentials file. Will be overwritten by the cred_filepath
of cred_mngr, if it posses a non-None value. If cred_filepath is None,
a default value is set."""
cred_mngr: Optional[CredentialManager]
"""A credential manager object."""
class Config:
arbitrary_types_allowed = True
@validator("domain")
def validate_domain(cls, v):
pattern = r"^(?!-)[A-Za-z0-9.-]{1,63}(?<!-)\.[A-Za-z]{2,}$"
assert re.match(pattern, v), "The domain is not valid." # noqa: S101 pydantic validator idiom
return v
@overload
def __init__(
self,
domain: str = None,
cred_filepath: Union[str, Path] = None,
cred_mngr: CredentialManager = None,
) -> None: ...
# case: nothing specified, all parameters to be loaded from .env file or
# environment parameters
@overload
def __init__(
self,
domain: str,
cred_filepath: Union[str, Path] = None,
cred_mngr: CredentialManager = None,
) -> None: ...
# case: domain is specified, rest is to be loaded from .env file or environment
# parameters
def __init__(
self,
domain: str | None = None,
cred_filepath: str | Path | None = None,
cred_mngr: CredentialManager | None = None,
):
if domain is None:
if os.getenv("OSW_DOMAIN") is not None:
domain = os.getenv("OSW_DOMAIN")
elif os.getenv("OSL_DOMAIN") is not None:
domain = os.getenv("OSL_DOMAIN")
else:
raise TypeError(
"The constructor of OswExpress is missing 1 required positional "
"argument: 'domain'."
"\nIf no domain was set as environmental variable, 'domain' is a "
"required argument."
)
if cred_mngr is not None and cred_mngr.cred_filepath is not None:
# If a credential manager is explicitly defined, that should have priority
cred_filepath = cred_mngr.cred_filepath[0]
if cred_filepath is None:
# If no credential file path is given, try to take it from environment
# vars. A credential file is only ever READ; credentials obtained from
# environment variables (OSW_USERNAME/OSW_PASSWORD, e.g. loaded from a
# .env file) or an interactive prompt are kept in memory only.
# Credentials can be persisted with an explicit call of
# cred_mngr.save_credentials_to_file()
if os.getenv("OSW_CRED_FILEPATH") is not None:
cred_filepath = os.getenv("OSW_CRED_FILEPATH")
elif os.getenv("OSL_CRED_FILEPATH") is not None:
cred_filepath = os.getenv("OSL_CRED_FILEPATH")
if cred_filepath is not None:
if not isinstance(cred_filepath, Path):
cred_filepath = Path(cred_filepath)
if not cred_filepath.is_file():
print(
f"Credential file '{cred_filepath}' does not exist and will "
"be ignored. Credentials are taken from the environment "
"variables OSW_USERNAME/OSW_PASSWORD or an interactive "
"prompt instead (in-memory only)."
)
cred_filepath = None
if cred_mngr is None:
# Create a credentials manager
if cred_filepath is None:
cred_mngr = CredentialManager()
else: # Reuse passed cred_filepath
cred_mngr = CredentialManager(cred_filepath=cred_filepath)
# Test if domain is reachable
try:
url = f"https://{domain}/wiki/Main_Page"
response = requests.get(url) # noqa: S113 reachability probe, TODO add timeout
if response.status_code == 200:
# Domain is reachable
print(f"Connecting to '{domain}'...")
else:
raise ConnectionError(
f"Could not connect to '{domain}'. Response: {response.status_code}"
)
except Exception as e:
raise ConnectionError(f"Could not connect to '{domain}'. Error: {e}")
site = WtSite(WtSite.WtSiteConfig(iri=domain, cred_mngr=cred_mngr))
super().__init__(**{"site": site, "domain": domain})
self.cred_mngr = cred_mngr
self.cred_filepath = cred_filepath
print(f"Connected to '{domain}'.")
def __enter__(self):
"""Return self when entering the context manager."""
return self
def __exit__(self, exc_type=None, exc_value=None, traceback=None):
"""Close the connection to the OSL instance when exiting the context manager."""
self.close_connection()
def shut_down(self):
"""Makes sure this OSL instance can't be reused after it was shut down,
as the connection can't be reopened except when initializing a new instance."""
self.close_connection()
del self
def download_file(
self,
url_or_title: str,
mode: str = "r",
delete_after_use: bool = False,
target_dir: Optional[Union[str, Path]] = None,
target_fn: Optional[str] = None,
target_fp: Optional[Union[str, Path]] = None,
overwrite: bool = False,
use_cached: bool = False,
) -> "DownloadFileResult":
"""Download a file from a URL to a target directory.
Parameters
----------
url_or_title
The URL or full page title of the WikiFile page to download.
mode
The mode to open the file in. Default is 'r'. Implements the built-in open.
delete_after_use
If True, the file will be deleted after use.
target_dir
The target directory to download the file to. If None, the current working
will be used.
target_fn
The target filename to save the file as. If None, the filename will be taken
from the URL or title.
target_fp
The target filepath to save the file to. If None, the file will be saved to
the target directory with the target filename.
overwrite
If True, the file will be overwritten if it already exists. If False, the
file
will not be downloaded if it already exists.
use_cached
If True, the file will be reloaded from the cache. If False, the file will
be reloaded from the server. This option is useful if you are debugging
code and don't want to reload the file from the server every time.
Returns
-------
result
A specific result object.
"""
return DownloadFileResult(
url_or_title=url_or_title,
mode=mode,
delete_after_use=delete_after_use,
target_dir=target_dir,
target_fn=target_fn,
target_fp=target_fp,
osw_express=self,
domain=self.domain,
overwrite=overwrite,
use_cached=use_cached,
)
def upload_file(
self,
source: Union["LocalFileController", "WikiFileController", str, Path, IO],
url_or_title: Optional[str] = None,
overwrite: OVERWRITE_CLASS_OPTIONS = OverwriteOptions.true,
delete_after_use: bool = False,
label: Optional[List[model.Label]] = None,
name: Optional[str] = None,
description: Optional[List[model.Description]] = None,
change_id: Optional[str] = None,
**properties: Dict[str, Any],
) -> "UploadFileResult":
"""Upload a file to an OSL page.
Parameters
----------
source
The source file to upload. Can be a LocalFileController, WikiFileController,
str, Path or an open file-like object.
url_or_title
The URL or full page title of the WikiFile page to upload the file to. Used to
overwrite autogenerated full page title on the target domain. If it is
included, the domain can be parsed from the URL.
overwrite
If True, the file will be overwritten if it already exists. If False, the file
will not be downloaded if it already exists.
delete_after_use
If True, the file will be deleted after use.
label
The labels to set on the WikiFile data model prior to uploading it to the OSL
instance. The labels will end up in the JSON data slot of the to be created
WikiFile page.
name
The name of the file. If None, the name will be taken from the source file.
description
The description to set on the WikiFile data model prior to uploading it to the
OSL instance.
change_id
The id of the change to use for the upload. If None, a new change id will be
generated.
properties
The properties to set on the WikiFile data model prior to uploading it to
the OSL instance. Properties listed here, won't overwrite properties handed
over in this function call, e.g., labe, name, description.
Properties will end up in the JSON data slot of the to be created WikiFile
page, if they match the WikiFile data model.
Returns
-------
result
A specific result object.
"""
# Preparing all args, kwargs & properties to set for the uploaded file
data = {**locals(), **properties}
# Remove 'self', 'source', and 'properties' to avoid duplicate keyword args
for key in ("self", "source", "properties"):
data.pop(key, None)
# Clean data dict to avoid passing None values
data = {key: value for key, value in data.items() if value is not None}
# Make sure self is passed as osw_express
data["osw_express"] = self
# Initialize the UploadFileResult object
return UploadFileResult(source=source, **data)
class FileResult(OswBaseModel):
url_or_title: Optional[str] = None
"""The URL or full page title of the WikiFile page."""
file_io: Optional[Union[TextIO, TextIOWrapper, None]] = None
"""The file object. They type depends on the file type."""
mode: str = "r"
"""The mode to open the file in. Default is 'r'. Implements the built-in open."""
delete_after_use: bool = False
"""If True, the file will be deleted after use."""
path: Optional[Path] = None
"""Overwriting the attribute of the base class to avoid validation error at
initialization. The path to the file."""
osw_express: Optional[OswExpress] = None
"""An OswExpress object. If None, a new OswExpress object will be created using
the credentials_manager or domain and cred_filepath."""
domain: Optional[str] = None
"""The domain of the OSL instance to download the file from. Required if
urL_or_title is a full page title and the domain is not set in the environment
variable 'OSW_DOMAIN'. If None the domain is parsed from the URL."""
cred_filepath: Optional[Union[str, Path]] = None
"""The filepath to the credentials file. Will only be used if cred_mngr is
None. If cred_filepath is None, the default credentials file 'accounts.pwd.yaml'
in the current working directory (project root) is read if present.
"""
cred_mngr: Optional[CredentialManager] = None
"""A credential manager object. If None, a new credential manager will be created
using the cred_filepath."""
class Config:
arbitrary_types_allowed = True
def open(self, mode: str = None, **kwargs) -> TextIO:
"""Open the file, if not already opened using the 'mode' argument (priority) or
the 'mode' attribute."""
if mode is None:
mode = self.mode
kwargs["mode"] = mode
if self.file_io is None or self.file_io.closed:
self.file_io = open(self.path, **kwargs) # noqa: SIM115 handle kept as instance state
return self.file_io
def close(self) -> None:
"""Close the file. Like io streams, closing an already closed (or
never opened) file is a silent no-op."""
if self.file_io is not None and not self.file_io.closed:
self.file_io.close()
def read(self, n: int = -1) -> AnyStr:
"""Read the file. If n is not specified, the entire file will be read.
If the file is not already opened, it will be opened."""
if self.file_io is None or self.file_io.closed:
self.file_io = self.open(mode="r")
return self.file_io.read(n)
def write(self, s: Union[Buffer, AnyStr]):
"""Write to the file. If the file is not already opened, it will be opened."""
if self.file_io is None or self.file_io.closed:
self.file_io = self.open(mode="w")
return self.file_io.write(s)
def __enter__(self):
"""Open the file when entering the context manager."""
if self.file_io is None or self.file_io.closed:
self.open()
return self
def __exit__(self, exc_type, exc_value, traceback):
"""Close the file when exiting the context manager, and deletes the file if
'delete_after_use' was set."""
self.close()
if self.delete_after_use and self.path.exists():
self.path.unlink()
def process_init_data(self, data: Dict[str, Any]) -> Dict[str, Any]:
# Get the field default values (attribute defaults)
attr_def_vals = {
field_name: field.default
for field_name, field in self.__class__.__fields__.items()
}
# Set the default values for the attributes, if no value was passed
for key, value in attr_def_vals.items():
if data.get(key) is None:
data[key] = value
# Do replacements
if (
data.get("label") == InMemoryController.__fields__["label"].get_default()
or data.get("label")
== LocalFileController.__fields__["label"].get_default()
or data.get("label") == WikiFileController.__fields__["label"].get_default()
):
# Make sure that the label is not set to the default value, it will be
# set by the source file controller
del data["label"]
if data.get("cred_filepath") is None:
# Take cred_filepath from environment variables
if os.getenv("OSW_CRED_FILEPATH") is not None:
data["cred_filepath"] = os.getenv("OSW_CRED_FILEPATH")
elif os.getenv("OSL_CRED_FILEPATH") is not None:
data["cred_filepath"] = os.getenv("OSL_CRED_FILEPATH")
# Fallback to the package default (based on CWD)
else:
data["cred_filepath"] = default_paths.cred_filepath
if not isinstance(data.get("cred_filepath"), Path):
data["cred_filepath"] = Path(data.get("cred_filepath"))
if not data.get("cred_filepath").parent.exists():
data["cred_filepath"].parent.mkdir(parents=True)
if data.get("cred_mngr") is None:
data["cred_mngr"] = CredentialManager()
if data.get("cred_filepath").exists():
data["cred_mngr"] = CredentialManager(
cred_filepath=data.get("cred_filepath")
)
return data
class DownloadFileResult(FileResult, LocalFileController):
"""A specific result object to describe the result of downloading a file from an
OSL instance."""
url_or_title: str
"""The URL or full page title of the WikiFile page to download the file from."""
target_dir: Optional[Union[str, Path]] = None
"""The target directory to download the file to. If None, the current working
will be used."""
target_fn: Optional[str] = None
"""The target filename to save the file as. If None, the filename will be taken
from the URL or title."""
target_fp: Optional[Union[str, Path]] = None
"""The target filepath to save the file to. If None, the file will be saved to the
target directory with the target filename."""
overwrite: bool = False
"""If True, the file will be overwritten if it already exists. If False, the file
will not be downloaded if it already exists."""
use_cached: bool = False
"""If True, the file will be reloaded from the cache. If False, the file will be
reloaded from the server. This option is useful if you are debugging code and
don't want to reload the file from the server every time."""
class Config:
arbitrary_types_allowed = True
def __init__(self, url_or_title, **data):
"""The constructor for the context manager."""
# Note: in pydantic v2 we can use
# https://docs.pydantic.dev/latest/api/base_model/#pydantic.BaseModel.model_post_init
data["url_or_title"] = url_or_title
# Do replacements
data = self.process_init_data(data)
if data.get("target_fn") is None:
data["target_fn"] = url_or_title.split("File:")[-1]
if data.get("target_dir") is None:
# Take target_dir from environment variables
if os.getenv("OSW_DOWNLOAD_DIR") is not None:
data["target_dir"] = os.getenv("OSW_DOWNLOAD_DIR")
elif os.getenv("OSL_DOWNLOAD_DIR") is not None:
data["target_dir"] = os.getenv("OSL_DOWNLOAD_DIR")
# Fallback to the package default (based on CWD)
else:
data["target_dir"] = default_paths.download_dir
if isinstance(data.get("target_dir"), str):
data["target_dir"] = Path(data.get("target_dir"))
if data.get("target_fp") is None:
data["target_fp"] = data.get("target_dir") / data.get("target_fn")
# Setting path after all operations on target_fp are complete
data["path"] = data.get("target_fp")
if data.get("domain") is None:
match = re.search(
pattern=r"(?:(?:https?:\/\/)?(?:www\.)?([^\/]+))\/wiki",
string=url_or_title,
)
if match is None:
if os.getenv("OSW_DOMAIN") is not None:
data["domain"] = os.getenv("OSW_DOMAIN")
elif os.getenv("OSL_DOMAIN") is not None:
data["domain"] = os.getenv("OSL_DOMAIN")
else:
raise ValueError(
f"Could not parse domain from URL: {url_or_title}. "
f"Either specify URL or domain and full page title."
)
else:
data["domain"] = match.group(1)
if data.get("use_cached") and data.get("target_fp").exists():
# Here, no file needs to be downloaded, but self need to be initialized
# with the path to the file
# Clean data dict to avoid passing None values
data = {key: value for key, value in data.items() if value is not None}
super().__init__(**data) # data includes "path"
else:
if data.get("osw_express") is None:
data["osw_express"] = OswExpress(
domain=data.get("domain"),
cred_mngr=data.get("cred_mngr"),
)
title: str = "File:" + url_or_title.split("File:")[-1]
file = data.get("osw_express").load_entity(title)
wf: WikiFileController = file.cast(
WikiFileController, osw=data.get("osw_express")
)
"""The file controller"""
if data.get("target_fp").exists() and not data.get("overwrite"):
raise FileExistsError(
f"File already exists: {data.get('target_fp')}. Set "
f"overwrite_existing=True to overwrite."
)
if not data.get("target_fp").parent.exists():
data.get("target_fp").parent.mkdir(parents=True)
lf = LocalFileController.from_other(wf, path=data.get("target_fp"))
# Clean data dict to avoid passing None values
data = {key: value for key, value in data.items() if value is not None}
super().__init__(**{**lf.dict(), **data})
self.put_from(wf)
# File is only opened at request to avoid locking the file
def osw_download_file(
url_or_title: str,
mode: str = "r",
delete_after_use: bool = False,
target_dir: Optional[Union[str, Path]] = None,
target_fn: Optional[str] = None,
target_fp: Optional[Union[str, Path]] = None,
osw_express: Optional[OswExpress] = None,
domain: Optional[str] = None,
cred_filepath: Optional[Union[str, Path]] = None,
cred_mngr: Optional[CredentialManager] = None,
overwrite: bool = False,
use_cached: bool = False,
) -> DownloadFileResult:
"""Download a file from a URL to a target directory.
Parameters
----------
url_or_title
The URL or full page title of the WikiFile page to download.
mode
The mode to open the file in. Default is 'r'. Implements the built-in open.
delete_after_use
If True, the file will be deleted after use.
target_dir
The target directory to download the file to. If None, the current working
will be used.
target_fn
The target filename to save the file as. If None, the filename will be taken
from the URL or title.
target_fp
The target filepath to save the file to. If None, the file will be saved to the
target directory with the target filename.
osw_express
An OswExpress object. If None, a new OswExpress object will be created using
the credentials_manager or domain and cred_filepath.
domain
The domain of the OSL instance to download the file from. Required if
urL_or_title is a full page title and the domain is not set in the environment
variable 'OSW_DOMAIN'. If None the domain is parsed from the URL.
cred_filepath
The filepath to the credentials file. Will only be used if cred_mngr is
None. If cred_filepath is None, the default credentials file
'accounts.pwd.yaml' in the current working directory (project root) is read
if present.
cred_mngr
A credential manager object. If None, a new credential manager will be created
using the cred_filepath.
overwrite
If True, the file will be overwritten if it already exists. If False, the file
will not be downloaded if it already exists.
use_cached
If True, the file will be reloaded from the cache. If False, the file will be
reloaded from the server. This option is useful if you are debugging code and
don't want to reload the file from the server every time.
Returns
-------
result
A specific result object.
"""
return DownloadFileResult(
url_or_title=url_or_title,
mode=mode,
delete_after_use=delete_after_use,
target_dir=target_dir,
target_fn=target_fn,
target_fp=target_fp,
osw_express=osw_express,
domain=domain,
cred_filepath=cred_filepath,
cred_mngr=cred_mngr,
overwrite=overwrite,
use_cached=use_cached,
)
class UploadFileResult(FileResult, WikiFileController):
"""A specific result object to describe the result of uploading a file to an
OSL instance."""
source: Union[LocalFileController, WikiFileController, str, Path, IO]
"""The source of the file to be uploaded."""
url_or_title: Optional[str] = None
"""The URL or full page title of the WikiFile page to upload the file to. Used to
overwrite autogenerated full page title on the target domain."""
target_fpt: Optional[str] = None
"""The full page title of the WikiFile page to upload the file to. If None, the
title will be taken from the URL or title."""
source_file_controller: Optional[FileController] = None
"""The source file controller to upload the file from. Can be a LocalFileController
or a WikiFileController."""
overwrite: OVERWRITE_CLASS_OPTIONS = OverwriteOptions.true
"""If True, the file will be overwritten if it already exists. If False, the file
will not be uploaded if it already exists. See osw.core for more information."""
change_id: Optional[str] = None
"""The change ID of the WikiFile page to upload the file to, stored in the meta
property."""
class Config:
arbitrary_types_allowed = True
def __init__(
self,
source: Union[LocalFileController, WikiFileController, str, Path, IO],
**data: Dict[str, Any],
):
# Do replacements on all data (including the those values taken from the
# default values of the class definition)
data = self.process_init_data(data)
# If a source_file_controller was provided, the 'path' should already be set.
if isinstance(source, LocalFileController) or isinstance(
source, WikiFileController
):
data["source_file_controller"] = source
data["source"] = source
elif isinstance(source, str) or isinstance(source, Path):
if not Path(source).exists():
raise FileNotFoundError(f"File not found: {source}")
if Path(source).is_dir():
raise IsADirectoryError(f"File expected. Path is a directory: {source}")
# source is a path to a file that exists - might also be a file in the CWD
data["path"] = Path(source)
data["source"] = Path(source)
data["source_file_controller"] = LocalFileController(path=data.get("path"))
# duck-typed: typing.IO is not runtime-checkable, isinstance(BytesIO(), IO)
# is False, so an explicit isinstance check would never match a stream
elif hasattr(source, "read"):
data["source_file_controller"] = InMemoryController(stream=source)
else:
raise ValueError(
"The 'source' argument must be a LocalFileController, WikiFileController,"
" str, Path or IO object."
)
# If url_or_title is given, it either
# * contains a valid domain, which can be used in osw_express
# * contains a full page title, which is the target page
if data.get("url_or_title") is not None:
# Check right format of the title
if "File:" not in data.get("url_or_title"):
raise ValueError(
"The 'url_or_title' should contain the namespace 'File'."
f"'{data.get('url_or_title')}' does not."
)
# Get the full page title from the URL or title
fpt = "File:" + data.get("url_or_title").split("File:")[-1]
if data.get("target_fpt") is None:
data["target_fpt"] = fpt
else:
if not data.get("target_fpt") == fpt:
raise ValueError(
"The full page title specified in 'url_or_title' does not "
"match the full page title specified in 'target_fpt'."
)
# Get the domain from the URL or title
match = re.search(
pattern=r"(?:(?:https?:\/\/)?(?:www\.)?([^\/]+))\/wiki",
string=data.get("url_or_title"),
)
if data.get("domain") is None:
if match is not None:
data["domain"] = match.group(1)
else:
data["domain"] = os.getenv("OSW_DOMAIN") or os.getenv("OSL_DOMAIN")
else:
if match is not None:
if not data.get("domain") == match.group(1):
raise ValueError(
"The domain specified in 'url_or_title' does not match the "
"domain specified in 'domain'."
)
# If the full page title is given, the domain must be given as well or
# osw_express must be given
if data.get("domain") is None and data.get("osw_express") is None:
raise ValueError(
"If 'url_or_title' is a full page title, 'domain' or "
"'osw_express' must be specified."
)
# Create an osw_express object if not given
if data.get("osw_express") is None:
create_new = True
# Try to get the osw_express object from the source_file_controller
if data.get("source_file_controller") is not None:
if hasattr(data["source_file_controller"], "osw_express"):
create_new = False
data["osw_express"] = data["source_file_controller"].osw_express
# Otherwise create a new osw_express object
if create_new:
data["osw_express"] = OswExpress(
domain=data.get("domain"),
cred_mngr=data.get("cred_mngr"),
)
# If no change_id is given, generate a new one
if data.get("change_id") is None:
data["change_id"] = str(uuid4())
# Change_id will be returned in the UploadFileResult.change_id attribute
# If given set titel and namespace
if data.get("target_fpt") is not None:
namespace = namespace_from_full_title(data.get("target_fpt"))
title = title_from_full_title(data.get("target_fpt"))
wiki_page = model.WikiPage(namespace=namespace, title=title)
data["meta"] = model.Meta(wiki_page=wiki_page)
data["title"] = title
# Set change_id to existing meta
for meta in [data.get("meta"), data["source_file_controller"].meta]:
if meta is not None:
if getattr(meta, "change_id", None) is None:
meta.change_id = [data["change_id"]]
else:
meta.change_id.append(data.get("change_id"))
# Clean data dict
data = {key: value for key, value in data.items() if value is not None}
# Create the WikiFileController from the source_file_controller
wfc = WikiFileController.from_other(
other=data.get("source_file_controller"),
osw=data.get("osw_express"),
**data,
# Passes arguments to the cast() method, e.g., overwrite the label
# cast method will call init
)
# Upload to the target OSW instance
wfc.put_from(data.get("source_file_controller"), **data)
data["url_or_title"] = wfc.url
super().__init__(**{**wfc.dict(), **data})
# Don't open the local (uploaded) file
def osw_upload_file(
source: Union[LocalFileController, WikiFileController, str, Path, IO],
url_or_title: Optional[str] = None,
overwrite: OVERWRITE_CLASS_OPTIONS = OverwriteOptions.true,
delete_after_use: bool = False,
osw_express: Optional[OswExpress] = None,
domain: str = None,
cred_filepath: Optional[Union[str, Path]] = None,
cred_mngr: Optional[CredentialManager] = None,
label: Optional[List[model.Label]] = None,
name: Optional[str] = None,
description: Optional[List[model.Description]] = None,
change_id: Optional[str] = None,
**properties: Dict[str, Any],
) -> UploadFileResult:
"""Upload a file to an OSL page.
Parameters
----------
source
The source file to upload. Can be a LocalFileController, WikiFileController,
str, Path or an open file-like object.
url_or_title
The URL or full page title of the WikiFile page to upload the file to. Used to
overwrite autogenerated full page title on the target domain. If it is
included, the domain can be parsed from the URL.
overwrite
If True, the file will be overwritten if it already exists. If False, the file
will not be downloaded if it already exists.
delete_after_use
If True, the file will be deleted after use.
osw_express
An OswExpress object. If None, a new OswExpress object will be created using
the credentials_manager, or domain and cred_filepath.
domain
The domain of the OSL instance to upload the file to. Required if no
OswExpress was pass to osw_express. If None the domain can be parsed from
the URL. If fpt_or_url is no URL user must specify the domain.
cred_filepath
The filepath to the credentials file. Will only be used if cred_mngr is
None. If cred_filepath is None, the default credentials file
'accounts.pwd.yaml' in the current working directory (project root) is read
if present.
cred_mngr
A credential manager object. If None, a new credential manager will be created
using the cred_filepath.
label
The labels to set on the WikiFile data model prior to uploading it to the OSL
instance. The labels will end up in the JSON data slot of the to be created
WikiFile page.
name
The name of the file. If None, the name will be taken from the source file.
description
The description to set on the WikiFile data model prior to uploading it to the
OSL instance.
change_id
The change ID of the WikiFile page to upload the file to, stored in the meta
property.
properties
The properties to set on the WikiFile data model prior to uploading it to
the OSL instance. Properties listed here, won't overwrite properties handed
over in this function call, e.g., labe, name, description.
Properties will end up in the JSON data slot of the to be created WikiFile
page, if they match the WikiFile data model.
Returns
-------
result
A specific result object.
"""
# Preparing all args, kwargs & properties to set for the uploaded file
data = {**locals(), **properties}
# Clean data dict to avoid passing None values
data = {key: value for key, value in data.items() if value is not None}
# Initialize the UploadFileResult object
return UploadFileResult(**data)
class DataModel(OswBaseModel):
module: str
"""The full address of the module to import from, e.g., 'osw.model.entity'."""
class_name: str
"""The target class name to import, e.g., 'Entity' in 'from osw.model.entity
import Entity'."""
osw_fpt: str = None
"""Full page title of the data model in an OSL instance to be fetched if not
already available in osw.model.entity, e.g., 'Category:Entity'."""
def import_with_fallback(
to_import: Union[List[DataModel], Dict[str, str]],
caller_globals: dict,
module: str = None,
dependencies: Dict[str, str] = None,
domain: str = None,
cred_filepath: Union[str, Path] = None,
osw_express: OswExpress = None,
):
# todo: rework this function to work as OswExpress -
"""Imports data models with a fallback to fetch the dependencies from an OSL
instance if the data models are not available in the local osw.model.entity module.
DISCLAIMER: Will prompt the user to enter domain and credentials if no domain and
cred_filepath or osw_express was passed.
BEST USAGE: See examples, for best usage in a script to be rerun multiple times
(without refetching datamodels).
Parameters
----------
to_import
List of DataModel objects or a dictionary, with (key: value) pairs (class_name:
osw_fpt) to import.
caller_globals
(Mandatory!) The globals dictionary as returned by globals() in the calling
script.
module
(Optional) The module to import the data models from. Used only if to_import
is of type List[Dict]. Defaults to 'osw.model.entity' if not specified.
dependencies
A dictionary with the keys being the names of the dependencies and the values
being the full page name of the dependencies.
domain
The domain of the OSL instance to connect to, if the dependencies are not
available in the local osw.model.entity module.
cred_filepath
The filepath to the credentials file. If None, a credentials file is created
under the default path.
osw_express
An OswExpress object. If None, a new OswExpress object will be created using
the credentials_manager or domain and cred_filepath.
Examples
--------
>>> from pathlib import Path
>>> from osw.defaults import paths, params
>>> from osw.express import import_with_fallback, DataModel, OswExpress
>>> wiki_domain__ = "wiki-dev.open-semantic-lab.org"
>>> cred_filepath__ = Path(r"accounts.pwd.yaml")
>>> osw_obj = OswExpress(domain=wiki_domain__, cred_filepath=cred_filepath__)
>>> import_with_fallback(
>>> [DataModel(module="osw.controller.file.base",class_name="FileController")],
>>> globals(), osw_express=osw_obj
>>> )
Returns
-------
None
"""
if isinstance(to_import, dict):
# The arg 'to_import' should have the right structure to act as 'dependencies'
# Assume all DataModels are part of osw.model.entity
if module is None:
module = "osw.model.entity"
# A list of DataModels will be derived from the dict
to_import = [
DataModel(
module=module,
class_name=key,
osw_fpt=value,
)
for key, value in to_import.items()
]
if to_import is None:
raise ValueError(
"Either the argument 'to_import' or 'dependencies' must be passed to the "
"function import_with_fallback()."
)
try:
# Try to import the listed data models from the (listed) module(s)
for ti in to_import:
# Raises AttributeError if the target could not be found
caller_globals[ti.class_name] = getattr(
importlib.import_module(ti.module), ti.class_name
)
except Exception as e:
if dependencies is None:
dependencies = {}
warn(
"No 'dependencies' were passed to the function "
"import_with_fallback()! Trying to derive them from 'to_import'."
)
new_dependencies = {
module.class_name: module.osw_fpt
for module in to_import
if module.osw_fpt is not None
}
dependencies.update(new_dependencies)
if not dependencies:
raise AttributeError(
f"An exception occurred while loading the module dependencies: \n"
f'"{e}"\n'
"No 'dependencies' were passed to the function import_with_fallback() "
"and could not be derived from 'to_import'!"
)
warn(
f"An exception occurred while loading the module dependencies: \n"
f'"{e}"\n'
"A connection to an OSW instance, to fetch the dependencies from, "
"has to be established!"
)
# todo: should this be taken from globals?
# If the user has set the default domain, use it
if domain is None:
if os.getenv("OSW_DOMAIN") is not None:
domain = os.getenv("OSW_DOMAIN")
elif os.getenv("OSL_DOMAIN") is not None:
domain = os.getenv("OSL_DOMAIN")
elif default_params.has_changed("wiki_domain"):
domain = default_params.wiki_domain
else:
domain = input(
"Please enter the domain of the OSW instance to connect to:"
)
if domain == "":
domain = default_params.wiki_domain
if cred_filepath is None:
if os.getenv("OSW_CRED_FILEPATH") is not None:
cred_filepath = os.getenv("OSW_CRED_FILEPATH")
elif os.getenv("OSL_CRED_FILEPATH") is not None:
cred_filepath = os.getenv("OSL_CRED_FILEPATH")
else:
cred_filepath = default_paths.cred_filepath
if osw_express is None:
osw_express = OswExpress(domain=domain, cred_filepath=cred_filepath)
osw_express.install_dependencies(dependencies, mode="append")
osw_express.shut_down() # Avoiding connection error
# Try again to import the data models
for ti in to_import:
# Raises AttributeError if the target could not be found
caller_globals[ti.class_name] = getattr(
importlib.import_module(ti.module), ti.class_name
)
OswExpress.update_forward_refs()
# todo:
# Ideas:
# * New express function:
# * Move gui.save_as_page_package and wrap in osw.express
# * parallel download of multiple files
# * query all instances of a category (direct members or member of subcategories
# as well = crawl)
# * Get query results into a pandas.DataFrame
# * upload any file to wiki whilst specifying the page to attach it to + select a
# property to link it to [basic example at file_upload_download.py]
# inputs:
# source:Union[LocalFileController, WikiFileController, str, Path],
# target_fpt_or_uri: Optional[str]
# properties / Links: Optional[str]
# commit message
# * Save a pandas.DataFrame to a WikiFile (as table, e.g. as csv, xlsx,
# json)
# * Save a wiki page as pdf