Skip to content

Commit b64ea30

Browse files
committed
major refactor of root_paser
also steps toward the compile smoke test
1 parent f7cc88c commit b64ea30

12 files changed

Lines changed: 221 additions & 80 deletions

File tree

‎paths_cli/commands/compile.py‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
import click
22

3-
from paths_cli.parsing.root_parser import parse
3+
from paths_cli.parsing.root_parser import parse, register_plugins
44
from paths_cli.parameters import OUTPUT_FILE
55
from paths_cli.errors import MissingIntegrationError
66
from paths_cli import OPSCommandPlugin
7+
from paths_cli.parsing.plugins import ParserPlugin, InstanceBuilder
8+
from paths_cli.plugin_management import (
9+
NamespacePluginLoader, FilePluginLoader
10+
)
711
import importlib
12+
from paths_cli.utils import app_dir_plugins
813

914
def import_module(module_name, format_type=None, install=None):
1015
try:
@@ -28,13 +33,11 @@ def load_json(f):
2833
json = import_module('json') # this should never fail... std lib!
2934
return json.loads(f.read())
3035

31-
# TODO: It looks like TOML was working with my test dict -- I'll have to see
32-
# if that's an issue with the TOML format or just something weird. I thought
33-
# TOML was equally as flexible (in theory) as JSON.
34-
# def load_toml(f):
35-
# toml = import_module('toml', format_type="TOML", install="toml")
36-
# return toml.loads(f.read())
37-
36+
# This is why we can't use TOML:
37+
# https://github.com/toml-lang/toml/issues/553#issuecomment-444814690
38+
# Conflicts with rules preventing mixed types in arrays. This seems to have
39+
# relaxed in TOML 1.0, but the toml package doesn't support the 1.0
40+
# standard. We'll add toml in once the pacakge supports the standard.
3841

3942
EXTENSIONS = {
4043
'yaml': load_yaml,
@@ -51,6 +54,17 @@ def select_loader(filename):
5154
except KeyError:
5255
raise RuntimeError(f"Unknown file extension: {ext}")
5356

57+
def load_plugins():
58+
plugin_types = (InstanceBuilder, ParserPlugin)
59+
plugin_loaders = [
60+
NamespacePluginLoader('paths_cli.parsing', plugin_types),
61+
FilePluginLoader(app_dir_plugins(posix=False), plugin_types),
62+
FilePluginLoader(app_dir_plugins(posix=True), plugin_types),
63+
NamespacePluginLoader('paths_cli_plugins', plugin_types)
64+
]
65+
plugins = sum([loader() for loader in plugin_loaders], [])
66+
return plugins
67+
5468
@click.command(
5569
'compile',
5670
)
@@ -61,8 +75,11 @@ def compile_(input_file, output_file):
6175
with open(input_file, mode='r') as f:
6276
dct = loader(f)
6377

78+
plugins = load_plugins()
79+
register_plugins(plugins)
80+
6481
objs = parse(dct)
65-
# print(objs)
82+
print(objs)
6683
storage = OUTPUT_FILE.get(output_file)
6784
storage.save(objs)
6885

‎paths_cli/parsing/engines.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,4 @@ def openmm_options(dct):
5353
name='openmm',
5454
)
5555

56-
TYPE_MAPPING = {
57-
'openmm': OPENMM_PLUGIN,
58-
}
59-
6056
ENGINE_PARSER = ParserPlugin(EngineParserPlugin, aliases=['engines'])

‎paths_cli/parsing/networks.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ def tis_trans_info(dct):
4242
'interfaces': interface_set}]
4343
return mistis_trans_info(dct)
4444

45-
build_tps_network = NetworkParserPlugin(
45+
TPS_NETWORK_PLUGIN = NetworkParserPlugin(
4646
builder=Builder('openpathsampling.TPSNetwork'),
4747
parameters=[
4848
Parameter('initial_states', parser_for('volume'),
@@ -53,16 +53,21 @@ def tis_trans_info(dct):
5353
name='tps'
5454
)
5555

56-
build_mistis_network = NetworkParserPlugin(
56+
build_tps_network = TPS_NETWORK_PLUGIN
57+
58+
MISTIS_NETWORK_PLUGIN = NetworkParserPlugin(
5759
parameters=[Parameter('trans_info', mistis_trans_info)],
5860
builder=Builder('openpathsampling.MISTISNetwork'),
5961
name='mistis'
6062
)
63+
build_mistis_network = MISTIS_NETWORK_PLUGIN
6164

62-
build_tis_network = NetworkParserPlugin(
65+
TIS_NETWORK_PLUGIN = NetworkParserPlugin(
6366
builder=Builder('openpathsampling.MISTISNetwork'),
6467
parameters=[Parameter('trans_info', tis_trans_info)],
6568
name='tis'
6669
)
70+
build_tis_network = TIS_NETWORK_PLUGIN
71+
6772

6873
NETWORK_PARSER = ParserPlugin(NetworkParserPlugin, aliases=['networks'])

‎paths_cli/parsing/root_parser.py‎

Lines changed: 102 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,14 @@
11
from paths_cli.parsing.core import Parser, InstanceBuilder
22
from paths_cli.parsing.plugins import ParserPlugin
33

4+
import logging
5+
logger = logging.getLogger(__name__)
46

7+
class ParserRegistrationError(Exception):
8+
pass
9+
10+
11+
# TODO: I think this is the only OPS-specific thing in here
512
_DEFAULT_PARSE_ORDER = [
613
'engine',
714
'cv',
@@ -13,16 +20,80 @@
1320

1421
PARSE_ORDER = _DEFAULT_PARSE_ORDER.copy()
1522

16-
PARSERS = {}
17-
ALIASES = {}
23+
def clean_input_key(key):
24+
# TODO: move this to core
25+
"""
26+
Canonical approach is to treat everything as lowercase with underscore
27+
separators. This will make everything lowercase, and convert
28+
whitespace/hyphens to underscores.
29+
"""
30+
key = key.lower()
31+
key = "_".join(key.split()) # whitespace to underscore
32+
key.replace("-", "_")
33+
return key
34+
35+
### Managing known parsers and aliases to the known parsers ################
36+
37+
_PARSERS = {} # mapping: {canonical_name: Parser}
38+
_ALIASES = {} # mapping: {alias: canonical_name}
39+
# NOTE: _ALIASES does *not* include self-mapping of the canonical names
40+
41+
def _canonical_name(alias):
42+
"""Take an alias or a parser name and return the parser name
43+
44+
This also cleans user input (using the canonical form generated by
45+
:meth:`.clean_input_key`).
46+
"""
47+
alias = clean_input_key(alias)
48+
alias_to_canonical = _ALIASES.copy()
49+
alias_to_canonical.update({pname: pname for pname in _PARSERS})
50+
return alias_to_canonical.get(alias, None)
51+
52+
def _get_parser(parser_name):
53+
"""
54+
_get_parser must only be used after the parsers have been registered. It
55+
will automatically create a parser for any unknown ``parser_name.``
56+
"""
57+
canonical_name = _canonical_name(parser_name)
58+
# create a new parser if none exists
59+
if canonical_name is None:
60+
canonical_name = parser_name
61+
_PARSERS[parser_name] = Parser(None, parser_name)
62+
return _PARSERS[canonical_name]
63+
64+
def _register_parser_plugin(plugin):
65+
DUPLICATE_ERROR = RuntimeError(f"The name {plugin.name} has been "
66+
"reserved by another parser")
67+
if plugin.name in _PARSERS:
68+
raise DUPLICATE_ERROR
69+
70+
parser = _get_parser(plugin.name)
71+
72+
# register aliases
73+
new_aliases = set(plugin.aliases) - set([plugin.name])
74+
for alias in new_aliases:
75+
if alias in _PARSERS or alias in _ALIASES:
76+
raise DUPLICATE_ERROR
77+
_ALIASES[alias] = plugin.name
78+
1879

19-
class ParserProxy:
80+
### Handling delayed loading of parsers ####################################
81+
#
82+
# Many objects need to use parsers to create their input parameters. In
83+
# order for them to be able to access dynamically-loaded plugins, we delay
84+
# the loading of the parser by using a proxy object.
85+
86+
class _ParserProxy:
2087
def __init__(self, parser_name):
2188
self.parser_name = parser_name
2289

2390
@property
2491
def _proxy(self):
25-
return PARSERS[self.parser_name]
92+
canonical_name = _canonical_name(self.parser_name)
93+
if canonical_name is None:
94+
raise RuntimeError("No parser registered for "
95+
f"'{self.parser_name}'")
96+
return _get_parser(canonical_name)
2697

2798
@property
2899
def named_objs(self):
@@ -41,18 +112,10 @@ def parser_for(parser_name):
41112
parser_name : str
42113
the name of the parser to use
43114
"""
44-
return ParserProxy(parser_name)
115+
return _ParserProxy(parser_name)
45116

46-
def _get_parser(parser_name):
47-
"""
48-
_get_parser must only be used after the parsers have been registered. It
49-
will automatically create a parser for any unknown ``parser_name.``
50-
"""
51-
parser = PARSERS.get(parser_name, None)
52-
if parser is None:
53-
parser = Parser(None, parser_name)
54-
PARSERS[parser_name] = parser
55-
return parser
117+
118+
### Registering builder plugins and user-facing register_plugins ###########
56119

57120
def _get_registration_names(plugin):
58121
"""This is a trick to ensure that the names appear in the desired order.
@@ -70,6 +133,11 @@ def _get_registration_names(plugin):
70133
found_names.add(name)
71134
return ordered_names
72135

136+
def _register_builder_plugin(plugin):
137+
parser = _get_parser(plugin.parser_name)
138+
for name in _get_registration_names(plugin):
139+
parser.register_builder(plugin, name)
140+
73141
def register_plugins(plugins):
74142
builders = []
75143
parsers = []
@@ -85,31 +153,31 @@ def register_plugins(plugins):
85153
for plugin in builders:
86154
_register_builder_plugin(plugin)
87155

88-
def _register_builder_plugin(plugin):
89-
parser = _get_parser(plugin.parser_name)
90-
for name in _get_registration_names(plugin):
91-
parser.register_builder(plugin, name)
92-
93-
def _register_parser_plugin(plugin):
94-
DUPLICATE_ERROR = RuntimeError(f"The name {plugin.name} has been "
95-
"reserved by another parser")
96-
if plugin.name in PARSERS:
97-
raise DUPLICATE_ERROR
156+
### Performing the parsing of user input ###################################
98157

99-
parser = _get_parser(plugin.name)
158+
def _sort_user_categories(user_categories):
159+
"""Organize user input categories into parse order.
100160
101-
# register aliases
102-
new_aliases = set(plugin.aliases) - set([plugin.name])
103-
for alias in new_aliases:
104-
if alias in PARSERS or alias in ALIASES:
105-
raise DUPLICATE_ERROR
106-
ALIASES[alias] = plugin.name
161+
"Cateogories" are the first-level keys in the user input file (e.g.,
162+
'engines', 'cvs', etc.) There must be one Parser per category.
163+
"""
164+
user_to_canonical = {user_key: _canonical_name(user_key)
165+
for user_key in user_categories}
166+
sorted_keys = sorted(
167+
user_categories,
168+
key=lambda x: PARSE_ORDER.index(user_to_canonical[x])
169+
)
170+
return sorted_keys
107171

108172
def parse(dct):
173+
"""Main function for compiling user input to objects.
174+
"""
109175
objs = []
110-
for category in PARSE_ORDER:
111-
func = PARSERS[category]
176+
for category in _sort_user_categories(dct):
177+
# func = PARSERS[category]
178+
func = _get_parser(category)
112179
yaml_objs = dct.get(category, [])
180+
print(f"{yaml_objs}")
113181
new = [func(obj) for obj in yaml_objs]
114182
objs.extend(new)
115183
return objs

‎paths_cli/parsing/volumes.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ def cv_volume_build_func(**dct):
1818
# TODO: wrap this with some logging
1919
return builder(**dct)
2020

21-
build_cv_volume = VolumeParserPlugin(
21+
CV_VOLUME_PLUGIN = VolumeParserPlugin(
2222
builder=cv_volume_build_func,
2323
parameters=[
2424
Parameter('cv', parser_for('cv'),
@@ -31,13 +31,15 @@ def cv_volume_build_func(**dct):
3131
name='cv-volume',
3232
)
3333

34+
build_cv_volume = CV_VOLUME_PLUGIN
35+
3436
# jsonschema type for combination volumes
3537
VOL_ARRAY_TYPE = {
3638
'type': 'array',
3739
'items': {"$ref": "#/definitions/volume_type"}
3840
}
3941

40-
build_intersection_volume = VolumeParserPlugin(
42+
INTERSECTION_VOLUME_PLUGIN = VolumeParserPlugin(
4143
builder=lambda subvolumes: functools.reduce(operator.__and__,
4244
subvolumes),
4345
parameters=[
@@ -48,7 +50,9 @@ def cv_volume_build_func(**dct):
4850
name='intersection',
4951
)
5052

51-
build_union_volume = VolumeParserPlugin(
53+
build_intersection_volume = INTERSECTION_VOLUME_PLUGIN
54+
55+
UNION_VOLUME_PLUGIN = VolumeParserPlugin(
5256
builder=lambda subvolumes: functools.reduce(operator.__or__,
5357
subvolumes),
5458
parameters=[
@@ -59,7 +63,4 @@ def cv_volume_build_func(**dct):
5963
name='union',
6064
)
6165

62-
TYPE_MAPPING = {
63-
'cv-volume': build_cv_volume,
64-
'intersection': build_intersection_volume,
65-
}
66+
build_union_volume = UNION_VOLUME_PLUGIN

‎paths_cli/tests/commands/test_compile.py‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@ def test_select_loader_error():
5353
select_loader('foo.bar')
5454

5555
def test_compile(ad_openmm, test_data_dir):
56+
# this is a smoke test to check that, given a correct YAML file, we can
57+
# compile the yaml into a db file, and then reload (some of) the saves
58+
# objects from the db.
5659
runner = CliRunner()
5760
setup = test_data_dir / "setup.yml"
5861
shutil.copy2(str(setup), ad_openmm)
@@ -63,18 +66,29 @@ def test_compile(ad_openmm, test_data_dir):
6366
ad_openmm / "integrator.xml"]
6467
for filename in files:
6568
shutil.copy2(str(filename), cwd)
66-
pytest.skip() # TODO: right now we're not building the parsers
69+
pytest.skip() # TODO: parser aliases don't work yet
6770
result = runner.invoke(
6871
compile_,
6972
['setup.yml', '-o', str(ad_openmm / 'setup.db')]
7073
)
74+
if result.exc_info: # -no-cov-
75+
# this only runs in the event of an error (to provide a clearer
76+
# failure) (TODO: change this up a little maybe? re-raise the
77+
# error?)
78+
import traceback
79+
traceback.print_tb(result.exc_info[2])
80+
print(result.exception)
81+
print(result.exc_info)
7182
assert result.exit_code == 0
83+
print(result.output)
84+
assert os.path.exists(str(ad_openmm / 'setup.db'))
7285
from openpathsampling.experimental.storage import (
7386
Storage, monkey_patch_all)
74-
st = Storage('setup.db', mode='r')
87+
# TODO: need to do the temporary monkey patch here
88+
st = Storage(str(ad_openmm / 'setup.db'), mode='r')
7589

7690
# smoke tests that we can reload things
91+
print("Engines:", len(st.engines), [e.name for e in st.engines])
7792
engine = st.engines['engine']
7893
phi = st.cvs['phi']
7994
C_7eq = st.volumes['C_7eq']
80-
pytest.skip()

0 commit comments

Comments
 (0)