Merge pull request #43 from dwhswenson/sim-setup

`compile` command: Simulation setup with YAML or JSON

Author: dwhswenson

docs/compiling.rst

@@ -0,0 +1,43 @@
+
+Compiling
+=========
+
+Let's start with an overview of terminology.:
+
+* **Compiling**: The process of converting input in a text format,
+  such as YAML or JSON, into OPS objects. Here we're focused on non-Python
+  alternatives to using the standard Python interpreter to compile objects.
+* **Category**: The base category of object to be created (engine, CV,
+  volume, etc.)
+* **Builder**: We will refer to a builder function, which creates an
+  instance of a specific 
+
+Everything is created with plugins. There are two types of plugins used in
+the ``compiling`` subpackage:
+
+* ``InstanceCompilerPlugin``: This is what you'll normally work with. These
+  convert the input text to an instance of a specific OPS object (for
+  example, an ``OpenMMEngine`` or an ``MDTrajFunctionCV``. In general, you
+  do not create subclasses of ``InstanceCompilerPlugin`` -- there are
+  subclasses specialized to engines (``EngineCompilerPlugin``), to CVs
+  (``CVCompilerPlugin``), etc. You create *instances* of those subclasses.
+  You write your builder function, and wrap in with in an instance of an
+  ``InstanceCompilerPlugin``.
+* ``CategoryPlugin``: These manage the plugins associated with a given
+  features.  Contributors will almost never need to create one of these.
+  The only case in which you would need to create one of these is if you're
+  creating a new *category* of object, i.e., something like an engine where
+  users will have multiple options at the command line.
+
+Other useful classes defined in the ``compiling`` subpackage include:
+
+* ``Builder``: The ``Builder`` class is a convenience for creating builder
+  functions. It takes either a callable or a string as input, where a string
+  is treated as a path to an object to import at runtime. It also allows
+  takes parameters ``remapper`` and ``after_build``, which are callables
+  that act on the input dictionary before object creation (``remapper``) and
+  on the created object after object creations (``after_build``).
+* ``CategoryCompiler``: This class manages plugins for a given category, as
+  well as tracking named objects of that type. These are created
+  automatically when plugins are registered; users do not need to create
+  these.

paths_cli/cli.py

@@ -15,6 +15,7 @@
 
 from .plugin_management import (FilePluginLoader, NamespacePluginLoader,
                                 OPSCommandPlugin)
+from .utils import get_installed_plugins
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
@@ -26,20 +27,10 @@ class OpenPathSamplingCLI(click.MultiCommand):
     def __init__(self, *args, **kwargs):
         # the logic here is all about loading the plugins
         commands = str(pathlib.Path(__file__).parent.resolve() / 'commands')
-        def app_dir_plugins(posix):
-            return str(pathlib.Path(
-                click.get_app_dir("OpenPathSampling", force_posix=posix)
-            ).resolve() / 'cli-plugins')
-
-        self.plugin_loaders = [
-            FilePluginLoader(commands, OPSCommandPlugin),
-            FilePluginLoader(app_dir_plugins(posix=False), OPSCommandPlugin),
-            FilePluginLoader(app_dir_plugins(posix=True), OPSCommandPlugin),
-            NamespacePluginLoader('paths_cli_plugins', OPSCommandPlugin)
-        ]
-
-        plugins = sum([loader() for loader in self.plugin_loaders], [])
-
+        plugins = get_installed_plugins(
+            default_loader=FilePluginLoader(commands, OPSCommandPlugin),
+            plugin_types=OPSCommandPlugin
+        )
         self._get_command = {}
         self._sections = collections.defaultdict(list)
         self.plugins = []

paths_cli/commands/compile.py

@@ -0,0 +1,89 @@
+import click
+
+from paths_cli.compiling.root_compiler import do_compile, register_plugins
+from paths_cli.parameters import OUTPUT_FILE
+from paths_cli.errors import MissingIntegrationError
+from paths_cli import OPSCommandPlugin
+from paths_cli.compiling.plugins import (
+    CategoryPlugin, InstanceCompilerPlugin
+)
+from paths_cli.plugin_management import NamespacePluginLoader
+import importlib
+from paths_cli.utils import get_installed_plugins
+from paths_cli.commands.contents import report_all_tables
+
+# this is just to handle a nicer error
+def import_module(module_name, format_type=None, install=None):
+    try:
+        mod = importlib.import_module(module_name)
+    except ImportError:
+        if format_type is None:
+            format_type = module_name
+
+        msg = f"Unable to find a compiler for {format_type} on your system."
+        if install is not None:
+            msg += f" Please install {install} to use this format."
+
+        raise MissingIntegrationError(msg)
+    return mod
+
+def load_yaml(f):
+    yaml = import_module('yaml', format_type="YAML", install="PyYAML")
+    return yaml.load(f.read(), Loader=yaml.FullLoader)
+
+def load_json(f):
+    json = import_module('json')  # this should never fail... std lib!
+    return json.loads(f.read())
+
+# This is why we can't use TOML:
+# https://github.com/toml-lang/toml/issues/553#issuecomment-444814690
+# Conflicts with rules preventing mixed types in arrays. This seems to have
+# relaxed in TOML 1.0, but the toml package doesn't support the 1.0
+# standard. We'll add toml in once the package supports the standard.
+
+EXTENSIONS = {
+    'yaml': load_yaml,
+    'yml': load_yaml,
+    'json': load_json,
+    'jsn': load_json,
+    # 'toml': load_toml,
+}
+
+def select_loader(filename):
+    ext = filename.split('.')[-1]
+    try:
+        return EXTENSIONS[ext]
+    except KeyError:
+        raise RuntimeError(f"Unknown file extension: {ext}")
+
+@click.command(
+    'compile',
+)
+@click.argument('input_file')
+@OUTPUT_FILE.clicked(required=True)
+def compile_(input_file, output_file):
+    loader = select_loader(input_file)
+    with open(input_file, mode='r') as f:
+        dct = loader(f)
+
+    plugin_types = (InstanceCompilerPlugin, CategoryPlugin)
+    plugins = get_installed_plugins(
+        default_loader=NamespacePluginLoader('paths_cli.compiling',
+                                             plugin_types),
+        plugin_types=plugin_types
+    )
+    register_plugins(plugins)
+
+    objs = do_compile(dct)
+    print(f"Saving {len(objs)} user-specified objects to {output_file}....")
+    storage = OUTPUT_FILE.get(output_file)
+    storage.save(objs)
+    report_all_tables(storage)
+
+
+PLUGIN = OPSCommandPlugin(
+    command=compile_,
+    section="Debug",
+    requires_ops=(1, 0),
+    requires_cli=(0, 3)
+)

paths_cli/commands/contents.py

@@ -20,6 +20,9 @@
     'Snapshots': 'snapshots'
 }
 
+import logging
+logger = logging.getLogger(__name__)
+
 @click.command(
     'contents',
     short_help="list named objects from an OPS storage file",
@@ -49,6 +52,7 @@ def contents(input_file, table):
 
 def get_section_string(label, store):
     attr = NAME_TO_ATTR.get(label, label.lower())
+    logger.debug(f"Working on {attr}")
     if attr in UNNAMED_SECTIONS:
         string = get_unnamed_section_string(label, store)
     elif attr in ['tag', 'tags']:

paths_cli/compat/openmm.py

@@ -0,0 +1,18 @@
+# should be able to remove this try block when we drop OpenMM < 7.6
+try:
+    import openmm as mm
+    from openmm import unit
+except ImportError:
+    try:
+        from simtk import openmm as mm
+        from simtk import unit  # -no-cov-
+    except ImportError:
+        HAS_OPENMM = False
+        mm = None
+        unit = None
+    else: # -no-cov-
+        HAS_OPENMM = True
+else:
+    HAS_OPENMM = True
+
+

paths_cli/compiling/__init__.py

@@ -0,0 +1,12 @@
+from .core import Parameter, Builder, InstanceCompilerPlugin
+from .root_compiler import (clean_input_key, compiler_for, register_plugins,
+                            do_compile)
+
+from .plugins import CategoryPlugin
+from .errors import InputError
+
+# OPS-specific
+from .plugins import (
+    EngineCompilerPlugin, CVCompilerPlugin, VolumeCompilerPlugin,
+    NetworkCompilerPlugin, SchemeCompilerPlugin, StrategyCompilerPlugin,
+)