Skip to content

Commit 58520df

Browse files
authored
Merge pull request #19 from dwhswenson/namespace_plugins
Update plugin management approach
2 parents e793f1e + 6a7484b commit 58520df

8 files changed

Lines changed: 297 additions & 72 deletions

File tree

‎devtools/miniconda_install.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ conda_version="latest"
2222
#conda_version="4.4.10" # can pin a miniconda version like this, if needed
2323

2424
MINICONDA=Miniconda${pyV}-${conda_version}-${OS_ARCH}.sh
25-
MINICONDA_MD5=$(curl -s https://repo.continuum.io/miniconda/ | grep -A3 $MINICONDA | sed -n '4p' | sed -n 's/ *<td>\(.*\)<\/td> */\1/p')
25+
MINICONDA_MD5=$(curl -sL https://repo.continuum.io/miniconda/ | grep -A3 $MINICONDA | sed -n '4p' | sed -n 's/ *<td>\(.*\)<\/td> */\1/p')
2626
wget https://repo.continuum.io/miniconda/$MINICONDA
2727
SCRIPT_MD5=`eval "$MD5_CMD $MD5_OPT $MINICONDA" | cut -d ' ' -f 1`
2828

‎paths_cli/cli.py‎

Lines changed: 16 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -7,23 +7,15 @@
77
import logging
88
import logging.config
99
import os
10+
import pathlib
1011

1112
import click
1213
# import click_completion
1314
# click_completion.init()
1415

15-
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
16-
17-
_POSSIBLE_PLUGIN_FOLDERS = [
18-
os.path.join(os.path.dirname(__file__), 'commands'),
19-
os.path.join(click.get_app_dir("OpenPathSampling"), 'cli-plugins'),
20-
os.path.join(click.get_app_dir("OpenPathSampling", force_posix=True),
21-
'cli-plugins'),
22-
]
23-
24-
OPSPlugin = collections.namedtuple("OPSPlugin",
25-
['name', 'filename', 'func', 'section'])
16+
from .plugin_management import FilePluginLoader, NamespacePluginLoader
2617

18+
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
2719

2820
class OpenPathSamplingCLI(click.MultiCommand):
2921
"""Main class for the command line interface
@@ -32,13 +24,20 @@ class OpenPathSamplingCLI(click.MultiCommand):
3224
"""
3325
def __init__(self, *args, **kwargs):
3426
# the logic here is all about loading the plugins
35-
self.plugin_folders = []
36-
for folder in _POSSIBLE_PLUGIN_FOLDERS:
37-
if folder not in self.plugin_folders and os.path.exists(folder):
38-
self.plugin_folders.append(folder)
27+
commands = str(pathlib.Path(__file__).parent.resolve() / 'commands')
28+
def app_dir_plugins(posix):
29+
return str(pathlib.Path(
30+
click.get_app_dir("OpenPathSampling", force_posix=posix)
31+
).resolve() / 'cli-plugins')
32+
33+
self.plugin_loaders = [
34+
FilePluginLoader(commands),
35+
FilePluginLoader(app_dir_plugins(posix=False)),
36+
FilePluginLoader(app_dir_plugins(posix=True)),
37+
NamespacePluginLoader('paths_cli.plugins')
38+
]
3939

40-
plugin_files = self._list_plugin_files(self.plugin_folders)
41-
plugins = self._load_plugin_files(plugin_files)
40+
plugins = sum([loader() for loader in self.plugin_loaders], [])
4241

4342
self._get_command = {}
4443
self._sections = collections.defaultdict(list)
@@ -62,45 +61,6 @@ def _deregister_plugin(self, plugin):
6261
def plugin_for_command(self, command_name):
6362
return {p.name: p for p in self.plugins}[command_name]
6463

65-
@staticmethod
66-
def _list_plugin_files(plugin_folders):
67-
def is_plugin(filename):
68-
return (
69-
filename.endswith(".py") and not filename.startswith("_")
70-
and not filename.startswith(".")
71-
)
72-
73-
plugin_files = []
74-
for folder in plugin_folders:
75-
files = [os.path.join(folder, f) for f in os.listdir(folder)
76-
if is_plugin(f)]
77-
plugin_files += files
78-
return plugin_files
79-
80-
@staticmethod
81-
def _filename_to_command_name(filename):
82-
command_name = filename[:-3] # get rid of .py
83-
command_name = command_name.replace('_', '-') # commands use -
84-
return command_name
85-
86-
@staticmethod
87-
def _load_plugin(name):
88-
ns = {}
89-
with open(name) as f:
90-
code = compile(f.read(), name, 'exec')
91-
eval(code, ns, ns)
92-
return ns['CLI'], ns['SECTION']
93-
94-
def _load_plugin_files(self, plugin_files):
95-
plugins = []
96-
for full_name in plugin_files:
97-
_, filename = os.path.split(full_name)
98-
command_name = self._filename_to_command_name(filename)
99-
func, section = self._load_plugin(full_name)
100-
plugins.append(OPSPlugin(name=command_name, filename=full_name,
101-
func=func, section=section))
102-
return plugins
103-
10464
def list_commands(self, ctx):
10565
return list(self._get_command.keys())
10666

‎paths_cli/plugin_management.py‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
import collections
2+
import pkgutil
3+
import importlib
4+
import os
5+
6+
OPSPlugin = collections.namedtuple(
7+
"OPSPlugin", ['name', 'location', 'func', 'section', 'plugin_type']
8+
)
9+
10+
class CLIPluginLoader(object):
11+
"""Abstract object for CLI plugins
12+
13+
The overall approach involves 5 steps, each of which can be overridden:
14+
15+
1. Find candidate plugins (which must be Python modules)
16+
2. Load the namespaces associated into a dict (nsdict)
17+
3. Based on those namespaces, validate that the module *is* a plugin
18+
4. Get the associated command name
19+
5. Return an OPSPlugin object for each plugin
20+
21+
Details on steps 1, 2, and 4 differ based on whether this is a
22+
filesystem-based plugin or a namespace-based plugin.
23+
"""
24+
def __init__(self, plugin_type, search_path):
25+
self.plugin_type = plugin_type
26+
self.search_path = search_path
27+
28+
def _find_candidates(self):
29+
raise NotImplementedError()
30+
31+
@staticmethod
32+
def _make_nsdict(candidate):
33+
raise NotImplementedError()
34+
35+
@staticmethod
36+
def _validate(nsdict):
37+
for attr in ['CLI', 'SECTION']:
38+
if attr not in nsdict:
39+
return False
40+
return True
41+
42+
def _get_command_name(self, candidate):
43+
raise NotImplementedError()
44+
45+
def _find_valid(self):
46+
candidates = self._find_candidates()
47+
namespaces = {cand: self._make_nsdict(cand) for cand in candidates}
48+
valid = {cand: ns for cand, ns in namespaces.items()
49+
if self._validate(ns)}
50+
return valid
51+
52+
def __call__(self):
53+
valid = self._find_valid()
54+
plugins = [
55+
OPSPlugin(name=self._get_command_name(cand),
56+
location=cand,
57+
func=ns['CLI'],
58+
section=ns['SECTION'],
59+
plugin_type=self.plugin_type)
60+
for cand, ns in valid.items()
61+
]
62+
return plugins
63+
64+
65+
class FilePluginLoader(CLIPluginLoader):
66+
"""File-based plugins (quick and dirty)
67+
68+
Parameters
69+
----------
70+
search_path : str
71+
path to the directory that contains plugins (OS-dependent format)
72+
"""
73+
def __init__(self, search_path):
74+
super().__init__(plugin_type="file", search_path=search_path)
75+
76+
def _find_candidates(self):
77+
def is_plugin(filename):
78+
return (
79+
filename.endswith(".py") and not filename.startswith("_")
80+
and not filename.startswith(".")
81+
)
82+
83+
if not os.path.exists(os.path.join(self.search_path)):
84+
return []
85+
86+
candidates = [os.path.join(self.search_path, f)
87+
for f in os.listdir(self.search_path)
88+
if is_plugin(f)]
89+
return candidates
90+
91+
@staticmethod
92+
def _make_nsdict(candidate):
93+
ns = {}
94+
with open(candidate) as f:
95+
code = compile(f.read(), candidate, 'exec')
96+
eval(code, ns, ns)
97+
return ns
98+
99+
def _get_command_name(self, candidate):
100+
_, command_name = os.path.split(candidate)
101+
command_name = command_name[:-3] # get rid of .py
102+
command_name = command_name.replace('_', '-') # commands use -
103+
return command_name
104+
105+
106+
class NamespacePluginLoader(CLIPluginLoader):
107+
"""Load namespace plugins (plugins for wide distribution)
108+
109+
Parameters
110+
----------
111+
search_path : str
112+
namespace (dot-separated) where plugins can be found
113+
"""
114+
def __init__(self, search_path):
115+
super().__init__(plugin_type="namespace", search_path=search_path)
116+
117+
def _find_candidates(self):
118+
# based on https://packaging.python.org/guides/creating-and-discovering-plugins/#using-namespace-packages
119+
def iter_namespace(ns_pkg):
120+
return pkgutil.iter_modules(ns_pkg.__path__,
121+
ns_pkg.__name__ + ".")
122+
123+
ns = importlib.import_module(self.search_path)
124+
candidates = [
125+
importlib.import_module(name)
126+
for _, name, _ in iter_namespace(ns)
127+
]
128+
return candidates
129+
130+
@staticmethod
131+
def _make_nsdict(candidate):
132+
return vars(candidate)
133+
134+
def _get_command_name(self, candidate):
135+
# +1 for the dot
136+
command_name = candidate.__name__
137+
command_name = command_name[len(self.search_path) + 1:]
138+
command_name = command_name.replace('_', '-') # commands use -
139+
return command_name
140+

‎paths_cli/plugins/__init__.py‎

Whitespace-only changes.

‎paths_cli/tests/null_command.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import logging
22
import click
3+
4+
from paths_cli.plugin_management import FilePluginLoader, OPSPlugin
5+
36
@click.command(
47
'null-command',
58
short_help="Do nothing (testing)"
@@ -14,7 +17,13 @@ def null_command():
1417
class NullCommandContext(object):
1518
"""Context that registers/deregisters the null command (for tests)"""
1619
def __init__(self, cli):
17-
self.plugin = cli._load_plugin_files([__file__])[0]
20+
self.plugin = OPSPlugin(name="null-command",
21+
location=__file__,
22+
func=CLI,
23+
section=SECTION,
24+
plugin_type="file")
25+
26+
cli._register_plugin(self.plugin)
1827
self.cli = cli
1928

2029
def __enter__(self):

‎paths_cli/tests/test_cli.py‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,11 @@
33
from click.testing import CliRunner
44

55
from paths_cli.cli import *
6+
from paths_cli.plugin_management import OPSPlugin
67
from .null_command import NullCommandContext
78

89

910
class TestOpenPathSamplingCLI(object):
10-
# TODO: more thorough testing of private methods to find/register
11-
# plugins might be nice; so far we mainly focus on testing the API.
12-
# (Still have smoke tests covering everything, though.)
1311
def setup(self):
1412
def make_mock(name, helpless=False):
1513
mock = MagicMock(return_value=name)
@@ -21,22 +19,27 @@ def make_mock(name, helpless=False):
2119

2220
self.plugin_dict = {
2321
'foo': OPSPlugin(name='foo',
24-
filename='foo.py',
22+
location='foo.py',
2523
func=make_mock('foo'),
26-
section='Simulation'),
24+
section='Simulation',
25+
plugin_type='file'),
2726
'foo-bar': OPSPlugin(name='foo-bar',
28-
filename='foo_bar.py',
27+
location='foo_bar.py',
2928
func=make_mock('foobar', helpless=True),
30-
section='Miscellaneous')
29+
section='Miscellaneous',
30+
plugin_type="file")
3131
}
32-
self.fake_plugins = list(self.plugin_dict.values())
33-
mock_plugins = MagicMock(return_value=self.fake_plugins)
34-
with patch.object(OpenPathSamplingCLI, '_load_plugin_files',
35-
mock_plugins):
36-
self.cli = OpenPathSamplingCLI()
32+
self.plugins = list(self.plugin_dict.values())
33+
self.cli = OpenPathSamplingCLI()
34+
# need to copy the plugins since we're changing the list
35+
for plugin in self.cli.plugins[:]:
36+
self.cli._deregister_plugin(plugin)
37+
38+
for plugin in self.plugins:
39+
self.cli._register_plugin(plugin)
3740

3841
def test_plugins(self):
39-
assert self.cli.plugins == self.fake_plugins
42+
assert self.cli.plugins == self.plugins
4043
assert self.cli._sections['Simulation'] == ['foo']
4144
assert self.cli._sections['Miscellaneous'] == ['foo-bar']
4245

0 commit comments

Comments
 (0)