Fix false unused terminal warnings

Author: amkdown-max

lark/load_grammar.py

@@ -681,13 +681,22 @@ class Grammar(Serialize):
     term_defs: List[Tuple[str, Tuple[Tree, int]]]
     rule_defs: List[Tuple[str, Tuple[str, ...], Tree, RuleOptions]]
     ignore: List[str]
+    term_references: Dict[str, List[str]]
 
-    def __init__(self, rule_defs: List[Tuple[str, Tuple[str, ...], Tree, RuleOptions]], term_defs: List[Tuple[str, Tuple[Tree, int]]], ignore: List[str]) -> None:
+    def __init__(self, rule_defs: List[Tuple[str, Tuple[str, ...], Tree, RuleOptions]], term_defs: List[Tuple[str, Tuple[Tree, int]]], ignore: List[str], term_references: Optional[Dict[str, List[str]]] = None) -> None:
         self.term_defs = term_defs
         self.rule_defs = rule_defs
         self.ignore = ignore
+        self.term_references = term_references or {}
 
-    __serialize_fields__ = 'term_defs', 'rule_defs', 'ignore'
+    __serialize_fields__ = 'term_defs', 'rule_defs', 'ignore', 'term_references'
+
+    @classmethod
+    def deserialize(cls, data, memo):
+        if 'term_references' not in data:
+            data = dict(data)
+            data['term_references'] = {}
+        return super().deserialize(data, memo)
 
     def compile(self, start, terminals_to_keep) -> Tuple[List[TerminalDef], List[Rule], List[str]]:
         # We change the trees in-place (to support huge grammars)
@@ -810,9 +819,12 @@ def compile(self, start, terminals_to_keep) -> Tuple[List[TerminalDef], List[Rul
             used_terms = {t.name for r in compiled_rules
                                  for t in r.expansion
                                  if isinstance(t, Terminal)}
+            kept_terms = used_terms | set(self.ignore) | set(terminals_to_keep)
+            reachable_terms = set(bfs(kept_terms, lambda name: self.term_references.get(name, ())))
             terminals, unused = classify_bool(terminals, lambda t: t.name in used_terms or t.name in self.ignore or t.name in terminals_to_keep)
-            if unused:
-                logger.debug("Unused terminals: %s", [t.name for t in unused])
+            unused_names = [t.name for t in unused if t.name not in reachable_terms]
+            if unused_names:
+                logger.debug("Unused terminals: %s", unused_names)
 
         return terminals, compiled_rules, self.ignore
 
@@ -1084,6 +1096,7 @@ def __init__(self, is_term, tree, params=(), options=None):
         self.tree = tree
         self.params = tuple(params)
         self.options = options
+        self.term_references = _find_used_symbols(tree) if is_term and tree is not None else set()
 
 class GrammarBuilder:
 
@@ -1155,6 +1168,8 @@ def _extend(self, name, is_term, exp, params=(), options=None):
 
         # TODO: think about what to do with 'options'
         base = d.tree
+        if is_term:
+            d.term_references.update(_find_used_symbols(exp))
 
         assert isinstance(base, Tree) and base.data == 'expansions'
         base.children.insert(0, exp)
@@ -1380,15 +1395,17 @@ def build(self) -> Grammar:
         self.validate()
         rule_defs = []
         term_defs = []
+        term_references = {}
         for name, d in self._definitions.items():
             (params, exp, options) = d.params, d.tree, d.options
             if d.is_term:
                 assert len(params) == 0
                 term_defs.append((name, (exp, options)))
+                term_references[name] = sorted(d.term_references)
             else:
                 rule_defs.append((name, params, exp, options))
         # resolve_term_references(term_defs)
-        return Grammar(rule_defs, term_defs, self._ignore_names)
+        return Grammar(rule_defs, term_defs, self._ignore_names, term_references)
 
 
 def verify_used_files(file_hashes):

tests/test_cache.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 
 import logging
+import pickle
 from unittest import TestCase, main, skipIf
 
 from lark import Lark, Tree, Transformer, UnexpectedInput
@@ -70,6 +71,11 @@ def append_zero(t):
 
 class TestCache(TestCase):
     g = '''start: "a"'''
+    g_with_terminal_reference = '''
+        start: A
+        A: B
+        B: "a"
+    '''
 
 
     def setUp(self):
@@ -194,9 +200,9 @@ def test_cache_grammar(self):
             Lark(self.g, parser='lalr', cache=False, cache_grammar=True)
 
         assert len(self.mock_fs.files) == 0
-        parser1 = Lark(self.g, parser='lalr', cache=True, cache_grammar=True)
-        parser2 = Lark(self.g, parser='lalr', cache=True, cache_grammar=True)
-        assert parser2.parse('a') == Tree('start', [])
+        parser1 = Lark(self.g_with_terminal_reference, parser='lalr', cache=True, cache_grammar=True)
+        parser2 = Lark(self.g_with_terminal_reference, parser='lalr', cache=True, cache_grammar=True)
+        assert parser2.parse('a') == Tree('start', ['a'])
 
         # Assert that the cache file was created, and uses a different name than regular cache
         assert len(self.mock_fs.files) == 1
@@ -207,6 +213,23 @@ def test_cache_grammar(self):
         assert parser1.grammar.term_defs == parser2.grammar.term_defs
         # Using repr() because RuleOptions doesn't implement __eq__
         assert repr(parser1.grammar.rule_defs) == repr(parser2.grammar.rule_defs)
+        assert parser1.grammar.term_references == parser2.grammar.term_references
+        assert parser2.grammar.term_references == {'A': ['B'], 'B': []}
+
+    def test_cache_grammar_backward_compatibility(self):
+        parser = Lark(self.g_with_terminal_reference, parser='lalr', cache=True, cache_grammar=True)
+        serialized = BytesIO()
+        parser.save(serialized)
+        payload = pickle.loads(serialized.getvalue())
+        del payload['data']['grammar']['term_references']
+
+        serialized = BytesIO()
+        pickle.dump(payload, serialized)
+        serialized.seek(0)
+        loaded = Lark.load(serialized)
+
+        assert loaded.parse('a') == Tree('start', ['a'])
+        assert loaded.grammar.term_references == {}
 
     def test_reconstruct(self):
         # Test that Reconstructor works with cached parsers (using cache_grammar)

tests/test_grammar.py

@@ -55,6 +55,38 @@ def test_lexer_callbacks_with_keyword_terminals(self):
         list(p.lex("abc 456"))
         assert sorted(set(fired)) == ['A', 'B']
 
+    def test_terminal_references_are_not_reported_as_unused(self):
+        grammar = r"""
+            start: IDENTIFIER
+            _IDENT_LETTER: "A".."Z"
+            DECIMAL_DIGIT: "0".."9"
+            ALPHANUMERIC: _IDENT_LETTER | DECIMAL_DIGIT
+            IDENTIFIER: _IDENT_LETTER ALPHANUMERIC+
+            UNUSED_FRAGMENT: "unused"
+            UNUSED: UNUSED_FRAGMENT
+        """
+
+        with self.assertLogs("lark", level="DEBUG") as logs:
+            parser = Lark(grammar, parser="lalr")
+
+        self.assertEqual(logs.output, ["DEBUG:lark:Unused terminals: ['UNUSED_FRAGMENT', 'UNUSED']"])
+        self.assertEqual([terminal.name for terminal in parser.terminals], ['IDENTIFIER'])
+        self.assertEqual(parser.parse("ANSWER42"), Tree('start', ['ANSWER42']))
+
+    def test_overridden_terminal_does_not_keep_imported_references(self):
+        grammar = r"""
+            %import common (CNAME, LETTER, DIGIT)
+            %override CNAME: "x"
+            start: CNAME
+        """
+
+        with self.assertLogs("lark", level="DEBUG") as logs:
+            parser = Lark(grammar, parser="lalr")
+
+        self.assertEqual(len(logs.records), 1)
+        self.assertEqual(set(logs.records[0].args[0]), {'DIGIT', 'LETTER'})
+        self.assertEqual(parser.parse("x"), Tree('start', ['x']))
+
 
     def test_override_rule(self):
         # Overrides the 'sep' template in existing grammar to add an optional terminating delimiter