AUTO_MEMORIES: the generated macros reach the GDS merge

The GDS merge reads the final DEF through a KLayout technology built
from the tech, standard-cell and ADDITIONAL_LEFS LEFs. The LEFs
AUTO_MEMORIES generates were not among them, so the merge stopped at
the first generated macro:

  Macro not found in LEF file: mbtb_entry_sram

And a generated macro has no GDS, while a platform's GDS_ALLOW_EMPTY
pattern (asap7: fakeram.*) only matches the platform's own memories,
so with the LEF found, every generated macro would have been reported
as an empty cell.

With AUTO_MEMORIES=1, klayout.lyt now takes the generated LEFs, the
ones load.tcl reads, and is rebuilt when they change; def2stream.py
allows the modules listed in memories/blackboxes.txt to be empty,
taking the first column of each line. Found with asap7/tage_update
(#4595), which now writes 6_final.gds.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Øyvind Harboe <oyvind.harboe@zylin.com>

Author: oharboe

flow/Makefile

@@ -204,6 +204,7 @@ do-klayout:
 		--template $(KLAYOUT_TECH_FILE) \
 		--output $(OBJECTS_DIR)/klayout.lyt \
 		--lef-files $(OBJECTS_DIR)/klayout_tech.lef $(SC_LEF) $(ADDITIONAL_LEFS) \
+		  $(if $(filter 1,$(AUTO_MEMORIES)),$(wildcard $(RESULTS_DIR)/memories/*.lef)) \
 		--map-files $(wildcard $(FLOW_HOME)/platforms/$(PLATFORM)/*map)
 
 $(OBJECTS_DIR)/klayout_wrap.lyt: $(KLAYOUT_TECH_FILE) $(OBJECTS_DIR)/klayout_tech.lef
@@ -285,6 +286,9 @@ $(RESULTS_DIR)/memories.json: $(RESULTS_DIR)/memories_inferred.json $(ADDITIONAL
 
 yosys-dependencies: $(RESULTS_DIR)/memories.json
 $(RESULTS_DIR)/1_1_yosys_canonicalize.rtlil: $(RESULTS_DIR)/memories.json
+# The generated LEFs are part of the KLayout technology the GDS merge
+# reads the final DEF with.
+$(OBJECTS_DIR)/klayout.lyt: $(RESULTS_DIR)/memories.json
 endif
 
 .PHONY: do-yosys

flow/test/test_def2stream.py

@@ -4,6 +4,7 @@
 from unittest.mock import MagicMock, patch, call
 import sys
 import os
+import tempfile
 
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "util"))
 
@@ -320,6 +321,64 @@ def test_allow_empty_regex_no_match_still_errors(self):
 
         self.assertEqual(errors, 1)
 
+    def test_auto_memories_cell_may_be_empty(self):
+        """A macro AUTO_MEMORIES generated has no GDS; it is not an error."""
+        generated = make_mock_cell(
+            "tage_entry_sram", cell_index=1, is_empty=True, parent_cells=1
+        )
+        other = make_mock_cell("other_cell", cell_index=2, is_empty=True)
+
+        top = make_mock_cell("test_design", cell_index=0)
+
+        pya_mod, main_layout, _, _ = make_mock_pya(
+            cells_before_read=[],
+            cells_after_read=[top],
+            top_only_cells=[generated, other],
+        )
+        main_layout.cell.return_value = top
+
+        errors = def2stream.merge_gds(
+            pya_mod=pya_mod,
+            tech_file="/tmp/test.lyt",
+            layer_map="",
+            in_def="/tmp/test.def",
+            design_name="test_design",
+            in_files="",
+            seal_file="",
+            out_file="/tmp/out.gds",
+            allow_empty_cells=frozenset(["tage_entry_sram"]),
+        )
+
+        self.assertEqual(errors, 1)
+
+
+class TestAutoMemoriesCells(unittest.TestCase):
+    def test_off_reads_nothing(self):
+        self.assertEqual(
+            def2stream.auto_memories_cells({"RESULTS_DIR": "/nonexistent"}),
+            frozenset(),
+        )
+
+    def test_names_from_blackboxes_txt(self):
+        with tempfile.TemporaryDirectory() as results:
+            os.mkdir(os.path.join(results, "memories"))
+            with open(os.path.join(results, "memories", "blackboxes.txt"), "w") as f:
+                # a name per line, optionally followed by more columns
+                f.write("tage_entry_sram\nmbtb_entry_sram 491.400000\n\n")
+            self.assertEqual(
+                def2stream.auto_memories_cells(
+                    {"AUTO_MEMORIES": "1", "RESULTS_DIR": results}
+                ),
+                frozenset(["tage_entry_sram", "mbtb_entry_sram"]),
+            )
+
+    def test_missing_list_is_an_error(self):
+        with tempfile.TemporaryDirectory() as results:
+            with self.assertRaises(FileNotFoundError):
+                def2stream.auto_memories_cells(
+                    {"AUTO_MEMORIES": "1", "RESULTS_DIR": results}
+                )
+
 
 class TestOrphanCells(unittest.TestCase):
     def test_orphan_cell_is_error(self):

flow/util/def2stream.py

@@ -18,6 +18,7 @@ def merge_gds(
     seal_file,
     out_file,
     allow_empty="",
+    allow_empty_cells=frozenset(),
 ):
     """Merge DEF and GDS/OAS files into a single stream file.
 
@@ -31,6 +32,7 @@ def merge_gds(
         seal_file: Path to seal ring GDS/OAS file (empty string if none).
         out_file: Path to output GDS/OAS file.
         allow_empty: Regex pattern for cells allowed to be empty.
+        allow_empty_cells: Names of cells allowed to be empty.
 
     Returns:
         Number of errors encountered.
@@ -88,6 +90,11 @@ def merge_gds(
                         i.name
                     )
                 )
+            elif i.name in allow_empty_cells:
+                print(
+                    "[WARNING] LEF Cell '{0}' ignored. AUTO_MEMORIES generated"
+                    " its views; it has no GDS.".format(i.name)
+                )
             else:
                 print(
                     "[ERROR] LEF Cell '{0}' has no matching GDS/OAS cell."
@@ -128,6 +135,19 @@ def merge_gds(
     return errors
 
 
+def auto_memories_cells(environ):
+    """The modules AUTO_MEMORIES turned into macros.
+
+    Their LEF and liberty views are generated and they have no GDS, so
+    they are empty in the merged layout by design.
+    """
+    if environ.get("AUTO_MEMORIES") != "1":
+        return frozenset()
+    path = os.path.join(environ["RESULTS_DIR"], "memories", "blackboxes.txt")
+    with open(path) as f:
+        return frozenset(line.split()[0] for line in f if line.strip())
+
+
 # When run via klayout -r, globals tech_file, layer_map, in_def, etc.
 # are set by klayout's -rd mechanism.
 if pya is not None:
@@ -144,6 +164,7 @@ def merge_gds(
                 seal_file=seal_file,  # noqa: F821
                 out_file=out_file,  # noqa: F821
                 allow_empty=os.environ.get("GDS_ALLOW_EMPTY", ""),
+                allow_empty_cells=auto_memories_cells(os.environ),
             )
         )
     except NameError: