Add Elements and local effective composition integration

Author: RayJr2

CMakeLists.txt

@@ -78,8 +78,14 @@ qt_add_executable(BrickSuite
     src/import/RebrickableThemeCatalogImporter.cpp
     src/import/RebrickableInventoryCompositionImporter.h
     src/import/RebrickableInventoryCompositionImporter.cpp
+    src/import/RebrickableElementImporter.h
+    src/import/RebrickableElementImporter.cpp
     src/repositories/SetInventoryRevisionRepository.h
     src/repositories/SetInventoryRevisionRepository.cpp
+    src/repositories/PartElementIdentifierRepository.h
+    src/repositories/PartElementIdentifierRepository.cpp
+    src/repositories/EffectiveSetCompositionRepository.h
+    src/repositories/EffectiveSetCompositionRepository.cpp
     src/ui/import/GlobalRebrickableImportDialog.h
     src/ui/import/GlobalRebrickableImportDialog.cpp
     resources/resources.qrc
@@ -380,6 +386,8 @@ qt_add_executable(BrickSuite
     src/services/minifigs/MinifigCompositionReplacementService.cpp
     src/services/minifigs/RebrickableMinifigPartsService.h
     src/services/minifigs/RebrickableMinifigPartsService.cpp
+    src/services/minifigs/RebrickableMinifigThemeDerivationService.h
+    src/services/minifigs/RebrickableMinifigThemeDerivationService.cpp
     src/models/SetCatalogPart.h
     src/repositories/SetCatalogPartRepository.h
     src/repositories/SetCatalogPartRepository.cpp
@@ -803,6 +811,7 @@ if(BUILD_TESTING)
         src/repositories/ManufacturerRepository.h src/repositories/ManufacturerRepository.cpp
         src/repositories/SetCatalogRepository.h src/repositories/SetCatalogRepository.cpp
         src/repositories/SetCatalogPartRepository.h src/repositories/SetCatalogPartRepository.cpp
+        src/repositories/EffectiveSetCompositionRepository.h src/repositories/EffectiveSetCompositionRepository.cpp
         src/services/builds/SetBuildCreationService.h src/services/builds/SetBuildCreationService.cpp
         src/services/sets/SetCompositionReplacementService.h src/services/sets/SetCompositionReplacementService.cpp
     )
@@ -897,6 +906,9 @@ if(BUILD_TESTING)
         src/import/RebrickableSetCatalogImporter.h src/import/RebrickableSetCatalogImporter.cpp
         src/import/RebrickableMinifigCatalogImporter.h src/import/RebrickableMinifigCatalogImporter.cpp
         src/import/RebrickableInventoryCompositionImporter.h src/import/RebrickableInventoryCompositionImporter.cpp
+        src/import/RebrickableElementImporter.h src/import/RebrickableElementImporter.cpp
+        src/services/minifigs/RebrickableMinifigThemeDerivationService.h
+        src/services/minifigs/RebrickableMinifigThemeDerivationService.cpp
         src/database/DatabaseManager.h src/database/DatabaseManager.cpp
         src/models/PartRelationship.h
     )
@@ -920,6 +932,22 @@ if(BUILD_TESTING)
         Qt6::Core Qt6::Sql ZLIB::ZLIB)
     add_test(NAME InventoryCompositionPersistence
         COMMAND InventoryCompositionPersistenceTest)
+
+    qt_add_executable(ElementsEffectiveCompositionTest EXCLUDE_FROM_ALL
+        tests/ElementsEffectiveCompositionTest.cpp
+        src/database/DatabaseSchema.h src/database/DatabaseSchema.cpp
+        src/database/DatabaseManager.h src/database/DatabaseManager.cpp
+        src/import/RebrickableCsvInputResolver.h src/import/RebrickableCsvInputResolver.cpp
+        src/import/global/RebrickableImportCancellation.h
+        src/import/RebrickableElementImporter.h src/import/RebrickableElementImporter.cpp
+        src/repositories/PartElementIdentifierRepository.h src/repositories/PartElementIdentifierRepository.cpp
+        src/repositories/EffectiveSetCompositionRepository.h src/repositories/EffectiveSetCompositionRepository.cpp
+        src/services/minifigs/RebrickableMinifigThemeDerivationService.h
+        src/services/minifigs/RebrickableMinifigThemeDerivationService.cpp
+    )
+    target_link_libraries(ElementsEffectiveCompositionTest PRIVATE
+        Qt6::Core Qt6::Sql ZLIB::ZLIB)
+    add_test(NAME ElementsEffectiveComposition COMMAND ElementsEffectiveCompositionTest)
 endif()
 
 if(WIN32)

resources/help/getting_started.html

@@ -4,7 +4,7 @@ <h2>1. Workspace</h2><p>Create or select the Workspace representing the workshop
 <h2>2. Storage</h2><p>Create the physical hierarchy you use. Active leaf locations may allow Inventory, Collection, or Both; parent locations organize the hierarchy but are not destinations.</p><img src="images/getting_started_storage.png" style="max-width:100%" alt="Storage">
 <h2>3. Optional providers</h2><p>Configure Rebrickable or Brickset under <b>Settings → APIs</b> when you want API-backed composition, images, enrichment, or instructions. Download-based catalog imports and local workflows do not require a continuous connection.</p>
 <h2>4. Reference catalogs</h2><p>Populate Parts, Sets, and Minifigs from supported Rebrickable downloads. Catalogs describe reference identities; they do not assert ownership.</p>
-<h3>Folder-based Rebrickable import</h3><p>Choose <b>Tools → Import Rebrickable Data Files...</b> and select a folder containing official Rebrickable <code>.csv</code> or <code>.csv.zip</code> bulk downloads. BrickSuite validates a visible plan before enabling Import. Themes, Colors, Part Categories, Parts, Part Relationships, Sets, Minifigs, inventory revisions, Inventory Parts, Inventory Minifigs, and contained Sets are supported. Elements remains deferred.</p><p>Rebrickable's inventory files contain both Set and Minifig inventory identities. BrickSuite recognizes both, persists revision history for Sets, and intentionally excludes Minifig-owned Part rows from Set composition.</p><p>Revision-backed composition is catalog data, not owned inventory. Historical Set revisions are retained, and BrickSuite selects the highest complete Rebrickable revision as preferred after all composition files finish successfully. Spare rows are preserved as provider metadata and do not imply physical ownership.</p><p>Each supported dataset uses its own transaction. A failed or cancelled current dataset is rolled back while earlier committed datasets remain available; dependent datasets are blocked, while independent datasets may continue. Required identities are checked before authoritative snapshot synchronization. BrickSuite does not download these files automatically.</p>
+<h3>Folder-based Rebrickable import</h3><p>Choose <b>Tools → Import Rebrickable Data Files...</b> and select a folder containing official Rebrickable <code>.csv</code> or <code>.csv.zip</code> bulk downloads. BrickSuite validates a visible plan before enabling Import. All twelve datasets are supported: Themes, Colors, Part Categories, Parts, Elements, Part Relationships, Sets, Minifigs, Inventories, Inventory Parts, Inventory Minifigs, and Inventory Sets. Elements map an exact Rebrickable Element ID to its exact Part and Color; they do not replace canonical Part numbers.</p><p>Rebrickable's inventory files contain both Set and Minifig inventory identities. BrickSuite recognizes both, persists revision history for Sets, and intentionally excludes Minifig-owned Part rows from Set composition.</p><p>Revision-backed composition is catalog data, not owned inventory. Historical Set revisions are retained, and BrickSuite selects the highest complete Rebrickable revision as preferred after all composition files finish successfully. A valid preferred revision supplies Set composition; existing legacy catalog composition remains the fallback when no preferred revision exists. Catalog spare rows remain available as provider metadata but are not copied into required Build pieces and do not imply physical ownership. A Build keeps the requirement snapshot created at that time, so later catalog imports do not rewrite it.</p><p>Each supported dataset uses its own transaction. A failed or cancelled current dataset is rolled back while earlier committed datasets remain available; dependent datasets are blocked, while independent datasets may continue. Required identities are checked before authoritative snapshot synchronization. BrickSuite does not download these files automatically.</p>
 <h2>5. My Inventory</h2><p>Add or import loose physical pieces with exact Part, Color, Manufacturer, condition, ownership, quantity, and Storage.</p><img src="images/getting_started_inventory.png" style="max-width:100%" alt="My Inventory">
 <h2>6. Builds</h2><p>Create Set, Minifig, or MOC Builds. Requirements can be allocated, deliberately substituted, pulled, reconciled, completed, cancelled, or disassembled.</p>
 <h2>7. My Collection</h2><p>Record each physical Set, Minifig, or MOC independently from loose inventory and Build history.</p>

resources/help/minifigs_catalog.html

@@ -41,8 +41,11 @@ <h2>Theme Classification</h2>
 <h2>Minifig Details and Images</h2>
 
 <p>Use <b>Actions... → Details</b> to view the selected Minifig's catalog metadata, provider
-source, and imported composition. Images are downloaded asynchronously from the imported
-Rebrickable image URL and cached in BrickSuite's dedicated Minifig image cache.</p>
+source, and local composition. A Global Rebrickable import can populate this composition from the
+Minifig-owned rows in <code>inventory_parts.csv</code>. Existing local composition is displayed
+automatically when Details opens; opening the dialog does not request parts from an API. Images
+are downloaded asynchronously from the imported Rebrickable image URL and cached in BrickSuite's
+dedicated Minifig image cache.</p>
 
 <h2>Get or Import a Minifig Parts List</h2>
 
@@ -55,7 +58,8 @@ <h2>Get or Import a Minifig Parts List</h2>
 
 <p><b>Import Parts List...</b> remains available as the offline/manual fallback. Select the
 Rebrickable Parts CSV or its downloaded ZIP. The selected BrickSuite Minifig is always the
-destination.</p>
+destination. Both manual buttons remain explicit refresh/replacement options even when bulk-imported
+local composition is already displayed.</p>
 
 <p>If the filename contains a recognizable identifier such as <code>fig-005210</code>, BrickSuite
 checks it against the selected Minifig before saving anything. A renamed file without a

resources/help/sets_catalog.html

@@ -49,14 +49,19 @@ <h2>Set Details</h2>
 
 <h3>Catalog Parts List</h3>
 
-<p>The <b>Catalog Parts List</b> stores reusable exact Part, Color, quantity, and spare information
-for the selected Set. Choose <b>Get Parts from Rebrickable...</b> to retrieve the complete list
-through the Rebrickable API. This API workflow includes the constituent pieces of Minifigs in the
-Set so the catalog composition represents the physical pieces needed to assemble it.</p>
+<p>The <b>Catalog Parts List</b> automatically displays locally available exact Part, Color,
+quantity, and spare information when Set Details opens. BrickSuite uses the preferred local
+Rebrickable inventory revision when one is available and otherwise uses the existing Catalog
+Parts List fallback. The source line identifies which composition is displayed.</p>
+
+<p>Choose <b>Get Parts from Rebrickable...</b> to explicitly refresh the complete list through
+the Rebrickable API. This API workflow includes the constituent pieces of Minifigs in the Set so
+the catalog composition represents the physical pieces needed to assemble it.</p>
 
 <p>If API access is unavailable, choose <b>Import Parts List...</b> and select a Rebrickable parts
-list in CSV or ZIP form. ZIP files may contain the CSV in a nested folder. Both acquisition methods
-replace the same catalog composition and require confirmation when a parts list already exists.</p>
+list in CSV or ZIP form. ZIP files may contain the CSV in a nested folder. Both manual acquisition
+methods remain available as explicit refresh or replacement workflows and require confirmation
+when a parts list already exists. Opening Set Details itself never starts an API request.</p>
 
 <div class="note"><b>Exact catalog identities:</b> Every imported row must resolve to an existing
 BrickSuite Part and Rebrickable Color. If the input is malformed or an identity cannot be resolved,

src/database/DatabaseSchema.cpp

@@ -576,6 +576,18 @@ bool DatabaseSchema::initialize(QSqlDatabase& database)
         version = 33;
     }
 
+    if (version == 33) {
+        if (!migrateVersion33ToVersion34(database)) {
+            database.rollback();
+            return false;
+        }
+        if (!setSchemaVersion(database, 34)) {
+            database.rollback();
+            return false;
+        }
+        version = 34;
+    }
+
     if (version != CurrentSchemaVersion) {
         qCritical() << "Unsupported BrickSuite database schema version:" << version;
 
@@ -3791,3 +3803,50 @@ bool DatabaseSchema::migrateVersion32ToVersion33(QSqlDatabase& database)
     }
     return true;
 }
+
+bool DatabaseSchema::createPartElementIdentifierTable(QSqlDatabase& database)
+{
+    QSqlQuery query(database);
+    const QStringList statements = {
+        R"(CREATE TABLE IF NOT EXISTS part_element_identifier (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            provider TEXT NOT NULL COLLATE NOCASE,
+            element_id TEXT NOT NULL COLLATE NOCASE,
+            part_id INTEGER NOT NULL,
+            color_id INTEGER NOT NULL,
+            design_id TEXT NOT NULL,
+            is_active INTEGER NOT NULL DEFAULT 1 CHECK(is_active IN (0,1)),
+            created_utc TEXT NOT NULL,
+            modified_utc TEXT NOT NULL,
+            FOREIGN KEY(part_id) REFERENCES part(id),
+            FOREIGN KEY(color_id) REFERENCES color(id),
+            UNIQUE(provider, element_id)
+        ))",
+        "CREATE INDEX IF NOT EXISTS idx_part_element_part_color "
+        "ON part_element_identifier(part_id,color_id,is_active)",
+        "CREATE INDEX IF NOT EXISTS idx_part_element_design "
+        "ON part_element_identifier(provider,design_id,is_active)",
+        "CREATE INDEX IF NOT EXISTS idx_part_element_provider_active "
+        "ON part_element_identifier(provider,is_active)"
+    };
+    for (const QString& statement : statements) {
+        if (!query.exec(statement)) {
+            qCritical() << "Unable to create Element identifier schema:"
+                        << query.lastError().text();
+            return false;
+        }
+    }
+    return true;
+}
+
+bool DatabaseSchema::migrateVersion33ToVersion34(QSqlDatabase& database)
+{
+    if (!createPartElementIdentifierTable(database))
+        return false;
+    QSqlQuery check(database);
+    if (!check.exec("PRAGMA foreign_key_check") || check.next()) {
+        qCritical() << "Foreign-key validation failed after Version 34 migration.";
+        return false;
+    }
+    return true;
+}

src/database/DatabaseSchema.h

@@ -25,7 +25,7 @@
 class DatabaseSchema
 {
 public:
-    static constexpr int CurrentSchemaVersion = 33;
+    static constexpr int CurrentSchemaVersion = 34;
 
     static bool initialize(QSqlDatabase& database);
     static bool seedManufacturers(QSqlDatabase& database);
@@ -107,6 +107,7 @@ class DatabaseSchema
     static bool migrateVersion30ToVersion31(QSqlDatabase& database);
     static bool migrateVersion31ToVersion32(QSqlDatabase& database);
     static bool migrateVersion32ToVersion33(QSqlDatabase& database);
+    static bool migrateVersion33ToVersion34(QSqlDatabase& database);
 
     static bool createExternalColorMappingTable(QSqlDatabase& database);
     static bool createExternalPartMappingTable(QSqlDatabase& database);
@@ -123,4 +124,5 @@ class DatabaseSchema
     static bool createMinifigCatalogPartTable(QSqlDatabase& database);
     static bool createSetCatalogPartTable(QSqlDatabase& database);
     static bool createSetInventoryRevisionTables(QSqlDatabase& database);
+    static bool createPartElementIdentifierTable(QSqlDatabase& database);
 };