Skip to content

Commit 28da787

Browse files
committed
Integrate catalog datasets into global Rebrickable importer
1 parent b003f41 commit 28da787

27 files changed

Lines changed: 885 additions & 56 deletions

CMakeLists.txt

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,10 @@ qt_add_executable(BrickSuite
7272
src/import/global/RebrickableImportPlanController.cpp
7373
src/import/global/RebrickableWorkerDatabaseSession.h
7474
src/import/global/RebrickableWorkerDatabaseSession.cpp
75+
src/import/global/RebrickableGlobalImportService.h
76+
src/import/global/RebrickableGlobalImportService.cpp
77+
src/import/RebrickableThemeCatalogImporter.h
78+
src/import/RebrickableThemeCatalogImporter.cpp
7579
src/ui/import/GlobalRebrickableImportDialog.h
7680
src/ui/import/GlobalRebrickableImportDialog.cpp
7781
resources/resources.qrc
@@ -633,6 +637,8 @@ if(BUILD_TESTING)
633637
src/import/RebrickableMinifigCatalogImporter.cpp
634638
src/import/RebrickableMinifigThemeImporter.h
635639
src/import/RebrickableMinifigThemeImporter.cpp
640+
src/import/RebrickableThemeCatalogImporter.h
641+
src/import/RebrickableThemeCatalogImporter.cpp
636642
src/models/ThemeCatalogItem.h
637643
src/repositories/ThemeCatalogRepository.h
638644
src/repositories/ThemeCatalogRepository.cpp
@@ -878,6 +884,16 @@ if(BUILD_TESTING)
878884
src/import/global/RebrickableImportPlanController.cpp
879885
src/import/global/RebrickableWorkerDatabaseSession.h
880886
src/import/global/RebrickableWorkerDatabaseSession.cpp
887+
src/import/global/RebrickableGlobalImportService.h
888+
src/import/global/RebrickableGlobalImportService.cpp
889+
src/import/RebrickableReferenceImporter.h src/import/RebrickableReferenceImporter.cpp
890+
src/import/RebrickableThemeCatalogImporter.h src/import/RebrickableThemeCatalogImporter.cpp
891+
src/import/RebrickablePartCatalogImporter.h src/import/RebrickablePartCatalogImporter.cpp
892+
src/import/RebrickablePartRelationshipImporter.h src/import/RebrickablePartRelationshipImporter.cpp
893+
src/import/RebrickableSetCatalogImporter.h src/import/RebrickableSetCatalogImporter.cpp
894+
src/import/RebrickableMinifigCatalogImporter.h src/import/RebrickableMinifigCatalogImporter.cpp
895+
src/database/DatabaseManager.h src/database/DatabaseManager.cpp
896+
src/models/PartRelationship.h
881897
)
882898
target_link_libraries(GlobalRebrickableImporterFoundationTest PRIVATE
883899
Qt6::Core Qt6::Sql ZLIB::ZLIB)

resources/help/getting_started.html

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ <h2>1. Workspace</h2><p>Create or select the Workspace representing the workshop
44
<h2>2. Storage</h2><p>Create the physical hierarchy you use. Active leaf locations may allow Inventory, Collection, or Both; parent locations organize the hierarchy but are not destinations.</p><img src="images/getting_started_storage.png" style="max-width:100%" alt="Storage">
55
<h2>3. Optional providers</h2><p>Configure Rebrickable or Brickset under <b>Settings → APIs</b> when you want API-backed composition, images, enrichment, or instructions. Download-based catalog imports and local workflows do not require a continuous connection.</p>
66
<h2>4. Reference catalogs</h2><p>Populate Parts, Sets, and Minifigs from supported Rebrickable downloads. Catalogs describe reference identities; they do not assert ownership.</p>
7+
<h3>Folder-based Rebrickable import</h3><p>Choose <b>Tools → Import Rebrickable Data Files...</b> and select a folder containing official Rebrickable <code>.csv</code> or <code>.csv.zip</code> bulk downloads. BrickSuite validates a visible plan before enabling Import. Themes, Colors, Part Categories, Parts, Part Relationships, Sets, and Minifigs are supported in this release; Elements, Inventories, Inventory Parts, Inventory Minifigs, and Inventory Sets are discovered and validated but remain marked <b>Not implemented</b>.</p><p>Each supported dataset uses its own transaction. A failed or cancelled current dataset is rolled back while earlier committed datasets remain available; dependent datasets are blocked, while independent datasets may continue. Required identities are checked before authoritative snapshot synchronization. BrickSuite does not download these files automatically.</p>
78
<h2>5. My Inventory</h2><p>Add or import loose physical pieces with exact Part, Color, Manufacturer, condition, ownership, quantity, and Storage.</p><img src="images/getting_started_inventory.png" style="max-width:100%" alt="My Inventory">
89
<h2>6. Builds</h2><p>Create Set, Minifig, or MOC Builds. Requirements can be allocated, deliberately substituted, pulled, reconciled, completed, cancelled, or disassembled.</p>
910
<h2>7. My Collection</h2><p>Record each physical Set, Minifig, or MOC independently from loose inventory and Build history.</p>

src/import/RebrickableMinifigCatalogImporter.cpp

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include "RebrickableMinifigCatalogImporter.h"
22

33
#include "RebrickableCsvInputResolver.h"
4+
#include "global/RebrickableImportCancellation.h"
45

56
#include "../database/DatabaseManager.h"
67

@@ -89,6 +90,16 @@ bool failTransaction(QSqlDatabase& database,
8990

9091
RebrickableMinifigCatalogImporter::Result
9192
RebrickableMinifigCatalogImporter::importFile(const QString& fileName)
93+
{
94+
QSqlDatabase database = DatabaseManager::instance().database();
95+
return importFile(fileName, database);
96+
}
97+
98+
RebrickableMinifigCatalogImporter::Result
99+
RebrickableMinifigCatalogImporter::importFile(const QString& fileName,
100+
QSqlDatabase& database,
101+
const RebrickableImportCancellation* cancellation,
102+
const RebrickableRowProgress& progress)
92103
{
93104
Result result;
94105
QTemporaryDir temporaryDirectory;
@@ -144,6 +155,12 @@ RebrickableMinifigCatalogImporter::importFile(const QString& fileName)
144155
continue;
145156

146157
++result.rowsRead;
158+
if ((result.rowsRead & 255) == 0 && cancellation
159+
&& cancellation->isCancellationRequested()) {
160+
result.message = QStringLiteral("Minifig Catalog import cancelled.");
161+
return result;
162+
}
163+
if ((result.rowsRead & 255) == 0 && progress) progress(result.rowsRead);
147164
bool rowOk = false;
148165
const QStringList fields = parseCsvLine(line, rowOk);
149166

@@ -189,7 +206,6 @@ RebrickableMinifigCatalogImporter::importFile(const QString& fileName)
189206

190207
file.close();
191208

192-
QSqlDatabase database = DatabaseManager::instance().database();
193209
QHash<QString, ExistingRow> existingRows;
194210
{
195211
QSqlQuery query(database);
@@ -271,6 +287,11 @@ RebrickableMinifigCatalogImporter::importFile(const QString& fileName)
271287
}
272288

273289
for (const ImportRow& row : rows) {
290+
if (cancellation && cancellation->isCancellationRequested()) {
291+
failTransaction(database, result,
292+
QStringLiteral("Minifig Catalog import cancelled."));
293+
return result;
294+
}
274295
const QString key = row.externalId.toCaseFolded();
275296

276297
if (!existingRows.contains(key)) {

src/import/RebrickableMinifigCatalogImporter.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
#pragma once
22

33
#include <QString>
4+
#include "global/RebrickableImportCancellation.h"
5+
6+
class QSqlDatabase;
47

58
class RebrickableMinifigCatalogImporter
69
{
@@ -17,4 +20,7 @@ class RebrickableMinifigCatalogImporter
1720
};
1821

1922
Result importFile(const QString& fileName);
23+
Result importFile(const QString& fileName, QSqlDatabase& database,
24+
const RebrickableImportCancellation* cancellation = nullptr,
25+
const RebrickableRowProgress& progress = {});
2026
};

src/import/RebrickableMinifigThemeImporter.cpp

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#include "RebrickableMinifigThemeImporter.h"
2+
#include "RebrickableThemeCatalogImporter.h"
23

34
#include "RebrickableCsvInputResolver.h"
45
#include "../database/DatabaseManager.h"
@@ -131,25 +132,14 @@ RebrickableMinifigThemeImporter::importDirectory(const QString& directoryPath)
131132
catalogCheck.finish();
132133
if(!db.transaction()){result.message="Unable to begin Theme import transaction.";return result;}
133134
auto fail=[&](const QString& m){db.rollback();result.message=m;return result;};
134-
const QString now=QDateTime::currentDateTimeUtc().toString(Qt::ISODateWithMs);
135-
QHash<QString,ExistingTheme> existing; QSqlQuery q(db);
136-
q.prepare("SELECT tc.id,tei.id,tei.external_id,tc.name,COALESCE(tc.parent_theme_catalog_id,0),tei.is_active FROM theme_external_identifier tei JOIN theme_catalog tc ON tc.id=tei.theme_catalog_id WHERE tei.provider=:p");q.bindValue(":p",Provider);
137-
if(!q.exec()) return fail(q.lastError().text()); while(q.next()) existing.insert(q.value(2).toString(),{q.value(0).toInt(),q.value(1).toInt(),q.value(3).toString(),q.value(4).toInt(),q.value(5).toBool()});
138-
QHash<QString,int> internal; QSet<QString> updatedThemes; QSqlQuery ic(db),ii(db),uc(db),ui(db);
139-
ic.prepare("INSERT INTO theme_catalog(name,parent_theme_catalog_id,is_active,created_utc,modified_utc) VALUES(:n,NULL,1,:c,:m)");
140-
ii.prepare("INSERT INTO theme_external_identifier(theme_catalog_id,provider,external_id,source,is_active,created_utc,modified_utc) VALUES(:id,:p,:e,:s,1,:c,:m)");
141-
uc.prepare("UPDATE theme_catalog SET name=:n,is_active=1,modified_utc=:m WHERE id=:id");
142-
ui.prepare("UPDATE theme_external_identifier SET source=:s,is_active=1,modified_utc=:m WHERE id=:id");
143-
for(const ThemeRow& t:themes){
144-
if(existing.contains(t.id)){auto e=existing.value(t.id);internal.insert(t.id,e.catalogId);uc.bindValue(":n",t.name);uc.bindValue(":m",now);uc.bindValue(":id",e.catalogId);if(!uc.exec())return fail(uc.lastError().text());
145-
if(e.name!=t.name)updatedThemes.insert(t.id);if(!e.active){ui.bindValue(":s",Source);ui.bindValue(":m",now);ui.bindValue(":id",e.identityId);if(!ui.exec())return fail(ui.lastError().text());++result.themesReactivated;}}
146-
else{ic.bindValue(":n",t.name);ic.bindValue(":c",now);ic.bindValue(":m",now);if(!ic.exec())return fail(ic.lastError().text());int id=ic.lastInsertId().toInt();internal.insert(t.id,id);ii.bindValue(":id",id);ii.bindValue(":p",Provider);ii.bindValue(":e",t.id);ii.bindValue(":s",Source);ii.bindValue(":c",now);ii.bindValue(":m",now);if(!ii.exec())return fail(ii.lastError().text());++result.themesInserted;}}
147-
QSqlQuery parent(db);parent.prepare("UPDATE theme_catalog SET parent_theme_catalog_id=:parent,modified_utc=:m WHERE id=:id AND COALESCE(parent_theme_catalog_id,0)<>:parent_compare");
148-
for(const ThemeRow&t:themes){int pid=t.parentId.isEmpty()?0:internal.value(t.parentId);parent.bindValue(":parent",pid?QVariant(pid):QVariant());parent.bindValue(":parent_compare",pid);parent.bindValue(":m",now);parent.bindValue(":id",internal.value(t.id));if(!parent.exec())return fail(parent.lastError().text());if(parent.numRowsAffected()>0&&existing.contains(t.id))updatedThemes.insert(t.id);}
149-
result.themesUpdated = updatedThemes.size();
150-
QSqlQuery deactivate(db);deactivate.prepare("UPDATE theme_external_identifier SET is_active=0,modified_utc=:m WHERE id=:id AND is_active=1");
151-
for(auto it=existing.cbegin();it!=existing.cend();++it)if(!themes.contains(it.key())&&it.value().active){deactivate.bindValue(":m",now);deactivate.bindValue(":id",it.value().identityId);if(!deactivate.exec())return fail(deactivate.lastError().text());++result.themesDeactivated;}
152-
QSqlQuery sync(db);sync.prepare("UPDATE theme_catalog SET is_active=CASE WHEN EXISTS(SELECT 1 FROM theme_external_identifier tei WHERE tei.theme_catalog_id=theme_catalog.id AND tei.is_active=1) THEN 1 ELSE 0 END,modified_utc=:m");sync.bindValue(":m",now);if(!sync.exec())return fail(sync.lastError().text());
135+
const auto themeResult = RebrickableThemeCatalogImporter().importFile(
136+
themePath, db, nullptr, false, Source);
137+
if (!themeResult.success) return fail(themeResult.message);
138+
result.themesInserted=themeResult.inserted; result.themesUpdated=themeResult.updated;
139+
result.themesReactivated=themeResult.reactivated; result.themesDeactivated=themeResult.deactivated;
140+
QHash<QString,int> internal; QSqlQuery q(db);
141+
q.prepare("SELECT tei.external_id,tei.theme_catalog_id FROM theme_external_identifier tei WHERE tei.provider=:p AND tei.is_active=1");q.bindValue(":p",Provider);
142+
if(!q.exec())return fail(q.lastError().text());while(q.next())internal.insert(q.value(0).toString(),q.value(1).toInt());
153143
QHash<QString,int> minifigs; q.prepare("SELECT mei.external_id,mei.minifig_catalog_id FROM minifig_external_identifier mei JOIN minifig_catalog mc ON mc.id=mei.minifig_catalog_id WHERE mei.provider=:p AND mei.is_active=1 AND mc.is_active=1");q.bindValue(":p",Provider);if(!q.exec())return fail(q.lastError().text());while(q.next())minifigs.insert(q.value(0).toString().toCaseFolded(),q.value(1).toInt());
154144
QSet<QString> unresolved, associations; int resolvableRows = 0; for(const auto&r:relationships){int mid=minifigs.value(r.first.toCaseFolded());if(!mid){unresolved.insert(r.first.toCaseFolded());continue;}++resolvableRows;associations.insert(QString::number(mid)+QChar(0x1f)+QString::number(internal.value(r.second)));}
155145
result.unresolvedMinifigs=unresolved.size();result.associations=associations.size();result.duplicateRelationshipsCollapsed=resolvableRows-result.associations;

src/import/RebrickablePartCatalogImporter.cpp

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "RebrickablePartCatalogImporter.h"
2222

2323
#include "RebrickableCsvInputResolver.h"
24+
#include "global/RebrickableImportCancellation.h"
2425

2526
#include "../database/DatabaseManager.h"
2627

@@ -92,6 +93,15 @@ QString signature(const QString& name, int categoryId, const QString& material)
9293

9394
RebrickablePartCatalogImporter::Result RebrickablePartCatalogImporter::importFile(
9495
const QString& fileName)
96+
{
97+
QSqlDatabase database = DatabaseManager::instance().database();
98+
return importFile(fileName, database);
99+
}
100+
101+
RebrickablePartCatalogImporter::Result RebrickablePartCatalogImporter::importFile(
102+
const QString& fileName, QSqlDatabase& database,
103+
const RebrickableImportCancellation* cancellation,
104+
const RebrickableRowProgress& progress)
95105
{
96106
Result result;
97107

@@ -155,8 +165,6 @@ RebrickablePartCatalogImporter::Result RebrickablePartCatalogImporter::importFil
155165
return result;
156166
}
157167

158-
QSqlDatabase database = DatabaseManager::instance().database();
159-
160168
//
161169
// Rebrickable category ID
162170
// ->
@@ -302,6 +310,14 @@ RebrickablePartCatalogImporter::Result RebrickablePartCatalogImporter::importFil
302310

303311
++result.rowsRead;
304312

313+
if ((result.rowsRead & 255) == 0 && cancellation
314+
&& cancellation->isCancellationRequested()) {
315+
database.rollback();
316+
result.message = QStringLiteral("Parts Catalog import cancelled.");
317+
return result;
318+
}
319+
if ((result.rowsRead & 255) == 0 && progress) progress(result.rowsRead);
320+
305321
bool rowOk = false;
306322

307323
const QStringList fields = parseCsvLine(line, rowOk);

src/import/RebrickablePartCatalogImporter.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@
2121
#pragma once
2222

2323
#include <QString>
24+
#include "global/RebrickableImportCancellation.h"
25+
26+
class QSqlDatabase;
2427

2528
class RebrickablePartCatalogImporter
2629
{
@@ -39,4 +42,7 @@ class RebrickablePartCatalogImporter
3942
};
4043

4144
Result importFile(const QString& fileName);
45+
Result importFile(const QString& fileName, QSqlDatabase& database,
46+
const RebrickableImportCancellation* cancellation = nullptr,
47+
const RebrickableRowProgress& progress = {});
4248
};

src/import/RebrickablePartRelationshipImporter.cpp

Lines changed: 55 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
*/
66
#include "RebrickablePartRelationshipImporter.h"
77
#include "RebrickableCsvInputResolver.h"
8+
#include "global/RebrickableImportCancellation.h"
89

910
#include "../database/DatabaseManager.h"
1011
#include "../models/PartRelationship.h"
@@ -107,6 +108,17 @@ struct ExistingRelationship
107108
RebrickablePartRelationshipImporter::Result
108109
RebrickablePartRelationshipImporter::importFile(
109110
const QString& fileName)
111+
{
112+
QSqlDatabase database = DatabaseManager::instance().database();
113+
return importFile(fileName, database);
114+
}
115+
116+
RebrickablePartRelationshipImporter::Result
117+
RebrickablePartRelationshipImporter::importFile(
118+
const QString& fileName, QSqlDatabase& database,
119+
const RebrickableImportCancellation* cancellation,
120+
const RebrickableRowProgress& progress,
121+
bool requireCompleteSnapshot)
110122
{
111123
Result result;
112124

@@ -172,9 +184,6 @@ RebrickablePartRelationshipImporter::importFile(
172184
return result;
173185
}
174186

175-
QSqlDatabase database =
176-
DatabaseManager::instance().database();
177-
178187
//
179188
// Build one in-memory Part Number -> BrickSuite part.id map.
180189
// This avoids tens of thousands of SELECTs during the import.
@@ -196,6 +205,7 @@ RebrickablePartRelationshipImporter::importFile(
196205

197206
return result;
198207
}
208+
if ((result.rowsRead & 255) == 0 && progress) progress(result.rowsRead);
199209

200210
while (query.next()) {
201211
partIdByNumber.insert(
@@ -338,6 +348,13 @@ RebrickablePartRelationshipImporter::importFile(
338348

339349
++result.rowsRead;
340350

351+
if ((result.rowsRead & 255) == 0 && cancellation
352+
&& cancellation->isCancellationRequested()) {
353+
database.rollback();
354+
result.message = QStringLiteral("Part Relationship import cancelled.");
355+
return result;
356+
}
357+
341358
bool rowOk = false;
342359
const QStringList fields =
343360
parseCsvLine(line, rowOk);
@@ -351,6 +368,14 @@ RebrickablePartRelationshipImporter::importFile(
351368
<< "ExpectedFields:" << headers.size()
352369
<< "ActualFields:" << fields.size();
353370

371+
if (requireCompleteSnapshot) {
372+
database.rollback();
373+
result.message = QStringLiteral(
374+
"Part Relationship snapshot contains malformed CSV data at row %1.")
375+
.arg(result.rowsRead);
376+
return result;
377+
}
378+
354379
continue;
355380
}
356381

@@ -394,6 +419,15 @@ RebrickablePartRelationshipImporter::importFile(
394419
<< "ParentPart:" << parentNumber
395420
<< "Reason:" << reason;
396421

422+
if (requireCompleteSnapshot) {
423+
database.rollback();
424+
result.message = QStringLiteral(
425+
"Part Relationship snapshot contains invalid data at row %1: %2")
426+
.arg(result.rowsRead)
427+
.arg(reason);
428+
return result;
429+
}
430+
397431
continue;
398432
}
399433

@@ -402,6 +436,14 @@ RebrickablePartRelationshipImporter::importFile(
402436

403437
if (parentIt == partIdByNumber.constEnd()) {
404438
++result.skippedMissingParent;
439+
if (requireCompleteSnapshot) {
440+
database.rollback();
441+
result.message = QStringLiteral(
442+
"Part Relationship snapshot has an unresolved parent Part %1 at row %2.")
443+
.arg(parentNumber)
444+
.arg(result.rowsRead);
445+
return result;
446+
}
405447
continue;
406448
}
407449

@@ -410,22 +452,22 @@ RebrickablePartRelationshipImporter::importFile(
410452

411453
if (childIt == partIdByNumber.constEnd()) {
412454
++result.skippedMissingChild;
455+
if (requireCompleteSnapshot) {
456+
database.rollback();
457+
result.message = QStringLiteral(
458+
"Part Relationship snapshot has an unresolved child Part %1 at row %2.")
459+
.arg(childNumber)
460+
.arg(result.rowsRead);
461+
return result;
462+
}
413463
continue;
414464
}
415465

416466
const int parentPartId = parentIt.value();
417467
const int childPartId = childIt.value();
418468

419469
if (parentPartId == childPartId) {
420-
++result.skippedInvalid;
421-
422-
qWarning() << "Part Relationship import skipped invalid row."
423-
<< "Row:" << result.rowsRead
424-
<< "RelType:" << sourceType
425-
<< "ChildPart:" << childNumber
426-
<< "ParentPart:" << parentNumber
427-
<< "Reason: Parent and child reference the same part.";
428-
470+
++result.selfReferencesIgnored;
429471
continue;
430472
}
431473

@@ -572,6 +614,7 @@ RebrickablePartRelationshipImporter::importFile(
572614
<< "SkippedInvalid:" << result.skippedInvalid
573615
<< "SkippedMissingParent:" << result.skippedMissingParent
574616
<< "SkippedMissingChild:" << result.skippedMissingChild
617+
<< "SelfReferencesIgnored:" << result.selfReferencesIgnored
575618
<< "Deactivated:" << result.deactivated;
576619

577620
return result;

0 commit comments

Comments
 (0)