Skip to content

Commit d766c0b

Browse files
committed
db: simplify export to only support Django databases
The export command exists solely to migrate data from Django to Go. Remove the isDjango abstraction and error out if the source database does not have a django_migrations table. Rename fields to srcName/ dstName for clarity, and handle missing tables and columns in older patchwork 3.x schemas (v3.0 lacks addressed/comment events, v3.1-3.2 lack series dependencies/metadata/webhooks). Signed-off-by: Robin Jarry <robin@jarry.cc>
1 parent a93c964 commit d766c0b

1 file changed

Lines changed: 174 additions & 59 deletions

File tree

‎pkg/db/export.go‎

Lines changed: 174 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -16,86 +16,123 @@ import (
1616
"github.com/uptrace/bun"
1717
)
1818

19-
// exportTable defines a table to export and an optional source name to read
20-
// from when the database uses Django naming conventions.
2119
type exportTable struct {
22-
name string
23-
djangoName string
20+
// dstName is the table name in the Go schema.
21+
dstName string
22+
// srcName is the table name in the Django schema.
23+
srcName string
24+
// optional tables may not exist in older Django versions.
25+
optional bool
2426
}
2527

26-
// Order matters. Tables that references others must be dumped *after*.
28+
// Order matters. Tables that reference others must be dumped *after*.
2729
var exportTables = []exportTable{
28-
{name: "auth_user", djangoName: "auth_user"},
29-
{name: "auth_token", djangoName: "authtoken_token"},
30-
{name: "state", djangoName: "patchwork_state"},
31-
{name: "tag", djangoName: "patchwork_tag"},
32-
{name: "project", djangoName: "patchwork_project"},
33-
{name: "project_maintainer", djangoName: "patchwork_userprofile_maintainer_projects"},
34-
{name: "delegation_rule", djangoName: "patchwork_delegationrule"},
35-
{name: "person", djangoName: "patchwork_person"},
36-
{name: "patch_relation", djangoName: "patchwork_patchrelation"},
37-
{name: "series", djangoName: "patchwork_series"},
38-
{name: "series_reference", djangoName: "patchwork_seriesreference"},
39-
{name: "series_metadata", djangoName: "patchwork_seriesmetadata"},
40-
{name: "series_dependencies", djangoName: "patchwork_series_dependencies"},
41-
{name: "cover", djangoName: "patchwork_cover"},
42-
{name: "patch", djangoName: "patchwork_patch"},
43-
{name: "patch_tag", djangoName: "patchwork_patchtag"},
44-
{name: "patch_comment", djangoName: "patchwork_patchcomment"},
45-
{name: "cover_comment", djangoName: "patchwork_covercomment"},
30+
{dstName: "auth_user", srcName: "auth_user"},
31+
{dstName: "auth_token", srcName: "authtoken_token"},
32+
{dstName: "state", srcName: "patchwork_state"},
33+
{dstName: "tag", srcName: "patchwork_tag"},
34+
{dstName: "project", srcName: "patchwork_project"},
35+
{dstName: "project_maintainer", srcName: "patchwork_userprofile_maintainer_projects"},
36+
{dstName: "delegation_rule", srcName: "patchwork_delegationrule"},
37+
{dstName: "person", srcName: "patchwork_person"},
38+
{dstName: "patch_relation", srcName: "patchwork_patchrelation"},
39+
{dstName: "series", srcName: "patchwork_series"},
40+
{dstName: "series_reference", srcName: "patchwork_seriesreference"},
41+
{dstName: "series_metadata", srcName: "patchwork_seriesmetadata", optional: true},
42+
{dstName: "series_dependencies", srcName: "patchwork_series_dependencies", optional: true},
43+
{dstName: "cover", srcName: "patchwork_cover"},
44+
{dstName: "patch", srcName: "patchwork_patch"},
45+
{dstName: "patch_tag", srcName: "patchwork_patchtag"},
46+
{dstName: "patch_comment", srcName: "patchwork_patchcomment"},
47+
{dstName: "cover_comment", srcName: "patchwork_covercomment"},
4648
// Deduplicate checks: keep only the latest per (patch, context, user).
4749
// Django accumulates duplicate checks; we enforce uniqueness.
48-
{name: "ci_check", djangoName: "patchwork_check"},
49-
{name: "bundle", djangoName: "patchwork_bundle"},
50-
{name: "bundle_patch", djangoName: "patchwork_bundlepatch"},
51-
{name: "event", djangoName: "patchwork_event"},
52-
{name: "email_confirmation", djangoName: "patchwork_emailconfirmation"},
53-
{name: "webhook", djangoName: "patchwork_webhook"},
50+
{dstName: "ci_check", srcName: "patchwork_check"},
51+
{dstName: "bundle", srcName: "patchwork_bundle"},
52+
{dstName: "bundle_patch", srcName: "patchwork_bundlepatch"},
53+
{dstName: "event", srcName: "patchwork_event"},
54+
{dstName: "email_confirmation", srcName: "patchwork_emailconfirmation"},
55+
{dstName: "webhook", srcName: "patchwork_webhook", optional: true},
5456
}
5557

56-
// Export writes all table data as SQL INSERT statements to w. Tables are
57-
// exported in foreign key dependency order so the output can be imported into
58-
// a fresh database.
58+
// Export reads data from a Django patchwork database and writes SQL
59+
// INSERT statements to w, using the Go schema table and column names.
60+
// The output can be imported into a fresh database initialized with
61+
// "pw db sync". All patchwork 3.x schema variations are supported.
5962
func Export(ctx context.Context, database *bun.DB, w io.Writer) error {
60-
isDjango := isDjangoDatabase(ctx, database)
63+
if !hasDjangoMigrations(ctx, database) {
64+
return fmt.Errorf("not a Django database (django_migrations table not found)")
65+
}
6166

6267
fmt.Fprint(w, "BEGIN;\n")
6368

6469
for _, t := range exportTables {
65-
srcTable := t.name
66-
if isDjango {
67-
if t.djangoName == "" {
68-
continue
69-
}
70-
srcTable = t.djangoName
70+
if t.optional && !tableExists(ctx, database, t.srcName) {
71+
continue
7172
}
72-
if err := exportTableRows(ctx, database, w, srcTable, t.name, isDjango); err != nil {
73-
return fmt.Errorf("export %s: %w", t.name, err)
73+
if err := exportTableRows(ctx, database, w, t.srcName, t.dstName); err != nil {
74+
return fmt.Errorf("export %s: %w", t.dstName, err)
7475
}
7576
}
7677

7778
fmt.Fprint(w, "COMMIT;\n")
7879
return nil
7980
}
8081

81-
// isDjangoDatabase returns true if the database contains a django_migrations
82-
// table, indicating it was created by Django.
83-
func isDjangoDatabase(ctx context.Context, database *bun.DB) bool {
82+
func hasDjangoMigrations(ctx context.Context, database *bun.DB) bool {
8483
_, err := database.NewSelect().
8584
TableExpr("django_migrations").
8685
Limit(1).
8786
Exec(ctx)
8887
return err == nil
8988
}
9089

90+
func tableExists(ctx context.Context, database *bun.DB, name string) bool {
91+
var n int
92+
_ = database.NewSelect().
93+
TableExpr("?", bun.Ident(name)).
94+
ColumnExpr("1").
95+
Limit(1).
96+
Scan(ctx, &n)
97+
return n == 1
98+
}
99+
91100
const batchSize = 100
92101

93-
// columnTransforms maps (srcTable:django) to custom SELECT queries used when
94-
// column names or structure differ between Django and Go schemas.
95-
var columnTransforms = map[string]string{
102+
// dstColumns lists the expected columns for each destination table. When
103+
// the source table has fewer columns (older Django schema), missing
104+
// columns get default values.
105+
var dstColumns = map[string][]string{
106+
"project": {
107+
"id", "linkname", "name", "listid", "listemail",
108+
"subject_match", "web_url", "scm_url", "webscm_url",
109+
"list_archive_url", "list_archive_url_format",
110+
"commit_url_format", "send_notifications", "use_tags",
111+
"show_dependencies", "auto_supersede",
112+
},
113+
"series": {
114+
"id", "project_id", "cover_letter_id", "previous_series_id",
115+
"name", "date", "submitter_id", "version", "total",
116+
},
117+
"patch_comment": {
118+
"id", "msgid", "date", "headers", "submitter_id",
119+
"content", "patch_id", "addressed",
120+
},
121+
"cover_comment": {
122+
"id", "msgid", "date", "headers", "submitter_id",
123+
"content", "cover_id", "addressed",
124+
},
125+
}
126+
127+
// customQueries provide hand-written SELECT queries for tables where
128+
// the Django schema differs structurally from the Go schema (column
129+
// renames, joins, deduplication). Placeholders {column} are resolved
130+
// at runtime: expanded to "e.column" if the column exists in the
131+
// source table, or "NULL AS column" otherwise.
132+
var customQueries = map[string]string{
96133
// Django auth_user has is_staff (dropped) and is_superuser (renamed
97134
// to is_admin). Profile columns are merged from patchwork_userprofile.
98-
"auth_user:django": `SELECT u.id, u.username, u.password,
135+
"auth_user": `SELECT u.id, u.username, u.password,
99136
u.first_name, u.last_name, u.email,
100137
u.is_superuser AS is_admin, u.is_active,
101138
u.date_joined, u.last_login,
@@ -106,13 +143,13 @@ var columnTransforms = map[string]string{
106143
LEFT JOIN patchwork_userprofile p ON p.user_id = u.id`,
107144
// Django project_maintainer goes through userprofile; resolve to
108145
// user_id directly.
109-
"patchwork_userprofile_maintainer_projects:django": `SELECT m.id,
146+
"patchwork_userprofile_maintainer_projects": `SELECT m.id,
110147
up.user_id, m.project_id
111148
FROM patchwork_userprofile_maintainer_projects m
112149
JOIN patchwork_userprofile up ON up.id = m.userprofile_id`,
113150
// Deduplicate checks: Django accumulates duplicates per
114151
// (patch, context, user). Keep only the most recent.
115-
"patchwork_check:django": `SELECT c.id, c.patch_id, c.user_id, c.date,
152+
"patchwork_check": `SELECT c.id, c.patch_id, c.user_id, c.date,
116153
c.state, c.target_url, c.context, c.description
117154
FROM patchwork_check c
118155
INNER JOIN (
@@ -124,12 +161,14 @@ var columnTransforms = map[string]string{
124161
AND COALESCE(c.user_id, 0) = COALESCE(latest.user_id, 0)
125162
AND c.date = latest.max_date`,
126163
// Drop events that reference deduplicated checks.
127-
"patchwork_event:django": `SELECT e.id, e.project_id, e.category, e.date,
164+
"patchwork_event": `SELECT e.id, e.project_id, e.category, e.date,
128165
e.actor_id, e.patch_id, e.series_id, e.cover_id,
129166
e.previous_state_id, e.current_state_id,
130167
e.previous_delegate_id, e.current_delegate_id,
131168
e.previous_relation_id, e.current_relation_id,
132-
e.created_check_id, e.cover_comment_id, e.patch_comment_id
169+
e.created_check_id,
170+
{cover_comment_id},
171+
{patch_comment_id}
133172
FROM patchwork_event e
134173
WHERE e.created_check_id IS NULL
135174
OR e.created_check_id IN (
@@ -145,13 +184,89 @@ var columnTransforms = map[string]string{
145184
)`,
146185
}
147186

148-
func exportTableRows(ctx context.Context, database *bun.DB, w io.Writer, srcTable, dstTable string, isDjango bool) error {
149-
query := "SELECT * FROM " + srcTable
150-
if isDjango {
151-
if q, ok := columnTransforms[srcTable+":django"]; ok {
152-
query = q
187+
func buildQuery(ctx context.Context, database *bun.DB, srcTable, dstTable string) string {
188+
if q, ok := customQueries[srcTable]; ok {
189+
return resolveTemplateCols(ctx, database, srcTable, q)
190+
}
191+
192+
expected, ok := dstColumns[dstTable]
193+
if !ok {
194+
return "SELECT * FROM " + srcTable
195+
}
196+
197+
srcCols := probeColumns(ctx, database, srcTable)
198+
if len(srcCols) == 0 {
199+
return "SELECT * FROM " + srcTable
200+
}
201+
202+
var selects []string
203+
for _, col := range expected {
204+
if srcCols[col] {
205+
selects = append(selects, col)
206+
} else {
207+
selects = append(selects, columnDefault(col)+" AS "+col)
153208
}
154209
}
210+
return "SELECT " + strings.Join(selects, ", ") + " FROM " + srcTable
211+
}
212+
213+
// resolveTemplateCols replaces {column} placeholders in a query. If the
214+
// column exists in the source table, it expands to "e.column". If not,
215+
// it expands to "NULL AS column".
216+
func resolveTemplateCols(ctx context.Context, database *bun.DB, srcTable string, query string) string {
217+
srcCols := probeColumns(ctx, database, srcTable)
218+
219+
result := query
220+
for {
221+
start := strings.Index(result, "{")
222+
if start < 0 {
223+
break
224+
}
225+
end := strings.Index(result[start:], "}")
226+
if end < 0 {
227+
break
228+
}
229+
col := result[start+1 : start+end]
230+
var replacement string
231+
if srcCols[col] {
232+
replacement = "e." + col
233+
} else {
234+
replacement = "NULL AS " + col
235+
}
236+
result = result[:start] + replacement + result[start+end+1:]
237+
}
238+
return result
239+
}
240+
241+
func probeColumns(ctx context.Context, database *bun.DB, table string) map[string]bool {
242+
rows, err := database.QueryContext(ctx, "SELECT * FROM "+table+" LIMIT 0")
243+
if err != nil {
244+
return nil
245+
}
246+
defer rows.Close()
247+
248+
cols, err := rows.Columns()
249+
if err != nil {
250+
return nil
251+
}
252+
m := make(map[string]bool, len(cols))
253+
for _, c := range cols {
254+
m[c] = true
255+
}
256+
return m
257+
}
258+
259+
func columnDefault(col string) string {
260+
switch col {
261+
case "show_dependencies", "auto_supersede":
262+
return "false"
263+
default:
264+
return "NULL"
265+
}
266+
}
267+
268+
func exportTableRows(ctx context.Context, database *bun.DB, w io.Writer, srcTable, dstTable string) error {
269+
query := buildQuery(ctx, database, srcTable, dstTable)
155270

156271
rows, err := database.QueryContext(ctx, query)
157272
if err != nil {
@@ -200,12 +315,12 @@ func exportTableRows(ctx context.Context, database *bun.DB, w io.Writer, srcTabl
200315
func formatValues(vals []any) string {
201316
parts := make([]string, len(vals))
202317
for i, v := range vals {
203-
parts[i] = formatValue(v)
318+
parts[i] = formatSQLValue(v)
204319
}
205320
return strings.Join(parts, ", ")
206321
}
207322

208-
func formatValue(v any) string {
323+
func formatSQLValue(v any) string {
209324
if v == nil {
210325
return "NULL"
211326
}

0 commit comments

Comments
 (0)