@@ -16,86 +16,123 @@ import (
1616 "github.com/uptrace/bun"
1717)
1818
19- // exportTable defines a table to export and an optional source name to read
20- // from when the database uses Django naming conventions.
2119type exportTable struct {
22- name string
23- djangoName string
20+ // dstName is the table name in the Go schema.
21+ dstName string
22+ // srcName is the table name in the Django schema.
23+ srcName string
24+ // optional tables may not exist in older Django versions.
25+ optional bool
2426}
2527
26- // Order matters. Tables that references others must be dumped *after*.
28+ // Order matters. Tables that reference others must be dumped *after*.
2729var exportTables = []exportTable {
28- {name : "auth_user" , djangoName : "auth_user" },
29- {name : "auth_token" , djangoName : "authtoken_token" },
30- {name : "state" , djangoName : "patchwork_state" },
31- {name : "tag" , djangoName : "patchwork_tag" },
32- {name : "project" , djangoName : "patchwork_project" },
33- {name : "project_maintainer" , djangoName : "patchwork_userprofile_maintainer_projects" },
34- {name : "delegation_rule" , djangoName : "patchwork_delegationrule" },
35- {name : "person" , djangoName : "patchwork_person" },
36- {name : "patch_relation" , djangoName : "patchwork_patchrelation" },
37- {name : "series" , djangoName : "patchwork_series" },
38- {name : "series_reference" , djangoName : "patchwork_seriesreference" },
39- {name : "series_metadata" , djangoName : "patchwork_seriesmetadata" },
40- {name : "series_dependencies" , djangoName : "patchwork_series_dependencies" },
41- {name : "cover" , djangoName : "patchwork_cover" },
42- {name : "patch" , djangoName : "patchwork_patch" },
43- {name : "patch_tag" , djangoName : "patchwork_patchtag" },
44- {name : "patch_comment" , djangoName : "patchwork_patchcomment" },
45- {name : "cover_comment" , djangoName : "patchwork_covercomment" },
30+ {dstName : "auth_user" , srcName : "auth_user" },
31+ {dstName : "auth_token" , srcName : "authtoken_token" },
32+ {dstName : "state" , srcName : "patchwork_state" },
33+ {dstName : "tag" , srcName : "patchwork_tag" },
34+ {dstName : "project" , srcName : "patchwork_project" },
35+ {dstName : "project_maintainer" , srcName : "patchwork_userprofile_maintainer_projects" },
36+ {dstName : "delegation_rule" , srcName : "patchwork_delegationrule" },
37+ {dstName : "person" , srcName : "patchwork_person" },
38+ {dstName : "patch_relation" , srcName : "patchwork_patchrelation" },
39+ {dstName : "series" , srcName : "patchwork_series" },
40+ {dstName : "series_reference" , srcName : "patchwork_seriesreference" },
41+ {dstName : "series_metadata" , srcName : "patchwork_seriesmetadata" , optional : true },
42+ {dstName : "series_dependencies" , srcName : "patchwork_series_dependencies" , optional : true },
43+ {dstName : "cover" , srcName : "patchwork_cover" },
44+ {dstName : "patch" , srcName : "patchwork_patch" },
45+ {dstName : "patch_tag" , srcName : "patchwork_patchtag" },
46+ {dstName : "patch_comment" , srcName : "patchwork_patchcomment" },
47+ {dstName : "cover_comment" , srcName : "patchwork_covercomment" },
4648 // Deduplicate checks: keep only the latest per (patch, context, user).
4749 // Django accumulates duplicate checks; we enforce uniqueness.
48- {name : "ci_check" , djangoName : "patchwork_check" },
49- {name : "bundle" , djangoName : "patchwork_bundle" },
50- {name : "bundle_patch" , djangoName : "patchwork_bundlepatch" },
51- {name : "event" , djangoName : "patchwork_event" },
52- {name : "email_confirmation" , djangoName : "patchwork_emailconfirmation" },
53- {name : "webhook" , djangoName : "patchwork_webhook" },
50+ {dstName : "ci_check" , srcName : "patchwork_check" },
51+ {dstName : "bundle" , srcName : "patchwork_bundle" },
52+ {dstName : "bundle_patch" , srcName : "patchwork_bundlepatch" },
53+ {dstName : "event" , srcName : "patchwork_event" },
54+ {dstName : "email_confirmation" , srcName : "patchwork_emailconfirmation" },
55+ {dstName : "webhook" , srcName : "patchwork_webhook" , optional : true },
5456}
5557
56- // Export writes all table data as SQL INSERT statements to w. Tables are
57- // exported in foreign key dependency order so the output can be imported into
58- // a fresh database.
58+ // Export reads data from a Django patchwork database and writes SQL
59+ // INSERT statements to w, using the Go schema table and column names.
60+ // The output can be imported into a fresh database initialized with
61+ // "pw db sync". All patchwork 3.x schema variations are supported.
5962func Export (ctx context.Context , database * bun.DB , w io.Writer ) error {
60- isDjango := isDjangoDatabase (ctx , database )
63+ if ! hasDjangoMigrations (ctx , database ) {
64+ return fmt .Errorf ("not a Django database (django_migrations table not found)" )
65+ }
6166
6267 fmt .Fprint (w , "BEGIN;\n " )
6368
6469 for _ , t := range exportTables {
65- srcTable := t .name
66- if isDjango {
67- if t .djangoName == "" {
68- continue
69- }
70- srcTable = t .djangoName
70+ if t .optional && ! tableExists (ctx , database , t .srcName ) {
71+ continue
7172 }
72- if err := exportTableRows (ctx , database , w , srcTable , t . name , isDjango ); err != nil {
73- return fmt .Errorf ("export %s: %w" , t .name , err )
73+ if err := exportTableRows (ctx , database , w , t . srcName , t . dstName ); err != nil {
74+ return fmt .Errorf ("export %s: %w" , t .dstName , err )
7475 }
7576 }
7677
7778 fmt .Fprint (w , "COMMIT;\n " )
7879 return nil
7980}
8081
81- // isDjangoDatabase returns true if the database contains a django_migrations
82- // table, indicating it was created by Django.
83- func isDjangoDatabase (ctx context.Context , database * bun.DB ) bool {
82+ func hasDjangoMigrations (ctx context.Context , database * bun.DB ) bool {
8483 _ , err := database .NewSelect ().
8584 TableExpr ("django_migrations" ).
8685 Limit (1 ).
8786 Exec (ctx )
8887 return err == nil
8988}
9089
90+ func tableExists (ctx context.Context , database * bun.DB , name string ) bool {
91+ var n int
92+ _ = database .NewSelect ().
93+ TableExpr ("?" , bun .Ident (name )).
94+ ColumnExpr ("1" ).
95+ Limit (1 ).
96+ Scan (ctx , & n )
97+ return n == 1
98+ }
99+
91100const batchSize = 100
92101
93- // columnTransforms maps (srcTable:django) to custom SELECT queries used when
94- // column names or structure differ between Django and Go schemas.
95- var columnTransforms = map [string ]string {
102+ // dstColumns lists the expected columns for each destination table. When
103+ // the source table has fewer columns (older Django schema), missing
104+ // columns get default values.
105+ var dstColumns = map [string ][]string {
106+ "project" : {
107+ "id" , "linkname" , "name" , "listid" , "listemail" ,
108+ "subject_match" , "web_url" , "scm_url" , "webscm_url" ,
109+ "list_archive_url" , "list_archive_url_format" ,
110+ "commit_url_format" , "send_notifications" , "use_tags" ,
111+ "show_dependencies" , "auto_supersede" ,
112+ },
113+ "series" : {
114+ "id" , "project_id" , "cover_letter_id" , "previous_series_id" ,
115+ "name" , "date" , "submitter_id" , "version" , "total" ,
116+ },
117+ "patch_comment" : {
118+ "id" , "msgid" , "date" , "headers" , "submitter_id" ,
119+ "content" , "patch_id" , "addressed" ,
120+ },
121+ "cover_comment" : {
122+ "id" , "msgid" , "date" , "headers" , "submitter_id" ,
123+ "content" , "cover_id" , "addressed" ,
124+ },
125+ }
126+
127+ // customQueries provide hand-written SELECT queries for tables where
128+ // the Django schema differs structurally from the Go schema (column
129+ // renames, joins, deduplication). Placeholders {column} are resolved
130+ // at runtime: expanded to "e.column" if the column exists in the
131+ // source table, or "NULL AS column" otherwise.
132+ var customQueries = map [string ]string {
96133 // Django auth_user has is_staff (dropped) and is_superuser (renamed
97134 // to is_admin). Profile columns are merged from patchwork_userprofile.
98- "auth_user:django " : `SELECT u.id, u.username, u.password,
135+ "auth_user" : `SELECT u.id, u.username, u.password,
99136 u.first_name, u.last_name, u.email,
100137 u.is_superuser AS is_admin, u.is_active,
101138 u.date_joined, u.last_login,
@@ -106,13 +143,13 @@ var columnTransforms = map[string]string{
106143 LEFT JOIN patchwork_userprofile p ON p.user_id = u.id` ,
107144 // Django project_maintainer goes through userprofile; resolve to
108145 // user_id directly.
109- "patchwork_userprofile_maintainer_projects:django " : `SELECT m.id,
146+ "patchwork_userprofile_maintainer_projects" : `SELECT m.id,
110147 up.user_id, m.project_id
111148 FROM patchwork_userprofile_maintainer_projects m
112149 JOIN patchwork_userprofile up ON up.id = m.userprofile_id` ,
113150 // Deduplicate checks: Django accumulates duplicates per
114151 // (patch, context, user). Keep only the most recent.
115- "patchwork_check:django " : `SELECT c.id, c.patch_id, c.user_id, c.date,
152+ "patchwork_check" : `SELECT c.id, c.patch_id, c.user_id, c.date,
116153 c.state, c.target_url, c.context, c.description
117154 FROM patchwork_check c
118155 INNER JOIN (
@@ -124,12 +161,14 @@ var columnTransforms = map[string]string{
124161 AND COALESCE(c.user_id, 0) = COALESCE(latest.user_id, 0)
125162 AND c.date = latest.max_date` ,
126163 // Drop events that reference deduplicated checks.
127- "patchwork_event:django " : `SELECT e.id, e.project_id, e.category, e.date,
164+ "patchwork_event" : `SELECT e.id, e.project_id, e.category, e.date,
128165 e.actor_id, e.patch_id, e.series_id, e.cover_id,
129166 e.previous_state_id, e.current_state_id,
130167 e.previous_delegate_id, e.current_delegate_id,
131168 e.previous_relation_id, e.current_relation_id,
132- e.created_check_id, e.cover_comment_id, e.patch_comment_id
169+ e.created_check_id,
170+ {cover_comment_id},
171+ {patch_comment_id}
133172 FROM patchwork_event e
134173 WHERE e.created_check_id IS NULL
135174 OR e.created_check_id IN (
@@ -145,13 +184,89 @@ var columnTransforms = map[string]string{
145184 )` ,
146185}
147186
148- func exportTableRows (ctx context.Context , database * bun.DB , w io.Writer , srcTable , dstTable string , isDjango bool ) error {
149- query := "SELECT * FROM " + srcTable
150- if isDjango {
151- if q , ok := columnTransforms [srcTable + ":django" ]; ok {
152- query = q
187+ func buildQuery (ctx context.Context , database * bun.DB , srcTable , dstTable string ) string {
188+ if q , ok := customQueries [srcTable ]; ok {
189+ return resolveTemplateCols (ctx , database , srcTable , q )
190+ }
191+
192+ expected , ok := dstColumns [dstTable ]
193+ if ! ok {
194+ return "SELECT * FROM " + srcTable
195+ }
196+
197+ srcCols := probeColumns (ctx , database , srcTable )
198+ if len (srcCols ) == 0 {
199+ return "SELECT * FROM " + srcTable
200+ }
201+
202+ var selects []string
203+ for _ , col := range expected {
204+ if srcCols [col ] {
205+ selects = append (selects , col )
206+ } else {
207+ selects = append (selects , columnDefault (col )+ " AS " + col )
153208 }
154209 }
210+ return "SELECT " + strings .Join (selects , ", " ) + " FROM " + srcTable
211+ }
212+
213+ // resolveTemplateCols replaces {column} placeholders in a query. If the
214+ // column exists in the source table, it expands to "e.column". If not,
215+ // it expands to "NULL AS column".
216+ func resolveTemplateCols (ctx context.Context , database * bun.DB , srcTable string , query string ) string {
217+ srcCols := probeColumns (ctx , database , srcTable )
218+
219+ result := query
220+ for {
221+ start := strings .Index (result , "{" )
222+ if start < 0 {
223+ break
224+ }
225+ end := strings .Index (result [start :], "}" )
226+ if end < 0 {
227+ break
228+ }
229+ col := result [start + 1 : start + end ]
230+ var replacement string
231+ if srcCols [col ] {
232+ replacement = "e." + col
233+ } else {
234+ replacement = "NULL AS " + col
235+ }
236+ result = result [:start ] + replacement + result [start + end + 1 :]
237+ }
238+ return result
239+ }
240+
241+ func probeColumns (ctx context.Context , database * bun.DB , table string ) map [string ]bool {
242+ rows , err := database .QueryContext (ctx , "SELECT * FROM " + table + " LIMIT 0" )
243+ if err != nil {
244+ return nil
245+ }
246+ defer rows .Close ()
247+
248+ cols , err := rows .Columns ()
249+ if err != nil {
250+ return nil
251+ }
252+ m := make (map [string ]bool , len (cols ))
253+ for _ , c := range cols {
254+ m [c ] = true
255+ }
256+ return m
257+ }
258+
259+ func columnDefault (col string ) string {
260+ switch col {
261+ case "show_dependencies" , "auto_supersede" :
262+ return "false"
263+ default :
264+ return "NULL"
265+ }
266+ }
267+
268+ func exportTableRows (ctx context.Context , database * bun.DB , w io.Writer , srcTable , dstTable string ) error {
269+ query := buildQuery (ctx , database , srcTable , dstTable )
155270
156271 rows , err := database .QueryContext (ctx , query )
157272 if err != nil {
@@ -200,12 +315,12 @@ func exportTableRows(ctx context.Context, database *bun.DB, w io.Writer, srcTabl
200315func formatValues (vals []any ) string {
201316 parts := make ([]string , len (vals ))
202317 for i , v := range vals {
203- parts [i ] = formatValue (v )
318+ parts [i ] = formatSQLValue (v )
204319 }
205320 return strings .Join (parts , ", " )
206321}
207322
208- func formatValue (v any ) string {
323+ func formatSQLValue (v any ) string {
209324 if v == nil {
210325 return "NULL"
211326 }
0 commit comments