Repository navigation
Expand file tree
/
Copy pathatomic_image_builder.py
More file actions
executable file
·8160 lines (7631 loc) · 413 KB
/
Copy pathatomic_image_builder.py
File metadata and controls
executable file
·8160 lines (7631 loc) · 413 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
from __future__ import annotations
import base64
import http.client
import json
import os
import re
import secrets
import shlex
import shutil
import subprocess
import sys
import tempfile
import textwrap
import urllib.error
import urllib.parse
import urllib.request
from collections.abc import Callable, Iterable, Sequence
from contextlib import ExitStack
from dataclasses import asdict, dataclass, field
from dataclasses import fields as dataclass_fields
from datetime import datetime, timedelta, timezone, tzinfo
from pathlib import Path, PurePosixPath
if sys.version_info < (3, 10): # noqa: UP036
raise SystemExit("Python 3.10 or newer is required.")
# This file intentionally keeps the whole beginner-focused tool in one module.
# The runtime model is:
# 1. collect choices from the user into Config
# 2. validate and normalize that Config
# 3. write a canonical state file so future updates do not need fragile parsing
# 4. render a GitHub repo from a pinned template snapshot plus generated files
# 5. let GitHub Actions build and sign the image
#
# A future refactor could split UI, GitHub operations, and rendering into
# separate modules, but the comments below aim to make the current layout easier
# to understand for anyone reading it now.
VERSION = "0.12.0"
TOOL_NAME = "Atomic Image Builder"
TOOL_SLUG = "atomic-image-builder"
# The command users type. Deliberately separate from TOOL_SLUG: that one names
# the product and, through STATE_FILE, is written into every managed repo, so
# renaming it would orphan every repo this tool has ever created. This one only
# has to match whatever the installers put on PATH.
#
# Not plain "aib" -- contrib/aib already installs a host wrapper by that name to
# ~/.local/bin, which on a normal PATH sits ahead of Homebrew's bin. Sharing the
# name would mean whichever came first silently won, with no way for a user to
# tell which one they were running.
TOOL_COMMAND = "aib-tool"
STATE_FILE = f".{TOOL_SLUG}.json"
# `gh repo list --limit N` is a cap, not a page size: gh pages on its own
# until it has N repos. At 100 an account with more repos than that lost every
# managed repo past the 100 most recent, and the picker then said none
# existed. This means "every repo" for any real account while still bounding
# a pathological one.
REPO_LIST_LIMIT = 10000
# Repos checked per GraphQL query in batch_check_state_files. One alias per
# repo; a few thousand in a single query would run into GitHub's query limits.
STATE_FILE_QUERY_BATCH = 100
DEFAULT_REPO_NAME = "my-atomic-image"
# Said the same way wherever a name is refused, because the wizard's field
# error and validate_config()'s final gate are the same rule.
REPO_NAME_RULE = (
"It must start and end with a letter or number, cannot end with .git, and can join "
"parts only with a single dot, one or two underscores, or dashes."
)
DEFAULT_GITHUB_BUILD_CRON = "05 10 * * *"
FEDORA_ATOMIC_FALLBACK_TAG = "44"
# Pinned by digest rather than by the tag it is published under. Every other
# third-party input this tool writes into a stranger's repository is pinned --
# actions through ACTION_PINS, downloads by sha256 -- and the payload below is
# the one that was not, even though it is the only one whose whole file tree is
# copied into `/` of an image the generated workflow then signs. A tag that
# moves upstream changes what a signed image contains without changing a byte
# of the repository that built it, and nothing in a generated repo records
# which payload went in, so the change leaves no trace to find afterwards.
#
# Refreshing this is a deliberate step, not a rubber stamp: the digest names
# the exact payload tests/test_brew_login_fragments.py and
# tests/test_brew_setup_staging.py reproduce, and a new one can ship files
# neither of them knows about. maintenance_audit.py --check-action-updates
# reports weekly when the tag stops resolving here, so the pin going stale is
# visible rather than silent.
UNIVERSAL_BLUE_BREW_IMAGE_REPO = "ghcr.io/ublue-os/brew"
UNIVERSAL_BLUE_BREW_IMAGE_TAG = "latest"
UNIVERSAL_BLUE_BREW_IMAGE_DIGEST = "sha256:60ada2d65891d8797beef49d8b43f2108519cbbaf04c9c7363e1a008677fcd35"
UNIVERSAL_BLUE_BREW_IMAGE = f"{UNIVERSAL_BLUE_BREW_IMAGE_REPO}@{UNIVERSAL_BLUE_BREW_IMAGE_DIGEST}"
# The other third-party image the tool ships to strangers, pinned for the
# same reason. The bundled disk workflow names bootc-image-builder by its
# :latest tag in `BIB_IMAGE`, and image-template.env names it again for the
# Justfile's local `just build-qcow2`; the action that runs it is pinned by
# SHA through ACTION_REF_PINS, the image it runs to produce the user's qcow2
# and installer ISO was not. It is the one builder that touches every disk
# artifact a generated repository publishes.
#
# What this pin costs is compatibility rather than freshness: the builder has
# to keep up with the bootc in whichever base image the user chose, and a
# pin holds it still while the base images move. That is why the weekly
# advisory below exists -- a re-pin is a check that the new builder still
# handles the base images the wizard offers, not a digest to copy across.
BOOTC_IMAGE_BUILDER_IMAGE_REPO = "quay.io/centos-bootc/bootc-image-builder"
BOOTC_IMAGE_BUILDER_IMAGE_TAG = "latest"
BOOTC_IMAGE_BUILDER_IMAGE_DIGEST = "sha256:2b52843ea2bfda73b0a08d97e76b734393b1d3a804681b9fabb26723bd3a2f0b"
BOOTC_IMAGE_BUILDER_IMAGE = f"{BOOTC_IMAGE_BUILDER_IMAGE_REPO}@{BOOTC_IMAGE_BUILDER_IMAGE_DIGEST}"
# The brew payload's /system_files carries three login-shell fragments
# alongside the units and the tarball, and none of them is guarded:
#
# /etc/profile.d/brew.sh evals `brew shellenv`
# /etc/profile.d/brew-bash-completion.sh runs `brew completions link`,
# then sources every file in the
# prefix's etc/bash_completion.d
# /usr/share/fish/vendor_conf.d/ublue-brew.fish the fish equivalent
#
# All three execute code out of /home/linuxbrew/.linuxbrew, and
# brew-setup.service -- which the block presets -- ends with
# `chown -R 1000:1000 /home/linuxbrew`. /etc/profile.d and the fish vendor
# directory are read by every login shell, the root account's included (`su -`,
# `sudo -i`, a console or SSH root login), so on a booted machine each fragment
# is UID 1000 choosing what root executes, with no password and no sudo record.
#
# This tool only offers Homebrew for a Fedora Atomic or CentOS base
# (offer_brew_if_applicable forces it off for Universal Blue, which ships its
# own), so the block is what puts the fragments in the image at all.
BREW_LOGIN_FRAGMENTS = (
"/etc/profile.d/brew.sh",
"/etc/profile.d/brew-bash-completion.sh",
"/usr/share/fish/vendor_conf.d/ublue-brew.fish",
)
# The replacement. It puts the prefix on PATH for the account that owns it and
# executes nothing out of it, so `brew` still works for that user.
BREW_PATH_FRAGMENT = "/etc/profile.d/brew-path.sh"
# Everything a login shell reads on its way in. The sweep is the load-bearing
# half of the fix: the payload is an image this tool does not build, pulled by a
# mutable tag, so deleting three names fixes today's digest and says nothing
# about the next one. Only a check of what actually landed can see a fourth.
BREW_LOGIN_SHELL_DIRS = (
"/etc/profile.d",
"/etc/fish/conf.d",
"/usr/share/fish/vendor_conf.d",
)
# The other half of what the payload does on a booted machine. brew-setup.service
# stages a 154 MB tarball through one fixed path, as root, with no `PrivateTmp=`:
#
# ExecStart=/usr/bin/mkdir -p /tmp/homebrew
# ExecStart=/usr/bin/tar --zstd -xf /usr/share/homebrew.tar.zst -C /tmp/homebrew
# ExecStart=/usr/bin/cp -R -n /tmp/homebrew/home/linuxbrew/.linuxbrew /home/linuxbrew
# ExecStart=/usr/bin/chown -R 1000:1000 /home/linuxbrew
#
# /tmp on a booted system is a world-writable tmpfs, and `mkdir -p` exits 0 on an
# existing symlink instead of replacing it -- so an account that claims
# /tmp/homebrew first has root extract through its symlink, has whatever it left
# there copied into /home/linuxbrew, and has it handed to UID 1000 by the chown.
# The `-n` protects nothing on the run that matters: the unit's own
# `ConditionPathExists=!/home/linuxbrew/.linuxbrew` guarantees the destination
# does not exist yet. The `--mount=type=tmpfs,dst=/tmp` on the preset RUN is a
# build-time mount and has no bearing on any of this.
BREW_SETUP_UNIT = "/usr/lib/systemd/system/brew-setup.service"
BREW_SETUP_DROPIN = "/usr/lib/systemd/system/brew-setup.service.d/10-private-tmp.conf"
MAX_UI_WIDTH = 120
# Every colour here is a 256-palette index chosen to stay legible on BOTH a
# light and a dark terminal. The originals were picked on a dark background and
# were near-invisible on a light one -- 252 is almost white, and 117 is a pale
# blue that gum was also given for `choose --selected.foreground`, so on a light
# terminal an all-selected list rendered as an empty screen.
#
# The rule for anything added here: mid-range indices only. Nothing above ~230
# (washes out on white) and nothing below ~20 (disappears on black).
ACCENT_COLOR = 33 # blue, for headers, cursors and selections
MUTED_COLOR = 244 # grey, for secondary text and placeholders
SUCCESS_COLOR = 28 # green, for the "created" panel
WARNING_COLOR = 178 # gold, for the attention panel
NOTICE_COLOR = 214 # orange, already legible on both
CONTROLS_COLOR = SUCCESS_COLOR
PACKAGE_SEARCH_LIMIT = 40
MANAGED_REPO_WARNING = "If you hand-edit a repo after this tool creates or manages it, stop using this tool for that repo."
MANAGED_REPO_HINT_CONTAINERFILE = (
f"Future updates use {STATE_FILE} as the source of truth and rewrite managed files such as README.md and build_files/build.sh."
)
MANAGED_REPO_HINT_BLUEBUILD = (
f"Future updates use {STATE_FILE} as the source of truth and rewrite managed files such as README.md and recipes/recipe.yml."
)
# Lead-in to the reset-then-switch instructions in the README, the creation
# panel and the build-status screen. It must hold whether the scan carried
# packages into the image or only found customizations the user agreed to
# leave behind: the build-status screen reads the single state-file flag and
# cannot tell the two apart, and an update regenerates the README from that
# same flag. So it says where the repo came from, not what the image carries.
SCAN_SWITCH_LEAD_IN = "This repo was created from a scan of your current system's rpm-ostree customizations."
CONTAINERFILE_TEMPLATE_REPO = "ublue-os/image-template"
BLUEBUILD_TEMPLATE_REPO = "blue-build/template"
TEMPLATE_SNAPSHOT_DIR = Path(__file__).resolve().parent / "template_snapshots"
CONTAINERFILE_TEMPLATE_DIR = TEMPLATE_SNAPSHOT_DIR / "containerfile"
BLUEBUILD_TEMPLATE_DIR = TEMPLATE_SNAPSHOT_DIR / "bluebuild"
ALLOWED_METHODS = {"containerfile", "bluebuild"}
METHOD_DISPLAY = {"containerfile": "Containerfile", "bluebuild": "BlueBuild"}
BLUEBUILD_RECIPE_SCHEMA = "https://schema.blue-build.org/recipe-v1.json"
# These regexes are our low-cost safety rails. They do not prove a package or
# service is real, but they do stop obviously unsafe values from becoming shell
# script content later.
#
# The first character must be alphanumeric: a token such as "--version" or
# "-y" is never a package name, and dnf5 reads it as an option wherever the
# list lands -- the repoquery lookup (which then prints its version and
# reports every real name in the batch as missing) and the generated
# `dnf5 install -y` line. The regex is the only guard, because dnf5 has no
# "--" end-of-options separator: 5.4.2.1 rejects it on repoquery, install and
# remove with 'Unknown argument "--"', so emitting one would break every
# lookup and every build.sh instead of hardening them.
PACKAGE_TOKEN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+:-]*$")
# What `dnf5 copr enable` takes: the owner is a username or a @groupname
# (@caddy/caddy), and the project may be a project directory with colons
# (owner/project:custom:123). The "@" is allowed only as the first character
# and the colons only after the slash, so this stays a repo spec and not a
# shell or YAML surprise -- shell_quote and yaml_scalar quote it downstream
# regardless. As with packages, the owner cannot start with "-": "-x/y" lands
# on `dnf5 copr enable` as an option ("-x" is --exclude), and dnf5 has no "--"
# to put in front of it.
COPR_REPO_RE = re.compile(r"^@?[A-Za-z0-9._][A-Za-z0-9._-]*/[A-Za-z0-9._:-]+$")
# A unit name cannot start with "-" either. "--now" otherwise passes as a
# service and becomes a bare `systemctl enable --now` line in build.sh, which
# fails under `set -e` on every build, and the BlueBuild systemd module passes
# the same token to its own systemctl call, where no "--" can be added.
SERVICE_TOKEN_RE = re.compile(r"^[A-Za-z0-9@._:+][A-Za-z0-9@._:+-]*$")
# The repo name becomes the image name in ghcr.io/<owner>/<repo>, so it has to
# satisfy the container-reference grammar as well as GitHub's naming rules.
# This is distribution/reference's path-component production: alphanumeric runs
# joined by a single dot, one or two underscores, or a run of hyphens. Names
# GitHub accepts but that grammar does not -- "a..b", "a.-b", "a___b" -- parse
# as an invalid reference, so the wizard would create a repo and signing key
# for an image that nothing can push or pull.
OCI_PATH_COMPONENT_RE = re.compile(r"[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*")
# A "./"-prefixed entry in a workflow path filter. The prefix is repeated in
# the pattern rather than matched once so "././x" normalises in a single pass.
RELATIVE_PATH_FILTER_RE = re.compile(r"^(\s*-\s+)(['\"]?)(?:\./)+(.*)$")
# Matched on the action name rather than a pinned SHA, so the step is still
# found after a pin refresh moves the SHA out from under this patcher.
UPLOAD_ARTIFACT_ACTION = "actions/upload-artifact@"
DISK_ARTIFACT_NAME = "${{ env.IMAGE_NAME }}-${{ matrix.disk-type }}"
# The disk workflow's runner ternary. The amd64 label is captured rather than
# written out, so a snapshot refresh that bumps the runner image is followed
# automatically instead of being pinned to whatever was current when this was
# written.
DISK_RUNNER_CHOICE_RE = re.compile(
r"^(\s*runs-on:\s*)\$\{\{\s*inputs\.platform\s*==\s*'amd64'\s*&&\s*'([^']+)'\s*\|\|\s*'[^']+'\s*\}\}\s*$"
)
# The two conditions upstream ships on the platform input. The first gates a
# step that only ever ran on ARM, so it goes with ARM; the second is now
# always true, so only the condition goes.
DISK_ARM_ONLY_IF = "if: inputs.platform == 'arm64'"
DISK_NON_ARM_IF = "if: inputs.platform != 'arm64'"
# `BIB_IMAGE` as the bundled template spells it, in the workflow's env block
# (`key: "value"`) and in image-template.env (`key="value"`). The reference is
# matched on its repository and any tag or digest, so a repository written
# before the pin, or on an earlier one, is re-pinned on its next update. A
# different repository is the user's own choice of builder and passes through
# untouched, the way every patcher here no-ops on text it does not recognise.
DISK_BUILDER_IMAGE_RE = re.compile(
r"^(?P<prefix>\s*BIB_IMAGE[=:]\s*)(?P<quote>['\"]?)"
+ re.escape(BOOTC_IMAGE_BUILDER_IMAGE_REPO)
+ r"(?:[:@][^'\"\s#]*)?(?P=quote)(?P<suffix>\s*(?:#.*)?)$"
)
# The bundled Justfile's spawn-vm rebuild dispatch, and what it should say.
# "the ISO" in upstream's message is wrong for every type but one, so it goes
# with the arguments.
SPAWN_VM_REBUILD_LINE = (
' [ "{{ rebuild }}" -eq 1 ] && echo "Rebuilding the ISO" && just build-vm {{ rebuild }} {{ type }}'
)
SPAWN_VM_REBUILD_FIXED = (
' [ "{{ rebuild }}" -eq 1 ] && echo "Rebuilding the {{ type }} image" && just rebuild-{{ type }}'
)
FROM_LINE_RE = re.compile(r"^(\s*FROM(?:\s+--platform=\S+)?\s+)(\S+)(.*)$", flags=re.IGNORECASE)
FROM_STAGE_NAME_RE = re.compile(r"^\s+AS\s+(\S+)", flags=re.IGNORECASE)
# BuildKit (frontend/dockerfile/parser) reads heredocs only from RUN, COPY and
# ADD, and only from a whole shell word -- split with quotes kept -- shaped
# <<[-]WORD. So "<<EOF" in quotes, cat<<EOF and a <<< here-string open none.
CONTAINERFILE_HEREDOC_INSTRUCTION_RE = re.compile(r"^\s*(?:ONBUILD\s+)?(?:RUN|COPY|ADD)\s", flags=re.IGNORECASE)
CONTAINERFILE_HEREDOC_WORD_RE = re.compile(r"^\d*<<(-?)\s*([^<]*)$")
# A parser directive (escape, syntax, check) is honoured only in the run of
# "# key=value" lines at the very top of the file; any other line -- a blank
# one, another comment or an instruction -- ends that run for good.
CONTAINERFILE_DIRECTIVE_RE = re.compile(r"^#\s*([a-zA-Z][a-zA-Z0-9]*)\s*=\s*(.+?)\s*$")
CONTAINERFILE_DIRECTIVE_NAMES = frozenset({"escape", "syntax", "check"})
# BuildKit's line-continuation test for each allowed escape character: the
# character, not itself escaped, then only spaces and tabs to the end.
CONTAINERFILE_CONTINUATION_RE = {
escape: re.compile(rf"(?:(?<=[^{re.escape(escape)}])|^){re.escape(escape)}[ \t]*$") for escape in ("\\", "`")
}
INSTALLER_SWITCH_RE = re.compile(r"^(\s*bootc switch --mutate-in-place --transport registry )(\S+)(.*)$")
INSTALLER_UNVERIFIED_SWITCH_COMMENT = (
"# Signature enforcement is deliberately omitted for this installer switch:",
"# the standalone installer has no trusted copy of this repo's cosign.pub",
"# before its first pull. After installation, follow README's Trusting The",
"# Signing Key section and repeat the enforced switch so later upgrades verify it.",
)
# dnf5 prints this when -C (cache-only) is used and no repository metadata has
# been downloaded yet. Matched so package search and the exact-name check can
# offer to fix it in place instead of naming a command the user may have no
# shell to run. It and DNF5_MISSING_MARKERS are dnf5's English messages, which
# is why dnf5_command() pins the locale for every dnf5 run the tool makes.
DNF5_NO_CACHE_MARKER = "cache-only enabled but no cache"
PACKAGE_SEARCH_NEEDS_METADATA = (
"Package search needs local DNF metadata. "
"Exact-name entry still works without it, but names are then only checked by the GitHub build."
)
DNF5_MISSING_MARKERS = (
"no matches found",
"no package matched",
"no packages to list",
"matched no packages",
"no matching packages",
)
# Every spelling of a package that `dnf5 install` and `rpm -q` accept as a
# NEVRA spec: libdnf5's NAME, NA, NEV, NEVR and NEVRA forms, each with and
# without the epoch (dnf5 prints %{epoch} as 0 when the package has none,
# and an explicit 0: still matches it). A positional repoquery matches a spec
# ignoring case where install and rpm -q do not, so the exact-name check asks
# dnf5 to print these for whatever it matched and requires the spec to be one
# of them verbatim -- see _resolve_package_spec.
DNF5_SPEC_SPELLINGS = (
"%{name}",
"%{name}.%{arch}",
"%{name}-%{version}",
"%{name}-%{version}-%{release}",
"%{name}-%{version}-%{release}.%{arch}",
"%{name}-%{epoch}:%{version}",
"%{name}-%{epoch}:%{version}-%{release}",
"%{name}-%{epoch}:%{version}-%{release}.%{arch}",
)
# What `rpm -q` prints, on stdout, for each spec it was asked about that is
# not installed. The spec comes back verbatim, so it is the key the removal
# lookup matches against -- see lookup_installed_host_packages for why the
# printed %{name} cannot be.
RPM_NOT_INSTALLED_RE = re.compile(r"^package (.+) is not installed$")
# GitHub Actions should be pinned to immutable SHAs instead of floating tags.
# The human-readable tag is kept as a comment so maintainers can still tell what
# upstream version the pin came from.
ACTION_PINS: dict[str, tuple[str, str]] = {
"actions/checkout": ("3d3c42e5aac5ba805825da76410c181273ba90b1", "v7"),
"actions/setup-python": ("5fda3b95a4ea91299a34e894583c3862153e4b97", "v7"),
"ublue-os/remove-unwanted-software": ("695eb75bc387dbcd9685a8e72d23439d8686cba6", "master"),
"docker/metadata-action": ("dc802804100637a589fabce1cb79ff13a1411302", "v6.2.0"),
"redhat-actions/buildah-build": ("719e3c40d8af9790c23eca13f7daa339f2867034", "v3.1.0"),
"docker/login-action": ("dbcb813823bdd20940b903addbd779551569679f", "v4.6.0"),
"redhat-actions/push-to-registry": ("94ade333c38ecc0e60e94785125d9a52ca423b37", "v3.0.0"),
"sigstore/cosign-installer": ("6f9f17788090df1f26f669e9d70d6ae9567deba6", "v4.1.2"),
"actions/upload-artifact": ("043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", "v7.0.1"),
"blue-build/github-action": ("c295af864f2802fc6aea227507af43d35823db81", "v1.13"),
"extractions/setup-just": ("53165ef7e734c5c07cb06b3c8e7b647c5aa16db3", "v4.0.0"),
}
ACTION_REF_PINS: dict[str, tuple[str, str]] = {
"ublue-os/remove-unwanted-software@v8": ("695eb75bc387dbcd9685a8e72d23439d8686cba6", "v8"),
"ublue-os/remove-unwanted-software@695eb75bc387dbcd9685a8e72d23439d8686cba6": ("695eb75bc387dbcd9685a8e72d23439d8686cba6", "v8"),
"ublue-os/remove-unwanted-software@v9": ACTION_PINS["ublue-os/remove-unwanted-software"],
"ublue-os/remove-unwanted-software@cc0becac701cf642c8f0a6613bbdaf5dc36b259e": ACTION_PINS["ublue-os/remove-unwanted-software"],
"osbuild/bootc-image-builder-action@main": ("56d652d0afb02eb3e4b8fd35e7ca0391dbebab2a", "main"),
"osbuild/bootc-image-builder-action@5fc2ef0c4689b43ba959a10e3dfed3a889810ba1": ("56d652d0afb02eb3e4b8fd35e7ca0391dbebab2a", "main"),
"osbuild/bootc-image-builder-action@8661cd3832544ad68c12dcde8681b13ab0f56a8d": ("56d652d0afb02eb3e4b8fd35e7ca0391dbebab2a", "main"),
"osbuild/bootc-image-builder-action@56d652d0afb02eb3e4b8fd35e7ca0391dbebab2a": ("56d652d0afb02eb3e4b8fd35e7ca0391dbebab2a", "main"),
"actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f": ACTION_PINS["actions/upload-artifact"],
"blue-build/github-action@836161eb076426a451e6a0054f722b1153b8b3ad": ACTION_PINS["blue-build/github-action"],
"sigstore/cosign-installer@v4.0.0": ("faadad0cce49287aee09b3a48701e75088a2c6ad", "v4.0.0"),
"sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad": ("faadad0cce49287aee09b3a48701e75088a2c6ad", "v4.0.0"),
}
# The BlueBuild CLI a local test build renders `recipes/recipe.yml` with. It
# is fetched the way both blue-build/github-action and BlueBuild's own
# install.sh fetch it: create a container from the installer image and copy
# `/out/bluebuild` out of it. The tag is the floating one the pinned action
# above installs by default (`CLI_VERSION_TAG` in its "Determine Vars" step);
# the digest is what podman is given. The binary runs on the host, as the
# user, with the user's files and credentials in reach, so like the other two
# third-party images above it is pinned to the bytes that were reviewed rather
# than to a tag anyone with push access to the repository can repoint.
#
# The cost is the one every pin has, and here it is parity: CI pulls the tag
# and moves when it moves, while this stays until re-pinned, so a local build
# can render with a CLI a step behind CI's. maintenance_audit.py's weekly run
# reports when the tag no longer resolves to this digest
# (audit_bluebuild_cli_image_pin); re-pin after a local build against the
# wizard's recipe still renders, and re-read the action's default tag whenever
# the action pin changes.
BLUEBUILD_CLI_INSTALLER_IMAGE_REPO = "ghcr.io/blue-build/cli"
BLUEBUILD_CLI_INSTALLER_IMAGE_TAG = "v0.9-installer"
BLUEBUILD_CLI_INSTALLER_IMAGE_DIGEST = "sha256:1a53a3a145068a38d1c20b00a6047dd2c5e8848f7cd6b6c89d49c793ebd76fd6"
BLUEBUILD_CLI_INSTALLER_IMAGE = f"{BLUEBUILD_CLI_INSTALLER_IMAGE_REPO}@{BLUEBUILD_CLI_INSTALLER_IMAGE_DIGEST}"
# Written into the local build context when no real key exists yet. The
# recipe's signing module only copies `cosign.pub` into /etc/pki/containers
# and names it in policy.json; nothing at build time parses it. A repo the
# tool created has a real key, and the update path uses that one.
LOCAL_BUILD_PLACEHOLDER_COSIGN_PUB = (
"-----BEGIN PLACEHOLDER-----\n"
f"Written by {TOOL_NAME} for a local test build only.\n"
"The real cosign.pub is generated when the repository is created.\n"
"-----END PLACEHOLDER-----\n"
)
# The rechunk step's Justfile invocation, matched so Chunkah can be swapped in
# for rpm-ostree. Upstream has shipped two spellings: the pre-rootless
# `sudo -E $(command -v just) ostree-rechunk` and, since ublue-os/image-template
# b9783f6, a bare `just ostree-rechunk`. Existing managed repositories still
# carry the older one, so both must keep matching -- a single-shape matcher
# would rename the step to Chunkah while leaving it running rpm-ostree.
# Step names whose body the rechunk recipe swap may rewrite. Upstream ships the
# first; the second is what we rename it to, kept in scope so a half-switched
# workflow is repaired rather than left alone.
RECHUNK_STEP_NAMES: frozenset[str] = frozenset(
{"- name: Rechunk with rpm-ostree", "- name: Rechunk with Chunkah"}
)
RECHUNK_RECIPE_RE = re.compile(r"(?<![\w./-])(just\)?)(\s+)ostree-rechunk\b")
# Upstream's trailing "if you are feeling adventurous" comment block, in both
# spellings, stripped once the active step above it is running Chunkah. Matched
# on upstream's exact literal text, same no-op-on-drift convention as every
# other patcher in this file.
STALE_CHUNKAH_COMMENT_BLOCKS: tuple[str, ...] = tuple(
(
" # If you are feeling adventurous, use the new distro agnostic rechunker\n"
" # https://github.com/coreos/chunkah\n"
" # You can delete the Rechunk with rpm-ostree portion then if you use this\n"
" #- name: Rechunk with Chunkah\n"
" # id: rechunk\n"
" # run: |\n"
f" # {invocation} rechunk \\\n"
" # ${IMAGE_NAME} \\\n"
" # ${DEFAULT_TAG}\n"
"\n"
)
for invocation in ("just", "sudo -E $(command -v just)")
)
# "Is the user logged in to GitHub" asks about the active github.com account
# only. Bare `gh auth status` checks every account on every configured host and
# exits 1 if any one fails, so a stale GHES login or an old inactive github.com
# account made a working login look missing and sent the user into the login
# guide on every launch (#513). contrib/aib and container/entrypoint.sh use the
# same probe. --active needs gh 2.40 or later.
GH_AUTH_PROBE = ("gh", "auth", "status", "--hostname", "github.com", "--active")
PRECHECK_REQUIRED_TOOLS: tuple[str, ...] = ("gum", "git", "gh", "cosign")
BREW_INSTALLABLE_TOOLS: tuple[str, ...] = ("gum", "git", "gh", "cosign")
HOST_REQUIRED_TOOLS: tuple[str, ...] = ("dnf5", "rpm-ostree")
@dataclass(frozen=True)
class BaseImage:
# This is the small curated image list shown to beginners. Keeping it as a
# dataclass instead of plain dicts gives the rest of the code typed fields
# and avoids magic string lookups.
key: str
provider: str
name: str
description: str
image_uri: str
# Whether the base runs a first-boot account wizard (gnome-initial-setup,
# or Fedora's desktop-neutral initial-setup) after Anaconda finishes.
# installer_profile() reads this to pick the ISO config: upstream's
# iso-gnome.toml disables Anaconda's Users module on the assumption that
# the wizard creates the account, so a base without one installs a system
# nobody can log in to (#361). No default on purpose -- a new entry has to
# say which it is, because the wrong guess is invisible until first boot.
first_boot_setup: bool
# Other repositories the same desktop is published at, without a tag.
# match_base_image() treats a host booted from one of these as this
# curated image; image_uri stays the one the tool recommends and writes.
aliases: tuple[str, ...] = ()
def read_os_release_fields(path: Path = Path("/etc/os-release")) -> dict[str, str]:
try:
text = path.read_text()
except OSError:
return {}
fields: dict[str, str] = {}
for raw_line in text.splitlines():
line = raw_line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}:
value = value[1:-1]
fields[key] = value
return fields
def determine_fedora_atomic_default_tag(
*,
fallback: str = FEDORA_ATOMIC_FALLBACK_TAG,
os_release_path: Path = Path("/etc/os-release"),
) -> str:
data = read_os_release_fields(os_release_path)
if data.get("ID") != "fedora":
return fallback
version_id = data.get("VERSION_ID", "")
if not version_id.isdigit():
return fallback
# Track a host that is ahead of the pinned fallback, but only by one
# release. quay.io/fedora-ostree-desktops publishes a tag per Fedora
# release, and a pre-release host (Rawhide, or Branched well before GA)
# reports a VERSION_ID whose tag does not exist yet. Taking an unbounded
# max meant such a host produced curated URIs like
# ".../silverblue:47", the repo was created and pushed successfully, and
# the GitHub build then failed on FROM with a manifest-unknown error the
# user has no way to diagnose. Nothing here can verify the tag resolves
# without a network call, so bound it instead: one release ahead covers a
# host that upgraded before this constant was bumped.
return str(min(int(fallback) + 1, max(int(fallback), int(version_id))))
FEDORA_ATOMIC_DEFAULT_TAG = determine_fedora_atomic_default_tag()
def universal_blue_image(key: str, name: str, description: str, image_uri: str, *, first_boot_setup: bool) -> BaseImage:
return BaseImage(key=key, provider="Universal Blue", name=name, description=description, image_uri=image_uri, first_boot_setup=first_boot_setup)
def fedora_atomic_image(key: str, name: str, description: str, variant: str, *, first_boot_setup: bool, official_alias: bool = True) -> BaseImage:
# Fedora publishes each desktop twice: the long-standing
# quay.io/fedora-ostree-desktops/<variant>, and the newer official bootc
# location quay.io/fedora/fedora-<variant>. Both carry the same release
# tags, and which one a host was rebased to is not the user's choice of
# base -- it is the same Silverblue either way. Without the alias a host
# on the official location was refused as a custom image (#354).
aliases = (f"quay.io/fedora/fedora-{variant}",) if official_alias else ()
return BaseImage(
key=key,
provider="Fedora Atomic",
name=name,
description=description,
image_uri=f"quay.io/fedora-ostree-desktops/{variant}:{FEDORA_ATOMIC_DEFAULT_TAG}",
first_boot_setup=first_boot_setup,
aliases=aliases,
)
# first_boot_setup follows the desktop, not the provider: GNOME bases ship
# gnome-initial-setup, and Fedora's Budgie and COSMIC variants ship the
# desktop-neutral initial-setup. KDE bases have no wizard, and neither does
# Sway Atomic -- it boots straight to sddm (checked against
# quay.io/fedora-ostree-desktops/sway-atomic:44 on 2026-09-20; #361).
BASE_IMAGES: tuple[BaseImage, ...] = (
universal_blue_image("bazzite", "Bazzite (KDE)", "KDE desktop for gaming systems and handheld-style setups", "ghcr.io/ublue-os/bazzite:stable", first_boot_setup=False),
universal_blue_image("bazzite-gnome", "Bazzite (GNOME)", "GNOME desktop for gaming systems and handheld-style setups", "ghcr.io/ublue-os/bazzite-gnome:stable", first_boot_setup=True),
universal_blue_image("bazzite-dx", "Bazzite DX (KDE)", "Bazzite plus extra developer tools on KDE", "ghcr.io/ublue-os/bazzite-dx:stable", first_boot_setup=False),
universal_blue_image("bazzite-dx-gnome", "Bazzite DX (GNOME)", "Bazzite plus extra developer tools on GNOME", "ghcr.io/ublue-os/bazzite-dx-gnome:stable", first_boot_setup=True),
universal_blue_image("aurora", "Aurora (KDE)", "KDE desktop for everyday use", "ghcr.io/ublue-os/aurora:stable", first_boot_setup=False),
universal_blue_image("aurora-dx", "Aurora DX", "Aurora plus extra developer tools", "ghcr.io/ublue-os/aurora-dx:stable", first_boot_setup=False),
universal_blue_image("bluefin", "Bluefin (GNOME)", "GNOME desktop for everyday use", "ghcr.io/ublue-os/bluefin:stable", first_boot_setup=True),
universal_blue_image("bluefin-dx", "Bluefin DX", "Bluefin plus extra developer tools", "ghcr.io/ublue-os/bluefin-dx:stable", first_boot_setup=True),
fedora_atomic_image("silverblue", "Fedora Silverblue", "GNOME desktop built from the official Fedora Atomic desktop image", "silverblue", first_boot_setup=True),
fedora_atomic_image("kinoite", "Fedora Kinoite", "KDE Plasma desktop built from the official Fedora Atomic desktop image", "kinoite", first_boot_setup=False),
fedora_atomic_image("sway-atomic", "Fedora Sway Atomic", "Sway desktop built from the official Fedora Atomic desktop image", "sway-atomic", first_boot_setup=False),
fedora_atomic_image("budgie-atomic", "Fedora Budgie Atomic", "Budgie desktop built from the official Fedora Atomic desktop image", "budgie-atomic", first_boot_setup=True),
# quay.io/fedora/fedora-cosmic-atomic does not exist (checked against the
# registry API on 2026-09-20; the other four variants do), so claiming
# the alias here would match a reference no host can be booted from.
fedora_atomic_image("cosmic-atomic", "Fedora COSMIC Atomic", "COSMIC desktop built from the official Fedora Atomic desktop image", "cosmic-atomic", first_boot_setup=True, official_alias=False),
)
def supported_base_image_names() -> str:
return ", ".join(image.name for image in BASE_IMAGES)
def supported_base_image_lines() -> list[str]:
# Grouped by provider so the main menu can say what this tool supports
# without printing thirteen names on one line.
grouped: dict[str, list[str]] = {}
for image in BASE_IMAGES:
grouped.setdefault(image.provider, []).append(image.name)
return [f"{provider}: {', '.join(names)}" for provider, names in grouped.items()]
# scan_os() has three outcomes, and the caller has to tell them apart: a scan
# that could not run at all falls back to picking a base by hand, while a user
# who backed out should simply return to the menu.
SCAN_OK = "ok"
SCAN_UNAVAILABLE = "unavailable"
SCAN_CANCELLED = "cancelled"
# The running image is not one of the curated bases. Distinct from
# SCAN_UNAVAILABLE: the scan worked perfectly, and what it found is the problem.
SCAN_UNSUPPORTED_BASE = "unsupported-base"
# The host runs on an architecture the generated workflow publishes no image
# for. Also a scan that worked: no base chosen by hand would change the
# answer, so the caller stops rather than offering one.
SCAN_UNPUBLISHED_ARCHITECTURE = "unpublished-architecture"
# rpm-ostree records customizations in more fields than this tool reads.
# `requested-packages` and `requested-base-removals` are the two it can carry,
# because both are names an image build can install or remove from a
# repository -- except a `requested-packages` entry typed as a capability or
# file path, which scan_os reports alongside these. The rest are real
# customizations pinned to files on this host --
# an RPM built somewhere else, a package replaced by a local build -- and no
# generated image reproduces them. They were read as absent rather than as
# unsupported, so a scan reported success and recommended `rpm-ostree reset`
# while the new image silently omitted them. See the rpm-ostree administrator
# handbook's "rpm-ostree status --json" section for the field list. The first
# three are string arrays; the remote one is a list of (from, packages) pairs
# and needs remote_replacement_list() rather than string_list().
UNSUPPORTED_SCAN_FIELDS: tuple[tuple[str, str], ...] = (
("requested-local-packages", "Locally installed RPMs"),
("requested-local-fileoverride-packages", "Local file overrides"),
("requested-base-local-replacements", "Base packages replaced by a local RPM"),
("requested-base-remote-replacements", "Base packages replaced from a repository"),
)
# Not a package list, so it is reported separately -- but it belongs with them,
# because a bare `rpm-ostree reset` removes initramfs customization too.
INITRAMFS_SCAN_FIELDS: tuple[tuple[str, str], ...] = (
("initramfs-etc", "Files kept in the initramfs from /etc"),
("initramfs-args", "Custom initramfs arguments"),
)
COMMON_SERVICES: tuple[tuple[str, str], ...] = (
("SSH remote access", "sshd.service"),
("Tailscale VPN", "tailscaled.service"),
("Cockpit web admin", "cockpit.socket"),
)
@dataclass
class Config:
# Config is the single source of truth for what the user wants to build.
# Most of the app mutates this object in memory, then state_payload()
# serializes it to STATE_FILE before repo files are written.
method: str = ""
base_image_uri: str = ""
base_image_name: str = ""
repo_name: str = ""
image_desc: str = "My custom bootc image"
packages: list[str] = field(default_factory=list)
copr_repos: list[str] = field(default_factory=list)
services: list[str] = field(default_factory=list)
removed_packages: list[str] = field(default_factory=list)
brew_enabled: bool = False
signing_enabled: bool = False
github_user: str = ""
# The scan lists hold the running host's complete layered-package and
# base-removal inventory. They stay in memory to drive the selection screens
# and are deliberately NOT written to the state file - see state_payload().
# scan_omitted_customizations is in-memory too: the scan found something the
# user agreed to leave behind, which reset still has to clear on switch.
# scan_customizations_carried is the one bit anything downstream needs.
scanned_packages: list[str] = field(default_factory=list)
scanned_removed: list[str] = field(default_factory=list)
scan_omitted_customizations: bool = False
scan_customizations_carried: bool = False
def normalize(self) -> None:
# Every menu appends to lists over time. Normalizing here keeps ordering
# stable for humans while still removing duplicates and empty values.
self.packages = unique(self.packages)
self.copr_repos = unique(self.copr_repos)
self.services = unique(self.services)
self.removed_packages = unique(self.removed_packages)
# The state file's schema is Config itself: state_payload() writes every field
# with asdict(), so the loader reads back the same names rather than keeping a
# second list that a new field could be left out of. With postponed
# annotations each type is its source spelling. A field of any other type would
# be written and never read back, which the round-trip test catches.
CONFIG_LIST_FIELDS = tuple(f.name for f in dataclass_fields(Config) if f.type == "list[str]")
CONFIG_STRING_FIELDS = tuple(f.name for f in dataclass_fields(Config) if f.type == "str")
CONFIG_BOOL_FIELDS = tuple(f.name for f in dataclass_fields(Config) if f.type == "bool")
def unique(values: Iterable[str]) -> list[str]:
# This preserves first-seen order, which makes generated files and review
# screens stable and easier for users to reason about.
seen: set[str] = set()
output: list[str] = []
for value in values:
stripped = value.strip()
if stripped and stripped not in seen:
output.append(stripped)
seen.add(stripped)
return output
def string_list(value: object) -> list[str]:
# rpm-ostree status is untrusted input: it can be a stale override file or a
# future schema change. Any field we expect to be a list of package names may
# arrive as null, a bare string, or a list with non-string entries, so coerce
# to a clean list of strings instead of letting unique() hit .strip() on an
# int. A bare string is NOT iterated into characters - it is rejected.
if not isinstance(value, list):
return []
return [item for item in value if isinstance(item, str)]
def next_boot_deployment(deployments: Sequence[dict[str, object]]) -> tuple[dict[str, object], bool]:
# The deployment a scan has to read, and whether it is pending rather than
# running. rpm-ostree lists deployments in boot order. A deployment of the
# booted OS listed ahead of the booted one is pending: an `rpm-ostree
# install`, `uninstall`, `rebase` or `rollback` since the last boot put it
# there, and it is what the next boot runs. It is also what `rpm-ostree
# reset` and `bootc switch` start from -- rpm-ostree takes the pending
# deployment's origin, not the booted one's, as the base of a new one
# (ostree_sysroot_query_deployments_for). Reading the booted deployment
# instead dropped every package layered since the last boot, and carried
# over one uninstalled since, right before the reset the generated README
# recommends made those changes permanent (#519).
booted = next((item for item in deployments if item.get("booted")), None)
if booted is None:
# No deployment claims to be booted: the first is the default boot.
return (deployments[0] if deployments else {}), False
osname = booted.get("osname")
# Always found: booted itself matches its own osname.
first = next(item for item in deployments if item.get("osname") == osname)
return first, first is not booted
def remote_replacement_list(value: object) -> list[str]:
# requested-base-remote-replacements is the one scan field that is not a
# flat list of names. rpm-ostree records each `override replace --from
# repo=...` as a (from, packages) pair -- Vec<(String, Vec<String>)> in
# rust/src/daemon.rs -- so the JSON is [["repo=<id>", ["pkg", ...]], ...].
# string_list() kept only str items, which dropped every pair and read the
# override as absent: no "Cannot Be Carried Over" row, no confirm, and a
# README recommending the reset that removes it (#353). Flatten to one
# entry per package with its source attached, package name first so a long
# COPR id is what the preview truncates. Same coercion contract as
# string_list() otherwise: null, a bare string, or an entry of another
# shape is dropped rather than raised on.
if not isinstance(value, list):
return []
found: list[str] = []
for entry in value:
if isinstance(entry, str):
found.append(entry)
continue
if not isinstance(entry, list) or len(entry) != 2:
continue
source, packages = entry
source = source.strip() if isinstance(source, str) else ""
names = string_list(packages)
if names:
found.extend(f"{name} (from {source})" if source else name for name in names)
elif source:
# A source with no readable package list is still an override the
# image will not reproduce; naming the repository beats silence.
found.append(source)
return found
def sanitize_slug(value: str, default: str = DEFAULT_REPO_NAME) -> str:
# GitHub repo names cannot contain spaces, so we translate user-friendly
# input into a slug before running stricter validation.
cleaned = re.sub(r"[^a-z0-9._-]", "-", value.lower()).strip("-")
return cleaned or default
def is_valid_repo_name(value: str) -> bool:
# Two rules, and the name has to satisfy both: the subset of GitHub naming
# rules we support in the beginner UI, and the reference grammar that
# decides whether ghcr.io/<owner>/<name> can be parsed at all. The first
# alone let "test..image" through as far as repository creation and signing
# setup, and every later push and pull of it failed to parse.
if not value or len(value) > 100:
return False
if value.endswith(".git"):
return False
if not re.fullmatch(r"[a-z0-9](?:[a-z0-9._-]{0,98}[a-z0-9])?", value):
return False
if not OCI_PATH_COMPONENT_RE.fullmatch(value):
return False
return True
def file_lines(text: str) -> list[str]:
"""Split file text on line endings only, as YAML, sh and Dockerfile do.
Every patcher that edits a file in place splits it with this and joins it
back with "\\n". str.splitlines() would also break on \\v, \\f,
\\x1c-\\x1e, U+0085, U+2028 and U+2029, so re-joining its output turned
those characters into real line breaks in text the tool did not write --
an owner's comment, or a description from before #527 -- and could leave
the workflow unparseable (#689). For text whose only boundaries are \\n,
\\r\\n and \\r the result is the same as str.splitlines(), so CRLF files
still come back as LF.
"""
lines = re.split(r"\r\n|\r|\n", text)
if lines and lines[-1] == "":
lines.pop()
return lines
# What json.dumps(ensure_ascii=False) leaves raw but YAML forbids in a
# document: DEL and the C1 controls (U+007F-U+009F) and the two noncharacters
# U+FFFE/U+FFFF. YAML 1.2's c-printable production stops there -- C0 controls
# are already escaped by json.dumps, and everything else up to U+10FFFF is
# printable. U+0085 (NEL) is technically printable but a 1.1 line break, which
# PyYAML folds to a space, so it is escaped along with its neighbours. U+2028
# and U+2029 are the other two 1.1 line breaks, so a YAML 1.1 reader would fold
# a raw one too; escaping them keeps the value the same to every reader. (They
# once also orphaned the rest of the value on the next update, when the
# patchers split files with str.splitlines() -- #527; file_lines() removed that.)
_YAML_UNPRINTABLE_RE = re.compile("[\x7f-\x9f\u2028\u2029]")
def yaml_scalar(value: str) -> str:
# JSON string quoting is valid YAML 1.2 and saves us from bringing in a
# YAML library just to safely escape a single scalar value. Only with
# ensure_ascii=False, though: json.dumps otherwise writes anything outside
# the BMP -- every modern emoji -- as a UTF-16 surrogate pair, and a YAML
# \u escape must name a single scalar value. libyaml, go-yaml, serde-yaml
# and YamlDotNet all refuse the pair (see #360), so a description like
# "My 🚀 image" would produce a recipe BlueBuild cannot read and a
# workflow Actions cannot load. The characters that need escaping (quotes,
# backslashes, C0 controls) are still escaped; the rest is written as the
# UTF-8 the file is saved in anyway -- except the few code points YAML
# refuses to see raw anywhere in a document, which ensure_ascii=False
# would now pass through and libyaml would reject the whole file over.
# Those go back to the \u escape, which every parser reads as itself.
return _YAML_UNPRINTABLE_RE.sub(lambda m: f"\\u{ord(m.group()):04x}", json.dumps(value, ensure_ascii=False))
# A plain scalar that every YAML reader Actions and BlueBuild use resolves to
# the string it spells: letters, digits and the separators a branch name
# usually carries, not starting with a digit or an indicator character.
_YAML_SAFE_PLAIN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_./-]*")
# ...minus the words YAML 1.1 resolves to a boolean or null when unquoted.
_YAML_RESERVED_PLAIN_WORDS = frozenset({"y", "n", "yes", "no", "on", "off", "true", "false", "null"})
def yaml_plain_or_scalar(value: str) -> str:
"""``value`` bare when YAML reads it back as that string, else yaml_scalar().
For hand-shaped lines such as a branch filter entry, where the ordinary
value -- ``main`` -- is expected to stay bare, but a legal branch name
such as ``#main`` (a comment), ``@dev`` (a reserved indicator) or
``null`` (not a string at all) must not be written unquoted.
"""
if _YAML_SAFE_PLAIN_RE.fullmatch(value) and value.lower() not in _YAML_RESERVED_PLAIN_WORDS:
return value
return yaml_scalar(value)
def ensure_trailing_newline(text: str) -> str:
return text.rstrip("\n") + "\n"
def containerfile_shell_words(text: str) -> list[str]:
"""Split ``text`` into shell words with their quotes and escapes kept."""
words: list[str] = []
word: list[str] = []
quote = ""
escaped = False
for char in text:
if escaped:
escaped = False
elif char == "\\" and quote != "'":
escaped = True
elif quote:
if char == quote:
quote = ""
elif char in "'\"":
quote = char
elif char.isspace():
if word:
words.append("".join(word))
word = []
continue
word.append(char)
if word:
words.append("".join(word))
return words
def containerfile_heredocs(instruction: str) -> list[tuple[str, bool]]:
"""(terminator, strips-leading-tabs) for each heredoc ``instruction`` opens."""
if "<<" not in instruction or not CONTAINERFILE_HEREDOC_INSTRUCTION_RE.match(instruction):
return []
heredocs: list[tuple[str, bool]] = []
for word in containerfile_shell_words(instruction):
match = CONTAINERFILE_HEREDOC_WORD_RE.match(word)
if not match:
continue
try:
# The terminator is the word with its quoting removed: <<"EOF"
# ends at a line reading EOF.
name = shlex.split(match.group(2))
except ValueError:
continue
if len(name) == 1:
heredocs.append((name[0], match.group(1) == "-"))
return heredocs
def containerfile_escape(lines: list[str]) -> str:
"""The escape character BuildKit reads ``lines`` with.
It is a backslash unless an ``# escape=`` parser directive in the
leading run of directives sets it to a backtick. The escape character is
also the line-continuation character, so a scanner that assumed the
backslash would miss every continuation in a backtick-escaped file.
"""
for index, line in enumerate(lines):
# BuildKit drops a byte-order mark and leading whitespace first.
match = CONTAINERFILE_DIRECTIVE_RE.match((line.removeprefix("\ufeff") if index == 0 else line).lstrip())
if not match or match.group(1).lower() not in CONTAINERFILE_DIRECTIVE_NAMES:
break
if match.group(1).lower() == "escape":
# BuildKit rejects any other value, so that build fails anyway.
return match.group(2) if match.group(2) in CONTAINERFILE_CONTINUATION_RE else "\\"
return "\\"
def containerfile_from_indices(lines: list[str]) -> list[int]:
"""Indices of the lines that are FROM instructions, in order.
A line that merely looks like one is skipped when it is the continuation
of the instruction above it or part of a heredoc body -- a Python
"from x import y" written by a RUN heredoc matches FROM_LINE_RE, and the
last FROM decides which stage is published, so it must not count. The
walk follows BuildKit's parser: the continuation character is the file's
escape character (see containerfile_escape), a comment line neither
starts nor ends a continued instruction, a blank line never ends one,
and heredoc bodies start after the whole instruction has been read.
"""
continuation = CONTAINERFILE_CONTINUATION_RE[containerfile_escape(lines)]
indices: list[int] = []
heredocs: list[tuple[str, bool]] = []
instruction: str | None = None
for index, line in enumerate(lines):
if heredocs:
name, strip_tabs = heredocs[0]
if (line.lstrip("\t") if strip_tabs else line) == name:
heredocs.pop(0)
continue
stripped = line.strip()
if not stripped or stripped.startswith("#"):
continue
if instruction is None:
if FROM_LINE_RE.match(line):
indices.append(index)
instruction = ""
if continuation.search(line):
instruction += continuation.sub("", line)
continue
heredocs = containerfile_heredocs(instruction + line)
instruction = None
return indices
def normalize_container_image_reference(container_ref: str) -> str:
# rpm-ostree reports image origins with different prefixes depending on how
# the deployment was created. Normalize those into a plain image reference
# so scan results can be matched against our curated base-image list.
base = container_ref.strip()
# "ostree-image-signed:" and "ostree-unverified-image:" are the signed and
# signature-disabled spellings of the same thing: an origin prefix followed
# by a transport-qualified reference. Both must be stripped, then the
# transport below. Handling only the signed spelling meant a host rebased
# with verification off fell through unchanged and never matched the curated
# list, so choose_base_image discarded a base image that is on it.
for origin_prefix in ("ostree-image-signed:", "ostree-unverified-image:"):
if base.startswith(origin_prefix):
base = base[len(origin_prefix) :]
break
else:
if base.startswith("ostree-unverified-registry:"):
base = base[len("ostree-unverified-registry:") :]
elif base.startswith(("ostree-remote-image:", "ostree-remote-registry:")):
# These carry the ostree remote name as a second field:
# ostree-remote-image:<remote>:docker://<ref>
parts = base.split(":", 2)
if len(parts) == 3:
base = parts[2]
for transport in ("docker://", "registry:"):
if base.startswith(transport):
return base[len(transport) :]
return base
# Every way rpm-ostree spells a container origin starts with one of these:
# the ostree-* prefixes normalize_container_image_reference() strips, and the
# bare transports a status override file may carry. What is left after ruling
# them out is a classic ostree origin -- `<remote>:<ref>` -- which is what
# every stock Fedora Silverblue or Kinoite install has until its first rebase.
CONTAINER_ORIGIN_PREFIXES: tuple[str, ...] = ("ostree-", "docker://", "registry:")
# The classic ref keeps a variant's original codename: Fedora renamed Sericea
# to Sway Atomic and Onyx to Budgie Atomic, and the container images on
# quay.io took the new names, but the ostree refs Fedora signs are
# still fedora/<release>/<arch>/sericea and .../onyx (see the ostree_refs
# table in fedora-infra/ansible's robosignatory role). The other three refs
# already spell the curated key.
CLASSIC_FEDORA_REF_ALIASES: dict[str, str] = {"sericea": "sway-atomic", "onyx": "budgie-atomic"}
# The remote Fedora's installer configures for its own ostree repo. Remote
# names are local configuration, so the ref alone says nothing about where a
# deployment came from: `corp:fedora/44/x86_64/silverblue` is whatever the
# corp remote serves under that name. Only this remote is Fedora's.
FEDORA_OSTREE_REMOTE = "fedora"
# The image workflow this tool generates builds and publishes one native
# x86_64 image and nothing else (see patch_disk_workflow_platform and #236).
# A host on another architecture can be scanned, but the image it would be
# guided into building could never be switched onto it.
PUBLISHED_ARCHITECTURE = "x86_64"