#!/usr/bin/env bash # Copyright (C) 2018 Luke Shumaker # Copyright (C) 2023-2024 Umorpha Systems # SPDX-License-Identifier: AGPL-3.0-or-later declare -r NAME=osi-mk declare -r VERSION=20231023 # Why is this different than mkosi[1]? # # - mkosi claims to be "legacy-free"--but they call everything that's # not systemd "legacy"; that clearly won't do for creating OpenRC # images. # # - mkosi claims to be "legacy-free", only supporting GPT # disk-labels--but btrfs can be booted directly, without a separate # disk-label, or a separate ESP partition. To a btrfs disk, GPT/ESP # is legacy. # # - Using a raw btrfs disk means that it can easily be mounted without # first dissecting the disk-label. # # [1]: https://github.com/systemd/mkosi set -euE -o pipefail shopt -s expand_aliases install_prefix="$(realpath --logical --canonicalize-missing -- "${BASH_SOURCE[0]}/../..")" readonly install_prefix source "${install_prefix}/lib/osi.sh" source "${install_prefix}/lib/argparse.sh" loaded_modules=() load_module() { local module if ! [[ -f $1 ]]; then argparse:error 'Module does not exist: %s' "$1" return fi module="$(realpath -- "$1")" if in_array "$module" "${loaded_modules[@]}"; then return 0 fi loaded_modules+=("$module") # shellcheck source=/dev/null source "$1" } beg_indent() { exec 8> >( gprintf -v prefix -- "$@" # shellcheck disable=SC2154 # false-positive, doesn't understand gprintf "${install_prefix}/lib/indent" "$NAME [$prefix] " ) _indent_pid=$! } alias end_indent=' <&- >&8; exec 8<&-; wait $_indent_pid' osi-mk:genfstab() { local arg_mountpoint=$1 { # This header mimics the stock fstat from 'filesystem'. echo '# Static information about the filesystems.' echo '# See fstab(5) for details.' echo { # This doesn't quite mimic the header from 'filesystem' # # - I swapped "file system" for "device", for clarity, # and so that it's one word for `column -t`. # - I swapped "pass" for "fsck", for clarity. # - I removed the space after the "#", so it works with # `column -t`. echo '# ' case "${arg_conf[format]}" in erofs) echo "UUID=${arg_conf[fsuuid]}" / erofs defaults 0 1 ;; *) genfstab -t uuid "$arg_mountpoint" | grep '^[#]' | awk '$3 != "swap"' ;; esac } | column -t } >"${arg_mountpoint}/etc/fstab" } osi-mk:directories() { local arg_mountpoint=$1 local spec outside inside for spec in "${arg_directories[@]}"; do outside="${spec%:*}" inside="${spec#"${outside}:"}" # TODO: maybe rsync would be better? mkdir -p -- "$(dirname -- "${arg_mountpoint}/${inside}")" print 'Copying %q to %q:%q' "$outside" "$arg_file" "$inside" cp -aT --no-preserve=ownership -- "$outside" "${arg_mountpoint}/${inside}" done } osi-mk:hostname() { local arg_mountpoint=$1 echo "${arg_conf[hostname]}" >"$arg_mountpoint/etc/hostname" } osi-mk:mkinitcpio() { local arg_mountpoint=$1 print 'Configuring mkinitcpio to include all drivers' mkdir -p -- "$arg_mountpoint/etc/mkinitcpio.conf.d" cat >"$arg_mountpoint/etc/mkinitcpio.conf.d/osi-mk.conf" <<'EOF' #!/hint/bash # Copyright (C) 2023 Umorpha Systems # SPDX-License-Identifier: AGPL-3.0-or-later # Remove 'autodetect' from HOOKS; include all drivers. for ((i=0; i<${#HOOKS[@]}; i++)); do if [[ ${HOOKS[i]} == autodetect ]]; then HOOKS=("${HOOKS[@]:0:i}" "${HOOKS[@]:i+1}") fi done EOF } osi-mk:grub-install() { local arg_mountpoint=$1 # shellcheck disable=SC2016 arch-chroot -- "$arg_mountpoint" sh -c \ 'grub-install --target=i386-pc "$(awk '\''$2 == "/" { print $1 }'\'' maxdate) { print maxdate } else { print } x-- } ' <"$file" >"$file.tmp" mv -T -- "$file.tmp" "$file" touch --date="@$((SOURCE_DATE_EPOCH-1))" -- "$file" done # Backdate pacman.log to 1 second before SOURCE_DATE_EPOCH. if [[ -f "$arg_mountpoint/var/log/pacman.log" ]]; then local old new old='^\[....-..-..T..:..:..[-+]....]' new="[$(TZ=UTC date --date="@$((SOURCE_DATE_EPOCH-1))" --iso-8601=seconds | sed 's/:00$/00/')]" sed -i "s/$old/$new/" -- "$arg_mountpoint/var/log/pacman.log" touch --date="@$((SOURCE_DATE_EPOCH-1))" -- "$arg_mountpoint/var/log/pacman.log" fi # Backdate file timestamps to SOURCE_DATE_EPOCH. case "${arg_conf[format]}" in raw_btrfs) # Ugg, it's lame that `find` can't find files newer than a # timestamp without using a temporary file. touch --date="@$SOURCE_DATE_EPOCH" -- "$arg_mountpoint/.source_date_epoch" find "$arg_mountpoint" -xdev -depth \ -newer "$arg_mountpoint/.source_date_epoch" \ -exec touch --no-dereference --date="@$SOURCE_DATE_EPOCH" -- {} + rm -f -- "$arg_mountpoint/.source_date_epoch" touch --date="@$SOURCE_DATE_EPOCH" -- "$arg_mountpoint/" ;; erofs) # Nothing to do; mkfs.erofs supports SOURCE_DATE_EPOCH. : ;; esac } main() { # argparse specifications ############################################## # # There got to be enough args with enough validation logic # that the declare/parse/set/validate locations were all far # enough apart that it was hard to maintain. # # So I factored out an argparse.sh library, so that I could # keep the logic for a given argument close to itself. # # Please don't hate me for failing to KISS or remember that # YAGNI; please trust that I did actually "NI". opt_specs+=(': --inside : MOUNTPOINT : internal-use only') local arg_mountpoint= opt_visit:inside() { argparse:setmode inside; arg_mountpoint=$1; } opt_final:inside() { :; } opt_specs+=('-e : --edit : [BASE.img] : edit an existing image') local arg_edit=false local arg_edit_base='' opt_visit:edit() { arg_edit=true; arg_edit_base=$1; } opt_final:edit() { if $arg_edit; then # We set arg_file below in opt_positional(). if ! [[ -f ${arg_edit_base:-$arg_file} ]]; then argparse:error 'Image must already exist to --edit: %s' "${arg_edit_base:-$arg_file}" fi fi } opt_specs+=(': --disorderfs : : use disorderfs') local arg_disorderfs=false opt_visit:disorderfs() { arg_disorderfs=true; } opt_final:disorderfs() { :; } opt_specs+=('') opt_specs+=('-d : --directory : OUTSIDE\:INSIDE : include the given directory') local arg_directories=() opt_visit:directory() { if ! [[ -d "${1%:*}" ]]; then argparse:error 'Directory does not exist: %s' "${1%:*}" return fi arg_directories+=("$1") } opt_final:directory() { :; } opt_specs+=('-m : --module : MOD.sh : include the given module') local arg_modules=() opt_visit:module() { if ! [[ -f $1 ]]; then argparse:error 'Module does not exist: %s' "$1" return fi arg_modules+=("$1") } opt_final:module() { :; } opt_specs+=('-p : --package : PKGNAME : include the given package (or group)') local arg_packages=() opt_visit:package() { arg_packages+=("$1"); } opt_final:package() { :; } opt_specs+=('-P : --package-file : PKG.pkg.tar.xz : include the given package file') local arg_package_files=() opt_visit:package-file() { if ! [[ -f "$1" ]]; then argparse:error 'Package file does not exist: %s' "$1" return fi arg_package_files+=("$1") } opt_final:package-file() { :; } opt_specs+=('-C : --conf : KEY=VAL : set set a config option') declare -A arg_conf=( # not package names [format]=raw_btrfs [bootloader]=grub [genfstab]=true # package names [initramfs]=mkinitcpio [dbus-units]=dbus-broker-units ) if [[ "${SOURCE_DATE_EPOCH:-}" =~ ^([0-9]+|auto)$ ]]; then arg_conf[SOURCE_DATE_EPOCH]="$SOURCE_DATE_EPOCH" arg_orig=(--conf=SOURCE_DATE_EPOCH="${arg_conf[SOURCE_DATE_EPOCH]}" "${arg_orig[@]}") fi opt_visit:conf() { local key val key="${1%%=*}" val="${1#*=}" case "$key" in format) case "$val" in raw_btrfs) :;; erofs) :;; *) argparse:error 'Unrecognized --conf=format= value: %q' "$val"; return;; esac ;; bootloader) case "$val" in grub) :;; none) :;; *) argparse:error 'Unrecognized --conf=bootloader= value: %q' "$val"; return;; esac ;; genfstab) case "${val,,}" in 1|t|true|y|yes) val=true;; 0|f|false|n|no) val=false;; *) argparse:error 'Unrecognized --conf=genfstab= value: %q' "$val"; return;; esac ;; SOURCE_DATE_EPOCH) if ! [[ "$val" =~ ^([0-9]+|auto|)$ ]]; then argparse:error 'Invalid --conf=SOURCE_DATE_EPOCH= value: %q' "$val" return fi ;; esac arg_conf["$key"]="$val"; } opt_final:conf() { case "${arg_conf[format]}" in raw_btrfs) if [[ -z ${arg_conf[size]:-} ]]; then argparse:error 'Must specify --conf=size= when creating a new image' fi ;; erofs) if $arg_edit; then argparse:error '--conf=format=erofs does not support --edit' fi if [[ ${arg_conf[bootloader]} != none ]]; then argparse:error '--conf=format=erofs requires --conf=bootloader=none' fi ;; esac } local arg_file opt_positional() { if (( $# != 1 )); then if (( $# == 0 )); then argparse:error "Expected 1 positional argument, got none" else argparse:error "Expected 1 positional argument, got %d: %s" "$#" "${*@Q}" fi return fi arg_file=$1 if [[ "$opt_mode" != inside ]]; then if [[ ( $arg_edit = false || -n $arg_edit_base ) && -e $arg_file ]]; then argparse:error 'Image file already exists, refusing to overwrite: %s' "$arg_file" fi fi } # actually parse the args ############################################## local arg_orig=("$@") argparse outside "$@" # main ################################################################ case "$opt_mode" in error) print "Try '%q --help' for more information" "${0##*/}" >&2 return $EXIT_INVALIDARGUMENT ;; version) print "%s (osi-tools) %s" "$NAME" "$VERSION" return $EXIT_SUCCESS ;; help) print 'Usage: %s [OPTIONS] FILENAME.img' "${0##*/}" print 'Operating System Image: Make' echo print 'Create a mountable, bootable OS image.' echo print 'OPTIONS:' echo "${opt_flaghelp}" | grep -v -e --inside return $EXIT_SUCCESS ;; # main code starts here outside) # the part that runs as a normal user without the image mounted mount_dev="$arg_file" mount_opt='' beg_indent 'format:before' { case "${arg_conf[format]}" in raw_btrfs) if $arg_edit; then if [[ -n $arg_edit_base ]]; then cp -T --reflink -- "$arg_edit_base" "$arg_file" fi if [[ -n ${arg_conf[fsuuid]:-} ]]; then # Explicit new FSUUID. btrfstune -f -U "${arg_conf[fsuuid]:-}" "$arg_file" elif [[ -n $arg_edit_base ]]; then # Random new FSUUID. btrfstune -fu "$arg_file" fi if [[ -n ${arg_conf[size]:-} ]]; then # Calculate sizes in exact bytes now, to avoid any discrepancies # between truncate(1) and btrfs-filesystem(8). # Use truncate(1) to do this. It's a little gross, # but because TMPDIR is likely on tmpfs, it's time-cheap, # and because of sparse files, it's space-cheap. tmpfile=$(mktemp -t -- "${0##*/}.XXXXXXXXXX") trap "rm -- ${tmpfile@Q}" EXIT case "${arg_conf[size]}" in '+'*|'-'*|'<'*|'>'*|'/'*|'%'*) truncate --reference="$arg_file" --size="${arg_conf[size]}" -- "$tmpfile";; *) truncate --size="${arg_conf[size]}" -- "$tmpfile";; esac old_size=$(stat --format='%s' -- "$arg_file") new_size=$(stat --format='%s' -- "$tmpfile") rm -- "$tmpfile" trap - EXIT # Do the resize arg_mountpoint=$(mktemp -dt -- "${0##*/}.XXXXXXXXXX") trap "rmdir -- ${arg_mountpoint@Q}" EXIT if (( new_size > old_size )); then truncate --size="$new_size" -- "$arg_file" _sudo -- "${install_prefix}/bin/osi-mount" --root -- "$arg_file" "$arg_mountpoint" btrfs filesystem resize max "$arg_mountpoint" elif (( new_size < old_size )); then _sudo -- "${install_prefix}/bin/osi-mount" --root -- "$arg_file" "$arg_mountpoint" btrfs filesystem resize "$new_size" "$arg_mountpoint" truncate --size="$new_size" -- "$arg_file" fi rmdir -- "$arg_mountpoint" trap - EXIT fi else truncate --size="${arg_conf[size]}" -- "$arg_file" mkfs.btrfs ${arg_conf[fsuuid]:+"--uuid=${arg_conf[fsuuid]}"} -- "$arg_file" fi ;; erofs) if $arg_edit; then error 2 '--conf=format=erofs does not support --edit' fi if [[ -z "${arg_conf[fsuuid]:-}" ]]; then arg_conf[fsuuid]=$(uuidgen) arg_orig=(--conf=fsuuid="${arg_conf[fsuuid]}" "${arg_orig[@]}") fi mount_dev=$(mktemp -dt -- "${0##*/}.XXXXXXXXXX") # Make our own tmpfs because $TMPDIR might be mounted # with 'noexec' or something. Also, so we can clean it # up without a `sudo rm -rf`. _sudo mount -t tmpfs tmpfs:osi-mk "$mount_dev" trap "_sudo umount ${mount_dev@Q}; rmdir -- ${mount_dev@Q}" EXIT mount_opt=bind ;; esac } end_indent arg_mountpoint=$(mktemp -dt -- "${0##*/}.XXXXXXXXXX") r=0 _sudo -- "${install_prefix}/bin/osi-mount" --root \ --rwdir="${TMPDIR:-/tmp}" \ --rwdir=/var/cache/pacman/pkg \ --rwdir=/etc/pacman.d/gnupg \ --options="$mount_opt" \ -- \ "$mount_dev" "$arg_mountpoint" \ "${BASH_SOURCE[0]}" --inside="$arg_mountpoint" \ "${arg_orig[@]}" || r=$? rmdir -- "$arg_mountpoint" if (( r != 0 )); then return $r fi beg_indent 'format:after' { case "${arg_conf[format]}" in erofs) if [[ "${arg_conf[SOURCE_DATE_EPOCH]:-}" == auto ]]; then local file for file in "$mount_dev"/var/lib/pacman/local/*/desc; do arg_conf[SOURCE_DATE_EPOCH]="$(sed -n '/^%INSTALLDATE%$/{n;p;}' -- "$file")" break done fi if [[ -n "${arg_conf[SOURCE_DATE_EPOCH]:-}" ]]; then export SOURCE_DATE_EPOCH="${arg_conf[SOURCE_DATE_EPOCH]}" fi tmpdir=$(mktemp -dt -- "${0##*/}.XXXXXXXXXX") trap "sudo umount ${mount_dev@Q}; rmdir -- ${mount_dev@Q}; rm -f -- ${tmpdir@Q}/erofs.img; rmdir -- ${tmpdir@Q}" EXIT _sudo mkfs.erofs \ -U"${arg_conf[fsuuid]}" \ ${arg_conf[erofs_compression]:+"-z${arg_conf[erofs_compression]}"} \ "$tmpdir/erofs.img" "$mount_dev" cp -T "$tmpdir/erofs.img" "$arg_file" ;; esac } end_indent print '%s Done' "$NAME" ;; inside) # the part that runs as root with the image mounted needs_sudo ### Load modules ### packages=("${arg_packages[@]}") pre_install=() post_install=( 50:osi-mk:directories ) if "${arg_conf[genfstab]}"; then if ! $arg_edit || [[ -n $arg_edit_base ]] || [[ -n "${arg_conf[fsuuid]:-}" ]]; then post_install+=(10:osi-mk:genfstab) fi fi case "${arg_conf[format]}" in raw_btrfs) packages+=(btrfs-progs) ;; erofs) packages+=(erofs-utils) ;; esac case "${arg_conf[bootloader]}" in grub) packages+=(grub) post_install+=(88:osi-mk:grub-mkconfig) # before 89:osi-mk:source-date-epoch if ! $arg_edit; then post_install+=(87:osi-mk:grub-install) # before '88:osi-mk:grub-mkconfig' fi ;; esac case "${arg_conf[initramfs]:-}" in mkinitcpio) pre_install+=(10:osi-mk:mkinitcpio) ;; esac if [[ -n "${arg_conf[hostname]:-}" ]]; then post_install+=(10:osi-mk:hostname) fi if [[ -n "${arg_conf[SOURCE_DATE_EPOCH]:-}" ]]; then post_install+=(89:osi-mk:source-date-epoch) fi local pkgfile pkgdeps for pkgfile in "${arg_package_files[@]}"; do mapfile -t pkgdeps < <(bsdtar xfO "$pkgfile" .PKGINFO | sed -n 's/^depend = //p') cache_packages+=("${pkgdeps[@]}") done for module in "${arg_modules[@]}"; do load_module "$module" done cache_packages+=("${packages[@]}") ### Disorderfs ### if $arg_disorderfs; then orig_arg_mountpoint=$arg_mountpoint arg_mountpoint=$(mktemp -dt -- "${0##*/}.XXXXXXXXXX") disorderfs --multi-user=yes --shuffle-dirents=yes "$orig_arg_mountpoint" "$arg_mountpoint" trap "fusermount -u ${arg_mountpoint@Q}; rmdir ${arg_mountpoint@Q}" EXIT fi pacman_conf=/usr/share/pacman/defaults/pacman.conf.x86_64 ### Download ### beg_indent 'download:repos' { # Download syncdbs to the image mkdir -p -- "$arg_mountpoint"/var/{lib/pacman,log} pacman -r "$arg_mountpoint" --config="$pacman_conf" \ -Sy --noconfirm } end_indent beg_indent 'download:check-config' { # Validate that the user specified an answer to # every question that pacman asks. opt_fail=false while read -r -a options; do name="${options[0]%:}" options=("${options[@]:1}") base=${name%%[<>=]*} if in_array "${arg_conf["$base"]:-}" "${options[@]}" "${options[@]#*/}"; then print 'inserting package %s=%s' "$name" "${arg_conf["$base"]}" packages+=("${arg_conf["$base"]}") cache_packages+=("${arg_conf["$base"]}") else error 0 "must set option '%q' to one of [%s]" "$base" "${options[*]}" opt_fail=true fi done < <("${install_prefix}/lib/pacman-choices" \ -r "$arg_mountpoint" --config="$pacman_conf" \ --oneline -- "${cache_packages[@]}") if [[ $opt_fail == true ]]; then exit $EXIT_NOTCONFIGURED fi } end_indent beg_indent 'download:packages' { if (( ${#cache_packages[@]} > 0 )); then # this check is important for --edit # Download needed packages to the host cache pacman -r "$arg_mountpoint" --config="$pacman_conf" \ -Syw --noconfirm -- "${cache_packages[@]}" # Copy needed packages from host cache to image cache mkdir -p -- "$arg_mountpoint"/var/cache/pacman/pkg pacman -r "$arg_mountpoint" --config="$pacman_conf" \ -Sp --print-format='%l' -- "${cache_packages[@]}" \ | sed -En 's,^file://(.*),\1\n\1.sig,p' \ | xargs -d $'\n' -r cp -t "$arg_mountpoint/var/cache/pacman/pkg" -- fi } end_indent if [[ "${arg_conf[SOURCE_DATE_EPOCH]:-}" =~ ^[0-9]+$ ]]; then print '%s SOURCE_DATE_EPOCH=%s' "$NAME" "${arg_conf[SOURCE_DATE_EPOCH]}" export SOURCE_DATE_EPOCH="${arg_conf[SOURCE_DATE_EPOCH]}" fi ### pre_install ### while IFS=: read -r n fn; do beg_indent 'pre_install:%s:%s' "$n" "$fn" ( print Begin "$fn" "$arg_mountpoint" print End ) end_indent done < <([[ "${#pre_install[@]}" == 0 ]] || printf '%s\n' "${pre_install[@]}" | sort) rm -f -- "$arg_mountpoint"/var/log/pacman.log ### Install ### beg_indent 'install' { if (( ${#packages[@]} > 0 )); then # this check is important for --edit local pacstrap_args=( -M # don't copy the host's mirrorlist -C "$pacman_conf" -- "$arg_mountpoint" --hookdir="$arg_mountpoint/etc/pacman.d/hooks" # hack around https://bugs.archlinux.org/task/49347 --needed # for --edit "${packages[@]}" ) TMPDIR='' pacstrap "${pacstrap_args[@]}" fi if [[ -f "$arg_mountpoint/var/log/pacman.log" ]]; then sed -i "s:${arg_mountpoint}:/mnt:g" -- "$arg_mountpoint/var/log/pacman.log" fi if (( ${#arg_package_files[@]} > 0 )); then dir="$(mktemp -d -- "$arg_mountpoint/var/tmp/package-files.XXXXXXXXXX")" trap "rm -rf -- ${dir@Q}" EXIT cp -t "$dir" -- "${arg_package_files[@]}" local arg filelist=() for arg in "${arg_package_files[@]}"; do filelist+=("/var/tmp/${dir##*/var/tmp/}/${arg##*/}") done arch-chroot -- "$arg_mountpoint" pacman -U --noconfirm -- "${filelist[@]}" rm -rf -- "$dir" trap - EXIT fi } end_indent if [[ "${arg_conf[SOURCE_DATE_EPOCH]:-}" == auto ]]; then # set it to 1 second after the last timestamp from any # package (that extra second is so that post-install # things can be timestamped after the package files). arg_conf[SOURCE_DATE_EPOCH]=$(($({ zcat "$arg_mountpoint"/var/lib/pacman/local/*/mtree | grep -o 'time=[0-9]*' | cut -d= -f2 cat /var/lib/pacman/local/*/desc | sed -n '/^%BUILDDATE%$/{n;p;}' } | sort -n | tail -n1) + 1)) fi if [[ -n "${arg_conf[SOURCE_DATE_EPOCH]:-}" ]]; then print '%s SOURCE_DATE_EPOCH=%s' "$NAME" "${arg_conf[SOURCE_DATE_EPOCH]}" export SOURCE_DATE_EPOCH="${arg_conf[SOURCE_DATE_EPOCH]}" fi ### post_install ### while IFS=: read -r n fn; do beg_indent 'post_install:%s:%s' "$n" "$fn" ( print Begin "$fn" "$arg_mountpoint" print End ) end_indent done < <([[ "${#post_install[@]}" == 0 ]] || printf '%s\n' "${post_install[@]}" | sort) ;; *) error $EXIT_FAILURE 'Internal error. The programmer writing this tool screwed up.';; esac } main "$@"