COS-3013: overlay node image before bootstrapping if necessary

data/data/bootstrap/files/etc/systemd/system-generators/node-image-overlay-generator

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -euo pipefail
+
+UNIT_DIR="${1:-/tmp}"
+
+if ! rpm -q openshift-clients &>/dev/null; then
+    ln -sf "/etc/systemd/system/node-image-overlay.target" \
+        "${UNIT_DIR}/default.target"
+fi

data/data/bootstrap/files/etc/systemd/system/node-image-finish.service

@@ -0,0 +1,13 @@
+# This is a separate unit because in the assisted-installer flow, we only want
+# `node-image-overlay.service`, not the isolating back to `multi-user.target`.
+
+[Unit]
+Description=Node Image Finish
+Requires=node-image-overlay.service
+After=node-image-overlay.service
+
+[Service]
+Type=oneshot
+# and now, back to our regularly scheduled programming...
+ExecStart=/usr/bin/echo "Node image overlay complete; switching back to multi-user.target"
+ExecStart=/usr/bin/systemctl --no-block isolate multi-user.target

data/data/bootstrap/files/etc/systemd/system/node-image-overlay.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Node Image Overlay
+Requires=node-image-pull.service
+After=node-image-pull.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/node-image-overlay.sh
+RemainAfterExit=yes

data/data/bootstrap/files/etc/systemd/system/node-image-overlay.target

@@ -0,0 +1,9 @@
+[Unit]
+Description=Node Image Overlay Target
+Requires=basic.target
+
+# for easier debugging
+Requires=sshd.service getty.target systemd-user-sessions.service
+
+Requires=node-image-overlay.service
+Requires=node-image-finish.service

data/data/bootstrap/files/etc/systemd/system/node-image-pull.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Node Image Pull
+Requires=network.target NetworkManager.service
+After=network.target
+
+[Service]
+Type=oneshot
+# we need to call ostree container (i.e. rpm-ostree), which has install_exec_t,
+# but by default, we'll run as unconfined_service_t, which is not allowed that
+# transition. Relabel the script itself.
+ExecStartPre=chcon --reference=/usr/bin/ostree /usr/local/bin/node-image-pull.sh
+ExecStart=/usr/local/bin/node-image-pull.sh
+MountFlags=slave
+RemainAfterExit=yes

data/data/bootstrap/files/usr/local/bin/node-image-overlay.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -euo pipefail
+
+ostree_checkout=/ostree/repo/tmp/node-image
+if [ ! -d "${ostree_checkout}" ]; then
+    ostree_checkout=/var/ostree-container/checkout
+fi
+
+echo "Overlaying node image content"
+
+# keep /usr/lib/modules from the booted deployment for kernel modules
+mount -o bind,ro "/usr/lib/modules" "${ostree_checkout}/usr/lib/modules"
+mount -o rbind,ro "${ostree_checkout}/usr" /usr
+rsync -a "${ostree_checkout}/usr/etc/" /etc
+
+# reload the new policy
+echo "Reloading SELinux policy"
+semodule -R

data/data/bootstrap/files/usr/local/bin/node-image-pull.sh.template

@@ -0,0 +1,87 @@
+#!/bin/bash
+set -euo pipefail
+
+# shellcheck source=release-image.sh.template
+. /usr/local/bin/release-image.sh
+
+# yuck... this is a good argument for renaming the node image to just `node` in both OCP and OKD
+coreos_img=rhel-coreos
+{{ if .IsOKD }}
+coreos_img=stream-coreos
+{{ end }}
+until COREOS_IMAGE=$(image_for ${coreos_img}); do
+    echo 'Failed to query release image; retrying...'
+    sleep 10
+done
+
+# need to use rpm-ostree here since `bootc status` doesn't work in the live ISO currently
+# https://github.com/containers/bootc/issues/1043
+booted_version=$(rpm-ostree status --json | jq -r .deployments[0].version)
+
+echo "Currently on CoreOS version $booted_version"
+echo "Target node image is $COREOS_IMAGE"
+
+# try to do this in the system repo so we get hardlinks and the checkout is
+# read-only, but fallback to using /var if we're in the live environment since
+# that's truly read-only
+ostree_repo=/ostree/repo
+ostree_checkout="${ostree_repo}/tmp/node-image"
+hardlink='-H'
+if grep -q coreos.liveiso= /proc/cmdline; then
+    ostree_repo=/var/ostree-container/repo
+    ostree_checkout=/var/ostree-container/checkout
+    mkdir -p "${ostree_repo}"
+    echo "In live ISO; creating temporary repo to pull node image"
+    ostree init --mode=bare --repo="${ostree_repo}"
+    # if there are layers, import all the content in the system repo for
+    # layer-level deduping
+    if [ -d /ostree/repo/refs/heads/ostree/container ]; then
+        echo "Importing base content from system repo for deduplication"
+        ostree pull-local --repo="${ostree_repo}" /ostree/repo
+    fi
+    # but we won't be able to force hardlinks cross-device
+    hardlink=''
+else
+    # (remember, we're MountFlags=slave)
+    mount -o rw,remount /sysroot
+fi
+
+# Use ostree stack to pull the container here. This gives us efficient
+# downloading with layers we already have, and also handles SELinux.
+echo "Pulling ${COREOS_IMAGE}"
+while ! ostree container image pull --authfile "/root/.docker/config.json" \
+  "${ostree_repo}" ostree-unverified-image:docker://"${COREOS_IMAGE}"; do
+    echo 'Failed to fetch release image; retrying...'
+    sleep 10
+done
+
+# ideally, `ostree container image pull` would support `--write-ref` or a
+# command to escape a pullspec, but for now it's pretty easy to tell which ref
+# it is since it's the only docker one
+ref=$(ostree refs --repo "${ostree_repo}" | grep ^ostree/container/image/docker)
+if [ $(echo "$ref" | wc -l) != 1 ]; then
+    echo "Expected single docker ref, found:"
+    echo "$ref"
+    exit 1
+fi
+ostree refs --repo "${ostree_repo}" "$ref" --create coreos/node-image
+
+# massive hack to make ostree admin config-diff work in live ISO where /etc
+# is actually on a separate mount and not the deployment root proper... should
+# enhance libostree for this (remember, we're MountFlags=slave)
+if grep -q coreos.liveiso= /proc/cmdline; then
+    mount -o bind,ro /etc /ostree/deploy/*/deploy/*/etc
+fi
+
+# get all state files in /etc; this is a cheap way to get "3-way /etc merge" semantics
+etc_keep=$(ostree admin config-diff | cut -f5 -d' ' | sed -e 's,^,/usr/etc/,')
+
+# check out the commit
+echo "Checking out node image content"
+ostree checkout --repo "${ostree_repo}" ${hardlink} coreos/node-image "${ostree_checkout}" --skip-list=<(cat <<< "$etc_keep")
+
+# in the assisted-installer case, nuke the temporary repo to save RAM
+if grep -q coreos.liveiso= /proc/cmdline; then
+    echo "Deleting temporary repo"
+    rm -rf "${ostree_repo}"
+fi

data/data/bootstrap/systemd/common/units/crio-configure.service.template

@@ -2,9 +2,6 @@
 Description=Configure CRI-O to use the pause image
 After=release-image.service
 Requires=release-image.service
-{{if .IsOKD -}}
-Requires=release-image-pivot.service
-{{end -}}
 Before=crio.service
 
 [Service]

data/data/bootstrap/systemd/common/units/kubelet.service.template

@@ -1,13 +1,7 @@
 [Unit]
 Description=Kubernetes Kubelet
 Wants=rpc-statd.service crio.service release-image.service
-{{if .IsOKD -}}
-Wants=release-image-pivot.service
-{{end -}}
 After=crio.service release-image.service
-{{if .IsOKD -}}
-After=release-image-pivot.service
-{{end -}}
 
 [Service]
 Type=notify

pkg/asset/ignition/bootstrap/common.go

@@ -438,16 +438,17 @@ func AddStorageFiles(config *igntypes.Config, base string, uri string, templateD
 
 	var mode int
 	appendToFile := false
-	if parentDir == "bin" || parentDir == "dispatcher.d" {
+	switch {
+	case parentDir == "bin", parentDir == "dispatcher.d", parentDir == "system-generators":
 		mode = 0555
-	} else if filename == "motd" || filename == "containers.conf" {
+	case filename == "motd", filename == "containers.conf":
 		mode = 0644
 		appendToFile = true
-	} else if filename == "registries.conf" {
+	case filename == "registries.conf":
 		// Having the mode be private breaks rpm-ostree, xref
 		// https://github.com/openshift/installer/pull/6789
 		mode = 0644
-	} else {
+	default:
 		mode = 0600
 	}
 	ign := ignition.FileFromBytes(strings.TrimSuffix(base, ".template"), "root", mode, data)