projectbluefin · castrojo · Sep 11, 2026 · Sep 11, 2026
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -102,16 +102,22 @@ jobs:
           just build-ddi
           just export-ddi
 
-      - name: Build and export live installer & target UKI
+      - name: Build and export live installer & target UKI, and PXE boot inputs
         run: |
           just build-installer
-          just export-installer
+          just export-pxe
 
       - name: Build and export k0s systemd-sysext
         run: |
           just build-sysext
           just export-sysext
 
+
+      - name: Build and export Flatcar LTS Kernel & ZFS
+        run: |
+          just build-kernel
+          just build-zfs
+          just export-kernel
       - name: Sign release SHA256SUMS manifest
         if: github.ref == 'refs/heads/main'
         env:
@@ -123,8 +129,11 @@ jobs:
           # flat manifest at the release root.
           mkdir -p dist/release
           cp dist/bluefin-server-installer-*.raw.zst dist/bluefin-server-*.efi dist/release/
+          cp dist/bluefin-server-pxe-vmlinuz-* dist/bluefin-server-pxe-initrd-*.cpio.gz dist/release/
           cp dist/ddi/bluefin-server-ddi-*.raw.zst dist/release/
-          cp dist/sysext/k0s-*.raw.zst dist/release/
+          if [ -d dist/kernel ]; then
+            cp dist/kernel/usr/lib/sysexts/*.raw dist/release/ 2>/dev/null || true
+          fi
           (cd dist/release && sha256sum --binary -- * > SHA256SUMS)
           GNUPGHOME=$(mktemp -d)
           export GNUPGHOME

diff --git a/.github/workflows/kernel.yml b/.github/workflows/kernel.yml
@@ -0,0 +1,71 @@
+name: Build Kernel & ZFS
+
+on:
+  pull_request:
+    paths:
+      - 'elements/flatcar/**'
+      - 'include/flatcar.yml'
+      - 'patches/flatcar-kernel/**'
+      - '.github/workflows/kernel.yml'
+      - 'Justfile'
+  push:
+    branches: [main]
+    paths:
+      - 'elements/flatcar/**'
+      - 'include/flatcar.yml'
+      - 'patches/flatcar-kernel/**'
+      - '.github/workflows/kernel.yml'
+      - 'Justfile'
+  workflow_dispatch:
+
+concurrency:
+  group: kernel-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  build-kernel:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.event.pull_request.head.ref || github.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+
+      - name: Set up just
+        uses: taiki-e/install-action@7b8d4719ee4aaa279bdf55df38dacb9ebfe12a6c # v2
+        with:
+          tool: just
+
+      - name: Validate Element Graph
+        run: just validate
+
+      - name: Set up runner disk space
+        run: |
+          set -euo pipefail
+          sudo mkdir -p /mnt/podman /mnt/buildstream
+          sudo chown -R "$USER:$USER" /mnt/podman /mnt/buildstream
+          mkdir -p ~/.config/containers
+          cat > ~/.config/containers/storage.conf <<EOF
+          [storage]
+          driver = "overlay"
+          runroot = "/run/user/1001"
+          graphroot = "/mnt/podman"
+          EOF
+          mkdir -p ~/.cache
+          ln -sf /mnt/buildstream ~/.cache/buildstream
+
+      - name: Build and export Flatcar LTS Kernel & ZFS sysext
+        run: just export-kernel
+
+      - name: Upload Kernel & ZFS artifacts
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: flatcar-kernel-zfs
+          path: dist/kernel/
+          if-no-files-found: error
+          retention-days: 7
diff --git a/Justfile b/Justfile
@@ -154,6 +154,32 @@ export-sysext: build-sysext
     rm -rf dist/sysext-checkout
     @echo "==> wrote k0s sysext:" && ls -lh dist/sysext/
 
+# -- Flatcar LTS kernel & ZFS --------------------------------------------------
+# Build the Flatcar LTS kernel and ZFS sysext.
+
+# Build the Flatcar LTS kernel binary and modules.
+[group('kernel')]
+build-kernel:
+    just bst build flatcar/flatcar-kernel.bst
+
+# Build the Flatcar ZFS system extension.
+[group('kernel')]
+build-zfs:
+    just bst build flatcar/flatcar-zfs.bst
+
+# Export the kernel and ZFS artifacts to dist/kernel/.
+[group('kernel')]
+export-kernel: build-kernel build-zfs
+    rm -rf dist/kernel dist/kernel-checkout dist/zfs-checkout
+    mkdir -p dist/kernel dist/kernel-checkout dist/zfs-checkout
+    just bst artifact checkout flatcar/flatcar-kernel.bst --directory /src/dist/kernel-checkout
+    just bst artifact checkout flatcar/flatcar-zfs.bst --directory /src/dist/zfs-checkout
+    cp -a dist/kernel-checkout/* dist/kernel/
+    cp -a dist/zfs-checkout/* dist/kernel/
+    rm -rf dist/kernel-checkout dist/zfs-checkout
+    (cd dist/kernel && find . -type f -exec sha256sum --binary {} + > SHA256SUMS)
+    @echo "==> wrote kernel & ZFS artifacts:" && ls -lh dist/kernel/
+
 # Write the raw GPT installer image to a physical USB drive.
 [group('installer')]
 flash-installer DEVICE="":

diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md
@@ -95,8 +95,9 @@ sudo_cmd := if `podman info >/dev/null 2>&1 && echo 1 || echo 0` == "1" { "" } e
 | Job | Workflow | Trigger | Purpose |
 |-----|----------|---------|---------|
 | `track-refs` | `build.yml` | `pull_request` (`renovate/*` only) | Resolves BuildStream junction refs and pushes them back to the PR branch. Sole `contents: write` grant on `pull_request`. |
-| `build` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Resolves the element graph, runs the full BuildStream compile, and signs the release manifest on pushes to `main`. Read-only token. |
+| `build` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Resolves the element graph, runs the full BuildStream compile (including Flatcar LTS Kernel & ZFS), and signs the release manifest on pushes to `main`. Read-only token. |
 | `release` | `build.yml` | `push/main`, `workflow_dispatch` | Downloads the signed assets handed off by `build` and publishes them to the GitHub Release (`if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/main' }}`). `contents: write`. |
+| `build-kernel` | `kernel.yml` | `pull_request` (paths: `elements/flatcar/**`, `include/flatcar.yml`, `patches/flatcar-kernel/**`), `push/main`, `workflow_dispatch` | Standalone kernel & OpenZFS sysext BuildStream build and export. Emits `dist/kernel/` artifacts. Read-only token. |
 | `docs` | `docs-checks.yml` | `pull_request`, `push/main` | Runs markdown and skill metadata checks via `docs-checks.py`. Read-only token. |
 | `unit` | `unit-tests.yml` | `pull_request`, `push/main` | Runs pytest and BATS unit test suites. Read-only token. |
 

diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md
@@ -23,6 +23,7 @@ just validate              # resolve the BuildStream graph
 just cluster-build         # submit an Argo workflow to build/publish
 just build-installer       # build the installer locally
 just export-installer      # export installer + UKI + SHA256SUMS to dist/
+just export-pxe            # export standalone PXE vmlinuz/initrd to dist/
 just build-ddi             # build the OS DDI payload
 just export-ddi            # export DDI + SHA256SUMS to dist/ddi/
 just build-sysext          # build the k0s sysext
@@ -87,11 +88,13 @@ Use direct I/O and full-block reads to avoid dirtying the page cache.
 The release process is driven by `.github/workflows/build.yml`:
 
 - Renovate point-release updates or direct pushes to `main` trigger a full build.
-- CI builds the DDI payload, installer, target UKI, and k0s sysext.
+- CI builds the DDI payload, installer, target UKI, k0s sysext, and standalone
+  PXE boot inputs (`bluefin-server-pxe-vmlinuz-*`, `bluefin-server-pxe-initrd-*.cpio.gz`).
 - CI uploads the versioned release assets to the corresponding
   `installer-v<release-version>` GitHub Release.
 - CI also produces a combined `dist/release/SHA256SUMS` manifest and signs it
-  to create `SHA256SUMS.gpg` for `systemd-sysupdate` verification.
+  to create `SHA256SUMS.gpg` for `systemd-sysupdate` verification. The PXE
+  inputs are included in this manifest, per `docs/skills/ddi-installer.md`.
 
 ## Common rationalizations
 

diff --git a/elements/bluefin-server/os-release-flatcar.bst b/elements/bluefin-server/os-release-flatcar.bst
@@ -1,8 +1,9 @@
 kind: manual
 description: Create custom Flatcar-compatible os-release metadata to support Flatcar system extensions (sysext).
 
-# %{flatcar-board} — derived from the `arch` option, never hardcoded here.
-(@): include/arch.yml
+(@):
+  - include/arch.yml
+  - include/flatcar.yml
 
 build-depends:
   # Base runtime providing /bin/sh and coreutils (cat, ln, mkdir, sed) in the sandbox.
@@ -36,7 +37,7 @@ config:
       fi
 
       # Flatcar-compatible version used by systemd-sysext / Flatcar Bakery matching.
-      FLATCAR_VERSION="4593.2.3"
+      FLATCAR_VERSION="%{flatcar-version}"
 
       cat <<EOF > "%{install-root}/usr/lib/os-release"
       NAME="Bluefin Server"

diff --git a/elements/bluefin-server/os-stack.bst b/elements/bluefin-server/os-stack.bst
@@ -25,8 +25,9 @@ depends:
   - bluefin-server/os-sshd-preset.bst
   - bluefin-server/os-sshd-config.bst
 
-  # Stock FSDK Kernel & Server-Chiseled Firmware
-  - freedesktop-sdk.bst:components/linux.bst
+  # Flatcar LTS Kernel & Matching ZFS
+  - flatcar/flatcar-kernel.bst
+  - flatcar/flatcar-zfs.bst
   - bluefin-server/linux-firmware-split.bst
 
   # Filesystem resizing support for growing root/var on first boot

diff --git a/elements/flatcar/flatcar-kernel.bst b/elements/flatcar/flatcar-kernel.bst
@@ -0,0 +1,58 @@
+kind: manual
+description: |
+  Import the Flatcar LTS Linux Kernel binary (vmlinuz) and matching module tree.
+  Contains all upstream Flatcar kernel patches (EFI secure boot lockdown, partition UUID uevents,
+  reproducible pahole builds, relative srctree kbuild).
+  Pinned in lockstep with the Flatcar release version defined in include/flatcar.yml.
+
+(@):
+  - include/arch.yml
+  - include/flatcar.yml
+
+build-depends:
+  - base/base-stack.bst
+  - freedesktop-sdk.bst:components/tar.bst
+  - freedesktop-sdk.bst:components/gzip.bst
+  - freedesktop-sdk.bst:components/xz.bst
+  - freedesktop-sdk.bst:components/findutils.bst
+
+variables:
+  strip-binaries: ""
+
+sources:
+  - kind: remote
+    url: flatcar:stable/%{flatcar-board}/%{flatcar-version}/flatcar_production_image.vmlinuz
+    ref: fe0b7ef1f9f98acc00e1db22937cdc57456df860a372fdc8eb806298f767b676
+  - kind: remote
+    url: flatcar:stable/%{flatcar-board}/%{flatcar-version}/flatcar-container.tar.gz
+    ref: c533ff576e128d11ce240e01f818c358d75b9ff118b2dd9199fbf4684547a32c
+  - kind: local
+    path: patches/flatcar-kernel
+    directory: patches
+
+config:
+  install-commands:
+    - |
+      set -euo pipefail
+
+      TARGET_MODDIR="%{install-root}/usr/lib/modules/%{flatcar-kver}"
+      mkdir -p "${TARGET_MODDIR}"
+
+      # Install the kernel binary
+      install -D -m 0644 flatcar_production_image.vmlinuz "${TARGET_MODDIR}/vmlinuz"
+
+      # Extract modules from container rootfs
+      tar -zxvf flatcar-container.tar.gz \
+        --strip-components=4 \
+        -C "${TARGET_MODDIR}" \
+        "./usr/lib/modules/%{flatcar-kver}"
+
+      # Uncompress .xz modules so dracut, depmod, and kmod can handle them consistently
+      find "${TARGET_MODDIR}" -type f -name '*.ko.xz' -exec unxz {} +
+
+      # Strip vmlinux and System.map if present to save image footprint
+      rm -f "${TARGET_MODDIR}/vmlinux" "${TARGET_MODDIR}/System.map"
+
+      # Stage Flatcar kernel patchset for reference and downstream builds
+      mkdir -p "%{install-root}/usr/src/flatcar-kernel-patches"
+      cp -a patches/*.patch "%{install-root}/usr/src/flatcar-kernel-patches/"
diff --git a/elements/flatcar/flatcar-zfs.bst b/elements/flatcar/flatcar-zfs.bst
@@ -0,0 +1,33 @@
+kind: manual
+description: |
+  Import and extract the Flatcar ZFS system extension (sysext).
+  Provides spl.ko and zfs.ko modules matching the Flatcar LTS kernel,
+  along with upstream zpool, zfs userspace tools, systemd services, and udev rules.
+
+(@):
+  - include/arch.yml
+  - include/flatcar.yml
+
+build-depends:
+  - base/base-stack.bst
+  - freedesktop-sdk.bst:components/squashfs-tools.bst
+
+variables:
+  strip-binaries: ""
+
+sources:
+  - kind: remote
+    url: flatcar:stable/%{flatcar-board}/%{flatcar-version}/flatcar-zfs.raw
+    ref: bed24d0a31b6c9ff0f1d59ec2b50d25585bf9bbcb8e9e96f5706f1a17948d971
+
+config:
+  install-commands:
+    - |
+      set -euo pipefail
+
+      # Extract the SquashFS sysext directly into the element install root
+      unsquashfs -no-xattrs -f -d "%{install-root}" flatcar-zfs.raw
+
+      # Also preserve the raw sysext image under /usr/lib/sysexts for runtime sysext usage
+      mkdir -p "%{install-root}/usr/lib/sysexts"
+      install -D -m 0644 flatcar-zfs.raw "%{install-root}/usr/lib/sysexts/zfs.raw"
diff --git a/elements/installer/installer-stack.bst b/elements/installer/installer-stack.bst
@@ -35,7 +35,8 @@ depends:
   - freedesktop-sdk.bst:components/dosfstools.bst
 
   # Kernel (installer boots from this; vmlinuz used as the UKI --linux= arg)
-  - freedesktop-sdk.bst:components/linux.bst
+  - flatcar/flatcar-kernel.bst
+  - flatcar/flatcar-zfs.bst
 
   # repart.d configs describing the TARGET disk layout
   - installer/installer-repart.bst
diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst
@@ -125,8 +125,8 @@ config:
 
       # Run dracut in generic (non-hostonly) mode and explicitly add XFS, VirtIO, and NVMe storage drivers.
       dracut -v --reproducible --no-machineid --no-hostonly \
-        --add-drivers "virtio virtio_blk virtio_pci virtio_scsi nvme nvme_core xfs erofs overlay" \
-        --filesystems "xfs vfat" \
+        --add-drivers "virtio virtio_blk virtio_pci virtio_scsi nvme nvme_core xfs erofs overlay zfs spl" \
+        --filesystems "xfs vfat zfs" \
         --install /etc/ld.so.cache \
         --kmoddir "/target-root/usr/lib/modules/${TARGET_KVER}" \
         --kernel-image "${TARGET_KERNEL}" \

diff --git a/include/aliases.yml b/include/aliases.yml
@@ -6,3 +6,4 @@ aliases:
   gitlab: https://gitlab.com/
   gnome: https://gitlab.gnome.org/GNOME/
   crates: https://static.crates.io/
+  flatcar: https://flatcar.cdn.cncf.io/
diff --git a/include/flatcar.yml b/include/flatcar.yml
@@ -0,0 +1,7 @@
+# Single source of truth for Flatcar LTS kernel and sysext versioning.
+#
+# Flatcar LTS kernel and ZFS sysext must stay strictly in lockstep because
+# OpenZFS modules (spl.ko, zfs.ko) require matching kernel vermagic (6.12.102-flatcar).
+variables:
+  flatcar-version: "4593.2.5"
+  flatcar-kver: "6.12.102-flatcar"
diff --git a/patches/flatcar-kernel/z0001-kbuild-derive-relative-path-for-srctree-from-CURDIR.patch b/patches/flatcar-kernel/z0001-kbuild-derive-relative-path-for-srctree-from-CURDIR.patch
@@ -0,0 +1,33 @@
+From dcf70e8275cf5cc964a0657312af9210996ae2e7 Mon Sep 17 00:00:00 2001
+From: Vito Caputo <vito.caputo@coreos.com>
+Date: Wed, 25 Nov 2015 02:59:45 -0800
+Subject: [PATCH 1/2] kbuild: derive relative path for srctree from CURDIR
+
+This enables relocating source and build trees to different roots,
+provided they stay reachable relative to one another.  Useful for
+builds done within a sandbox where the eventual root is prefixed
+by some undesirable path component.
+---
+ Makefile | 6 ++++--
+ 1 file changed, 4 insertions(+), 2 deletions(-)
+
+diff --git a/Makefile b/Makefile
+index a5cfcd0a85a9..b81055b65169 100644
+--- a/Makefile
++++ b/Makefile
+@@ -262,8 +262,10 @@ else
+ 	building_out_of_srctree := 1
+ endif
+
+-ifneq ($(KBUILD_ABS_SRCTREE),)
+-srctree := $(abs_srctree)
++ifneq ($(KBUILD_OUTPUT),)
++        srctree := $(shell realpath --relative-to=$(KBUILD_OUTPUT) $(abs_srctree))
++else
++        srctree := $(abs_srctree)
+ endif
+
+ objtree		:= .
+-- 
+2.25.1
+
diff --git a/patches/flatcar-kernel/z0002-pahole-support-reproducible-builds.patch b/patches/flatcar-kernel/z0002-pahole-support-reproducible-builds.patch
@@ -0,0 +1,26 @@
+From 9faff3734e6456e7927c0914829a4764ec9f1b44 Mon Sep 17 00:00:00 2001
+From: Adrian Vladu <avladu@cloudbasesolutions.com>
+Date: Tue, 17 Sep 2024 13:44:14 +0000
+Subject: [PATCH] pahole: support reproducible builds
+
+---
+ scripts/Makefile.btf | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/scripts/Makefile.btf b/scripts/Makefile.btf
+index 2d6e5ed9081e..b2f88b0fcf37 100644
+--- a/scripts/Makefile.btf
++++ b/scripts/Makefile.btf
+@@ -23,6 +23,9 @@ else
+ # Switch to using --btf_features for v1.26 and later.
+ pahole-flags-$(call test-ge, $(pahole-ver), 126)  = -j --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func
+
++# Support reproducible builds.
++pahole-flags-$(call test-ge, $(pahole-ver), 127)  = -j --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func,reproducible_build
++
+ endif
+
+ pahole-flags-$(CONFIG_PAHOLE_HAS_LANG_EXCLUDE)		+= --lang_exclude=rust
+-- 
+2.34.1
+