riscv: implement multikernel lifecycle and safe shutdown

Documentation/multikernel/device-tree.rst

@@ -204,6 +204,7 @@ only the boot handoff knows::
         multikernel,ipi-pages = <65>;
         multikernel,host-ipi-buffer = <...>;        /* the host's ring */
         multikernel,host-ipi-pages = <...>;
+        multikernel,host-ipi-cpu = <...>;           /* physical doorbell CPU, u64 */
     };
 
 On x86 the ``SETUP_MULTIKERNEL`` setup_data entry points at the page and

Documentation/multikernel/usage.rst

@@ -83,3 +83,37 @@ Restrictions
 - Instance files are read-only; an instance's resources change through
   overlays targeting ``/instances/<name>``.
 - Rollback (``rmdir`` on a transaction) cannot destroy a running instance.
+
+RISC-V entry stub
+=================
+
+OpenSBI ``HART_START`` does not invalidate a stopped hart's instruction
+cache, and RFENCE cannot target that hart.  The host therefore first starts
+every assigned hart at an immutable host-text trampoline.  Every hart made
+available to the pool has executed a local ``fence.i`` immediately before
+``HART_STOP``, so the trampoline cannot be fetched from an older cache line.
+The trampoline executes another ``fence.i`` and immediately calls
+``HART_STOP``, making a newly copied per-instance stub visible before its
+first fetch.  The host confirms ``STOPPED`` before the real start.  It repeats
+the handshake before donating a hart to an active instance through CPU
+hot-add.
+
+The host copies one immutable entry stub into the instance control block.
+That stub begins with ``fence.i`` before loading the current entry from the
+preceding context page and jumping to it.  It preserves the boot ABI
+registers ``a0`` and ``a1``.
+
+The multikernel manifest advertises the stub address to the spawn kernel.
+Before starting a secondary hart, the spawn kernel changes the context entry
+to ``secondary_start_sbi`` and passes the normal per-CPU boot data in ``a1``.
+Thus every HSM start reaches the immutable stub before entering replaceable
+Image code; the primary still receives its DTB and secondaries still receive
+their SBI boot data.
+
+Every local HSM stop path also executes ``fence.i`` immediately before the
+hart enters firmware, keeping the immutable host trampoline safe to fetch on
+the next start.
+
+Respawns update only the host-owned entry data, never the copied instructions.
+The priming handshake makes the immutable stub visible; the stub's own
+``fence.i`` then makes the newly written Image visible before the jump.

arch/riscv/Kconfig

@@ -1095,6 +1095,9 @@ config ARCH_SUPPORTS_MULTIKERNEL
 	depends on HOTPLUG_CPU
 	depends on !RISCV_M_MODE
 
+config ARCH_HAS_MK_POOL_STATE
+	def_bool ARCH_SUPPORTS_MULTIKERNEL
+
 config ARCH_SUPPORTS_CRASH_DUMP
 	def_bool y
 

arch/riscv/include/asm/cpu_ops_sbi.h

@@ -12,6 +12,14 @@
 
 extern const struct cpu_operations cpu_ops_sbi;
 
+int sbi_hsm_hart_start(unsigned long hartid, unsigned long saddr,
+		       unsigned long priv);
+
+#ifdef CONFIG_HOTPLUG_CPU
+int sbi_hsm_hart_stop(void);
+int sbi_hsm_hart_get_status(unsigned long hartid);
+#endif
+
 /**
  * struct sbi_hart_boot_data - Hart specific boot used during booting and
  *			       cpu hotplug.

arch/riscv/include/asm/multikernel.h

@@ -36,17 +36,25 @@ static inline int arch_cpu_from_physical_id(u64 hartid)
  */
 #define MK_CTRL_BLOCK_SIZE	(SZ_64K + 2 * PAGE_SIZE)
 
-/*
- * Architecture-private spawn and park state is added with the SBI HSM
- * and Image loader support. The compile-only skeleton intentionally has
- * none.
- */
+struct mk_riscv_spawn_context {
+	unsigned long image_entry;
+};
+
+/* Per-instance state, allocated from the control block on first spawn. */
 struct mk_instance_arch {
+	struct mk_riscv_spawn_context *ctx;
+	phys_addr_t ctx_phys;
+	const void *stub;
+	phys_addr_t stub_phys;
 };
 
 struct mk_pool_arch {
 };
 
+extern char mk_riscv_entry_stub_start[];
+extern char mk_riscv_entry_stub_end[];
+extern char mk_riscv_entry_fence_stop[];
+
 #endif /* !__ASSEMBLY__ */
 
 #endif /* _ASM_RISCV_MULTIKERNEL_H */

arch/riscv/kernel/cpu_ops_sbi.c

@@ -5,8 +5,11 @@
  * Copyright (c) 2020 Western Digital Corporation or its affiliates.
  */
 
+#include <linux/cacheflush.h>
 #include <linux/init.h>
+#include <linux/io.h>
 #include <linux/mm.h>
+#include <linux/multikernel.h>
 #include <linux/sched/task_stack.h>
 #include <asm/cpu_ops.h>
 #include <asm/cpu_ops_sbi.h>
@@ -23,45 +26,86 @@ const struct cpu_operations cpu_ops_sbi;
  */
 static struct sbi_hart_boot_data boot_data[NR_CPUS];
 
-static int sbi_hsm_hart_start(unsigned long hartid, unsigned long saddr,
-			      unsigned long priv)
+static int sbi_hsm_err_map_linux_errno(long err)
+{
+	switch (err) {
+	case SBI_ERR_ALREADY_AVAILABLE:
+	case SBI_ERR_ALREADY_STARTED:
+	case SBI_ERR_ALREADY_STOPPED:
+		return -EALREADY;
+	case SBI_ERR_FAILURE:
+		return -EIO;
+	default:
+		return sbi_err_map_linux_errno(err);
+	}
+}
+
+int sbi_hsm_hart_start(unsigned long hartid, unsigned long saddr,
+		       unsigned long priv)
 {
 	struct sbiret ret;
 
 	ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_START,
 			hartid, saddr, priv, 0, 0, 0);
 	if (ret.error)
-		return sbi_err_map_linux_errno(ret.error);
+		return sbi_hsm_err_map_linux_errno(ret.error);
 	else
 		return 0;
 }
 
 #ifdef CONFIG_HOTPLUG_CPU
-static int sbi_hsm_hart_stop(void)
+int sbi_hsm_hart_stop(void)
 {
 	struct sbiret ret;
 
+	/* A stopped hart cannot receive the remote fence for its next entry. */
+	local_flush_icache_all();
 	ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_STOP, 0, 0, 0, 0, 0, 0);
 
 	if (ret.error)
-		return sbi_err_map_linux_errno(ret.error);
+		return sbi_hsm_err_map_linux_errno(ret.error);
 	else
 		return 0;
 }
 
-static int sbi_hsm_hart_get_status(unsigned long hartid)
+int sbi_hsm_hart_get_status(unsigned long hartid)
 {
 	struct sbiret ret;
 
 	ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_STATUS,
 			hartid, 0, 0, 0, 0, 0);
 	if (ret.error)
-		return sbi_err_map_linux_errno(ret.error);
+		return sbi_hsm_err_map_linux_errno(ret.error);
 	else
 		return ret.value;
 }
 #endif
 
+#ifdef CONFIG_MULTIKERNEL
+static int sbi_spawn_cpu_entry(unsigned long *boot_addr)
+{
+	struct mk_riscv_spawn_context *ctx;
+	phys_addr_t stub_addr;
+
+	if (!mk_is_spawn_kernel())
+		return 0;
+
+	stub_addr = mk_manifest_entry_stub_phys();
+	if (stub_addr < PAGE_SIZE || !IS_ALIGNED(stub_addr, PAGE_SIZE) ||
+	    !pfn_valid(PHYS_PFN(stub_addr - PAGE_SIZE)) ||
+	    !pfn_valid(PHYS_PFN(stub_addr))) {
+		pr_err_once("SBI: invalid multikernel entry stub address %pa\n",
+			    &stub_addr);
+		return -EINVAL;
+	}
+
+	ctx = phys_to_virt(stub_addr - PAGE_SIZE);
+	WRITE_ONCE(ctx->image_entry, *boot_addr);
+	*boot_addr = stub_addr;
+	return 0;
+}
+#endif
+
 static int sbi_cpu_start(unsigned int cpuid, struct task_struct *tidle)
 {
 	unsigned long boot_addr = __pa_symbol(secondary_start_sbi);
@@ -73,6 +117,10 @@ static int sbi_cpu_start(unsigned int cpuid, struct task_struct *tidle)
 	smp_mb();
 	bdata->task_ptr = tidle;
 	bdata->stack_ptr = task_pt_regs(tidle);
+#ifdef CONFIG_MULTIKERNEL
+	if (sbi_spawn_cpu_entry(&boot_addr))
+		return -EINVAL;
+#endif
 	/* Make sure boot data is updated */
 	smp_mb();
 	hsm_data = __pa(bdata);

arch/riscv/kernel/reset.c

@@ -3,6 +3,8 @@
  * Copyright (C) 2012 Regents of the University of California
  */
 
+#include <linux/multikernel.h>
+#include <linux/panic.h>
 #include <linux/reboot.h>
 #include <linux/pm.h>
 
@@ -17,18 +19,29 @@ EXPORT_SYMBOL(pm_power_off);
 
 void machine_restart(char *cmd)
 {
+	if (mk_is_spawn_kernel() && panic_in_progress())
+		mk_panic_to_pool();
+	if (mk_is_spawn_kernel())
+		mk_halt_to_pool();
+
 	do_kernel_restart(cmd);
 	while (1);
 }
 
 void machine_halt(void)
 {
+	if (mk_is_spawn_kernel())
+		mk_halt_to_pool();
+
 	do_kernel_power_off();
 	default_power_off();
 }
 
 void machine_power_off(void)
 {
+	if (mk_is_spawn_kernel())
+		mk_halt_to_pool();
+
 	do_kernel_power_off();
 	default_power_off();
 }

arch/riscv/kernel/sbi.c

@@ -8,6 +8,7 @@
 #include <linux/bits.h>
 #include <linux/init.h>
 #include <linux/mm.h>
+#include <linux/multikernel.h>
 #include <linux/pm.h>
 #include <linux/reboot.h>
 #include <asm/sbi.h>
@@ -648,6 +649,7 @@ int sbi_debug_console_read(char *bytes, unsigned int num_bytes)
 
 void __init sbi_init(void)
 {
+	bool spawn_kernel = mk_is_spawn_kernel();
 	bool srst_power_off = false;
 	int ret;
 
@@ -682,11 +684,13 @@ void __init sbi_init(void)
 		if (sbi_spec_version >= sbi_mk_version(0, 3) &&
 		    sbi_probe_extension(SBI_EXT_SRST)) {
 			pr_info("SBI SRST extension detected\n");
-			register_platform_power_off(sbi_srst_power_off);
-			srst_power_off = true;
-			sbi_srst_reboot_nb.notifier_call = sbi_srst_reboot;
-			sbi_srst_reboot_nb.priority = 192;
-			register_restart_handler(&sbi_srst_reboot_nb);
+			if (!spawn_kernel) {
+				register_platform_power_off(sbi_srst_power_off);
+				srst_power_off = true;
+				sbi_srst_reboot_nb.notifier_call = sbi_srst_reboot;
+				sbi_srst_reboot_nb.priority = 192;
+				register_restart_handler(&sbi_srst_reboot_nb);
+			}
 		}
 		if (sbi_spec_version >= sbi_mk_version(2, 0) &&
 		    sbi_probe_extension(SBI_EXT_DBCN) > 0) {
@@ -704,6 +708,6 @@ void __init sbi_init(void)
 		__sbi_rfence	= __sbi_rfence_v01;
 	}
 
-	if (!srst_power_off)
+	if (!spawn_kernel && !srst_power_off)
 		sbi_set_power_off();
 }

arch/riscv/kernel/setup.c

@@ -13,6 +13,8 @@
 #include <linux/init.h>
 #include <linux/mm.h>
 #include <linux/memblock.h>
+#include <linux/multikernel.h>
+#include <linux/panic.h>
 #include <linux/sched.h>
 #include <linux/console.h>
 #include <linux/of_fdt.h>
@@ -316,6 +318,8 @@ extern void __init init_rt_signal_env(void);
 void __init setup_arch(char **cmdline_p)
 {
 	parse_dtb();
+	if (mk_is_spawn_kernel())
+		set_arch_panic_timeout(-1, 0);
 	setup_initial_init_mm(_stext, _etext, _edata, _end);
 
 	*cmdline_p = boot_command_line;

arch/riscv/kernel/smp.c

@@ -14,6 +14,7 @@
 #include <linux/module.h>
 #include <linux/kexec.h>
 #include <linux/kgdb.h>
+#include <linux/multikernel.h>
 #include <linux/percpu.h>
 #include <linux/profile.h>
 #include <linux/smp.h>
@@ -81,11 +82,23 @@ int riscv_hartid_to_cpuid(unsigned long hartid)
 
 static void ipi_stop(void)
 {
+	if (mk_is_spawn_kernel())
+		mk_enter_pool_state(NULL);
+
 	set_cpu_online(smp_processor_id(), false);
 	while (1)
 		wait_for_interrupt();
 }
 
+void __noreturn panic_smp_self_stop(void)
+{
+	if (mk_is_spawn_kernel())
+		mk_enter_pool_state(NULL);
+
+	for (;;)
+		cpu_relax();
+}
+
 #ifdef CONFIG_KEXEC_CORE
 static atomic_t waiting_for_crash_ipi = ATOMIC_INIT(0);
 

arch/riscv/multikernel/Makefile

@@ -1,3 +1,3 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
-obj-y += spawn.o
+obj-y += spawn.o entry.o

arch/riscv/multikernel/entry.S

@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#include <asm/page.h>
+#include <linux/linkage.h>
+
+.section ".text", "ax"
+
+.option push
+.option norvc
+.option norelax
+
+SYM_CODE_START(mk_riscv_entry_stub_start)
+	fence.i
+	auipc	t0, 0
+	li	t1, -PAGE_SIZE
+	and	t0, t0, t1
+	add	t0, t0, t1
+	ld	t0, 0(t0)
+	jr	t0
+.globl mk_riscv_entry_stub_end
+mk_riscv_entry_stub_end:
+SYM_CODE_END(mk_riscv_entry_stub_start)
+
+/*
+ * Every hart executes a local fence.i immediately before entering the CPU
+ * pool, so this immutable host-text entry cannot be fetched from an older
+ * cache line.  Use it to make a newly copied per-instance stub visible before
+ * its first fetch.
+ */
+SYM_CODE_START(mk_riscv_entry_fence_stop)
+	fence.i
+	li	a7, 0x48534d	/* SBI_EXT_HSM */
+	li	a6, 1		/* SBI_EXT_HSM_HART_STOP */
+	ecall
+.Lmk_riscv_entry_stop_failed:
+	wfi
+	j	.Lmk_riscv_entry_stop_failed
+SYM_CODE_END(mk_riscv_entry_fence_stop)
+
+.option pop