aarch64: serialize diagnostic UART records for issue 847 (R247)

docker/qemu/run-aarch64-boot-test-strict.sh

@@ -836,6 +836,9 @@ score_serial() {
         echo "Pin-guard oracle did not pass ($(grep -aE "$PIN_GUARD_ORACLE_PATTERN" "$serial_file" | tail -1))"
         return 1
     fi
+    if ! python3 "$BREENIX_ROOT/scripts/score-serial-interleave.py" "$serial_file"; then
+        return 1
+    fi
     return 0
 }
 

docs/planning/green-program/tracing/serials/847/.gitattributes

@@ -0,0 +1 @@
+* -text

docs/planning/green-program/tracing/serials/847/SHA256SUMS.txt

@@ -0,0 +1,44 @@
+705fd4d6451a31d36b3df7de96f83f30ac976c9b4a6d1e51671d8e2f33e2d0da  .gitattributes
+4b36a45842bf442b91438a18b0586b30b9f5ff4e4114649cdf4af1f2876c1194  baseline-census-red.txt
+afc07c235567234322b6ec1f9a4ac11f9f5540ef7bfa835a56f93692023e6ffa  bypass-capture.txt
+08fa7d339a3ea2d5408dad36f4f3abe3e01ad1aae1320d65b38b74e096a7f9db  bypass-ratchet.txt
+45010f14bc6749fbcc3986573d3c17a7af5d3880f00ccbf6c7ce5a4d5abb2bee  bypass-score.txt
+9eb5ef82722150f26e2e6a1c8868ba378e3101aaffbeb1c62fff16371e4a277c  bypass-serial.txt
+623b187da749efa8960f87d055a040410a325f6368e826a99bff5fc7ad4bd1f8  delete-mutation.txt
+b6e0b9e15af7e775b6bb895d49bdca0f392beef07f4dabea7e1cb5cddce8ce7c  development-first-syscall-red.txt
+9d1415afce0e3c02de6a7d35530559cd11e73dbccb358aeddf726bfb523d1986  development-smoke-green.txt
+aaf0b63932d7be2f539d8d45b5aac37f8596be525a4c18e3a754c2b60f2970b5  prod-final-serial.txt
+cd9030a4169eec8232157537a5354efc93011cc7a66ac88f95a814db8bfe8805  prod-final-transcript.txt
+0e81280b7438ee7e1613e29c3197710a5b385f6facbd2dbba8221001ad131b1d  prod-initial-serial.txt
+dac0656ce2c4d66f88f44cf1401042b74579b82b5b1d0faffb47dfd4d8b9bc05  prod-initial-transcript.txt
+1d3227748e0b5007823b2f4e8b031535463ed1abcd38a1d250f62eb3e2c015b1  service-cortex-a72-census.tsv
+8f947e01ece1a20f3c0c9a5dfda0c8d7e62d15c4f0a118241c079275de879551  service-cortex-a72-serial-1.txt
+d20acb9cfc622fe12b9b4352920cc43132beb42a4f5dc46eca153bc0b590f3f8  service-cortex-a72-serial-2.txt
+72cf1ac8abf752f55f596b7752a1cd417cc4aa2f786d751e351c4e93eb66113b  service-max-census.tsv
+fff2f05f7ac121db567176ddb37178c8c7d81f395952a7a800974bd45d888d78  service-max-serial-1.txt
+cf446dfff63cfa88c3dd3500787f8896cf0eba5b68844e401194da5a5864e83d  service-max-serial-2.txt
+f0c06ba36babacda21bcd6143d07471e6b8cd6d2c722ea7ff43e938dc5208f0f  service-transcript.txt
+cd8b8b9374e41e74e9c927270f437f438907f84e3670b28f27b5f1e5989d9151  strict-01.txt
+4577ca4993cca1e639d8c1f360e6e79db7c9557c59dd3a5e5eef05e1848373b8  strict-02.txt
+011bb0c1af586d9f73f59a0df79198d69b166ebe4ce94b00e3527bfcfc1c2f3f  strict-03.txt
+a224d8c3ebda3c8bcfb95b17d555b813aea63e2f962858ac4e452df19742923c  strict-04.txt
+7d5536e2f83f637180ae6d239844842ff0575395dda921b0f69d7456d5eb8420  strict-05.txt
+c35755541c09a5097692c7aad9bd793d59b62b289ca42528eb7e60b288105d1b  strict-06.txt
+35d82568c0962ac7adf1c8992314a2150c876209d7cc750a315ed9617315cc1f  strict-07.txt
+2d86c00a86ca25b378dba78f7dd9aee006c04a5fa6e53a89c57d1d6802d4d1c2  strict-08.txt
+7d39a155f25c023f1239ab9301449378f9739e66e3212edc9d89eee9a3e53e41  strict-09.txt
+f4ba6eda62a3e5a26ea9c8ceb378e57e5e5eb5e9d8619da63412fce99797a75d  strict-10.txt
+323cb6993020fce059bcf151bb39cfcab38269fad7b7713ebafed27c7ecdbf02  strict-11.txt
+362cc651710342dbd9e160c50f6db24e33048d98d175908306089f1c96451df7  strict-12.txt
+e6ddbbdc0115a9b280fa888bb322bc3f1c5d4f2903b8d693fb74114604512c46  strict-13.txt
+edcb5476cb50df58cb3c7be49cc09bea4a341ff3bb985f8c407f4aaac68c3dce  strict-14.txt
+619fe9d16b868217cd6ae2f69ddd7c40460e9c2fa891b448978f5f923107e80b  strict-15.txt
+5afd008def5911e7fd3051a7242605ed435aa2942036cde09cd76cdd46782b3c  strict-16.txt
+8fcd0ce8bfdb4a5a1fa037f540441a6135f8611c194c39bfcb6e872145723bec  strict-17.txt
+a334cde59b0f427a2c85efec39227b65cd9bba934f6a09db708b86c89bb4a67e  strict-18.txt
+6d685d22956f79ea5ebd8464e76a2e4cfa8fb1138190a141401cfc7dd6bdb1b7  strict-19.txt
+9f9713dee0a66f32983d9638569cb1aae9c4b3a19804a437f35b96f0be3c052d  strict-20.txt
+66676d63cb21ad46c06b294a11cf1bb5eebeec7050991330f3bcaeea3f09b113  strict20-corruption-grep.txt
+ad95e6b809d176ac02b26edce272ce954c6451cc7366e0b349275ffecbf2271c  strict20-scores.txt
+15a1d7dae500e53826bd73e76109ae9ff43eff8a97de8b9f94d13b3e624b780a  strict20-transcript.txt
+1108b784e7443f18cf27288c01cb45916695170ce79337a615a0b9c121860c83  x86-incomplete-transcript.txt

docs/planning/green-program/tracing/serials/847/baseline-census-red.txt

@@ -0,0 +1,319 @@
+revision: 4394409fca3932296f3468914b5be325ce0d48a6 plus new census ratchet, before writer conversion
+== compiling serial_line_atomicity_structure ==
+== running serial_line_atomicity_structure aarch64_raw_writers ==
+
+running 1 test
+
+thread 'aarch64_raw_writers_must_be_inside_serialized_emitter' panicked at /private/tmp/claude-501/-Users-wrb-fun-code-breenix/d69ffb9d-4539-4cf3-8a3d-a872ff7c830b/scratchpad/t847/wt/tests/serial_line_atomicity_structure.rs:1639:5:
+aarch64 raw writers outside line emitter: {
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn check_inline_eret_resume_pc -> raw_uart_str",
+    ): 5,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn check_inline_save_resume_point -> raw_uart_str",
+    ): 7,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn check_need_resched_and_switch_arm64 -> raw_uart_str",
+    ): 4,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn dispatch_thread_locked -> raw_uart_str",
+    ): 13,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn dump_all_dispatch_mismatch_snapshots -> raw_uart_str",
+    ): 7,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn dump_all_eret_frame_anomaly_snapshots -> raw_uart_str",
+    ): 9,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn dump_all_eret_guard_records -> raw_uart_str",
+    ): 9,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn dump_all_idle_redirect_histories -> raw_uart_str",
+    ): 13,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn dump_all_inline_save_skew_snapshots -> raw_uart_str",
+    ): 7,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn dump_all_last_dispatched_tids -> raw_uart_str",
+    ): 4,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn dump_all_save_skew_snapshots -> raw_uart_str",
+    ): 11,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn dump_dispatch_trace -> raw_uart_char",
+    ): 1,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn dump_dispatch_trace -> raw_uart_str",
+    ): 11,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn dump_stack_pivot_alias_history -> raw_uart_str",
+    ): 9,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn emit_el0_entry_marker -> raw_uart_str",
+    ): 2,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn emit_resume_pc_census -> raw_uart_str",
+    ): 9,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn emit_schedule_boot_marker -> raw_uart_str",
+    ): 1,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn log_bad_thread_sp -> raw_uart_str",
+    ): 11,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn log_idle_thread_context -> raw_uart_str",
+    ): 11,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn log_last_defer_requeue_snapshot -> raw_uart_str",
+    ): 9,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn raw_uart_dec -> raw_uart_char",
+    ): 1,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn raw_uart_hex -> raw_uart_char",
+    ): 1,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn raw_uart_hex -> raw_uart_str",
+    ): 1,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn raw_uart_str -> raw_uart_char",
+    ): 1,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn record_inline_elr_divergence -> raw_uart_str",
+    ): 6,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn record_resume_pc_refusal -> raw_uart_str",
+    ): 11,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn record_ret_stage_refusal -> raw_uart_str",
+    ): 6,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn restore_kernel_context_inline -> raw_uart_str",
+    ): 13,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn save_kernel_context_inline -> raw_uart_str",
+    ): 22,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn save_userspace_context_inline -> raw_uart_str",
+    ): 11,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn set_next_ttbr0_for_thread -> raw_uart_str",
+    ): 6,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn set_saved_lr -> raw_uart_str",
+    ): 6,
+    (
+        "kernel/src/arch_impl/aarch64/context_switch.rs",
+        "fn take_inline_ret_dispatch_info -> raw_uart_str",
+    ): 10,
+    (
+        "kernel/src/arch_impl/aarch64/exception.rs",
+        "fn defer_current_user_thread_sigsegv_exit -> raw_uart_str",
+    ): 6,
+    (
+        "kernel/src/arch_impl/aarch64/exception.rs",
+        "fn dump_el1_fatal_frame_and_dispatch_trace -> raw_uart_str",
+    ): 13,
+    (
+        "kernel/src/arch_impl/aarch64/exception.rs",
+        "fn dump_el1_first_fault -> raw_uart_str",
+    ): 23,
+    (
+        "kernel/src/arch_impl/aarch64/exception.rs",
+        "fn dump_fatal_postmortem_once -> raw_uart_str",
+    ): 18,
+    (
+        "kernel/src/arch_impl/aarch64/exception.rs",
+        "fn dump_fatal_postmortem_section -> raw_uart_str",
+    ): 1,
+    (
+        "kernel/src/arch_impl/aarch64/exception.rs",
+        "fn dump_stack_classification -> raw_uart_str",
+    ): 6,
+    (
+        "kernel/src/arch_impl/aarch64/exception.rs",
+        "fn handle_sync_exception -> raw_uart_str",
+    ): 196,
+    (
+        "kernel/src/arch_impl/aarch64/exception.rs",
+        "fn raw_uart_hex_u32 -> raw_uart_char",
+    ): 1,
+    (
+        "kernel/src/arch_impl/aarch64/exception.rs",
+        "fn raw_uart_hex_u32 -> raw_uart_str",
+    ): 1,
+    (
+        "kernel/src/arch_impl/aarch64/percpu.rs",
+        "fn record_cpu_identity_split -> raw_uart_str",
+    ): 6,
+    (
+        "kernel/src/arch_impl/aarch64/percpu.rs",
+        "fn record_percpu_stack_alien -> raw_uart_str",
+    ): 9,
+    (
+        "kernel/src/arch_impl/aarch64/timer_interrupt.rs",
+        "fn dump_lockup_state -> raw_serial_str",
+    ): 5,
+    (
+        "kernel/src/arch_impl/aarch64/timer_interrupt.rs",
+        "fn print_hex_u64 -> raw_serial_char",
+    ): 1,
+    (
+        "kernel/src/arch_impl/aarch64/timer_interrupt.rs",
+        "fn print_timer_count_decimal -> raw_serial_char",
+    ): 1,
+    (
+        "kernel/src/arch_impl/aarch64/timer_interrupt.rs",
+        "fn raw_serial_str -> raw_serial_str",
+    ): 1,
+    (
+        "kernel/src/arch_impl/aarch64/timer_interrupt.rs",
+        "fn timer_interrupt_handler -> raw_serial_str",
+    ): 6,
+    (
+        "kernel/src/syscall/futex_timeout_record.rs",
+        "fn record -> raw_serial_str",
+    ): 12,
+    (
+        "kernel/src/task/ret_zero_pc_oracle.rs",
+        "#[cfg(all(target_arch=aarch64,any(feature=resume_pc_el0_kernel_oracle,feature=resume_pc_el0_tid_oracle),not(feature=resume_pc_el0_frame_oracle)))] fn inject_el0_resume_pc_if_armed -> raw_uart_str",
+    ): 6,
+    (
+        "kernel/src/task/ret_zero_pc_oracle.rs",
+        "#[cfg(all(target_arch=aarch64,any(feature=resume_pc_el1_oracle,feature=eret_zero_pc_oracle,all(feature=resume_pc_el0_frame_oracle,any(feature=resume_pc_el0_kernel_oracle,feature=resume_pc_el0_tid_oracle)))))] fn inject_el1_frame_resume_pc_if_armed -> raw_uart_str",
+    ): 6,
+    (
+        "kernel/src/task/ret_zero_pc_oracle.rs",
+        "#[cfg(all(target_arch=aarch64,feature=lr_poison_oracle))] fn inject_saved_lr_if_armed -> raw_uart_str",
+    ): 3,
+    (
+        "kernel/src/task/ret_zero_pc_oracle.rs",
+        "#[cfg(all(target_arch=aarch64,feature=resume_pc_el0_frame_oracle,any(feature=resume_pc_el0_kernel_oracle,feature=resume_pc_el0_tid_oracle)))] fn inject_el0_frame_resume_pc_if_armed -> raw_uart_str",
+    ): 6,
+    (
+        "kernel/src/task/ret_zero_pc_oracle.rs",
+        "#[cfg(all(target_arch=aarch64,feature=ret_floor_oracle))] fn inject_ret_floor_if_armed -> raw_uart_str",
+    ): 3,
+    (
+        "kernel/src/task/ret_zero_pc_oracle.rs",
+        "#[cfg(all(target_arch=aarch64,feature=ret_stack_pc_oracle))] fn inject_ret_stack_pc_if_armed -> raw_uart_str",
+    ): 4,
+    (
+        "kernel/src/task/ret_zero_pc_oracle.rs",
+        "#[cfg(all(target_arch=aarch64,feature=ret_zero_pc_oracle))] fn inject_ret_zero_pc_if_armed -> raw_uart_str",
+    ): 3,
+    (
+        "kernel/src/task/ret_zero_pc_oracle.rs",
+        "#[cfg(all(target_arch=aarch64,feature=ret_zero_pc_oracle_exec))] fn inject_exec_commit_if_armed -> raw_uart_str",
+    ): 2,
+    (
+        "kernel/src/task/scheduler.rs",
+        "#[cfg(target_arch=aarch64)] fn dump_cpu_state_history -> raw_uart_str",
+    ): 9,
+    (
+        "kernel/src/task/scheduler.rs",
+        "impl Scheduler::fn hold_pinned_wake_for_home -> raw_serial_str",
+    ): 5,
+    (
+        "kernel/src/test_framework/registry.rs",
+        "fn test_serial_output -> raw_serial_str",
+    ): 1,
+    (
+        "kernel/src/tracing/output.rs",
+        "fn dump_all_buffers -> raw_serial_str",
+    ): 6,
+    (
+        "kernel/src/tracing/output.rs",
+        "fn dump_buffer -> raw_serial_str",
+    ): 7,
+    (
+        "kernel/src/tracing/output.rs",
+        "fn dump_counters -> raw_serial_char",
+    ): 2,
+    (
+        "kernel/src/tracing/output.rs",
+        "fn dump_counters -> raw_serial_str",
+    ): 9,
+    (
+        "kernel/src/tracing/output.rs",
+        "fn dump_event_summary -> raw_serial_str",
+    ): 6,
+    (
+        "kernel/src/tracing/output.rs",
+        "fn dump_latest_events -> raw_serial_str",
+    ): 3,
+    (
+        "kernel/src/tracing/output.rs",
+        "fn dump_providers -> raw_serial_str",
+    ): 7,
+    (
+        "kernel/src/tracing/output.rs",
+        "fn format_event_to_serial -> raw_serial_str",
+    ): 7,
+    (
+        "kernel/src/tracing/output.rs",
+        "fn raw_serial_dec -> raw_serial_char",
+    ): 1,
+    (
+        "kernel/src/tracing/output.rs",
+        "fn raw_serial_hex -> raw_serial_char",
+    ): 1,
+    (
+        "kernel/src/tracing/output.rs",
+        "fn raw_serial_hex -> raw_serial_str",
+    ): 1,
+    (
+        "kernel/src/tracing/output.rs",
+        "fn raw_serial_hex16 -> raw_serial_str",
+    ): 1,
+    (
+        "kernel/src/tracing/output.rs",
+        "fn raw_serial_str -> raw_serial_char",
+    ): 1,
+    (
+        "kernel/src/tty/driver.rs",
+        "impl TtyDevice::fn send_signal_to_process_nonblock -> raw_serial_str",
+    ): 2,
+}
+note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
+test aarch64_raw_writers_must_be_inside_serialized_emitter ... FAILED
+
+failures:
+
+failures:
+    aarch64_raw_writers_must_be_inside_serialized_emitter
+
+test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 11 filtered out; finished in 0.28s
+

docs/planning/green-program/tracing/serials/847/bypass-capture.txt

@@ -0,0 +1,7 @@
+revision=69a2bfae8d7e7b994dde43607ba58dd3285aba68
+mutation=Ticket::acquire returns Some without CAS; source restored before this capture
+c53d76fd9297f44076eabf6b37ae4aeab34cb91d421c474e08d46d99c3ac1b36  .tmp/kernel-aarch64-847-no-ticket
+QEMU HOST LOCK: host qemu-system-aarch64 count before acquire: 1
+qemu-system-aarch64: terminating on signal 15 from pid 62328 (<unknown process>)
+SERIAL_INTERLEAVE: records=2 corrupted=324 verdict=FAIL
+qemu_exit=124 score_exit=1