Skip to content

rustc segfaults if out-of-disk is encountered inside LLVM #122089

Description

@saethlin

In a recent crater run I found a suspicious number of segfaults from rustc (search all crater logs for SIGSEGV: invalid memory reference): #121282 (comment). Examples:
https://crater-reports.s3.amazonaws.com/pr-121282/try%23d073071d77ce0f93b4fd8cc567a1e2b9e1b22126%2Brustflags=-Copt-level=3/gh/tgnottingham.valgrind-ebpf-rodata-bug/log.txt
https://crater-reports.s3.amazonaws.com/pr-121282/try%23d073071d77ce0f93b4fd8cc567a1e2b9e1b22126%2Brustflags=-Copt-level=3/gh/wyatt-herkamp.auto_project/log.txt
https://crater-reports.s3.amazonaws.com/pr-121282/try%23d073071d77ce0f93b4fd8cc567a1e2b9e1b22126%2Brustflags=-Copt-level=3/gh/wrsturgeon.gamecontroller3-nix/log.txt

It's a bit suspicious that these all appear after a cascade of LLVM ERROR output. I've modified the test rig I posted here to test this situation: #119510

new evil write impl

// cargo add libc backtrace

use std::sync::atomic::{AtomicBool, Ordering::SeqCst};

static DISK_FULL: AtomicBool = AtomicBool::new(false);

fn check() -> bool {
    let mut found_llvm = false;

    backtrace::trace(|frame| {
        backtrace::resolve_frame(frame, |symbol| {
            let Some(name) = symbol.name() else {
                return;
            };
            let Some(name) = name.as_str() else {
                return;
            };
            if name.contains("llvm") {
                found_llvm = true;
            }
        });
        !found_llvm
    });

    found_llvm
}

#[no_mangle]
pub extern "C" fn write(
    fd: libc::c_int,
    buf: *const libc::c_void,
    count: libc::size_t,
) -> libc::ssize_t {
    if fd > 2 && (DISK_FULL.load(SeqCst) || check()) {
        DISK_FULL.store(true, SeqCst);
        unsafe {
            *libc::__errno_location() = libc::ENOSPC;
        }
        return -1;
    } else {
        unsafe {
            let res =
                libc::syscall(libc::SYS_write, fd as usize, buf as usize, count as usize) as isize;
            if res < 0 {
                *libc::__errno_location() = -res as i32;
                -1
            } else {
                res
            }
        }
    }
}

Then build any sizable project with that write impl:

LD_PRELOAD=/home/ben/evil/target/release/libevil.so cargo +stable build

and you should be greeted with a huge pile of errors:

output from building exa

╭ ➜ ben@archlinux:~/rustc-perf/collector/compile-benchmarks/exa-0.10.1
╰ ➤ LD_PRELOAD=/home/ben/evil/target/release/libevil.so cargo +stable build
   Compiling libc v0.2.93
   Compiling pkg-config v0.3.19
   Compiling unicode-width v0.1.8
   Compiling tinyvec_macros v0.1.0
   Compiling matches v0.1.8
   Compiling log v0.4.14
   Compiling cfg-if v1.0.0
   Compiling percent-encoding v2.1.0
   Compiling bitflags v1.2.1
   Compiling byteorder v1.4.3
   Compiling lazy_static v1.4.0
   Compiling scoped_threadpool v0.1.9
   Compiling ansi_term v0.12.1
   Compiling natord v1.0.9
   Compiling number_prefix v0.4.0
   Compiling glob v0.3.0
   Compiling tinyvec v1.2.0
LLVM ERROR: IO failure on output stream: No space left on device
LLVM ERROR: IO failure on output stream: No space left on device
   Compiling unicode-bidi v0.3.5
error: could not compile `cfg-if` (lib)
warning: build failed, waiting for other jobs to finish...
LLVM ERROR: IO failure on output stream: No space left on device
error: could not compile `tinyvec_macros` (lib)
LLVM ERROR: IO failure on output stream: No space left on device
LLVM ERROR: IO failure on output stream: No space left on device
error: could not compile `log` (build script)
error: could not compile `lazy_static` (lib)
error: could not compile `matches` (lib)
LLVM ERROR: IO failure on output stream: No space left on device
error: could not compile `unicode-width` (lib)
LLVM ERROR: IO failure on output stream: No space left on device
LLVM ERROR: IO failure on output stream: No space left on device
error: could not compile `number_prefix` (lib)
LLVM ERROR: IO failure on output stream: No space left on device
error: could not compile `natord` (lib)
error: could not compile `percent-encoding` (lib)
LLVM ERROR: IO failure on output stream: No space left on device
error: could not compile `bitflags` (build script)
LLVM ERROR: IO failure on output stream: No space left on deviceLLVM ERROR: IO failure on output stream: No space left on device

LLVM ERROR: IO failure on output stream: No space left on device
error: could not compile `libc` (build script)
error: could not compile `scoped_threadpool` (lib)
LLVM ERROR: IO failure on output stream: No space left on device
error: could not compile `byteorder` (lib)
LLVM ERROR: IO failure on output stream: No space left on deviceLLVM ERROR: IO failure on output stream: No space left on device

LLVM ERROR: IO failure on output stream: No space left on device
LLVM ERROR: IO failure on output stream: No space left on device
error: rustc interrupted by SIGSEGV, printing backtrace

/home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/librustc_driver-ef0b2e016afc8182.so(+0x2c31aa6)[0x7e8f4da31aa6]
/usr/lib/libc.so.6(+0x3c770)[0x7e8f4ac5a770]
/home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/bin/rustc(+0x6c232)[0x63c5eff78232]
/home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/bin/rustc(+0x1c635)[0x63c5eff28635]
/usr/lib/libc.so.6(+0x3ebb0)[0x7e8f4ac5cbb0]
/usr/lib/libc.so.6(+0x3ec80)[0x7e8f4ac5cc80]
/home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/librustc_driver-ef0b2e016afc8182.so(+0x38094f9)[0x7e8f4e6094f9]
/home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/libLLVM-17-rust-1.76.0-stable.so(_ZN4llvm18report_fatal_errorERKNS_5TwineEb+0x127)[0x7e8f46e03f07]
/home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/libLLVM-17-rust-1.76.0-stable.so(+0x70869ce)[0x7e8f4a0869ce]
/home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/librustc_driver-ef0b2e016afc8182.so(LLVMRustWriteOutputFile+0x20b)[0x7e8f4f8163d3]
/home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/librustc_driver-ef0b2e016afc8182.so(+0x4a15f95)[0x7e8f4f815f95]
/home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/librustc_driver-ef0b2e016afc8182.so(+0x4a13713)[0x7e8f4f813713]
/home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/librustc_driver-ef0b2e016afc8182.so(+0x4a133c2)[0x7e8f4f8133c2]
/home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/librustc_driver-ef0b2e016afc8182.so(+0x4aabbf1)[0x7e8f4f8abbf1]
/home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/librustc_driver-ef0b2e016afc8182.so(+0x4aab6c2)[0x7e8f4f8ab6c2]
/home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/libstd-66d8041607d2929b.so(rust_metadata_std_79729d9c385e1623+0xbe8e5)[0x7e8f50b4a8e5]
/usr/lib/libc.so.6(+0x8b55a)[0x7e8f4aca955a]
/usr/lib/libc.so.6(+0x108a3c)[0x7e8f4ad26a3c]

note: we would appreciate a report at https://github.com/rust-lang/rust
error: could not compile `ansi_term` (lib)
LLVM ERROR: IO failure on output stream: No space left on deviceLLVM ERROR: IO failure on output stream: No space left on device

error: could not compile `pkg-config` (lib)
LLVM ERROR: IO failure on output stream: No space left on device
LLVM ERROR: IO failure on output stream: No space left on device
error: could not compile `unicode-bidi` (lib)
LLVM ERROR: IO failure on output stream: No space left on device
error: could not compile `tinyvec` (lib)
error: could not compile `glob` (lib)

Caused by:
  process didn't exit successfully: `/home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/bin/rustc --crate-name glob /home/ben/.cargo/registry/src/index.crates.io-6f17d22bba15001f/glob-0.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --diagnostic-width=128 --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=75c7fd398ccc3c18 -C extra-filename=-75c7fd398ccc3c18 --out-dir /home/ben/rustc-perf/collector/compile-benchmarks/exa-0.10.1/target/debug/deps -L dependency=/home/ben/rustc-perf/collector/compile-benchmarks/exa-0.10.1/target/debug/deps --cap-lints allow` (signal: 11, SIGSEGV: invalid memory reference)

gdb reports this backtrace from the core dump:

#0  0x000063c5eff78232 in tcache_bin_flush_match (edata=0x0, 
    cur_arena_ind=<error reading variable: Incompatible types on DWARF stack>, cur_binshard=<optimized out>, small=true)
    at src/tcache.c:432
#1  tcache_bin_flush_impl (tsd=0x7e8f301ffc78, cache_bin=0x7e8f302001d0, binind=21, nflush=16, small=true, 
    tcache=<optimized out>, ptrs=<optimized out>) at src/tcache.c:434
#2  tcache_bin_flush_bottom (tsd=0x7e8f301ffc78, tsd@entry=0x0, tcache=<optimized out>, cache_bin=0x7e8f302001d0, 
    cache_bin@entry=0xa, binind=21, rem=<optimized out>, small=true) at src/tcache.c:519
#3  _rjem_je_tcache_bin_flush_small (tsd=tsd@entry=0x7e8f301ffc78, tcache=<optimized out>, 
    cache_bin=cache_bin@entry=0x7e8f302001d0, binind=21, rem=<optimized out>) at src/tcache.c:529
#4  0x000063c5eff28635 in tcache_dalloc_small (tsd=0x7e8f301ffc78, tcache=0x8, ptr=0x7e8f42cb2d00, binind=26, slow_path=false)
    at include/jemalloc/internal/tcache_inlines.h:157
#5  arena_dalloc (tsdn=0x7e8f301ffc78, ptr=0x7e8f42cb2d00, tcache=0x8, slow_path=false, caller_alloc_ctx=<optimized out>)
    at include/jemalloc/internal/arena_inlines_b.h:331
#6  idalloctm (tsdn=0x7e8f301ffc78, ptr=0x7e8f42cb2d00, tcache=0x8, is_internal=false, slow_path=false, 
    alloc_ctx=<optimized out>) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120
#7  ifree (tsd=0x7e8f301ffc78, ptr=0x7e8f42cb2d00, tcache=0x8, slow_path=false) at src/jemalloc.c:2887
#8  _rjem_je_free_default (ptr=0x7e8f42cb2d00) at src/jemalloc.c:3014
#9  0x00007e8f4ac5cbb0 in __run_exit_handlers (status=101, listp=0x7e8f4adf6680 <__exit_funcs>, 
    run_list_atexit=run_list_atexit@entry=true, run_dtors=run_dtors@entry=true) at exit.c:123
#10 0x00007e8f4ac5cc80 in __GI_exit (status=<optimized out>) at exit.c:138
#11 0x00007e8f4e6094f9 in FatalErrorHandler(void*, char const*, bool) ()
   from /home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/librustc_driver-ef0b2e016afc8182.so
#12 0x00007e8f46e03f07 in llvm::report_fatal_error(llvm::Twine const&, bool) ()
   from /home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/libLLVM-17-rust-1.76.0-stable.so
#13 0x00007e8f4a0869ce in llvm::raw_fd_ostream::~raw_fd_ostream() [clone .cold.0] ()
   from /home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/libLLVM-17-rust-1.76.0-stable.so
#14 0x00007e8f4f8163d3 in LLVMRustWriteOutputFile ()
   from /home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/librustc_driver-ef0b2e016afc8182.so
#15 0x00007e8f4f815f95 in rustc_codegen_llvm::back::write::write_output_file ()
   from /home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/librustc_driver-ef0b2e016afc8182.so
#16 0x00007e8f4f813713 in rustc_codegen_llvm::back::write::codegen ()
   from /home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/librustc_driver-ef0b2e016afc8182.so
#17 0x00007e8f4f8133c2 in rustc_codegen_ssa::back::write::finish_intra_module_work::<rustc_codegen_llvm::LlvmCodegenBackend> ()
   from /home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/librustc_driver-ef0b2e016afc8182.so
#18 0x00007e8f4f8abbf1 in std::sys_common::backtrace::__rust_begin_short_backtrace::<<rustc_codegen_llvm::LlvmCodegenBackend as rustc_codegen_ssa::traits::backend::ExtraBackendMethods>::spawn_named_thread<rustc_codegen_ssa::back::write::spawn_work<rustc_codegen_llvm::LlvmCodegenBackend>::{closure#0}, ()>::{closure#0}, ()> ()
   from /home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/librustc_driver-ef0b2e016afc8182.so
#19 0x00007e8f4f8ab6c2 in <<std::thread::Builder>::spawn_unchecked_<<rustc_codegen_llvm::LlvmCodegenBackend as rustc_codegen_ssa::traits::backend::ExtraBackendMethods>::spawn_named_thread<rustc_codegen_ssa::back::write::spawn_work<rustc_codegen_llvm::LlvmCodegenBackend>::{closure#0}, ()>::{closure#0}, ()>::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0}
    () from /home/ben/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/librustc_driver-ef0b2e016afc8182.so
#20 0x00007e8f50b4a8e5 in alloc::boxed::{impl#47}::call_once<(), dyn core::ops::function::FnOnce<(), Output=()>, alloc::alloc::Global> () at library/alloc/src/boxed.rs:2015
#21 alloc::boxed::{impl#47}::call_once<(), alloc::boxed::Box<dyn core::ops::function::FnOnce<(), Output=()>, alloc::alloc::Global>, alloc::alloc::Global> () at library/alloc/src/boxed.rs:2015
#22 std::sys::unix::thread::{impl#2}::new::thread_start () at library/std/src/sys/unix/thread.rs:108
#23 0x00007e8f4aca955a in start_thread (arg=<optimized out>) at pthread_create.c:447
#24 0x00007e8f4ad26a3c in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78

This looks like allocator corruption from state corruption (a data race?) in an atexit handler.

Note that though I found this in a yet-to-be-merged PR, this trivially reproduces on stable.

Activity

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    A-LLVMArea: Code generation parts specific to LLVM. Both correctness bugs and optimization-related issues.C-bugCategory: This is a bug.I-crashIssue: The compiler crashes (SIGSEGV, SIGABRT, etc). Use I-ICE instead when the compiler panics.T-compilerRelevant to the compiler team, which will review and decide on the PR/issue.

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions