diff --git a/jit.c b/jit.c index 484c7a63a9e35c..e0866817f35c45 100644 --- a/jit.c +++ b/jit.c @@ -704,9 +704,9 @@ rb_jit_get_page_size(void) } #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) -// Align the current write position to a multiple of bytes -static uint8_t * -align_ptr(uint8_t *ptr, uint32_t multiple) +// Round `ptr` up to the next multiple of `multiple` bytes. Shared with zjit.c. +uint8_t * +rb_jit_align_ptr(uint8_t *ptr, uint32_t multiple) { // Compute the pointer modulo the given alignment boundary uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple; @@ -736,7 +736,7 @@ rb_jit_reserve_addr_space(uint32_t mem_size) uint8_t *const cfunc_sample_addr = (void *)(uintptr_t)&rb_jit_reserve_addr_space; uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX; // Align the requested address to page size - uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size); + uint8_t *req_addr = rb_jit_align_ptr(cfunc_sample_addr, page_size); // Probe for addresses close to this function using MAP_FIXED_NOREPLACE // to improve odds of being in range for 32-bit relative call instructions. diff --git a/string.c b/string.c index 11534c0ca1157a..32c113dc82e99e 100644 --- a/string.c +++ b/string.c @@ -9209,10 +9209,11 @@ tr_buffer_ensure_capa(struct tr_buffer *buffer, size_t extra_capa) size_t required_capa = offset + extra_capa; if (UNLIKELY(buffer->capa < required_capa)) { size_t new_capa = buffer->capa ? buffer->capa : buffer->initial_capa; + RUBY_ASSERT(new_capa >= 32); // Lower would cause infinite loop while (new_capa < required_capa) { new_capa *= 1.2; } - buffer->buf = SIZED_REALLOC_N(buffer->buf, unsigned char, new_capa, buffer->capa); + SIZED_REALLOC_N(buffer->buf, unsigned char, new_capa, buffer->capa); buffer->ptr = buffer->buf + offset; buffer->capa = new_capa; } diff --git a/test/ruby/test_variable.rb b/test/ruby/test_variable.rb index a305ad6b2a21dc..e138633536dcd9 100644 --- a/test/ruby/test_variable.rb +++ b/test/ruby/test_variable.rb @@ -71,6 +71,27 @@ def test_cloned_allows_setting_cvar assert_equal "Athena", Zeus.class_variable_get(:@@rule) end + def test_raw_object_smallest_slot + assert_separately([], <<-"end;") + require 'objspace' + base_size = ObjectSpace.memsize_of(Object.new) + + assert_equal base_size, ObjectSpace.memsize_of(Object.new) + + object = Object.new + 10.times do |i| + object.instance_variable_set("@iv_\#{i}", i) + end + + assert_equal base_size, ObjectSpace.memsize_of(Object.new) + + class TestClass + end + + assert_equal base_size, ObjectSpace.memsize_of(TestClass.new) + end; + end + def test_singleton_class_included_class_variable c = Class.new c.extend(Olympians) diff --git a/zjit.c b/zjit.c index e92eb26a409b3f..3e3c6e9c6dce5a 100644 --- a/zjit.c +++ b/zjit.c @@ -26,6 +26,10 @@ #include "ractor_core.h" #include "shape.h" +#ifndef _WIN32 +#include +#endif + // This build config impacts the pointer tagging scheme and we only want to // support one scheme for simplicity. STATIC_ASSERT(pointer_tagging_scheme, USE_FLONUM); @@ -62,6 +66,66 @@ const zjit_jit_frame_t rb_zjit_c_frame = (zjit_jit_frame_t) { .materialize_block_code = false, }; +#if !defined(_WIN32) && defined(MAP_ANONYMOUS) +uint8_t *rb_jit_align_ptr(uint8_t *ptr, uint32_t multiple); // defined in jit.c + +// Reserve address space that lives entirely below INT32_MAX for JITFrame. +// +// When a JITFrame pointer fits in 32 bits, x86_64 can encode the store +// as `mov qword ptr [mem], imm32` (8 bytes) instead of `movabs` + a store, +// and arm64 materializes it in two instructions instead of four. +// +// Like rb_jit_reserve_addr_space in jit.c, this only reserves address space (PROT_NONE). +// VirtualMem is in charge of mapping physical memory into the reserved space page by page. +void * +rb_zjit_reserve_low_addr_space(size_t size) +{ + void *mem_block = MAP_FAILED; + + // Linux (x86_64): Use MAP_32BIT to map within the first 2GiB of address space. + // This works only for x86_64, and the kernel restricts it to [1GiB, 2GiB). + #ifdef MAP_32BIT + mem_block = mmap(NULL, size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_32BIT, -1, 0); + #endif + + // Linux (all arch): Probe a free hole below 2GiB if MAP_32BIT is not possible. + // MAP_FIXED_NOREPLACE fails rather than clobbering an existing mapping. + #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) + if (mem_block == MAP_FAILED) { + // Distance between probes. 64MiB sweeps the usable 2GiB in at most 32 mmap calls. + const uintptr_t probe_stride = 64 * 1024 * 1024; + const uint32_t page_size = (uint32_t)sysconf(_SC_PAGESIZE); + const uintptr_t limit = (uintptr_t)INT32_MAX - size; + for (uintptr_t addr = probe_stride; addr < limit; addr += probe_stride) { + // mmap only honors a hint that is page-aligned. + void *req = rb_jit_align_ptr((uint8_t *)addr, page_size); + mem_block = mmap(req, size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (mem_block != MAP_FAILED) break; + } + } + #endif + + if (mem_block == MAP_FAILED) return NULL; + + // Both MAP_32BIT and MAP_FIXED_NOREPLACE are advisory in some platforms, e.g. + // sandboxes or older kernels. Fallback to normal allocation if it doesn't work. + if ((uintptr_t)mem_block + size > (uintptr_t)INT32_MAX) { + munmap(mem_block, size); + return NULL; + } + ruby_annotate_mmap(mem_block, size, "Ruby:rb_zjit_reserve_low_addr_space"); + return mem_block; +} + +#else + +// Windows not supported for now +void *rb_zjit_reserve_low_addr_space(size_t size) { return NULL; } + +#endif + void rb_zjit_profile_disable(const rb_iseq_t *iseq); int rb_zjit_insn_to_bare_insn(int insn); diff --git a/zjit.rb b/zjit.rb index f2f2742e5ae234..995d1485cd911a 100644 --- a/zjit.rb +++ b/zjit.rb @@ -185,8 +185,10 @@ def stats_string :throw_count, :side_exit_size, - :code_region_bytes, :side_exit_size_ratio, + :jit_frame_heap_bytes, + :jit_frame_region_bytes, + :code_region_bytes, :zjit_alloc_bytes, :total_mem_bytes, :total_native_stack_bytes, diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index 00d169f924fb4e..9463da3070bdc4 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -308,6 +308,7 @@ fn main() { .allowlist_function("rb_iseq_opcode_at_pc") .allowlist_function("rb_iseq_bare_opcode_at_pc") .allowlist_function("rb_jit_reserve_addr_space") + .allowlist_function("rb_zjit_reserve_low_addr_space") .allowlist_function("rb_jit_mark_writable") .allowlist_function("rb_jit_mark_executable") .allowlist_function("rb_jit_mark_unused") diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index fd4f085023f10e..1a0532ceb77390 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -2446,6 +2446,7 @@ unsafe extern "C" { pub fn rb_profile_frame_full_label(frame: VALUE) -> VALUE; pub fn rb_jit_cont_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); pub static rb_zjit_runtime_offsets: rb_zjit_runtime_offsets; + pub fn rb_zjit_reserve_low_addr_space(size: usize) -> *mut ::std::os::raw::c_void; pub fn rb_zjit_profile_disable(iseq: *const rb_iseq_t); pub fn rb_zjit_insn_to_bare_insn(insn: ::std::os::raw::c_int) -> ::std::os::raw::c_int; pub fn rb_vm_base_ptr(cfp: *mut rb_control_frame_struct) -> *mut VALUE; diff --git a/zjit/src/jit_frame.rs b/zjit/src/jit_frame.rs index 7884f10055506c..a5f22909aa4b9d 100644 --- a/zjit/src/jit_frame.rs +++ b/zjit/src/jit_frame.rs @@ -1,16 +1,77 @@ -use std::alloc::{alloc, handle_alloc_error, Layout}; +use std::alloc::{handle_alloc_error, Layout}; use std::mem::{align_of, size_of}; use std::ptr; use crate::cruby::{__IncompleteArrayField, IseqPtr, VALUE, rb_gc_mark_movable, rb_gc_location}; use crate::cruby::zjit_jit_frame; use crate::codegen::iseq_may_write_block_code; +use crate::options::get_option; use crate::state::ZJITState; +use crate::stats::{incr_counter_by, Counter}; +use crate::virtualmem::VirtualMem; /// JITFrame struct is defined in zjit.h (the single source of truth) and /// imported into Rust via bindgen. See zjit.h for field documentation. pub type JITFrame = zjit_jit_frame; +/// A bump allocator for JITFrame backed by address space below INT32_MAX. +pub struct JITFrameAllocator { + /// Address space below `INT32_MAX`. Physical pages are mapped in on demand + /// and, since the region is never marked executable, stay read/write for the + /// life of the process. + virt_mem: VirtualMem, + + /// Offset of the next free byte in `virt_mem`, or the region size once the + /// arena is exhausted. + cursor: usize, +} + +impl JITFrameAllocator { + /// Reserve the arena's address space, or return None when the platform cannot + /// provide low memory. + pub fn new() -> Option { + // A tenth of --zjit-mem-size is reserved, not mapped: physical pages are + // mapped in one at a time as the bump cursor crosses into them. lobsters + // allocates up to 4MiB of JITFrames, well below the 12.8MiB default. + Some(JITFrameAllocator { + virt_mem: VirtualMem::alloc_low(get_option!(mem_bytes) / 10)?, + cursor: 0, + }) + } + + /// The number of bytes the allocator has mapped physical memory for. + pub fn mapped_bytes(&self) -> usize { + self.virt_mem.mapped_region_size() + } + + /// Bump-allocate `layout` from the arena, or return null when it's full. + fn try_alloc(&mut self, layout: Layout) -> *mut u8 { + // The region start is page-aligned, so aligning the offset aligns the pointer. + debug_assert!(layout.align() <= self.virt_mem.system_page_size()); + let start = self.cursor.next_multiple_of(layout.align()); + let Some(end) = start.checked_add(layout.size()) else { + return ptr::null_mut(); + }; + if end > self.virt_mem.virtual_region_size() { + return ptr::null_mut(); + } + // Touch the last byte of the allocation on first use: write_byte maps every + // page between the end of the mapped region and the byte it writes. + if end > self.virt_mem.mapped_region_size() { + let last_byte = self.virt_mem.start_ptr().add_bytes(end - 1); + if self.virt_mem.write_byte(last_byte, 0).is_err() { + // write_byte's bound check refuses the final page of the region, so + // treat any failure as permanent exhaustion instead of retrying on + // every allocation. + self.cursor = self.virt_mem.virtual_region_size(); + return ptr::null_mut(); + } + } + self.cursor = end; + self.virt_mem.start_ptr().add_bytes(start).raw_ptr(&self.virt_mem) as *mut u8 + } +} + impl JITFrame { /// Allocate a JITFrame and its trailing stack map on the heap, register it /// with ZJITState, and return a raw pointer that remains valid for the @@ -27,9 +88,18 @@ impl JITFrame { .checked_add(stack_size.checked_mul(size_of::()).unwrap()) .unwrap(); let layout = Layout::from_size_align(frame_size, align_of::()).unwrap(); - let raw_ptr = unsafe { alloc(layout) as *mut JITFrame }; + // Prefer the low-address arena so that call sites can store this pointer + // as a 32-bit immediate. Falling back to the heap only costs code size. + let mut raw_ptr = match ZJITState::get_jit_frame_allocator() { + Some(arena) => arena.try_alloc(layout) as *mut JITFrame, + None => ptr::null_mut(), + }; if raw_ptr.is_null() { - handle_alloc_error(layout); + raw_ptr = unsafe { std::alloc::alloc(layout) as *mut JITFrame }; + if raw_ptr.is_null() { + handle_alloc_error(layout); + } + incr_counter_by(Counter::jit_frame_heap_bytes, layout.size() as u64); } unsafe { diff --git a/zjit/src/state.rs b/zjit/src/state.rs index 4fda1e792db5a6..f3067c664d3e85 100644 --- a/zjit/src/state.rs +++ b/zjit/src/state.rs @@ -8,7 +8,7 @@ use std::sync::atomic::Ordering; use crate::invariants::Invariants; use crate::asm::CodeBlock; use crate::options::{get_option, rb_zjit_prepare_options}; -use crate::jit_frame::JITFrame; +use crate::jit_frame::{JITFrame, JITFrameAllocator}; use crate::stats::{Counters, InsnCounters, PerfettoTracer}; use crate::virtualmem::CodePtr; use std::sync::atomic::AtomicUsize; @@ -86,6 +86,11 @@ pub struct ZJITState { /// Frame metadata for ISEQ and C calls that are known at compile time jit_frames: Vec<*mut JITFrame>, + + /// Bump allocator that serves JITFrame allocations from address space below + /// INT32_MAX, so that call sites can store frame pointers as 32-bit immediates. + /// None when the platform cannot provide low memory. + jit_frame_allocator: Option, } /// Tracks the initialization progress @@ -166,6 +171,7 @@ impl ZJITState { iseq_calls_count_pointers: HashMap::new(), perfetto_tracer, jit_frames: vec![], + jit_frame_allocator: JITFrameAllocator::new(), }; unsafe { ZJIT_STATE = Enabled(zjit_state); } @@ -209,6 +215,11 @@ impl ZJITState { &mut ZJITState::get_instance().jit_frames } + /// Get a mutable reference to the JITFrame allocator + pub fn get_jit_frame_allocator() -> Option<&'static mut JITFrameAllocator> { + ZJITState::get_instance().jit_frame_allocator.as_mut() + } + pub fn get_method_annotations() -> &'static cruby_methods::Annotations { &ZJITState::get_instance().method_annotations } diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index 7574893b7119cf..6bf4b673f32638 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -156,6 +156,7 @@ make_counters! { default { compiled_iseq_count, failed_iseq_count, + jit_frame_heap_bytes, skipped_native_stack_full, compile_time_ns, @@ -859,10 +860,12 @@ pub extern "C" fn rb_zjit_stats(_ec: EcPtr, _self: VALUE, target_key: VALUE) -> } // Memory usage stats + let jit_frame_region_bytes = ZJITState::get_jit_frame_allocator().map_or(0, |allocator| allocator.mapped_bytes()); let code_region_bytes = ZJITState::get_code_block().mapped_region_size(); + set_stat_usize!(hash, "jit_frame_region_bytes", jit_frame_region_bytes); set_stat_usize!(hash, "code_region_bytes", code_region_bytes); set_stat_usize!(hash, "zjit_alloc_bytes", zjit_alloc_bytes()); - set_stat_usize!(hash, "total_mem_bytes", code_region_bytes + zjit_alloc_bytes()); + set_stat_usize!(hash, "total_mem_bytes", code_region_bytes + jit_frame_region_bytes + zjit_alloc_bytes()); // End of default stats. Every counter beyond this is provided only for --zjit-stats. if !get_option!(stats) { diff --git a/zjit/src/virtualmem.rs b/zjit/src/virtualmem.rs index 4c512b151e526b..ae0e71ce1a8809 100644 --- a/zjit/src/virtualmem.rs +++ b/zjit/src/virtualmem.rs @@ -126,6 +126,14 @@ impl VirtualMem { Self::new(sys::SystemAllocator {}, page_size, NonNull::new(virt_block).unwrap(), exec_mem_bytes, mem_bytes) } + + /// Reserve `size` bytes of address space below `INT32_MAX` for JITFrame + pub fn alloc_low(size: usize) -> Option { + let virt_block = unsafe { rb_zjit_reserve_low_addr_space(size) } as *mut u8; + let virt_block = NonNull::new(virt_block)?; + let page_size = unsafe { rb_jit_get_page_size() }; + Some(Self::new(sys::SystemAllocator {}, page_size, virt_block, size, None)) + } } impl VirtualMemory {