[pull] master from ruby:master

jit.c

@@ -704,9 +704,9 @@ rb_jit_get_page_size(void)
 }
 
 #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
-// Align the current write position to a multiple of bytes
-static uint8_t *
-align_ptr(uint8_t *ptr, uint32_t multiple)
+// Round `ptr` up to the next multiple of `multiple` bytes. Shared with zjit.c.
+uint8_t *
+rb_jit_align_ptr(uint8_t *ptr, uint32_t multiple)
 {
     // Compute the pointer modulo the given alignment boundary
     uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple;
@@ -736,7 +736,7 @@ rb_jit_reserve_addr_space(uint32_t mem_size)
         uint8_t *const cfunc_sample_addr = (void *)(uintptr_t)&rb_jit_reserve_addr_space;
         uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX;
         // Align the requested address to page size
-        uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size);
+        uint8_t *req_addr = rb_jit_align_ptr(cfunc_sample_addr, page_size);
 
         // Probe for addresses close to this function using MAP_FIXED_NOREPLACE
         // to improve odds of being in range for 32-bit relative call instructions.

string.c

@@ -9209,10 +9209,11 @@ tr_buffer_ensure_capa(struct tr_buffer *buffer, size_t extra_capa)
     size_t required_capa = offset + extra_capa;
     if (UNLIKELY(buffer->capa < required_capa)) {
         size_t new_capa = buffer->capa ? buffer->capa : buffer->initial_capa;
+        RUBY_ASSERT(new_capa >= 32); // Lower would cause infinite loop
         while (new_capa < required_capa) {
             new_capa *= 1.2;
         }
-        buffer->buf = SIZED_REALLOC_N(buffer->buf, unsigned char, new_capa, buffer->capa);
+        SIZED_REALLOC_N(buffer->buf, unsigned char, new_capa, buffer->capa);
         buffer->ptr = buffer->buf + offset;
         buffer->capa = new_capa;
     }

test/ruby/test_variable.rb

@@ -71,6 +71,27 @@ def test_cloned_allows_setting_cvar
     assert_equal "Athena", Zeus.class_variable_get(:@@rule)
   end
 
+  def test_raw_object_smallest_slot
+    assert_separately([], <<-"end;")
+      require 'objspace'
+      base_size = ObjectSpace.memsize_of(Object.new)
+
+      assert_equal base_size, ObjectSpace.memsize_of(Object.new)
+
+      object = Object.new
+      10.times do |i|
+        object.instance_variable_set("@iv_\#{i}", i)
+      end
+
+      assert_equal base_size, ObjectSpace.memsize_of(Object.new)
+
+      class TestClass
+      end
+
+      assert_equal base_size, ObjectSpace.memsize_of(TestClass.new)
+    end;
+  end
+
   def test_singleton_class_included_class_variable
     c = Class.new
     c.extend(Olympians)

zjit.c

@@ -26,6 +26,10 @@
 #include "ractor_core.h"
 #include "shape.h"
 
+#ifndef _WIN32
+#include <sys/mman.h>
+#endif
+
 // This build config impacts the pointer tagging scheme and we only want to
 // support one scheme for simplicity.
 STATIC_ASSERT(pointer_tagging_scheme, USE_FLONUM);
@@ -62,6 +66,66 @@ const zjit_jit_frame_t rb_zjit_c_frame = (zjit_jit_frame_t) {
     .materialize_block_code = false,
 };
 
+#if !defined(_WIN32) && defined(MAP_ANONYMOUS)
+uint8_t *rb_jit_align_ptr(uint8_t *ptr, uint32_t multiple); // defined in jit.c
+
+// Reserve address space that lives entirely below INT32_MAX for JITFrame.
+//
+// When a JITFrame pointer fits in 32 bits, x86_64 can encode the store
+// as `mov qword ptr [mem], imm32` (8 bytes) instead of `movabs` + a store,
+// and arm64 materializes it in two instructions instead of four.
+//
+// Like rb_jit_reserve_addr_space in jit.c, this only reserves address space (PROT_NONE).
+// VirtualMem is in charge of mapping physical memory into the reserved space page by page.
+void *
+rb_zjit_reserve_low_addr_space(size_t size)
+{
+    void *mem_block = MAP_FAILED;
+
+    // Linux (x86_64): Use MAP_32BIT to map within the first 2GiB of address space.
+    // This works only for x86_64, and the kernel restricts it to [1GiB, 2GiB).
+  #ifdef MAP_32BIT
+    mem_block = mmap(NULL, size, PROT_NONE,
+                     MAP_PRIVATE | MAP_ANONYMOUS | MAP_32BIT, -1, 0);
+  #endif
+
+    // Linux (all arch): Probe a free hole below 2GiB if MAP_32BIT is not possible.
+    // MAP_FIXED_NOREPLACE fails rather than clobbering an existing mapping.
+  #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
+    if (mem_block == MAP_FAILED) {
+        // Distance between probes. 64MiB sweeps the usable 2GiB in at most 32 mmap calls.
+        const uintptr_t probe_stride = 64 * 1024 * 1024;
+        const uint32_t page_size = (uint32_t)sysconf(_SC_PAGESIZE);
+        const uintptr_t limit = (uintptr_t)INT32_MAX - size;
+        for (uintptr_t addr = probe_stride; addr < limit; addr += probe_stride) {
+            // mmap only honors a hint that is page-aligned.
+            void *req = rb_jit_align_ptr((uint8_t *)addr, page_size);
+            mem_block = mmap(req, size, PROT_NONE,
+                             MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0);
+            if (mem_block != MAP_FAILED) break;
+        }
+    }
+  #endif
+
+    if (mem_block == MAP_FAILED) return NULL;
+
+    // Both MAP_32BIT and MAP_FIXED_NOREPLACE are advisory in some platforms, e.g.
+    // sandboxes or older kernels. Fallback to normal allocation if it doesn't work.
+    if ((uintptr_t)mem_block + size > (uintptr_t)INT32_MAX) {
+        munmap(mem_block, size);
+        return NULL;
+    }
+    ruby_annotate_mmap(mem_block, size, "Ruby:rb_zjit_reserve_low_addr_space");
+    return mem_block;
+}
+
+#else
+
+// Windows not supported for now
+void *rb_zjit_reserve_low_addr_space(size_t size) { return NULL; }
+
+#endif
+
 void rb_zjit_profile_disable(const rb_iseq_t *iseq);
 int rb_zjit_insn_to_bare_insn(int insn);
 

zjit.rb

@@ -185,8 +185,10 @@ def stats_string
       :throw_count,
 
       :side_exit_size,
-      :code_region_bytes,
       :side_exit_size_ratio,
+      :jit_frame_heap_bytes,
+      :jit_frame_region_bytes,
+      :code_region_bytes,
       :zjit_alloc_bytes,
       :total_mem_bytes,
       :total_native_stack_bytes,

zjit/bindgen/src/main.rs

@@ -308,6 +308,7 @@ fn main() {
         .allowlist_function("rb_iseq_opcode_at_pc")
         .allowlist_function("rb_iseq_bare_opcode_at_pc")
         .allowlist_function("rb_jit_reserve_addr_space")
+        .allowlist_function("rb_zjit_reserve_low_addr_space")
         .allowlist_function("rb_jit_mark_writable")
         .allowlist_function("rb_jit_mark_executable")
         .allowlist_function("rb_jit_mark_unused")

zjit/src/jit_frame.rs

@@ -1,16 +1,77 @@
-use std::alloc::{alloc, handle_alloc_error, Layout};
+use std::alloc::{handle_alloc_error, Layout};
 use std::mem::{align_of, size_of};
 use std::ptr;
 
 use crate::cruby::{__IncompleteArrayField, IseqPtr, VALUE, rb_gc_mark_movable, rb_gc_location};
 use crate::cruby::zjit_jit_frame;
 use crate::codegen::iseq_may_write_block_code;
+use crate::options::get_option;
 use crate::state::ZJITState;
+use crate::stats::{incr_counter_by, Counter};
+use crate::virtualmem::VirtualMem;
 
 /// JITFrame struct is defined in zjit.h (the single source of truth) and
 /// imported into Rust via bindgen. See zjit.h for field documentation.
 pub type JITFrame = zjit_jit_frame;
 
+/// A bump allocator for JITFrame backed by address space below INT32_MAX.
+pub struct JITFrameAllocator {
+    /// Address space below `INT32_MAX`. Physical pages are mapped in on demand
+    /// and, since the region is never marked executable, stay read/write for the
+    /// life of the process.
+    virt_mem: VirtualMem,
+
+    /// Offset of the next free byte in `virt_mem`, or the region size once the
+    /// arena is exhausted.
+    cursor: usize,
+}
+
+impl JITFrameAllocator {
+    /// Reserve the arena's address space, or return None when the platform cannot
+    /// provide low memory.
+    pub fn new() -> Option<Self> {
+        // A tenth of --zjit-mem-size is reserved, not mapped: physical pages are
+        // mapped in one at a time as the bump cursor crosses into them. lobsters
+        // allocates up to 4MiB of JITFrames, well below the 12.8MiB default.
+        Some(JITFrameAllocator {
+            virt_mem: VirtualMem::alloc_low(get_option!(mem_bytes) / 10)?,
+            cursor: 0,
+        })
+    }
+
+    /// The number of bytes the allocator has mapped physical memory for.
+    pub fn mapped_bytes(&self) -> usize {
+        self.virt_mem.mapped_region_size()
+    }
+
+    /// Bump-allocate `layout` from the arena, or return null when it's full.
+    fn try_alloc(&mut self, layout: Layout) -> *mut u8 {
+        // The region start is page-aligned, so aligning the offset aligns the pointer.
+        debug_assert!(layout.align() <= self.virt_mem.system_page_size());
+        let start = self.cursor.next_multiple_of(layout.align());
+        let Some(end) = start.checked_add(layout.size()) else {
+            return ptr::null_mut();
+        };
+        if end > self.virt_mem.virtual_region_size() {
+            return ptr::null_mut();
+        }
+        // Touch the last byte of the allocation on first use: write_byte maps every
+        // page between the end of the mapped region and the byte it writes.
+        if end > self.virt_mem.mapped_region_size() {
+            let last_byte = self.virt_mem.start_ptr().add_bytes(end - 1);
+            if self.virt_mem.write_byte(last_byte, 0).is_err() {
+                // write_byte's bound check refuses the final page of the region, so
+                // treat any failure as permanent exhaustion instead of retrying on
+                // every allocation.
+                self.cursor = self.virt_mem.virtual_region_size();
+                return ptr::null_mut();
+            }
+        }
+        self.cursor = end;
+        self.virt_mem.start_ptr().add_bytes(start).raw_ptr(&self.virt_mem) as *mut u8
+    }
+}
+
 impl JITFrame {
     /// Allocate a JITFrame and its trailing stack map on the heap, register it
     /// with ZJITState, and return a raw pointer that remains valid for the
@@ -27,9 +88,18 @@ impl JITFrame {
             .checked_add(stack_size.checked_mul(size_of::<VALUE>()).unwrap())
             .unwrap();
         let layout = Layout::from_size_align(frame_size, align_of::<JITFrame>()).unwrap();
-        let raw_ptr = unsafe { alloc(layout) as *mut JITFrame };
+        // Prefer the low-address arena so that call sites can store this pointer
+        // as a 32-bit immediate. Falling back to the heap only costs code size.
+        let mut raw_ptr = match ZJITState::get_jit_frame_allocator() {
+            Some(arena) => arena.try_alloc(layout) as *mut JITFrame,
+            None => ptr::null_mut(),
+        };
         if raw_ptr.is_null() {
-            handle_alloc_error(layout);
+            raw_ptr = unsafe { std::alloc::alloc(layout) as *mut JITFrame };
+            if raw_ptr.is_null() {
+                handle_alloc_error(layout);
+            }
+            incr_counter_by(Counter::jit_frame_heap_bytes, layout.size() as u64);
         }
 
         unsafe {

zjit/src/state.rs

@@ -8,7 +8,7 @@ use std::sync::atomic::Ordering;
 use crate::invariants::Invariants;
 use crate::asm::CodeBlock;
 use crate::options::{get_option, rb_zjit_prepare_options};
-use crate::jit_frame::JITFrame;
+use crate::jit_frame::{JITFrame, JITFrameAllocator};
 use crate::stats::{Counters, InsnCounters, PerfettoTracer};
 use crate::virtualmem::CodePtr;
 use std::sync::atomic::AtomicUsize;
@@ -86,6 +86,11 @@ pub struct ZJITState {
 
     /// Frame metadata for ISEQ and C calls that are known at compile time
     jit_frames: Vec<*mut JITFrame>,
+
+    /// Bump allocator that serves JITFrame allocations from address space below
+    /// INT32_MAX, so that call sites can store frame pointers as 32-bit immediates.
+    /// None when the platform cannot provide low memory.
+    jit_frame_allocator: Option<JITFrameAllocator>,
 }
 
 /// Tracks the initialization progress
@@ -166,6 +171,7 @@ impl ZJITState {
             iseq_calls_count_pointers: HashMap::new(),
             perfetto_tracer,
             jit_frames: vec![],
+            jit_frame_allocator: JITFrameAllocator::new(),
         };
         unsafe { ZJIT_STATE = Enabled(zjit_state); }
 
@@ -209,6 +215,11 @@ impl ZJITState {
         &mut ZJITState::get_instance().jit_frames
     }
 
+    /// Get a mutable reference to the JITFrame allocator
+    pub fn get_jit_frame_allocator() -> Option<&'static mut JITFrameAllocator> {
+        ZJITState::get_instance().jit_frame_allocator.as_mut()
+    }
+
     pub fn get_method_annotations() -> &'static cruby_methods::Annotations {
         &ZJITState::get_instance().method_annotations
     }

zjit/src/stats.rs

@@ -156,6 +156,7 @@ make_counters! {
     default {
         compiled_iseq_count,
         failed_iseq_count,
+        jit_frame_heap_bytes,
         skipped_native_stack_full,
 
         compile_time_ns,
@@ -859,10 +860,12 @@ pub extern "C" fn rb_zjit_stats(_ec: EcPtr, _self: VALUE, target_key: VALUE) ->
     }
 
     // Memory usage stats
+    let jit_frame_region_bytes = ZJITState::get_jit_frame_allocator().map_or(0, |allocator| allocator.mapped_bytes());
     let code_region_bytes = ZJITState::get_code_block().mapped_region_size();
+    set_stat_usize!(hash, "jit_frame_region_bytes", jit_frame_region_bytes);
     set_stat_usize!(hash, "code_region_bytes", code_region_bytes);
     set_stat_usize!(hash, "zjit_alloc_bytes", zjit_alloc_bytes());
-    set_stat_usize!(hash, "total_mem_bytes", code_region_bytes + zjit_alloc_bytes());
+    set_stat_usize!(hash, "total_mem_bytes", code_region_bytes + jit_frame_region_bytes + zjit_alloc_bytes());
 
     // End of default stats. Every counter beyond this is provided only for --zjit-stats.
     if !get_option!(stats) {

zjit/src/virtualmem.rs

@@ -126,6 +126,14 @@ impl VirtualMem {
 
         Self::new(sys::SystemAllocator {}, page_size, NonNull::new(virt_block).unwrap(), exec_mem_bytes, mem_bytes)
     }
+
+    /// Reserve `size` bytes of address space below `INT32_MAX` for JITFrame
+    pub fn alloc_low(size: usize) -> Option<Self> {
+        let virt_block = unsafe { rb_zjit_reserve_low_addr_space(size) } as *mut u8;
+        let virt_block = NonNull::new(virt_block)?;
+        let page_size = unsafe { rb_jit_get_page_size() };
+        Some(Self::new(sys::SystemAllocator {}, page_size, virt_block, size, None))
+    }
 }
 
 impl<A: Allocator> VirtualMemory<A> {