From ef78e41da116c973e66e2fbb1923982fd9587146 Mon Sep 17 00:00:00 2001 From: Joshua Young Date: Mon, 17 Aug 2026 09:20:44 +1000 Subject: [PATCH 01/25] [ruby/rubygems] Separate Bundler I/O concurrency https://github.com/ruby/rubygems/commit/2222146df1 --- lib/bundler/fetcher/compact_index.rb | 2 +- lib/bundler/fetcher/gem_remote_fetcher.rb | 2 +- lib/bundler/installer/parallel_installer.rb | 54 +++++++++---------- lib/bundler/man/bundle-config.1 | 6 ++- lib/bundler/man/bundle-config.1.ronn | 8 ++- lib/bundler/man/bundle-install.1 | 2 +- lib/bundler/man/bundle-install.1.ronn | 4 +- lib/bundler/man/bundle-update.1 | 2 +- lib/bundler/man/bundle-update.1.ronn | 4 +- lib/bundler/settings.rb | 10 ++++ lib/bundler/source/rubygems.rb | 11 ++-- lib/bundler/worker.rb | 5 +- .../bundler/fetcher/compact_index_spec.rb | 10 ++++ .../fetcher/gem_remote_fetcher_spec.rb | 6 +++ .../installer/parallel_installer_spec.rb | 39 +++++++++----- spec/bundler/bundler/settings_spec.rb | 34 ++++++++++++ spec/bundler/bundler/worker_spec.rb | 18 +++++++ 17 files changed, 161 insertions(+), 56 deletions(-) diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb index 5fa6b96c20b317..0fcb9e32320be6 100644 --- a/lib/bundler/fetcher/compact_index.rb +++ b/lib/bundler/fetcher/compact_index.rb @@ -113,7 +113,7 @@ def in_parallel(inputs, &blk) def bundle_worker(func = nil) @bundle_worker ||= begin worker_name = "Compact Index (#{display_uri.host})" - Bundler::Worker.new(Bundler.settings.processor_count, worker_name, func) + Bundler::Worker.new(Bundler.settings.metadata_parallelization, worker_name, func) end @bundle_worker.tap do |worker| worker.instance_variable_set(:@func, func) if func diff --git a/lib/bundler/fetcher/gem_remote_fetcher.rb b/lib/bundler/fetcher/gem_remote_fetcher.rb index d53a7ea52a8abf..0781632957dbb7 100644 --- a/lib/bundler/fetcher/gem_remote_fetcher.rb +++ b/lib/bundler/fetcher/gem_remote_fetcher.rb @@ -8,7 +8,7 @@ class GemRemoteFetcher < Gem::RemoteFetcher def initialize(*) super - @pool_size = Bundler.settings.installation_parallelization + @pool_size = Bundler.settings.download_parallelization ssl_ca_cert = Bundler.settings[:ssl_ca_cert] @cert_files << ssl_ca_cert if ssl_ca_cert end diff --git a/lib/bundler/installer/parallel_installer.rb b/lib/bundler/installer/parallel_installer.rb index 1b9badf0226325..ed3a5a9ff183c8 100644 --- a/lib/bundler/installer/parallel_installer.rb +++ b/lib/bundler/installer/parallel_installer.rb @@ -60,9 +60,10 @@ def self.call(*args, **kwargs) attr_reader :size - def initialize(installer, all_specs, size, standalone, force, local: false, skip: nil) + def initialize(installer, all_specs, size, standalone, force, local: false, skip: nil, download_size: Bundler.settings.download_parallelization) @installer = installer @size = size + @download_size = download_size @standalone = standalone @force = force @local = local @@ -87,7 +88,7 @@ def call Gem::Specification.reset end - if @size > 1 + if @size > 1 || @download_size > 1 install_with_worker else install_serially @@ -96,6 +97,7 @@ def call handle_error if failed_specs.any? @specs ensure + download_worker_pool&.stop worker_pool&.stop end @@ -166,17 +168,18 @@ def install_serially end end + def download_worker_pool + @download_worker_pool ||= Bundler::Worker.new(@download_size, "Gem Downloader", + ->(spec_install, worker_num) { do_download(spec_install, worker_num) }, response_queue: response_queue) + end + def worker_pool - @worker_pool ||= Bundler::Worker.new @size, "Parallel Installer", lambda {|spec_install, worker_num| - case spec_install.state - when :enqueued - do_download(spec_install, worker_num) - when :installable - do_install(spec_install, worker_num) - else - spec_install - end - } + @worker_pool ||= Bundler::Worker.new(@size, "Parallel Installer", + ->(spec_install, worker_num) { do_install(spec_install, worker_num) }, response_queue: response_queue) + end + + def response_queue + @response_queue ||= Thread::Queue.new end def do_download(spec_install, worker_num) @@ -214,24 +217,24 @@ def do_install(spec_install, worker_num) spec_install end - # Dequeue a spec and save its post-install message and then enqueue the - # remaining specs. - # Some specs might've had to wait til this spec was installed to be - # processed so the call to `enqueue_specs` is important after every - # dequeue. + # Process one completed download or installation. Downloads can finish + # before their dependencies are installed, so check all downloaded specs + # after each completion and enqueue any that are now installable. def process_specs(installed_specs) spec = worker_pool.deq if spec.installed? installed_specs[spec.name] = true - return elsif spec.failed? return - elsif spec.ready_to_install?(installed_specs) - spec.state = :installable end - worker_pool.enq(spec, priority: spec.enqueue_with_priority?) + @specs.each do |candidate| + next unless candidate.ready_to_install?(installed_specs) + + candidate.state = :installable + worker_pool.enq(candidate, priority: candidate.enqueue_with_priority?) + end end def finished_installing? @@ -270,11 +273,8 @@ def require_tree_for_spec(spec) t end - # Keys in the remains hash represent uninstalled gems specs. - # We enqueue all gem specs that do not have any dependencies. - # Later we call this lambda again to install specs that depended on - # previously installed specifications. We continue until all specs - # are installed. + # Queue every missing spec for download. `process_specs` schedules each + # downloaded spec for installation once its dependencies are installed. def enqueue_specs(installed_specs) @specs.each do |spec| if spec.installed? @@ -283,7 +283,7 @@ def enqueue_specs(installed_specs) end spec.state = :enqueued - worker_pool.enq spec + download_worker_pool.enq spec end end end diff --git a/lib/bundler/man/bundle-config.1 b/lib/bundler/man/bundle-config.1 index a962bc504f107e..347d63b2a57322 100644 --- a/lib/bundler/man/bundle-config.1 +++ b/lib/bundler/man/bundle-config.1 @@ -135,6 +135,8 @@ The store can also be selected per host with \fBcredential_store\.\fR (\fB .IP "\(bu" 4 \fBdisable_version_check\fR (\fBBUNDLE_DISABLE_VERSION_CHECK\fR): Stop Bundler from checking if a newer Bundler version is available on rubygems\.org\. .IP "\(bu" 4 +\fBdownload_jobs\fR (\fBBUNDLE_DOWNLOAD_JOBS\fR): The number of gems Bundler can download in parallel\. Defaults to three times the number of installation jobs, capped at eight\. +.IP "\(bu" 4 \fBforce_ruby_platform\fR (\fBBUNDLE_FORCE_RUBY_PLATFORM\fR): Ignore the current machine's platform and install only \fBruby\fR platform gems\. As a result, gems with native extensions will be compiled from source\. .IP "\(bu" 4 \fBfrozen\fR (\fBBUNDLE_FROZEN\fR): Disallow any automatic changes to \fBGemfile\.lock\fR\. Bundler commands will be blocked unless the lockfile can be installed exactly as written\. Usually this will happen when changing the \fBGemfile\fR manually and forgetting to update the lockfile through \fBbundle lock\fR or \fBbundle install\fR\. @@ -153,7 +155,7 @@ The store can also be selected per host with \fBcredential_store\.\fR (\fB .IP "\(bu" 4 \fBinit_gems_rb\fR (\fBBUNDLE_INIT_GEMS_RB\fR): Generate a \fBgems\.rb\fR instead of a \fBGemfile\fR when running \fBbundle init\fR\. .IP "\(bu" 4 -\fBjobs\fR (\fBBUNDLE_JOBS\fR): The number of gems Bundler can download and install in parallel\. Defaults to the number of available processors\. +\fBjobs\fR (\fBBUNDLE_JOBS\fR): The number of parallel installation jobs\. Defaults to the number of available processors\. .IP "\(bu" 4 \fBkeep_outdated_cache\fR (\fBBUNDLE_KEEP_OUTDATED_CACHE\fR): Whether Bundler should leave outdated gems unpruned when caching\. Defaults to false\. .IP "\(bu" 4 @@ -161,6 +163,8 @@ The store can also be selected per host with \fBcredential_store\.\fR (\fB .IP "\(bu" 4 \fBlockfile_checksums\fR (\fBBUNDLE_LOCKFILE_CHECKSUMS\fR): Whether Bundler should include a checksums section in new lockfiles, to protect from compromised gem sources\. Defaults to true\. Bundler's own checksum is only included when its \fB\.gem\fR file is cached, which may not be the case when Bundler is installed as a default gem\. .IP "\(bu" 4 +\fBmetadata_jobs\fR (\fBBUNDLE_METADATA_JOBS\fR): The number of compact index metadata requests Bundler can make in parallel\. Defaults to the number of download jobs\. +.IP "\(bu" 4 \fBno_build_extension\fR (\fBBUNDLE_NO_BUILD_EXTENSION\fR): Whether Bundler should skip building native extensions during installation\. When set, gems are installed without compiling their C extensions\. To build extensions later, unset this setting and run \fBbundle pristine \fR\. .IP "\(bu" 4 \fBno_install\fR (\fBBUNDLE_NO_INSTALL\fR): Whether \fBbundle package\fR should skip installing gems\. diff --git a/lib/bundler/man/bundle-config.1.ronn b/lib/bundler/man/bundle-config.1.ronn index 6b8e4569c23777..c743765d73aec4 100644 --- a/lib/bundler/man/bundle-config.1.ronn +++ b/lib/bundler/man/bundle-config.1.ronn @@ -270,6 +270,9 @@ learn more about their operation in [bundle install(1)](bundle-install.1.html). * `disable_version_check` (`BUNDLE_DISABLE_VERSION_CHECK`): Stop Bundler from checking if a newer Bundler version is available on rubygems.org. +* `download_jobs` (`BUNDLE_DOWNLOAD_JOBS`): + The number of gems Bundler can download in parallel. Defaults to three times + the number of installation jobs, capped at eight. * `force_ruby_platform` (`BUNDLE_FORCE_RUBY_PLATFORM`): Ignore the current machine's platform and install only `ruby` platform gems. As a result, gems with native extensions will be compiled from source. @@ -302,7 +305,7 @@ learn more about their operation in [bundle install(1)](bundle-install.1.html). * `init_gems_rb` (`BUNDLE_INIT_GEMS_RB`): Generate a `gems.rb` instead of a `Gemfile` when running `bundle init`. * `jobs` (`BUNDLE_JOBS`): - The number of gems Bundler can download and install in parallel. + The number of parallel installation jobs. Defaults to the number of available processors. * `keep_outdated_cache` (`BUNDLE_KEEP_OUTDATED_CACHE`): Whether Bundler should leave outdated gems unpruned when caching. Defaults @@ -315,6 +318,9 @@ learn more about their operation in [bundle install(1)](bundle-install.1.html). Whether Bundler should include a checksums section in new lockfiles, to protect from compromised gem sources. Defaults to true. Bundler's own checksum is only included when its `.gem` file is cached, which may not be the case when Bundler is installed as a default gem. +* `metadata_jobs` (`BUNDLE_METADATA_JOBS`): + The number of compact index metadata requests Bundler can make in parallel. + Defaults to the number of download jobs. * `no_build_extension` (`BUNDLE_NO_BUILD_EXTENSION`): Whether Bundler should skip building native extensions during installation. When set, gems are installed without compiling their C extensions. diff --git a/lib/bundler/man/bundle-install.1 b/lib/bundler/man/bundle-install.1 index be5a210b1a3938..3160e76164c0e5 100644 --- a/lib/bundler/man/bundle-install.1 +++ b/lib/bundler/man/bundle-install.1 @@ -26,7 +26,7 @@ Bundler will not call Rubygems' API endpoint (default) but download and cache a The location of the Gemfile(5) which Bundler should use\. This defaults to a Gemfile(5) in the current working directory\. In general, Bundler will assume that the location of the Gemfile(5) is also the project's root and will try to find \fBGemfile\.lock\fR and \fBvendor/cache\fR relative to this location\. .TP \fB\-\-jobs=\fR, \fB\-j=\fR -The maximum number of parallel download and install jobs\. The default is the number of available processors\. +The maximum number of parallel installation jobs\. The default is the number of available processors\. .TP \fB\-\-local\fR Do not attempt to connect to \fBrubygems\.org\fR\. Instead, Bundler will use the gems already present in Rubygems' cache or in \fBvendor/cache\fR\. Note that if an appropriate platform\-specific gem exists on \fBrubygems\.org\fR it will not be found\. diff --git a/lib/bundler/man/bundle-install.1.ronn b/lib/bundler/man/bundle-install.1.ronn index 5ef990223f6563..69c433c6fcb784 100644 --- a/lib/bundler/man/bundle-install.1.ronn +++ b/lib/bundler/man/bundle-install.1.ronn @@ -61,8 +61,8 @@ update process below under [CONSERVATIVE UPDATING][]. to this location. * `--jobs=`, `-j=`: - The maximum number of parallel download and install jobs. The default is the - number of available processors. + The maximum number of parallel installation jobs. The default is the number + of available processors. * `--local`: Do not attempt to connect to `rubygems.org`. Instead, Bundler will use the diff --git a/lib/bundler/man/bundle-update.1 b/lib/bundler/man/bundle-update.1 index 9d5ea89c4d7111..f8b6d442a6aeeb 100644 --- a/lib/bundler/man/bundle-update.1 +++ b/lib/bundler/man/bundle-update.1 @@ -39,7 +39,7 @@ Fall back to using the single\-file index of all gems\. Use the specified gemfile instead of [\fBGemfile(5)\fR][Gemfile(5)]\. .TP \fB\-\-jobs=\fR, \fB\-j=\fR -Specify the number of jobs to run in parallel\. The default is the number of available processors\. +Specify the number of installation jobs to run in parallel\. The default is the number of available processors\. .TP \fB\-\-retry=[]\fR Retry failed network or git requests for \fInumber\fR times\. diff --git a/lib/bundler/man/bundle-update.1.ronn b/lib/bundler/man/bundle-update.1.ronn index 3ca4dc730a2f74..24f509766d5b0c 100644 --- a/lib/bundler/man/bundle-update.1.ronn +++ b/lib/bundler/man/bundle-update.1.ronn @@ -65,8 +65,8 @@ gem. Use the specified gemfile instead of [`Gemfile(5)`][Gemfile(5)]. * `--jobs=`, `-j=`: - Specify the number of jobs to run in parallel. The default is the number of - available processors. + Specify the number of installation jobs to run in parallel. The default is + the number of available processors. * `--retry=[]`: Retry failed network or git requests for times. diff --git a/lib/bundler/settings.rb b/lib/bundler/settings.rb index 75f9feaa27eae8..7e5d6e84554f8e 100644 --- a/lib/bundler/settings.rb +++ b/lib/bundler/settings.rb @@ -46,7 +46,9 @@ class Settings NUMBER_KEYS = %w[ cooldown + download_jobs jobs + metadata_jobs redirect retry ssl_verify_mode @@ -383,10 +385,18 @@ def app_cache_path @app_cache_path ||= self[:cache_path] || "vendor/cache" end + def download_parallelization + self[:download_jobs] || [installation_parallelization * 3, 8].min + end + def installation_parallelization self[:jobs] || processor_count end + def metadata_parallelization + self[:metadata_jobs] || download_parallelization + end + ## # The cooldown that applies to a source whose Gemfile declaration asks for # +source_cooldown+ days. diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 897968b3e11bd0..41eb85f5fc8f2e 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -25,6 +25,7 @@ def initialize(options = {}) @checksum_store = Checksum::Store.new @gem_installers = {} @gem_installers_mutex = Mutex.new + @remote_spec_for_mutex = Mutex.new @remote_specs_mutex = Mutex.new cooldown = options["cooldown"] @@ -443,12 +444,14 @@ def remote_specs # Looks up a single spec in the remote sources, fetching only its own # name when the full remote index is not already materialized. def remote_spec_for(spec) - return remote_specs.search(spec).first if @remote_specs || api_fetchers.empty? + @remote_spec_for_mutex.synchronize do + return remote_specs.search(spec).first if @remote_specs || api_fetchers.empty? - index = Index.build do |idx| - fetch_names(api_fetchers, [spec.name], idx) + index = Index.build do |idx| + fetch_names(api_fetchers, [spec.name], idx) + end + index.search(spec).first end - index.search(spec).first end def fetch_names(fetchers, dependency_names, index) diff --git a/lib/bundler/worker.rb b/lib/bundler/worker.rb index 77f4f004aa6961..f4e99de305d810 100644 --- a/lib/bundler/worker.rb +++ b/lib/bundler/worker.rb @@ -19,11 +19,12 @@ def initialize(exn) # @param size [Integer] Size of pool # @param name [String] name the name of the worker # @param func [Proc] job to run in inside the worker pool - def initialize(size, name, func) + # @param response_queue [Thread::Queue] queue that receives completed jobs + def initialize(size, name, func, response_queue: Thread::Queue.new) @name = name @request_queue = Thread::Queue.new @request_queue_with_priority = Thread::Queue.new - @response_queue = Thread::Queue.new + @response_queue = response_queue @func = func @size = size @threads = nil diff --git a/spec/bundler/bundler/fetcher/compact_index_spec.rb b/spec/bundler/bundler/fetcher/compact_index_spec.rb index 67582b73d6d7b1..1cbb2e17de9895 100644 --- a/spec/bundler/bundler/fetcher/compact_index_spec.rb +++ b/spec/bundler/bundler/fetcher/compact_index_spec.rb @@ -18,6 +18,16 @@ allow(compact_index).to receive(:compact_index_client).and_return(compact_index_client) end + describe "#bundle_worker" do + it "uses metadata jobs for the worker pool size" do + Bundler.settings.temporary(metadata_jobs: 9) do + worker = compact_index.send(:bundle_worker) + + expect(worker.instance_variable_get(:@size)).to eq(9) + end + end + end + describe "#specs_for_names" do let(:thread_list) { Thread.list.select {|thread| thread.status == "run" } } let(:thread_inspection) { thread_list.map {|th| " * #{th}:\n #{th.backtrace_locations.join("\n ")}" }.join("\n") } diff --git a/spec/bundler/bundler/fetcher/gem_remote_fetcher_spec.rb b/spec/bundler/bundler/fetcher/gem_remote_fetcher_spec.rb index 2619491972ee28..e210eb6c2aebd8 100644 --- a/spec/bundler/bundler/fetcher/gem_remote_fetcher_spec.rb +++ b/spec/bundler/bundler/fetcher/gem_remote_fetcher_spec.rb @@ -6,6 +6,12 @@ RSpec.describe Bundler::Fetcher::GemRemoteFetcher do describe "#initialize" do + it "uses download jobs for the connection pool size" do + Bundler.settings.temporary(download_jobs: 7) do + expect(subject.instance_variable_get(:@pool_size)).to eq(7) + end + end + context "when ssl_ca_cert setting is not set" do before do allow(Bundler.settings).to receive(:[]).and_call_original diff --git a/spec/bundler/bundler/installer/parallel_installer_spec.rb b/spec/bundler/bundler/installer/parallel_installer_spec.rb index 20d39e885be3d8..5229405b0a249c 100644 --- a/spec/bundler/bundler/installer/parallel_installer_spec.rb +++ b/spec/bundler/bundler/installer/parallel_installer_spec.rb @@ -56,21 +56,19 @@ end let(:installer) { Bundler::Installer.new(bundled_app, definition) } - it "queues native extensions in priority" do - parallel_installer = Bundler::ParallelInstaller.new(installer, definition.specs, 2, false, true) - worker_pool = parallel_installer.send(:worker_pool) - expected = 6 # Enqueue to download bundler and the 2 gems. Enqueue to install Bundler and the 2 gems. - - expect(worker_pool).to receive(:enq).exactly(expected).times.and_wrap_original do |original_enq, spec, opts| - unless opts.nil? # Enqueued for download, no priority - if spec.name == "gem_with_extension" - expect(opts).to eq({ priority: true }) - else - expect(opts).to eq({ priority: false }) - end + it "prioritizes native extensions for installation" do + parallel_installer = Bundler::ParallelInstaller.new(installer, definition.specs, 2, false, true, download_size: 6) + download_worker_pool = parallel_installer.send(:download_worker_pool) + install_worker_pool = parallel_installer.send(:worker_pool) + + expect(download_worker_pool).to receive(:enq).exactly(3).times.and_call_original + expect(install_worker_pool).to receive(:enq).exactly(3).times.and_wrap_original do |original_enq, spec, opts| + if spec.name == "gem_with_extension" + expect(opts).to eq({ priority: true }) + else + expect(opts).to eq({ priority: false }) end - opts ||= {} original_enq.call(spec, **opts) end @@ -78,6 +76,21 @@ end end + describe "worker pools" do + it "uses separate sizes for download and installation workers" do + parallel_installer = described_class.new(nil, [], 2, false, false, download_size: 6) + + download_worker_pool = parallel_installer.send(:download_worker_pool) + install_worker_pool = parallel_installer.send(:worker_pool) + + expect(download_worker_pool.instance_variable_get(:@size)).to eq(6) + expect(install_worker_pool.instance_variable_get(:@size)).to eq(2) + ensure + download_worker_pool&.stop + install_worker_pool&.stop + end + end + describe "connect to make jobserver" do before do unless Gem::Installer.private_method_defined?(:build_jobs) diff --git a/spec/bundler/bundler/settings_spec.rb b/spec/bundler/bundler/settings_spec.rb index f80a9af27d9cc4..da25f40eaa5417 100644 --- a/spec/bundler/bundler/settings_spec.rb +++ b/spec/bundler/bundler/settings_spec.rb @@ -380,6 +380,40 @@ end end + describe "#download_parallelization" do + it "defaults to three times installation parallelization" do + allow(settings).to receive(:installation_parallelization).and_return(2) + + expect(settings.download_parallelization).to eq(6) + end + + it "caps the default at eight" do + allow(settings).to receive(:installation_parallelization).and_return(4) + + expect(settings.download_parallelization).to eq(8) + end + + it "uses configured download jobs" do + settings.temporary(download_jobs: 7) do + expect(settings.download_parallelization).to eq(7) + end + end + end + + describe "#metadata_parallelization" do + it "defaults to download parallelization" do + allow(settings).to receive(:download_parallelization).and_return(7) + + expect(settings.metadata_parallelization).to eq(7) + end + + it "uses configured metadata jobs" do + settings.temporary(metadata_jobs: 9) do + expect(settings.metadata_parallelization).to eq(9) + end + end + end + describe "#set_global" do context "when it's not possible to write to create the settings directory" do it "raises an PermissionError with explanation" do diff --git a/spec/bundler/bundler/worker_spec.rb b/spec/bundler/bundler/worker_spec.rb index 2ad2845e378c26..a13e66abc1139c 100644 --- a/spec/bundler/bundler/worker_spec.rb +++ b/spec/bundler/bundler/worker_spec.rb @@ -18,6 +18,24 @@ expect { subject.enq "a" }.to raise_error(Bundler::ThreadCreationError, "Failed to create threads for the Spec Worker worker: error creating thread") end end + + context "with a shared response queue" do + it "allows workers to publish to the same queue" do + response_queue = Thread::Queue.new + workers = [ + described_class.new(1, "First", function, response_queue: response_queue), + described_class.new(1, "Second", function, response_queue: response_queue), + ] + + workers.first.enq("first") + workers.last.enq("second") + + responses = Array.new(2) { workers.first.deq.first } + expect(responses).to contain_exactly("first", "second") + ensure + workers&.each(&:stop) + end + end end describe "priority queue" do From 427e31e8e4ae43a91500b22dba392f7938f97489 Mon Sep 17 00:00:00 2001 From: Joshua Young Date: Fri, 4 Sep 2026 12:46:07 +1000 Subject: [PATCH 02/25] [ruby/rubygems] Preserve existing Bundler concurrency settings https://github.com/ruby/rubygems/commit/8c646202df --- lib/bundler/fetcher/compact_index.rb | 2 +- lib/bundler/fetcher/gem_remote_fetcher.rb | 2 +- lib/bundler/installer/parallel_installer.rb | 11 +++--- lib/bundler/man/bundle-config.1 | 6 +--- lib/bundler/man/bundle-config.1.ronn | 8 +---- lib/bundler/man/bundle-install.1 | 2 +- lib/bundler/man/bundle-install.1.ronn | 4 +-- lib/bundler/man/bundle-update.1 | 2 +- lib/bundler/man/bundle-update.1.ronn | 4 +-- lib/bundler/settings.rb | 10 ------ .../bundler/fetcher/compact_index_spec.rb | 10 ------ .../fetcher/gem_remote_fetcher_spec.rb | 6 ---- .../installer/parallel_installer_spec.rb | 31 ++++++++++++++--- spec/bundler/bundler/settings_spec.rb | 34 ------------------- spec/bundler/bundler/worker_spec.rb | 2 +- 15 files changed, 42 insertions(+), 92 deletions(-) diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb index 0fcb9e32320be6..5fa6b96c20b317 100644 --- a/lib/bundler/fetcher/compact_index.rb +++ b/lib/bundler/fetcher/compact_index.rb @@ -113,7 +113,7 @@ def in_parallel(inputs, &blk) def bundle_worker(func = nil) @bundle_worker ||= begin worker_name = "Compact Index (#{display_uri.host})" - Bundler::Worker.new(Bundler.settings.metadata_parallelization, worker_name, func) + Bundler::Worker.new(Bundler.settings.processor_count, worker_name, func) end @bundle_worker.tap do |worker| worker.instance_variable_set(:@func, func) if func diff --git a/lib/bundler/fetcher/gem_remote_fetcher.rb b/lib/bundler/fetcher/gem_remote_fetcher.rb index 0781632957dbb7..d53a7ea52a8abf 100644 --- a/lib/bundler/fetcher/gem_remote_fetcher.rb +++ b/lib/bundler/fetcher/gem_remote_fetcher.rb @@ -8,7 +8,7 @@ class GemRemoteFetcher < Gem::RemoteFetcher def initialize(*) super - @pool_size = Bundler.settings.download_parallelization + @pool_size = Bundler.settings.installation_parallelization ssl_ca_cert = Bundler.settings[:ssl_ca_cert] @cert_files << ssl_ca_cert if ssl_ca_cert end diff --git a/lib/bundler/installer/parallel_installer.rb b/lib/bundler/installer/parallel_installer.rb index ed3a5a9ff183c8..42c7093c82fa4c 100644 --- a/lib/bundler/installer/parallel_installer.rb +++ b/lib/bundler/installer/parallel_installer.rb @@ -60,10 +60,9 @@ def self.call(*args, **kwargs) attr_reader :size - def initialize(installer, all_specs, size, standalone, force, local: false, skip: nil, download_size: Bundler.settings.download_parallelization) + def initialize(installer, all_specs, size, standalone, force, local: false, skip: nil) @installer = installer @size = size - @download_size = download_size @standalone = standalone @force = force @local = local @@ -88,7 +87,7 @@ def call Gem::Specification.reset end - if @size > 1 || @download_size > 1 + if @size > 1 install_with_worker else install_serially @@ -97,8 +96,8 @@ def call handle_error if failed_specs.any? @specs ensure - download_worker_pool&.stop - worker_pool&.stop + @worker_pool&.stop + @download_worker_pool&.stop end private @@ -169,7 +168,7 @@ def install_serially end def download_worker_pool - @download_worker_pool ||= Bundler::Worker.new(@download_size, "Gem Downloader", + @download_worker_pool ||= Bundler::Worker.new(@size, "Gem Downloader", ->(spec_install, worker_num) { do_download(spec_install, worker_num) }, response_queue: response_queue) end diff --git a/lib/bundler/man/bundle-config.1 b/lib/bundler/man/bundle-config.1 index 347d63b2a57322..a962bc504f107e 100644 --- a/lib/bundler/man/bundle-config.1 +++ b/lib/bundler/man/bundle-config.1 @@ -135,8 +135,6 @@ The store can also be selected per host with \fBcredential_store\.\fR (\fB .IP "\(bu" 4 \fBdisable_version_check\fR (\fBBUNDLE_DISABLE_VERSION_CHECK\fR): Stop Bundler from checking if a newer Bundler version is available on rubygems\.org\. .IP "\(bu" 4 -\fBdownload_jobs\fR (\fBBUNDLE_DOWNLOAD_JOBS\fR): The number of gems Bundler can download in parallel\. Defaults to three times the number of installation jobs, capped at eight\. -.IP "\(bu" 4 \fBforce_ruby_platform\fR (\fBBUNDLE_FORCE_RUBY_PLATFORM\fR): Ignore the current machine's platform and install only \fBruby\fR platform gems\. As a result, gems with native extensions will be compiled from source\. .IP "\(bu" 4 \fBfrozen\fR (\fBBUNDLE_FROZEN\fR): Disallow any automatic changes to \fBGemfile\.lock\fR\. Bundler commands will be blocked unless the lockfile can be installed exactly as written\. Usually this will happen when changing the \fBGemfile\fR manually and forgetting to update the lockfile through \fBbundle lock\fR or \fBbundle install\fR\. @@ -155,7 +153,7 @@ The store can also be selected per host with \fBcredential_store\.\fR (\fB .IP "\(bu" 4 \fBinit_gems_rb\fR (\fBBUNDLE_INIT_GEMS_RB\fR): Generate a \fBgems\.rb\fR instead of a \fBGemfile\fR when running \fBbundle init\fR\. .IP "\(bu" 4 -\fBjobs\fR (\fBBUNDLE_JOBS\fR): The number of parallel installation jobs\. Defaults to the number of available processors\. +\fBjobs\fR (\fBBUNDLE_JOBS\fR): The number of gems Bundler can download and install in parallel\. Defaults to the number of available processors\. .IP "\(bu" 4 \fBkeep_outdated_cache\fR (\fBBUNDLE_KEEP_OUTDATED_CACHE\fR): Whether Bundler should leave outdated gems unpruned when caching\. Defaults to false\. .IP "\(bu" 4 @@ -163,8 +161,6 @@ The store can also be selected per host with \fBcredential_store\.\fR (\fB .IP "\(bu" 4 \fBlockfile_checksums\fR (\fBBUNDLE_LOCKFILE_CHECKSUMS\fR): Whether Bundler should include a checksums section in new lockfiles, to protect from compromised gem sources\. Defaults to true\. Bundler's own checksum is only included when its \fB\.gem\fR file is cached, which may not be the case when Bundler is installed as a default gem\. .IP "\(bu" 4 -\fBmetadata_jobs\fR (\fBBUNDLE_METADATA_JOBS\fR): The number of compact index metadata requests Bundler can make in parallel\. Defaults to the number of download jobs\. -.IP "\(bu" 4 \fBno_build_extension\fR (\fBBUNDLE_NO_BUILD_EXTENSION\fR): Whether Bundler should skip building native extensions during installation\. When set, gems are installed without compiling their C extensions\. To build extensions later, unset this setting and run \fBbundle pristine \fR\. .IP "\(bu" 4 \fBno_install\fR (\fBBUNDLE_NO_INSTALL\fR): Whether \fBbundle package\fR should skip installing gems\. diff --git a/lib/bundler/man/bundle-config.1.ronn b/lib/bundler/man/bundle-config.1.ronn index c743765d73aec4..6b8e4569c23777 100644 --- a/lib/bundler/man/bundle-config.1.ronn +++ b/lib/bundler/man/bundle-config.1.ronn @@ -270,9 +270,6 @@ learn more about their operation in [bundle install(1)](bundle-install.1.html). * `disable_version_check` (`BUNDLE_DISABLE_VERSION_CHECK`): Stop Bundler from checking if a newer Bundler version is available on rubygems.org. -* `download_jobs` (`BUNDLE_DOWNLOAD_JOBS`): - The number of gems Bundler can download in parallel. Defaults to three times - the number of installation jobs, capped at eight. * `force_ruby_platform` (`BUNDLE_FORCE_RUBY_PLATFORM`): Ignore the current machine's platform and install only `ruby` platform gems. As a result, gems with native extensions will be compiled from source. @@ -305,7 +302,7 @@ learn more about their operation in [bundle install(1)](bundle-install.1.html). * `init_gems_rb` (`BUNDLE_INIT_GEMS_RB`): Generate a `gems.rb` instead of a `Gemfile` when running `bundle init`. * `jobs` (`BUNDLE_JOBS`): - The number of parallel installation jobs. + The number of gems Bundler can download and install in parallel. Defaults to the number of available processors. * `keep_outdated_cache` (`BUNDLE_KEEP_OUTDATED_CACHE`): Whether Bundler should leave outdated gems unpruned when caching. Defaults @@ -318,9 +315,6 @@ learn more about their operation in [bundle install(1)](bundle-install.1.html). Whether Bundler should include a checksums section in new lockfiles, to protect from compromised gem sources. Defaults to true. Bundler's own checksum is only included when its `.gem` file is cached, which may not be the case when Bundler is installed as a default gem. -* `metadata_jobs` (`BUNDLE_METADATA_JOBS`): - The number of compact index metadata requests Bundler can make in parallel. - Defaults to the number of download jobs. * `no_build_extension` (`BUNDLE_NO_BUILD_EXTENSION`): Whether Bundler should skip building native extensions during installation. When set, gems are installed without compiling their C extensions. diff --git a/lib/bundler/man/bundle-install.1 b/lib/bundler/man/bundle-install.1 index 3160e76164c0e5..be5a210b1a3938 100644 --- a/lib/bundler/man/bundle-install.1 +++ b/lib/bundler/man/bundle-install.1 @@ -26,7 +26,7 @@ Bundler will not call Rubygems' API endpoint (default) but download and cache a The location of the Gemfile(5) which Bundler should use\. This defaults to a Gemfile(5) in the current working directory\. In general, Bundler will assume that the location of the Gemfile(5) is also the project's root and will try to find \fBGemfile\.lock\fR and \fBvendor/cache\fR relative to this location\. .TP \fB\-\-jobs=\fR, \fB\-j=\fR -The maximum number of parallel installation jobs\. The default is the number of available processors\. +The maximum number of parallel download and install jobs\. The default is the number of available processors\. .TP \fB\-\-local\fR Do not attempt to connect to \fBrubygems\.org\fR\. Instead, Bundler will use the gems already present in Rubygems' cache or in \fBvendor/cache\fR\. Note that if an appropriate platform\-specific gem exists on \fBrubygems\.org\fR it will not be found\. diff --git a/lib/bundler/man/bundle-install.1.ronn b/lib/bundler/man/bundle-install.1.ronn index 69c433c6fcb784..5ef990223f6563 100644 --- a/lib/bundler/man/bundle-install.1.ronn +++ b/lib/bundler/man/bundle-install.1.ronn @@ -61,8 +61,8 @@ update process below under [CONSERVATIVE UPDATING][]. to this location. * `--jobs=`, `-j=`: - The maximum number of parallel installation jobs. The default is the number - of available processors. + The maximum number of parallel download and install jobs. The default is the + number of available processors. * `--local`: Do not attempt to connect to `rubygems.org`. Instead, Bundler will use the diff --git a/lib/bundler/man/bundle-update.1 b/lib/bundler/man/bundle-update.1 index f8b6d442a6aeeb..9d5ea89c4d7111 100644 --- a/lib/bundler/man/bundle-update.1 +++ b/lib/bundler/man/bundle-update.1 @@ -39,7 +39,7 @@ Fall back to using the single\-file index of all gems\. Use the specified gemfile instead of [\fBGemfile(5)\fR][Gemfile(5)]\. .TP \fB\-\-jobs=\fR, \fB\-j=\fR -Specify the number of installation jobs to run in parallel\. The default is the number of available processors\. +Specify the number of jobs to run in parallel\. The default is the number of available processors\. .TP \fB\-\-retry=[]\fR Retry failed network or git requests for \fInumber\fR times\. diff --git a/lib/bundler/man/bundle-update.1.ronn b/lib/bundler/man/bundle-update.1.ronn index 24f509766d5b0c..3ca4dc730a2f74 100644 --- a/lib/bundler/man/bundle-update.1.ronn +++ b/lib/bundler/man/bundle-update.1.ronn @@ -65,8 +65,8 @@ gem. Use the specified gemfile instead of [`Gemfile(5)`][Gemfile(5)]. * `--jobs=`, `-j=`: - Specify the number of installation jobs to run in parallel. The default is - the number of available processors. + Specify the number of jobs to run in parallel. The default is the number of + available processors. * `--retry=[]`: Retry failed network or git requests for times. diff --git a/lib/bundler/settings.rb b/lib/bundler/settings.rb index 7e5d6e84554f8e..75f9feaa27eae8 100644 --- a/lib/bundler/settings.rb +++ b/lib/bundler/settings.rb @@ -46,9 +46,7 @@ class Settings NUMBER_KEYS = %w[ cooldown - download_jobs jobs - metadata_jobs redirect retry ssl_verify_mode @@ -385,18 +383,10 @@ def app_cache_path @app_cache_path ||= self[:cache_path] || "vendor/cache" end - def download_parallelization - self[:download_jobs] || [installation_parallelization * 3, 8].min - end - def installation_parallelization self[:jobs] || processor_count end - def metadata_parallelization - self[:metadata_jobs] || download_parallelization - end - ## # The cooldown that applies to a source whose Gemfile declaration asks for # +source_cooldown+ days. diff --git a/spec/bundler/bundler/fetcher/compact_index_spec.rb b/spec/bundler/bundler/fetcher/compact_index_spec.rb index 1cbb2e17de9895..67582b73d6d7b1 100644 --- a/spec/bundler/bundler/fetcher/compact_index_spec.rb +++ b/spec/bundler/bundler/fetcher/compact_index_spec.rb @@ -18,16 +18,6 @@ allow(compact_index).to receive(:compact_index_client).and_return(compact_index_client) end - describe "#bundle_worker" do - it "uses metadata jobs for the worker pool size" do - Bundler.settings.temporary(metadata_jobs: 9) do - worker = compact_index.send(:bundle_worker) - - expect(worker.instance_variable_get(:@size)).to eq(9) - end - end - end - describe "#specs_for_names" do let(:thread_list) { Thread.list.select {|thread| thread.status == "run" } } let(:thread_inspection) { thread_list.map {|th| " * #{th}:\n #{th.backtrace_locations.join("\n ")}" }.join("\n") } diff --git a/spec/bundler/bundler/fetcher/gem_remote_fetcher_spec.rb b/spec/bundler/bundler/fetcher/gem_remote_fetcher_spec.rb index e210eb6c2aebd8..2619491972ee28 100644 --- a/spec/bundler/bundler/fetcher/gem_remote_fetcher_spec.rb +++ b/spec/bundler/bundler/fetcher/gem_remote_fetcher_spec.rb @@ -6,12 +6,6 @@ RSpec.describe Bundler::Fetcher::GemRemoteFetcher do describe "#initialize" do - it "uses download jobs for the connection pool size" do - Bundler.settings.temporary(download_jobs: 7) do - expect(subject.instance_variable_get(:@pool_size)).to eq(7) - end - end - context "when ssl_ca_cert setting is not set" do before do allow(Bundler.settings).to receive(:[]).and_call_original diff --git a/spec/bundler/bundler/installer/parallel_installer_spec.rb b/spec/bundler/bundler/installer/parallel_installer_spec.rb index 5229405b0a249c..5b20650dc3a1a3 100644 --- a/spec/bundler/bundler/installer/parallel_installer_spec.rb +++ b/spec/bundler/bundler/installer/parallel_installer_spec.rb @@ -57,7 +57,7 @@ let(:installer) { Bundler::Installer.new(bundled_app, definition) } it "prioritizes native extensions for installation" do - parallel_installer = Bundler::ParallelInstaller.new(installer, definition.specs, 2, false, true, download_size: 6) + parallel_installer = Bundler::ParallelInstaller.new(installer, definition.specs, 2, false, true) download_worker_pool = parallel_installer.send(:download_worker_pool) install_worker_pool = parallel_installer.send(:worker_pool) @@ -77,17 +77,38 @@ end describe "worker pools" do - it "uses separate sizes for download and installation workers" do - parallel_installer = described_class.new(nil, [], 2, false, false, download_size: 6) + it "uses separate worker pools with the configured size" do + parallel_installer = described_class.new(nil, [], 2, false, false) download_worker_pool = parallel_installer.send(:download_worker_pool) install_worker_pool = parallel_installer.send(:worker_pool) - expect(download_worker_pool.instance_variable_get(:@size)).to eq(6) + expect(download_worker_pool).not_to equal(install_worker_pool) + expect(download_worker_pool.instance_variable_get(:@size)).to eq(2) expect(install_worker_pool.instance_variable_get(:@size)).to eq(2) ensure - download_worker_pool&.stop install_worker_pool&.stop + download_worker_pool&.stop + end + + it "restores the previous interrupt handler after shutting down" do + parallel_installer = described_class.new(nil, [], 2, false, false) + download_worker_pool = parallel_installer.send(:download_worker_pool) + install_worker_pool = parallel_installer.send(:worker_pool) + + previous_handler = Signal.trap("INT", "IGNORE") + custom_handler = proc {} + Signal.trap("INT", custom_handler) + + download_worker_pool.send(:create_threads) + install_worker_pool.send(:create_threads) + + parallel_installer.call + + restored_handler = Signal.trap("INT", previous_handler) + expect(restored_handler).to equal(custom_handler) + ensure + Signal.trap("INT", previous_handler) if previous_handler end end diff --git a/spec/bundler/bundler/settings_spec.rb b/spec/bundler/bundler/settings_spec.rb index da25f40eaa5417..f80a9af27d9cc4 100644 --- a/spec/bundler/bundler/settings_spec.rb +++ b/spec/bundler/bundler/settings_spec.rb @@ -380,40 +380,6 @@ end end - describe "#download_parallelization" do - it "defaults to three times installation parallelization" do - allow(settings).to receive(:installation_parallelization).and_return(2) - - expect(settings.download_parallelization).to eq(6) - end - - it "caps the default at eight" do - allow(settings).to receive(:installation_parallelization).and_return(4) - - expect(settings.download_parallelization).to eq(8) - end - - it "uses configured download jobs" do - settings.temporary(download_jobs: 7) do - expect(settings.download_parallelization).to eq(7) - end - end - end - - describe "#metadata_parallelization" do - it "defaults to download parallelization" do - allow(settings).to receive(:download_parallelization).and_return(7) - - expect(settings.metadata_parallelization).to eq(7) - end - - it "uses configured metadata jobs" do - settings.temporary(metadata_jobs: 9) do - expect(settings.metadata_parallelization).to eq(9) - end - end - end - describe "#set_global" do context "when it's not possible to write to create the settings directory" do it "raises an PermissionError with explanation" do diff --git a/spec/bundler/bundler/worker_spec.rb b/spec/bundler/bundler/worker_spec.rb index a13e66abc1139c..7daf16cfdd2c7a 100644 --- a/spec/bundler/bundler/worker_spec.rb +++ b/spec/bundler/bundler/worker_spec.rb @@ -33,7 +33,7 @@ responses = Array.new(2) { workers.first.deq.first } expect(responses).to contain_exactly("first", "second") ensure - workers&.each(&:stop) + workers&.reverse_each(&:stop) end end end From d3c71ba02aa3e8d537a7853177025009dc329892 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 02:14:05 +0000 Subject: [PATCH 03/25] Bump taiki-e/install-action Bumps the github-actions group with 1 update in the / directory: [taiki-e/install-action](https://github.com/taiki-e/install-action). Updates `taiki-e/install-action` from 2.87.3 to 2.87.5 - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/0758d235715de2f3551eacc980d9ae8fce9342c3...5bf6ce016fd2e72eefc647cbca1e4213f65955b8) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.87.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions ... Signed-off-by: dependabot[bot] --- .github/workflows/zjit-macos.yml | 2 +- .github/workflows/zjit-ubuntu.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/zjit-macos.yml b/.github/workflows/zjit-macos.yml index b50de2ddde128a..72b3c200edb4fb 100644 --- a/.github/workflows/zjit-macos.yml +++ b/.github/workflows/zjit-macos.yml @@ -98,7 +98,7 @@ jobs: rustup install ${{ matrix.rust_version }} --profile minimal rustup default ${{ matrix.rust_version }} - - uses: taiki-e/install-action@0758d235715de2f3551eacc980d9ae8fce9342c3 # v2.87.3 + - uses: taiki-e/install-action@5bf6ce016fd2e72eefc647cbca1e4213f65955b8 # v2.87.5 with: tool: nextest@0.9 if: ${{ matrix.test_task == 'zjit-check' }} diff --git a/.github/workflows/zjit-ubuntu.yml b/.github/workflows/zjit-ubuntu.yml index 1471a0d88e7c92..b0475e62dab611 100644 --- a/.github/workflows/zjit-ubuntu.yml +++ b/.github/workflows/zjit-ubuntu.yml @@ -152,7 +152,7 @@ jobs: ruby-version: '3.1' bundler: none - - uses: taiki-e/install-action@0758d235715de2f3551eacc980d9ae8fce9342c3 # v2.87.3 + - uses: taiki-e/install-action@5bf6ce016fd2e72eefc647cbca1e4213f65955b8 # v2.87.5 with: tool: nextest@0.9 if: ${{ matrix.test_task == 'zjit-check' }} From fe2b997cdd60641f30adc39403085ab230acf066 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 30 Jun 2026 16:16:32 +0900 Subject: [PATCH 04/25] [ruby/rubygems] Stop installing native extension build logs mkmf.log and gem_make.out were written into the installed extension directory, polluting the install tree and breaking bit-for-bit reproducibility checks on distros like Guix and Nix. A successful build now leaves no logs behind, and a failed build writes them to build_info (.mkmf.log / .gem_make.out) for inspection. https://bugs.ruby-lang.org/issues/21995 https://github.com/rubygems/rubygems/issues/6259 https://github.com/ruby/rubygems/commit/5c2ef35eed Co-Authored-By: Claude Opus 4.8 --- lib/rubygems/commands/install_command.rb | 4 +- lib/rubygems/ext/builder.rb | 26 +++++- lib/rubygems/ext/ext_conf_builder.rb | 15 +-- .../test_gem_commands_install_command.rb | 7 +- .../test_gem_commands_update_command.rb | 5 +- test/rubygems/test_gem_ext_builder.rb | 91 ++++++++++++++++++- .../rubygems/test_gem_ext_ext_conf_builder.rb | 13 ++- test/rubygems/test_gem_specification.rb | 3 +- 8 files changed, 134 insertions(+), 30 deletions(-) diff --git a/lib/rubygems/commands/install_command.rb b/lib/rubygems/commands/install_command.rb index 2ebbc40a03080f..288da7c31feee7 100644 --- a/lib/rubygems/commands/install_command.rb +++ b/lib/rubygems/commands/install_command.rb @@ -93,7 +93,7 @@ def description # :nodoc: [build fails] Gem files will remain installed in \\ /path/to/gems/some_extension_gem-1.0 for inspection. - Results logged to /path/to/gems/some_extension_gem-1.0/gem_make.out + Results logged to /path/to/build_info/some_extension_gem-1.0.gem_make.out $ gem install some_extension_gem -- --with-extension-lib=/path/to/lib [build succeeds] $ gem list some_extension_gem @@ -110,7 +110,7 @@ def description # :nodoc: [build fails] Gem files will remain installed in \\ /path/to/gems/some_extension_gem-1.0 for inspection. - Results logged to /path/to/gems/some_extension_gem-1.0/gem_make.out + Results logged to /path/to/build_info/some_extension_gem-1.0.gem_make.out $ [cd /path/to/gems/some_extension_gem-1.0] $ [edit files or what-have-you and run make] $ gem spec ../../cache/some_extension_gem-1.0.gem --ruby > \\ diff --git a/lib/rubygems/ext/builder.rb b/lib/rubygems/ext/builder.rb index f1fce48823a888..307b81bcd244c8 100644 --- a/lib/rubygems/ext/builder.rb +++ b/lib/rubygems/ext/builder.rb @@ -243,9 +243,25 @@ def build_extension(extension, dest_path) # :nodoc: verbose { results.join("\n") } - write_gem_make_out results.join "\n" + # mkmf.log is a noisy, non-reproducible build artifact that is not meant + # to be installed. Drop the one left behind by a successful build instead + # of leaving it in the installation tree. + FileUtils.rm_f File.join(extension_dir, "mkmf.log") rescue StandardError => e results << e.message + + # On failure keep the mkmf.log for inspection, but move it out of the + # installation tree into the build_info directory. + mkmf_log = File.join extension_dir, "mkmf.log" + if File.exist?(mkmf_log) + mkmf_log_dest = File.join @spec.build_info_dir, "#{@spec.full_name}.mkmf.log" + FileUtils.mkdir_p @spec.build_info_dir + FileUtils.mv mkmf_log, mkmf_log_dest + + results << "To see why this extension failed to compile, please check the mkmf.log which can be found here:" + results << " #{mkmf_log_dest}" + end + build_error(results.join("\n"), $@) end end @@ -277,12 +293,14 @@ def build_extensions end ## - # Writes +output+ to gem_make.out in the extension install directory. + # Writes +output+ to gem_make.out in the build_info directory. Only called + # on failure (via #build_error), to keep build logs out of the installation + # tree. def write_gem_make_out(output) # :nodoc: - destination = File.join @spec.extension_dir, "gem_make.out" + destination = File.join @spec.build_info_dir, "#{@spec.full_name}.gem_make.out" - FileUtils.mkdir_p @spec.extension_dir + FileUtils.mkdir_p @spec.build_info_dir File.open destination, "wb" do |io| io.puts output diff --git a/lib/rubygems/ext/ext_conf_builder.rb b/lib/rubygems/ext/ext_conf_builder.rb index 822454355d104d..a658e0b0eb5963 100644 --- a/lib/rubygems/ext/ext_conf_builder.rb +++ b/lib/rubygems/ext/ext_conf_builder.rb @@ -27,17 +27,10 @@ def self.build(extension, dest_path, results, args = [], lib_dir = nil, extensio cmd << "--target-rbconfig=#{target_rbconfig.path}" if target_rbconfig.path cmd.push(*args) - run(cmd, results, class_name, extension_dir) do |s, r| - mkmf_log = File.join(extension_dir, "mkmf.log") - if File.exist? mkmf_log - unless s.success? - r << "To see why this extension failed to compile, please check" \ - " the mkmf.log which can be found here:\n" - r << " " + File.join(dest_path, "mkmf.log") + "\n" - end - FileUtils.mv mkmf_log, dest_path - end - end + # Leave mkmf.log in the extension directory. The final placement (dropped + # on success, moved to build_info on failure) is decided by + # Gem::Ext::Builder#build_extension. + run(cmd, results, class_name, extension_dir) ENV["DESTDIR"] = nil diff --git a/test/rubygems/test_gem_commands_install_command.rb b/test/rubygems/test_gem_commands_install_command.rb index 01786bea58cb89..1b8978670462c8 100644 --- a/test/rubygems/test_gem_commands_install_command.rb +++ b/test/rubygems/test_gem_commands_install_command.rb @@ -1740,6 +1740,9 @@ def test_pass_down_the_job_option_to_make write_file(extconf_path) do |io| io.puts "require 'mkmf'" + # Force the build to fail at the make stage so the build log is + # written. The make command line (including -j) is recorded there. + io.puts "File.write('a.c', '#error forced build failure for test')" io.puts "create_makefile '#{spec.name}'" end @@ -1748,12 +1751,12 @@ def test_pass_down_the_job_option_to_make end use_ui @ui do - assert_raise Gem::MockGemUi::SystemExitException, @ui.error do + assert_raise Gem::MockGemUi::TermError, @ui.error do @cmd.invoke "a", "-j4" end end - gem_make_out = File.read(File.join(gemspec.extension_dir, "gem_make.out")) + gem_make_out = File.read(File.join(gemspec.build_info_dir, "#{gemspec.full_name}.gem_make.out")) if vc_windows? && nmake_found? refute_includes(gem_make_out, " -j4") else diff --git a/test/rubygems/test_gem_commands_update_command.rb b/test/rubygems/test_gem_commands_update_command.rb index fc42baa4a7bf70..1449f5425f4efe 100644 --- a/test/rubygems/test_gem_commands_update_command.rb +++ b/test/rubygems/test_gem_commands_update_command.rb @@ -954,6 +954,9 @@ def test_pass_down_the_job_option_to_make write_file(extconf_path) do |io| io.puts "require 'mkmf'" + # Force the build to fail at the make stage so the build log is + # written. The make command line (including -j) is recorded there. + io.puts "File.write('a.c', '#error forced build failure for test')" io.puts "create_makefile '#{spec.name}'" end @@ -967,7 +970,7 @@ def test_pass_down_the_job_option_to_make @cmd.invoke("a", "-j2") end - gem_make_out = File.read(File.join(gemspec.extension_dir, "gem_make.out")) + gem_make_out = File.read(File.join(gemspec.build_info_dir, "#{gemspec.full_name}.gem_make.out")) if vc_windows? && nmake_found? refute_includes(gem_make_out, " -j2") else diff --git a/test/rubygems/test_gem_ext_builder.rb b/test/rubygems/test_gem_ext_builder.rb index 8f90687ede306f..80b2db2d9a2ecf 100644 --- a/test/rubygems/test_gem_ext_builder.rb +++ b/test/rubygems/test_gem_ext_builder.rb @@ -244,7 +244,9 @@ def test_build_extensions assert_path_exist @spec.extension_dir assert_path_exist @spec.gem_build_complete_path - assert_path_exist File.join @spec.extension_dir, "gem_make.out" + assert_path_not_exist File.join @spec.extension_dir, "gem_make.out" + assert_path_not_exist File.join @spec.extension_dir, "mkmf.log" + assert_path_not_exist File.join @spec.gem_dir, "ext", "mkmf.log" assert_path_exist File.join @spec.extension_dir, "a.rb" assert_path_exist File.join @spec.gem_dir, "lib", "a.rb" assert_path_exist File.join @spec.gem_dir, "lib", "a", "b.rb" @@ -298,7 +300,9 @@ def test_build_extensions_install_ext_only assert_path_exist @spec.extension_dir assert_path_exist @spec.gem_build_complete_path - assert_path_exist File.join @spec.extension_dir, "gem_make.out" + assert_path_not_exist File.join @spec.extension_dir, "gem_make.out" + assert_path_not_exist File.join @spec.extension_dir, "mkmf.log" + assert_path_not_exist File.join @spec.gem_dir, "ext", "mkmf.log" assert_path_exist File.join @spec.extension_dir, "a.rb" assert_path_not_exist File.join @spec.gem_dir, "lib", "a.rb" assert_path_not_exist File.join @spec.gem_dir, "lib", "a", "b.rb" @@ -351,13 +355,88 @@ def test_build_multiple_extensions assert_path_exist @spec.extension_dir assert_path_exist @spec.gem_build_complete_path assert_path_exist File.join @spec.gem_dir, "ext", "foo" - assert_path_exist File.join @spec.extension_dir, "gem_make.out" + assert_path_not_exist File.join @spec.extension_dir, "gem_make.out" + assert_path_not_exist File.join @spec.extension_dir, "mkmf.log" + assert_path_not_exist File.join @spec.gem_dir, "ext", "mkmf.log" assert_path_exist File.join @spec.extension_dir, "a.rb" assert_path_exist File.join @spec.gem_dir, "lib", "a.rb" assert_path_exist File.join @spec.gem_dir, "lib", "a", "b.rb" end end + def test_build_extensions_does_not_install_logs_on_success + pend "terminates on mswin" if vc_windows? && ruby_repo? + + @spec.extensions << "ext/extconf.rb" + + ext_dir = File.join @spec.gem_dir, "ext" + FileUtils.mkdir_p ext_dir + + File.open File.join(ext_dir, "extconf.rb"), "w" do |f| + f.write <<-'RUBY' + require 'mkmf' + + create_makefile 'a' + RUBY + end + + use_ui @ui do + @builder.build_extensions + end + + assert_path_exist @spec.gem_build_complete_path + + # No build logs are left anywhere in the installation tree. + assert_path_not_exist File.join @spec.extension_dir, "gem_make.out" + assert_path_not_exist File.join @spec.extension_dir, "mkmf.log" + assert_path_not_exist File.join ext_dir, "mkmf.log" + assert_path_not_exist File.join ext_dir, "gem_make.out" + assert_path_not_exist File.join @spec.build_info_dir, "#{@spec.full_name}.mkmf.log" + assert_path_not_exist File.join @spec.build_info_dir, "#{@spec.full_name}.gem_make.out" + end + + def test_build_extensions_logs_to_build_info_on_failure + pend "terminates on mswin" if vc_windows? && ruby_repo? + + @spec.extensions << "ext/extconf.rb" + + ext_dir = File.join @spec.gem_dir, "ext" + FileUtils.mkdir_p ext_dir + + File.open File.join(ext_dir, "extconf.rb"), "w" do |f| + f.write <<-'RUBY' + require 'mkmf' + + have_library 'nonexistent' or abort 'need libnonexistent' + + create_makefile 'a' + RUBY + end + + e = assert_raise Gem::Ext::BuildError do + use_ui @ui do + @builder.build_extensions + end + end + + mkmf_log = File.join @spec.build_info_dir, "#{@spec.full_name}.mkmf.log" + gem_make_out = File.join @spec.build_info_dir, "#{@spec.full_name}.gem_make.out" + + assert_path_exist mkmf_log + assert_path_exist gem_make_out + + # Logs are not left in the installation tree. + assert_path_not_exist File.join @spec.extension_dir, "mkmf.log" + assert_path_not_exist File.join @spec.extension_dir, "gem_make.out" + assert_path_not_exist File.join ext_dir, "mkmf.log" + + # The error message points at the new build_info paths. + assert_includes e.message, gem_make_out + assert_includes e.message, mkmf_log + + assert_path_not_exist @spec.gem_build_complete_path + end + def test_build_extensions_none use_ui @ui do @builder.build_extensions @@ -401,12 +480,14 @@ def test_build_extensions_extconf_bad assert_equal "Building native extensions. This could take a while...\n", @ui.output assert_equal "", @ui.error - gem_make_out = File.join @spec.extension_dir, "gem_make.out" + gem_make_out = File.join @spec.build_info_dir, "#{@spec.full_name}.gem_make.out" cmd_make_out = File.read(gem_make_out) assert_match %r{#{Regexp.escape Gem.ruby} .* extconf\.rb}, cmd_make_out assert_match(/: No such file/, cmd_make_out) + assert_path_not_exist File.join @spec.extension_dir, "gem_make.out" + assert_path_not_exist @spec.gem_build_complete_path assert_equal cwd, Dir.pwd @@ -414,7 +495,7 @@ def test_build_extensions_extconf_bad def test_build_extensions_unsupported FileUtils.mkdir_p @spec.gem_dir - gem_make_out = File.join @spec.extension_dir, "gem_make.out" + gem_make_out = File.join @spec.build_info_dir, "#{@spec.full_name}.gem_make.out" @spec.extensions << nil e = assert_raise Gem::Ext::BuildError do diff --git a/test/rubygems/test_gem_ext_ext_conf_builder.rb b/test/rubygems/test_gem_ext_ext_conf_builder.rb index bc383e5540a9e3..e14013899559b6 100644 --- a/test/rubygems/test_gem_ext_ext_conf_builder.rb +++ b/test/rubygems/test_gem_ext_ext_conf_builder.rb @@ -110,10 +110,12 @@ def test_class_build_extconf_fail assert_equal "extconf failed, exit code 1", error.message assert_match(/^#{Regexp.quote(Gem.ruby)}.* extconf.rb/, output[1]) - assert_match(File.join(@dest_path, "mkmf.log"), output[4]) - assert_includes(output, "To see why this extension failed to compile, please check the mkmf.log which can be found here:\n") + refute_includes(output, "To see why this extension failed to compile, please check the mkmf.log which can be found here:\n") - assert_path_exist File.join @dest_path, "mkmf.log" + # mkmf.log is left in the extension directory; deciding where it ends up is + # left to Gem::Ext::Builder#build_extension. + assert_path_exist File.join @ext, "mkmf.log" + assert_path_not_exist File.join @dest_path, "mkmf.log" end def test_class_build_extconf_success_without_warning @@ -133,7 +135,10 @@ def test_class_build_extconf_success_without_warning refute_includes(output, "To see why this extension failed to compile, please check the mkmf.log which can be found here:\n") - assert_path_exist File.join @dest_path, "mkmf.log" + # On a successful build, mkmf.log is cleaned up by "make clean" and is never + # copied into the install destination. + assert_path_not_exist File.join @ext, "mkmf.log" + assert_path_not_exist File.join @dest_path, "mkmf.log" end def test_class_build_unconventional diff --git a/test/rubygems/test_gem_specification.rb b/test/rubygems/test_gem_specification.rb index 025fec7e66ebfd..dc32a62907863c 100644 --- a/test/rubygems/test_gem_specification.rb +++ b/test/rubygems/test_gem_specification.rb @@ -1562,8 +1562,9 @@ def test_build_extensions_preview @ext.build_extensions + # A successful build no longer leaves gem_make.out in the install tree. gem_make_out = File.join @ext.extension_dir, "gem_make.out" - assert_path_exist gem_make_out + assert_path_not_exist gem_make_out end def test_contains_requirable_file_eh From 78efe7eb1667a1a6573d7a1a00fc2d74980c882d Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 30 Jun 2026 16:16:39 +0900 Subject: [PATCH 05/25] [ruby/rubygems] Remove build_info logs on uninstall Clean up the per-gem mkmf.log and gem_make.out left in build_info by a failed extension build when the gem is uninstalled. https://github.com/ruby/rubygems/commit/ec599546fc Co-Authored-By: Claude Opus 4.8 --- lib/rubygems/uninstaller.rb | 2 ++ test/rubygems/test_gem_uninstaller.rb | 33 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/lib/rubygems/uninstaller.rb b/lib/rubygems/uninstaller.rb index 28bf33ea710605..f87ee2f4984167 100644 --- a/lib/rubygems/uninstaller.rb +++ b/lib/rubygems/uninstaller.rb @@ -272,6 +272,8 @@ def remove(spec) safe_delete { rm_r full_gem_path, exclusions: exclusions } safe_delete { FileUtils.rm_r spec.extension_dir } + safe_delete { FileUtils.rm_f File.join(spec.build_info_dir, "#{spec.full_name}.mkmf.log") } + safe_delete { FileUtils.rm_f File.join(spec.build_info_dir, "#{spec.full_name}.gem_make.out") } old_platform_name = spec.original_name diff --git a/test/rubygems/test_gem_uninstaller.rb b/test/rubygems/test_gem_uninstaller.rb index 137bb1de97a9c5..ea2ac081012b1d 100644 --- a/test/rubygems/test_gem_uninstaller.rb +++ b/test/rubygems/test_gem_uninstaller.rb @@ -419,6 +419,39 @@ def test_uninstall_extension assert_path_not_exist @spec.extension_dir end + def test_uninstall_removes_build_info_logs + @spec.extensions << "extconf.rb" + write_file File.join(@tempdir, "extconf.rb") do |io| + io.write <<-RUBY +require 'mkmf' +create_makefile '#{@spec.name}' + RUBY + end + + @spec.files += %w[extconf.rb] + + use_ui @ui do + path = Gem::Package.build @spec + + installer = Gem::Installer.at path, force: true + installer.install + end + + # Build logs left behind in build_info by a previous failed build. + FileUtils.mkdir_p @spec.build_info_dir + mkmf_log = File.join @spec.build_info_dir, "#{@spec.full_name}.mkmf.log" + gem_make_out = File.join @spec.build_info_dir, "#{@spec.full_name}.gem_make.out" + FileUtils.touch mkmf_log + FileUtils.touch gem_make_out + + uninstaller = Gem::Uninstaller.new @spec.name, executables: true + uninstaller.uninstall + + assert_path_not_exist @spec.extension_dir + assert_path_not_exist mkmf_log + assert_path_not_exist gem_make_out + end + def test_uninstall_nonexistent uninstaller = Gem::Uninstaller.new "bogus", executables: true From b42143dbc498fa3f566c232147f374284999c9cd Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Fri, 17 Jul 2026 14:23:47 +0900 Subject: [PATCH 06/25] [ruby/rubygems] Adapt bundler jobserver specs to build logs no longer installed These specs read the `make -jN` command line from `gem_make.out`, which a successful build no longer writes. The integration specs in `install_spec` now force the build to fail so the command lands in `build_info`, and the `parallel_installer` specs assert on the number of jobserver slots each gem's build acquired, which is exactly what becomes `make -jN`, instead of reading a build log. https://github.com/ruby/rubygems/commit/5d703328c3 Co-Authored-By: Claude Opus 4.8 --- .../installer/parallel_installer_spec.rb | 72 +++++++++++-------- spec/bundler/commands/install_spec.rb | 20 +++--- 2 files changed, 55 insertions(+), 37 deletions(-) diff --git a/spec/bundler/bundler/installer/parallel_installer_spec.rb b/spec/bundler/bundler/installer/parallel_installer_spec.rb index 5b20650dc3a1a3..8112660af35a4d 100644 --- a/spec/bundler/bundler/installer/parallel_installer_spec.rb +++ b/spec/bundler/bundler/installer/parallel_installer_spec.rb @@ -186,34 +186,37 @@ let(:gem_two) { definition.specs.find {|spec| spec.name == "two" } } it "takes all available slots" do - redefine_build_jobs do + acquired = track_build_jobs(rendezvous: true) do Bundler::ParallelInstaller.call(installer, definition.specs, 5, false, true) end - # Take 3 slots out of the 5 available. - expect(File.read(File.join(gem_one.extension_dir, "gem_make.out"))).to include("make -j3") + # Take 3 slots (capped per gem) out of the 5 available. + expect(acquired["one"]).to eq(3) # Take the remaining 2 slots. - expect(File.read(File.join(gem_two.extension_dir, "gem_make.out"))).to include("make -j2") + expect(acquired["two"]).to eq(2) end it "fallback to non parallel when no slots are available" do - redefine_build_jobs do + acquired = track_build_jobs(rendezvous: true) do Bundler::ParallelInstaller.call(installer, definition.specs, 3, false, true) end # Take 3 slots out of the 3 available. - expect(File.read(File.join(gem_one.extension_dir, "gem_make.out"))).to include("make -j3") + expect(acquired["one"]).to eq(3) # Fallback to one slot (non parallel). - expect(File.read(File.join(gem_two.extension_dir, "gem_make.out"))).to_not include("make -j") + expect(acquired["two"]).to eq(1) end it "uses one jobs when installing serially" do + acquired = nil Bundler.settings.temporary(jobs: 1) do - Bundler::ParallelInstaller.call(installer, definition.specs, 1, false, true) + acquired = track_build_jobs do + Bundler::ParallelInstaller.call(installer, definition.specs, 1, false, true) + end end - expect(File.read(File.join(gem_one.extension_dir, "gem_make.out"))).to_not include("make -j") - expect(File.read(File.join(gem_two.extension_dir, "gem_make.out"))).to_not include("make -j") + expect(acquired["one"]).to eq(1) + expect(acquired["two"]).to eq(1) end it "release the job slots" do @@ -225,39 +228,52 @@ end end - Bundler::ParallelInstaller.call(installer, definition.specs, 3, false, true) + acquired = track_build_jobs do + Bundler::ParallelInstaller.call(installer, definition.specs, 3, false, true) + end # Take 3 slots out of the 3 available. - expect(File.read(File.join(gem_one.extension_dir, "gem_make.out"))).to include("make -j3") - # Take 3 slots that were released. - expect(File.read(File.join(gem_two.extension_dir, "gem_make.out"))).to include("make -j3") + expect(acquired["one"]).to eq(3) + # Take 3 slots that were released by `one`. + expect(acquired["two"]).to eq(3) end - def redefine_build_jobs + # Records how many jobserver slots each gem's build acquired. RubyGems turns + # that count directly into `make -jN`, so asserting on it verifies slot + # allocation and release without reading a build log, which a successful + # build no longer writes. With +rendezvous+, "one" grabs its slots first and + # holds them until "two" has grabbed the rest, making the split deterministic. + def track_build_jobs(rendezvous: false) + acquired = {} old_method = Bundler::RubyGemsGemInstaller.instance_method(:build_jobs) Bundler::RubyGemsGemInstaller.remove_method(:build_jobs) - # Rendezvous so that "one" grabs its slots first and keeps holding them - # until "two" has grabbed the rest. Blocking on a queue avoids the - # busy-wait and makes the ordering deterministic. one_acquired = Thread::Queue.new two_acquired = Thread::Queue.new Bundler::RubyGemsGemInstaller.define_method(:build_jobs) do - if spec.name == "one" - value = old_method.bind(self).call - one_acquired << true - two_acquired.pop - elsif spec.name == "two" - one_acquired.pop - value = old_method.bind(self).call - two_acquired << true - end - + value = + if rendezvous && spec.name == "one" + v = old_method.bind(self).call + one_acquired << true + two_acquired.pop + v + elsif rendezvous && spec.name == "two" + one_acquired.pop + v = old_method.bind(self).call + two_acquired << true + v + else + old_method.bind(self).call + end + + acquired[spec.name] = value value end yield + + acquired ensure Bundler::RubyGemsGemInstaller.remove_method(:build_jobs) Bundler::RubyGemsGemInstaller.define_method(:build_jobs, old_method) diff --git a/spec/bundler/commands/install_spec.rb b/spec/bundler/commands/install_spec.rb index a0b56a3d2970a4..bf360a65b9c29a 100644 --- a/spec/bundler/commands/install_spec.rb +++ b/spec/bundler/commands/install_spec.rb @@ -1369,10 +1369,18 @@ def run s.extensions = extension s.write(extension, extconf_code) + # A successful build no longer leaves gem_make.out behind. Force the + # build to fail at the make stage so the make command line, including + # the jobserver `-j`, is recorded in build_info for these assertions. + s.write("ext/mypsych/mypsych.c", "#error forced build failure for test") end end end + def gem_make_out + File.read(File.join(@gemspec.build_info_dir, "#{@gemspec.full_name}.gem_make.out")) + end + after do if @old_makeflags ENV["MAKEFLAGS"] = @old_makeflags @@ -1384,39 +1392,33 @@ def run it "doesn't pass down -j to make when MAKEFLAGS is set" do ENV["MAKEFLAGS"] = "-j1" - install_gemfile(<<~G, env: { "BUNDLE_JOBS" => "8" }) + install_gemfile(<<~G, env: { "BUNDLE_JOBS" => "8" }, raise_on_error: false) source "https://gem.repo4" gem "mypsych" G - gem_make_out = File.read(File.join(@gemspec.extension_dir, "gem_make.out")) - expect(gem_make_out).not_to include("make -j8") end it "uses 3 slots from the available pool when running the compilation of an extension", rubygems: ">= 4.1.0.dev" do ENV.delete("MAKEFLAGS") - install_gemfile(<<~G, env: { "BUNDLE_JOBS" => "8" }) + install_gemfile(<<~G, env: { "BUNDLE_JOBS" => "8" }, raise_on_error: false) source "https://gem.repo4" gem "mypsych" G - gem_make_out = File.read(File.join(@gemspec.extension_dir, "gem_make.out")) - expect(gem_make_out).to include("make -j3") end it "consumes 3 slots from the pool when BUNDLE_JOBS isn't set", rubygems: ">= 4.1.0.dev" do ENV.delete("MAKEFLAGS") - install_gemfile(<<~G) + install_gemfile(<<~G, raise_on_error: false) source "https://gem.repo4" gem "mypsych" G - gem_make_out = File.read(File.join(@gemspec.extension_dir, "gem_make.out")) - expect(gem_make_out).to include("make -j3") end end From 09c66aff748fa71602fc69d4122b63498bb62a76 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 5 Aug 2026 18:29:24 +0900 Subject: [PATCH 07/25] [ruby/rubygems] Gate the MAKEFLAGS jobserver spec on RubyGems 4.1 Older system RubyGems writes the build log to the extension directory, so under RGV=system the example read the new build_info path and hit ENOENT. The `-j` suppression it verifies only exists with the jobserver support in RubyGems 4.1, so gate it like its sibling examples. https://github.com/ruby/rubygems/commit/16c69087b6 Co-Authored-By: Claude Opus 4.8 --- spec/bundler/commands/install_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/bundler/commands/install_spec.rb b/spec/bundler/commands/install_spec.rb index bf360a65b9c29a..d4b88902e21e28 100644 --- a/spec/bundler/commands/install_spec.rb +++ b/spec/bundler/commands/install_spec.rb @@ -1389,7 +1389,7 @@ def gem_make_out end end - it "doesn't pass down -j to make when MAKEFLAGS is set" do + it "doesn't pass down -j to make when MAKEFLAGS is set", rubygems: ">= 4.1.0.dev" do ENV["MAKEFLAGS"] = "-j1" install_gemfile(<<~G, env: { "BUNDLE_JOBS" => "8" }, raise_on_error: false) From 7bb4124d1634b333eaece131e73cdae9a929f9f6 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 2 Sep 2026 11:23:10 +0900 Subject: [PATCH 08/25] [ruby/rubygems] Restore the build log lifecycle around a failed build Two behaviours regressed against the extension directory the logs used to live in. "clean" is the first make target and mkmf lists mkmf.log in CLEANFILES, so the log was already gone by the time a compile failure reached the handler that moves it to build_info. And nothing cleared a failed build's logs afterwards, where previously the installer wiped the extension directory on every install, so a later successful install kept reporting an old failure. Park mkmf.log next to the built extension right after extconf, the way ExtConfBuilder did before, and let build_extension decide from there whether to drop it or keep it. Drop both logs again on success, along with any gem_make.out an older RubyGems left in the extension directory. https://github.com/ruby/rubygems/commit/47614894cf Co-Authored-By: Claude Opus 4.8 --- lib/rubygems/ext/builder.rb | 34 +++++++-- lib/rubygems/ext/ext_conf_builder.rb | 10 ++- test/rubygems/test_gem_ext_builder.rb | 74 +++++++++++++++++++ .../rubygems/test_gem_ext_ext_conf_builder.rb | 6 +- 4 files changed, 110 insertions(+), 14 deletions(-) diff --git a/lib/rubygems/ext/builder.rb b/lib/rubygems/ext/builder.rb index 307b81bcd244c8..e5c03d7eed9ff2 100644 --- a/lib/rubygems/ext/builder.rb +++ b/lib/rubygems/ext/builder.rb @@ -243,18 +243,20 @@ def build_extension(extension, dest_path) # :nodoc: verbose { results.join("\n") } - # mkmf.log is a noisy, non-reproducible build artifact that is not meant - # to be installed. Drop the one left behind by a successful build instead - # of leaving it in the installation tree. - FileUtils.rm_f File.join(extension_dir, "mkmf.log") + # Build logs are noisy, non-reproducible artifacts that are not meant to + # be installed. Drop the ones this build left behind, plus any written + # into the extension directory by a RubyGems old enough to put them there. + FileUtils.rm_f mkmf_log_candidates(extension_dir, dest_path) + FileUtils.rm_f File.join(dest_path, "gem_make.out") + FileUtils.rm_f [build_log_path("mkmf.log"), build_log_path("gem_make.out")] rescue StandardError => e results << e.message # On failure keep the mkmf.log for inspection, but move it out of the # installation tree into the build_info directory. - mkmf_log = File.join extension_dir, "mkmf.log" - if File.exist?(mkmf_log) - mkmf_log_dest = File.join @spec.build_info_dir, "#{@spec.full_name}.mkmf.log" + mkmf_log = mkmf_log_candidates(extension_dir, dest_path).find {|log| File.exist?(log) } + if mkmf_log + mkmf_log_dest = build_log_path "mkmf.log" FileUtils.mkdir_p @spec.build_info_dir FileUtils.mv mkmf_log, mkmf_log_dest @@ -266,6 +268,22 @@ def build_extension(extension, dest_path) # :nodoc: end end + ## + # Where a build log of +kind+ for this gem lives in the build_info directory. + + def build_log_path(kind) # :nodoc: + File.join @spec.build_info_dir, "#{@spec.full_name}.#{kind}" + end + + ## + # Places a completed build may have left an mkmf.log, most specific first. + # Gem::Ext::ExtConfBuilder parks it in +dest_path+ so that the "clean" target + # cannot delete it; the other builders leave it where extconf ran. + + def mkmf_log_candidates(extension_dir, dest_path) # :nodoc: + [File.join(dest_path, "mkmf.log"), File.join(extension_dir, "mkmf.log")] + end + ## # Builds extensions. Valid types of extensions are extconf.rb files, # configure scripts and rakefiles or mkrf_conf files. @@ -298,7 +316,7 @@ def build_extensions # tree. def write_gem_make_out(output) # :nodoc: - destination = File.join @spec.build_info_dir, "#{@spec.full_name}.gem_make.out" + destination = build_log_path "gem_make.out" FileUtils.mkdir_p @spec.build_info_dir diff --git a/lib/rubygems/ext/ext_conf_builder.rb b/lib/rubygems/ext/ext_conf_builder.rb index a658e0b0eb5963..2234e7c6d0bea3 100644 --- a/lib/rubygems/ext/ext_conf_builder.rb +++ b/lib/rubygems/ext/ext_conf_builder.rb @@ -27,11 +27,15 @@ def self.build(extension, dest_path, results, args = [], lib_dir = nil, extensio cmd << "--target-rbconfig=#{target_rbconfig.path}" if target_rbconfig.path cmd.push(*args) - # Leave mkmf.log in the extension directory. The final placement (dropped - # on success, moved to build_info on failure) is decided by - # Gem::Ext::Builder#build_extension. run(cmd, results, class_name, extension_dir) + # "clean" is the first make target, and mkmf puts mkmf.log in CLEANFILES, + # so park the log next to the built extension before make can delete it. + # Whether it is then dropped or kept for inspection is decided by + # Gem::Ext::Builder#build_extension. + mkmf_log = File.join(extension_dir, "mkmf.log") + FileUtils.mv mkmf_log, dest_path if File.exist?(mkmf_log) + ENV["DESTDIR"] = nil make dest_path, results, extension_dir, tmp_dest_relative, target_rbconfig: target_rbconfig, n_jobs: n_jobs diff --git a/test/rubygems/test_gem_ext_builder.rb b/test/rubygems/test_gem_ext_builder.rb index 80b2db2d9a2ecf..4e2535ba7b972f 100644 --- a/test/rubygems/test_gem_ext_builder.rb +++ b/test/rubygems/test_gem_ext_builder.rb @@ -395,6 +395,80 @@ def test_build_extensions_does_not_install_logs_on_success assert_path_not_exist File.join @spec.build_info_dir, "#{@spec.full_name}.gem_make.out" end + def test_build_extensions_logs_to_build_info_on_make_failure + pend "terminates on mswin" if vc_windows? && ruby_repo? + + @spec.extensions << "ext/extconf.rb" + + ext_dir = File.join @spec.gem_dir, "ext" + FileUtils.mkdir_p ext_dir + + # have_header makes mkmf actually write an mkmf.log. extconf then succeeds, + # so "make clean" runs before the build and would delete that log unless it + # has been parked out of the way first. + File.open File.join(ext_dir, "extconf.rb"), "w" do |f| + f.write <<-'RUBY' + require 'mkmf' + + have_header 'stdio.h' + + File.write 'a.c', "#error forced build failure for test\n" + + create_makefile 'a' + RUBY + end + + e = assert_raise Gem::Ext::BuildError do + use_ui @ui do + @builder.build_extensions + end + end + + mkmf_log = File.join @spec.build_info_dir, "#{@spec.full_name}.mkmf.log" + + assert_path_exist mkmf_log + assert_includes e.message, mkmf_log + + assert_path_not_exist File.join @spec.extension_dir, "mkmf.log" + assert_path_not_exist File.join ext_dir, "mkmf.log" + end + + def test_build_extensions_removes_stale_build_info_logs_on_success + pend "terminates on mswin" if vc_windows? && ruby_repo? + + @spec.extensions << "ext/extconf.rb" + + ext_dir = File.join @spec.gem_dir, "ext" + FileUtils.mkdir_p ext_dir + + File.open File.join(ext_dir, "extconf.rb"), "w" do |f| + f.write <<-'RUBY' + require 'mkmf' + + create_makefile 'a' + RUBY + end + + # Logs left in build_info by an earlier failed build, and in the extension + # directory by a RubyGems old enough to install them there. + FileUtils.mkdir_p @spec.build_info_dir + FileUtils.mkdir_p @spec.extension_dir + stale = [ + File.join(@spec.build_info_dir, "#{@spec.full_name}.mkmf.log"), + File.join(@spec.build_info_dir, "#{@spec.full_name}.gem_make.out"), + File.join(@spec.extension_dir, "gem_make.out"), + ] + FileUtils.touch stale + + use_ui @ui do + @builder.build_extensions + end + + assert_path_exist @spec.gem_build_complete_path + + stale.each {|path| assert_path_not_exist path } + end + def test_build_extensions_logs_to_build_info_on_failure pend "terminates on mswin" if vc_windows? && ruby_repo? diff --git a/test/rubygems/test_gem_ext_ext_conf_builder.rb b/test/rubygems/test_gem_ext_ext_conf_builder.rb index e14013899559b6..0a23a34ea5a4c0 100644 --- a/test/rubygems/test_gem_ext_ext_conf_builder.rb +++ b/test/rubygems/test_gem_ext_ext_conf_builder.rb @@ -135,10 +135,10 @@ def test_class_build_extconf_success_without_warning refute_includes(output, "To see why this extension failed to compile, please check the mkmf.log which can be found here:\n") - # On a successful build, mkmf.log is cleaned up by "make clean" and is never - # copied into the install destination. + # mkmf.log is parked in dest_path so that "make clean" cannot delete it. + # Dropping it is Gem::Ext::Builder#build_extension's job, not this one's. assert_path_not_exist File.join @ext, "mkmf.log" - assert_path_not_exist File.join @dest_path, "mkmf.log" + assert_path_exist File.join @dest_path, "mkmf.log" end def test_class_build_unconventional From 8e781cd60976ce9265bdc1b07805e0079d4b288e Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 2 Sep 2026 11:23:10 +0900 Subject: [PATCH 09/25] [ruby/rubygems] Stop gem doctor from deleting build logs build_info entries are matched by stripping ".info", so the new .mkmf.log and .gem_make.out never matched an installed gem and were removed as strays, including the log the build error had just told the user to read. Let a subdirectory declare several suffixes and give build_info all three. https://github.com/ruby/rubygems/commit/d1b3845795 Co-Authored-By: Claude Opus 4.8 --- lib/rubygems/doctor.rb | 23 +++++++++++++++++------ test/rubygems/test_gem_doctor.rb | 27 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/lib/rubygems/doctor.rb b/lib/rubygems/doctor.rb index 114946209b4bcb..a443c9606d0762 100644 --- a/lib/rubygems/doctor.rb +++ b/lib/rubygems/doctor.rb @@ -21,7 +21,7 @@ class Gem::Doctor REPOSITORY_EXTENSION_MAP = [ # :nodoc: ["specifications", ".gemspec"], - ["build_info", ".info"], + ["build_info", ".info", ".mkmf.log", ".gem_make.out"], ["cache", ".gem"], ["doc", ""], ["extensions", ""], @@ -92,15 +92,15 @@ def doctor # Cleans up children of this gem repository def doctor_children # :nodoc: - REPOSITORY_EXTENSION_MAP.each do |sub_directory, extension| - doctor_child sub_directory, extension + REPOSITORY_EXTENSION_MAP.each do |sub_directory, *extensions| + doctor_child sub_directory, *extensions end end ## - # Removes files in +sub_directory+ with +extension+ + # Removes files in +sub_directory+ with any of +extensions+ - def doctor_child(sub_directory, extension) # :nodoc: + def doctor_child(sub_directory, *extensions) # :nodoc: directory = File.join(@gem_repository, sub_directory) Dir.entries(directory).sort.each do |ent| @@ -109,7 +109,7 @@ def doctor_child(sub_directory, extension) # :nodoc: child = File.join(directory, ent) next unless File.exist?(child) - basename = File.basename(child, extension) + basename = strip_extension File.basename(child), extensions next if installed_specs.include? basename next if /^rubygems-\d/.match?(basename) next if sub_directory == "specifications" && basename == "default" @@ -135,4 +135,15 @@ def doctor_child(sub_directory, extension) # :nodoc: rescue Errno::ENOENT # ignore end + + ## + # Removes the first of +extensions+ that +name+ ends with + + def strip_extension(name, extensions) # :nodoc: + extension = extensions.find do |ext| + !ext.empty? && name.end_with?(ext) + end + + extension ? name.delete_suffix(extension) : name + end end diff --git a/test/rubygems/test_gem_doctor.rb b/test/rubygems/test_gem_doctor.rb index 9fd6f33641e21c..fe2f5c8cadcf87 100644 --- a/test/rubygems/test_gem_doctor.rb +++ b/test/rubygems/test_gem_doctor.rb @@ -121,6 +121,33 @@ def test_doctor_dry_run assert_equal Gem.path, [@gemhome, @userhome] end + def test_doctor_keeps_build_logs_of_installed_gems + a = gem "a" + + Gem.use_paths @userhome, @gemhome + + build_info_dir = File.join @gemhome, "build_info" + FileUtils.mkdir_p build_info_dir + + kept = ["#{a.full_name}.mkmf.log", "#{a.full_name}.gem_make.out"].map do |name| + File.join build_info_dir, name + end + stale = File.join build_info_dir, "b-2.gem_make.out" + + FileUtils.touch kept + [stale] + + doctor = Gem::Doctor.new @gemhome + + capture_output do + use_ui @ui do + doctor.doctor + end + end + + kept.each {|path| assert_path_exist path } + assert_path_not_exist stale + end + def test_doctor_non_gem_home other_dir = File.join @tempdir, "other", "dir" From 0913bcc1a3d344a10a32eb6ef071a9ea58cd5b45 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 2 Sep 2026 11:23:10 +0900 Subject: [PATCH 10/25] [ruby/rubygems] Keep bundle clean away from build_info Logs for a git source land in bundler/gems/build_info, which `bundle clean` globs as a git checkout and reports as "Removing (build_info)" before deleting it. Exclude it the way the sibling extensions directory already is. https://github.com/ruby/rubygems/commit/01dfc59016 Co-Authored-By: Claude Opus 4.8 --- lib/bundler/runtime.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bundler/runtime.rb b/lib/bundler/runtime.rb index b76e2b3e98303f..caefc61fdefc00 100644 --- a/lib/bundler/runtime.rb +++ b/lib/bundler/runtime.rb @@ -203,7 +203,7 @@ def clean(dry_run = false) spec_gem_executables.flatten! stale_gem_bins = gem_bins - spec_gem_executables - stale_git_dirs = git_dirs - spec_git_paths - ["#{Gem.dir}/bundler/gems/extensions"] + stale_git_dirs = git_dirs - spec_git_paths - ["#{Gem.dir}/bundler/gems/extensions", "#{Gem.dir}/bundler/gems/build_info"] stale_git_cache_dirs = git_cache_dirs - spec_git_cache_dirs stale_gem_dirs = gem_dirs - spec_gem_paths stale_gem_files = gem_files - spec_cache_paths From cb11e45786788eef2d61fe3b977f1ff15d67ac43 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 2 Sep 2026 11:37:32 +0900 Subject: [PATCH 11/25] [ruby/rubygems] Keep the build error when the log cannot be written Preserving a log happens while a build failure is being reported, so a filesystem error there replaced the compile output the user needed with a bare Errno, and the extension builder stopped raising Gem::Ext::BuildError at all. Callers only rescue the Gem::InstallError family, so the failure escaped as an unhandled exception. Swallow errors from moving mkmf.log and from writing gem_make.out, and drop the "Results logged to" line when there is no log to point at. https://github.com/ruby/rubygems/commit/c5f4887ac7 Co-Authored-By: Claude Opus 4.8 --- lib/rubygems/ext/builder.rb | 53 +++++++++++++++++++++------ test/rubygems/test_gem_ext_builder.rb | 51 ++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 12 deletions(-) diff --git a/lib/rubygems/ext/builder.rb b/lib/rubygems/ext/builder.rb index e5c03d7eed9ff2..b7e80d6b06bb89 100644 --- a/lib/rubygems/ext/builder.rb +++ b/lib/rubygems/ext/builder.rb @@ -220,9 +220,11 @@ def build_error(output, backtrace = nil) # :nodoc: #{output} Gem files will remain installed in #{@gem_dir} for inspection. -Results logged to #{gem_make_out} EOF + # Losing the log must not cost the user the build error itself. + message += "Results logged to #{gem_make_out}\n" if gem_make_out + raise Gem::Ext::BuildError, message, backtrace end @@ -249,17 +251,22 @@ def build_extension(extension, dest_path) # :nodoc: FileUtils.rm_f mkmf_log_candidates(extension_dir, dest_path) FileUtils.rm_f File.join(dest_path, "gem_make.out") FileUtils.rm_f [build_log_path("mkmf.log"), build_log_path("gem_make.out")] - rescue StandardError => e + rescue Gem::Ext::Builder::NoMakefileError => e + # extconf ran fine but produced no Makefile, so the extension was skipped + # rather than built and installing carries on. Keep the log that says why + # it was skipped, out of the installation tree but still reachable. results << e.message + results << "Skipping make for #{extension} as no Makefile was found." + + verbose { results.join("\n") } - # On failure keep the mkmf.log for inspection, but move it out of the - # installation tree into the build_info directory. - mkmf_log = mkmf_log_candidates(extension_dir, dest_path).find {|log| File.exist?(log) } - if mkmf_log - mkmf_log_dest = build_log_path "mkmf.log" - FileUtils.mkdir_p @spec.build_info_dir - FileUtils.mv mkmf_log, mkmf_log_dest + preserve_mkmf_log extension_dir, dest_path + write_gem_make_out results.join("\n") + rescue StandardError => e + results << e.message + mkmf_log_dest = preserve_mkmf_log(extension_dir, dest_path) + if mkmf_log_dest results << "To see why this extension failed to compile, please check the mkmf.log which can be found here:" results << " #{mkmf_log_dest}" end @@ -275,6 +282,26 @@ def build_log_path(kind) # :nodoc: File.join @spec.build_info_dir, "#{@spec.full_name}.#{kind}" end + ## + # Moves the mkmf.log this build left behind into the build_info directory and + # returns its new path, or nil when there is none or it cannot be kept. + # Keeping a log must never replace the build error the caller is reporting, + # so a filesystem failure here is swallowed. + + def preserve_mkmf_log(extension_dir, dest_path) # :nodoc: + mkmf_log = mkmf_log_candidates(extension_dir, dest_path).find {|log| File.exist?(log) } + return unless mkmf_log + + destination = build_log_path "mkmf.log" + + FileUtils.mkdir_p @spec.build_info_dir + FileUtils.mv mkmf_log, destination + + destination + rescue SystemCallError + nil + end + ## # Places a completed build may have left an mkmf.log, most specific first. # Gem::Ext::ExtConfBuilder parks it in +dest_path+ so that the "clean" target @@ -311,9 +338,9 @@ def build_extensions end ## - # Writes +output+ to gem_make.out in the build_info directory. Only called - # on failure (via #build_error), to keep build logs out of the installation - # tree. + # Writes +output+ to gem_make.out in the build_info directory and returns its + # path, or nil when it cannot be written. Only called when the extension was + # not built, to keep build logs out of the installation tree. def write_gem_make_out(output) # :nodoc: destination = build_log_path "gem_make.out" @@ -325,5 +352,7 @@ def write_gem_make_out(output) # :nodoc: end destination + rescue SystemCallError + nil end end diff --git a/test/rubygems/test_gem_ext_builder.rb b/test/rubygems/test_gem_ext_builder.rb index 4e2535ba7b972f..6b4eed2cf21175 100644 --- a/test/rubygems/test_gem_ext_builder.rb +++ b/test/rubygems/test_gem_ext_builder.rb @@ -469,6 +469,57 @@ def test_build_extensions_removes_stale_build_info_logs_on_success stale.each {|path| assert_path_not_exist path } end + def test_build_extensions_keeps_logs_when_no_makefile_is_generated + pend "terminates on mswin" if vc_windows? && ruby_repo? + + @spec.extensions << "ext/extconf.rb" + + ext_dir = File.join @spec.gem_dir, "ext" + FileUtils.mkdir_p ext_dir + + # extconf exits cleanly but generates no Makefile, the way one that bails out + # early on an unsupported platform does. The extension is then skipped rather + # than built, and the log is the only record of why. + File.open File.join(ext_dir, "extconf.rb"), "w" do |f| + f.write "# nothing to build on this platform\n" + end + + use_ui @ui do + @builder.build_extensions + end + + gem_make_out = File.join @spec.build_info_dir, "#{@spec.full_name}.gem_make.out" + + assert_path_exist gem_make_out + assert_includes File.read(gem_make_out), "no Makefile was found" + + # Skipping still leaves nothing behind in the installation tree. + assert_path_not_exist File.join @spec.extension_dir, "mkmf.log" + assert_path_not_exist File.join @spec.extension_dir, "gem_make.out" + assert_path_not_exist File.join ext_dir, "mkmf.log" + end + + def test_build_extensions_reports_build_error_when_logs_cannot_be_written + @spec.extensions << "extconf.rb" + + FileUtils.mkdir_p @spec.gem_dir + + # A plain file where build_info belongs makes every log write fail. + FileUtils.rm_rf @spec.build_info_dir + File.write @spec.build_info_dir, "" + + e = assert_raise Gem::Ext::BuildError do + use_ui @ui do + @builder.build_extensions + end + end + + # The build failure survives instead of being replaced by the log failure. + assert_match(/\AERROR: Failed to build gem native extension.$/, e.message) + assert_match(/: No such file/, e.message) + refute_includes e.message, "Results logged to" + end + def test_build_extensions_logs_to_build_info_on_failure pend "terminates on mswin" if vc_windows? && ruby_repo? From 76b8e34c961e80c3a34a72eb797b1a2e4796fbe1 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 2 Sep 2026 11:37:32 +0900 Subject: [PATCH 12/25] [ruby/rubygems] Record why an extension was skipped An extconf that exits cleanly without generating a Makefile leaves the extension unbuilt while the install still reports success. That case used to be described in gem_make.out; since the success path stopped writing one it went unrecorded entirely. Let the no-Makefile case reach Gem::Ext::Builder, which already owns where logs end up, and have it write the explanation to build_info. https://github.com/ruby/rubygems/commit/e183327d52 Co-Authored-By: Claude Opus 4.8 --- lib/rubygems/ext/ext_conf_builder.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/rubygems/ext/ext_conf_builder.rb b/lib/rubygems/ext/ext_conf_builder.rb index 2234e7c6d0bea3..9cf7eafc5dc344 100644 --- a/lib/rubygems/ext/ext_conf_builder.rb +++ b/lib/rubygems/ext/ext_conf_builder.rb @@ -63,10 +63,6 @@ def self.build(extension, dest_path, results, args = [], lib_dir = nil, extensio end results - rescue Gem::Ext::Builder::NoMakefileError => error - results << error.message - results << "Skipping make for #{extension} as no Makefile was found." - # We are good, do not re-raise the error. ensure FileUtils.rm_rf tmp_dest if tmp_dest end From 7375b831053a501154a2a333fc20535e7425bf87 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 2 Sep 2026 12:11:05 +0900 Subject: [PATCH 13/25] [ruby/rubygems] Keep a git gem's build log with its checkout base_dir for a git source points at the directory holding every checkout, so logs keyed by full_name collided between revisions of the same gem and landed in bundler/gems/build_info, which nothing prunes and which `bundle clean` mistook for a stale checkout. Resolve them from extension_dir instead, which Bundler already makes unique per revision and which `bundle clean` removes along with the checkout. That also drops the clean exclusion added for the directory this no longer creates. Path sources are unaffected: Bundler installs them with extensions disabled, so they never produce a build log. https://github.com/ruby/rubygems/commit/71b4dbc584 Co-Authored-By: Claude Opus 4.8 --- lib/bundler/rubygems_ext.rb | 14 ++++++++++++++ lib/bundler/runtime.rb | 2 +- spec/bundler/install/gemfile/git_spec.rb | 22 ++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/lib/bundler/rubygems_ext.rb b/lib/bundler/rubygems_ext.rb index 730d1e522b4ce3..d3c4ce6d9dcf94 100644 --- a/lib/bundler/rubygems_ext.rb +++ b/lib/bundler/rubygems_ext.rb @@ -281,6 +281,20 @@ def extension_dir end end + alias_method :rg_build_info_dir, :build_info_dir + def build_info_dir + # A git checkout's build logs belong with that checkout's extension build. + # base_dir points at the directory holding every checkout, so logs keyed by + # full_name would collide between revisions of the same gem and would sit + # outside anything `bundle clean` prunes. extension_dir is unique per + # revision and goes away with the checkout. + if source.respond_to?(:extension_dir_name) + extension_dir + else + rg_build_info_dir + end + end + # Can be removed once RubyGems 3.5.21 support is dropped remove_method :gem_dir if method_defined?(:gem_dir, false) diff --git a/lib/bundler/runtime.rb b/lib/bundler/runtime.rb index caefc61fdefc00..b76e2b3e98303f 100644 --- a/lib/bundler/runtime.rb +++ b/lib/bundler/runtime.rb @@ -203,7 +203,7 @@ def clean(dry_run = false) spec_gem_executables.flatten! stale_gem_bins = gem_bins - spec_gem_executables - stale_git_dirs = git_dirs - spec_git_paths - ["#{Gem.dir}/bundler/gems/extensions", "#{Gem.dir}/bundler/gems/build_info"] + stale_git_dirs = git_dirs - spec_git_paths - ["#{Gem.dir}/bundler/gems/extensions"] stale_git_cache_dirs = git_cache_dirs - spec_git_cache_dirs stale_gem_dirs = gem_dirs - spec_gem_paths stale_gem_files = gem_files - spec_cache_paths diff --git a/spec/bundler/install/gemfile/git_spec.rb b/spec/bundler/install/gemfile/git_spec.rb index 2b74aa849ac8fc..c361f41b3ecdc4 100644 --- a/spec/bundler/install/gemfile/git_spec.rb +++ b/spec/bundler/install/gemfile/git_spec.rb @@ -536,4 +536,26 @@ end end end + + describe "a git gem whose extension fails to build" do + it "keeps the build log with that checkout instead of the shared repository" do + build_git "foo", "1.0", &:add_c_extension + File.write lib_path("foo-1.0/ext/foo.c"), "#error forced build failure for test\n" + + install_gemfile <<~G, raise_on_error: false + source "https://gem.repo1" + gem "foo", :git => "#{lib_path("foo-1.0")}" + G + + # The log lands in the checkout's own extension directory, which is unique + # per revision and which `bundle clean` prunes along with the checkout. + logs = Dir.glob("#{Gem.dir}/bundler/gems/extensions/*/*/*/*.gem_make.out") + + expect(logs.size).to eq(1) + expect(File.basename(File.dirname(logs.first))).to start_with("foo-1.0-") + + # Nothing is left directly under bundler/gems, which holds checkouts. + expect(Pathname.new("#{Gem.dir}/bundler/gems/build_info")).not_to exist + end + end end From 6b9b223a445f03e791e9aafdaf6b70abcafbbb64 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 2 Sep 2026 13:06:49 +0900 Subject: [PATCH 14/25] [ruby/rubygems] Fix the tests broken by the last two changes The git extension spec overwrote the C source after build_git had already committed the checkout, so bundle installed the original working source and the build succeeded, leaving no log to find. Write the broken source inside the build_git block, and assert the log is the one this failure produced. ExtConfBuilder no longer swallows NoMakefileError, so on JRuby, where the fixture extconf returns before creating a Makefile, calling the class method directly now raises instead of returning. https://github.com/ruby/rubygems/commit/d8616f348d Co-Authored-By: Claude Opus 4.8 --- spec/bundler/install/gemfile/git_spec.rb | 9 ++++++-- .../rubygems/test_gem_ext_ext_conf_builder.rb | 21 ++++++++++++------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/spec/bundler/install/gemfile/git_spec.rb b/spec/bundler/install/gemfile/git_spec.rb index c361f41b3ecdc4..908cd0af3c26f9 100644 --- a/spec/bundler/install/gemfile/git_spec.rb +++ b/spec/bundler/install/gemfile/git_spec.rb @@ -539,8 +539,12 @@ describe "a git gem whose extension fails to build" do it "keeps the build log with that checkout instead of the shared repository" do - build_git "foo", "1.0", &:add_c_extension - File.write lib_path("foo-1.0/ext/foo.c"), "#error forced build failure for test\n" + build_git "foo", "1.0" do |s| + s.add_c_extension + # Overwrite the source add_c_extension wrote, before the checkout is + # committed, so that building it fails. + s.write "ext/foo.c", "#error forced build failure for test\n" + end install_gemfile <<~G, raise_on_error: false source "https://gem.repo1" @@ -553,6 +557,7 @@ expect(logs.size).to eq(1) expect(File.basename(File.dirname(logs.first))).to start_with("foo-1.0-") + expect(File.read(logs.first)).to include("forced build failure for test") # Nothing is left directly under bundler/gems, which holds checkouts. expect(Pathname.new("#{Gem.dir}/bundler/gems/build_info")).not_to exist diff --git a/test/rubygems/test_gem_ext_ext_conf_builder.rb b/test/rubygems/test_gem_ext_ext_conf_builder.rb index 0a23a34ea5a4c0..b776883f45571d 100644 --- a/test/rubygems/test_gem_ext_ext_conf_builder.rb +++ b/test/rubygems/test_gem_ext_ext_conf_builder.rb @@ -26,16 +26,18 @@ def test_class_build output = [] - result = Gem::Ext::ExtConfBuilder.build "extconf.rb", @dest_path, output, [], nil, @ext - - assert_same result, output - - assert_match(/^current directory:/, output[0]) - assert_match(/^#{Regexp.quote(Gem.ruby)}.* extconf.rb/, output[1]) - if Gem.java_platform? - assert_includes(output, "Skipping make for extconf.rb as no Makefile was found.") + # extconf returns before creating a Makefile, so the extension is skipped. + # Deciding what that means is Gem::Ext::Builder#build_extension's job now, + # so the error reaches it instead of being swallowed here. + assert_raise Gem::Ext::Builder::NoMakefileError do + Gem::Ext::ExtConfBuilder.build "extconf.rb", @dest_path, output, [], nil, @ext + end else + result = Gem::Ext::ExtConfBuilder.build "extconf.rb", @dest_path, output, [], nil, @ext + + assert_same result, output + assert_equal "creating Makefile\n", output[2] assert_match(/^current directory:/, output[3]) assert_contains_make_command "clean", output[4] @@ -43,6 +45,9 @@ def test_class_build assert_contains_make_command "install", output[10] end + assert_match(/^current directory:/, output[0]) + assert_match(/^#{Regexp.quote(Gem.ruby)}.* extconf.rb/, output[1]) + assert_empty Dir.glob(File.join(@ext, "siteconf*.rb")) assert_empty Dir.glob(File.join(@ext, ".gem.*")) end From daf616ac718d0bfcaee7c3e4f092aa26e75f3e78 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 2 Sep 2026 16:34:50 +0900 Subject: [PATCH 15/25] [ruby/rubygems] Gate the git build log spec on RubyGems 4.1 Where a build log goes is decided by the RubyGems running the install, and under RGV=system that is an older one which writes a bare gem_make.out into the extension directory. Gate the example the way the other specs that assert on log locations already are. https://github.com/ruby/rubygems/commit/fca2ce60e8 Co-Authored-By: Claude Opus 4.8 --- spec/bundler/install/gemfile/git_spec.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/bundler/install/gemfile/git_spec.rb b/spec/bundler/install/gemfile/git_spec.rb index 908cd0af3c26f9..847e5fa35efc03 100644 --- a/spec/bundler/install/gemfile/git_spec.rb +++ b/spec/bundler/install/gemfile/git_spec.rb @@ -538,7 +538,9 @@ end describe "a git gem whose extension fails to build" do - it "keeps the build log with that checkout instead of the shared repository" do + # Where a build log goes is decided by the RubyGems running the install, and + # older ones write a bare gem_make.out into the extension directory. + it "keeps the build log with that checkout instead of the shared repository", rubygems: ">= 4.1.0.dev" do build_git "foo", "1.0" do |s| s.add_c_extension # Overwrite the source add_c_extension wrote, before the checkout is From 1fe4fcf1852fe9887f2903e800d026b3a952b5c0 Mon Sep 17 00:00:00 2001 From: niku <10890+niku@users.noreply.github.com> Date: Fri, 8 May 2026 09:10:02 +0900 Subject: [PATCH 16/25] Fix Box resolution crash with IFUNC frames [Bug #21977] When `RUBY_BOX=1` is set, combining `binding` with `Symbol#to_proc` causes a crash (`[BUG] BUG: Local ep without cme/box`). This occurs because escaping an `IFUNC` frame via `binding` adds the `VM_ENV_FLAG_LOCAL` flag. `VM_EP_RUBY_LEP` previously relied on `VM_EP_LEP`, which blindly stopped at this flag. This caused it to return the `IFUNC` itself instead of its caller, and since `IFUNC` lacks `cme/box` data, the Box resolution crashed. This commit fixes the issue by traversing `current_cfp` backwards to skip `IFUNC` frames directly, instead of relying on `VM_EP_LEP` and the local flag. --- test/ruby/test_box.rb | 21 +++++++++++++++++++++ vm.c | 14 +++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/test/ruby/test_box.rb b/test/ruby/test_box.rb index b3ecd0c546394d..15777234ee8c22 100644 --- a/test/ruby/test_box.rb +++ b/test/ruby/test_box.rb @@ -1195,6 +1195,27 @@ def foo_box = Ruby::Box.current end; end + def test_symbol_to_proc_with_escaped_binding + assert_separately([ENV_ENABLE_BOX], __FILE__, __LINE__, "#{<<~"begin;"}\n#{<<~'end;'}", ignore_stderr: true) + begin; + # Regression test for [BUG] Local ep without cme/box, flags: 66660087. + # binding() may escape TOP env and propagate LOCAL to IFUNC frames. + assert_nothing_raised do + using Module.new { + refine ::Binding do + def eval_methods + ::Kernel.instance_method(:methods).bind_call(receiver) + end + end + } + + result = binding.eval_methods.map(&:to_s) + assert_kind_of Array, result + assert result.all? { |x| x.is_a?(String) } + end + end; + end + def test_very_basic_method_calls_and_constants assert_separately([ENV_ENABLE_BOX], __FILE__, __LINE__, "#{<<~"begin;"}\n#{<<~'end;'}", ignore_stderr: true) begin; diff --git a/vm.c b/vm.c index 5454bff570e974..780df64bb9f2b6 100644 --- a/vm.c +++ b/vm.c @@ -119,7 +119,6 @@ VM_EP_RUBY_LEP(const rb_execution_context_t *ec, const rb_control_frame_t *curre const rb_control_frame_t *cfp = current_cfp; if (VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_IFUNC)) { - ep = VM_EP_LEP(current_cfp->ep); /** * Returns CFUNC frame only in this case. * @@ -147,7 +146,20 @@ VM_EP_RUBY_LEP(const rb_execution_context_t *ec, const rb_control_frame_t *curre * We expect that `chunk_i` works as expected by the implementation of `#chunk` * without any overwritten definitions from boxes. * So the definitions on IFUNC frames should be equal to the caller CFUNC. + * + * NOTE: We traverse the cfp chain directly instead of using VM_EP_LEP. + * When an IFUNC env is escaped to the heap (e.g., due to a surrounding + * `binding` call), the env may acquire VM_ENV_FLAG_LOCAL, causing VM_EP_LEP + * to return the IFUNC ep itself rather than the enclosing CFUNC ep. */ + while (VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_IFUNC)) { + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + VM_BOX_ASSERT(RUBY_VM_VALID_CONTROL_FRAME_P(cfp, eocfp), "Valid control frame expected for IFUNC caller"); + if (!RUBY_VM_VALID_CONTROL_FRAME_P(cfp, eocfp)) { + return NULL; + } + ep = cfp->ep; + } VM_ASSERT(VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_CFUNC)); return ep; } From 675e82b96cf9af6a853961c713ab592160359d82 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 31 Aug 2026 14:37:33 +0900 Subject: [PATCH 17/25] Fix Box resolution crash for ifunc procs called from Ruby frames An ifunc proc created by Method#to_proc and invoked directly via Proc#call has no enclosing CFUNC frame. The IFUNC skip loop in VM_EP_RUBY_LEP returned the caller's ep unconditionally, so a non-local EVAL ep reached current_box_on_cfp and crashed with "BUG: Local ep without cme/box". Return the ep early only when the enclosing frame is a CFUNC, and otherwise continue to the local ep walk. Co-Authored-By: Claude Fable 5 --- test/ruby/test_box.rb | 14 ++++++ vm.c | 99 +++++++++++++++++++++---------------------- 2 files changed, 62 insertions(+), 51 deletions(-) diff --git a/test/ruby/test_box.rb b/test/ruby/test_box.rb index 15777234ee8c22..38ea4a9fdd59c8 100644 --- a/test/ruby/test_box.rb +++ b/test/ruby/test_box.rb @@ -1216,6 +1216,20 @@ def eval_methods end; end + def test_method_to_proc_called_from_ruby_frame + assert_separately([ENV_ENABLE_BOX], __FILE__, __LINE__, "#{<<~"begin;"}\n#{<<~'end;'}", ignore_stderr: true) + begin; + # Regression test for [BUG] Local ep without cme/box, flags: 77770021. + # An ifunc proc invoked directly from a Ruby frame (e.g. Proc#call) + # has no enclosing CFUNC frame, so the caller Ruby frame determines + # the box. + assert_nothing_raised do + method(:require).to_proc.call("English") + end + assert_equal 1, $LOADED_FEATURES.grep(/English/).size + end; + end + def test_very_basic_method_calls_and_constants assert_separately([ENV_ENABLE_BOX], __FILE__, __LINE__, "#{<<~"begin;"}\n#{<<~'end;'}", ignore_stderr: true) begin; diff --git a/vm.c b/vm.c index 780df64bb9f2b6..59e6428c04d720 100644 --- a/vm.c +++ b/vm.c @@ -118,66 +118,63 @@ VM_EP_RUBY_LEP(const rb_execution_context_t *ec, const rb_control_frame_t *curre const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec); /* end of control frame pointer */ const rb_control_frame_t *cfp = current_cfp; - if (VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_IFUNC)) { - /** - * Returns CFUNC frame only in this case. - * - * Usually CFUNC frame doesn't represent the current box and it should operate - * the caller box. See the example: - * - * # in the main box - * module Kernel - * def foo = "foo" - * module_function :foo - * end - * - * In the case above, `module_function` is defined in the root box. - * If `module_function` worked in the root box, `Kernel#foo` is invisible - * from it and it causes NameError: undefined method `foo` for module `Kernel`. - * - * But in cases of IFUNC (blocks written in C), IFUNC doesn't have its own box - * and its local env frame will be CFUNC frame. - * For example, `Enumerator#chunk` calls IFUNC blocks, written as `chunk_i` function. - * - * [1].chunk{ it.even? }.each{ ... } - * - * Before calling the Ruby block `{ it.even? }`, `#chunk` calls `chunk_i` as IFUNC - * to iterate the array's members (it's just like `#each`). - * We expect that `chunk_i` works as expected by the implementation of `#chunk` - * without any overwritten definitions from boxes. - * So the definitions on IFUNC frames should be equal to the caller CFUNC. - * - * NOTE: We traverse the cfp chain directly instead of using VM_EP_LEP. - * When an IFUNC env is escaped to the heap (e.g., due to a surrounding - * `binding` call), the env may acquire VM_ENV_FLAG_LOCAL, causing VM_EP_LEP - * to return the IFUNC ep itself rather than the enclosing CFUNC ep. - */ - while (VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_IFUNC)) { - cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); - VM_BOX_ASSERT(RUBY_VM_VALID_CONTROL_FRAME_P(cfp, eocfp), "Valid control frame expected for IFUNC caller"); - if (!RUBY_VM_VALID_CONTROL_FRAME_P(cfp, eocfp)) { - return NULL; - } - ep = cfp->ep; - } - VM_ASSERT(VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_CFUNC)); - return ep; - } + /** + * For IFUNC frames, returns the ep of the enclosing CFUNC frame. + * + * Usually CFUNC frame doesn't represent the current box and it should operate + * the caller box. See the example: + * + * # in the main box + * module Kernel + * def foo = "foo" + * module_function :foo + * end + * + * In the case above, `module_function` is defined in the root box. + * If `module_function` worked in the root box, `Kernel#foo` is invisible + * from it and it causes NameError: undefined method `foo` for module `Kernel`. + * + * But in cases of IFUNC (blocks written in C), IFUNC doesn't have its own box + * and its local env frame will be CFUNC frame. + * For example, `Enumerator#chunk` calls IFUNC blocks, written as `chunk_i` function. + * + * [1].chunk{ it.even? }.each{ ... } + * + * Before calling the Ruby block `{ it.even? }`, `#chunk` calls `chunk_i` as IFUNC + * to iterate the array's members (it's just like `#each`). + * We expect that `chunk_i` works as expected by the implementation of `#chunk` + * without any overwritten definitions from boxes. + * So the definitions on IFUNC frames should be equal to the caller CFUNC. + * + * NOTE: We traverse the cfp chain directly instead of using VM_EP_LEP. + * When an IFUNC env is escaped to the heap (e.g., due to a surrounding + * `binding` call), the env may acquire VM_ENV_FLAG_LOCAL, causing VM_EP_LEP + * to return the IFUNC ep itself rather than the enclosing CFUNC ep. + * + * NOTE: An IFUNC may also have no enclosing CFUNC frame at all, when an + * ifunc proc is invoked directly from Ruby code (e.g. Proc#call on a proc + * created by Method#to_proc). In that case the caller Ruby frame + * determines the box, so continue to the local ep walk below. + */ + while (VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_IFUNC) || + VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_CFUNC)) { + bool from_ifunc = VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_IFUNC); - while (VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_CFUNC)) { cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); - - VM_BOX_ASSERT(cfp, "CFUNC should have a valid previous control frame"); - VM_BOX_ASSERT(cfp < eocfp, "CFUNC should have a valid caller frame"); - if (!cfp || cfp >= eocfp) { + VM_BOX_ASSERT(RUBY_VM_VALID_CONTROL_FRAME_P(cfp, eocfp), "Valid caller control frame expected"); + if (!RUBY_VM_VALID_CONTROL_FRAME_P(cfp, eocfp)) { return NULL; } - VM_BOX_ASSERT(cfp->ep, "CFUNC should have a valid caller frame with env"); + VM_BOX_ASSERT(cfp->ep, "Caller control frame should have a valid env"); ep = cfp->ep; if (!ep) { return NULL; } + + if (from_ifunc && VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_CFUNC)) { + return ep; + } } while (!VM_ENV_LOCAL_P(ep)) { From 89f210de76b1b2cb5057d2857bc4eadd25056491 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 7 Sep 2026 13:17:29 +0900 Subject: [PATCH 18/25] [ruby/rubygems] Pass every expected extension into the ABI-scoped doctor_child recursion doctor_child took a single extension when the ABI-scoped recursion was written, and gained a splat in the same release, so the recursion has been raising NameError on any repository with a specifications/ directory. https://github.com/ruby/rubygems/commit/4fd406f68a Co-Authored-By: Claude Opus 5 --- lib/rubygems/doctor.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rubygems/doctor.rb b/lib/rubygems/doctor.rb index a443c9606d0762..774aa2c1864669 100644 --- a/lib/rubygems/doctor.rb +++ b/lib/rubygems/doctor.rb @@ -117,7 +117,7 @@ def doctor_child(sub_directory, *extensions) # :nodoc: if sub_directory == "specifications" && File.directory?(child) && Gem::ContentAddress.valid_ruby_abi?(ent) - doctor_child(File.join(sub_directory, ent), extension) if ent == Gem.ruby_abi && !File.symlink?(child) + doctor_child(File.join(sub_directory, ent), *extensions) if ent == Gem.ruby_abi && !File.symlink?(child) next end From cc34fb44b96222c1686c74804b64b6c07e9845a4 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 7 Sep 2026 13:18:01 +0900 Subject: [PATCH 19/25] [ruby/rubygems] Pin the running Ruby when a test needs its own ABI to resolve The content addressing resolver tests built a "~> X.Y.0" requirement from the running Ruby, which a prerelease Ruby does not satisfy, so both tests failed on 4.1.0dev. Reuse the pinning the dependency installer test already did for the same reason. https://github.com/ruby/rubygems/commit/ddaf343ade Co-Authored-By: Claude Opus 5 --- test/rubygems/helper.rb | 9 +++++++++ test/rubygems/test_gem_dependency_installer.rb | 4 ++-- test/rubygems/test_gem_resolver.rb | 10 ++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/test/rubygems/helper.rb b/test/rubygems/helper.rb index 4392fe583a72c5..2bcebc0a9a7745 100644 --- a/test/rubygems/helper.rb +++ b/test/rubygems/helper.rb @@ -1379,6 +1379,15 @@ def util_set_RUBY_VERSION(version, patchlevel, revision, description, engine = " Object.const_set :RUBY_ENGINE_VERSION, engine_version end + ## + # Pins the running Ruby to the first release of +ruby_abi+, so that the + # "~> X.Y.0" requirement a content addressed gem pins its ABI with is + # satisfied on a prerelease Ruby too. Pair with util_restore_RUBY_VERSION. + + def util_pin_ruby_to_abi(ruby_abi) + util_set_RUBY_VERSION "#{ruby_abi}.0", 0, RUBY_REVISION, "ruby #{ruby_abi}.0" + end + def util_restore_RUBY_VERSION util_clear_RUBY_VERSION diff --git a/test/rubygems/test_gem_dependency_installer.rb b/test/rubygems/test_gem_dependency_installer.rb index 685455068f6487..95d455ad5f3d7b 100644 --- a/test/rubygems/test_gem_dependency_installer.rb +++ b/test/rubygems/test_gem_dependency_installer.rb @@ -425,8 +425,8 @@ def test_install_local end def test_install_local_by_name_preserves_content_address - ruby_abi = Gem.ruby_version.segments.first(2).join(".") - util_set_RUBY_VERSION "#{ruby_abi}.0", 0, RUBY_REVISION, "ruby #{ruby_abi}.0" + ruby_abi = Gem.ruby_abi + util_pin_ruby_to_abi ruby_abi _spec, ca_gem = util_gem("ca", "1.0.0", ruby_abi: ruby_abi) do |spec| spec.platform = Gem::Platform.local end diff --git a/test/rubygems/test_gem_resolver.rb b/test/rubygems/test_gem_resolver.rb index 8e17d5c1107980..9fff107e02d366 100644 --- a/test/rubygems/test_gem_resolver.rb +++ b/test/rubygems/test_gem_resolver.rb @@ -338,7 +338,8 @@ def test_prefers_content_addressed_gem_for_same_platform def test_prefers_compatible_content_addressed_gem_over_more_specific_platform util_set_arch "arm64-darwin-27" - current_abi = "#{Gem.ruby_version.segments[0]}.#{Gem.ruby_version.segments[1]}" + current_abi = Gem.ruby_abi + util_pin_ruby_to_abi current_abi ca_spec = util_spec "a", "1" fat_spec = util_spec "a", "1" @@ -352,6 +353,8 @@ def test_prefers_compatible_content_addressed_gem_over_more_specific_platform dependency = make_dep "a" resolver = Gem::Resolver.new([dependency], s) assert_resolves_to [ca_spec], resolver + ensure + util_restore_RUBY_VERSION end def test_prefers_more_specific_platform_over_content_addressed_gem_for_another_ruby @@ -419,7 +422,8 @@ def test_falls_back_to_non_content_addressable_before_source_when_content_addres end def test_prefers_compatible_content_addressed_gem_when_multiple_abis_available - current_abi = "#{Gem.ruby_version.segments[0]}.#{Gem.ruby_version.segments[1]}" + current_abi = Gem.ruby_abi + util_pin_ruby_to_abi current_abi ca_compatible = util_spec "a", "1" ca_incompatible = util_spec "a", "1" @@ -435,6 +439,8 @@ def test_prefers_compatible_content_addressed_gem_when_multiple_abis_available dependency = make_dep "a" resolver = Gem::Resolver.new([dependency], s) assert_resolves_to [ca_compatible], resolver + ensure + util_restore_RUBY_VERSION end def test_raises_when_only_content_addressed_gem_is_incompatible From c2a0c3db4a7cf3965ea70127ae00c588561a8416 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 7 Sep 2026 13:18:07 +0900 Subject: [PATCH 20/25] [ruby/rubygems] Keep deliberately broken fixtures from writing to the test output Three tests feed rubygems something they expect it to reject: two write a gemspec that cannot be loaded, one asks git for a ref that does not exist. Each printed its diagnosis straight into the middle of the test run, as did the rake package task that no longer silenced FileUtils. https://github.com/ruby/rubygems/commit/89cc731c85 Co-Authored-By: Claude Opus 5 --- test/rubygems/test_gem_doctor.rb | 14 ++++++++++---- test/rubygems/test_gem_package_task.rb | 5 +++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/test/rubygems/test_gem_doctor.rb b/test/rubygems/test_gem_doctor.rb index fe2f5c8cadcf87..1554e7af128dd4 100644 --- a/test/rubygems/test_gem_doctor.rb +++ b/test/rubygems/test_gem_doctor.rb @@ -255,10 +255,13 @@ def test_doctor_removes_corrupt_abi_scoped_gemspec doctor = Gem::Doctor.new @gemhome - use_ui @ui do - doctor.doctor + _, err = capture_output do + use_ui @ui do + doctor.doctor + end end + assert_include err, "Invalid gemspec" assert_path_exist abi_dir assert_path_exist gemspec_path assert_path_not_exist corrupt_path @@ -297,10 +300,13 @@ def test_doctor_does_not_recurse_into_abi_symlink doctor = Gem::Doctor.new @gemhome - use_ui @ui do - doctor.doctor + _, err = capture_output do + use_ui @ui do + doctor.doctor + end end + assert_include err, "Invalid gemspec" assert File.symlink?(link) assert_path_exist outside_path end diff --git a/test/rubygems/test_gem_package_task.rb b/test/rubygems/test_gem_package_task.rb index f03af2a3a39f0d..e180cc5d4ce7ec 100644 --- a/test/rubygems/test_gem_package_task.rb +++ b/test/rubygems/test_gem_package_task.rb @@ -44,6 +44,9 @@ def test_gem_package end def test_moves_filename_returned_by_build + original_rake_fileutils_verbosity = RakeFileUtils.verbose_flag + RakeFileUtils.verbose_flag = false + gem = Gem::Specification.new do |g| g.name = "pkgr" g.version = "1.2.3" @@ -82,6 +85,8 @@ def test_moves_filename_returned_by_build assert_equal "pkg/pkgr-1.2.3-01234567.gem", built_files.first assert_path_not_exist "pkg/pkgr-1.2.3-arm64-darwin.gem" end + ensure + RakeFileUtils.verbose_flag = original_rake_fileutils_verbosity end def test_gem_package_prints_to_stdout_by_default From 443cd40febd80126566a6f2bec4713d8de1d0fef Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 7 Sep 2026 13:18:17 +0900 Subject: [PATCH 21/25] [ruby/rubygems] Report coverage once per test run SimpleCov writes its summary to stderr, so the redirection meant to keep the collate quiet never caught it and every run ended with the same two lines twice. A failing run replaced the second copy with SimpleCov reporting that it had stopped, which reads like a third failure. https://github.com/ruby/rubygems/commit/254b0332c2 Co-Authored-By: Claude Opus 5 --- test/rubygems/helper.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/rubygems/helper.rb b/test/rubygems/helper.rb index 2bcebc0a9a7745..09b7f427ebbf1f 100644 --- a/test/rubygems/helper.rb +++ b/test/rubygems/helper.rb @@ -20,6 +20,8 @@ skip ".gemspec" end + SimpleCov.print_error_status = false + # Prevent SimpleCov from running in subprocesses spawned by assert_separately ENV["SIMPLECOV_SUBPROCESS"] = "1" end From 3b9ada18c9acad899b178b49dbc4a87f8b9561ab Mon Sep 17 00:00:00 2001 From: niku <10890+niku@users.noreply.github.com> Date: Thu, 7 May 2026 12:40:24 +0900 Subject: [PATCH 22/25] Ensure Symbol#to_proc respects caller's Ruby::Box context [Bug #22015] During a normal method call, the VM creates a control frame, and the Ruby::Box associated with that frame determines the execution context. However, Symbol#to_proc does not automatically create a frame. Therefore, we need to manually push a frame associated with the appropriate Box context. When determining the Box to associate with this new frame, we cannot simply look at the closest Ruby-level control frame. This is because some built-in methods (like `Enumerable#map`) are implemented in Ruby (e.g., ``). These built-in Ruby frames belong to the default Root or Main Box, which would obscure the actual user-defined Box context we need to capture. To fix this properly, we now traverse the frames and check their inherent Box states. Since built-in Ruby frames always run in the Root or Main Box, we safely skip them by continuing our traversal until we find a user-defined Box (an Optional Box, indicated by `BOX_OPTIONAL_P`). --- test/ruby/test_box.rb | 24 ++++++++++++++++++++++++ vm.c | 2 ++ vm_insnhelper.c | 43 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/test/ruby/test_box.rb b/test/ruby/test_box.rb index 38ea4a9fdd59c8..b12d3663fcd681 100644 --- a/test/ruby/test_box.rb +++ b/test/ruby/test_box.rb @@ -1230,6 +1230,30 @@ def test_method_to_proc_called_from_ruby_frame end; end + def test_symbol_to_proc_uses_current_box + assert_separately([ENV_ENABLE_BOX], __FILE__, __LINE__, "#{<<~"begin;"}\n#{<<~'end;'}", ignore_stderr: true) + begin; + box = Ruby::Box.new + + normal, via_sym_proc = box.eval(<<~'RUBY') + class Array + def box_only_method + :ok + end + end + + normal = [[1]].flat_map { |ary| ary.box_only_method } + via_sym_proc = [[1]].flat_map(&:box_only_method) + [normal, via_sym_proc] + RUBY + + assert_equal [:ok], normal + assert_equal [:ok], via_sym_proc + + assert_raise(NoMethodError) { [1].box_only_method } + end; + end + def test_very_basic_method_calls_and_constants assert_separately([ENV_ENABLE_BOX], __FILE__, __LINE__, "#{<<~"begin;"}\n#{<<~'end;'}", ignore_stderr: true) begin; diff --git a/vm.c b/vm.c index 59e6428c04d720..23bdefdabd5527 100644 --- a/vm.c +++ b/vm.c @@ -682,6 +682,8 @@ static void add_opt_method_entry(const rb_method_entry_t *me); #define VM_ASSERT_TYPE3(obj, type1, type2, type3) \ VM_ASSERT(RB_TYPE_3_P(obj, type1, type2, type3), #obj ": %s", rb_obj_info(obj)) +static const rb_box_t * current_box_on_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp); + #include "vm_insnhelper.c" #include "vm_exec.c" diff --git a/vm_insnhelper.c b/vm_insnhelper.c index 0542221f58d33e..8bfcbd04ce8069 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -5280,7 +5280,48 @@ rb_vm_yield_with_cfunc(rb_execution_context_t *ec, const struct rb_captured_bloc static VALUE vm_yield_with_symbol(rb_execution_context_t *ec, VALUE symbol, int argc, const VALUE *argv, int kw_splat, VALUE block_handler) { - return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, rb_vm_bh_to_procval(ec, block_handler)); + VALUE passed_proc = rb_vm_bh_to_procval(ec, block_handler); + + if (!rb_box_available()) { + return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc); + } + + const rb_control_frame_t *reg_cfp = ec->cfp; + const rb_control_frame_t *ruby_cfp = rb_vm_get_ruby_level_next_cfp(ec, reg_cfp); + const rb_box_t *box = NULL; + + /* + * Traverse the frames until a user-defined Box (Optional Box) is found. + * Frames for built-in methods like run in the Root or Main Box, + * so we can safely skip them with this condition without doing string comparisons. + */ + while (ruby_cfp) { + box = current_box_on_cfp(ec, ruby_cfp); + if (BOX_OPTIONAL_P(box)) { + break; + } + ruby_cfp = rb_vm_get_ruby_level_next_cfp(ec, RUBY_VM_PREVIOUS_CONTROL_FRAME(ruby_cfp)); + } + + // Fallback to the normal call if no user-defined Box (Optional Box) is found + if (!ruby_cfp || !box) { + return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc); + } + + VALUE filename = rb_iseq_path(CFP_ISEQ(ruby_cfp)); + const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP); + VALUE val; + + vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, + Qnil, GC_GUARDED_PTR(box), + (VALUE)vm_cref_new_toplevel(ec), /* cref or me */ + 0, reg_cfp->sp, 0, 0); + + val = rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc); + + rb_vm_pop_frame(ec); + + return val; } static inline int From 72e9f1b24b3549875f077606c23a4fbbb5945d47 Mon Sep 17 00:00:00 2001 From: KITAMURA Daisuke <10890+niku@users.noreply.github.com> Date: Mon, 11 May 2026 13:26:18 +0900 Subject: [PATCH 23/25] Refactor comment in vm_insnhelper.c Remove comments that lost context after the final changes. --- vm_insnhelper.c | 1 - 1 file changed, 1 deletion(-) diff --git a/vm_insnhelper.c b/vm_insnhelper.c index 8bfcbd04ce8069..13c785e07d070f 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -5293,7 +5293,6 @@ vm_yield_with_symbol(rb_execution_context_t *ec, VALUE symbol, int argc, const /* * Traverse the frames until a user-defined Box (Optional Box) is found. * Frames for built-in methods like run in the Root or Main Box, - * so we can safely skip them with this condition without doing string comparisons. */ while (ruby_cfp) { box = current_box_on_cfp(ec, ruby_cfp); From 8c5fae1b42d8d14b6232eb8e9a026d8ebc9f97cd Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Fri, 28 Aug 2026 10:07:10 +0900 Subject: [PATCH 24/25] Symbol#to_proc: treat the main box as a caller box too Since the main/root box separation ([Feature #21881]), code running in the main box also fails to call box-local methods via Symbol#to_proc. Stop the caller-frame traversal at any user box (main or optional) instead of optional boxes only. --- test/ruby/test_box.rb | 14 ++++++++++++++ vm_insnhelper.c | 8 ++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/test/ruby/test_box.rb b/test/ruby/test_box.rb index b12d3663fcd681..647eb960da915f 100644 --- a/test/ruby/test_box.rb +++ b/test/ruby/test_box.rb @@ -1254,6 +1254,20 @@ def box_only_method end; end + def test_symbol_to_proc_uses_main_box + assert_separately([ENV_ENABLE_BOX], __FILE__, __LINE__, "#{<<~"begin;"}\n#{<<~'end;'}", ignore_stderr: true) + begin; + class Array + def main_box_only_method + :ok + end + end + + assert_equal [:ok], [[1]].flat_map { |ary| ary.main_box_only_method } + assert_equal [:ok], [[1]].flat_map(&:main_box_only_method) + end; + end + def test_very_basic_method_calls_and_constants assert_separately([ENV_ENABLE_BOX], __FILE__, __LINE__, "#{<<~"begin;"}\n#{<<~'end;'}", ignore_stderr: true) begin; diff --git a/vm_insnhelper.c b/vm_insnhelper.c index 13c785e07d070f..1294819124dbb1 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -5291,18 +5291,18 @@ vm_yield_with_symbol(rb_execution_context_t *ec, VALUE symbol, int argc, const const rb_box_t *box = NULL; /* - * Traverse the frames until a user-defined Box (Optional Box) is found. - * Frames for built-in methods like run in the Root or Main Box, + * Traverse the frames until a user box (Main or Optional Box) is found. + * Frames for built-in methods like run in the Root or Master Box, */ while (ruby_cfp) { box = current_box_on_cfp(ec, ruby_cfp); - if (BOX_OPTIONAL_P(box)) { + if (BOX_USER_P(box)) { break; } ruby_cfp = rb_vm_get_ruby_level_next_cfp(ec, RUBY_VM_PREVIOUS_CONTROL_FRAME(ruby_cfp)); } - // Fallback to the normal call if no user-defined Box (Optional Box) is found + // Fallback to the normal call if no user box is found if (!ruby_cfp || !box) { return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc); } From 4b224b63e6eac024d5058578fbc10ad7240ead63 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 7 Sep 2026 13:50:55 +0900 Subject: [PATCH 25/25] Symbol#to_proc: push a block frame instead of a TOP frame Per review feedback, a TOP frame in the middle of a method call is misleading. Push a VM_FRAME_MAGIC_BLOCK frame whose outer env points to the caller's frame, so the box resolves through the ep chain and the backtrace reads like a usual block invocation at the caller's site. The frame is finished but owned by no vm_exec loop, so catch the non-local exit and rewind to the caller before re-raising. Co-Authored-By: Claude Opus 5 --- vm_insnhelper.c | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/vm_insnhelper.c b/vm_insnhelper.c index 1294819124dbb1..05c9bf75b7753f 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -5307,16 +5307,41 @@ vm_yield_with_symbol(rb_execution_context_t *ec, VALUE symbol, int argc, const return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc); } - VALUE filename = rb_iseq_path(CFP_ISEQ(ruby_cfp)); - const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP); - VALUE val; + /* + * Push a dummy block frame whose outer env is `ruby_cfp` so that the + * method resolves in the caller's box (via the ep chain) and the + * backtrace reads like a usual block invocation at the caller's site. + */ + const rb_iseq_t *caller_iseq = CFP_ISEQ(ruby_cfp); + VALUE name = rb_sprintf("block in %"PRIsVALUE, rb_iseq_label(caller_iseq)); + const rb_iseq_t *iseq = rb_iseq_new_with_opt(Qnil, name, + rb_iseq_path(caller_iseq), rb_iseq_realpath(caller_iseq), + rb_vm_get_sourceline(ruby_cfp), caller_iseq, 0, + ISEQ_TYPE_BLOCK, NULL, Qnil); + volatile VALUE val = Qnil; + enum ruby_tag_type state; + + vm_push_frame(ec, iseq, VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_FINISH, + ruby_cfp->self, VM_GUARDED_PREV_EP(ruby_cfp->ep), + Qfalse, /* cref or me */ + ISEQ_BODY(iseq)->iseq_encoded, reg_cfp->sp, + ISEQ_BODY(iseq)->local_table_size, ISEQ_BODY(iseq)->stack_max); - vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, - Qnil, GC_GUARDED_PTR(box), - (VALUE)vm_cref_new_toplevel(ec), /* cref or me */ - 0, reg_cfp->sp, 0, 0); + /* + * The pushed frame is finished (owned by no vm_exec loop), so catch the + * non-local exit here, pop the frame, and let the caller's handlers see + * the exception. + */ + EC_PUSH_TAG(ec); + if ((state = EC_EXEC_TAG()) == TAG_NONE) { + val = rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc); + } + EC_POP_TAG(); - val = rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc); + if (state != TAG_NONE) { + rb_vm_rewind_cfp(ec, (rb_control_frame_t *)reg_cfp); + EC_JUMP_TAG(ec, state); + } rb_vm_pop_frame(ec);