[pull] master from ruby:master

.github/workflows/zjit-macos.yml

@@ -98,7 +98,7 @@ jobs:
           rustup install ${{ matrix.rust_version }} --profile minimal
           rustup default ${{ matrix.rust_version }}
 
-      - uses: taiki-e/install-action@0758d235715de2f3551eacc980d9ae8fce9342c3 # v2.87.3
+      - uses: taiki-e/install-action@5bf6ce016fd2e72eefc647cbca1e4213f65955b8 # v2.87.5
         with:
           tool: nextest@0.9
         if: ${{ matrix.test_task == 'zjit-check' }}

.github/workflows/zjit-ubuntu.yml

@@ -152,7 +152,7 @@ jobs:
           ruby-version: '3.1'
           bundler: none
 
-      - uses: taiki-e/install-action@0758d235715de2f3551eacc980d9ae8fce9342c3 # v2.87.3
+      - uses: taiki-e/install-action@5bf6ce016fd2e72eefc647cbca1e4213f65955b8 # v2.87.5
         with:
           tool: nextest@0.9
         if: ${{ matrix.test_task == 'zjit-check' }}

lib/bundler/installer/parallel_installer.rb

@@ -96,7 +96,8 @@ def call
       handle_error if failed_specs.any?
       @specs
     ensure
-      worker_pool&.stop
+      @worker_pool&.stop
+      @download_worker_pool&.stop
     end
 
     private
@@ -166,17 +167,18 @@ def install_serially
       end
     end
 
+    def download_worker_pool
+      @download_worker_pool ||= Bundler::Worker.new(@size, "Gem Downloader",
+        ->(spec_install, worker_num) { do_download(spec_install, worker_num) }, response_queue: response_queue)
+    end
+
     def worker_pool
-      @worker_pool ||= Bundler::Worker.new @size, "Parallel Installer", lambda {|spec_install, worker_num|
-        case spec_install.state
-        when :enqueued
-          do_download(spec_install, worker_num)
-        when :installable
-          do_install(spec_install, worker_num)
-        else
-          spec_install
-        end
-      }
+      @worker_pool ||= Bundler::Worker.new(@size, "Parallel Installer",
+        ->(spec_install, worker_num) { do_install(spec_install, worker_num) }, response_queue: response_queue)
+    end
+
+    def response_queue
+      @response_queue ||= Thread::Queue.new
     end
 
     def do_download(spec_install, worker_num)
@@ -214,24 +216,24 @@ def do_install(spec_install, worker_num)
       spec_install
     end
 
-    # Dequeue a spec and save its post-install message and then enqueue the
-    # remaining specs.
-    # Some specs might've had to wait til this spec was installed to be
-    # processed so the call to `enqueue_specs` is important after every
-    # dequeue.
+    # Process one completed download or installation. Downloads can finish
+    # before their dependencies are installed, so check all downloaded specs
+    # after each completion and enqueue any that are now installable.
     def process_specs(installed_specs)
       spec = worker_pool.deq
 
       if spec.installed?
         installed_specs[spec.name] = true
-        return
       elsif spec.failed?
         return
-      elsif spec.ready_to_install?(installed_specs)
-        spec.state = :installable
       end
 
-      worker_pool.enq(spec, priority: spec.enqueue_with_priority?)
+      @specs.each do |candidate|
+        next unless candidate.ready_to_install?(installed_specs)
+
+        candidate.state = :installable
+        worker_pool.enq(candidate, priority: candidate.enqueue_with_priority?)
+      end
     end
 
     def finished_installing?
@@ -270,11 +272,8 @@ def require_tree_for_spec(spec)
       t
     end
 
-    # Keys in the remains hash represent uninstalled gems specs.
-    # We enqueue all gem specs that do not have any dependencies.
-    # Later we call this lambda again to install specs that depended on
-    # previously installed specifications. We continue until all specs
-    # are installed.
+    # Queue every missing spec for download. `process_specs` schedules each
+    # downloaded spec for installation once its dependencies are installed.
     def enqueue_specs(installed_specs)
       @specs.each do |spec|
         if spec.installed?
@@ -283,7 +282,7 @@ def enqueue_specs(installed_specs)
         end
 
         spec.state = :enqueued
-        worker_pool.enq spec
+        download_worker_pool.enq spec
       end
     end
   end

lib/bundler/rubygems_ext.rb

@@ -281,6 +281,20 @@ def extension_dir
       end
     end
 
+    alias_method :rg_build_info_dir, :build_info_dir
+    def build_info_dir
+      # A git checkout's build logs belong with that checkout's extension build.
+      # base_dir points at the directory holding every checkout, so logs keyed by
+      # full_name would collide between revisions of the same gem and would sit
+      # outside anything `bundle clean` prunes. extension_dir is unique per
+      # revision and goes away with the checkout.
+      if source.respond_to?(:extension_dir_name)
+        extension_dir
+      else
+        rg_build_info_dir
+      end
+    end
+
     # Can be removed once RubyGems 3.5.21 support is dropped
     remove_method :gem_dir if method_defined?(:gem_dir, false)
 

lib/bundler/source/rubygems.rb

@@ -25,6 +25,7 @@ def initialize(options = {})
         @checksum_store = Checksum::Store.new
         @gem_installers = {}
         @gem_installers_mutex = Mutex.new
+        @remote_spec_for_mutex = Mutex.new
         @remote_specs_mutex = Mutex.new
 
         cooldown = options["cooldown"]
@@ -443,12 +444,14 @@ def remote_specs
       # Looks up a single spec in the remote sources, fetching only its own
       # name when the full remote index is not already materialized.
       def remote_spec_for(spec)
-        return remote_specs.search(spec).first if @remote_specs || api_fetchers.empty?
+        @remote_spec_for_mutex.synchronize do
+          return remote_specs.search(spec).first if @remote_specs || api_fetchers.empty?
 
-        index = Index.build do |idx|
-          fetch_names(api_fetchers, [spec.name], idx)
+          index = Index.build do |idx|
+            fetch_names(api_fetchers, [spec.name], idx)
+          end
+          index.search(spec).first
         end
-        index.search(spec).first
       end
 
       def fetch_names(fetchers, dependency_names, index)

lib/bundler/worker.rb

@@ -19,11 +19,12 @@ def initialize(exn)
     # @param size [Integer] Size of pool
     # @param name [String] name the name of the worker
     # @param func [Proc] job to run in inside the worker pool
-    def initialize(size, name, func)
+    # @param response_queue [Thread::Queue] queue that receives completed jobs
+    def initialize(size, name, func, response_queue: Thread::Queue.new)
       @name = name
       @request_queue = Thread::Queue.new
       @request_queue_with_priority = Thread::Queue.new
-      @response_queue = Thread::Queue.new
+      @response_queue = response_queue
       @func = func
       @size = size
       @threads = nil

lib/rubygems/commands/install_command.rb

@@ -93,7 +93,7 @@ def description # :nodoc:
   [build fails]
   Gem files will remain installed in \\
   /path/to/gems/some_extension_gem-1.0 for inspection.
-  Results logged to /path/to/gems/some_extension_gem-1.0/gem_make.out
+  Results logged to /path/to/build_info/some_extension_gem-1.0.gem_make.out
   $ gem install some_extension_gem -- --with-extension-lib=/path/to/lib
   [build succeeds]
   $ gem list some_extension_gem
@@ -110,7 +110,7 @@ def description # :nodoc:
   [build fails]
   Gem files will remain installed in \\
   /path/to/gems/some_extension_gem-1.0 for inspection.
-  Results logged to /path/to/gems/some_extension_gem-1.0/gem_make.out
+  Results logged to /path/to/build_info/some_extension_gem-1.0.gem_make.out
   $ [cd /path/to/gems/some_extension_gem-1.0]
   $ [edit files or what-have-you and run make]
   $ gem spec ../../cache/some_extension_gem-1.0.gem --ruby > \\

lib/rubygems/doctor.rb

@@ -21,7 +21,7 @@ class Gem::Doctor
 
   REPOSITORY_EXTENSION_MAP = [ # :nodoc:
     ["specifications", ".gemspec"],
-    ["build_info",     ".info"],
+    ["build_info",     ".info", ".mkmf.log", ".gem_make.out"],
     ["cache",          ".gem"],
     ["doc",            ""],
     ["extensions",     ""],
@@ -92,15 +92,15 @@ def doctor
   # Cleans up children of this gem repository
 
   def doctor_children # :nodoc:
-    REPOSITORY_EXTENSION_MAP.each do |sub_directory, extension|
-      doctor_child sub_directory, extension
+    REPOSITORY_EXTENSION_MAP.each do |sub_directory, *extensions|
+      doctor_child sub_directory, *extensions
     end
   end
 
   ##
-  # Removes files in +sub_directory+ with +extension+
+  # Removes files in +sub_directory+ with any of +extensions+
 
-  def doctor_child(sub_directory, extension) # :nodoc:
+  def doctor_child(sub_directory, *extensions) # :nodoc:
     directory = File.join(@gem_repository, sub_directory)
 
     Dir.entries(directory).sort.each do |ent|
@@ -109,15 +109,15 @@ def doctor_child(sub_directory, extension) # :nodoc:
       child = File.join(directory, ent)
       next unless File.exist?(child)
 
-      basename = File.basename(child, extension)
+      basename = strip_extension File.basename(child), extensions
       next if installed_specs.include? basename
       next if /^rubygems-\d/.match?(basename)
       next if sub_directory == "specifications" && basename == "default"
       next if sub_directory == "plugins" && Gem.plugin_suffix_regexp =~ basename
 
       if sub_directory == "specifications" && File.directory?(child) &&
          Gem::ContentAddress.valid_ruby_abi?(ent)
-        doctor_child(File.join(sub_directory, ent), extension) if ent == Gem.ruby_abi && !File.symlink?(child)
+        doctor_child(File.join(sub_directory, ent), *extensions) if ent == Gem.ruby_abi && !File.symlink?(child)
         next
       end
 
@@ -135,4 +135,15 @@ def doctor_child(sub_directory, extension) # :nodoc:
   rescue Errno::ENOENT
     # ignore
   end
+
+  ##
+  # Removes the first of +extensions+ that +name+ ends with
+
+  def strip_extension(name, extensions) # :nodoc:
+    extension = extensions.find do |ext|
+      !ext.empty? && name.end_with?(ext)
+    end
+
+    extension ? name.delete_suffix(extension) : name
+  end
 end

lib/rubygems/ext/builder.rb

@@ -220,9 +220,11 @@ def build_error(output, backtrace = nil) # :nodoc:
     #{output}
 
 Gem files will remain installed in #{@gem_dir} for inspection.
-Results logged to #{gem_make_out}
 EOF
 
+    # Losing the log must not cost the user the build error itself.
+    message += "Results logged to #{gem_make_out}\n" if gem_make_out
+
     raise Gem::Ext::BuildError, message, backtrace
   end
 
@@ -243,13 +245,72 @@ def build_extension(extension, dest_path) # :nodoc:
 
       verbose { results.join("\n") }
 
-      write_gem_make_out results.join "\n"
+      # Build logs are noisy, non-reproducible artifacts that are not meant to
+      # be installed. Drop the ones this build left behind, plus any written
+      # into the extension directory by a RubyGems old enough to put them there.
+      FileUtils.rm_f mkmf_log_candidates(extension_dir, dest_path)
+      FileUtils.rm_f File.join(dest_path, "gem_make.out")
+      FileUtils.rm_f [build_log_path("mkmf.log"), build_log_path("gem_make.out")]
+    rescue Gem::Ext::Builder::NoMakefileError => e
+      # extconf ran fine but produced no Makefile, so the extension was skipped
+      # rather than built and installing carries on. Keep the log that says why
+      # it was skipped, out of the installation tree but still reachable.
+      results << e.message
+      results << "Skipping make for #{extension} as no Makefile was found."
+
+      verbose { results.join("\n") }
+
+      preserve_mkmf_log extension_dir, dest_path
+      write_gem_make_out results.join("\n")
     rescue StandardError => e
       results << e.message
+
+      mkmf_log_dest = preserve_mkmf_log(extension_dir, dest_path)
+      if mkmf_log_dest
+        results << "To see why this extension failed to compile, please check the mkmf.log which can be found here:"
+        results << "  #{mkmf_log_dest}"
+      end
+
       build_error(results.join("\n"), $@)
     end
   end
 
+  ##
+  # Where a build log of +kind+ for this gem lives in the build_info directory.
+
+  def build_log_path(kind) # :nodoc:
+    File.join @spec.build_info_dir, "#{@spec.full_name}.#{kind}"
+  end
+
+  ##
+  # Moves the mkmf.log this build left behind into the build_info directory and
+  # returns its new path, or nil when there is none or it cannot be kept.
+  # Keeping a log must never replace the build error the caller is reporting,
+  # so a filesystem failure here is swallowed.
+
+  def preserve_mkmf_log(extension_dir, dest_path) # :nodoc:
+    mkmf_log = mkmf_log_candidates(extension_dir, dest_path).find {|log| File.exist?(log) }
+    return unless mkmf_log
+
+    destination = build_log_path "mkmf.log"
+
+    FileUtils.mkdir_p @spec.build_info_dir
+    FileUtils.mv mkmf_log, destination
+
+    destination
+  rescue SystemCallError
+    nil
+  end
+
+  ##
+  # Places a completed build may have left an mkmf.log, most specific first.
+  # Gem::Ext::ExtConfBuilder parks it in +dest_path+ so that the "clean" target
+  # cannot delete it; the other builders leave it where extconf ran.
+
+  def mkmf_log_candidates(extension_dir, dest_path) # :nodoc:
+    [File.join(dest_path, "mkmf.log"), File.join(extension_dir, "mkmf.log")]
+  end
+
   ##
   # Builds extensions.  Valid types of extensions are extconf.rb files,
   # configure scripts and rakefiles or mkrf_conf files.
@@ -277,17 +338,21 @@ def build_extensions
   end
 
   ##
-  # Writes +output+ to gem_make.out in the extension install directory.
+  # Writes +output+ to gem_make.out in the build_info directory and returns its
+  # path, or nil when it cannot be written. Only called when the extension was
+  # not built, to keep build logs out of the installation tree.
 
   def write_gem_make_out(output) # :nodoc:
-    destination = File.join @spec.extension_dir, "gem_make.out"
+    destination = build_log_path "gem_make.out"
 
-    FileUtils.mkdir_p @spec.extension_dir
+    FileUtils.mkdir_p @spec.build_info_dir
 
     File.open destination, "wb" do |io|
       io.puts output
     end
 
     destination
+  rescue SystemCallError
+    nil
   end
 end