fix: keep the url's query string when probing /json/version

CHANGELOG.md

@@ -27,6 +27,8 @@
   window (browserless). That window lives in the browser's implicit context, which Chrome below 145
   won't address by id. Ferrum now detects it and exposes as `Ferrum::Context#implicit?`. It's never
   disposed, so `#reset` won't clear such a browser. [#578] [#627]
+- Connecting to a remote browser with `:url` drops query string when probing `/json/version`, endpoint might answer
+  401 and Ferrum died with `undefined method 'host' for nil`. in case of parse issues `Ferrum::NoWebSocketUrlError` is raised.
 - Targets of a type `Ferrum::Contexts` doesn't track were left attached and paused by auto-attach for the lifetime
   of the browser; Chrome opens two `browser_ui` ones per browser context. They're now resumed and detached from
 - `Ferrum::Browser::Process` registered two `ObjectSpace` finalizers on the same object, a directory remover and

lib/ferrum/browser/process.rb

@@ -25,6 +25,10 @@ class Browser
     class Process
       extend Forwardable
 
+      HTTP_SCHEMES = { "ws" => "http", "wss" => "https" }.freeze
+      WS_SCHEMES = { "http" => "ws", "https" => "wss" }.freeze
+      LOCAL_HOSTS = %w[0.0.0.0 :: 127.0.0.1 ::1 localhost].freeze
+
       delegate path: :command
 
       #
@@ -50,8 +54,12 @@ def initialize(options)
         if options.ws_url || options.url
           # `:ws_url` option is higher priority than `:url`, parse versions
           # and use it as a ws_url, otherwise use what has been parsed.
-          response = parse_json_version(options.ws_url || options.url)
-          self.ws_url = options.ws_url || response&.[]("webSocketDebuggerUrl")
+          endpoint = options.ws_url || options.url
+          response = parse_json_version(endpoint)
+          ws_url = options.ws_url || reachable_ws_url(response&.[]("webSocketDebuggerUrl"), endpoint)
+          raise NoWebSocketUrlError, endpoint unless ws_url
+
+          self.ws_url = ws_url
           return
         end
 
@@ -227,18 +235,7 @@ def close_io(*ios)
       end
 
       def parse_json_version(url)
-        url = URI.join(url, "/json/version")
-
-        if %w[wss ws].include?(url.scheme)
-          url.scheme = case url.scheme
-                       when "ws"
-                         "http"
-                       when "wss"
-                         "https"
-                       end
-        end
-
-        response = JSON.parse(::Net::HTTP.get(URI(url.to_s)))
+        response = JSON.parse(::Net::HTTP.get(URI(json_version_url(url).to_s)))
 
         @v8_version = response["V8-Version"]
         @browser_version = response["Browser"]
@@ -250,6 +247,25 @@ def parse_json_version(url)
       rescue JSON::ParserError
         # nop
       end
+
+      def json_version_url(url)
+        url = Addressable::URI.parse(url).dup
+        url.scheme = HTTP_SCHEMES.fetch(url.scheme, url.scheme)
+        url.path = "/json/version"
+        url
+      end
+
+      def reachable_ws_url(ws_url, endpoint)
+        return unless ws_url
+
+        ws_url = Addressable::URI.parse(ws_url)
+        return ws_url.to_s unless LOCAL_HOSTS.include?(ws_url.host)
+
+        url = Addressable::URI.parse(endpoint).dup
+        url.scheme = WS_SCHEMES.fetch(url.scheme, url.scheme)
+        url.path = ws_url.path
+        url.to_s
+      end
     end
   end
 end

lib/ferrum/errors.rb

@@ -77,6 +77,18 @@ def initialize(timeout, output)
     end
   end
 
+  # Raised when connecting to an already running browser, but its
+  # `/json/version` endpoint doesn't answer with the websocket url.
+  class NoWebSocketUrlError < Error
+    attr_reader :url
+
+    def initialize(url)
+      @url = url
+      super("Couldn't get a websocket url out of #{url}, `/json/version` there didn't answer with JSON. " \
+            "Check the browser is reachable at that url and that any credentials it needs are part of it")
+    end
+  end
+
   # Raised when trying to interact with a browser process that has died or
   # a window that has already been closed.
   class DeadBrowserError < Error

sig/ferrum/browser/process.rbs

@@ -1,6 +1,12 @@
 module Ferrum
   class Browser
     class Process
+      HTTP_SCHEMES: ::Hash[String, String]
+
+      WS_SCHEMES: ::Hash[String, String]
+
+      LOCAL_HOSTS: ::Array[String]
+
       attr_reader host: String?
 
       attr_reader port: ::Integer?
@@ -73,6 +79,10 @@ module Ferrum
       def parse_json_version: (Addressable::URI | String url) -> Hash[String, untyped]?
 
       def close_io: (*IO ios) -> void
+
+      def json_version_url: (Addressable::URI | String url) -> Addressable::URI
+
+      def reachable_ws_url: (String? ws_url, Addressable::URI | String endpoint) -> String?
     end
   end
 end

sig/ferrum/errors.rbs

@@ -44,6 +44,12 @@ module Ferrum
     def initialize: (::Numeric timeout, String output) -> void
   end
 
+  class NoWebSocketUrlError < Error
+    attr_reader url: String
+
+    def initialize: (String url) -> void
+  end
+
   class DeadBrowserError < Error
     def initialize: (?::String message) -> void
   end

spec/browser_spec.rb

@@ -139,6 +139,90 @@
       end
     end
 
+    def with_token_authenticated_proxy(browser_url, token:)
+      browser_url = Addressable::URI.parse(browser_url)
+      server = TCPServer.new("127.0.0.1", 0)
+      port = server.addr[1]
+
+      acceptor = Thread.new do
+        loop { Thread.new(server.accept) { |socket| proxy_request(socket, browser_url, token, port) } }
+      rescue IOError, Errno::EBADF
+        nil
+      end
+
+      yield "http://127.0.0.1:#{port}?token=#{token}"
+    ensure
+      acceptor&.kill
+      server&.close
+    end
+
+    def proxy_request(socket, browser_url, token, port)
+      request = socket.gets.to_s
+
+      if !request.include?("token=#{token}")
+        socket.print(proxy_unauthorized)
+      elsif request.start_with?("GET /json/version")
+        socket.print(proxy_json_version(browser_url, port))
+      else
+        proxy_tunnel(socket, request, browser_url)
+      end
+    ensure
+      socket.close unless socket.closed?
+    end
+
+    def proxy_unauthorized
+      body = "<html>401 Unauthorized</html>"
+      "HTTP/1.1 401 Unauthorized\r\nContent-Length: #{body.bytesize}\r\nConnection: close\r\n\r\n#{body}"
+    end
+
+    def proxy_json_version(browser_url, port)
+      version = JSON.parse(Net::HTTP.get(URI(browser_url.join("/json/version").to_s)))
+      path = Addressable::URI.parse(version["webSocketDebuggerUrl"]).path
+      body = version.merge("webSocketDebuggerUrl" => "ws://0.0.0.0:#{port}#{path}").to_json
+      "HTTP/1.1 200 OK\r\nContent-Length: #{body.bytesize}\r\nConnection: close\r\n\r\n#{body}"
+    end
+
+    def proxy_tunnel(socket, request, browser_url)
+      upstream = TCPSocket.new(browser_url.host, browser_url.port)
+      upstream.print(request.sub(/\?token=[^ ]*/, ""))
+
+      [[socket, upstream], [upstream, socket]].map do |from, to|
+        Thread.new do
+          IO.copy_stream(from, to)
+        rescue IOError, Errno::ECONNRESET, Errno::EPIPE
+          nil
+        end
+      end.each(&:join)
+    ensure
+      upstream&.close
+    end
+
+    it "supports :url argument with a query string" do
+      with_external_browser do |url, _process|
+        with_token_authenticated_proxy(url, token: "secret") do |proxy_url|
+          browser = Ferrum::Browser.new(url: proxy_url)
+          browser.go_to(base_url)
+
+          expect(browser.body).to include("Hello world!")
+          expect(browser.process.ws_url.authority).to eq(Addressable::URI.parse(proxy_url).authority)
+          expect(browser.process.ws_url.query).to eq("token=secret")
+        ensure
+          browser&.quit
+        end
+      end
+    end
+
+    it "raises when :url doesn't answer with a websocket url" do
+      with_external_browser do |url, _process|
+        with_token_authenticated_proxy(url, token: "secret") do |proxy_url|
+          unauthorized = proxy_url.sub("secret", "wrong")
+
+          expect { Ferrum::Browser.new(url: unauthorized) }
+            .to raise_error(Ferrum::NoWebSocketUrlError, /#{Regexp.escape(unauthorized)}/)
+        end
+      end
+    end
+
     it "supports :ws_url argument" do
       with_external_browser do |url, process|
         browser = Ferrum::Browser.new(ws_url: web_socket_debugger_url(url))