feat: multi-address DNS resolution for contact points and connections (DRIVER-201)

core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java

@@ -27,6 +27,7 @@
 import com.datastax.oss.protocol.internal.util.Bytes;
 import edu.umd.cs.findbugs.annotations.NonNull;
 import edu.umd.cs.findbugs.annotations.Nullable;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.security.PrivilegedActionException;
@@ -319,7 +320,7 @@ protected GssApiAuthenticator(
                 SUPPORTED_MECHANISMS,
                 options.getAuthorizationId(),
                 protocol,
-                ((InetSocketAddress) endPoint.resolve()).getAddress().getCanonicalHostName(),
+                serverName(endPoint),
                 options.getSaslProperties(),
                 null);
       } catch (LoginException | SaslException e) {
@@ -328,6 +329,23 @@ protected GssApiAuthenticator(
       this.endPoint = endPoint;
     }
 
+    /**
+     * The host name to build the Kerberos service principal from.
+     *
+     * <p>Prefers the canonical name of the resolved address, which is what Kerberos expects. The
+     * driver's own endpoints always hand this a resolved address — the channel carries an endpoint
+     * bound to the address it connected to (see {@code PinnableEndPoint}) — but a custom {@link
+     * EndPoint} implementation may still yield an unresolved one, in which case {@code
+     * getAddress()} is null. Fall back to the host string rather than throwing a {@link
+     * NullPointerException}: the hostname is usually the right service name anyway, and a failed
+     * reverse lookup should not take authentication down.
+     */
+    private static String serverName(EndPoint endPoint) {
+      InetSocketAddress address = (InetSocketAddress) endPoint.resolve();
+      InetAddress inetAddress = address.getAddress();
+      return inetAddress != null ? inetAddress.getCanonicalHostName() : address.getHostString();
+    }
+
     @NonNull
     @Override
     protected ByteBuffer getMechanism() {

core/src/main/java/com/datastax/dse/driver/internal/core/insights/InsightsClient.java

@@ -294,7 +294,32 @@ private Map<String, SessionStateForNode> getConnectedNodes() {
         .collect(
             Collectors.toMap(
                 entry -> AddressFormatter.nullSafeToString(entry.getKey().getEndPoint().resolve()),
-                this::constructSessionStateForNode));
+                this::constructSessionStateForNode,
+                InsightsClient::mergeNodeStates));
+  }
+
+  /**
+   * Combines the states of two nodes that report under the same address.
+   *
+   * <p>The key is not unique per node: behind an SNI proxy or a cloud client route, every node's
+   * endpoint resolves to the same proxy address, so any session with more than one node open
+   * produces duplicate keys. Without a merge function {@link Collectors#toMap} throws {@link
+   * IllegalStateException}, which propagates out of the status report and aborts it every interval.
+   * Summing matches what the shared key denotes in that deployment: the totals reached through that
+   * address.
+   */
+  private static SessionStateForNode mergeNodeStates(
+      SessionStateForNode first, SessionStateForNode second) {
+    return new SessionStateForNode(
+        sumNullable(first.getConnections(), second.getConnections()),
+        sumNullable(first.getInFlightQueries(), second.getInFlightQueries()));
+  }
+
+  private static Integer sumNullable(Integer first, Integer second) {
+    if (first == null) {
+      return second;
+    }
+    return second == null ? first : first + second;
   }
 
   private SessionStateForNode constructSessionStateForNode(Map.Entry<Node, ChannelPool> entry) {

core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java

@@ -135,6 +135,13 @@ public enum DefaultDriverOption implements DriverOption {
    * <p>Value-type: int
    */
   CONNECTION_MAX_ORPHAN_REQUESTS("advanced.connection.max-orphan-requests"),
+  /**
+   * The maximum number of addresses a single connection attempt will try, when the endpoint it
+   * connects to is a DNS name that resolves to several addresses.
+   *
+   * <p>Value-type: int
+   */
+  CONNECTION_MAX_CANDIDATE_ADDRESSES("advanced.connection.max-candidate-addresses"),
   /**
    * Whether to log non-fatal errors when the driver tries to open a new connection.
    *
@@ -701,8 +708,14 @@ public enum DefaultDriverOption implements DriverOption {
   CONTROL_CONNECTION_AGREEMENT_WARN("advanced.control-connection.schema-agreement.warn-on-failure"),
 
   /**
-   * Whether to forcibly add original contact points held by MetadataManager to the reconnection
-   * plan, in case there is no live nodes available according to LBP. Experimental.
+   * Whether to append the original contact points held by MetadataManager to the reconnection plan,
+   * after the live nodes reported by the load balancing policy. Defaults to {@code true}.
+   *
+   * <p>This is also the driver's DNS re-resolution path. Contact points are appended as-is, still
+   * unresolved hostnames, and each is expanded to its current DNS IPs at connection time through
+   * Netty's configured resolver. Metadata nodes, in contrast, hold an already-resolved endpoint
+   * that is never re-resolved. Keeping this enabled lets control-connection reconnects re-resolve
+   * the original hostnames and pick up new IPs once the live-node plan is exhausted.
    *
    * <p>Value-type: boolean
    */
@@ -837,7 +850,14 @@ public enum DefaultDriverOption implements DriverOption {
    * Whether to resolve the addresses passed to `basic.contact-points`.
    *
    * <p>Value-type: boolean
+   *
+   * @deprecated Setting this option has no effect. Contact points given in the configuration are
+   *     now always kept as unresolved hostnames and expanded to all of their DNS-mapped IPs lazily
+   *     at connection time. This never applied to programmatic contact points passed to {@code
+   *     SessionBuilder.addContactPoints}, which are used exactly as supplied -- an already-resolved
+   *     address stays bound to that one IP.
    */
+  @Deprecated
   RESOLVE_CONTACT_POINTS("advanced.resolve-contact-points"),
 
   /**

core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java

@@ -245,6 +245,9 @@ private void readObject(ObjectInputStream stream) throws InvalidObjectException
     throw new InvalidObjectException("Proxy required");
   }
 
+  // RESOLVE_CONTACT_POINTS is deprecated and has no effect, but it is still a driver option, so the
+  // defaults map stays complete by carrying its reference.conf value.
+  @SuppressWarnings("deprecation")
   protected static void fillWithDriverDefaults(OptionsMap map) {
     Duration initQueryTimeout = Duration.ofSeconds(5);
     Duration requestTimeout = Duration.ofSeconds(2);
@@ -276,6 +279,7 @@ protected static void fillWithDriverDefaults(OptionsMap map) {
     map.put(TypedDriverOption.CONNECTION_POOL_INIT_BATCH_SIZE, 0);
     map.put(TypedDriverOption.CONNECTION_MAX_REQUESTS, 1024);
     map.put(TypedDriverOption.CONNECTION_MAX_ORPHAN_REQUESTS, 256);
+    map.put(TypedDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES, 5);
     map.put(TypedDriverOption.CONNECTION_WARN_INIT_ERROR, true);
     map.put(TypedDriverOption.CONNECTION_ADVANCED_SHARD_AWARENESS_ENABLED, true);
     map.put(TypedDriverOption.ADVANCED_SHARD_AWARENESS_PORT_LOW, 10000);
@@ -369,7 +373,7 @@ protected static void fillWithDriverDefaults(OptionsMap map) {
     map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_INTERVAL, Duration.ofMillis(200));
     map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_TIMEOUT, Duration.ofSeconds(10));
     map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, true);
-    map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, false);
+    map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, true);
     map.put(TypedDriverOption.PREPARE_ON_ALL_NODES, true);
     map.put(TypedDriverOption.REPREPARE_ENABLED, true);
     map.put(TypedDriverOption.REPREPARE_CHECK_SYSTEM_TABLE, false);

core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java

@@ -172,6 +172,13 @@ public String toString() {
   public static final TypedDriverOption<Integer> CONNECTION_MAX_ORPHAN_REQUESTS =
       new TypedDriverOption<>(
           DefaultDriverOption.CONNECTION_MAX_ORPHAN_REQUESTS, GenericType.INTEGER);
+  /**
+   * The maximum number of addresses a single connection attempt will try, when the endpoint it
+   * connects to is a DNS name that resolves to several addresses.
+   */
+  public static final TypedDriverOption<Integer> CONNECTION_MAX_CANDIDATE_ADDRESSES =
+      new TypedDriverOption<>(
+          DefaultDriverOption.CONNECTION_MAX_CANDIDATE_ADDRESSES, GenericType.INTEGER);
   /** Whether to log non-fatal errors when the driver tries to open a new connection. */
   public static final TypedDriverOption<Boolean> CONNECTION_WARN_INIT_ERROR =
       new TypedDriverOption<>(DefaultDriverOption.CONNECTION_WARN_INIT_ERROR, GenericType.BOOLEAN);
@@ -600,7 +607,15 @@ public String toString() {
   public static final TypedDriverOption<Boolean> CONTROL_CONNECTION_AGREEMENT_WARN =
       new TypedDriverOption<>(
           DefaultDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, GenericType.BOOLEAN);
-  /** Whether to forcibly try original contacts if no live nodes are available */
+  /**
+   * Whether to append the original contact points to the control-connection reconnection plan,
+   * after the live nodes reported by the load balancing policy (defaults to {@code true}).
+   *
+   * <p>Contact points are appended as-is (unresolved hostnames); each is expanded to all of its
+   * current DNS IPs at connection time, which is also the driver's DNS re-resolution mechanism. The
+   * append is skipped for topology monitors that re-resolve node addresses themselves (such as the
+   * cloud/proxy monitors).
+   */
   public static final TypedDriverOption<Boolean> CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS =
       new TypedDriverOption<>(
           DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, GenericType.BOOLEAN);
@@ -664,7 +679,16 @@ public String toString() {
   /** The coalescer reschedule interval. */
   public static final TypedDriverOption<Duration> COALESCER_INTERVAL =
       new TypedDriverOption<>(DefaultDriverOption.COALESCER_INTERVAL, GenericType.DURATION);
-  /** Whether to resolve the addresses passed to `basic.contact-points`. */
+  /**
+   * Whether to resolve the addresses passed to `basic.contact-points`.
+   *
+   * @deprecated Setting this option has no effect. Contact points given in the configuration are
+   *     now always kept as unresolved hostnames and expanded to all of their DNS-mapped IPs lazily
+   *     at connection time. This never applied to programmatic contact points passed to {@code
+   *     SessionBuilder.addContactPoints}, which are used exactly as supplied -- an already-resolved
+   *     address stays bound to that one IP.
+   */
+  @Deprecated
   public static final TypedDriverOption<Boolean> RESOLVE_CONTACT_POINTS =
       new TypedDriverOption<>(DefaultDriverOption.RESOLVE_CONTACT_POINTS, GenericType.BOOLEAN);
   /**

core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java

@@ -18,24 +18,58 @@
 package com.datastax.oss.driver.api.core.metadata;
 
 import edu.umd.cs.findbugs.annotations.NonNull;
-import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 
 /**
  * Encapsulates the information needed to open connections to a node.
  *
  * <p>By default, the driver assumes plain TCP connections, and this is just a wrapper around an
- * {@link InetSocketAddress}. However, more complex deployment scenarios might use a custom
+ * {@link java.net.InetSocketAddress}. However, more complex deployment scenarios might use a custom
  * implementation that contains additional information; for example, if the nodes are accessed
  * through a proxy with SNI routing, an SNI server name is needed in addition to the proxy address.
  */
 public interface EndPoint {
 
   /**
-   * Resolves this instance to a socket address.
+   * Resolves this instance to the socket address connections should be opened to.
    *
    * <p>This will be called each time the driver opens a new connection to the node. The returned
    * address cannot be null.
+   *
+   * <p><b>Returning a hostname is fine, and is how multi-address support works.</b> The returned
+   * address need not be resolved: an {@linkplain java.net.InetSocketAddress#isUnresolved()
+   * unresolved} {@link java.net.InetSocketAddress} is expanded by the driver to <b>every</b>
+   * address the name maps to, and each one is tried in turn until a connection succeeds. That is
+   * what {@code DefaultEndPoint} does for contact points backed by a hostname, so a single
+   * unreachable IP behind a multi-record name no longer fails the connection.
+   *
+   * <p><b>Implementations must not resolve names themselves, and must not block.</b> The driver
+   * calls this from its admin event loop, and it performs the expansion through Netty's configured
+   * {@code AddressResolverGroup} — the same resolver an unresolved address reaches when it is
+   * handed to {@code Bootstrap.connect()}. Looking the name up here instead (for example with
+   * {@link java.net.InetAddress#getAllByName(String)}) would both block that loop and bypass a
+   * custom resolver installed via {@code NettyOptions#afterBootstrapInitialized(Bootstrap)}.
+   *
+   * <p><b>Callers must not assume the returned address is resolved.</b> It is for a node discovered
+   * from {@code system.peers} (built from that node's physical broadcast RPC address) and for the
+   * node the control connection is on (bound to the address that connection reached). It is
+   * <b>not</b> for a node reached through the Cloud SNI proxy, or through a cloud private-endpoint
+   * client route: there the address is the configured hostname, and {@link
+   * java.net.InetSocketAddress#getAddress()} returns {@code null}. Read the host with {@link
+   * java.net.InetSocketAddress#getHostString()}, which yields whichever of the two the address
+   * carries and never triggers a reverse lookup.
+   *
+   * @apiNote <b>Timeout note:</b> when a name expands to several addresses they are tried in
+   *     sequence, so the worst-case time before the node is declared unreachable is N times a full
+   *     attempt — and an attempt is more than a connect. Each address that accepts the TCP
+   *     connection then runs the init handshake, whose steps each arm their own {@code
+   *     advanced.connection.init-query-timeout}; those add up rather than sharing one deadline. An
+   *     address that stalls after accepting the connection can therefore burn {@code
+   *     advanced.connection.connect-timeout} plus several times {@code
+   *     advanced.connection.init-query-timeout} on its own. In practice DNS round-robin entries
+   *     have only a small number of records, so this is rarely a concern, but it is worth bearing
+   *     in mind when configuring timeouts — note also that session initialization has no overall
+   *     deadline of its own.
    */
   @NonNull
   SocketAddress resolve();