Use endpoint as default connection option (ADR-119)

This implements ADR-119[1], which specifies the client connection
options to update requests to the endpoints implemented as part of
ADR-042[2].

The endpoint may be one of the following:

* a routing policy name (such as `main`)
* a nonprod routing policy name (such as `nonprod:sandbox`)
* a FQDN such as `foo.example.com`

The endpoint option is not valid with any of environment, restHost or
realtimeHost, but we still intend to support the legacy options.

If the client has been configured to use any of these legacy options,
then they should continue to work in the same way, using the same
primary and fallback hostnames.

If the client has not been explicitly configured, then the hostnames
will change to the new `ably.net` domain when the package is upgraded.

[1] https://ably.atlassian.net/wiki/spaces/ENG/pages/3428810778/ADR-119+ClientOptions+for+new+DNS+structure
[2] https://ably.atlassian.net/wiki/spaces/ENG/pages/1791754276/ADR-042+DNS+Restructure

Author: surminus

lib/ably/modules/ably.rb

@@ -4,19 +4,26 @@
 #
 # @see file:README.md README
 module Ably
-  # Fallback hosts to use when a connection to rest/realtime.ably.io is not possible due to
+  # Fallback hosts to use when a connection to main.realtime.ably.net is not possible due to
   # network failures either at the client, between the client and Ably, within an Ably data center, or at the IO domain registrar
   # see https://ably.com/docs/client-lib-development-guide/features/#RSC15a
   #
-  FALLBACK_DOMAIN = 'ably-realtime.com'.freeze
+  PROD_FALLBACK_DOMAIN = 'ably-realtime.com'.freeze
+  NONPROD_FALLBACK_DOMAIN = 'ably-realtime-nonprod.com'.freeze
+
   FALLBACK_IDS = %w(a b c d e).freeze
 
-  # Default production fallbacks a.ably-realtime.com ... e.ably-realtime.com
-  FALLBACK_HOSTS = FALLBACK_IDS.map { |host| "#{host}.#{FALLBACK_DOMAIN}".freeze }.freeze
+  # Default production fallbacks main.a.fallback.ably-realtime.com ... main.e.fallback.ably-realtime.com
+  FALLBACK_HOSTS = FALLBACK_IDS.map { |host| "main.#{host}.fallback.#{PROD_FALLBACK_DOMAIN}".freeze }.freeze
+
+  # Prod fallback suffixes a.fallback.ably-realtime.com ... e.fallback.ably-realtime.com
+  PROD_FALLBACKS_SUFFIXES = FALLBACK_IDS.map do |host|
+    "#{host}.fallback.#{PROD_FALLBACK_DOMAIN}".freeze
+  end.freeze
 
-  # Custom environment default fallbacks {ENV}-a-fallback.ably-realtime.com ... {ENV}-a-fallback.ably-realtime.com
-  CUSTOM_ENVIRONMENT_FALLBACKS_SUFFIXES = FALLBACK_IDS.map do |host|
-    "-#{host}-fallback.#{FALLBACK_DOMAIN}".freeze
+  # Nonprod fallback suffixes a.fallback.ably-realtime-nonprod.com ... e.fallback.ably-realtime-nonprod.com
+  NONPROD_FALLBACKS_SUFFIXES = FALLBACK_IDS.map do |host|
+    "#{host}.fallback.#{NONPROD_FALLBACK_DOMAIN}".freeze
   end.freeze
 
   INTERNET_CHECK = {

lib/ably/realtime/client.rb

@@ -14,8 +14,6 @@ class Client
       extend Forwardable
       using Ably::Util::AblyExtensions
 
-      DOMAIN = 'realtime.ably.io'
-
       # A {Aby::Realtime::Channels} object.
       #
       # @spec RTC3, RTS1
@@ -73,7 +71,7 @@ class Client
       def_delegators :auth, :client_id, :auth_options
       def_delegators :@rest_client, :encoders
       def_delegators :@rest_client, :use_tls?, :protocol, :protocol_binary?
-      def_delegators :@rest_client, :environment, :custom_host, :custom_port, :custom_tls_port
+      def_delegators :@rest_client, :endpoint, :environment, :custom_host, :custom_port, :custom_tls_port
       def_delegators :@rest_client, :log_level
       def_delegators :@rest_client, :options
 
@@ -289,10 +287,34 @@ def publish(channel_name, name, data = nil, attributes = {}, &success_block)
         end
       end
 
-      # @!attribute [r] endpoint
+      # @!attribute [r] hostname
+      # @return [String] The primary hostname to connect to Ably
+      def hostname
+        if endpoint.include?('.') || endpoint.include?('::') || endpoint == 'localhost'
+          return endpoint
+        end
+
+        if endpoint.start_with?('nonprod:')
+          "#{endpoint.gsub('nonprod:', '')}.realtime.#{root_domain}"
+        else
+          "#{endpoint}.realtime.#{root_domain}"
+        end
+      end
+
+      # @!attribute [r] root_domain
+      # @return [String] The root domain used in the hostname
+      def root_domain
+        if endpoint.start_with?('nonprod:')
+          'ably-nonprod.net'
+        else
+          'ably.net'
+        end
+      end
+
+      # @!attribute [r] uri
       # @return [URI::Generic] Default Ably Realtime endpoint used for all requests
-      def endpoint
-        endpoint_for_host(custom_realtime_host || [environment, DOMAIN].compact.join('-'))
+      def uri
+        uri_for_host(custom_realtime_host || hostname)
       end
 
       # (see Ably::Rest::Client#register_encoder)
@@ -322,8 +344,8 @@ def disable_automatic_connection_recovery
       # @api private
       def fallback_endpoint
         unless defined?(@fallback_endpoints) && @fallback_endpoints
-          @fallback_endpoints = fallback_hosts.shuffle.map { |fallback_host| endpoint_for_host(fallback_host) }
-          @fallback_endpoints << endpoint # Try the original host last if all fallbacks have been used
+          @fallback_endpoints = fallback_hosts.shuffle.map { |fallback_host| uri_for_host(fallback_host) }
+          @fallback_endpoints << uri # Try the original host last if all fallbacks have been used
         end
 
         fallback_endpoint_index = connection.manager.retry_count_for_state(:disconnected) + connection.manager.retry_count_for_state(:suspended) - 1
@@ -341,7 +363,8 @@ def device
       end
 
       private
-      def endpoint_for_host(host)
+
+      def uri_for_host(host)
         port = if use_tls?
           custom_tls_port
         else

lib/ably/realtime/connection.rb

@@ -173,7 +173,7 @@ def initialize(client, options)
         @state         = STATE(state_machine.current_state)
         @manager       = ConnectionManager.new(self)
 
-        @current_host = client.endpoint.host
+        @current_host = client.uri.host
 
         reset_client_msg_serial
       end
@@ -396,12 +396,12 @@ def determine_host
             @current_host = if internet_is_up_result
               client.fallback_endpoint.host
             else
-              client.endpoint.host
+              client.uri.host
             end
             yield current_host
           end
         else
-          @current_host = client.endpoint.host
+          @current_host = client.uri.host
           yield current_host
         end
       end
@@ -496,8 +496,8 @@ def create_websocket_transport
                 end
               end
 
-              url = URI(client.endpoint).tap do |endpoint|
-                endpoint.query = URI.encode_www_form(url_params)
+              url = URI(client.uri).tap do |uri|
+                uri.query = URI.encode_www_form(url_params)
               end
 
               determine_host do |host|

lib/ably/rest/client.rb

@@ -18,9 +18,6 @@ class Client
       extend Forwardable
       using Ably::Util::AblyExtensions
 
-      # Default Ably domain for REST
-      DOMAIN = 'rest.ably.io'
-
       MAX_MESSAGE_SIZE = 65536 # See spec TO3l8
       MAX_FRAME_SIZE = 524288 # See spec TO3l8
 
@@ -43,7 +40,11 @@ class Client
 
       def_delegators :auth, :client_id, :auth_options
 
-      # Custom environment to use such as 'sandbox' when testing the client library against an alternate Ably environment
+      # The hostname used to connect to Ably
+      # @return [String]
+      attr_reader :endpoint
+
+      # Custom environment to use such as 'sandbox' when testing the client library against an alternate Ably environment (deprecated)
       # @return [String]
       attr_reader :environment
 
@@ -135,7 +136,8 @@ class Client
       # @option options [String]                  :token               Token string or {Models::TokenDetails} used to authenticate requests
       # @option options [String]                  :token_details       {Models::TokenDetails} used to authenticate requests
       # @option options [Boolean]                 :use_token_auth      Will force Basic Auth if set to false, and Token auth if set to true
-      # @option options [String]                  :environment         Specify 'sandbox' when testing the client library against an alternate Ably environment
+      # @option options [String]                  :endpoint            Specify a routing policy or fully-qualified domain name to connect to Ably
+      # @option options [String]                  :environment         Specify 'sandbox' when testing the client library against an alternate Ably environment (deprecated)
       # @option options [Symbol]                  :protocol            (:msgpack) Protocol used to communicate with Ably, :json and :msgpack currently supported
       # @option options [Boolean]                 :use_binary_protocol (true) When true will use the MessagePack binary protocol, when false it will use JSON encoding. This option will overide :protocol option
       # @option options [Logger::Severity,Symbol] :log_level           (Logger::WARN) Log level for the standard Logger that outputs to STDOUT. Can be set to :fatal (Logger::FATAL), :error (Logger::ERROR), :warn (Logger::WARN), :info (Logger::INFO), :debug (Logger::DEBUG) or :none
@@ -188,8 +190,6 @@ def initialize(options)
         @agent               = options.delete(:agent) || Ably::AGENT
         @realtime_client     = options.delete(:realtime_client)
         @tls                 = options.delete_with_default(:tls, true)
-        @environment         = options.delete(:environment) # nil is production
-        @environment         = nil if [:production, 'production'].include?(@environment)
         @protocol            = options.delete(:protocol) || :msgpack
         @debug_http          = options.delete(:debug_http)
         @log_level           = options.delete(:log_level) || ::Logger::WARN
@@ -203,18 +203,25 @@ def initialize(options)
         @max_frame_size      = options.delete(:max_frame_size) || MAX_FRAME_SIZE
         @idempotent_rest_publishing = options.delete_with_default(:idempotent_rest_publishing, true)
 
+        @environment         = options.delete(:environment) # nil is production
+        @environment         = nil if [:production, 'production'].include?(@environment)
+        @endpoint            = environment || options.delete_with_default(:endpoint, 'main')
+
         if options[:fallback_hosts_use_default] && options[:fallback_hosts]
           raise ArgumentError, "fallback_hosts_use_default cannot be set to try when fallback_hosts is also provided"
         end
+
         @fallback_hosts = case
         when options.delete(:fallback_hosts_use_default)
           Ably::FALLBACK_HOSTS
         when options_fallback_hosts = options.delete(:fallback_hosts)
           options_fallback_hosts
         when custom_host || options[:realtime_host] || custom_port || custom_tls_port
           []
-        when environment
-          CUSTOM_ENVIRONMENT_FALLBACKS_SUFFIXES.map { |host| "#{environment}#{host}" }
+        when endpoint.start_with?('nonprod:')
+          NONPROD_FALLBACKS_SUFFIXES.map { |host| "#{endpoint.gsub('nonprod:', '')}.#{host}" }
+        when endpoint != 'main'
+          PROD_FALLBACKS_SUFFIXES.map { |host| "#{endpoint}.#{host}" }
         else
           Ably::FALLBACK_HOSTS
         end
@@ -426,10 +433,34 @@ def push
         @push ||= Push.new(self)
       end
 
-      # @!attribute [r] endpoint
+      # @!attribute [r] hostname
+      # @return [String] The primary hostname to connect to Ably
+      def hostname
+        if endpoint.include?('.') || endpoint.include?('::') || endpoint == 'localhost'
+          return endpoint
+        end
+
+        if endpoint.start_with?('nonprod:')
+          "#{endpoint.gsub('nonprod:', '')}.realtime.#{root_domain}"
+        else
+          "#{endpoint}.realtime.#{root_domain}"
+        end
+      end
+
+      # @!attribute [r] root_domain
+      # @return [String] The root domain used in the hostname
+      def root_domain
+        if endpoint.start_with?('nonprod:')
+          'ably-nonprod.net'
+        else
+          'ably.net'
+        end
+      end
+
+      # @!attribute [r] uri
       # @return [URI::Generic] Default Ably REST endpoint used for all requests
-      def endpoint
-        endpoint_for_host(custom_host || [@environment, DOMAIN].compact.join('-'))
+      def uri
+        uri_for_host(custom_host || hostname)
       end
 
       # @!attribute [r] logger
@@ -480,7 +511,7 @@ def connection(options = {})
         if options[:use_fallback]
           fallback_connection
         else
-          @connection ||= Faraday.new(endpoint.to_s, connection_options)
+          @connection ||= Faraday.new(uri.to_s, connection_options)
         end
       end
 
@@ -493,7 +524,7 @@ def connection(options = {})
       # @api private
       def fallback_connection
         unless defined?(@fallback_connections) && @fallback_connections
-          @fallback_connections = fallback_hosts.shuffle.map { |host| Faraday.new(endpoint_for_host(host).to_s, connection_options) }
+          @fallback_connections = fallback_hosts.shuffle.map { |host| Faraday.new(uri_for_host(host).to_s, connection_options) }
         end
         @fallback_index ||= 0
 
@@ -653,7 +684,7 @@ def reauthorize_on_authorization_failure
         end
       end
 
-      def endpoint_for_host(host)
+      def uri_for_host(host)
         port = if use_tls?
           custom_tls_port
         else

spec/acceptance/realtime/client_spec.rb

@@ -234,7 +234,7 @@
     context '#request (#RSC19*)' do
       let(:client_options) { default_options.merge(key: api_key) }
       let(:device_id) { random_str }
-      let(:endpoint) { subject.rest_client.endpoint }
+      let(:uri) { subject.rest_client.uri }
 
       context 'get' do
         it 'returns an HttpPaginatedResponse object' do
@@ -287,7 +287,7 @@
 
       context 'post', :webmock do
         before do
-          stub_request(:delete, "#{endpoint}/push/deviceRegistrations/#{device_id}/resetUpdateToken").
+          stub_request(:delete, "#{uri}/push/deviceRegistrations/#{device_id}/resetUpdateToken").
             to_return(status: 200, body: '{}', headers: { 'Content-Type' => 'application/json' })
         end
 
@@ -301,7 +301,7 @@
 
       context 'delete', :webmock do
         before do
-          stub_request(:delete, "#{endpoint}/push/channelSubscriptions?deviceId=#{device_id}").
+          stub_request(:delete, "#{uri}/push/channelSubscriptions?deviceId=#{device_id}").
             to_return(status: 200, body: '{}', headers: { 'Content-Type' => 'application/json' })
         end
 
@@ -317,7 +317,7 @@
         let(:body_params) { { 'metadata' => { 'key' => 'value' } } }
 
         before do
-          stub_request(:patch, "#{endpoint}/push/deviceRegistrations/#{device_id}")
+          stub_request(:patch, "#{uri}/push/deviceRegistrations/#{device_id}")
             .with(body: serialize_body(body_params, protocol))
             .to_return(status: 200, body: '{}', headers: { 'Content-Type' => 'application/json' })
         end
@@ -341,7 +341,7 @@
         end
 
         before do
-          stub_request(:put, "#{endpoint}/push/deviceRegistrations/#{device_id}")
+          stub_request(:put, "#{uri}/push/deviceRegistrations/#{device_id}")
             .with(body: serialize_body(body_params, protocol))
             .to_return(status: 200, body: '{}', headers: { 'Content-Type' => 'application/json' })
         end

spec/acceptance/realtime/connection_failures_spec.rb

@@ -930,7 +930,7 @@ def send_disconnect_message
                   original_method.call(*args, &block)
                 end
                 connection.once(:connected) do
-                  host = "#{"#{environment}-" if environment && environment.to_s != 'production'}#{Ably::Realtime::Client::DOMAIN}"
+                  host = client.hostname
                   expect(hosts.first).to eql(host)
                   expect(hosts.length).to eql(1)
                   stop_reactor
@@ -1469,13 +1469,13 @@ def kill_connection_transport_and_prevent_valid_resume
       end
 
       context 'with non-production environment' do
-        let(:environment)    { 'sandbox' }
-        let(:expected_host)  { "#{environment}-#{Ably::Realtime::Client::DOMAIN}" }
+        let(:environment)    { 'nonprod:sandbox' }
+        let(:expected_host)  { "sandbox.realtime.ably-nonprod.net" }
         let(:client_options) { timeout_options.merge(environment: environment) }
 
         context ':fallback_hosts_use_default is unset' do
           let(:max_time_in_state_for_tests) { 8 }
-          let(:expected_hosts) { Ably::CUSTOM_ENVIRONMENT_FALLBACKS_SUFFIXES.map { |suffix| "#{environment}#{suffix}" } + [expected_host] }
+          let(:expected_hosts) { Ably::NONPROD_FALLBACKS_SUFFIXES.map { |suffix| "#{environment.gsub("nonprod:", "")}.#{suffix}" } + [expected_host] }
           let(:fallback_hosts_used) { Array.new }
 
           it 'uses fallback hosts by default' do
@@ -1568,7 +1568,7 @@ def kill_connection_transport_and_prevent_valid_resume
           stub_const 'Ably::FALLBACK_HOSTS', custom_hosts
         end
 
-        let(:expected_host)  { Ably::Realtime::Client::DOMAIN }
+        let(:expected_host)  { 'main.realtime.ably.net' }
         let(:client_options) { timeout_options.merge(environment: nil) }
 
         let(:fallback_hosts_used) { Array.new }

spec/acceptance/realtime/connection_spec.rb

@@ -29,7 +29,7 @@
 
       context 'current_host' do
         it 'is available immediately after the client is instanced' do
-          expect(connection.current_host.to_s).to match(/\.ably\.io$/)
+          expect(connection.current_host.to_s).to match(/\.realtime\.ably[-nonprod]*\.net$/)
           stop_reactor
         end
       end

spec/acceptance/realtime/message_spec.rb

@@ -775,7 +775,7 @@ def publish_and_check_extras(extras)
                 EventMachine.add_timer(0.0001) do
                   connection.transition_state_machine :suspended
                   stub_const 'Ably::FALLBACK_HOSTS', []
-                  allow(client).to receive(:endpoint).and_return(URI::Generic.build(scheme: 'wss', host: 'does.not.exist.com'))
+                  allow(client).to receive(:uri).and_return(URI::Generic.build(scheme: 'wss', host: 'does.not.exist.com'))
                 end
               end
             end

spec/acceptance/realtime/push_admin_spec.rb

@@ -102,7 +102,7 @@
         end
 
         let!(:publish_stub) do
-          stub_request(:post, "#{client.rest_client.endpoint}/push/publish").
+          stub_request(:post, "#{client.rest_client.uri}/push/publish").
             with do |request|
               expect(deserialize_body(request.body, protocol)['recipient']['camelCase']['secondLevelCamelCase']).to eql('val')
               expect(deserialize_body(request.body, protocol)['recipient']).to_not have_key('camel_case')
@@ -135,7 +135,7 @@
             'transportType' => 'ablyChannel',
             'channel' => channel,
             'ablyKey' => api_key,
-            'ablyUrl' => client.rest_client.endpoint.to_s
+            'ablyUrl' => client.rest_client.uri.to_s
           }
         end
         let(:notification_payload) do