Use endpoint as default connection option (ADR-119)

This implements ADR-119[1], which specifies the client connection
options to update requests to the endpoints implemented as part of
ADR-042[2].

The endpoint may be one of the following:

* a routing policy name (such as `main`)
* a nonprod routing policy name (such as `nonprod:sandbox`)
* a FQDN such as `foo.example.com`

The endpoint option is not valid with any of environment, restHost or
realtimeHost, but we still intend to support the legacy options.

If the client has been configured to use any of these legacy options,
then they should continue to work in the same way, using the same
primary and fallback hostnames.

If the client has not been explicitly configured, then the hostnames
will change to the new `ably.net` domain when the package is upgraded.

[1] https://ably.atlassian.net/wiki/spaces/ENG/pages/3428810778/ADR-119+ClientOptions+for+new+DNS+structure
[2] https://ably.atlassian.net/wiki/spaces/ENG/pages/1791754276/ADR-042+DNS+Restructure

Author: surminus

lib/ably/modules/ably.rb

@@ -11,12 +11,12 @@ module Ably
   FALLBACK_DOMAIN = 'ably-realtime.com'.freeze
   FALLBACK_IDS = %w(a b c d e).freeze
 
-  # Default production fallbacks a.ably-realtime.com ... e.ably-realtime.com
-  FALLBACK_HOSTS = FALLBACK_IDS.map { |host| "#{host}.#{FALLBACK_DOMAIN}".freeze }.freeze
+  # Default production fallbacks main.a.fallback.ably-realtime.com ... main.e.fallback.ably-realtime.com
+  FALLBACK_HOSTS = FALLBACK_IDS.map { |host| "main.#{host}.fallback.#{FALLBACK_DOMAIN}".freeze }.freeze
 
-  # Custom environment default fallbacks {ENV}-a-fallback.ably-realtime.com ... {ENV}-a-fallback.ably-realtime.com
+  # Custom environment default fallbacks {ENV}.a.fallback.ably-realtime.com ... {ENV}.e.fallback.ably-realtime.com
   CUSTOM_ENVIRONMENT_FALLBACKS_SUFFIXES = FALLBACK_IDS.map do |host|
-    "-#{host}-fallback.#{FALLBACK_DOMAIN}".freeze
+    ".#{host}.fallback.#{FALLBACK_DOMAIN}".freeze
   end.freeze
 
   INTERNET_CHECK = {

lib/ably/realtime/client.rb

@@ -73,7 +73,7 @@ class Client
       def_delegators :auth, :client_id, :auth_options
       def_delegators :@rest_client, :encoders
       def_delegators :@rest_client, :use_tls?, :protocol, :protocol_binary?
-      def_delegators :@rest_client, :environment, :custom_host, :custom_port, :custom_tls_port
+      def_delegators :@rest_client, :endpoint, :environment, :custom_host, :custom_port, :custom_tls_port
       def_delegators :@rest_client, :log_level
       def_delegators :@rest_client, :options
 
@@ -289,10 +289,34 @@ def publish(channel_name, name, data = nil, attributes = {}, &success_block)
         end
       end
 
-      # @!attribute [r] endpoint
+      # @!attribute [r] hostname
+      # @return [String] The primary hostname to connect to Ably
+      def hostname
+        if endpoint.include?('.') || endpoint.include?('::') || endpoint == 'localhost'
+          return endpoint
+        end
+
+        if endpoint.start_with?('nonprod:')
+          "#{endpoint.gsub('nonprod:', '')}.realtime.#{root_domain}"
+        else
+          "#{endpoint}.realtime.#{root_domain}"
+        end
+      end
+
+      # @!attribute [r] root_domain
+      # @return [String] The root domain used in the hostname
+      def root_domain
+        if endpoint.start_with?('nonprod:')
+          'ably-nonprod.net'
+        else
+          'ably.net'
+        end
+      end
+
+      # @!attribute [r] uri
       # @return [URI::Generic] Default Ably Realtime endpoint used for all requests
-      def endpoint
-        endpoint_for_host(custom_realtime_host || [environment, DOMAIN].compact.join('-'))
+      def uri
+        uri_for_host(custom_realtime_host || hostname)
       end
 
       # (see Ably::Rest::Client#register_encoder)
@@ -341,7 +365,8 @@ def device
       end
 
       private
-      def endpoint_for_host(host)
+
+      def uri_for_host(host)
         port = if use_tls?
           custom_tls_port
         else

lib/ably/realtime/connection.rb

@@ -173,7 +173,7 @@ def initialize(client, options)
         @state         = STATE(state_machine.current_state)
         @manager       = ConnectionManager.new(self)
 
-        @current_host = client.endpoint.host
+        @current_host = client.uri.host
 
         reset_client_msg_serial
       end
@@ -396,12 +396,12 @@ def determine_host
             @current_host = if internet_is_up_result
               client.fallback_endpoint.host
             else
-              client.endpoint.host
+              client.uri.host
             end
             yield current_host
           end
         else
-          @current_host = client.endpoint.host
+          @current_host = client.uri.host
           yield current_host
         end
       end
@@ -496,8 +496,8 @@ def create_websocket_transport
                 end
               end
 
-              url = URI(client.endpoint).tap do |endpoint|
-                endpoint.query = URI.encode_www_form(url_params)
+              url = URI(client.uri).tap do |uri|
+                uri.query = URI.encode_www_form(url_params)
               end
 
               determine_host do |host|

lib/ably/rest/client.rb

@@ -43,7 +43,11 @@ class Client
 
       def_delegators :auth, :client_id, :auth_options
 
-      # Custom environment to use such as 'sandbox' when testing the client library against an alternate Ably environment
+      # The hostname used to connect to Ably
+      # @return [String]
+      attr_reader :endpoint
+
+      # Custom environment to use such as 'sandbox' when testing the client library against an alternate Ably environment (deprecated)
       # @return [String]
       attr_reader :environment
 
@@ -135,7 +139,8 @@ class Client
       # @option options [String]                  :token               Token string or {Models::TokenDetails} used to authenticate requests
       # @option options [String]                  :token_details       {Models::TokenDetails} used to authenticate requests
       # @option options [Boolean]                 :use_token_auth      Will force Basic Auth if set to false, and Token auth if set to true
-      # @option options [String]                  :environment         Specify 'sandbox' when testing the client library against an alternate Ably environment
+      # @option options [String]                  :endpoint            Specify a routing policy or fully-qualified domain name to connect to Ably
+      # @option options [String]                  :environment         Specify 'sandbox' when testing the client library against an alternate Ably environment (deprecated)
       # @option options [Symbol]                  :protocol            (:msgpack) Protocol used to communicate with Ably, :json and :msgpack currently supported
       # @option options [Boolean]                 :use_binary_protocol (true) When true will use the MessagePack binary protocol, when false it will use JSON encoding. This option will overide :protocol option
       # @option options [Logger::Severity,Symbol] :log_level           (Logger::WARN) Log level for the standard Logger that outputs to STDOUT. Can be set to :fatal (Logger::FATAL), :error (Logger::ERROR), :warn (Logger::WARN), :info (Logger::INFO), :debug (Logger::DEBUG) or :none
@@ -188,8 +193,6 @@ def initialize(options)
         @agent               = options.delete(:agent) || Ably::AGENT
         @realtime_client     = options.delete(:realtime_client)
         @tls                 = options.delete_with_default(:tls, true)
-        @environment         = options.delete(:environment) # nil is production
-        @environment         = nil if [:production, 'production'].include?(@environment)
         @protocol            = options.delete(:protocol) || :msgpack
         @debug_http          = options.delete(:debug_http)
         @log_level           = options.delete(:log_level) || ::Logger::WARN
@@ -203,18 +206,27 @@ def initialize(options)
         @max_frame_size      = options.delete(:max_frame_size) || MAX_FRAME_SIZE
         @idempotent_rest_publishing = options.delete_with_default(:idempotent_rest_publishing, true)
 
+        @environment         = options.delete(:environment) # nil is production
+        @environment         = nil if [:production, 'production'].include?(@environment)
+        @endpoint            = environment || options.delete_with_default(:endpoint, 'main')
+
         if options[:fallback_hosts_use_default] && options[:fallback_hosts]
           raise ArgumentError, "fallback_hosts_use_default cannot be set to try when fallback_hosts is also provided"
         end
+
         @fallback_hosts = case
         when options.delete(:fallback_hosts_use_default)
           Ably::FALLBACK_HOSTS
         when options_fallback_hosts = options.delete(:fallback_hosts)
           options_fallback_hosts
         when custom_host || options[:realtime_host] || custom_port || custom_tls_port
           []
-        when environment
-          CUSTOM_ENVIRONMENT_FALLBACKS_SUFFIXES.map { |host| "#{environment}#{host}" }
+        when endpoint
+          if endpoint.start_with?('nonprod:')
+            CUSTOM_ENVIRONMENT_FALLBACKS_SUFFIXES.map { |host| "#{endpoint.gsub('nonprod:', '')}.#{host}" }
+          else
+            CUSTOM_ENVIRONMENT_FALLBACKS_SUFFIXES.map { |host| "#{endpoint}.#{host}" }
+          end
         else
           Ably::FALLBACK_HOSTS
         end
@@ -426,10 +438,34 @@ def push
         @push ||= Push.new(self)
       end
 
-      # @!attribute [r] endpoint
+      # @!attribute [r] hostname
+      # @return [String] The primary hostname to connect to Ably
+      def hostname
+        if endpoint.include?('.') || endpoint.include?('::') || endpoint == 'localhost'
+          return endpoint
+        end
+
+        if endpoint.start_with?('nonprod:')
+          "#{endpoint.gsub('nonprod:', '')}.realtime.#{root_domain}"
+        else
+          "#{endpoint}.realtime.#{root_domain}"
+        end
+      end
+
+      # @!attribute [r] root_domain
+      # @return [String] The root domain used in the hostname
+      def root_domain
+        if endpoint.start_with?('nonprod:')
+          'ably-nonprod.net'
+        else
+          'ably.net'
+        end
+      end
+
+      # @!attribute [r] uri
       # @return [URI::Generic] Default Ably REST endpoint used for all requests
-      def endpoint
-        endpoint_for_host(custom_host || [@environment, DOMAIN].compact.join('-'))
+      def uri
+        uri_for_host(custom_host || hostname)
       end
 
       # @!attribute [r] logger
@@ -480,7 +516,7 @@ def connection(options = {})
         if options[:use_fallback]
           fallback_connection
         else
-          @connection ||= Faraday.new(endpoint.to_s, connection_options)
+          @connection ||= Faraday.new(uri.to_s, connection_options)
         end
       end
 
@@ -493,7 +529,7 @@ def connection(options = {})
       # @api private
       def fallback_connection
         unless defined?(@fallback_connections) && @fallback_connections
-          @fallback_connections = fallback_hosts.shuffle.map { |host| Faraday.new(endpoint_for_host(host).to_s, connection_options) }
+          @fallback_connections = fallback_hosts.shuffle.map { |host| Faraday.new(uri_for_host(host).to_s, connection_options) }
         end
         @fallback_index ||= 0
 
@@ -653,7 +689,7 @@ def reauthorize_on_authorization_failure
         end
       end
 
-      def endpoint_for_host(host)
+      def uri_for_host(host)
         port = if use_tls?
           custom_tls_port
         else

spec/acceptance/realtime/client_spec.rb

@@ -234,7 +234,7 @@
     context '#request (#RSC19*)' do
       let(:client_options) { default_options.merge(key: api_key) }
       let(:device_id) { random_str }
-      let(:endpoint) { subject.rest_client.endpoint }
+      let(:uri) { subject.rest_client.uri }
 
       context 'get' do
         it 'returns an HttpPaginatedResponse object' do
@@ -287,7 +287,7 @@
 
       context 'post', :webmock do
         before do
-          stub_request(:delete, "#{endpoint}/push/deviceRegistrations/#{device_id}/resetUpdateToken").
+          stub_request(:delete, "#{uri}/push/deviceRegistrations/#{device_id}/resetUpdateToken").
             to_return(status: 200, body: '{}', headers: { 'Content-Type' => 'application/json' })
         end
 
@@ -301,7 +301,7 @@
 
       context 'delete', :webmock do
         before do
-          stub_request(:delete, "#{endpoint}/push/channelSubscriptions?deviceId=#{device_id}").
+          stub_request(:delete, "#{uri}/push/channelSubscriptions?deviceId=#{device_id}").
             to_return(status: 200, body: '{}', headers: { 'Content-Type' => 'application/json' })
         end
 
@@ -317,7 +317,7 @@
         let(:body_params) { { 'metadata' => { 'key' => 'value' } } }
 
         before do
-          stub_request(:patch, "#{endpoint}/push/deviceRegistrations/#{device_id}")
+          stub_request(:patch, "#{uri}/push/deviceRegistrations/#{device_id}")
             .with(body: serialize_body(body_params, protocol))
             .to_return(status: 200, body: '{}', headers: { 'Content-Type' => 'application/json' })
         end
@@ -341,7 +341,7 @@
         end
 
         before do
-          stub_request(:put, "#{endpoint}/push/deviceRegistrations/#{device_id}")
+          stub_request(:put, "#{uri}/push/deviceRegistrations/#{device_id}")
             .with(body: serialize_body(body_params, protocol))
             .to_return(status: 200, body: '{}', headers: { 'Content-Type' => 'application/json' })
         end

spec/acceptance/realtime/message_spec.rb

@@ -775,7 +775,7 @@ def publish_and_check_extras(extras)
                 EventMachine.add_timer(0.0001) do
                   connection.transition_state_machine :suspended
                   stub_const 'Ably::FALLBACK_HOSTS', []
-                  allow(client).to receive(:endpoint).and_return(URI::Generic.build(scheme: 'wss', host: 'does.not.exist.com'))
+                  allow(client).to receive(:uri).and_return(URI::Generic.build(scheme: 'wss', host: 'does.not.exist.com'))
                 end
               end
             end

spec/acceptance/realtime/push_admin_spec.rb

@@ -102,7 +102,7 @@
         end
 
         let!(:publish_stub) do
-          stub_request(:post, "#{client.rest_client.endpoint}/push/publish").
+          stub_request(:post, "#{client.rest_client.uri}/push/publish").
             with do |request|
               expect(deserialize_body(request.body, protocol)['recipient']['camelCase']['secondLevelCamelCase']).to eql('val')
               expect(deserialize_body(request.body, protocol)['recipient']).to_not have_key('camel_case')
@@ -135,7 +135,7 @@
             'transportType' => 'ablyChannel',
             'channel' => channel,
             'ablyKey' => api_key,
-            'ablyUrl' => client.rest_client.endpoint.to_s
+            'ablyUrl' => client.rest_client.uri.to_s
           }
         end
         let(:notification_payload) do

spec/acceptance/rest/auth_spec.rb

@@ -61,7 +61,7 @@ def request_body_includes(request, protocol, key, val)
       end
 
       it 'creates a TokenRequest automatically and sends it to Ably to obtain a token', webmock: true do
-        token_request_stub = stub_request(:post, "#{client.endpoint}/keys/#{key_name}/requestToken").
+        token_request_stub = stub_request(:post, "#{client.uri}/keys/#{key_name}/requestToken").
           to_return(status: 201, body: serialize_body({}, protocol), headers: { 'Content-Type' => content_type })
         expect(auth).to receive(:create_token_request).and_call_original
         auth.request_token
@@ -90,7 +90,7 @@ def coerce_if_time_value(field_name, value, params = {})
 
           let(:token_response) { {} }
           let!(:request_token_stub) do
-            stub_request(:post, "#{client.endpoint}/keys/#{key_name}/requestToken").
+            stub_request(:post, "#{client.uri}/keys/#{key_name}/requestToken").
               with do |request|
                 request_body_includes(request, protocol, token_param, coerce_if_time_value(token_param, random, multiply: 1000))
               end.to_return(
@@ -121,7 +121,7 @@ def coerce_if_time_value(field_name, value, params = {})
 
         let(:token_response) { {} }
         let!(:request_token_stub) do
-          stub_request(:post, "#{client.endpoint}/keys/#{key_name}/requestToken").
+          stub_request(:post, "#{client.uri}/keys/#{key_name}/requestToken").
             with do |request|
               request_body_includes(request, protocol, 'mac', mac)
             end.to_return(
@@ -151,7 +151,7 @@ def coerce_if_time_value(field_name, value, params = {})
 
         let(:token_response) { {} }
         let!(:request_token_stub) do
-          stub_request(:post, "#{client.endpoint}/keys/#{key_name}/requestToken").
+          stub_request(:post, "#{client.uri}/keys/#{key_name}/requestToken").
             with do |request|
               request_body_includes(request, protocol, 'mac', mac)
             end.to_return(
@@ -293,7 +293,7 @@ def coerce_if_time_value(field_name, value, params = {})
         let(:auth_url_content_type) { 'application/json' }
 
         let!(:request_token_stub) do
-          stub_request(:post, "#{client.endpoint}/keys/#{key_name}/requestToken").
+          stub_request(:post, "#{client.uri}/keys/#{key_name}/requestToken").
             with do |request|
               request_body_includes(request, protocol, 'key_name', key_name)
             end.to_return(

spec/acceptance/rest/base_spec.rb

@@ -14,7 +14,7 @@
     let(:body_value) { [as_since_epoch(now)] }
 
     before do
-      stub_request(:get, "#{client.endpoint}/time").
+      stub_request(:get, "#{client.uri}/time").
         with(:headers => { 'Accept' => mime }).
         to_return(:status => 200, :body => request_body, :headers => { 'Content-Type' => mime })
     end
@@ -87,7 +87,7 @@
         let(:error_response) { '{ "error": { "statusCode": 500, "code": 50000, "message": "Internal error" } }' }
 
         before do
-          (client.fallback_hosts.map { |host| "https://#{host}" } + [client.endpoint]).each do |host|
+          (client.fallback_hosts.map { |host| "https://#{host}" } + [client.uri]).each do |host|
             stub_request(:get, "#{host}/time")
               .to_return(:status => 500, :body => error_response, :headers => { 'Content-Type' => 'application/json' })
           end
@@ -100,7 +100,7 @@
 
       describe '500 server error without a valid JSON response body', :webmock do
         before do
-          (client.fallback_hosts.map { |host| "https://#{host}" } + [client.endpoint]).each do |host|
+          (client.fallback_hosts.map { |host| "https://#{host}" } + [client.uri]).each do |host|
             stub_request(:get, "#{host}/time").
             to_return(:status => 500, :headers => { 'Content-Type' => 'application/json' })
           end
@@ -121,15 +121,15 @@
         @token_requests = 0
         @publish_attempts = 0
 
-        stub_request(:post, "#{client.endpoint}/keys/#{key_name}/requestToken").to_return do
+        stub_request(:post, "#{client.uri}/keys/#{key_name}/requestToken").to_return do
           @token_requests += 1
           {
             :body => public_send("token_#{@token_requests}").merge(expires: (Time.now.to_i + 60) * 1000).to_json,
             :headers => { 'Content-Type' => 'application/json' }
           }
         end
 
-        stub_request(:post, "#{client.endpoint}/channels/#{channel}/publish").to_return do
+        stub_request(:post, "#{client.uri}/channels/#{channel}/publish").to_return do
           @publish_attempts += 1
           if [1, 3].include?(@publish_attempts)
             { status: 201, :body => '[]', :headers => { 'Content-Type' => 'application/json' } }