FIX: Request html when fetching inline onebox data (discourse#24674)

We do expect to receive html

lib/final_destination.rb

@@ -86,6 +86,7 @@ def initialize(url, opts = nil)
         end
       )
     @stop_at_blocked_pages = @opts[:stop_at_blocked_pages]
+    @extra_headers = @opts[:headers]
   end
 
   def self.connection_timeout
@@ -120,6 +121,7 @@ def request_headers
       "Host" => @uri.hostname + (@include_port_in_host_header ? ":#{@uri.port}" : ""),
     }
 
+    result.merge!(@extra_headers) if @extra_headers
     result["Cookie"] = @cookie if @cookie
 
     result

lib/retrieve_title.rb

@@ -78,6 +78,9 @@ def self.fetch_title(url, max_redirects: nil, initial_https_redirect_ignore_limi
         stop_at_blocked_pages: true,
         max_redirects: max_redirects,
         initial_https_redirect_ignore_limit: initial_https_redirect_ignore_limit,
+        headers: {
+          Accept: "text/html,*/*",
+        },
       )
 
     current = nil

spec/models/topic_link_spec.rb

@@ -1,18 +1,13 @@
 # frozen_string_literal: true
 
 RSpec.describe TopicLink do
-  it { is_expected.to validate_presence_of :url }
-
-  def test_uri
-    URI.parse(Discourse.base_url)
-  end
-
+  let(:test_uri) { URI.parse(Discourse.base_url) }
   fab!(:topic) { Fabricate(:topic, title: "unique topic name") }
-
   fab!(:user) { topic.user }
-
   fab!(:post)
 
+  it { is_expected.to validate_presence_of :url }
+
   it "can't link to the same topic" do
     ftl = TopicLink.new(url: "/t/#{topic.id}", topic_id: topic.id, link_topic_id: topic.id)
     expect(ftl.valid?).to eq(false)
@@ -25,7 +20,7 @@ def test_uri
       # prepare a title for one of the links
       stub_request(:get, non_png).with(
         headers: {
-          "Accept" => "*/*",
+          "Accept" => "text/html,*/*",
           "Accept-Encoding" => "gzip",
           "Host" => "b.com",
         },
@@ -273,6 +268,7 @@ def test_uri
       let(:post) do
         Fabricate(:post, topic: topic, user: user, raw: "<a href='/faq'>faq link here</a>")
       end
+
       before { TopicLink.extract_from(post) }
 
       it "does not extract a link" do