claytongentry · mayel · Jul 31, 2018 · Jul 31, 2018 · Feb 7, 2019 · Feb 12, 2019
diff --git a/.envrc b/.envrc
@@ -0,0 +1,3 @@
+if has guix; then
+    use guix erlang elixir elixir-hex just
+fi
diff --git a/.tool-versions b/.tool-versions
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## v.0.6.0
+
+Forked from Furlex with many changes/additions - changelog TODO
+
 ## v.0.5.0
 
 - Updates Floki ([queer](https://github.com/queer))
@@ -41,12 +45,12 @@
 
 ## v.0.2.2
 
-- Furlex now supports passing HTTP options to Furlex.unfurl/2.
+- Unfurl now supports passing HTTP options to Unfurl.unfurl/2.
 - `:depth` config has been transformed to a `:group_keys?` boolean.
 
 ## v.0.2.1
 
-- Add status code to %Furlex{} structure (thanks [abitdodgy](https://github.com/abitdodgy))
+- Add status code to Unfurl's return (thanks [abitdodgy](https://github.com/abitdodgy))
 - Fix compatibility with Phoenix 1.3 (thanks, again, [abitdodgy](https://github.com/abitdodgy)!)
 
 ## v.0.2.0

diff --git a/LICENSE.md b/LICENSE.md
@@ -1,3 +1,4 @@
+Copyright 2020 Bonfire Networks
 Copyright 2017 Clayton Gentry
 
 Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/README.md b/README.md
@@ -1,127 +1,163 @@
-# Furlex
+# Unfurl
 
-Furlex is a [structured data](https://moz.com/learn/seo/schema-structured-data) extraction tool written in Elixir.
+Unfurl is a [structured data](https://moz.com/learn/seo/schema-structured-data) extraction tool written in Elixir.
 
-It currently supports unfurling oEmbed, Twitter Card, Facebook Open Graph, JSON-LD
-and plain ole' HTML `<meta />` data out of any url you supply.
+It currently supports unfurling oEmbed, Open Graph (Facebook), Twitter Card, JSON-LD, rel-me, favicons, and plain ole' HTML `<meta />` data out of any url you supply.
 
 ## Installation
 
-Add `:furlex` to your list of dependencies in `mix.exs`:
+Add `:unfurl` to your list of dependencies in `mix.exs`:O
 
 ```elixir
 def deps do
-  [{:furlex, "~> 0.5.0"}]
+  [{:unfurl, "~> 0.6.0"}]
 end
 ```
 
-Then run `$ mix deps.get`. Also add `:furlex` to your applications list:
+Then run `$ mix deps.get`. Also add `:unfurl` to your applications list:
 
 ```elixir
 def application do
-  [applications: [:furlex]]
+  [applications: [:unfurl]]
 end
 ```
 
-[Jason](http://github.com/michalmuskala/jason) is the default json library in Furlex. You can however configure Furlex to use another library. For example:
+[Jason](http://github.com/michalmuskala/jason) is the default json library in Unfurl. You can however configure Unfurl to use another library. For example:
 
 ```elixir
-config :furlex, :json_library, YourLibraryOfChoice
+config :unfurl, :json_library, YourLibraryOfChoice
 ```
 
 ## Usage
 
-To unfurl a url, simply pass it to `Furlex.unfurl/1`
+To unfurl a url, simply pass it to `Unfurl.unfurl/1`
 
 ```elixir
-iex(1)> Furlex.unfurl "https://www.youtube.com/watch?v=Gh6H7Md_L2k"
+iex(1)> Unfurl.unfurl "https://www.youtube.com/watch?v=Gh6H7Md_L2k"
 {:ok,
- %Furlex{canonical_url: "https://www.youtube.com/watch?v=Gh6H7Md_L2k",
-  facebook: %{"fb:app_id" => "87741124305",
-    "og:description" => "Watch the full episode: https://www.thisoldhouse.com/watch/ask-toh-future-house-offerman Ask This Old House host Kevin O’Connor visits Nick Offerman in Los A...",
-    "og:image" => "https://i.ytimg.com/vi/Gh6H7Md_L2k/maxresdefault.jpg",
-    "og:site_name" => "YouTube",
-    "og:title" => "Touring Nick Offerman’s Wood Shop", "og:type" => "video",
-    "og:url" => "https://www.youtube.com/watch?v=Gh6H7Md_L2k",
-    "og:video:height" => ["720", "720"],
-    "og:video:secure_url" => ["https://www.youtube.com/embed/Gh6H7Md_L2k",
-     "https://www.youtube.com/v/Gh6H7Md_L2k?version=3&autohide=1"],
-    "og:video:type" => ["text/html", "application/x-shockwave-flash"],
-    "og:video:url" => ["https://www.youtube.com/embed/Gh6H7Md_L2k",
-     "http://www.youtube.com/v/Gh6H7Md_L2k?version=3&autohide=1"],
-    "og:video:width" => ["1280", "1280"]},
-  json_ld: [%{"@context" => "http://schema.org", "@type" => "BreadcrumbList",
-     "itemListElement" => [%{"@type" => "ListItem",
-        "item" => %{"@id" => "http://www.youtube.com/user/thisoldhouse",
-          "name" => "This Old House"}, "position" => 1}]}],
-  oembed: %{"author_name" => "This Old House",
-    "author_url" => "https://www.youtube.com/user/thisoldhouse",
-    "height" => 270,
-    "html" => "<iframe width=\"480\" height=\"270\" src=\"https://www.youtube.com/embed/Gh6H7Md_L2k?feature=oembed\" frameborder=\"0\" gesture=\"media\" allow=\"encrypted-media\" allowfullscreen></iframe>",
-    "provider_name" => "YouTube", "provider_url" => "https://www.youtube.com/",
-    "thumbnail_height" => 360,
-    "thumbnail_url" => "https://i.ytimg.com/vi/Gh6H7Md_L2k/hqdefault.jpg",
-    "thumbnail_width" => 480, "title" => "Touring Nick Offerman’s Wood Shop",
-    "type" => "video", "version" => "1.0", "width" => 480},
-  other: %{"description" => "Watch the full episode: https://www.thisoldhouse.com/watch/ask-toh-future-house-offerman Ask This Old House host Kevin O’Connor visits Nick Offerman in Los A...",
-    "keywords" => "this old house, how-to, home improvement, Episode, TV Show, DIY, Ask This Old House, Nick Offerman, Kevin O'Connor, woodworking, wood shop",
-    "theme-color" => "#ff0000",
-    "title" => "Touring Nick Offerman’s Wood Shop"},
-  status_code: 200,
-  twitter: %{"twitter:app:id:googleplay" => "com.google.android.youtube",
-    "twitter:app:id:ipad" => "544007664",
-    "twitter:app:id:iphone" => "544007664",
-    "twitter:app:name:googleplay" => "YouTube",
-    "twitter:app:name:ipad" => "YouTube",
-    "twitter:app:name:iphone" => "YouTube",
-    "twitter:app:url:googleplay" => "https://www.youtube.com/watch?v=Gh6H7Md_L2k",
-    "twitter:app:url:ipad" => "vnd.youtube://www.youtube.com/watch?v=Gh6H7Md_L2k&feature=applinks",
-    "twitter:app:url:iphone" => "vnd.youtube://www.youtube.com/watch?v=Gh6H7Md_L2k&feature=applinks",
-    "twitter:card" => "player",
-    "twitter:description" => "Watch the full episode: https://www.thisoldhouse.com/watch/ask-toh-future-house-offerman Ask This Old House host Kevin O’Connor visits Nick Offerman in Los A...",
-    "twitter:image" => "https://i.ytimg.com/vi/Gh6H7Md_L2k/maxresdefault.jpg",
-    "twitter:player" => "https://www.youtube.com/embed/Gh6H7Md_L2k",
-    "twitter:player:height" => "720", "twitter:player:width" => "1280",
-    "twitter:site" => "@youtube",
-    "twitter:title" => "Touring Nick Offerman’s Wood Shop",
-    "twitter:url" => "https://www.youtube.com/watch?v=Gh6H7Md_L2k"}}}
+ %{
+   other: %{
+     "description" => "Ask This Old House host Kevin O’Connor visits Nick Offerman in Los Angeles to tour the comedian’s woodworking shop.SUBSCRIBE to This Old House: http://bit.ly...",
+     "keywords" => "this old house, how-to, home improvement, Episode, TV Show, DIY, Ask This Old House, Nick Offerman, Kevin O'Connor, woodworking, wood shop, Los Angeles, Comedian, This Old House, Home Improvement, DIY Ideas, Renovation, Renovation Ideas, How To Fix, How To Install, How To Build, Kevin o’connor, kevin o'connor house, kevin o'connor this old house, kevin o'connor ask this old house, kevin o'connor interview",
+     "theme-color" => "rgba(255, 255, 255, 0.98)",
+     "title" => ["Touring Nick Offerman’s Wood Shop | Ask This Old House",
+      "Touring Nick Offerman’s Wood Shop | Ask This Old House - YouTube"]
+   },
+   canonical_url: nil,
+   facebook: %{
+     "description" => "Ask This Old House host Kevin O’Connor visits Nick Offerman in Los Angeles to tour the comedian’s woodworking shop.SUBSCRIBE to This Old House: http://bit.ly...",
+     "fb" => %{"app_id" => "87741124305"},
+     "image" => %{"height" => "720", "width" => "1280"},
+     "site_name" => "YouTube",
+     "title" => "Touring Nick Offerman’s Wood Shop | Ask This Old House",
+     "type" => "video.other",
+     "url" => "https://www.youtube.com/watch?v=Gh6H7Md_L2k",
+     "video" => %{
+       "height" => "720",
+       "secure_url" => "https://www.youtube.com/embed/Gh6H7Md_L2k",
+       "type" => "text/html",
+       "url" => "https://www.youtube.com/embed/Gh6H7Md_L2k",
+       "width" => "1280"
+     }
+   },
+   twitter: %{
+     "app" => %{
+       "id" => %{
+         "googleplay" => "com.google.android.youtube",
+         "ipad" => "544007664",
+         "iphone" => "544007664"
+       },
+       "name" => %{
+         "googleplay" => "YouTube",
+         "ipad" => "YouTube",
+         "iphone" => "YouTube"
+       },
+       "url" => %{
+         "googleplay" => "https://www.youtube.com/watch?v=Gh6H7Md_L2k",
+         "ipad" => "vnd.youtube://www.youtube.com/watch?v=Gh6H7Md_L2k&feature=applinks",
+         "iphone" => "vnd.youtube://www.youtube.com/watch?v=Gh6H7Md_L2k&feature=applinks"
+       }
+     },
+     "card" => "player",
+     "description" => "Ask This Old House host Kevin O’Connor visits Nick Offerman in Los Angeles to tour the comedian’s woodworking shop.SUBSCRIBE to This Old House: http://bit.ly...",
+     "image" => "https://i.ytimg.com/vi/Gh6H7Md_L2k/maxresdefault.jpg",
+     "player" => %{"height" => "720", "width" => "1280"},
+     "site" => "@youtube",
+     "title" => "Touring Nick Offerman’s Wood Shop | Ask This Old House",
+     "url" => "https://www.youtube.com/watch?v=Gh6H7Md_L2k"
+   },
+   oembed: %{
+     "author_name" => "This Old House",
+     "author_url" => "https://www.youtube.com/@thisoldhouse",
+     "height" => 113,
+     "html" => "<iframe width=\"200\" height=\"113\" src=\"https://www.youtube.com/embed/Gh6H7Md_L2k?feature=oembed\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\" referrerpolicy=\"strict-origin-when-cross-origin\" allowfullscreen title=\"Touring Nick Offerman’s Wood Shop | Ask This Old House\"></iframe>",
+     "provider_name" => "YouTube",
+     "provider_url" => "https://www.youtube.com/",
+     "thumbnail_height" => 360,
+     "thumbnail_url" => "https://i.ytimg.com/vi/Gh6H7Md_L2k/hqdefault.jpg",
+     "thumbnail_width" => 480,
+     "title" => "Touring Nick Offerman’s Wood Shop | Ask This Old House",
+     "type" => "video",
+     "version" => "1.0",
+     "width" => 200
+   },
+   json_ld: [
+     %{
+       "@context" => "http://schema.org",
+       "@type" => "BreadcrumbList",
+       "itemListElement" => [
+         %{
+           "@type" => "ListItem",
+           "item" => %{
+             "@id" => "http://www.youtube.com/@thisoldhouse",
+             "name" => "This Old House"
+           },
+           "position" => 1
+         }
+       ]
+     }
+   ],
+   status_code: 200,
+   rel_me: nil,
+   favicon: "https://www.youtube.com/s/desktop/ef8ce500/img/favicon_32x32.png"
+ }}
 ```
 
 ## Configuration
 
-Furlex accepts a few optional configuration parameters.
+Unfurl accepts a few optional configuration parameters.
 
 You may configure additional tags to capture under the Facebook
 OpenGraph and TwitterCard parsers.
 
 ```elixir
-config :furlex, Furlex.Parser.Facebook,
+config :unfurl, Unfurl.Parser.Facebook,
   tags: ~w(my:custom:facebook:tag another:custom:facebook:tag)
 
-config :furlex, Furlex.Parser.Twitter,
+config :unfurl, Unfurl.Parser.Twitter,
   tags: ~w(my:custom:twitter:tag)
 ```
 
-You may also configure the depth of the resulting Furlex map with a `:group_keys?` boolean.
+You may also configure the depth of the resulting Unfurl map with a `:group_keys?` boolean.
 
 ```elixir
-config :furlex, group_keys?: true
+config :unfurl, group_keys?: true
 ```
 
-If this option is set to false or unconfigured, Furlex will return values mapped directly beneath OpenGraph and TwitterCard keys, i.e.
+If this option is set to false or unconfigured, Unfurl will return values mapped directly beneath OpenGraph and TwitterCard keys, i.e.
 
 ```elixir
-%Furlex{twitter: %{
+%{twitter: %{
   "twitter:app:id:googleplay" => "com.google.android.youtube",
   "twitter:app:id:ipad"       => "544007664",
   "twitter:app:id:iphone"     => "544007664"
 }}
 ```
 
-If true, Furlex will return values grouped into colon-delimited map structures, i.e.
+If true, Unfurl will return values grouped into colon-delimited map structures, i.e.
 
 ```elixir
-%Furlex{twitter: %{
+%{twitter: %{
   "twitter" => %{
     "app" => %{
       "id" => %{
@@ -136,7 +172,8 @@ If true, Furlex will return values grouped into colon-delimited map structures,
 
 ## License
 
-Copyright 2017 Clayton Gentry
+Copyright 2020 Bonfire Networks
+Copyright 2017 Clayton Gentry (author of https://www.hex.pm/packages/furlex which Unfurl was forked from)
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

diff --git a/benchmark.exs b/benchmark.exs
@@ -4,8 +4,8 @@
 vimeo = File.read! "./test/fixtures/vimeo.html"
 
 Benchee.run(%{
-  "facebook" => fn -> Furlex.Parser.Facebook.parse(vimeo) end,
-  "twitter" => fn -> Furlex.Parser.Twitter.parse(vimeo) end,
-  "json_ld" => fn -> Furlex.Parser.JsonLD.parse(vimeo) end,
-  "html" => fn -> Furlex.Parser.HTML.parse(vimeo) end
+  "facebook" => fn -> Unfurl.Parser.Facebook.parse(vimeo) end,
+  "twitter" => fn -> Unfurl.Parser.Twitter.parse(vimeo) end,
+  "json_ld" => fn -> Unfurl.Parser.JsonLD.parse(vimeo) end,
+  "html" => fn -> Unfurl.Parser.HTML.parse(vimeo) end
 })
diff --git a/config/config.exs b/config/config.exs
@@ -1,6 +1,13 @@
 # This file is responsible for configuring your application
 # and its dependencies with the aid of the Mix.Config module.
-use Mix.Config
+import Config
+
+config :tesla, adapter: {Tesla.Adapter.Hackney, [recv_timeout: 1_000]}
+
+config :bypass, enable_debug_log: true
+
+config :logger,
+  truncate: :infinity
 
 # This configuration is loaded before any dependency and is restricted
 # to this project. If another project depends on this project, this
@@ -10,11 +17,11 @@ use Mix.Config
 
 # You can configure for your application as:
 #
-#     config :furlex, key: :value
+#     config :unfurl, key: :value
 #
 # And access this configuration in your application as:
 #
-#     Application.get_env(:furlex, :key)
+#     Application.get_env(:unfurl, :key)
 #
 # Or configure a 3rd-party app:
 #

diff --git a/lib/fetcher/fetcher.ex b/lib/fetcher/fetcher.ex
@@ -0,0 +1,55 @@
+defmodule Unfurl.Fetcher do
+  @moduledoc """
+  A module for fetching body data for a given url
+  """
+  use Tesla
+
+  plug Tesla.Middleware.Telemetry
+
+  plug Unfurl.Tesla.Middleware.MaybeFollowRedirects, max_redirects: 5
+
+  import Untangle
+
+  @doc """
+  Fetches a url and extracts the body
+  """
+  @spec fetch(String.t(), List.t()) :: {:ok, String.t(), Integer.t()} | {:error, Atom.t()}
+  def fetch(url, opts \\ [])
+
+  def fetch(url, opts) when is_binary(url) do
+    URI.parse(url)
+    |> fetch(opts)
+  end
+
+  def fetch(%URI{} = url, opts) do
+    case url do
+      %URI{host: nil, path: nil} ->
+        error(url, "Tried to fetch an invalid URL")
+
+      %URI{scheme: "doi"} ->
+        error(url, "Tried to fetch an invalid URL")
+
+      %URI{scheme: nil, host: nil, path: host_detected_as_path} ->
+        do_fetch("http://#{host_detected_as_path}", opts)
+
+      %URI{} ->
+        do_fetch(to_string(url), opts)
+    end
+  end
+
+  defp do_fetch(url, opts \\ []) when is_binary(url) do
+    case get(url, opts) do
+      {:ok, %{body: body, status: status_code}} -> {:ok, body, status_code}
+      other -> other
+    end
+  rescue
+    e in Tesla.Mock.Error ->
+      error(e)
+
+    e in ArgumentError ->
+      error(e)
+
+    e in CaseClauseError ->
+      error(e)
+  end
+end