Skip to content

Commit c81cb16

Browse files
aerosolzoldar
andauthored
Snippet integration verification (plausible#4106)
* Allow running browserless.io locally * Compile tailwind classes based on extra/ too * Add browserless runtime configuration * Ignore verification events on ingestion * Improve extracting HTML text in tests * Update dependencies - Floki will be used on production to parse site contents - Req will be used to handle redundant stuff like retrying etc. * Add shuttle SVG to generic components Later on we'll use it to indicate verification errors * Connect live socket & allow skipping awaiting the first pageview * Connect live socket in general settings * Implement verification checks & diagnostics * Stub remote services with Req for testing * Change snippet screen copy * Update tracker script, so that: 1. headless browsers aren't ignored if `window.__plausible` is defined 2. callback optionally supplies the event response HTTP status This will be later used to check whether the server acknowledged the verification event. * Implement LiveView verification UI * Embed the verification UIs into settings and onboarding * Implement browserless puppeteer verification script It: - tries to visit the site - defines window.__plausible, so the tracker doesn't ignore test events - sends a verification event and instruments the callback - awaits the callback to fire and returns the result * Improve diagnostics for CSP Only report CSP error if the snippet is already found * Put verification behind a feature flag/env setting * Contact Us hint only for Enterprise Edition * For headless code, use JS context instead of EEx interpolation * Update diagnostics test with WordPress scenarios * Shorten exception/throw interception * Rename test * Tidy up * Bust URL always on headless check * Update moduledoc * Detect official Plausible WordPress Plugin and act accordingly on diagnostics interoperation * Stop using 'rating' in favour of 'interpretation' * Only report CSP error if no proxy is likely * Update CHANGELOG * Allow event-* attributes on snippet elements * Improve naive GTM detection, not to confuse it with GA4 * Update lib/plausible/verification.ex Co-authored-by: Adrian Gruntkowski <[email protected]> * Update test/plausible/site/verification/checks_test.exs Co-authored-by: Adrian Gruntkowski <[email protected]> * s/perform_wrapped/perform_safe * Update lib/plausible/verification/checks/installation.ex Co-authored-by: Adrian Gruntkowski <[email protected]> * Remove garbage --------- Co-authored-by: Adrian Gruntkowski <[email protected]>
1 parent 5881f1c commit c81cb16

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+2838
-34
lines changed

CHANGELOG.md

+2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file.
55

66
### Added
77

8+
- Snippet integration verification
9+
810
### Removed
911

1012
### Changed

Makefile

+3
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ postgres-prod: ## Start a container with the same version of postgres as the one
3737
postgres-stop: ## Stop and remove the postgres container
3838
docker stop plausible_db && docker rm plausible_db
3939

40+
browserless:
41+
docker run -e "TOKEN=dummy_token" -p 3000:3000 --network host ghcr.io/browserless/chromium
42+
4043
minio: ## Start a transient container with a recent version of minio (s3)
4144
docker run -d --rm -p 10000:10000 -p 10001:10001 --name plausible_minio minio/minio server /data --address ":10000" --console-address ":10001"
4245
while ! docker exec plausible_minio mc alias set local http://localhost:10000 minioadmin minioadmin; do sleep 1; done

assets/tailwind.config.js

+3-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ module.exports = {
55
content: [
66
"./js/**/*.js",
77
"../lib/*_web.ex",
8-
"../lib/*_web/**/*.*ex"
8+
"../lib/*_web/**/*.*ex",
9+
"../extra/*_web.ex",
10+
"../extra/*_web/**/*.*ex"
911
],
1012
safelist: [
1113
// PlausibleWeb.StatsView.stats_container_class/1 uses this class

config/.env.dev

+2
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,5 @@ S3_REGION=us-east-1
2828
S3_ENDPOINT=http://localhost:10000
2929
S3_EXPORTS_BUCKET=dev-exports
3030
S3_IMPORTS_BUCKET=dev-imports
31+
32+
VERIFICATION_ENABLED=true

config/runtime.exs

+9
Original file line numberDiff line numberDiff line change
@@ -701,6 +701,15 @@ config :plausible, Plausible.PromEx,
701701
grafana: :disabled,
702702
metrics_server: :disabled
703703

704+
config :plausible, Plausible.Verification,
705+
enabled?:
706+
get_var_from_path_or_env(config_dir, "VERIFICATION_ENABLED", "false")
707+
|> String.to_existing_atom()
708+
709+
config :plausible, Plausible.Verification.Checks.Installation,
710+
token: get_var_from_path_or_env(config_dir, "BROWSERLESS_TOKEN", "dummy_token"),
711+
endpoint: get_var_from_path_or_env(config_dir, "BROWSERLESS_ENDPOINT", "http://0.0.0.0:3000")
712+
704713
if not is_selfhost do
705714
site_default_ingest_threshold =
706715
case System.get_env("SITE_DEFAULT_INGEST_THRESHOLD") do

config/test.exs

+10
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,13 @@ config :ex_money, api_module: Plausible.ExchangeRateMock
3131
config :plausible, Plausible.Ingestion.Counters, enabled: false
3232

3333
config :plausible, Oban, testing: :manual
34+
35+
config :plausible, Plausible.Verification.Checks.FetchBody,
36+
req_opts: [
37+
plug: {Req.Test, Plausible.Verification.Checks.FetchBody}
38+
]
39+
40+
config :plausible, Plausible.Verification.Checks.Installation,
41+
req_opts: [
42+
plug: {Req.Test, Plausible.Verification.Checks.Installation}
43+
]

extra/lib/plausible/ingestion/event/revenue.ex

+2-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ defmodule Plausible.Ingestion.Event.Revenue do
2222
}
2323

2424
matching_goal.currency != revenue_source.currency ->
25-
converted = Money.to_currency!(revenue_source, matching_goal.currency)
25+
converted =
26+
Money.to_currency!(revenue_source, matching_goal.currency)
2627

2728
%{
2829
revenue_source_amount: Money.to_decimal(revenue_source),

lib/plausible/ingestion/event.ex

+14
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ defmodule Plausible.Ingestion.Event do
2121
salts: nil,
2222
changeset: nil
2323

24+
@verification_user_agent Plausible.Verification.user_agent()
25+
2426
@type drop_reason() ::
2527
:bot
2628
| :spam_referrer
@@ -31,6 +33,7 @@ defmodule Plausible.Ingestion.Event do
3133
| :site_country_blocklist
3234
| :site_page_blocklist
3335
| :site_hostname_allowlist
36+
| :verification_agent
3437

3538
@type t() :: %__MODULE__{
3639
domain: String.t() | nil,
@@ -104,6 +107,7 @@ defmodule Plausible.Ingestion.Event do
104107

105108
defp pipeline() do
106109
[
110+
drop_verification_agent: &drop_verification_agent/1,
107111
drop_datacenter_ip: &drop_datacenter_ip/1,
108112
drop_shield_rule_hostname: &drop_shield_rule_hostname/1,
109113
drop_shield_rule_page: &drop_shield_rule_page/1,
@@ -167,6 +171,16 @@ defmodule Plausible.Ingestion.Event do
167171
struct!(event, clickhouse_session_attrs: Map.merge(event.clickhouse_session_attrs, attrs))
168172
end
169173

174+
defp drop_verification_agent(%__MODULE__{} = event) do
175+
case event.request.user_agent do
176+
@verification_user_agent ->
177+
drop(event, :verification_agent)
178+
179+
_ ->
180+
event
181+
end
182+
end
183+
170184
defp drop_datacenter_ip(%__MODULE__{} = event) do
171185
case event.request.ip_classification do
172186
"dc_ip" ->

lib/plausible/verification.ex

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
defmodule Plausible.Verification do
2+
@moduledoc """
3+
Module defining the user-agent used for site verification.
4+
"""
5+
use Plausible
6+
7+
@feature_flag :verification
8+
9+
def enabled?(user) do
10+
enabled_via_config? =
11+
:plausible |> Application.fetch_env!(__MODULE__) |> Keyword.fetch!(:enabled?)
12+
13+
enabled_for_user? = not is_nil(user) and FunWithFlags.enabled?(@feature_flag, for: user)
14+
enabled_via_config? or enabled_for_user?
15+
end
16+
17+
on_ee do
18+
def user_agent() do
19+
"Plausible Verification Agent - if abused, contact [email protected]"
20+
end
21+
else
22+
def user_agent() do
23+
"Plausible Community Edition"
24+
end
25+
end
26+
end

lib/plausible/verification/check.ex

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
defmodule Plausible.Verification.Check do
2+
@moduledoc """
3+
Behaviour to be implemented by specific site verification checks.
4+
`friendly_name()` doesn't necessarily reflect the actual check description,
5+
it serves as a user-facing message grouping mechanism, to prevent frequent message flashing when checks rotate often.
6+
Each check operates on `state()` and is expected to return it, optionally modified, by all means.
7+
`perform_safe/1` is used to guarantee no exceptions are thrown by faulty implementations, not to interrupt LiveView.
8+
"""
9+
@type state() :: Plausible.Verification.State.t()
10+
@callback friendly_name() :: String.t()
11+
@callback perform(state()) :: state()
12+
13+
defmacro __using__(_) do
14+
quote do
15+
import Plausible.Verification.State
16+
17+
alias Plausible.Verification.Checks
18+
alias Plausible.Verification.State
19+
alias Plausible.Verification.Diagnostics
20+
21+
require Logger
22+
23+
@behaviour Plausible.Verification.Check
24+
25+
def perform_safe(state) do
26+
perform(state)
27+
catch
28+
_, e ->
29+
Logger.error(
30+
"Error running check #{inspect(__MODULE__)} on #{state.url}: #{inspect(e)}"
31+
)
32+
33+
put_diagnostics(state, service_error: true)
34+
end
35+
end
36+
end
37+
end

lib/plausible/verification/checks.ex

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
defmodule Plausible.Verification.Checks do
2+
@moduledoc """
3+
Checks that are performed during site verification.
4+
Each module defined in `@checks` implements the `Plausible.Verification.Check` behaviour.
5+
Checks are normally run asynchronously, except when synchronous execution is optionally required
6+
for tests. Slowdowns can be optionally added, the user doesn't benefit from running the checks too quickly.
7+
8+
In async execution, each check notifies the caller by sending a message to it.
9+
"""
10+
alias Plausible.Verification.Checks
11+
alias Plausible.Verification.State
12+
13+
require Logger
14+
15+
@checks [
16+
Checks.FetchBody,
17+
Checks.CSP,
18+
Checks.ScanBody,
19+
Checks.Snippet,
20+
Checks.SnippetCacheBust,
21+
Checks.Installation
22+
]
23+
24+
def run(url, data_domain, opts \\ []) do
25+
checks = Keyword.get(opts, :checks, @checks)
26+
report_to = Keyword.get(opts, :report_to, self())
27+
async? = Keyword.get(opts, :async?, true)
28+
slowdown = Keyword.get(opts, :slowdown, 500)
29+
30+
if async? do
31+
Task.start_link(fn -> do_run(url, data_domain, checks, report_to, slowdown) end)
32+
else
33+
do_run(url, data_domain, checks, report_to, slowdown)
34+
end
35+
end
36+
37+
def interpret_diagnostics(%State{} = state) do
38+
Plausible.Verification.Diagnostics.interpret(state.diagnostics, state.url)
39+
end
40+
41+
defp do_run(url, data_domain, checks, report_to, slowdown) do
42+
init_state = %State{url: url, data_domain: data_domain, report_to: report_to}
43+
44+
state =
45+
Enum.reduce(
46+
checks,
47+
init_state,
48+
fn check, state ->
49+
state
50+
|> notify_start(check, slowdown)
51+
|> check.perform_safe()
52+
end
53+
)
54+
55+
notify_verification_end(state, slowdown)
56+
end
57+
58+
defp notify_start(state, check, slowdown) do
59+
if is_pid(state.report_to) do
60+
if is_integer(slowdown) and slowdown > 0, do: :timer.sleep(slowdown)
61+
send(state.report_to, {:verification_check_start, {check, state}})
62+
end
63+
64+
state
65+
end
66+
67+
defp notify_verification_end(state, slowdown) do
68+
if is_pid(state.report_to) do
69+
if is_integer(slowdown) and slowdown > 0, do: :timer.sleep(slowdown)
70+
send(state.report_to, {:verification_end, state})
71+
end
72+
73+
state
74+
end
75+
end
+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
defmodule Plausible.Verification.Checks.CSP do
2+
@moduledoc """
3+
Scans the Content Security Policy header to ensure that the Plausible domain is allowed.
4+
See `Plausible.Verification.Checks` for the execution sequence.
5+
"""
6+
use Plausible.Verification.Check
7+
8+
@impl true
9+
def friendly_name, do: "We're visiting your site to ensure that everything is working correctly"
10+
11+
@impl true
12+
def perform(%State{assigns: %{headers: headers}} = state) do
13+
case headers["content-security-policy"] do
14+
[policy] ->
15+
directives = String.split(policy, ";")
16+
17+
allowed? =
18+
Enum.any?(directives, fn directive ->
19+
String.contains?(directive, PlausibleWeb.Endpoint.host())
20+
end)
21+
22+
if allowed? do
23+
state
24+
else
25+
put_diagnostics(state, disallowed_via_csp?: true)
26+
end
27+
28+
_ ->
29+
state
30+
end
31+
end
32+
33+
def perform(state), do: state
34+
end
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
defmodule Plausible.Verification.Checks.FetchBody do
2+
@moduledoc """
3+
Fetches the body of the site and extracts the HTML document, if available, for
4+
further processing.
5+
See `Plausible.Verification.Checks` for the execution sequence.
6+
"""
7+
use Plausible.Verification.Check
8+
9+
@impl true
10+
def friendly_name, do: "We're visiting your site to ensure that everything is working correctly"
11+
12+
@impl true
13+
def perform(%State{url: "https://" <> _ = url} = state) do
14+
fetch_body_opts = Application.get_env(:plausible, __MODULE__)[:req_opts] || []
15+
16+
opts =
17+
Keyword.merge(
18+
[
19+
base_url: url,
20+
max_redirects: 2,
21+
connect_options: [timeout: 4_000],
22+
receive_timeout: 4_000,
23+
max_retries: 3,
24+
retry_log_level: :warning
25+
],
26+
fetch_body_opts
27+
)
28+
29+
req = Req.new(opts)
30+
31+
case Req.get(req) do
32+
{:ok, %Req.Response{status: status, body: body} = response}
33+
when is_binary(body) and status in 200..299 ->
34+
extract_document(state, response)
35+
36+
_ ->
37+
state
38+
end
39+
end
40+
41+
defp extract_document(state, response) when byte_size(response.body) <= 500_000 do
42+
with true <- html?(response),
43+
{:ok, document} <- Floki.parse_document(response.body) do
44+
state
45+
|> assign(raw_body: response.body, document: document, headers: response.headers)
46+
|> put_diagnostics(body_fetched?: true)
47+
else
48+
_ ->
49+
state
50+
end
51+
end
52+
53+
defp extract_document(state, response) when byte_size(response.body) > 500_000 do
54+
state
55+
end
56+
57+
defp html?(%Req.Response{headers: headers}) do
58+
headers
59+
|> Map.get("content-type", "")
60+
|> List.wrap()
61+
|> List.first()
62+
|> String.contains?("text/html")
63+
end
64+
end

0 commit comments

Comments
 (0)