Switched to using the server time for event timestamps (#459)

ref
https://linear.app/ghost/issue/NY-344/switch-to-using-server-time-for-analytics-events

We currently accept the timestamp in each event received, which is
generated from the client's browser, and forward it directly to
Tinybird. To avoid issues with browser clock drift, we've decided to
always use the server time, since we know it is right and can ensure it
is applied consistently. This is also simpler than validating the
timestamp and either rejecting or coercing the timestamp for "invalid"
timestamps.

Author: cmraible

src/app.ts

@@ -2,6 +2,7 @@
 import fastify from 'fastify';
 import {TypeBoxTypeProvider} from '@fastify/type-provider-typebox';
 import loggingPlugin from './plugins/logging';
+import timestampPlugin from './plugins/timestamp';
 import corsPlugin from './plugins/cors';
 import proxyPlugin from './plugins/proxy';
 import hmacValidationPlugin from './plugins/hmac-validation';
@@ -16,28 +17,23 @@ const app = fastify({
     trustProxy: process.env.TRUST_PROXY !== 'false'
 }).withTypeProvider<TypeBoxTypeProvider>();
 
-// Register global validation error handler
+// Global error handler
 app.setErrorHandler(errorHandler());
-
-// Register reply-from plugin
 app.register(replyFrom);
-
-// Register CORS plugin
 app.register(corsPlugin);
-
-// Register logging plugin
 app.register(loggingPlugin);
 
+// Record timestamp request was received as early as possible
+app.register(timestampPlugin);
+
 // Register HMAC validation plugin (before all other business logic)
 app.register(hmacValidationPlugin);
 
-// Register proxy plugin
+// Local proxy endpoint for development/testing
 app.register(proxyPlugin);
 
-// Register v1 routes
+// Register routes
 app.register(v1Routes, {prefix: '/api/v1'});
-
-// Routes
 app.get('/', async () => {
     return 'Hello Ghost Traffic Analytics';
 });

src/plugins/timestamp.ts

@@ -0,0 +1,17 @@
+import {FastifyInstance} from 'fastify';
+import fp from 'fastify-plugin';
+
+declare module 'fastify' {
+    interface FastifyRequest {
+        serverReceivedAt: Date;
+    }
+}
+
+async function timestampPlugin(fastify: FastifyInstance) {
+    // Capture timestamp as early as possible in the request lifecycle.
+    fastify.addHook('onRequest', async (request) => {
+        request.serverReceivedAt = new Date();
+    });
+}
+
+export default fp(timestampPlugin);

src/transformations/page-hit-transformations.ts

@@ -15,7 +15,7 @@ export const pageHitRawPayloadFromRequest = (request: PageHitRequestType): PageH
     };
 
     return {
-        timestamp: request.body.timestamp,
+        timestamp: request.serverReceivedAt.toISOString(),
         action: request.body.action,
         version: request.body.version,
         site_uuid: request.headers['x-site-uuid'],

test/e2e/web_analytics.test.ts

@@ -113,8 +113,11 @@ describe('E2E Tests with Fake Tinybird', () => {
             // Verify the request body was processed and enriched
             const requestBody = wireMock.parseRequestBody(tinybirdRequests[0]);
 
+            // Verify timestamp is server time (not client time)
+            expect(requestBody.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/);
+            expect(requestBody.timestamp).not.toBe(DEFAULT_BODY.timestamp); // Should be overwritten with server time
+
             expect(requestBody).toMatchObject({
-                timestamp: DEFAULT_BODY.timestamp,
                 action: 'page_hit',
                 version: '1',
                 // Should have session_id added by processing
@@ -161,8 +164,11 @@ describe('E2E Tests with Fake Tinybird', () => {
             // Verify the request body was processed and enriched
             const requestBody = wireMock.parseRequestBody(tinybirdRequests[0]);
 
+            // Verify timestamp is server time (not client time)
+            expect(requestBody.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/);
+            expect(requestBody.timestamp).not.toBe(DEFAULT_BODY.timestamp); // Should be overwritten with server time
+
             expect(requestBody).toMatchObject({
-                timestamp: DEFAULT_BODY.timestamp,
                 action: 'page_hit',
                 version: '1',
                 // Should have session_id added by processing

test/unit/transformations/page-hit-transformations.test.ts

@@ -3,9 +3,10 @@ import {pageHitRawPayloadFromRequest} from '../../../src/transformations/page-hi
 import {PageHitRequestType} from '../../../src/schemas';
 
 describe('pageHitRawPayloadFromRequest', () => {
-    function createPageHitRequest() {
+    function createPageHitRequest(serverReceivedAt: Date = new Date()) {
         return {
             ip: '192.168.1.1',
+            serverReceivedAt,
             headers: {
                 'x-site-uuid': '12345678-1234-1234-1234-123456789012',
                 'content-type': 'application/json',
@@ -37,44 +38,43 @@ describe('pageHitRawPayloadFromRequest', () => {
     }
 
     it('should transform request to PageHitRaw with all fields present', () => {
-        const request = createPageHitRequest();
+        const serverTime = new Date('2024-01-15T10:30:00.000Z');
+        const request = createPageHitRequest(serverTime);
         const result = pageHitRawPayloadFromRequest(request);
 
-        expect(result).toEqual({
-            timestamp: '2024-01-01T00:00:00.000Z',
-            action: 'page_hit',
-            version: '1',
-            site_uuid: '12345678-1234-1234-1234-123456789012',
-            payload: {
-                event_id: 'test-event-id',
-                member_uuid: 'member-uuid-123',
-                member_status: 'free',
-                post_uuid: 'post-uuid-456',
-                post_type: 'post',
-                locale: 'en-US',
-                location: 'homepage',
-                referrer: 'https://google.com',
-                parsedReferrer: {
-                    source: 'google',
-                    medium: 'organic',
-                    url: 'https://google.com'
-                },
-                pathname: '/blog/post',
-                href: 'https://example.com/blog/post',
-                utm_source: null,
-                utm_medium: null,
-                utm_campaign: null,
-                utm_term: null,
-                utm_content: null,
-                meta: {
-                    received_timestamp: null
-                }
+        expect(result.timestamp).toBe(serverTime.toISOString());
+        expect(result.action).toBe('page_hit');
+        expect(result.version).toBe('1');
+        expect(result.site_uuid).toBe('12345678-1234-1234-1234-123456789012');
+        expect(result.payload).toEqual({
+            event_id: 'test-event-id',
+            member_uuid: 'member-uuid-123',
+            member_status: 'free',
+            post_uuid: 'post-uuid-456',
+            post_type: 'post',
+            locale: 'en-US',
+            location: 'homepage',
+            referrer: 'https://google.com',
+            parsedReferrer: {
+                source: 'google',
+                medium: 'organic',
+                url: 'https://google.com'
             },
+            pathname: '/blog/post',
+            href: 'https://example.com/blog/post',
+            utm_source: null,
+            utm_medium: null,
+            utm_campaign: null,
+            utm_term: null,
+            utm_content: null,
             meta: {
-                ip: '192.168.1.1',
-                'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
+                received_timestamp: null
             }
         });
+        expect(result.meta).toEqual({
+            ip: '192.168.1.1',
+            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
+        });
     });
 
     describe('Event ID', () => {
@@ -287,8 +287,10 @@ describe('pageHitRawPayloadFromRequest', () => {
     });
 
     it('should handle complex real-world request', () => {
+        const serverTime = new Date('2024-03-15T14:30:25.123Z');
         const request = {
             ip: '203.0.113.42',
+            serverReceivedAt: serverTime,
             headers: {
                 'x-site-uuid': 'c7929de8-27d7-404e-b714-0fc774f701e6',
                 'content-type': 'application/json',
@@ -320,7 +322,7 @@ describe('pageHitRawPayloadFromRequest', () => {
 
         const result = pageHitRawPayloadFromRequest(request);
 
-        expect(result.timestamp).toBe('2024-03-15T14:30:25.123Z');
+        expect(result.timestamp).toBe(serverTime.toISOString());
         expect(result.action).toBe('page_hit');
         expect(result.version).toBe('1');
         expect(result.site_uuid).toBe('c7929de8-27d7-404e-b714-0fc774f701e6');