From 672d70cbac692fc15a6d4abe6750f6426e9b0251 Mon Sep 17 00:00:00 2001 From: jevansnyc Date: Wed, 25 Jun 2025 19:18:39 -0500 Subject: [PATCH 01/11] GAM module, gam testing endpoints, working locally --- crates/common/Cargo.toml | 3 + crates/common/src/gam.rs | 416 +++++++++++++++++++++++++++++++++ crates/common/src/lib.rs | 1 + crates/common/src/settings.rs | 16 ++ crates/common/src/templates.rs | 295 ++++++++++++++++++++++- crates/fastly/src/main.rs | 45 ++-- fastly.toml | 10 +- trusted-server.toml | 10 + 8 files changed, 772 insertions(+), 24 deletions(-) create mode 100644 crates/common/src/gam.rs diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 2494f54..6ab791f 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -12,6 +12,7 @@ license = "Apache-2.0" debug = 1 [dependencies] +brotli = "3.3" chrono = "0.4" config = "0.15.11" cookie = "0.18.1" @@ -27,3 +28,5 @@ serde_json = "1.0.91" sha2 = "0.10.6" tokio = { version = "1.43", features = ["sync", "macros", "io-util", "rt", "time"] } url = "2.4.1" +uuid = { version = "1.0", features = ["v4"] } +urlencoding = "2.1" diff --git a/crates/common/src/gam.rs b/crates/common/src/gam.rs new file mode 100644 index 0000000..3b43672 --- /dev/null +++ b/crates/common/src/gam.rs @@ -0,0 +1,416 @@ +use crate::settings::Settings; +use crate::gdpr::get_consent_from_request; +use fastly::http::{header, Method, StatusCode}; +use fastly::{Error, Request, Response}; +use serde_json::json; +use std::collections::HashMap; +use uuid::Uuid; + +/// GAM request builder for server-side ad requests +pub struct GamRequest { + pub publisher_id: String, + pub ad_units: Vec, + pub page_url: String, + pub correlator: String, + pub prmtvctx: Option, // Permutive context - initially hardcoded, then dynamic + pub user_agent: String, + pub synthetic_id: String, +} + +impl GamRequest { + /// Create a new GAM request with default parameters + pub fn new(settings: &Settings, req: &Request) -> Result { + let correlator = Uuid::new_v4().to_string(); + let page_url = req.get_url().to_string(); + let user_agent = req + .get_header(header::USER_AGENT) + .and_then(|h| h.to_str().ok()) + .unwrap_or("Mozilla/5.0 (compatible; TrustedServer/1.0)") + .to_string(); + + // Get synthetic ID from request headers + let synthetic_id = req + .get_header("X-Synthetic-Trusted-Server") + .and_then(|h| h.to_str().ok()) + .unwrap_or("unknown") + .to_string(); + + Ok(Self { + publisher_id: settings.gam.publisher_id.clone(), + ad_units: settings.gam.ad_units.iter().map(|u| u.name.clone()).collect(), + page_url, + correlator, + prmtvctx: None, // Will be set later with captured value + user_agent, + synthetic_id, + }) + } + + /// Set the Permutive context (initially hardcoded from captured request) + pub fn with_prmtvctx(mut self, prmtvctx: String) -> Self { + self.prmtvctx = Some(prmtvctx); + self + } + + /// Build the GAM request URL for the "Golden URL" replay phase + pub fn build_golden_url(&self) -> String { + // This will be replaced with the actual captured URL from autoblog.com + // For now, using a template based on the captured Golden URL + let mut params = HashMap::new(); + + // Core GAM parameters (based on captured URL) + params.insert("pvsid".to_string(), "3290837576990024".to_string()); // Publisher Viewability ID + params.insert("correlator".to_string(), self.correlator.clone()); + params.insert("eid".to_string(), "31086815,31093089,95353385,31085777,83321072".to_string()); // Event IDs + params.insert("output".to_string(), "ldjh".to_string()); // Important: not 'json' + params.insert("gdfp_req".to_string(), "1".to_string()); + params.insert("vrg".to_string(), "202506170101".to_string()); // Version/Region + params.insert("ptt".to_string(), "17".to_string()); // Page Type + params.insert("impl".to_string(), "fifs".to_string()); // Implementation + + // Ad unit parameters (simplified version of captured format) + params.insert("iu_parts".to_string(), format!("{},{},homepage", self.publisher_id, "trustedserver")); + params.insert("enc_prev_ius".to_string(), "/0/1/2,/0/1/2,/0/1/2".to_string()); + params.insert("prev_iu_szs".to_string(), "320x50|300x250|728x90|970x90|970x250|1x2,320x50|300x250|728x90|970x90|970x250|1x2,320x50|300x250|728x90|970x90|970x250|1x2".to_string()); + params.insert("fluid".to_string(), "height,height,height".to_string()); + + // Browser context (simplified) + params.insert("biw".to_string(), "1512".to_string()); + params.insert("bih".to_string(), "345".to_string()); + params.insert("u_tz".to_string(), "-300".to_string()); + params.insert("u_cd".to_string(), "30".to_string()); + params.insert("u_sd".to_string(), "2".to_string()); + + // Page context + params.insert("url".to_string(), self.page_url.clone()); + params.insert("dt".to_string(), chrono::Utc::now().timestamp_millis().to_string()); + + // Add Permutive context if available (in cust_params like the captured URL) + if let Some(ref prmtvctx) = self.prmtvctx { + let cust_params = format!("permutive={}&puid={}", prmtvctx, self.synthetic_id); + params.insert("cust_params".to_string(), cust_params); + } + + // Build query string + let query_string = params + .iter() + .map(|(k, v)| format!("{}={}", k, urlencoding::encode(v))) + .collect::>() + .join("&"); + + format!("{}?{}", self.get_base_url(), query_string) + } + + /// Get the base GAM server URL + pub fn get_base_url(&self) -> String { + // This will be updated with the actual GAM endpoint from captured request + "https://securepubads.g.doubleclick.net/gampad/ads".to_string() + } + + /// Send the GAM request and return the response + pub async fn send_request(&self, _settings: &Settings) -> Result { + let url = self.build_golden_url(); + println!("Sending GAM request to: {}", url); + + // Create the request + let mut req = Request::new(Method::GET, &url); + + // Set headers to mimic a browser request (using only Fastly-compatible headers) + req.set_header(header::USER_AGENT, &self.user_agent); + req.set_header(header::ACCEPT, "application/json, text/plain, */*"); + req.set_header(header::ACCEPT_LANGUAGE, "en-US,en;q=0.9"); + req.set_header(header::ACCEPT_ENCODING, "gzip, deflate, br"); + req.set_header(header::REFERER, &self.page_url); + req.set_header(header::ORIGIN, &self.page_url); + req.set_header("X-Synthetic-ID", &self.synthetic_id); + + // Send the request to the GAM backend + let backend_name = "gam_backend"; + println!("Sending request to backend: {}", backend_name); + + match req.send(backend_name) { + Ok(mut response) => { + println!("Received GAM response with status: {}", response.get_status()); + + // Log response headers for debugging + println!("GAM Response headers:"); + for (name, value) in response.get_headers() { + println!(" {}: {:?}", name, value); + } + + // Handle response body safely + let body_bytes = response.take_body_bytes(); + let body = match std::str::from_utf8(&body_bytes) { + Ok(body_str) => body_str.to_string(), + Err(e) => { + println!("Warning: Could not read response body as UTF-8: {:?}", e); + + // Try to decompress if it's Brotli compressed + let mut decompressed = Vec::new(); + match brotli::BrotliDecompress(&mut std::io::Cursor::new(&body_bytes), &mut decompressed) { + Ok(_) => { + match std::str::from_utf8(&decompressed) { + Ok(decompressed_str) => { + println!("Successfully decompressed Brotli response: {} bytes", decompressed_str.len()); + decompressed_str.to_string() + } + Err(e2) => { + println!("Warning: Could not read decompressed body as UTF-8: {:?}", e2); + format!("{{\"error\": \"decompression_failed\", \"message\": \"Could not decode decompressed response\", \"original_error\": \"{:?}\"}}", e2) + } + } + } + Err(e2) => { + println!("Warning: Could not decompress Brotli response: {:?}", e2); + // Return a placeholder since we can't parse the binary response + format!("{{\"error\": \"compression_failed\", \"message\": \"Could not decompress response\", \"original_error\": \"{:?}\"}}", e2) + } + } + } + }; + + println!("GAM Response body length: {} bytes", body.len()); + + // For debugging, log first 500 chars of response + if body.len() > 500 { + println!("GAM Response preview: {}...", &body[..500]); + } else { + println!("GAM Response: {}", body); + } + + Ok(Response::from_status(response.get_status()) + .with_header(header::CONTENT_TYPE, "application/json") + .with_header(header::CACHE_CONTROL, "no-store, private") + .with_header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .with_header("X-GAM-Test", "true") + .with_header("X-Synthetic-ID", &self.synthetic_id) + .with_header("X-Correlator", &self.correlator) + .with_header("x-compress-hint", "on") + .with_body(body)) + } + Err(e) => { + println!("Error sending GAM request: {:?}", e); + Err(e.into()) + } + } + } +} + +/// Handle GAM test requests (Phase 1: Capture & Replay) +pub async fn handle_gam_test(settings: &Settings, req: Request) -> Result { + println!("Starting GAM test request handling"); + + // Debug: Log all request headers + println!("GAM Test - All request headers:"); + for (name, value) in req.get_headers() { + println!(" {}: {:?}", name, value); + } + + // Check consent status from cookie (more reliable than header) + let consent = get_consent_from_request(&req).unwrap_or_default(); + let advertising_consent = consent.advertising; + + println!("GAM Test - Consent from cookie: {:?}", consent); + println!("GAM Test - Advertising consent from cookie: {}", advertising_consent); + + // Also check header as fallback + let header_consent = req + .get_header("X-Consent-Advertising") + .and_then(|h| h.to_str().ok()) + .map(|v| v == "true") + .unwrap_or(false); + + println!("GAM Test - Advertising consent from header: {}", header_consent); + + // Use cookie consent as primary, header as fallback + let final_consent = advertising_consent || header_consent; + println!("GAM Test - Final advertising consent: {}", final_consent); + + if !final_consent { + return Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "No advertising consent", + "message": "GAM requests require advertising consent", + "debug": { + "cookie_consent": consent, + "header_consent": header_consent, + "final_consent": final_consent + } + }))?); + } + + // Create GAM request + let gam_req = match GamRequest::new(settings, &req) { + Ok(req) => { + println!("Successfully created GAM request"); + req + } + Err(e) => { + println!("Error creating GAM request: {:?}", e); + return Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "Failed to create GAM request", + "details": format!("{:?}", e) + }))?); + } + }; + + // For Phase 1, we'll use a hardcoded prmtvctx value from captured request + // This will be replaced with the actual value from autoblog.com + let gam_req_with_context = gam_req.with_prmtvctx("129627,137412,138272,139095,139096,139218,141364,143196,143210,143211,143214,143217,144331,144409,144438,144444,144488,144543,144663,144679,144731,144824,144916,145933,146347,146348,146349,146350,146351,146370,146383,146391,146392,146393,146424,146995,147077,147740,148616,148627,148628,149007,150420,150663,150689,150690,150692,150752,150753,150755,150756,150757,150764,150770,150781,150862,154609,155106,155109,156204,164183,164573,165512,166017,166019,166484,166486,166487,166488,166492,166494,166495,166497,166511,167639,172203,172544,173548,176066,178053,178118,178120,178121,178133,180321,186069,199642,199691,202074,202075,202081,233782,238158,adv,bhgp,bhlp,bhgw,bhlq,bhlt,bhgx,bhgv,bhgu,bhhb,rts".to_string()); + + println!("Sending GAM request with correlator: {}", gam_req_with_context.correlator); + + match gam_req_with_context.send_request(settings).await { + Ok(response) => { + println!("GAM request successful"); + Ok(response) + } + Err(e) => { + println!("GAM request failed: {:?}", e); + Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "Failed to send GAM request", + "details": format!("{:?}", e) + }))?) + } + } +} + +/// Handle GAM golden URL replay (for testing captured requests) +pub async fn handle_gam_golden_url(_settings: &Settings, _req: Request) -> Result { + println!("Handling GAM golden URL replay"); + + // This endpoint will be used to test the exact captured URL from autoblog.com + // For now, return a placeholder response + Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "status": "golden_url_replay", + "message": "Ready for captured URL testing", + "next_steps": [ + "1. Capture complete GAM request URL from autoblog.com", + "2. Replace placeholder URL in GamRequest::build_golden_url()", + "3. Test with exact captured parameters" + ] + }))?) +} + +/// Handle GAM custom URL testing (for testing captured URLs directly) +pub async fn handle_gam_custom_url(settings: &Settings, mut req: Request) -> Result { + println!("Handling GAM custom URL test"); + + // Check consent status from cookie + let consent = get_consent_from_request(&req).unwrap_or_default(); + let advertising_consent = consent.advertising; + + if !advertising_consent { + return Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "No advertising consent", + "message": "GAM requests require advertising consent" + }))?); + } + + // Parse the request body to get the custom URL + let body = req.take_body_str(); + let url_data: serde_json::Value = serde_json::from_str(&body) + .map_err(|e| { + println!("Error parsing request body: {:?}", e); + fastly::Error::msg("Invalid JSON in request body") + })?; + + let custom_url = url_data["url"].as_str() + .ok_or_else(|| fastly::Error::msg("Missing 'url' field in request body"))?; + + println!("Testing custom GAM URL: {}", custom_url); + + // Create a request to the custom URL + let mut gam_req = Request::new(Method::GET, custom_url); + + // Set headers to mimic a browser request + gam_req.set_header(header::USER_AGENT, "Mozilla/5.0 (compatible; TrustedServer/1.0)"); + gam_req.set_header(header::ACCEPT, "application/json, text/plain, */*"); + gam_req.set_header(header::ACCEPT_LANGUAGE, "en-US,en;q=0.9"); + gam_req.set_header(header::ACCEPT_ENCODING, "gzip, deflate, br"); + gam_req.set_header(header::REFERER, "https://www.autoblog.com/"); + gam_req.set_header(header::ORIGIN, "https://www.autoblog.com"); + + // Send the request to the GAM backend + let backend_name = "gam_backend"; + println!("Sending custom URL request to backend: {}", backend_name); + + match gam_req.send(backend_name) { + Ok(mut response) => { + println!("Received GAM response with status: {}", response.get_status()); + + // Log response headers for debugging + println!("GAM Response headers:"); + for (name, value) in response.get_headers() { + println!(" {}: {:?}", name, value); + } + + // Handle response body safely + let body_bytes = response.take_body_bytes(); + let body = match std::str::from_utf8(&body_bytes) { + Ok(body_str) => body_str.to_string(), + Err(e) => { + println!("Warning: Could not read response body as UTF-8: {:?}", e); + + // Try to decompress if it's Brotli compressed + let mut decompressed = Vec::new(); + match brotli::BrotliDecompress(&mut std::io::Cursor::new(&body_bytes), &mut decompressed) { + Ok(_) => { + match std::str::from_utf8(&decompressed) { + Ok(decompressed_str) => { + println!("Successfully decompressed Brotli response: {} bytes", decompressed_str.len()); + decompressed_str.to_string() + } + Err(e2) => { + println!("Warning: Could not read decompressed body as UTF-8: {:?}", e2); + format!("{{\"error\": \"decompression_failed\", \"message\": \"Could not decode decompressed response\", \"original_error\": \"{:?}\"}}", e2) + } + } + } + Err(e2) => { + println!("Warning: Could not decompress Brotli response: {:?}", e2); + // Return a placeholder since we can't parse the binary response + format!("{{\"error\": \"compression_failed\", \"message\": \"Could not decompress response\", \"original_error\": \"{:?}\"}}", e2) + } + } + } + }; + + println!("GAM Response body length: {} bytes", body.len()); + + Ok(Response::from_status(response.get_status()) + .with_header(header::CONTENT_TYPE, "application/json") + .with_header(header::CACHE_CONTROL, "no-store, private") + .with_header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .with_header("X-GAM-Test", "true") + .with_header("X-Custom-URL", "true") + .with_header("x-compress-hint", "on") + .with_body_json(&json!({ + "status": "custom_url_test", + "original_url": custom_url, + "response_status": response.get_status().as_u16(), + "response_body": body, + "message": "Custom URL test completed" + }))?) + } + Err(e) => { + println!("Error sending custom GAM request: {:?}", e); + Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "Failed to send custom GAM request", + "details": format!("{:?}", e), + "original_url": custom_url + }))?) + } + } +} \ No newline at end of file diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 7dff572..9687ce3 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -1,5 +1,6 @@ pub mod constants; pub mod cookies; +pub mod gam; pub mod gdpr; pub mod models; pub mod prebid; diff --git a/crates/common/src/settings.rs b/crates/common/src/settings.rs index a3b70c3..fabf694 100644 --- a/crates/common/src/settings.rs +++ b/crates/common/src/settings.rs @@ -15,6 +15,21 @@ pub struct Prebid { pub server_url: String, } +#[derive(Debug, Deserialize)] +#[allow(unused)] +pub struct GamAdUnit { + pub name: String, + pub size: String, +} + +#[derive(Debug, Deserialize)] +#[allow(unused)] +pub struct Gam { + pub publisher_id: String, + pub server_url: String, + pub ad_units: Vec, +} + #[derive(Debug, Deserialize)] #[allow(unused)] pub struct Synthetic { @@ -29,6 +44,7 @@ pub struct Synthetic { pub struct Settings { pub ad_server: AdServer, pub prebid: Prebid, + pub gam: Gam, pub synthetic: Synthetic, } diff --git a/crates/common/src/templates.rs b/crates/common/src/templates.rs index 76f5a65..b01e313 100644 --- a/crates/common/src/templates.rs +++ b/crates/common/src/templates.rs @@ -174,6 +174,9 @@ pub const HTML_TEMPLATE: &str = r#" } function saveConsent(consent) { + // Set the cookie first + document.cookie = `gdpr_consent=${JSON.stringify(consent)}; path=/; max-age=31536000`; // 1 year expiry + fetch('/gdpr/consent', { method: 'POST', headers: { @@ -184,16 +187,20 @@ pub const HTML_TEMPLATE: &str = r#" document.getElementById('gdpr-banner').classList.remove('visible'); document.getElementById('gdpr-preferences').classList.remove('visible'); document.querySelector('.overlay').classList.remove('visible'); - location.reload(); + // Remove the reload - we'll let the page continue with the new consent + }).catch(error => { + console.error('Error saving consent:', error); }); } // Load ads and tracking based on consent window.addEventListener('load', function() { - showGdprBanner(); + const consent = getCookie('gdpr_consent'); + if (!consent) { + showGdprBanner(); + } // Get consent status - const consent = getCookie('gdpr_consent'); const consentData = consent ? JSON.parse(consent) : { advertising: false, functional: false }; // Always make the prebid request, but include consent information @@ -319,3 +326,285 @@ pub const HTML_TEMPLATE: &str = r#" "#; + +pub const GAM_TEST_TEMPLATE: &str = r#" + + + + + + GAM Test - Trusted Server + + + +
+

GAM Test - Headless GPT PoC

+ +
+

📋 Instructions for Capture & Replay Phase

+

Phase 1 Goal: Capture a complete, successful ad request URL from autoblog.com and replay it from our server.

+
    +
  1. Open browser dev tools on autoblog.com
  2. +
  3. Go to Network tab and filter by "g.doubleclick.net"
  4. +
  5. Refresh the page and look for successful ad requests
  6. +
  7. Copy the complete URL with all parameters
  8. +
  9. Use the "Test Golden URL" button below to test it
  10. +
+
+ +
+

Phase 1: Capture & Replay (Golden URL)

+

Test the exact captured URL from autoblog.com to prove network connectivity.

+ +
+

Golden URL Test

+

Paste the captured GAM URL from autoblog.com below and test it:

+
+ +
+ + + +
+
+ +
+

Phase 2: Dynamic Request Building

+

Test dynamic parameter generation with hardcoded prmtvctx value.

+ +
+

Dynamic GAM Request

+

Test server-side GAM request with dynamic correlator and synthetic ID.

+ + +
+
+ +
+

Debug Information

+
+

Request Headers

+
+ +

Synthetic ID Status

+
+ Checking synthetic ID... +
+
+
+
+ + + + +"#; diff --git a/crates/fastly/src/main.rs b/crates/fastly/src/main.rs index bd7dc84..66e3946 100644 --- a/crates/fastly/src/main.rs +++ b/crates/fastly/src/main.rs @@ -8,6 +8,7 @@ use std::env; use trusted_server_common::constants::{SYNTHETIC_HEADER_FRESH, SYNTHETIC_HEADER_TRUSTED_SERVER}; use trusted_server_common::cookies::create_synthetic_cookie; +use trusted_server_common::gam::{handle_gam_test, handle_gam_golden_url, handle_gam_custom_url}; use trusted_server_common::gdpr::{ get_consent_from_request, handle_consent_request, handle_data_subject_request, }; @@ -16,24 +17,41 @@ use trusted_server_common::prebid::PrebidRequest; use trusted_server_common::privacy::PRIVACY_TEMPLATE; use trusted_server_common::settings::Settings; use trusted_server_common::synthetic::{generate_synthetic_id, get_or_generate_synthetic_id}; -use trusted_server_common::templates::HTML_TEMPLATE; +use trusted_server_common::templates::{HTML_TEMPLATE, GAM_TEST_TEMPLATE}; use trusted_server_common::why::WHY_TEMPLATE; #[fastly::main] fn main(req: Request) -> Result { let settings = Settings::new().unwrap(); - println!("Settings {settings:?}"); + + // Print Fastly Service Version only once at the beginning + println!( + "FASTLY_SERVICE_VERSION: {}", + std::env::var("FASTLY_SERVICE_VERSION").unwrap_or_else(|_| String::new()) + ); + + // Print User IP address immediately after Fastly Service Version + let client_ip = req + .get_client_ip_addr() + .map(|ip| ip.to_string()) + .unwrap_or_else(|| "Unknown".to_string()); + println!("User IP: {}", client_ip); + + // Print Settings only once at the beginning + println!("Settings: {settings:?}"); futures::executor::block_on(async { - println!( - "FASTLY_SERVICE_VERSION: {}", - std::env::var("FASTLY_SERVICE_VERSION").unwrap_or_else(|_| String::new()) - ); - match (req.get_method(), req.get_path()) { (&Method::GET, "/") => handle_main_page(&settings, req), (&Method::GET, "/ad-creative") => handle_ad_request(&settings, req), (&Method::GET, "/prebid-test") => handle_prebid_test(&settings, req).await, + (&Method::GET, "/gam-test") => handle_gam_test(&settings, req).await, + (&Method::GET, "/gam-golden-url") => handle_gam_golden_url(&settings, req).await, + (&Method::POST, "/gam-test-custom-url") => handle_gam_custom_url(&settings, req).await, + (&Method::GET, "/gam-test-page") => Ok(Response::from_status(StatusCode::OK) + .with_body(GAM_TEST_TEMPLATE) + .with_header(header::CONTENT_TYPE, "text/html") + .with_header("x-compress-hint", "on")), (&Method::GET, "/gdpr/consent") => handle_consent_request(&settings, req), (&Method::POST, "/gdpr/consent") => handle_consent_request(&settings, req), (&Method::GET, "/gdpr/data") => handle_data_subject_request(&settings, req), @@ -108,11 +126,6 @@ fn get_dma_code(req: &mut Request) -> Option { } fn handle_main_page(settings: &Settings, mut req: Request) -> Result { - println!( - "Using ad_partner_url: {}, counter_store: {}", - settings.ad_server.ad_partner_url, settings.synthetic.counter_store, - ); - log_fastly::init_simple("mylogs", Info); // Add DMA code check to main page as well @@ -321,11 +334,10 @@ fn handle_ad_request(settings: &Settings, mut req: Request) -> Result Result Result Date: Fri, 11 Jul 2025 16:31:48 -0500 Subject: [PATCH 02/11] Syncing GAM functionality files --- .gitignore | 1 + Cargo.lock | 96 ++++++++++++++- crates/common/src/gam.rs | 210 ++++++++++++++++++++++++++++++++- crates/common/src/templates.rs | 129 +++++++++++++++++++- crates/fastly/src/main.rs | 3 +- 5 files changed, 434 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index cd4d019..921ce02 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ src/*.html !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json +.specstory diff --git a/Cargo.lock b/Cargo.lock index ab6bf08..e113ba6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -124,6 +139,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.17.0" @@ -199,7 +235,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom", + "getrandom 0.2.15", "once_cell", "tiny-keccak", ] @@ -649,7 +685,19 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", ] [[package]] @@ -1149,6 +1197,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "regex" version = "1.11.1" @@ -1525,6 +1579,7 @@ checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc" name = "trusted-server-common" version = "0.1.0" dependencies = [ + "brotli", "chrono", "config", "cookie", @@ -1540,6 +1595,8 @@ dependencies = [ "sha2 0.10.8", "tokio", "url", + "urlencoding", + "uuid", ] [[package]] @@ -1590,6 +1647,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf16_iter" version = "1.0.5" @@ -1602,6 +1665,17 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "uuid" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" +dependencies = [ + "getrandom 0.3.3", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "version_check" version = "0.9.5" @@ -1614,6 +1688,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -1804,6 +1887,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags 2.9.0", +] + [[package]] name = "write16" version = "1.0.0" diff --git a/crates/common/src/gam.rs b/crates/common/src/gam.rs index 3b43672..02a1de6 100644 --- a/crates/common/src/gam.rs +++ b/crates/common/src/gam.rs @@ -300,7 +300,7 @@ pub async fn handle_gam_golden_url(_settings: &Settings, _req: Request) -> Resul } /// Handle GAM custom URL testing (for testing captured URLs directly) -pub async fn handle_gam_custom_url(settings: &Settings, mut req: Request) -> Result { +pub async fn handle_gam_custom_url(_settings: &Settings, mut req: Request) -> Result { println!("Handling GAM custom URL test"); // Check consent status from cookie @@ -413,4 +413,212 @@ pub async fn handle_gam_custom_url(settings: &Settings, mut req: Request) -> Res }))?) } } +} + +/// Handle GAM response rendering in iframe +pub async fn handle_gam_render(settings: &Settings, req: Request) -> Result { + println!("Handling GAM response rendering"); + + // Check consent status from cookie + let consent = get_consent_from_request(&req).unwrap_or_default(); + let advertising_consent = consent.advertising; + + if !advertising_consent { + return Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "No advertising consent", + "message": "GAM requests require advertising consent" + }))?); + } + + // Create GAM request and get response + let gam_req = match GamRequest::new(settings, &req) { + Ok(req) => req.with_prmtvctx("129627,137412,138272,139095,139096,139218,141364,143196,143210,143211,143214,143217,144331,144409,144438,144444,144488,144543,144663,144679,144731,144824,144916,145933,146347,146348,146349,146350,146351,146370,146383,146391,146392,146393,146424,146995,147077,147740,148616,148627,148628,149007,150420,150663,150689,150690,150692,150752,150753,150755,150756,150757,150764,150770,150781,150862,154609,155106,155109,156204,164183,164573,165512,166017,166019,166484,166486,166487,166488,166492,166494,166495,166497,166511,167639,172203,172544,173548,176066,178053,178118,178120,178121,178133,180321,186069,199642,199691,202074,202075,202081,233782,238158,adv,bhgp,bhlp,bhgw,bhlq,bhlt,bhgx,bhgv,bhgu,bhhb,rts".to_string()), + Err(e) => { + return Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "Failed to create GAM request", + "details": format!("{:?}", e) + }))?); + } + }; + + // Get GAM response + let gam_response = match gam_req.send_request(settings).await { + Ok(response) => response, + Err(e) => { + return Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "Failed to get GAM response", + "details": format!("{:?}", e) + }))?); + } + }; + + // Parse the GAM response to extract HTML + let response_body = gam_response.into_body_str(); + println!("Parsing GAM response for HTML extraction"); + + // The GAM response format is: {"/ad_unit_path":["html",0,null,null,0,90,728,0,0,null,null,null,null,null,[...],null,null,null,null,null,null,null,0,null,null,null,null,null,null,"creative_id","line_item_id"],"..."} + // We need to extract the HTML part after the JSON array + + let html_content = if response_body.contains("") { + // Find the start of HTML content + if let Some(html_start) = response_body.find("") { + let html = &response_body[html_start..]; + println!("Extracted HTML content: {} bytes", html.len()); + html.to_string() + } else { + format!("

Error: Could not find HTML content in GAM response

{}
", + response_body.chars().take(500).collect::()) + } + } else { + // Fallback: return the raw response in a safe HTML wrapper + format!("

GAM Response (no HTML found):

{}
", + response_body.chars().take(1000).collect::()) + }; + + // Create a safe HTML page that renders the ad content in an iframe + let render_page = format!( + r#" + + + + + GAM Ad Render Test + + + +
+
+

🎯 GAM Ad Render Test

+

Rendering Google Ad Manager response in iframe

+
+ +
+ Status: Ad content loaded successfully
+ Response Size: {} bytes
+ Timestamp: {} +
+ +
+ + + +
+ + + + +
+ + + +"#, + html_content.len(), + chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC"), + html_content.replace("\"", """).replace("'", "'"), + html_content.len(), + html_content.chars().take(200).collect::() + ); + + Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .with_header(header::CACHE_CONTROL, "no-store, private") + .with_header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .with_header("X-GAM-Render", "true") + .with_header("X-Synthetic-ID", &gam_req.synthetic_id) + .with_header("X-Correlator", &gam_req.correlator) + .with_body(render_page)) } \ No newline at end of file diff --git a/crates/common/src/templates.rs b/crates/common/src/templates.rs index b01e313..c9e6457 100644 --- a/crates/common/src/templates.rs +++ b/crates/common/src/templates.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + pub const HTML_TEMPLATE: &str = r#" @@ -478,6 +480,21 @@ pub const GAM_TEST_TEMPLATE: &str = r#" +
+

Phase 3: Ad Rendering in iFrame

+

Render the GAM response HTML content in a sandboxed iframe for visual testing.

+ +
+

Ad Render Test

+

Test rendering the GAM response as an actual ad in an iframe:

+ + + +
+
+

Debug Information

@@ -592,13 +609,65 @@ pub const GAM_TEST_TEMPLATE: &str = r#" } }); - const data = await response.json(); + // Get the response as text first (since it contains both JSON and HTML) + const responseText = await response.text(); + + // Try to parse as JSON first (in case it's a pure JSON response) + let data; + try { + data = JSON.parse(responseText); + } catch (jsonError) { + // If JSON parsing fails, it's likely the mixed JSON+HTML format + // Find the end of the JSON part (before the HTML starts) + const htmlStart = responseText.indexOf(''); + if (htmlStart !== -1) { + // Extract just the JSON part + const jsonPart = responseText.substring(0, htmlStart); + try { + data = JSON.parse(jsonPart); + // Add info about the HTML part + data.html_content_length = responseText.length - htmlStart; + data.full_response_length = responseText.length; + } catch (innerError) { + // If we still can't parse JSON, show the raw response + data = { + error: 'Could not parse GAM response as JSON', + raw_response_preview: responseText.substring(0, 500) + '...', + response_length: responseText.length + }; + } + } else { + // No HTML found, show the raw response + data = { + error: 'Unexpected response format', + raw_response: responseText, + response_length: responseText.length + }; + } + } + resultDiv.textContent = JSON.stringify(data, null, 2); } catch (error) { resultDiv.textContent = 'Error: ' + error.message; } } + // Test ad rendering in iframe + async function testAdRender() { + const resultDiv = document.getElementById('renderResult'); + resultDiv.style.display = 'block'; + resultDiv.textContent = 'Opening ad render page in new tab...'; + + // Open the render page in a new tab + window.open('/gam-render', '_blank'); + + // Update the result message + setTimeout(() => { + resultDiv.textContent = 'Ad render page opened in new tab. Check the new tab to see the rendered ad!'; + resultDiv.className = 'status success'; + }, 1000); + } + // Initialize page document.addEventListener('DOMContentLoaded', function() { displayHeaders(); @@ -608,3 +677,61 @@ pub const GAM_TEST_TEMPLATE: &str = r#" "#; +// GAM Configuration Template +struct GamConfigTemplate { + publisher_id: String, + ad_units: Vec, + page_context: PageContext, + data_providers: Vec, +} + +struct AdUnitConfig { + name: String, + sizes: Vec, + position: String, + targeting: HashMap, +} + +struct PageContext { + page_type: String, + section: String, + keywords: Vec, +} + +enum DataProvider { + Permutive(PermutiveConfig), + Lotame(LotameConfig), + Neustar(NeustarConfig), + Custom(CustomProviderConfig), +} + +struct PermutiveConfig {} +struct LotameConfig {} +struct NeustarConfig {} +struct CustomProviderConfig {} + +trait DataProviderTrait { + fn get_user_segments(&self, user_id: &str) -> Vec; +} + +struct RequestContext { + user_id: String, + page_url: String, + consent_status: bool, +} + +struct DynamicGamBuilder { + base_config: GamConfigTemplate, + context: RequestContext, + data_providers: Vec>, +} + +// Instead of hardcoded strings, use templates: +// "cust_params": "{{#each data_providers}}{{name}}={{segments}}&{{/each}}puid={{user_id}}" + +// This could generate: +// "permutive=129627,137412...&lotame=segment1,segment2&puid=abc123" + +// let context = data_provider_manager.build_context(&user_id, &request_context); +// let gam_req_with_context = gam_req.with_dynamic_context(context); + diff --git a/crates/fastly/src/main.rs b/crates/fastly/src/main.rs index 66e3946..e7af3eb 100644 --- a/crates/fastly/src/main.rs +++ b/crates/fastly/src/main.rs @@ -8,7 +8,7 @@ use std::env; use trusted_server_common::constants::{SYNTHETIC_HEADER_FRESH, SYNTHETIC_HEADER_TRUSTED_SERVER}; use trusted_server_common::cookies::create_synthetic_cookie; -use trusted_server_common::gam::{handle_gam_test, handle_gam_golden_url, handle_gam_custom_url}; +use trusted_server_common::gam::{handle_gam_test, handle_gam_golden_url, handle_gam_custom_url, handle_gam_render}; use trusted_server_common::gdpr::{ get_consent_from_request, handle_consent_request, handle_data_subject_request, }; @@ -48,6 +48,7 @@ fn main(req: Request) -> Result { (&Method::GET, "/gam-test") => handle_gam_test(&settings, req).await, (&Method::GET, "/gam-golden-url") => handle_gam_golden_url(&settings, req).await, (&Method::POST, "/gam-test-custom-url") => handle_gam_custom_url(&settings, req).await, + (&Method::GET, "/gam-render") => handle_gam_render(&settings, req).await, (&Method::GET, "/gam-test-page") => Ok(Response::from_status(StatusCode::OK) .with_body(GAM_TEST_TEMPLATE) .with_header(header::CONTENT_TYPE, "text/html") From a072d26a66be025db4bd7ad055375227820fe27e Mon Sep 17 00:00:00 2001 From: jevansnyc Date: Fri, 11 Jul 2025 16:33:20 -0500 Subject: [PATCH 03/11] Syncing GAM after running cargo fmt --- crates/common/src/gam.rs | 183 +++++++++++++++++++++------------ crates/common/src/templates.rs | 3 +- crates/fastly/src/main.rs | 12 ++- 3 files changed, 126 insertions(+), 72 deletions(-) diff --git a/crates/common/src/gam.rs b/crates/common/src/gam.rs index 02a1de6..9ae440c 100644 --- a/crates/common/src/gam.rs +++ b/crates/common/src/gam.rs @@ -1,5 +1,5 @@ -use crate::settings::Settings; use crate::gdpr::get_consent_from_request; +use crate::settings::Settings; use fastly::http::{header, Method, StatusCode}; use fastly::{Error, Request, Response}; use serde_json::json; @@ -37,7 +37,12 @@ impl GamRequest { Ok(Self { publisher_id: settings.gam.publisher_id.clone(), - ad_units: settings.gam.ad_units.iter().map(|u| u.name.clone()).collect(), + ad_units: settings + .gam + .ad_units + .iter() + .map(|u| u.name.clone()) + .collect(), page_url, correlator, prmtvctx: None, // Will be set later with captured value @@ -57,34 +62,46 @@ impl GamRequest { // This will be replaced with the actual captured URL from autoblog.com // For now, using a template based on the captured Golden URL let mut params = HashMap::new(); - + // Core GAM parameters (based on captured URL) params.insert("pvsid".to_string(), "3290837576990024".to_string()); // Publisher Viewability ID params.insert("correlator".to_string(), self.correlator.clone()); - params.insert("eid".to_string(), "31086815,31093089,95353385,31085777,83321072".to_string()); // Event IDs + params.insert( + "eid".to_string(), + "31086815,31093089,95353385,31085777,83321072".to_string(), + ); // Event IDs params.insert("output".to_string(), "ldjh".to_string()); // Important: not 'json' params.insert("gdfp_req".to_string(), "1".to_string()); params.insert("vrg".to_string(), "202506170101".to_string()); // Version/Region params.insert("ptt".to_string(), "17".to_string()); // Page Type params.insert("impl".to_string(), "fifs".to_string()); // Implementation - + // Ad unit parameters (simplified version of captured format) - params.insert("iu_parts".to_string(), format!("{},{},homepage", self.publisher_id, "trustedserver")); - params.insert("enc_prev_ius".to_string(), "/0/1/2,/0/1/2,/0/1/2".to_string()); + params.insert( + "iu_parts".to_string(), + format!("{},{},homepage", self.publisher_id, "trustedserver"), + ); + params.insert( + "enc_prev_ius".to_string(), + "/0/1/2,/0/1/2,/0/1/2".to_string(), + ); params.insert("prev_iu_szs".to_string(), "320x50|300x250|728x90|970x90|970x250|1x2,320x50|300x250|728x90|970x90|970x250|1x2,320x50|300x250|728x90|970x90|970x250|1x2".to_string()); params.insert("fluid".to_string(), "height,height,height".to_string()); - + // Browser context (simplified) params.insert("biw".to_string(), "1512".to_string()); params.insert("bih".to_string(), "345".to_string()); params.insert("u_tz".to_string(), "-300".to_string()); params.insert("u_cd".to_string(), "30".to_string()); params.insert("u_sd".to_string(), "2".to_string()); - + // Page context params.insert("url".to_string(), self.page_url.clone()); - params.insert("dt".to_string(), chrono::Utc::now().timestamp_millis().to_string()); - + params.insert( + "dt".to_string(), + chrono::Utc::now().timestamp_millis().to_string(), + ); + // Add Permutive context if available (in cust_params like the captured URL) if let Some(ref prmtvctx) = self.prmtvctx { let cust_params = format!("permutive={}&puid={}", prmtvctx, self.synthetic_id); @@ -114,7 +131,7 @@ impl GamRequest { // Create the request let mut req = Request::new(Method::GET, &url); - + // Set headers to mimic a browser request (using only Fastly-compatible headers) req.set_header(header::USER_AGENT, &self.user_agent); req.set_header(header::ACCEPT, "application/json, text/plain, */*"); @@ -127,11 +144,14 @@ impl GamRequest { // Send the request to the GAM backend let backend_name = "gam_backend"; println!("Sending request to backend: {}", backend_name); - + match req.send(backend_name) { Ok(mut response) => { - println!("Received GAM response with status: {}", response.get_status()); - + println!( + "Received GAM response with status: {}", + response.get_status() + ); + // Log response headers for debugging println!("GAM Response headers:"); for (name, value) in response.get_headers() { @@ -144,14 +164,20 @@ impl GamRequest { Ok(body_str) => body_str.to_string(), Err(e) => { println!("Warning: Could not read response body as UTF-8: {:?}", e); - + // Try to decompress if it's Brotli compressed let mut decompressed = Vec::new(); - match brotli::BrotliDecompress(&mut std::io::Cursor::new(&body_bytes), &mut decompressed) { + match brotli::BrotliDecompress( + &mut std::io::Cursor::new(&body_bytes), + &mut decompressed, + ) { Ok(_) => { match std::str::from_utf8(&decompressed) { Ok(decompressed_str) => { - println!("Successfully decompressed Brotli response: {} bytes", decompressed_str.len()); + println!( + "Successfully decompressed Brotli response: {} bytes", + decompressed_str.len() + ); decompressed_str.to_string() } Err(e2) => { @@ -170,7 +196,7 @@ impl GamRequest { }; println!("GAM Response body length: {} bytes", body.len()); - + // For debugging, log first 500 chars of response if body.len() > 500 { println!("GAM Response preview: {}...", &body[..500]); @@ -211,7 +237,10 @@ pub async fn handle_gam_test(settings: &Settings, req: Request) -> Result Result Result { @@ -283,7 +318,7 @@ pub async fn handle_gam_test(settings: &Settings, req: Request) -> Result Result { println!("Handling GAM golden URL replay"); - + // This endpoint will be used to test the exact captured URL from autoblog.com // For now, return a placeholder response Ok(Response::from_status(StatusCode::OK) @@ -300,13 +335,16 @@ pub async fn handle_gam_golden_url(_settings: &Settings, _req: Request) -> Resul } /// Handle GAM custom URL testing (for testing captured URLs directly) -pub async fn handle_gam_custom_url(_settings: &Settings, mut req: Request) -> Result { +pub async fn handle_gam_custom_url( + _settings: &Settings, + mut req: Request, +) -> Result { println!("Handling GAM custom URL test"); - + // Check consent status from cookie let consent = get_consent_from_request(&req).unwrap_or_default(); let advertising_consent = consent.advertising; - + if !advertising_consent { return Ok(Response::from_status(StatusCode::OK) .with_header(header::CONTENT_TYPE, "application/json") @@ -315,39 +353,45 @@ pub async fn handle_gam_custom_url(_settings: &Settings, mut req: Request) -> Re "message": "GAM requests require advertising consent" }))?); } - + // Parse the request body to get the custom URL let body = req.take_body_str(); - let url_data: serde_json::Value = serde_json::from_str(&body) - .map_err(|e| { - println!("Error parsing request body: {:?}", e); - fastly::Error::msg("Invalid JSON in request body") - })?; - - let custom_url = url_data["url"].as_str() + let url_data: serde_json::Value = serde_json::from_str(&body).map_err(|e| { + println!("Error parsing request body: {:?}", e); + fastly::Error::msg("Invalid JSON in request body") + })?; + + let custom_url = url_data["url"] + .as_str() .ok_or_else(|| fastly::Error::msg("Missing 'url' field in request body"))?; - + println!("Testing custom GAM URL: {}", custom_url); - + // Create a request to the custom URL let mut gam_req = Request::new(Method::GET, custom_url); - + // Set headers to mimic a browser request - gam_req.set_header(header::USER_AGENT, "Mozilla/5.0 (compatible; TrustedServer/1.0)"); + gam_req.set_header( + header::USER_AGENT, + "Mozilla/5.0 (compatible; TrustedServer/1.0)", + ); gam_req.set_header(header::ACCEPT, "application/json, text/plain, */*"); gam_req.set_header(header::ACCEPT_LANGUAGE, "en-US,en;q=0.9"); gam_req.set_header(header::ACCEPT_ENCODING, "gzip, deflate, br"); gam_req.set_header(header::REFERER, "https://www.autoblog.com/"); gam_req.set_header(header::ORIGIN, "https://www.autoblog.com"); - + // Send the request to the GAM backend let backend_name = "gam_backend"; println!("Sending custom URL request to backend: {}", backend_name); - + match gam_req.send(backend_name) { Ok(mut response) => { - println!("Received GAM response with status: {}", response.get_status()); - + println!( + "Received GAM response with status: {}", + response.get_status() + ); + // Log response headers for debugging println!("GAM Response headers:"); for (name, value) in response.get_headers() { @@ -360,22 +404,29 @@ pub async fn handle_gam_custom_url(_settings: &Settings, mut req: Request) -> Re Ok(body_str) => body_str.to_string(), Err(e) => { println!("Warning: Could not read response body as UTF-8: {:?}", e); - + // Try to decompress if it's Brotli compressed let mut decompressed = Vec::new(); - match brotli::BrotliDecompress(&mut std::io::Cursor::new(&body_bytes), &mut decompressed) { - Ok(_) => { - match std::str::from_utf8(&decompressed) { - Ok(decompressed_str) => { - println!("Successfully decompressed Brotli response: {} bytes", decompressed_str.len()); - decompressed_str.to_string() - } - Err(e2) => { - println!("Warning: Could not read decompressed body as UTF-8: {:?}", e2); - format!("{{\"error\": \"decompression_failed\", \"message\": \"Could not decode decompressed response\", \"original_error\": \"{:?}\"}}", e2) - } + match brotli::BrotliDecompress( + &mut std::io::Cursor::new(&body_bytes), + &mut decompressed, + ) { + Ok(_) => match std::str::from_utf8(&decompressed) { + Ok(decompressed_str) => { + println!( + "Successfully decompressed Brotli response: {} bytes", + decompressed_str.len() + ); + decompressed_str.to_string() } - } + Err(e2) => { + println!( + "Warning: Could not read decompressed body as UTF-8: {:?}", + e2 + ); + format!("{{\"error\": \"decompression_failed\", \"message\": \"Could not decode decompressed response\", \"original_error\": \"{:?}\"}}", e2) + } + }, Err(e2) => { println!("Warning: Could not decompress Brotli response: {:?}", e2); // Return a placeholder since we can't parse the binary response @@ -386,7 +437,7 @@ pub async fn handle_gam_custom_url(_settings: &Settings, mut req: Request) -> Re }; println!("GAM Response body length: {} bytes", body.len()); - + Ok(Response::from_status(response.get_status()) .with_header(header::CONTENT_TYPE, "application/json") .with_header(header::CACHE_CONTROL, "no-store, private") @@ -413,16 +464,16 @@ pub async fn handle_gam_custom_url(_settings: &Settings, mut req: Request) -> Re }))?) } } -} +} /// Handle GAM response rendering in iframe pub async fn handle_gam_render(settings: &Settings, req: Request) -> Result { println!("Handling GAM response rendering"); - + // Check consent status from cookie let consent = get_consent_from_request(&req).unwrap_or_default(); let advertising_consent = consent.advertising; - + if !advertising_consent { return Ok(Response::from_status(StatusCode::OK) .with_header(header::CONTENT_TYPE, "application/json") @@ -431,7 +482,7 @@ pub async fn handle_gam_render(settings: &Settings, req: Request) -> Result req.with_prmtvctx("129627,137412,138272,139095,139096,139218,141364,143196,143210,143211,143214,143217,144331,144409,144438,144444,144488,144543,144663,144679,144731,144824,144916,145933,146347,146348,146349,146350,146351,146370,146383,146391,146392,146393,146424,146995,147077,147740,148616,148627,148628,149007,150420,150663,150689,150690,150692,150752,150753,150755,150756,150757,150764,150770,150781,150862,154609,155106,155109,156204,164183,164573,165512,166017,166019,166484,166486,166487,166488,166492,166494,166495,166497,166511,167639,172203,172544,173548,176066,178053,178118,178120,178121,178133,180321,186069,199642,199691,202074,202075,202081,233782,238158,adv,bhgp,bhlp,bhgw,bhlq,bhlt,bhgx,bhgv,bhgu,bhhb,rts".to_string()), @@ -461,10 +512,10 @@ pub async fn handle_gam_render(settings: &Settings, req: Request) -> Result..."} // We need to extract the HTML part after the JSON array - + let html_content = if response_body.contains("") { // Find the start of HTML content if let Some(html_start) = response_body.find("") { @@ -477,8 +528,10 @@ pub async fn handle_gam_render(settings: &Settings, req: Request) -> Result

GAM Response (no HTML found):

{}
", - response_body.chars().take(1000).collect::()) + format!( + "

GAM Response (no HTML found):

{}
", + response_body.chars().take(1000).collect::() + ) }; // Create a safe HTML page that renders the ad content in an iframe @@ -621,4 +674,4 @@ pub async fn handle_gam_render(settings: &Settings, req: Request) -> Result Result { let settings = Settings::new().unwrap(); - + // Print Fastly Service Version only once at the beginning println!( "FASTLY_SERVICE_VERSION: {}", std::env::var("FASTLY_SERVICE_VERSION").unwrap_or_else(|_| String::new()) ); - + // Print User IP address immediately after Fastly Service Version let client_ip = req .get_client_ip_addr() .map(|ip| ip.to_string()) .unwrap_or_else(|| "Unknown".to_string()); println!("User IP: {}", client_ip); - + // Print Settings only once at the beginning println!("Settings: {settings:?}"); From 00337c5c5a8c6504a7ade5b95bda7598564bf908 Mon Sep 17 00:00:00 2001 From: jevansnyc Date: Fri, 11 Jul 2025 17:19:41 -0500 Subject: [PATCH 04/11] Merged GAM with Main - resolved conflicts --- Cargo.lock | 28 ++-------------------------- crates/common/src/lib.rs | 2 +- crates/common/src/templates.rs | 16 +++++++++++----- crates/fastly/src/main.rs | 6 ++---- 4 files changed, 16 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index de78909..2689050 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -235,7 +235,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", "once_cell", "tiny-keccak", ] @@ -663,7 +663,7 @@ checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", ] [[package]] @@ -1718,12 +1718,6 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -1762,15 +1756,6 @@ dependencies = [ "wit-bindgen-rt 0.39.0", ] -[[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" -dependencies = [ - "wit-bindgen-rt", -] - [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -1975,15 +1960,6 @@ name = "wit-bindgen-rt" version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags 2.9.0", -] - -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ "bitflags 2.9.1", ] diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index a580493..89f58b4 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -20,8 +20,8 @@ pub mod constants; pub mod cookies; -pub mod gam; pub mod error; +pub mod gam; pub mod gdpr; pub mod models; pub mod prebid; diff --git a/crates/common/src/templates.rs b/crates/common/src/templates.rs index cda56a6..c4aae3c 100644 --- a/crates/common/src/templates.rs +++ b/crates/common/src/templates.rs @@ -678,48 +678,54 @@ pub const GAM_TEST_TEMPLATE: &str = r#" "#; // GAM Configuration Template +#[allow(dead_code)] struct GamConfigTemplate { publisher_id: String, ad_units: Vec, page_context: PageContext, data_providers: Vec, } - +#[allow(dead_code)] struct AdUnitConfig { name: String, sizes: Vec, position: String, targeting: HashMap, } - +#[allow(dead_code)] struct PageContext { page_type: String, section: String, keywords: Vec, } - +#[allow(dead_code)] enum DataProvider { Permutive(PermutiveConfig), Lotame(LotameConfig), Neustar(NeustarConfig), Custom(CustomProviderConfig), } - +#[allow(dead_code)] struct PermutiveConfig {} +#[allow(dead_code)] struct LotameConfig {} +#[allow(dead_code)] struct NeustarConfig {} +#[allow(dead_code)] struct CustomProviderConfig {} - +#[allow(dead_code)] trait DataProviderTrait { fn get_user_segments(&self, user_id: &str) -> Vec; } +#[allow(dead_code)] struct RequestContext { user_id: String, page_url: String, consent_status: bool, } +#[allow(dead_code)] struct DynamicGamBuilder { base_config: GamConfigTemplate, context: RequestContext, diff --git a/crates/fastly/src/main.rs b/crates/fastly/src/main.rs index 4c257bc..752b4bf 100644 --- a/crates/fastly/src/main.rs +++ b/crates/fastly/src/main.rs @@ -34,7 +34,6 @@ use trusted_server_common::why::WHY_TEMPLATE; #[fastly::main] fn main(req: Request) -> Result { - // Print Settings only once at the beginning let settings = match Settings::new() { Ok(s) => s, @@ -44,13 +43,13 @@ fn main(req: Request) -> Result { } }; log::info!("Settings {settings:?}"); - // Print User IP address immediately after Fastly Service Version + // Print User IP address immediately after Fastly Service Version let client_ip = req .get_client_ip_addr() .map(|ip| ip.to_string()) .unwrap_or_else(|| "Unknown".to_string()); println!("User IP: {}", client_ip); - + futures::executor::block_on(async { log::info!( "FASTLY_SERVICE_VERSION: {}", @@ -150,7 +149,6 @@ fn get_dma_code(req: &mut Request) -> Option { /// /// Returns a Fastly [`Error`] if response creation fails. fn handle_main_page(settings: &Settings, mut req: Request) -> Result { - log::info!( "Using ad_partner_url: {}, counter_store: {}", settings.ad_server.ad_partner_url, From 4f2641fff23f78be87284e5643da4261ad38f8a2 Mon Sep 17 00:00:00 2001 From: jevansnyc Date: Fri, 11 Jul 2025 17:36:04 -0500 Subject: [PATCH 05/11] Fixed test warnings and re-ran cargo fmt --- crates/common/src/test_support.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/crates/common/src/test_support.rs b/crates/common/src/test_support.rs index 67f7262..1b8da25 100644 --- a/crates/common/src/test_support.rs +++ b/crates/common/src/test_support.rs @@ -1,6 +1,6 @@ #[cfg(test)] pub mod tests { - use crate::settings::{AdServer, Prebid, Publisher, Settings, Synthetic}; + use crate::settings::{AdServer, Gam, GamAdUnit, Prebid, Publisher, Settings, Synthetic}; pub fn crate_test_settings_str() -> String { r#" @@ -16,6 +16,16 @@ pub mod tests { [prebid] server_url = "https://test-prebid.com/openrtb2/auction" + [gam] + publisher_id = "3790" + server_url = "https://securepubads.g.doubleclick.net/gampad/ads" + ad_units = [ + { name = "Flex8:1", size = "flexible" }, + { name = "Fixed728x90", size = "728x90" }, + { name = "Static8:1", size = "flexible" }, + { name = "Static728x90", size = "728x90" } + ] + [synthetic] counter_store = "test-counter-store" opid_store = "test-opid-store" @@ -38,6 +48,11 @@ pub mod tests { prebid: Prebid { server_url: "https://test-prebid.com/openrtb2/auction".to_string(), }, + gam: Gam { + publisher_id: "test-publisher-id".to_string(), + server_url: "https://securepubads.g.doubleclick.net/gampad/ads".to_string(), + ad_units: vec![GamAdUnit { name: "test-ad-unit".to_string(), size: "300x250".to_string() }], + }, synthetic: Synthetic { counter_store: "test_counter_store".to_string(), opid_store: "test-opid-store".to_string(), From 5c5c69c0a3657738eccaad028dd07d5f2a5863d5 Mon Sep 17 00:00:00 2001 From: jevansnyc Date: Fri, 11 Jul 2025 17:40:08 -0500 Subject: [PATCH 06/11] Updated changelog.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57e7085..a6e560a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added basic unit tests - Added publisher config - Add AI assist rules. Based on https://github.com/hashintel/hash +- Added ability to construct GAM requests from static permutive segments with test pages ### Changed - Upgrade to rust 1.87.0 From 0e2689504ad356acc768402b5c5f287fc83dd97e Mon Sep 17 00:00:00 2001 From: jevansnyc Date: Fri, 11 Jul 2025 18:01:06 -0500 Subject: [PATCH 07/11] updated all println statements to log --- crates/common/src/gam.rs | 112 +++++++++++++++++++------------------- crates/fastly/src/main.rs | 2 +- 2 files changed, 56 insertions(+), 58 deletions(-) diff --git a/crates/common/src/gam.rs b/crates/common/src/gam.rs index 9ae440c..bd42ac0 100644 --- a/crates/common/src/gam.rs +++ b/crates/common/src/gam.rs @@ -127,7 +127,7 @@ impl GamRequest { /// Send the GAM request and return the response pub async fn send_request(&self, _settings: &Settings) -> Result { let url = self.build_golden_url(); - println!("Sending GAM request to: {}", url); + log::info!("Sending GAM request to: {}", url); // Create the request let mut req = Request::new(Method::GET, &url); @@ -143,19 +143,19 @@ impl GamRequest { // Send the request to the GAM backend let backend_name = "gam_backend"; - println!("Sending request to backend: {}", backend_name); + log::info!("Sending request to backend: {}", backend_name); match req.send(backend_name) { Ok(mut response) => { - println!( + log::info!( "Received GAM response with status: {}", response.get_status() ); // Log response headers for debugging - println!("GAM Response headers:"); + log::debug!("GAM Response headers:"); for (name, value) in response.get_headers() { - println!(" {}: {:?}", name, value); + log::debug!(" {}: {:?}", name, value); } // Handle response body safely @@ -163,7 +163,7 @@ impl GamRequest { let body = match std::str::from_utf8(&body_bytes) { Ok(body_str) => body_str.to_string(), Err(e) => { - println!("Warning: Could not read response body as UTF-8: {:?}", e); + log::warn!("Could not read response body as UTF-8: {:?}", e); // Try to decompress if it's Brotli compressed let mut decompressed = Vec::new(); @@ -171,23 +171,24 @@ impl GamRequest { &mut std::io::Cursor::new(&body_bytes), &mut decompressed, ) { - Ok(_) => { - match std::str::from_utf8(&decompressed) { - Ok(decompressed_str) => { - println!( - "Successfully decompressed Brotli response: {} bytes", - decompressed_str.len() - ); - decompressed_str.to_string() - } - Err(e2) => { - println!("Warning: Could not read decompressed body as UTF-8: {:?}", e2); - format!("{{\"error\": \"decompression_failed\", \"message\": \"Could not decode decompressed response\", \"original_error\": \"{:?}\"}}", e2) - } + Ok(_) => match std::str::from_utf8(&decompressed) { + Ok(decompressed_str) => { + log::debug!( + "Successfully decompressed Brotli response: {} bytes", + decompressed_str.len() + ); + decompressed_str.to_string() } - } + Err(e2) => { + log::warn!( + "Could not read decompressed body as UTF-8: {:?}", + e2 + ); + format!("{{\"error\": \"decompression_failed\", \"message\": \"Could not decode decompressed response\", \"original_error\": \"{:?}\"}}", e2) + } + }, Err(e2) => { - println!("Warning: Could not decompress Brotli response: {:?}", e2); + log::warn!("Could not decompress Brotli response: {:?}", e2); // Return a placeholder since we can't parse the binary response format!("{{\"error\": \"compression_failed\", \"message\": \"Could not decompress response\", \"original_error\": \"{:?}\"}}", e2) } @@ -195,13 +196,13 @@ impl GamRequest { } }; - println!("GAM Response body length: {} bytes", body.len()); + log::debug!("GAM Response body length: {} bytes", body.len()); // For debugging, log first 500 chars of response if body.len() > 500 { - println!("GAM Response preview: {}...", &body[..500]); + log::debug!("GAM Response preview: {}...", &body[..500]); } else { - println!("GAM Response: {}", body); + log::debug!("GAM Response: {}", body); } Ok(Response::from_status(response.get_status()) @@ -215,7 +216,7 @@ impl GamRequest { .with_body(body)) } Err(e) => { - println!("Error sending GAM request: {:?}", e); + log::error!("Error sending GAM request: {:?}", e); Err(e.into()) } } @@ -224,20 +225,20 @@ impl GamRequest { /// Handle GAM test requests (Phase 1: Capture & Replay) pub async fn handle_gam_test(settings: &Settings, req: Request) -> Result { - println!("Starting GAM test request handling"); + log::info!("Starting GAM test request handling"); // Debug: Log all request headers - println!("GAM Test - All request headers:"); + log::debug!("GAM Test - All request headers:"); for (name, value) in req.get_headers() { - println!(" {}: {:?}", name, value); + log::debug!(" {}: {:?}", name, value); } // Check consent status from cookie (more reliable than header) let consent = get_consent_from_request(&req).unwrap_or_default(); let advertising_consent = consent.advertising; - println!("GAM Test - Consent from cookie: {:?}", consent); - println!( + log::debug!("GAM Test - Consent from cookie: {:?}", consent); + log::debug!( "GAM Test - Advertising consent from cookie: {}", advertising_consent ); @@ -249,14 +250,14 @@ pub async fn handle_gam_test(settings: &Settings, req: Request) -> Result Result { - println!("Successfully created GAM request"); + log::info!("Successfully created GAM request"); req } Err(e) => { - println!("Error creating GAM request: {:?}", e); + log::error!("Error creating GAM request: {:?}", e); return Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) .with_header(header::CONTENT_TYPE, "application/json") .with_body_json(&json!({ @@ -293,18 +294,18 @@ pub async fn handle_gam_test(settings: &Settings, req: Request) -> Result { - println!("GAM request successful"); + log::info!("GAM request successful"); Ok(response) } Err(e) => { - println!("GAM request failed: {:?}", e); + log::error!("GAM request failed: {:?}", e); Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) .with_header(header::CONTENT_TYPE, "application/json") .with_body_json(&json!({ @@ -317,7 +318,7 @@ pub async fn handle_gam_test(settings: &Settings, req: Request) -> Result Result { - println!("Handling GAM golden URL replay"); + log::info!("Handling GAM golden URL replay"); // This endpoint will be used to test the exact captured URL from autoblog.com // For now, return a placeholder response @@ -339,7 +340,7 @@ pub async fn handle_gam_custom_url( _settings: &Settings, mut req: Request, ) -> Result { - println!("Handling GAM custom URL test"); + log::info!("Handling GAM custom URL test"); // Check consent status from cookie let consent = get_consent_from_request(&req).unwrap_or_default(); @@ -357,7 +358,7 @@ pub async fn handle_gam_custom_url( // Parse the request body to get the custom URL let body = req.take_body_str(); let url_data: serde_json::Value = serde_json::from_str(&body).map_err(|e| { - println!("Error parsing request body: {:?}", e); + log::error!("Error parsing request body: {:?}", e); fastly::Error::msg("Invalid JSON in request body") })?; @@ -365,7 +366,7 @@ pub async fn handle_gam_custom_url( .as_str() .ok_or_else(|| fastly::Error::msg("Missing 'url' field in request body"))?; - println!("Testing custom GAM URL: {}", custom_url); + log::info!("Testing custom GAM URL: {}", custom_url); // Create a request to the custom URL let mut gam_req = Request::new(Method::GET, custom_url); @@ -383,19 +384,19 @@ pub async fn handle_gam_custom_url( // Send the request to the GAM backend let backend_name = "gam_backend"; - println!("Sending custom URL request to backend: {}", backend_name); + log::info!("Sending custom URL request to backend: {}", backend_name); match gam_req.send(backend_name) { Ok(mut response) => { - println!( + log::info!( "Received GAM response with status: {}", response.get_status() ); // Log response headers for debugging - println!("GAM Response headers:"); + log::debug!("GAM Response headers:"); for (name, value) in response.get_headers() { - println!(" {}: {:?}", name, value); + log::debug!(" {}: {:?}", name, value); } // Handle response body safely @@ -403,7 +404,7 @@ pub async fn handle_gam_custom_url( let body = match std::str::from_utf8(&body_bytes) { Ok(body_str) => body_str.to_string(), Err(e) => { - println!("Warning: Could not read response body as UTF-8: {:?}", e); + log::warn!("Could not read response body as UTF-8: {:?}", e); // Try to decompress if it's Brotli compressed let mut decompressed = Vec::new(); @@ -413,22 +414,19 @@ pub async fn handle_gam_custom_url( ) { Ok(_) => match std::str::from_utf8(&decompressed) { Ok(decompressed_str) => { - println!( + log::debug!( "Successfully decompressed Brotli response: {} bytes", decompressed_str.len() ); decompressed_str.to_string() } Err(e2) => { - println!( - "Warning: Could not read decompressed body as UTF-8: {:?}", - e2 - ); + log::warn!("Could not read decompressed body as UTF-8: {:?}", e2); format!("{{\"error\": \"decompression_failed\", \"message\": \"Could not decode decompressed response\", \"original_error\": \"{:?}\"}}", e2) } }, Err(e2) => { - println!("Warning: Could not decompress Brotli response: {:?}", e2); + log::warn!("Could not decompress Brotli response: {:?}", e2); // Return a placeholder since we can't parse the binary response format!("{{\"error\": \"compression_failed\", \"message\": \"Could not decompress response\", \"original_error\": \"{:?}\"}}", e2) } @@ -436,7 +434,7 @@ pub async fn handle_gam_custom_url( } }; - println!("GAM Response body length: {} bytes", body.len()); + log::debug!("GAM Response body length: {} bytes", body.len()); Ok(Response::from_status(response.get_status()) .with_header(header::CONTENT_TYPE, "application/json") @@ -454,7 +452,7 @@ pub async fn handle_gam_custom_url( }))?) } Err(e) => { - println!("Error sending custom GAM request: {:?}", e); + log::error!("Error sending custom GAM request: {:?}", e); Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) .with_header(header::CONTENT_TYPE, "application/json") .with_body_json(&json!({ @@ -468,7 +466,7 @@ pub async fn handle_gam_custom_url( /// Handle GAM response rendering in iframe pub async fn handle_gam_render(settings: &Settings, req: Request) -> Result { - println!("Handling GAM response rendering"); + log::info!("Handling GAM response rendering"); // Check consent status from cookie let consent = get_consent_from_request(&req).unwrap_or_default(); @@ -511,7 +509,7 @@ pub async fn handle_gam_render(settings: &Settings, req: Request) -> Result..."} // We need to extract the HTML part after the JSON array @@ -520,7 +518,7 @@ pub async fn handle_gam_render(settings: &Settings, req: Request) -> Result") { let html = &response_body[html_start..]; - println!("Extracted HTML content: {} bytes", html.len()); + log::debug!("Extracted HTML content: {} bytes", html.len()); html.to_string() } else { format!("

Error: Could not find HTML content in GAM response

{}
", diff --git a/crates/fastly/src/main.rs b/crates/fastly/src/main.rs index 752b4bf..77cb614 100644 --- a/crates/fastly/src/main.rs +++ b/crates/fastly/src/main.rs @@ -48,7 +48,7 @@ fn main(req: Request) -> Result { .get_client_ip_addr() .map(|ip| ip.to_string()) .unwrap_or_else(|| "Unknown".to_string()); - println!("User IP: {}", client_ip); + log::info!("User IP: {}", client_ip); futures::executor::block_on(async { log::info!( From c0f89a2f7be3a95dcfcda5e1f4c98447c42ad618 Mon Sep 17 00:00:00 2001 From: jevansnyc Date: Thu, 31 Jul 2025 11:19:35 -0400 Subject: [PATCH 08/11] feat: add GAM integration with publisher configuration and audit - Add GAM module with request construction and ad serving capabilities - Add partners.rs for partner-specific configurations - Create comprehensive publisher IDs audit document - Update settings and templates for GAM integration - Add SSL certificates for local development - Update Cargo dependencies for new functionality --- Cargo.lock | 12 + PUBLISHER_IDS_AUDIT.md | 77 ++++ crates/common/src/gam.rs | 21 +- crates/common/src/lib.rs | 1 + crates/common/src/partners.rs | 199 +++++++++ crates/common/src/settings.rs | 17 + crates/common/src/templates.rs | 726 +++++++++++++++++++++++++++++++-- crates/fastly/Cargo.toml | 3 + crates/fastly/src/main.rs | 628 +++++++++++++++++++++++++++- fastly.toml | 44 +- localhost+2-key.pem | 28 ++ localhost+2.pem | 26 ++ trusted-server.toml | 34 +- 13 files changed, 1766 insertions(+), 50 deletions(-) create mode 100644 PUBLISHER_IDS_AUDIT.md create mode 100644 crates/common/src/partners.rs create mode 100644 localhost+2-key.pem create mode 100644 localhost+2.pem diff --git a/Cargo.lock b/Cargo.lock index 2689050..d5fe777 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -535,6 +535,15 @@ dependencies = [ "wit-bindgen-rt 0.42.1", ] +[[package]] +name = "fern" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4316185f709b23713e41e3195f90edef7fb00c3ed4adc79769cf09cc762a3b29" +dependencies = [ + "log", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1661,8 +1670,11 @@ dependencies = [ name = "trusted-server-fastly" version = "0.1.0" dependencies = [ + "brotli", + "chrono", "error-stack", "fastly", + "fern", "futures", "log", "log-fastly", diff --git a/PUBLISHER_IDS_AUDIT.md b/PUBLISHER_IDS_AUDIT.md new file mode 100644 index 0000000..cf9b319 --- /dev/null +++ b/PUBLISHER_IDS_AUDIT.md @@ -0,0 +1,77 @@ +# Publisher-Specific IDs Audit + +This document lists all publisher-specific IDs and configurations found in the codebase that are currently hardcoded to test publisher values. + +## Configuration Files + +### trusted-server.toml + +**GAM Configuration:** +- `publisher_id = "3790"` (line 14) +- `server_url = "https://securepubads.g.doubleclick.net/gampad/ads"` (line 15) + +**Equativ Configuration:** +- `sync_url = "https://adapi-srv-eu.smartadserver.com/ac?pgid=2040327&fmtid=137675&synthetic_id={{synthetic_id}}"` (line 8) + - Page ID: `2040327` + - Format ID: `137675` + +**Test Publisher Domain:** +- `domain = "test-publisher.com"` (line 2) +- `cookie_domain = ".test-publisher.com"` (line 3) +- `origin_url = "https://origin.test-publisher.com"` (line 4) + +**KV Store Names (user-specific):** +- `counter_store = "jevans_synth_id_counter"` (line 24) +- `opid_store = "jevans_synth_id_opid"` (line 25) + +## Hardcoded in Source Code + +### /Users/jevans/trusted-server/crates/common/src/gam.rs + +**Permutive Segment Data (lines 295 and 486):** +```rust +.with_prmtvctx("129627,137412,138272,139095,139096,139218,141364,143196,143210,143211,143214,143217,144331,144409,144438,144444,144488,144543,144663,144679,144731,144824,144916,145933,146347,146348,146349,146350,146351,146370,146383,146391,146392,146393,146424,146995,147077,147740,148616,148627,148628,149007,150420,150663,150689,150690,150692,150752,150753,150755,150756,150757,150764,150770,150781,150862,154609,155106,155109,156204,164183,164573,165512,166017,166019,166484,166486,166487,166488,166492,166494,166495,166497,166511,167639,172203,172544,173548,176066,178053,178118,178120,178121,178133,180321,186069,199642,199691,202074,202075,202081,233782,238158,adv,bhgp,bhlp,bhgw,bhlq,bhlt,bhgx,bhgv,bhgu,bhhb,rts".to_string()) +``` + +This large string contains Permutive segment IDs that appear to be captured from a specific test publisher's live traffic. + +### /Users/jevans/trusted-server/crates/common/src/prebid.rs + +**Equativ Integration:** +- `"pageId": 2040327` (matches config) +- `"formatId": 137675` (matches config) + +### Test Files + +**Test Support Files:** +- GAM publisher ID `"3790"` in test configurations +- `"test-publisher.com"` and related test domains in multiple test files + +## Impact Assessment + +### High Priority (Publisher-Specific) +1. **GAM Publisher ID (3790)** - Core identifier for ad serving +2. **Permutive Segments** - Large hardcoded segment string from test traffic +3. **Equativ Page/Format IDs (2040327, 137675)** - Ad network integration + +### Medium Priority (Environment-Specific) +1. **Test Publisher Domains** - Should be configurable per deployment +2. **KV Store Names** - Currently user-specific (jevans_*) + +### Low Priority (Infrastructure) +1. **Server URLs** - Generally standard but should be configurable + +## Recommendations + +1. Move hardcoded Permutive segments to configuration +2. Make GAM publisher ID environment-specific +3. Make Equativ IDs configurable per publisher +4. Generalize KV store naming convention +5. Create publisher-specific configuration templates + +## Files to Update + +- `trusted-server.toml` - Add permutive segments configuration +- `crates/common/src/gam.rs` - Remove hardcoded segments (lines 295, 486) +- `crates/common/src/prebid.rs` - Use configuration for Equativ IDs +- Test files - Use environment-agnostic test data \ No newline at end of file diff --git a/crates/common/src/gam.rs b/crates/common/src/gam.rs index bd42ac0..f267ed6 100644 --- a/crates/common/src/gam.rs +++ b/crates/common/src/gam.rs @@ -206,7 +206,7 @@ impl GamRequest { } Ok(Response::from_status(response.get_status()) - .with_header(header::CONTENT_TYPE, "application/json") + .with_header(header::CONTENT_TYPE, "text/plain") .with_header(header::CACHE_CONTROL, "no-store, private") .with_header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*") .with_header("X-GAM-Test", "true") @@ -342,16 +342,29 @@ pub async fn handle_gam_custom_url( ) -> Result { log::info!("Handling GAM custom URL test"); - // Check consent status from cookie + // Check consent status from cookie or header for testing let consent = get_consent_from_request(&req).unwrap_or_default(); - let advertising_consent = consent.advertising; + let cookie_consent = consent.advertising; + + // Also check header as fallback for testing + let header_consent = req + .get_header("X-Consent-Advertising") + .and_then(|h| h.to_str().ok()) + .map(|v| v == "true") + .unwrap_or(false); + + let advertising_consent = cookie_consent || header_consent; if !advertising_consent { return Ok(Response::from_status(StatusCode::OK) .with_header(header::CONTENT_TYPE, "application/json") .with_body_json(&json!({ "error": "No advertising consent", - "message": "GAM requests require advertising consent" + "message": "GAM requests require advertising consent", + "debug": { + "cookie_consent": cookie_consent, + "header_consent": header_consent + } }))?); } diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 89f58b4..d115e71 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -24,6 +24,7 @@ pub mod error; pub mod gam; pub mod gdpr; pub mod models; +pub mod partners; pub mod prebid; pub mod privacy; pub mod settings; diff --git a/crates/common/src/partners.rs b/crates/common/src/partners.rs new file mode 100644 index 0000000..d420593 --- /dev/null +++ b/crates/common/src/partners.rs @@ -0,0 +1,199 @@ +use std::collections::HashMap; + +use crate::settings::Settings; + +/// Manages partner-specific URL rewriting and proxy configurations +pub struct PartnerManager { + /// Map of original domain -> proxy domain for rewriting URLs + domain_mappings: HashMap, + /// Map of original domain -> backend name for proxying requests + backend_mappings: HashMap, +} + +impl PartnerManager { + /// Create a new PartnerManager from settings + pub fn from_settings(settings: &Settings) -> Self { + let mut domain_mappings = HashMap::new(); + let mut backend_mappings = HashMap::new(); + + if let Some(partners) = &settings.partners { + // Process GAM partner config + if let Some(gam) = &partners.gam { + if gam.enabled { + for domain in &gam.domains_to_proxy { + domain_mappings.insert(domain.clone(), gam.proxy_domain.clone()); + backend_mappings.insert(domain.clone(), gam.backend_name.clone()); + } + } + } + + // Process Equativ partner config + if let Some(equativ) = &partners.equativ { + if equativ.enabled { + for domain in &equativ.domains_to_proxy { + domain_mappings.insert(domain.clone(), equativ.proxy_domain.clone()); + backend_mappings.insert(domain.clone(), equativ.backend_name.clone()); + } + } + } + + // Process Prebid partner config + if let Some(prebid) = &partners.prebid { + if prebid.enabled { + for domain in &prebid.domains_to_proxy { + domain_mappings.insert(domain.clone(), prebid.proxy_domain.clone()); + backend_mappings.insert(domain.clone(), prebid.backend_name.clone()); + } + } + } + } + + Self { + domain_mappings, + backend_mappings, + } + } + + /// Rewrite a URL to use the configured proxy domain + pub fn rewrite_url(&self, original_url: &str) -> String { + let mut rewritten_url = original_url.to_string(); + + for (original_domain, proxy_domain) in &self.domain_mappings { + if rewritten_url.contains(original_domain) { + rewritten_url = rewritten_url.replace(original_domain, proxy_domain); + // Only replace the first match to avoid multiple replacements + break; + } + } + + rewritten_url + } + + /// Get the backend name for a given domain (for proxying) + pub fn get_backend_for_domain(&self, domain: &str) -> Option<&str> { + self.backend_mappings.get(domain).map(|s| s.as_str()) + } + + /// Check if a domain should be proxied + pub fn should_proxy_domain(&self, domain: &str) -> bool { + self.domain_mappings.contains_key(domain) + } + + /// Get all domains that should be proxied + pub fn get_proxied_domains(&self) -> Vec<&String> { + self.domain_mappings.keys().collect() + } + + /// Rewrite multiple URLs in a text content (for HTML/JS content) + pub fn rewrite_content(&self, content: &str) -> String { + let mut rewritten_content = content.to_string(); + + for (original_domain, proxy_domain) in &self.domain_mappings { + // Use regex-like replacement for all occurrences + rewritten_content = rewritten_content.replace(original_domain, proxy_domain); + } + + rewritten_content + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::settings::{PartnerConfig, Partners, Settings}; + + fn create_test_settings() -> Settings { + let mut settings = Settings::default(); + + let gam_config = PartnerConfig { + enabled: true, + name: "Google Ad Manager".to_string(), + domains_to_proxy: vec![ + "securepubads.g.doubleclick.net".to_string(), + "tpc.googlesyndication.com".to_string(), + ], + proxy_domain: "creatives.auburndao.com".to_string(), + backend_name: "gam_proxy_backend".to_string(), + }; + + let equativ_config = PartnerConfig { + enabled: true, + name: "Equativ".to_string(), + domains_to_proxy: vec!["creatives.sascdn.com".to_string()], + proxy_domain: "creatives.auburndao.com".to_string(), + backend_name: "equativ_proxy_backend".to_string(), + }; + + settings.partners = Some(Partners { + gam: Some(gam_config), + equativ: Some(equativ_config), + prebid: None, + }); + + settings + } + + #[test] + fn test_url_rewriting() { + let settings = create_test_settings(); + let manager = PartnerManager::from_settings(&settings); + + // Test GAM URL rewriting + let gam_url = "https://tpc.googlesyndication.com/simgad/12184163379128326694"; + let rewritten = manager.rewrite_url(gam_url); + assert_eq!( + rewritten, + "https://creatives.auburndao.com/simgad/12184163379128326694" + ); + + // Test Equativ URL rewriting + let equativ_url = "https://creatives.sascdn.com/diff/12345/creative.jpg"; + let rewritten = manager.rewrite_url(equativ_url); + assert_eq!( + rewritten, + "https://creatives.auburndao.com/diff/12345/creative.jpg" + ); + + // Test non-matching URL (should remain unchanged) + let other_url = "https://example.com/image.jpg"; + let rewritten = manager.rewrite_url(other_url); + assert_eq!(rewritten, "https://example.com/image.jpg"); + } + + #[test] + fn test_backend_mapping() { + let settings = create_test_settings(); + let manager = PartnerManager::from_settings(&settings); + + assert_eq!( + manager.get_backend_for_domain("tpc.googlesyndication.com"), + Some("gam_proxy_backend") + ); + assert_eq!( + manager.get_backend_for_domain("creatives.sascdn.com"), + Some("equativ_proxy_backend") + ); + assert_eq!(manager.get_backend_for_domain("unknown.domain.com"), None); + } + + #[test] + fn test_content_rewriting() { + let settings = create_test_settings(); + let manager = PartnerManager::from_settings(&settings); + + let html_content = r#" + + + + "#; + + let rewritten = manager.rewrite_content(html_content); + + assert!(rewritten.contains("https://creatives.auburndao.com/simgad/123")); + assert!(rewritten.contains("https://creatives.auburndao.com/gpt/pubads.js")); + assert!(rewritten.contains("https://creatives.auburndao.com/creative.jpg")); + assert!(!rewritten.contains("tpc.googlesyndication.com")); + assert!(!rewritten.contains("securepubads.g.doubleclick.net")); + assert!(!rewritten.contains("creatives.sascdn.com")); + } +} diff --git a/crates/common/src/settings.rs b/crates/common/src/settings.rs index f218162..c80f116 100644 --- a/crates/common/src/settings.rs +++ b/crates/common/src/settings.rs @@ -51,6 +51,22 @@ pub struct Synthetic { pub template: String, } +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct PartnerConfig { + pub enabled: bool, + pub name: String, + pub domains_to_proxy: Vec, + pub proxy_domain: String, + pub backend_name: String, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct Partners { + pub gam: Option, + pub equativ: Option, + pub prebid: Option, +} + #[derive(Debug, Default, Deserialize, Serialize)] pub struct Settings { pub ad_server: AdServer, @@ -58,6 +74,7 @@ pub struct Settings { pub prebid: Prebid, pub gam: Gam, pub synthetic: Synthetic, + pub partners: Option, } #[allow(unused)] diff --git a/crates/common/src/templates.rs b/crates/common/src/templates.rs index c4aae3c..23aa97e 100644 --- a/crates/common/src/templates.rs +++ b/crates/common/src/templates.rs @@ -234,7 +234,9 @@ pub const HTML_TEMPLATE: &str = r#" const adLink = document.createElement('a'); adLink.href = 'https://iabtechlab.com/?potsi-test%3F'; const adImage = document.createElement('img'); - adImage.src = data.creativeUrl.replace('creatives.sascdn.com', 'creatives.auburndao.com'); + // Direct first-party URL rewriting for Equativ only (like auburndao.com) + adImage.src = data.creativeUrl + .replace('creatives.sascdn.com', '//www.edgepubs.com'); adImage.alt = 'Ad Creative'; adLink.appendChild(adImage); adContainer.appendChild(adLink); @@ -329,6 +331,686 @@ pub const HTML_TEMPLATE: &str = r#" "#; +pub const EDGEPUBS_TEMPLATE: &str = r##" + + + + + EdgePubs - The Edge Is Yours + + + + + +
+

Cookie Consent

+

We use cookies to enhance your browsing experience and serve personalized ads. By clicking "Accept All", you consent to our use of cookies.

+
+ + +
+
+ + + + +
+
+

The Edge Is Yours

+

Run your site, ads, and data stack server-side — under your domain, on your terms.

+ Get Started → +
+
+ + +
+
+ +
+
+ + +
+
+

Why EdgePubs?

+ +
+
+
+

Publisher-Controlled Execution

+

Replace slow browser scripts with fast, server-side orchestration. Run your entire site and ad stack at the edge.

+
+
+

1st-Party Data & Identity

+

Protect and activate your first-party data. Build synthetic IDs and pass privacy-compliant signals to your partners.

+
+
+

Server-Side Tagging

+

No more fragile on-page tags. Execute all third-party tags server-side, giving you speed, control, and compliance.

+
+
+

Ad Stack Orchestration

+

Integrate Prebid Server, GAM, and SSPs directly. Manage auctions and measurement server-side for faster performance.

+
+
+

Faster Sites, Better UX

+

Cut page load times in half. Delight users with blazing fast experiences and fewer third-party browser calls.

+
+
+ + + +
+
+
+ + +
+
+

How It Works

+
+
    +
  • Trusted Server acts as a secure reverse proxy in front of your CMS (WordPress, Drupal, etc.)
  • +
  • Prebid auctions, ad-serving, and consent tools run server-side, not in the browser.
  • +
  • Contextual signals and creative assets are stitched directly into the page at the edge.
  • +
  • Result: More revenue. More control. Better user experience.
  • +
+ +
+ Publisher → Trusted Server → Ad Tech Partners → User +
+
+
+
+ + +
+
+
+
+
+
+
+
+ + +
+
+
+ + +
+ +
+
    +
  • Full control of your execution environment
  • +
  • Server-side identity, consent, and measurement
  • +
  • No more slow, fragile browser tags
  • +
+
+ +
+
    +
  • Cleaner supply paths (no intermediaries siphoning value)
  • +
  • Higher-quality inventory with verified user signals
  • +
  • Cookieless targeting ready out-of-the-box
  • +
+
+
+
+ + + + + + +"##; + pub const GAM_TEST_TEMPLATE: &str = r#" @@ -609,42 +1291,16 @@ pub const GAM_TEST_TEMPLATE: &str = r#" } }); - // Get the response as text first (since it contains both JSON and HTML) + // Get the response as text (raw GAM response content) const responseText = await response.text(); - // Try to parse as JSON first (in case it's a pure JSON response) - let data; - try { - data = JSON.parse(responseText); - } catch (jsonError) { - // If JSON parsing fails, it's likely the mixed JSON+HTML format - // Find the end of the JSON part (before the HTML starts) - const htmlStart = responseText.indexOf(''); - if (htmlStart !== -1) { - // Extract just the JSON part - const jsonPart = responseText.substring(0, htmlStart); - try { - data = JSON.parse(jsonPart); - // Add info about the HTML part - data.html_content_length = responseText.length - htmlStart; - data.full_response_length = responseText.length; - } catch (innerError) { - // If we still can't parse JSON, show the raw response - data = { - error: 'Could not parse GAM response as JSON', - raw_response_preview: responseText.substring(0, 500) + '...', - response_length: responseText.length - }; - } - } else { - // No HTML found, show the raw response - data = { - error: 'Unexpected response format', - raw_response: responseText, - response_length: responseText.length - }; - } - } + // For the test page, create a simple data structure for display + const data = { + status: "gam_test_success", + response_length: responseText.length, + response_preview: responseText.substring(0, 500) + (responseText.length > 500 ? '...' : ''), + full_response: responseText + }; resultDiv.textContent = JSON.stringify(data, null, 2); } catch (error) { diff --git a/crates/fastly/Cargo.toml b/crates/fastly/Cargo.toml index d98e2b5..55e77fc 100644 --- a/crates/fastly/Cargo.toml +++ b/crates/fastly/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] +brotli = "3.3" error-stack = "0.5" fastly = "0.11.5" futures = "0.3" @@ -12,3 +13,5 @@ log-fastly = "0.11.5" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.140" trusted-server-common = { path = "../common" } +fern ="0.7" +chrono = "0.4.41" diff --git a/crates/fastly/src/main.rs b/crates/fastly/src/main.rs index 77cb614..cdb772d 100644 --- a/crates/fastly/src/main.rs +++ b/crates/fastly/src/main.rs @@ -1,10 +1,11 @@ use std::env; +use std::io::Read; use fastly::geo::geo_lookup; use fastly::http::{header, Method, StatusCode}; use fastly::KVStore; use fastly::{Error, Request, Response}; -use log::LevelFilter::Info; +use log_fastly::Logger; use serde_json::json; mod error; @@ -29,11 +30,19 @@ use trusted_server_common::prebid::PrebidRequest; use trusted_server_common::privacy::PRIVACY_TEMPLATE; use trusted_server_common::settings::Settings; use trusted_server_common::synthetic::{generate_synthetic_id, get_or_generate_synthetic_id}; -use trusted_server_common::templates::{GAM_TEST_TEMPLATE, HTML_TEMPLATE}; +use trusted_server_common::templates::{EDGEPUBS_TEMPLATE, GAM_TEST_TEMPLATE, HTML_TEMPLATE}; use trusted_server_common::why::WHY_TEMPLATE; #[fastly::main] fn main(req: Request) -> Result { + init_logger(); + + // Log service version first + log::info!( + "FASTLY_SERVICE_VERSION: {}", + std::env::var("FASTLY_SERVICE_VERSION").unwrap_or_else(|_| String::new()) + ); + // Print Settings only once at the beginning let settings = match Settings::new() { Ok(s) => s, @@ -51,14 +60,18 @@ fn main(req: Request) -> Result { log::info!("User IP: {}", client_ip); futures::executor::block_on(async { - log::info!( - "FASTLY_SERVICE_VERSION: {}", - std::env::var("FASTLY_SERVICE_VERSION").unwrap_or_else(|_| String::new()) - ); - match (req.get_method(), req.get_path()) { - (&Method::GET, "/") => handle_main_page(&settings, req), + (&Method::GET, "/") => handle_edgepubs_page(&settings, req), + (&Method::GET, "/auburndao") => handle_main_page(&settings, req), (&Method::GET, "/ad-creative") => handle_ad_request(&settings, req), + // Direct asset serving for partner domains (like auburndao.com approach) + (&Method::GET, path) if is_partner_asset_path(path) => { + handle_partner_asset(&settings, req).await + } + // GAM asset serving (separate from Equativ, checked after Equativ) + (&Method::GET, path) if is_gam_asset_path(path) => { + handle_gam_asset(&settings, req).await + } (&Method::GET, "/prebid-test") => handle_prebid_test(&settings, req).await, (&Method::GET, "/gam-test") => handle_gam_test(&settings, req).await, (&Method::GET, "/gam-golden-url") => handle_gam_golden_url(&settings, req).await, @@ -141,6 +154,582 @@ fn get_dma_code(req: &mut Request) -> Option { None } +/// Handles the EdgePubs page request. +/// +/// Serves the EdgePubs landing page with integrated ad slots. +/// +/// # Errors +/// +/// Returns a Fastly [`Error`] if response creation fails. +fn handle_edgepubs_page(settings: &Settings, mut req: Request) -> Result { + log::info!("Serving EdgePubs landing page"); + + // log_fastly::init_simple("mylogs", Info); + + // Add DMA code check + let dma_code = get_dma_code(&mut req); + log::info!("EdgePubs page - DMA Code: {:?}", dma_code); + + // Check GDPR consent + let _consent = match get_consent_from_request(&req) { + Some(c) => c, + None => { + log::debug!("No GDPR consent found for EdgePubs page, using default"); + GdprConsent::default() + } + }; + + // Generate synthetic ID for EdgePubs page + let fresh_id = match generate_synthetic_id(settings, &req) { + Ok(id) => id, + Err(e) => return Ok(to_error_response(e)), + }; + + // Get or generate Trusted Server ID + let trusted_server_id = match get_or_generate_synthetic_id(settings, &req) { + Ok(id) => id, + Err(e) => return Ok(to_error_response(e)), + }; + + // Create response with EdgePubs template + let mut response = Response::from_status(StatusCode::OK) + .with_body(EDGEPUBS_TEMPLATE) + .with_header(header::CONTENT_TYPE, "text/html") + .with_header(header::CACHE_CONTROL, "no-store, private") + .with_header(HEADER_X_COMPRESS_HINT, "on"); + + // Add synthetic ID headers + response.set_header(HEADER_SYNTHETIC_FRESH, &fresh_id); + response.set_header(HEADER_SYNTHETIC_TRUSTED_SERVER, &trusted_server_id); + + // Add DMA code header if available + if let Some(dma) = dma_code { + response.set_header(HEADER_X_GEO_METRO_CODE, dma); + } + + // Set synthetic ID cookie + let cookie = create_synthetic_cookie(settings, &trusted_server_id); + response.set_header(header::SET_COOKIE, cookie); + + Ok(response) +} + +/// Check if the path is for an Equativ asset that should be served directly (like auburndao.com) +fn is_partner_asset_path(path: &str) -> bool { + // Only handle Equativ/Smart AdServer assets for now + path.contains("/diff/") || // Equativ assets + path.ends_with(".png") || // Images + path.ends_with(".jpg") || // Images + path.ends_with(".gif") // Images +} + +/// Handles direct asset serving for partner domains (like auburndao.com). +/// +/// Fetches assets from original partner domains and serves them as first-party content. +/// This bypasses ad blockers and Safari ITP by making all assets appear to come from edgepubs.com. +/// +/// # Errors +/// +/// Returns a Fastly [`Error`] if asset fetching fails. +async fn handle_partner_asset(_settings: &Settings, req: Request) -> Result { + let path = req.get_path(); + println!("=== HANDLING PARTNER ASSET: {} ===", path); + log::info!("Handling partner asset request: {}", path); + + // Only handle Equativ/Smart AdServer assets (matching auburndao.com approach) + let (backend_name, original_host) = ("equativ_sascdn_backend", "creatives.sascdn.com"); + + log::info!( + "Serving asset from backend: {} (original host: {})", + backend_name, + original_host + ); + + // Construct full URL using the original host and path + let full_url = format!("https://{}{}", original_host, path); + log::info!("Fetching asset URL: {}", full_url); + + let mut asset_req = Request::new(req.get_method().clone(), &full_url); + + // Copy all headers from original request + for (name, value) in req.get_headers() { + asset_req.set_header(name, value); + } + + // Set the Host header to the original domain for proper routing + asset_req.set_header(header::HOST, original_host); + + // Send to appropriate backend + match asset_req.send(backend_name) { + Ok(mut response) => { + // Match auburndao.com cache control exactly + let cache_control = "max-age=31536000"; + + // No content rewriting needed for Equativ assets (they're mostly images) + // This matches the auburndao.com approach of serving assets directly + + // Match auburndao.com headers exactly - no modifications + response.set_header(header::CACHE_CONTROL, cache_control); + + // Don't modify any other headers - keep them exactly as auburndao.com gets them + + println!("=== ASSET RESPONSE HEADERS FOR {} ===", path); + for (name, value) in response.get_headers() { + println!(" {}: {:?}", name, value); + } + + // No special CORB handling needed for Equativ image assets + + log::info!( + "Partner asset served successfully, cache-control: {}", + cache_control + ); + Ok(response) + } + Err(e) => { + log::error!( + "Error fetching partner asset from {} (original host: {}): {:?}", + backend_name, + original_host, + e + ); + Ok(Response::from_status(StatusCode::NOT_FOUND) + .with_header(header::CONTENT_TYPE, "text/plain") + .with_header("X-Original-Host", original_host) + .with_header("X-Backend-Used", backend_name) + .with_body(format!("Asset not found: Unable to fetch from {} (original: {})\\nPath: {}\\nError: {:?}", backend_name, original_host, path, e))) + } + } +} + +/// Check if the path is for a GAM asset (separate from Equativ) +fn is_gam_asset_path(path: &str) -> bool { + // Instead of trying to match specific path patterns, we check if this looks like + // a request that was originally intended for a GAM domain but got rewritten to edgepubs.com + + // Common GAM paths that we know about + path.contains("/tag/js/") || // Google Tag Manager/GAM scripts (including our renamed test.js) + path.contains("/pagead/") || // GAM ad serving and interactions + path.contains("/gtag/js") || // Google Analytics/GAM gtag scripts + path.contains("/gampad/") || // GAM ad requests (gampad/ads) + path.contains("/bg/") || // GAM background scripts + path.contains("/sodar") || // GAM traffic quality checks + path.contains("/getconfig/") || // GAM configuration requests + path.contains("/generate_204") || // GAM tracking pixels + path.contains("/recaptcha/") || // reCAPTCHA requests + path.contains("/static/topics/") || // GAM topics framework + path.contains("safeframe") // GAM safe frame containers +} + +/// Handles GAM asset serving (completely separate from Equativ) +/// Rewrite hardcoded URLs in GAM JavaScript to use first-party proxy +fn rewrite_gam_urls(content: &str) -> String { + log::info!("Starting GAM URL rewriting..."); + + // Define the URL mappings based on the user's configuration + let url_mappings = [ + // Primary GAM domains + ("securepubads.g.doubleclick.net", "edgepubs.com"), + ("googletagservices.com", "edgepubs.com"), + ("googlesyndication.com", "edgepubs.com"), + ("pagead2.googlesyndication.com", "edgepubs.com"), + ("tpc.googlesyndication.com", "edgepubs.com"), + + // GAM-specific subdomains that might appear + ("www.googletagservices.com", "edgepubs.com"), + ("www.googlesyndication.com", "edgepubs.com"), + ("static.googleadsserving.cn", "edgepubs.com"), + + // Ad serving domains + ("doubleclick.net", "edgepubs.com"), + ("www.google.com/adsense", "edgepubs.com/adsense"), + + // Google ad quality and traffic domains (these were missing!) + ("adtrafficquality.google", "edgepubs.com"), + ("ep1.adtrafficquality.google", "edgepubs.com"), + ("ep2.adtrafficquality.google", "edgepubs.com"), + ("ep3.adtrafficquality.google", "edgepubs.com"), + + // Other Google ad-related domains + ("6ab9b2c571ea5e8cf287325e9ebeaa41.safeframe.googlesyndication.com", "edgepubs.com"), + ("www.google.com/recaptcha", "edgepubs.com/recaptcha"), + ]; + + let mut rewritten_content = content.to_string(); + let mut total_replacements = 0; + + for (original_domain, proxy_domain) in &url_mappings { + // Count replacements for this domain + let before_count = rewritten_content.matches(original_domain).count(); + + if before_count > 0 { + log::info!( + "Found {} occurrences of '{}' to rewrite", + before_count, + original_domain + ); + + // Replace both HTTP and HTTPS versions + rewritten_content = rewritten_content.replace( + &format!("https://{}", original_domain), + &format!("https://{}", proxy_domain), + ); + rewritten_content = rewritten_content.replace( + &format!("http://{}", original_domain), + &format!("https://{}", proxy_domain), + ); + + // Also replace protocol-relative URLs (//domain.com) + rewritten_content = rewritten_content.replace( + &format!("//{}", original_domain), + &format!("//{}", proxy_domain), + ); + + // Replace domain-only references (for cases where protocol is added separately) + rewritten_content = rewritten_content.replace( + &format!("\"{}\"", original_domain), + &format!("\"{}\"", proxy_domain), + ); + rewritten_content = rewritten_content.replace( + &format!("'{}'", original_domain), + &format!("'{}'", proxy_domain), + ); + + let after_count = rewritten_content.matches(original_domain).count(); + let replacements = before_count - after_count; + total_replacements += replacements; + + if replacements > 0 { + log::info!( + "Replaced {} occurrences of '{}' with '{}'", + replacements, + original_domain, + proxy_domain + ); + } + } + } + + log::info!( + "GAM URL rewriting complete. Total replacements: {}", + total_replacements + ); + + // Log a sample of the rewritten content for debugging (first 500 chars) + if total_replacements > 0 { + let sample_length = std::cmp::min(500, rewritten_content.len()); + log::debug!( + "Rewritten content sample: {}", + &rewritten_content[..sample_length] + ); + } + + rewritten_content +} + +async fn handle_gam_asset(_settings: &Settings, req: Request) -> Result { + let path = req.get_path(); + println!("=== HANDLING GAM ASSET: {} ===", path); + log::info!("Handling GAM asset request: {}", path); + + // Enhanced logging for GAM requests + log::info!("GAM Asset Request Details:"); + log::info!(" - Path: {}", path); + log::info!(" - Method: {}", req.get_method()); + log::info!(" - Full URL: {}", req.get_url()); + + // Log all request headers for debugging + log::info!("GAM Asset Request Headers:"); + for (name, value) in req.get_headers() { + log::info!(" {}: {:?}", name, value); + } + + // Log query parameters if any + if let Some(query) = req.get_url().query() { + log::info!("GAM Asset Query Parameters: {}", query); + } + + // For domain-level proxying, we assume all GAM requests go to the main GAM backend + // unless we detect specific patterns that need different backends + let (backend_name, original_host, target_path) = if path.contains("/tag/js/test.js") { + // Special case: our renamed test.js should map to the original gpt.js + ("gam_backend", "securepubads.g.doubleclick.net", "/tag/js/gpt.js".to_string()) + } else if path.contains("/pagead/") && path.contains("googlesyndication") { + ( + "pagead2_googlesyndication_backend", + "pagead2.googlesyndication.com", + path.to_string() + ) + } else { + // Default: all other GAM requests go to main GAM backend with original path + ("gam_backend", "securepubads.g.doubleclick.net", path.to_string()) + }; + + log::info!( + "Serving GAM asset from backend: {} (original host: {})", + backend_name, + original_host + ); + + // Construct full URL using the original host and target path (may be different from request path) + let mut full_url = format!("https://{}{}", original_host, target_path); + + // Add query string if present + if let Some(query) = req.get_url().query() { + full_url.push('?'); + full_url.push_str(query); + } + + // Special handling for /gampad/ads requests - rewrite URL parameter to use autoblog.com + if target_path.contains("/gampad/ads") { + log::info!("Applying URL parameter rewriting for GAM ad request"); + // Change url=https%3A%2F%2Fedgepubs.com%2F to url=https%3A%2F%2Fwww.autoblog.com%2F + full_url = full_url.replace( + "url=https%3A%2F%2Fedgepubs.com%2F", + "url=https%3A%2F%2Fwww.autoblog.com%2F" + ); + log::info!("Rewrote URL parameter from edgepubs.com to www.autoblog.com"); + } + + log::info!("Fetching GAM asset URL: {} (original request: {})", full_url, path); + + let mut asset_req = Request::new(req.get_method().clone(), &full_url); + + // Copy all headers from original request + for (name, value) in req.get_headers() { + asset_req.set_header(name, value); + } + + // Set the Host header to the original domain for proper routing + asset_req.set_header(header::HOST, original_host); + + // Send to appropriate GAM backend + log::info!( + "Sending GAM asset request to backend '{}' for URL: {}", + backend_name, + full_url + ); + match asset_req.send(backend_name) { + Ok(mut response) => { + log::info!( + "Received GAM asset response: status={}, content-length={:?}", + response.get_status(), + response.get_header(header::CONTENT_LENGTH) + ); + // Check if this is a JavaScript response that needs content rewriting + let content_type = response + .get_header(header::CONTENT_TYPE) + .and_then(|h| h.to_str().ok()) + .unwrap_or(""); + + // Enable rewriting for JavaScript content + let needs_rewriting = content_type.contains("javascript") || path.contains(".js"); + log::info!("Content rewriting enabled for JavaScript: {}", needs_rewriting); + + log::info!( + "GAM asset content-type: {}, needs_rewriting: {}", + content_type, + needs_rewriting + ); + + if needs_rewriting { + // Step 1: Capture original content-disposition header before processing + let original_content_disposition = response.get_header("content-disposition") + .and_then(|h| h.to_str().ok()) + .map(|s| s.to_string()); + + log::info!("Captured original content-disposition: {:?}", original_content_disposition); + + // Step 2: Remove content-disposition header so we can process the response properly + response.remove_header("content-disposition"); + log::info!("Step 2: Temporarily removed content-disposition header for processing"); + + // Get the response body as bytes first + let body_bytes = response.take_body_bytes(); + let original_length = body_bytes.len(); + + log::info!( + "Original GAM JavaScript body length: {} bytes", + original_length + ); + + // Check if we need to decompress Brotli data + let decompressed_body = if response.get_header("content-encoding") + .and_then(|h| h.to_str().ok()) == Some("br") { + + log::info!("Detected Brotli compression, decompressing..."); + let mut decompressed = Vec::new(); + match brotli::Decompressor::new(&body_bytes[..], 4096) + .read_to_end(&mut decompressed) { + Ok(bytes_read) => { + log::info!("Successfully decompressed {} bytes to {} bytes", original_length, bytes_read); + decompressed + } + Err(e) => { + log::error!("Failed to decompress Brotli data: {:?}", e); + return Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_body(format!("Failed to decompress GAM asset: {:?}", e))); + } + } + } else { + log::info!("No compression detected, using raw bytes"); + body_bytes + }; + + // Now safely convert decompressed data to string + let body = match std::str::from_utf8(&decompressed_body) { + Ok(body_str) => { + log::info!("Successfully decoded decompressed response as UTF-8"); + body_str.to_string() + } + Err(e) => { + log::error!("Decompressed response contains non-UTF-8 data: {:?}", e); + return Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_body(format!("Invalid UTF-8 in decompressed GAM asset: {:?}", e))); + } + }; + + // Log a sample of the original content for debugging (safely) + let sample = if body.len() > 200 { + // Find a safe character boundary near 200 bytes + match body.char_indices().nth(100) { // Get first ~100 characters instead of 200 bytes + Some((byte_idx, _)) => &body[..byte_idx], + None => &body[..std::cmp::min(50, body.len())] // Fallback to very short sample + } + } else { + &body + }; + log::info!("Original content sample: {}", sample); + + // Rewrite hardcoded URLs to use first-party proxy + let rewritten_body = rewrite_gam_urls(&body); + let rewritten_length = rewritten_body.len(); + + log::info!( + "Rewritten GAM JavaScript body length: {} bytes (diff: {})", + rewritten_length, + rewritten_length as i32 - original_length as i32 + ); + + // Log a sample of the rewritten content for comparison (safely) + if rewritten_length != original_length { + let rewritten_sample = if rewritten_body.len() > 200 { + // Find a safe character boundary near 200 bytes + match rewritten_body.char_indices().nth(100) { + Some((byte_idx, _)) => &rewritten_body[..byte_idx], + None => &rewritten_body[..std::cmp::min(50, rewritten_body.len())] + } + } else { + &rewritten_body + }; + log::info!("Rewritten content sample: {}", rewritten_sample); + } else { + log::warn!("No content changes detected - rewriting may not have found target URLs"); + } + + // Create new response with rewritten content + let mut new_response = Response::from_status(response.get_status()); + + // Copy headers from original response, but filter out problematic ones + for (name, value) in response.get_headers() { + let header_name = name.as_str().to_lowercase(); + // Skip headers that would cause problems with rewritten content + if header_name == "content-disposition" { + log::info!("Skipping problematic header: {}: {:?}", name, value); + continue; + } + // Skip compression headers since our rewritten content is uncompressed + if header_name == "content-encoding" || header_name == "content-length" { + log::info!("Skipping compression header for rewritten content: {}: {:?}", name, value); + continue; + } + new_response.set_header(name, value); + } + + // Set rewritten content (uncompressed to avoid timeout issues) + // Note: compression headers are already filtered out above + new_response.set_body(rewritten_body); + log::info!("Sending rewritten content uncompressed to avoid Fastly timeout limits"); + + // Disable caching during development/debugging + let cache_control = "no-cache, no-store, must-revalidate"; + new_response.set_header(header::CACHE_CONTROL, cache_control); + new_response.set_header("Pragma", "no-cache"); + new_response.set_header("Expires", "0"); + + // Step 3: Restore original content-disposition header if it existed + if let Some(original_disposition) = original_content_disposition { + new_response.set_header("content-disposition", &original_disposition); + log::info!("Step 3: Restored original content-disposition header: {}", original_disposition); + } + + // Add debug headers + new_response.set_header("X-Content-Rewritten", "true"); + new_response.set_header("X-Original-Length", &original_length.to_string()); + new_response.set_header("X-Rewritten-Length", &rewritten_length.to_string()); + + println!( + "=== GAM ASSET RESPONSE HEADERS FOR {} (REWRITTEN) ===", + path + ); + log::info!("GAM Asset Response Headers (Rewritten):"); + for (name, value) in new_response.get_headers() { + println!(" {}: {:?}", name, value); + log::info!(" {}: {:?}", name, value); + } + + log::info!("GAM JavaScript asset rewritten and served successfully"); + Ok(new_response) + } else { + // No rewriting needed, serve as-is but disable caching for debugging + let cache_control = "no-cache, no-store, must-revalidate"; + response.set_header(header::CACHE_CONTROL, cache_control); + response.set_header("Pragma", "no-cache"); + response.set_header("Expires", "0"); + + println!("=== GAM ASSET RESPONSE HEADERS FOR {} ===", path); + log::info!("GAM Asset Response Headers (No Rewriting):"); + for (name, value) in response.get_headers() { + println!(" {}: {:?}", name, value); + log::info!(" {}: {:?}", name, value); + } + + log::info!( + "GAM asset served successfully, cache-control: {}", + cache_control + ); + Ok(response) + } + } + Err(e) => { + log::error!( + "Error fetching GAM asset from {} (original host: {}): {:?}", + backend_name, + original_host, + e + ); + log::error!("GAM Asset Error Details:"); + log::error!(" - Backend: {}", backend_name); + log::error!(" - Original Host: {}", original_host); + log::error!(" - Full URL: {}", full_url); + log::error!(" - Path: {}", path); + log::error!(" - Error: {:?}", e); + + println!("=== GAM ASSET ERROR DETAILS ==="); + println!("Backend: {}", backend_name); + println!("Original Host: {}", original_host); + println!("Full URL: {}", full_url); + println!("Error: {:?}", e); + Ok(Response::from_status(StatusCode::NOT_FOUND) + .with_header(header::CONTENT_TYPE, "text/plain") + .with_header("X-Original-Host", original_host) + .with_header("X-Backend-Used", backend_name) + .with_header("X-Full-URL", &full_url) + .with_body(format!("GAM Asset not found: Unable to fetch from {} (original: {})\\nPath: {}\\nFull URL: {}\\nError: {:?}", backend_name, original_host, path, full_url, e))) + } + } +} + /// Handles the main page request. /// /// Serves the main page with synthetic ID generation and ad integration. @@ -155,7 +744,7 @@ fn handle_main_page(settings: &Settings, mut req: Request) -> Result Result) + .apply() + .expect("Failed to initialize logger"); +} diff --git a/fastly.toml b/fastly.toml index 8c17b43..a4bd9ac 100644 --- a/fastly.toml +++ b/fastly.toml @@ -3,17 +3,43 @@ authors = ["jason@stackpop.com"] cloned_from = "https://github.com/fastly/compute-starter-kit-rust-default" -description = "Trusted Server" +description = "EdgePubs - Trusted Server" language = "rust" manifest_version = 3 name = "trusted-server-fastly" +service_id = "dysUw6h73VzeomD61eal85" [scripts] build = """ cargo build --bin trusted-server-fastly --release --target wasm32-wasip1 --color always """ +# Production Backends for Partner Asset Serving (like auburndao.com) +[backends] + + [backends.tpc_googlesyndication_backend] + url = "https://tpc.googlesyndication.com" + + [backends.GAM_javascript_backend] + url = "https://pagead2.googlesyndication.com" + + [backends.GTS_services_backend] + url = "https://googletagservices.com" + + [backends.pagead2_googlesyndication_backend] + url = "https://googlesyndication.com" + + [backends.equativ_sascdn_backend] + url = "https://creatives.sascdn.com" + + [backends.googleads_doubleclick_backend] + url = "https://securepubads.g.doubleclick.net" + [local_server] + address = "127.0.0.1:7676" + tls_cert_path = "./localhost+2.pem" + tls_key_path = "./localhost+2-key.pem" + [local_server.backends] [local_server.backends.equativ_ad_api_2] @@ -24,6 +50,22 @@ build = """ url = "https://securepubads.g.doubleclick.net" [local_server.backends.wordpress_backend] url = "http://localhost:8080" # Adjust this to your local WordPress URL + [local_server.backends.edgepubs_main_be] + url = "https://api.edgepubs.com" # EdgePubs API backend (update when you provide the real backend URL) + + # Partner Asset Backends (matching auburndao.com approach) + [local_server.backends.tpc_googlesyndication_backend] + url = "https://tpc.googlesyndication.com" + [local_server.backends.GAM_javascript_backend] + url = "https://pagead2.googlesyndication.com" + [local_server.backends.GTS_services_backend] + url = "https://googletagservices.com" + [local_server.backends.pagead2_googlesyndication_backend] + url = "https://googlesyndication.com" + [local_server.backends.equativ_sascdn_backend] + url = "https://creatives.sascdn.com" + [local_server.backends.googleads_doubleclick_backend] + url = "https://securepubads.g.doubleclick.net" [local_server.kv_stores] diff --git a/localhost+2-key.pem b/localhost+2-key.pem new file mode 100644 index 0000000..9efa871 --- /dev/null +++ b/localhost+2-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC8azFdM30OSsQu +LlCv2iiu/Nbm+aXqWmWp///5/peVB/0jhcBcLBq17EPmZV6n15HU4fDRz4z/wouB +k4xEale0IlDhKxWyQkCyt0miarODUILbBwOzqo/Srsem8BnXSsd3sTXR30AJgNAr +lfgdyVdMLRviP5TcRz7QUZ2ukcEEutjUa1PIcKHpHW5+JPre1DYmNHVpvl2gu5zO +vRtva3CFx4bq8MspzWVnay9URlmyyKoqvS42hk1VPEa7JPxGGBwPAgMj0LrxZPio +J297N2d53asHzV9qWoNJ7HCAIfE2cOmtlgQhRzTzls6JAOBs0Ei+a8L8b135meVG +wRzsfzpNAgMBAAECggEAEGnSEgljMnSG5HL6amc5tgymd9Z1gpW/V3DKjq+D6uKY +RZjoKd9ZsFFVlONtvHdF5+KKxOwsnzcGShJdXLRa0YFPxF2pdS7aTfczfeeHnb/M +tmHFC8im4LaDA+LUjXGhMKcKRi8JeAp1l+Znh8i7RxKC5FGfmRTHvwPtFYDy7INn +8s1voRjK+my+R6gkAtPJU2VkUxgB2iozfs8h2irzg1eiRJ5AIPuxaddQC1nsOxh5 +Uo5CtG+ghNYzKXrmlDeEylXx4JYCnuRTbAoU5Law1QVAblGnMdmNe7P5USXk3DSU +qGTxWz3x8h0qT+P4V8Bd9B45wO0VomFUzTYOHp9hwQKBgQDswWpSALxnaITSg/45 +QPfYPtqVocCKEvbPHfvU2lVoSFuhOMuReBiNyiONP/kvwbycMu5JeJzYQNnFfvFR +7QMKLaEjxMWEcbu8Y0V3zdpjxQKhfMcK83jBjsgVErEQmCGL7SE0nryZ91KqU9cs +Iz+6s/KrGquNWDfDfpaikNmsvQKBgQDLu/MKuXhoGKqDp7bB16AuxGMrOkcOEG83 +YaBgk7RtNbPAMnVeUbVzxfacFerM5Qr1qZ+S9u931skDMzsPxi1fpFxsdq4oVUnM +dn/m9eQB+HkJ1NR87wYEwe/7PLBT5sELpe2Ucga33FWFVPFfdbOiVRTJ5VCMF39Q +y9lvQHNE0QKBgAO7VLqwKkgS/Kz4yaQLYxy0eOuM3iUPAXboSRTk0B3RrigPZh4H +35M0kXTjgkD/c8KehuT/krFR7FzRkGozKy4Y5AZVTqu0wznH8stdokiaTV1QQy7o +AYpDfuHeJo5VZ4n2FeTnb5+Zp2Lp6WVXtORze6q4w+V5c3+QWj17vQKdAoGBAJPm +8QzTEmc3qOoE4jcc62uoJHooR5cdNTAWTpBuCijSOJkJyIzdcoYGq/4qJxfyGx+2 +sWLGo5fzNGB/W5wa+auaLYKMMeiT6gxl5npr/ce0c2JAwLWl7E+d3zMbnsJ6HaDw +J2aB1QC7EtbFkKkFUJBDheD/+ZaKJGIqasJKHqRRAoGALSojLBlT6rViG2626V2G +CJ7v3M7oAsHDplds8GgYjSxAAZRUchkDITkImivpsvWmm89mCOxdKiPqfcN4T5DX +fKoDXkBRZY0CvyqYl964ZUc1lftFRLvjjoV7yyZxslDkEg31ukw4isnx07Mn/nds +MkL/sCs6D25Z4fXm3oSbrtI= +-----END PRIVATE KEY----- diff --git a/localhost+2.pem b/localhost+2.pem new file mode 100644 index 0000000..4ded880 --- /dev/null +++ b/localhost+2.pem @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEazCCAtOgAwIBAgIRAMYFg8+z3gU9P6SNc57FYgMwDQYJKoZIhvcNAQELBQAw +gY0xHjAcBgNVBAoTFW1rY2VydCBkZXZlbG9wbWVudCBDQTExMC8GA1UECwwoamV2 +YW5zQGpldmFucy1tNC1wcm8ubG9jYWwgKEphc29uIEV2YW5zKTE4MDYGA1UEAwwv +bWtjZXJ0IGpldmFuc0BqZXZhbnMtbTQtcHJvLmxvY2FsIChKYXNvbiBFdmFucykw +HhcNMjUwNzE3MTgzNjIwWhcNMjcxMDE3MTgzNjIwWjBcMScwJQYDVQQKEx5ta2Nl +cnQgZGV2ZWxvcG1lbnQgY2VydGlmaWNhdGUxMTAvBgNVBAsMKGpldmFuc0BqZXZh +bnMtbTQtcHJvLmxvY2FsIChKYXNvbiBFdmFucykwggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQC8azFdM30OSsQuLlCv2iiu/Nbm+aXqWmWp///5/peVB/0j +hcBcLBq17EPmZV6n15HU4fDRz4z/wouBk4xEale0IlDhKxWyQkCyt0miarODUILb +BwOzqo/Srsem8BnXSsd3sTXR30AJgNArlfgdyVdMLRviP5TcRz7QUZ2ukcEEutjU +a1PIcKHpHW5+JPre1DYmNHVpvl2gu5zOvRtva3CFx4bq8MspzWVnay9URlmyyKoq +vS42hk1VPEa7JPxGGBwPAgMj0LrxZPioJ297N2d53asHzV9qWoNJ7HCAIfE2cOmt +lgQhRzTzls6JAOBs0Ei+a8L8b135meVGwRzsfzpNAgMBAAGjdjB0MA4GA1UdDwEB +/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATAfBgNVHSMEGDAWgBQIZHj9s28t +wMjgCCK0FRfIG4eRwDAsBgNVHREEJTAjgglsb2NhbGhvc3SHBH8AAAGHEAAAAAAA +AAAAAAAAAAAAAAEwDQYJKoZIhvcNAQELBQADggGBAEbVeOvFa/9mTS7Q0gMid715 +Y5LtAToIyAlPpdHCR5lC8Z5nP1xSfFgCtzdM8OlDU7KnPjISOXLjJgPHOCp+VVeo +OB/RAwDUcIK1s0SR2xfaeUwfMTgBVJm4Ee02klIooiAd1HMVi4LkfFtqQ+eiusBg +g8HkL8qRLMbv1EwG4ZIsRTy2r+ZbTZetN+uomIOfii97unHHarJgme2WaCCTGg3/ +krwR/6nvpSjz3ZUdlo1ZGltaKwWFM7jI/fi0N9VWr2hAT1wcpZnLc0Ei4U3I72NY +yOEtzm7in6m9egZ/g31Gehy3FaHQ04wE6XCwA1PnvVXDKumS0v95f+GuItg4Yx5r +SnSVknmUVDMiQr2F7yh91XLjR1VW5cZqKBhkcC7b6RJFI+6iS99a8kMhDmxAUQZx +lYrjFpr4s0k5j7B+eQa+GgbgE4dbKl/ROkLSC72rw0Z4trzAhIcKfzcHdr/RP7ye +htPRoZVU8Sz8+Cx4MwNtOLZCaO8G0f7w2j6Ut4/CLA== +-----END CERTIFICATE----- diff --git a/trusted-server.toml b/trusted-server.toml index 0a3c3b2..dde7bde 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -11,7 +11,7 @@ sync_url = "https://adapi-srv-eu.smartadserver.com/ac?pgid=2040327&fmtid=137675& server_url = "http://68.183.113.79:8000/openrtb2/auction" [gam] -publisher_id = "3790" +publisher_id = "88059007" server_url = "https://securepubads.g.doubleclick.net/gampad/ads" ad_units = [ { name = "Flex8:1", size = "flexible" }, @@ -32,3 +32,35 @@ secret_key = "trusted-server" # - "publisher_domain" # - "accept_language" template = "{{ client_ip }}:{{ user_agent }}:{{ first_party_id }}:{{ auth_user_id }}:{{ publisher_domain }}:{{ accept_language }}" + +# Partner Proxy Configuration +[partners] + +[partners.gam] +enabled = true +name = "Google Ad Manager" +domains_to_proxy = [ + "securepubads.g.doubleclick.net", + "tpc.googlesyndication.com", + "googletagservices.com", + "googlesyndication.com" +] +proxy_domain = "www.edgepubs.com" +backend_name = "gam_proxy_backend" + +[partners.equativ] +enabled = true +name = "Equativ (Smart AdServer)" +domains_to_proxy = [ + "creatives.sascdn.com", + "adapi-srv-eu.smartadserver.com" +] +proxy_domain = "www.edgepubs.com" +backend_name = "equativ_proxy_backend" + +[partners.prebid] +enabled = true +name = "Prebid Server" +domains_to_proxy = ["68.183.113.79"] +proxy_domain = "www.edgepubs.com" +backend_name = "prebid_proxy_backend" From 3231261d88a0419a16c9deeaec3450a50c265ec6 Mon Sep 17 00:00:00 2001 From: jevansnyc Date: Mon, 1 Sep 2025 13:41:55 -0500 Subject: [PATCH 09/11] feat: expand GAM domain rewriting and fix response handlers - Add missing Google domains to GAM partner configuration (10 new domains) - Update handle_gam_test to apply domain rewriting to responses - Import PartnerManager for consistent domain rewriting across GAM handlers - Prepare for server-side ad orchestration architecture --- crates/common/src/gam.rs | 105 +++++++++++++++++++++------------------ trusted-server.toml | 12 ++++- 2 files changed, 67 insertions(+), 50 deletions(-) diff --git a/crates/common/src/gam.rs b/crates/common/src/gam.rs index f267ed6..885150e 100644 --- a/crates/common/src/gam.rs +++ b/crates/common/src/gam.rs @@ -1,4 +1,5 @@ use crate::gdpr::get_consent_from_request; +use crate::partners::PartnerManager; use crate::settings::Settings; use fastly::http::{header, Method, StatusCode}; use fastly::{Error, Request, Response}; @@ -57,61 +58,46 @@ impl GamRequest { self } - /// Build the GAM request URL for the "Golden URL" replay phase + /// Build the GAM request URL - simplified traditional format pub fn build_golden_url(&self) -> String { - // This will be replaced with the actual captured URL from autoblog.com - // For now, using a template based on the captured Golden URL + // Using simplified traditional GAM ad request format (not SRA) let mut params = HashMap::new(); - // Core GAM parameters (based on captured URL) - params.insert("pvsid".to_string(), "3290837576990024".to_string()); // Publisher Viewability ID - params.insert("correlator".to_string(), self.correlator.clone()); - params.insert( - "eid".to_string(), - "31086815,31093089,95353385,31085777,83321072".to_string(), - ); // Event IDs - params.insert("output".to_string(), "ldjh".to_string()); // Important: not 'json' - params.insert("gdfp_req".to_string(), "1".to_string()); - params.insert("vrg".to_string(), "202506170101".to_string()); // Version/Region - params.insert("ptt".to_string(), "17".to_string()); // Page Type - params.insert("impl".to_string(), "fifs".to_string()); // Implementation - - // Ad unit parameters (simplified version of captured format) - params.insert( - "iu_parts".to_string(), - format!("{},{},homepage", self.publisher_id, "trustedserver"), - ); - params.insert( - "enc_prev_ius".to_string(), - "/0/1/2,/0/1/2,/0/1/2".to_string(), - ); - params.insert("prev_iu_szs".to_string(), "320x50|300x250|728x90|970x90|970x250|1x2,320x50|300x250|728x90|970x90|970x250|1x2,320x50|300x250|728x90|970x90|970x250|1x2".to_string()); - params.insert("fluid".to_string(), "height,height,height".to_string()); - - // Browser context (simplified) - params.insert("biw".to_string(), "1512".to_string()); - params.insert("bih".to_string(), "345".to_string()); - params.insert("u_tz".to_string(), "-300".to_string()); - params.insert("u_cd".to_string(), "30".to_string()); - params.insert("u_sd".to_string(), "2".to_string()); - + // Basic GAM ad request parameters + params.insert("iu".to_string(), format!("/{}/autoblog/reviews", self.publisher_id)); + params.insert("sz".to_string(), "728x90|300x250".to_string()); // Multiple sizes + params.insert("c".to_string(), self.correlator.clone()); + + // Output format + params.insert("output".to_string(), "html".to_string()); // Standard HTML output + params.insert("impl".to_string(), "s".to_string()); // Standard implementation + // Page context - params.insert("url".to_string(), self.page_url.clone()); - params.insert( - "dt".to_string(), - chrono::Utc::now().timestamp_millis().to_string(), - ); - - // Add Permutive context if available (in cust_params like the captured URL) - if let Some(ref prmtvctx) = self.prmtvctx { - let cust_params = format!("permutive={}&puid={}", prmtvctx, self.synthetic_id); - params.insert("cust_params".to_string(), cust_params); - } + params.insert("url".to_string(), urlencoding::encode("https://www.autoblog.com/reviews/test").to_string()); + params.insert("ref".to_string(), urlencoding::encode("https://www.autoblog.com/").to_string()); + + // Basic targeting - URL encoded + let cust_params = if let Some(ref prmtvctx) = self.prmtvctx { + format!("channel=web&cv=lifestyle&lang=en&pagetype=photo-gallery-article&permutive={}", prmtvctx) + } else { + "channel=web&cv=lifestyle&lang=en&pagetype=photo-gallery-article".to_string() + }; + params.insert("cust_params".to_string(), urlencoding::encode(&cust_params).to_string()); + + // Publisher Provided ID + params.insert("ppid".to_string(), format!("arenaGroup-{}", self.synthetic_id)); + + // Timestamps + params.insert("dt".to_string(), chrono::Utc::now().timestamp_millis().to_string()); + + // Browser hints (minimal) + params.insert("biw".to_string(), "1024".to_string()); + params.insert("bih".to_string(), "768".to_string()); // Build query string let query_string = params .iter() - .map(|(k, v)| format!("{}={}", k, urlencoding::encode(v))) + .map(|(k, v)| format!("{}={}", k, v)) .collect::>() .join("&"); @@ -300,9 +286,30 @@ pub async fn handle_gam_test(settings: &Settings, req: Request) -> Result { + Ok(mut response) => { log::info!("GAM request successful"); - Ok(response) + + // Apply domain rewriting to the response body + let partner_manager = PartnerManager::from_settings(settings); + let original_body = response.take_body_str(); + let rewritten_body = partner_manager.rewrite_content(&original_body); + + if original_body != rewritten_body { + log::info!("Applied domain rewriting to GAM response (changed {} to {} chars)", + original_body.len(), rewritten_body.len()); + } else { + log::debug!("No domain rewriting needed for GAM response"); + } + + Ok(Response::from_status(response.get_status()) + .with_header(header::CONTENT_TYPE, response.get_header(header::CONTENT_TYPE).unwrap_or(&header::HeaderValue::from_static("text/plain"))) + .with_header(header::CACHE_CONTROL, "no-store, private") + .with_header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .with_header("X-GAM-Test", "true") + .with_header("X-Synthetic-ID", &gam_req_with_context.synthetic_id) + .with_header("X-Correlator", &gam_req_with_context.correlator) + .with_header("x-compress-hint", "on") + .with_body(rewritten_body)) } Err(e) => { log::error!("GAM request failed: {:?}", e); diff --git a/trusted-server.toml b/trusted-server.toml index dde7bde..b10c211 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -43,7 +43,17 @@ domains_to_proxy = [ "securepubads.g.doubleclick.net", "tpc.googlesyndication.com", "googletagservices.com", - "googlesyndication.com" + "googlesyndication.com", + "googleads.g.doubleclick.net", + "stats.g.doubleclick.net", + "cm.g.doubleclick.net", + "ad.doubleclick.net", + "pagead2.googlesyndication.com", + "safeframe.googlesyndication.com", + "ep1.adtrafficquality.google", + "ep2.adtrafficquality.google", + "ep3.adtrafficquality.google", + "adtrafficquality.google" ] proxy_domain = "www.edgepubs.com" backend_name = "gam_proxy_backend" From e65fe258c764808a0a41b5573405a3386de8c245 Mon Sep 17 00:00:00 2001 From: jevansnyc Date: Mon, 1 Sep 2025 13:43:49 -0500 Subject: [PATCH 10/11] feat: implement comprehensive domain rewriting system - Add encoded path routing with backend mapping for all Google domains - Expand text content rewriting beyond just JavaScript files - Add new Fastly backends for adtrafficquality and safeframe domains - Update routing order to prioritize encoded paths over generic asset handling - Fix test settings structure to include partners field --- crates/common/src/test_support.rs | 1 + crates/fastly/src/main.rs | 437 +++++++++++++++++++++--------- fastly.toml | 24 ++ 3 files changed, 335 insertions(+), 127 deletions(-) diff --git a/crates/common/src/test_support.rs b/crates/common/src/test_support.rs index 1b8da25..3ffd922 100644 --- a/crates/common/src/test_support.rs +++ b/crates/common/src/test_support.rs @@ -59,6 +59,7 @@ pub mod tests { secret_key: "test-secret-key".to_string(), template: "{{client_ip}}:{{user_agent}}:{{first_party_id}}:{{auth_user_id}}:{{publisher_domain}}:{{accept_language}}".to_string(), }, + partners: None, } } } diff --git a/crates/fastly/src/main.rs b/crates/fastly/src/main.rs index cdb772d..03bb8ef 100644 --- a/crates/fastly/src/main.rs +++ b/crates/fastly/src/main.rs @@ -68,10 +68,7 @@ fn main(req: Request) -> Result { (&Method::GET, path) if is_partner_asset_path(path) => { handle_partner_asset(&settings, req).await } - // GAM asset serving (separate from Equativ, checked after Equativ) - (&Method::GET, path) if is_gam_asset_path(path) => { - handle_gam_asset(&settings, req).await - } + // Specific routes MUST come before catch-all patterns (&Method::GET, "/prebid-test") => handle_prebid_test(&settings, req).await, (&Method::GET, "/gam-test") => handle_gam_test(&settings, req).await, (&Method::GET, "/gam-golden-url") => handle_gam_golden_url(&settings, req).await, @@ -81,6 +78,14 @@ fn main(req: Request) -> Result { .with_body(GAM_TEST_TEMPLATE) .with_header(header::CONTENT_TYPE, "text/html") .with_header("x-compress-hint", "on")), + // Encoded path routing for domain proxying + (&Method::GET, path) if is_encoded_path(path) => { + handle_encoded_path(&settings, req).await + } + // GAM asset serving (separate from Equativ, checked after encoded paths) + (&Method::GET, path) if is_text_response_path(path) => { + handle_gam_asset(&settings, req).await + }, (&Method::GET, "/gdpr/consent") => handle_consent_request(&settings, req), (&Method::POST, "/gdpr/consent") => handle_consent_request(&settings, req), (&Method::GET, "/gdpr/data") => handle_data_subject_request(&settings, req), @@ -302,13 +307,30 @@ async fn handle_partner_asset(_settings: &Settings, req: Request) -> Result bool { + // Match our encoded path patterns (10-char hex codes) + if let Some(first_segment) = path.strip_prefix('/').and_then(|p| p.split('/').next()) { + first_segment.len() == 10 && first_segment.chars().all(|c| c.is_ascii_hexdigit()) + } else { + false + } +} + /// Check if the path is for a GAM asset (separate from Equativ) -fn is_gam_asset_path(path: &str) -> bool { - // Instead of trying to match specific path patterns, we check if this looks like - // a request that was originally intended for a GAM domain but got rewritten to edgepubs.com - - // Common GAM paths that we know about - path.contains("/tag/js/") || // Google Tag Manager/GAM scripts (including our renamed test.js) +fn is_text_response_path(path: &str) -> bool { + // Broaden matching to catch all text content that might contain domain references + + // JavaScript and web assets + path.ends_with(".js") || + path.ends_with(".css") || + path.ends_with(".html") || + path.ends_with(".htm") || + path.ends_with(".json") || + path.ends_with(".xml") || + + // Google/GAM specific paths + path.contains("/tag/js/") || // Google Tag Manager/GAM scripts path.contains("/pagead/") || // GAM ad serving and interactions path.contains("/gtag/js") || // Google Analytics/GAM gtag scripts path.contains("/gampad/") || // GAM ad requests (gampad/ads) @@ -318,115 +340,237 @@ fn is_gam_asset_path(path: &str) -> bool { path.contains("/generate_204") || // GAM tracking pixels path.contains("/recaptcha/") || // reCAPTCHA requests path.contains("/static/topics/") || // GAM topics framework - path.contains("safeframe") // GAM safe frame containers + + // Broader Google domain patterns + path.contains("google") || // Any Google service + path.contains("doubleclick") || // All DoubleClick variations + path.contains("syndication") || // Google ad syndication + + // API and configuration endpoints + path.contains("/api/") || // API responses may contain URLs + path.contains("/config/") || // Configuration files + path.contains("/ads/") || // Ad-related content + + // Default: apply rewriting to most text content to be comprehensive + !path.ends_with(".png") && + !path.ends_with(".jpg") && + !path.ends_with(".jpeg") && + !path.ends_with(".gif") && + !path.ends_with(".ico") && + !path.ends_with(".woff") && + !path.ends_with(".woff2") && + !path.ends_with(".ttf") && + !path.ends_with(".eot") && + !path.ends_with(".svg") && + !path.ends_with(".mp4") && + !path.ends_with(".webm") && + !path.ends_with(".pdf") } /// Handles GAM asset serving (completely separate from Equativ) -/// Rewrite hardcoded URLs in GAM JavaScript to use first-party proxy +/// Rewrite hardcoded URLs in content to use first-party proxy domains fn rewrite_gam_urls(content: &str) -> String { - log::info!("Starting GAM URL rewriting..."); - - // Define the URL mappings based on the user's configuration - let url_mappings = [ - // Primary GAM domains - ("securepubads.g.doubleclick.net", "edgepubs.com"), - ("googletagservices.com", "edgepubs.com"), - ("googlesyndication.com", "edgepubs.com"), - ("pagead2.googlesyndication.com", "edgepubs.com"), - ("tpc.googlesyndication.com", "edgepubs.com"), - - // GAM-specific subdomains that might appear - ("www.googletagservices.com", "edgepubs.com"), - ("www.googlesyndication.com", "edgepubs.com"), - ("static.googleadsserving.cn", "edgepubs.com"), - - // Ad serving domains - ("doubleclick.net", "edgepubs.com"), - ("www.google.com/adsense", "edgepubs.com/adsense"), - - // Google ad quality and traffic domains (these were missing!) - ("adtrafficquality.google", "edgepubs.com"), - ("ep1.adtrafficquality.google", "edgepubs.com"), - ("ep2.adtrafficquality.google", "edgepubs.com"), - ("ep3.adtrafficquality.google", "edgepubs.com"), - - // Other Google ad-related domains - ("6ab9b2c571ea5e8cf287325e9ebeaa41.safeframe.googlesyndication.com", "edgepubs.com"), - ("www.google.com/recaptcha", "edgepubs.com/recaptcha"), - ]; + log::info!("Starting precise domain rewriting with encoded paths..."); let mut rewritten_content = content.to_string(); let mut total_replacements = 0; - for (original_domain, proxy_domain) in &url_mappings { - // Count replacements for this domain - let before_count = rewritten_content.matches(original_domain).count(); - - if before_count > 0 { - log::info!( - "Found {} occurrences of '{}' to rewrite", - before_count, - original_domain - ); - - // Replace both HTTP and HTTPS versions - rewritten_content = rewritten_content.replace( - &format!("https://{}", original_domain), - &format!("https://{}", proxy_domain), - ); - rewritten_content = rewritten_content.replace( - &format!("http://{}", original_domain), - &format!("https://{}", proxy_domain), - ); - - // Also replace protocol-relative URLs (//domain.com) - rewritten_content = rewritten_content.replace( - &format!("//{}", original_domain), - &format!("//{}", proxy_domain), - ); - - // Replace domain-only references (for cases where protocol is added separately) - rewritten_content = rewritten_content.replace( - &format!("\"{}\"", original_domain), - &format!("\"{}\"", proxy_domain), - ); - rewritten_content = rewritten_content.replace( - &format!("'{}'", original_domain), - &format!("'{}'", proxy_domain), - ); - - let after_count = rewritten_content.matches(original_domain).count(); - let replacements = before_count - after_count; - total_replacements += replacements; + // Define precise domain-to-encoded-path mappings (order matters - longest first to avoid conflicts) + let domain_mappings = [ + // Specific subdomains FIRST (to avoid conflicts with broader matches) + ("securepubads.g.doubleclick.net", "edgepubs.com/d4f8a2b1c3"), + ("googleads.g.doubleclick.net", "edgepubs.com/f9a2d3e7c1"), + ("stats.g.doubleclick.net", "edgepubs.com/c7e1f4a9d2"), + ("cm.g.doubleclick.net", "edgepubs.com/e3d8f2a5c9"), + ("pagead2.googlesyndication.com", "edgepubs.com/a8f3d9e2c4"), + ("tpc.googlesyndication.com", "edgepubs.com/b8d4e1f5a2"), + ("safeframe.googlesyndication.com", "edgepubs.com/g2h7f9a4e1"), + ("ep1.adtrafficquality.google", "edgepubs.com/a7e3f9d2c8"), + ("ep2.adtrafficquality.google", "edgepubs.com/b4f7a8e1d5"), + ("ep3.adtrafficquality.google", "edgepubs.com/c9a4f2e7b3"), + + // Main domains LAST (after all subdomains processed) + ("googlesyndication.com", "edgepubs.com/e7f2a9c4d1"), + ("googletagservices.com", "edgepubs.com/f3a8d9e2c7"), + ("adtrafficquality.google", "edgepubs.com/d6c9f4a2e8"), + ]; - if replacements > 0 { - log::info!( - "Replaced {} occurrences of '{}' with '{}'", - replacements, - original_domain, - proxy_domain - ); + // Process domains in order (specific subdomains first, then general domains) + for (original_domain, encoded_path) in &domain_mappings { + if rewritten_content.contains(original_domain) { + let before_len = rewritten_content.len(); + + // Replace with multiple URL patterns for thoroughness + rewritten_content = rewritten_content + .replace(&format!("https://{}", original_domain), &format!("https://{}", encoded_path)) + .replace(&format!("http://{}", original_domain), &format!("https://{}", encoded_path)) + .replace(&format!("//{}", original_domain), &format!("//{}", encoded_path)) + .replace(&format!("\"{}\"", original_domain), &format!("\"{}\"", encoded_path)) + .replace(&format!("'{}'", original_domain), &format!("'{}'", encoded_path)); + + let after_len = rewritten_content.len(); + if before_len != after_len { + total_replacements += 1; + log::info!("Rewrote domain: {} -> {}", original_domain, encoded_path); } } } log::info!( - "GAM URL rewriting complete. Total replacements: {}", + "Precise domain rewriting complete. {} domains processed", total_replacements ); - // Log a sample of the rewritten content for debugging (first 500 chars) - if total_replacements > 0 { - let sample_length = std::cmp::min(500, rewritten_content.len()); - log::debug!( - "Rewritten content sample: {}", - &rewritten_content[..sample_length] - ); - } - rewritten_content } +/// Handle requests to encoded domain paths +async fn handle_encoded_path(_settings: &Settings, req: Request) -> Result { + let path = req.get_path(); + log::info!("Handling encoded path request: {}", path); + + // Extract encoded domain and remaining path + let (encoded_domain, backend_path) = if let Some(stripped) = path.strip_prefix('/') { + if let Some((first, rest)) = stripped.split_once('/') { + (first, format!("/{}", rest)) + } else { + (stripped, "/".to_string()) + } + } else { + return Ok(Response::from_status(StatusCode::BAD_REQUEST) + .with_body("Invalid encoded path")); + }; + + // Map encoded domains to backends (now including version 48 backends) + let (backend_name, original_domain) = match encoded_domain { + // Existing backends (version 47) + "d4f8a2b1c3" => ("gam_backend", "securepubads.g.doubleclick.net"), + "a8f3d9e2c4" => ("GAM_javascript_backend", "pagead2.googlesyndication.com"), + "b8d4e1f5a2" => ("tpc_googlesyndication_backend", "tpc.googlesyndication.com"), + "e7f2a9c4d1" => ("pagead2_googlesyndication_backend", "googlesyndication.com"), + "f3a8d9e2c7" => ("GTS_services_backend", "googletagservices.com"), + + // New backends (version 48) + "a7e3f9d2c8" => ("adtraffic_backend", "ep1.adtrafficquality.google"), + "b4f7a8e1d5" => ("adtraffic_ep2_backend", "ep2.adtrafficquality.google"), + "c9a4f2e7b3" => ("adtraffic_ep3_backend", "ep3.adtrafficquality.google"), + "g2h7f9a4e1" => ("safeframe_backend", "safeframe.googlesyndication.com"), + + // Additional domains that might route to existing backends + "f9a2d3e7c1" => ("gam_backend", "googleads.g.doubleclick.net"), // Route to gam_backend + "c7e1f4a9d2" => ("gam_backend", "stats.g.doubleclick.net"), // Route to gam_backend + "e3d8f2a5c9" => ("gam_backend", "cm.g.doubleclick.net"), // Route to gam_backend + "d6c9f4a2e8" => ("adtraffic_backend", "adtrafficquality.google"), // Route to adtraffic_backend + + // Fallback to default GAM backend for unknown codes + _ => { + log::warn!("Unknown encoded domain '{}', routing to default GAM backend", encoded_domain); + ("gam_backend", "securepubads.g.doubleclick.net") + } + }; + + log::info!( + "Routing encoded path '{}' to backend '{}' ({}{})", + encoded_domain, backend_name, original_domain, backend_path + ); + + // Construct the full target URL + let mut target_url = format!("https://{}{}", original_domain, backend_path); + if let Some(query) = req.get_url().query() { + target_url = format!("{}?{}", target_url, query); + } + + log::info!("Full target URL: {}", target_url); + + // Create backend request + let mut backend_req = Request::new(req.get_method().clone(), &target_url); + + // Copy headers from original request + for (name, value) in req.get_headers() { + backend_req.set_header(name, value); + } + backend_req.set_header("host", original_domain); + + // Send request to backend + match backend_req.send(backend_name) { + Ok(mut response) => { + log::info!( + "Encoded path response: status={}, content-type={:?}", + response.get_status(), + response.get_header("content-type") + ); + + // Apply domain rewriting to response body if it's text content + let content_type = response.get_header("content-type") + .and_then(|h| h.to_str().ok()) + .unwrap_or(""); + + if content_type.contains("javascript") || content_type.contains("text") || backend_path.ends_with(".js") { + // Handle compressed content properly (like original GAM asset handler) + let body_bytes = response.take_body_bytes(); + + // Check if content is compressed + let decompressed_body = if response.get_header("content-encoding") + .and_then(|h| h.to_str().ok()) == Some("br") { + + log::info!("Detected Brotli compression in encoded path response, decompressing..."); + let mut decompressed = Vec::new(); + match brotli::Decompressor::new(&body_bytes[..], 4096) + .read_to_end(&mut decompressed) { + Ok(_) => decompressed, + Err(e) => { + log::error!("Failed to decompress encoded path response: {:?}", e); + return Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_body("Failed to decompress response")); + } + } + } else { + body_bytes + }; + + // Convert to string safely + match std::str::from_utf8(&decompressed_body) { + Ok(body_str) => { + // Debug GAM response content + log::info!("GAM encoded path response analysis:"); + log::info!(" - Body length: {} chars", body_str.len()); + log::info!(" - Contains 'creative': {}", body_str.contains("creative")); + log::info!(" - Contains 'img': {}", body_str.contains("img")); + log::info!(" - Contains 'src=': {}", body_str.contains("src=")); + log::info!(" - Contains 'http': {}", body_str.contains("http")); + if body_str.len() > 400 { + log::info!(" - First 200 chars: {}", &body_str[0..200]); + log::info!(" - Last 200 chars: {}", &body_str[body_str.len()-200..]); + } else { + log::info!(" - Full content (short): {}", body_str); + } + + let rewritten_body = rewrite_gam_urls(body_str); + response.set_body(rewritten_body); + // Remove compression headers since we're serving uncompressed + response.remove_header("content-encoding"); + response.remove_header("content-length"); + log::info!("Applied domain rewriting to encoded path response"); + } + Err(e) => { + log::error!("Encoded path response contains non-UTF-8 data: {:?}", e); + // Return original binary content without rewriting + response.set_body(decompressed_body); + log::warn!("Served encoded path response without domain rewriting (binary content)"); + } + } + } + + Ok(response) + } + Err(e) => { + log::error!("Error fetching encoded path from {}: {:?}", backend_name, e); + Ok(Response::from_status(StatusCode::BAD_GATEWAY) + .with_body(format!("Failed to fetch from encoded path: {}", encoded_domain))) + } + } +} + async fn handle_gam_asset(_settings: &Settings, req: Request) -> Result { let path = req.get_path(); println!("=== HANDLING GAM ASSET: {} ===", path); @@ -453,16 +597,24 @@ async fn handle_gam_asset(_settings: &Settings, req: Request) -> Result Result Result Result Result { - log::info!("Successfully decompressed {} bytes to {} bytes", original_length, bytes_read); + log::info!( + "Successfully decompressed {} bytes to {} bytes", + original_length, + bytes_read + ); decompressed } Err(e) => { @@ -585,23 +755,27 @@ async fn handle_gam_asset(_settings: &Settings, req: Request) -> Result { log::error!("Decompressed response contains non-UTF-8 data: {:?}", e); return Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) - .with_body(format!("Invalid UTF-8 in decompressed GAM asset: {:?}", e))); + .with_body(format!( + "Invalid UTF-8 in decompressed GAM asset: {:?}", + e + ))); } }; // Log a sample of the original content for debugging (safely) let sample = if body.len() > 200 { // Find a safe character boundary near 200 bytes - match body.char_indices().nth(100) { // Get first ~100 characters instead of 200 bytes + match body.char_indices().nth(100) { + // Get first ~100 characters instead of 200 bytes Some((byte_idx, _)) => &body[..byte_idx], - None => &body[..std::cmp::min(50, body.len())] // Fallback to very short sample + None => &body[..std::cmp::min(50, body.len())], // Fallback to very short sample } } else { &body }; log::info!("Original content sample: {}", sample); - // Rewrite hardcoded URLs to use first-party proxy + // Rewrite hardcoded URLs to use first-party proxy (now applied to all text content) let rewritten_body = rewrite_gam_urls(&body); let rewritten_length = rewritten_body.len(); @@ -617,14 +791,16 @@ async fn handle_gam_asset(_settings: &Settings, req: Request) -> Result &rewritten_body[..byte_idx], - None => &rewritten_body[..std::cmp::min(50, rewritten_body.len())] + None => &rewritten_body[..std::cmp::min(50, rewritten_body.len())], } } else { &rewritten_body }; log::info!("Rewritten content sample: {}", rewritten_sample); } else { - log::warn!("No content changes detected - rewriting may not have found target URLs"); + log::warn!( + "No content changes detected - rewriting may not have found target URLs" + ); } // Create new response with rewritten content @@ -640,7 +816,11 @@ async fn handle_gam_asset(_settings: &Settings, req: Request) -> Result Result Date: Thu, 4 Sep 2025 08:46:40 -0400 Subject: [PATCH 11/11] feat: implement comprehensive caching optimizations for performance - Add static asset endpoints (/static/styles.css, /static/app.js) with 24h cache - Update main page cache to 10 minutes with ETag versioning - Ensure ad content stays fresh with aggressive no-cache headers - Add cache-busting timestamps to ad requests only - Optimize loading order: CSS first, JS async defer - Separate cacheable static assets from dynamic ad content --- crates/common/src/gam.rs | 400 ++++++++++++++++++++++++++++++++- crates/common/src/templates.rs | 88 +++++++- crates/fastly/src/main.rs | 44 +++- static/app.js | 100 +++++++++ static/styles.css | 143 ++++++++++++ 5 files changed, 757 insertions(+), 18 deletions(-) create mode 100644 static/app.js create mode 100644 static/styles.css diff --git a/crates/common/src/gam.rs b/crates/common/src/gam.rs index 885150e..c34c4d6 100644 --- a/crates/common/src/gam.rs +++ b/crates/common/src/gam.rs @@ -1,9 +1,10 @@ use crate::gdpr::get_consent_from_request; use crate::partners::PartnerManager; +use crate::prebid::PrebidRequest; use crate::settings::Settings; use fastly::http::{header, Method, StatusCode}; use fastly::{Error, Request, Response}; -use serde_json::json; +use serde_json::{json, Value}; use std::collections::HashMap; use uuid::Uuid; @@ -68,9 +69,49 @@ impl GamRequest { params.insert("sz".to_string(), "728x90|300x250".to_string()); // Multiple sizes params.insert("c".to_string(), self.correlator.clone()); - // Output format - params.insert("output".to_string(), "html".to_string()); // Standard HTML output - params.insert("impl".to_string(), "s".to_string()); // Standard implementation + // Output format - match production autoblog + params.insert("output".to_string(), "ldjh".to_string()); // LDJH format like autoblog + params.insert("impl".to_string(), "fifs".to_string()); // FIFS implementation like autoblog + + // Required GAM parameters from production + params.insert("gdfp_req".to_string(), "1".to_string()); // Required GAM identifier + let timestamp = chrono::Utc::now().timestamp_millis(); + params.insert("pvsid".to_string(), timestamp.to_string()); // Page view session ID + params.insert("vrg".to_string(), "202508280101".to_string()); // Version/release group + params.insert("frm".to_string(), "20".to_string()); // Frame parameter + + // Add correlator as separate parameter (different from 'c') + params.insert("correlator".to_string(), timestamp.to_string()); + + // Experiment IDs from production autoblog + params.insert("eid".to_string(), "31086815%2C31093080%2C83321073".to_string()); + + // Page and session parameters + params.insert("ptt".to_string(), "17".to_string()); // Page type + params.insert("ifi".to_string(), "5".to_string()); // Iframe index + params.insert("sc".to_string(), "1".to_string()); // Screen + params.insert("eri".to_string(), "1".to_string()); // Error info + params.insert("sfv".to_string(), "1-0-45".to_string()); // SafeFrame version + params.insert("abxe".to_string(), "1".to_string()); // Ad blocker + params.insert("lmt".to_string(), timestamp.to_string()); // Load time + + // Browser/device parameters from production + params.insert("u_his".to_string(), "3".to_string()); // History length + params.insert("u_h".to_string(), "982".to_string()); // Screen height + params.insert("u_w".to_string(), "1512".to_string()); // Screen width + params.insert("u_ah".to_string(), "889".to_string()); // Available height + params.insert("u_aw".to_string(), "1512".to_string()); // Available width + params.insert("u_cd".to_string(), "30".to_string()); // Color depth + params.insert("u_sd".to_string(), "2".to_string()); // Screen density + params.insert("u_tz".to_string(), "-300".to_string()); // Timezone offset + params.insert("dmc".to_string(), "8".to_string()); // Device memory + params.insert("bc".to_string(), "31".to_string()); // Browser capabilities + params.insert("nvt".to_string(), "1".to_string()); // Navigation type + params.insert("vis".to_string(), "1".to_string()); // Visibility + + // Cookie parameters from production + params.insert("cookie".to_string(), "ID%3Dcd6d5b4a2d1c1371%3AT%3D1749585594%3ART%3D1756844784%3AS%3DALNI_MZxgfKMLtdNI6bEtVhMeqqg2IEL-g".to_string()); + params.insert("gpic".to_string(), "UID%3D000010d34192ecc4%3AT%3D1749585594%3ART%3D1756844784%3AS%3DALNI_MYAoYlnKkmIbJV5p7kgvAFcTQjiUQ".to_string()); // Page context params.insert("url".to_string(), urlencoding::encode("https://www.autoblog.com/reviews/test").to_string()); @@ -84,8 +125,8 @@ impl GamRequest { }; params.insert("cust_params".to_string(), urlencoding::encode(&cust_params).to_string()); - // Publisher Provided ID - params.insert("ppid".to_string(), format!("arenaGroup-{}", self.synthetic_id)); + // Publisher Provided ID - use proper UUID format like autoblog + params.insert("ppid".to_string(), format!("arenaGroup-{}", uuid::Uuid::new_v4())); // Timestamps params.insert("dt".to_string(), chrono::Utc::now().timestamp_millis().to_string()); @@ -118,13 +159,22 @@ impl GamRequest { // Create the request let mut req = Request::new(Method::GET, &url); - // Set headers to mimic a browser request (using only Fastly-compatible headers) + // Set headers to mimic browser request exactly (critical for GAM authentication) req.set_header(header::USER_AGENT, &self.user_agent); - req.set_header(header::ACCEPT, "application/json, text/plain, */*"); + req.set_header(header::ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); req.set_header(header::ACCEPT_LANGUAGE, "en-US,en;q=0.9"); req.set_header(header::ACCEPT_ENCODING, "gzip, deflate, br"); req.set_header(header::REFERER, &self.page_url); - req.set_header(header::ORIGIN, &self.page_url); + + // Critical browser security headers that GAM validates + req.set_header("Sec-Fetch-Site", "cross-site"); + req.set_header("Sec-Fetch-Mode", "no-cors"); + req.set_header("Sec-Fetch-Dest", "script"); + req.set_header(header::CACHE_CONTROL, "no-cache"); + req.set_header("Pragma", "no-cache"); + req.set_header("Connection", "keep-alive"); + + // Don't send Origin header as GAM scripts typically don't include it req.set_header("X-Synthetic-ID", &self.synthetic_id); // Send the request to the GAM backend @@ -686,10 +736,340 @@ pub async fn handle_gam_render(settings: &Settings, req: Request) -> Result Result { + log::info!("Starting GAM passthrough from Prebid"); + + // Check consent status from cookie + let consent = get_consent_from_request(&req).unwrap_or_default(); + let advertising_consent = consent.advertising; + + if !advertising_consent { + return Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "No advertising consent", + "message": "GAM passthrough requires advertising consent" + }))?); + } + + // Parse Prebid auction response data + let body = req.take_body_str(); + let prebid_data: Value = match serde_json::from_str(&body) { + Ok(data) => data, + Err(e) => { + log::error!("Error parsing Prebid auction data: {:?}", e); + return Ok(Response::from_status(StatusCode::BAD_REQUEST) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "Invalid Prebid auction data", + "details": format!("{:?}", e) + }))?); + } + }; + + log::info!("Received Prebid auction data: {}", + serde_json::to_string_pretty(&prebid_data).unwrap_or_else(|_| "invalid".to_string())); + + // Extract winning bid information + let winning_bidder = prebid_data["seatbid"][0]["seat"] + .as_str() + .unwrap_or("unknown"); + let winning_price = prebid_data["seatbid"][0]["bid"][0]["price"] + .as_f64() + .unwrap_or(0.0); + let auction_id = prebid_data["id"] + .as_str() + .unwrap_or_else(|| "unknown"); + + log::info!("Winning bid: bidder={}, price=${:.3}, auction_id={}", + winning_bidder, winning_price, auction_id); + + // Get synthetic ID from original request context + let synthetic_id = req.get_header("X-Synthetic-ID") + .and_then(|h| h.to_str().ok()) + .unwrap_or(auction_id); + + // Create GAM request with winning bid context + let mut gam_req = GamRequest { + publisher_id: settings.gam.publisher_id.clone(), + ad_units: settings.gam.ad_units.iter().map(|u| u.name.clone()).collect(), + page_url: req.get_url().to_string(), + correlator: Uuid::new_v4().to_string(), + prmtvctx: None, + user_agent: req.get_header(header::USER_AGENT) + .and_then(|h| h.to_str().ok()) + .unwrap_or("Mozilla/5.0 (compatible; TrustedServer/1.0)") + .to_string(), + synthetic_id: synthetic_id.to_string(), + }; + + // Add Prebid winning bid data using standard targeting parameters + let prebid_context = if winning_price > 0.0 { + format!( + "hb_bidder={}&hb_pb={:.2}&hb_adid={}&hb_size=728x90&hb_source=server", + winning_bidder, winning_price, auction_id + ) + } else { + "hb_source=server".to_string() // No winning bid, just indicate server-side + }; + gam_req.prmtvctx = Some(prebid_context); + + log::info!("Forwarding winning bid to GAM with context: {}", + gam_req.prmtvctx.as_ref().unwrap()); + + // Send request to GAM with winning bid information + match gam_req.send_request(settings).await { + Ok(mut response) => { + log::info!("GAM passthrough successful"); + + // Apply domain rewriting to GAM response + let partner_manager = PartnerManager::from_settings(settings); + let original_body = response.take_body_str(); + let rewritten_body = partner_manager.rewrite_content(&original_body); + + if original_body != rewritten_body { + log::info!("Applied domain rewriting to GAM passthrough response"); + } + + Ok(Response::from_status(response.get_status()) + .with_header(header::CONTENT_TYPE, "application/json") + .with_header(header::CACHE_CONTROL, "no-store, private") + .with_header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .with_header("X-GAM-Passthrough", "true") + .with_header("X-Winning-Bidder", winning_bidder) + .with_header("X-Winning-Price", &winning_price.to_string()) + .with_header("X-Auction-ID", auction_id) + .with_body_json(&json!({ + "status": "gam_passthrough_success", + "prebid_auction": { + "winning_bidder": winning_bidder, + "winning_price": winning_price, + "auction_id": auction_id + }, + "gam_response": rewritten_body, + "domain_rewriting_applied": original_body != rewritten_body + }))?) + } + Err(e) => { + log::error!("GAM passthrough failed: {:?}", e); + Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "GAM passthrough failed", + "details": format!("{:?}", e), + "prebid_auction": { + "winning_bidder": winning_bidder, + "winning_price": winning_price, + "auction_id": auction_id + } + }))?) + } + } +} + +/// Handle server-side ad orchestration: Prebid → GAM flow +pub async fn handle_server_side_ad(settings: &Settings, req: Request) -> Result { + log::info!("Starting server-side ad orchestration: Prebid → GAM"); + + // Check consent status from cookie and header fallback + let consent = get_consent_from_request(&req).unwrap_or_default(); + let cookie_consent = consent.advertising; + + // Also check header as fallback + let header_consent = req + .get_header("X-Consent-Advertising") + .and_then(|h| h.to_str().ok()) + .map(|v| v == "true") + .unwrap_or(false); + + let advertising_consent = cookie_consent || header_consent; + + log::info!("Server-side ad consent check: cookie={}, header={}, final={}", + cookie_consent, header_consent, advertising_consent); + + if !advertising_consent { + return Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "No advertising consent", + "message": "Server-side ad orchestration requires advertising consent", + "ad_slot_html": "Ads disabled - no advertising consent" + }))?); + } + + // Step 1: Create and send Prebid request + let prebid_req = match PrebidRequest::new(settings, &req) { + Ok(req) => req, + Err(e) => { + log::error!("Error creating Prebid request: {:?}", e); + return Ok(Response::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "Failed to create Prebid request", + "details": format!("{:?}", e), + "ad_slot_html": "Ad temporarily unavailable" + }))?); + } + }; + + log::info!("Sending Prebid request for server-side ad orchestration"); + + let prebid_response = match prebid_req.send_bid_request(settings, &req).await { + Ok(mut response) => { + let prebid_body = response.take_body_str(); + log::info!("Prebid response received: {} chars", prebid_body.len()); + + // Parse Prebid response + match serde_json::from_str::(&prebid_body) { + Ok(data) => data, + Err(e) => { + log::error!("Error parsing Prebid response: {:?}", e); + return Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "Invalid Prebid response", + "details": format!("{:?}", e), + "ad_slot_html": "Ad service error" + }))?); + } + } + } + Err(e) => { + log::error!("Prebid request failed: {:?}", e); + return Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "Prebid request failed", + "details": format!("{:?}", e), + "ad_slot_html": "No bids available" + }))?); + } + }; + + // Step 2: Extract winning bid information (corrected structure) + let winning_bidder = prebid_response["seatbid"][0]["seat"] + .as_str() + .unwrap_or("unknown"); + let winning_price = prebid_response["seatbid"][0]["bid"][0]["price"] + .as_f64() + .unwrap_or(0.0); + let creative_content = prebid_response["seatbid"][0]["bid"][0]["adm"] + .as_str() + .unwrap_or(""); + let auction_id = prebid_response["id"] + .as_str() + .unwrap_or("unknown"); + + log::info!("Prebid auction complete: bidder={}, price=${:.3}, auction_id={}, creative_length={}", + winning_bidder, winning_price, auction_id, creative_content.len()); + + // Step 3: Create GAM request with winning bid context + let synthetic_id = req.get_header("X-Synthetic-Trusted-Server") + .and_then(|h| h.to_str().ok()) + .unwrap_or(auction_id); + + let mut gam_req = GamRequest { + publisher_id: settings.gam.publisher_id.clone(), + ad_units: vec!["728x90_header".to_string()], // Specific slot + page_url: req.get_url().to_string(), + correlator: uuid::Uuid::new_v4().to_string(), + prmtvctx: None, + user_agent: req.get_header(header::USER_AGENT) + .and_then(|h| h.to_str().ok()) + .unwrap_or("Mozilla/5.0 (compatible; TrustedServer/1.0)") + .to_string(), + synthetic_id: synthetic_id.to_string(), + }; + + // Add Prebid winning bid data using standard targeting parameters + working prmtvctx + let prebid_context = if winning_price > 0.0 { + format!( + "hb_bidder={}&hb_pb={:.2}&hb_adid={}&hb_size=728x90&hb_source=server&{}", + winning_bidder, winning_price, auction_id, + "129627,137412,138272,139095,139096,139218,141364,143196,143210,143211,143214,143217,144331,144409,144438,144444,144488,144543,144663,144679,144731,144824,144916,145933,146347,146348,146349,146350,146351,146370,146383,146391,146392,146393,146424,146995,147077,147740,148616,148627,148628,149007,150420,150663,150689,150690,150692,150752,150753,150755,150756,150757,150764,150770,150781,150862,154609,155106,155109,156204,164183,164573,165512,166017,166019,166484,166486,166487,166488,166492,166494,166495,166497,166511,167639,172203,172544,173548,176066,178053,178118,178120,178121,178133,180321,186069,199642,199691,202074,202075,202081,233782,238158,adv,bhgp,bhlp,bhgw,bhlq,bhlt,bhgx,bhgv,bhgu,bhhb,rts" + ) + } else { + format!( + "hb_source=server&{}", + "129627,137412,138272,139095,139096,139218,141364,143196,143210,143211,143214,143217,144331,144409,144438,144444,144488,144543,144663,144679,144731,144824,144916,145933,146347,146348,146349,146350,146351,146370,146383,146391,146392,146393,146424,146995,147077,147740,148616,148627,148628,149007,150420,150663,150689,150690,150692,150752,150753,150755,150756,150757,150764,150770,150781,150862,154609,155106,155109,156204,164183,164573,165512,166017,166019,166484,166486,166487,166488,166492,166494,166495,166497,166511,167639,172203,172544,173548,176066,178053,178118,178120,178121,178133,180321,186069,199642,199691,202074,202075,202081,233782,238158,adv,bhgp,bhlp,bhgw,bhlq,bhlt,bhgx,bhgv,bhgu,bhhb,rts" + ) + }; + gam_req.prmtvctx = Some(prebid_context.clone()); + + log::info!("Sending GAM request with Prebid context: {}", prebid_context); + + // Step 4: Send GAM request with winning bid information + match gam_req.send_request(settings).await { + Ok(mut response) => { + log::info!("Server-side ad orchestration successful"); + + // Apply domain rewriting to GAM response + let partner_manager = PartnerManager::from_settings(settings); + let original_body = response.take_body_str(); + let rewritten_body = partner_manager.rewrite_content(&original_body); + + if original_body != rewritten_body { + log::info!("Applied domain rewriting to server-side ad response"); + } + + // Extract creative HTML for injection + let creative_html = if rewritten_body.contains("") { + if let Some(html_start) = rewritten_body.find("") { + rewritten_body[html_start..].to_string() + } else { + "Creative format error".to_string() + } + } else { + rewritten_body.clone() + }; + + Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "application/json; charset=utf-8") + .with_header(header::CACHE_CONTROL, "no-store, no-cache, must-revalidate, private") + .with_header("Pragma", "no-cache") + .with_header("Expires", "0") + .with_header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .with_header("X-Server-Side-Ad", "true") + .with_header("X-Winning-Bidder", winning_bidder) + .with_header("X-Winning-Price", &winning_price.to_string()) + .with_header("X-Auction-ID", auction_id) + .with_body_json(&json!({ + "status": "server_side_ad_success", + "prebid_auction": { + "winning_bidder": winning_bidder, + "winning_price": winning_price, + "auction_id": auction_id + }, + "gam_response": rewritten_body, + "ad_slot_html": creative_html, + "domain_rewriting_applied": original_body != rewritten_body + }))?) + } + Err(e) => { + log::error!("Server-side ad orchestration failed: {:?}", e); + Ok(Response::from_status(StatusCode::OK) + .with_header(header::CONTENT_TYPE, "application/json") + .with_body_json(&json!({ + "error": "Server-side ad orchestration failed", + "details": format!("{:?}", e), + "prebid_auction": { + "winning_bidder": winning_bidder, + "winning_price": winning_price, + "auction_id": auction_id + }, + "ad_slot_html": "Ad temporarily unavailable" + }))?) + } + } +} diff --git a/crates/common/src/templates.rs b/crates/common/src/templates.rs index 23aa97e..722faf9 100644 --- a/crates/common/src/templates.rs +++ b/crates/common/src/templates.rs @@ -337,7 +337,10 @@ pub const EDGEPUBS_TEMPLATE: &str = r##" EdgePubs - The Edge Is Yours + +