Skip to content

added a check for x-host header on CDN use #31

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dnmitev wants to merge 2 commits into
base: master
Choose a base branch
from
Open

added a check for x-host header on CDN use #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
71834cb
added a check for x-host header on CDN use
dnmitev Nov 10, 2017
1bd8487
applied use of x-host header as configurable
dnmitev Nov 18, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions Prerender.io/PrerenderConfigSection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,5 +143,24 @@ public String Token
this["token"] = value;
}
}

/// <summary>
/// CDNs use the X-host to provide the actual web address,
/// i.e. if we have the address www.my-address.com which uses the my-traffic-manager.manager.com,
/// the X-Host would containt "my-address.com" while the prerender.io would provide my-traffic-manager.manager.com.
/// In other words we would have https://my-traffic-manager.com/users indexed instead of http://my-address.com/users
/// </summary>
[ConfigurationProperty("useXHostHeader", DefaultValue = false)]
public bool UseXHostHeader
{
get
{
return (bool)this["useXHostHeader"];
}
set
{
this["useXHostHeader"] = value;
}
}
}
}
50 changes: 32 additions & 18 deletions Prerender.io/PrerenderModule.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ private void DoPrerender(HttpApplication context)
response.Headers.Add(header, value);
}
}

response.Write(result.ResponseBody);
response.Flush();
context.CompleteRequest();
Expand All @@ -94,7 +94,7 @@ private ResponseResult GetPrerenderedPageResponse(HttpRequest request)
try
{
// Get the web response and read content etc. if successful
var webResponse = (HttpWebResponse) webRequest.GetResponse();
var webResponse = (HttpWebResponse)webRequest.GetResponse();
var reader = new StreamReader(webResponse.GetResponseStream(), Encoding.UTF8);
return new ResponseResult(webResponse.StatusCode, reader.ReadToEnd(), webResponse.Headers);
}
Expand Down Expand Up @@ -122,10 +122,24 @@ private static void SetNoCache(HttpWebRequest webRequest)

private String GetApiUrl(HttpRequest request)
{
// var url = request.Url.AbsoluteUri; (not working with angularjs)
// use request.RawUrl instead of request.Url.AbsoluteUri to get the original url
// becuase angularjs requires a rewrite and requests are rewritten to base /
var url = string.Format("{0}://{1}{2}", request.Url.Scheme, request.Url.Authority, request.RawUrl);
if (_prerenderConfig.UseXHostHeader)
{
var xHostHeader = request.Headers["X-Host"];
var url = string.Empty;
if (xHostHeader != null)
{
// when there is a CDN in front of the WebApp the url might be mistaken with the traffic manager
// therefore the X-Host header provided value should be used instaed of the request.Url
url = string.Format("{0}://{1}{2}", request.Url.Scheme, xHostHeader, request.RawUrl);
}
}
else
{
// var url = request.Url.AbsoluteUri; (not working with angularjs)
// use request.RawUrl instead of request.Url.AbsoluteUri to get the original url
// becuase angularjs requires a rewrite and requests are rewritten to base /
url = string.Format("{0}://{1}{2}", request.Url.Scheme, request.Url.Authority, request.RawUrl);
}

// request.RawUrl have the _escaped_fragment_ query string
// Prerender server remove it before making a request, but caching plugins happen before prerender server remove it
Expand All @@ -139,19 +153,19 @@ private String GetApiUrl(HttpRequest request)
}

// Remove the application from the URL
if (_prerenderConfig.StripApplicationNameFromRequestUrl && !string.IsNullOrEmpty(request.ApplicationPath) && request.ApplicationPath != "/")
{
// http://test.com/MyApp/?_escape_=/somewhere
url = url.Replace(request.ApplicationPath, string.Empty);
}
if (_prerenderConfig.StripApplicationNameFromRequestUrl && !string.IsNullOrEmpty(request.ApplicationPath) && request.ApplicationPath != "/")
{
// http://test.com/MyApp/?_escape_=/somewhere
url = url.Replace(request.ApplicationPath, string.Empty);
}

var prerenderServiceUrl = _prerenderConfig.PrerenderServiceUrl;
return prerenderServiceUrl.EndsWith("/")
? (prerenderServiceUrl + url)
: string.Format("{0}/{1}", prerenderServiceUrl, url);
}
public static string RemoveQueryStringByKey(string url, string key)

public static string RemoveQueryStringByKey(string url, string key)
{
var uri = new Uri(url);

Expand Down Expand Up @@ -268,10 +282,10 @@ private IEnumerable<String> GetCrawlerUserAgents()
// "googlebot",
// "yahoo",
// "bingbot",
"baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot",
"embedly", "quora link preview", "showyoubot", "outbrain", "pinterest/0.",
"developers.google.com/+/web/snippet", "slackbot", "vkShare", "W3C_Validator",
"redditbot", "Applebot", "WhatsApp", "flipboard", "tumblr", "bitlybot",
"baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot",
"embedly", "quora link preview", "showyoubot", "outbrain", "pinterest/0.",
"developers.google.com/+/web/snippet", "slackbot", "vkShare", "W3C_Validator",
"redditbot", "Applebot", "WhatsApp", "flipboard", "tumblr", "bitlybot",
"SkypeUriPreview", "nuzzel", "Discordbot", "Google Page Speed", "x-bufferbot"
});

Expand Down