Make the listing site work on Netlify (remoteintech#460)

EchoJoker · Aug 12, 2018 · 4ee2650 · 4ee2650
1 parent a65dfb4
commit 4ee2650
Show file tree

Hide file tree

Showing 18 changed files with 1,031 additions and 172 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -2,4 +2,4 @@ language: node_js
 
 before_script: npm install
 
-script: node bin/validate.js && node node_modules/.bin/mocha
+script: npm run validate && npm test
diff --git a/README.md b/README.md
@@ -1,12 +1,13 @@
 # Remote-friendly companies
 
-A list of semi to fully remote-friendly companies in or around tech. This is the list. There is also a [blog / information website](https://remoteintech.company/).
+A list of semi to fully remote-friendly companies in or around tech.
 
-Contributions are very welcome! Please read the [contribution guidelines](/CONTRIBUTING.md) and complete a [Pull Request template](/PULL_REQUEST_TEMPLATE.MD/) for all new submissions.
+Contributions are very welcome! Please
+[submit a pull request on GitHub](https://github.com/remoteintech/remote-jobs/blob/master/CONTRIBUTING.md).
 
-You can see the format needed in the [example.md](/company-profiles/example.md) file.
-
-_Some company names have a ⚠️️️ icon next to them. This icon means we don't have much information about this company yet and we would love a contribution! See each individual company profile for details._
+_Some company names have a ⚠️️️ icon next to them. This icon means we don't have
+much information about this company yet and we would love a contribution! See
+each individual company profile for details._
 
 ## Companies
 

diff --git a/site/coming-soon.html → _layouts/default.html b/site/coming-soon.html → _layouts/default.html
@@ -1,3 +1,6 @@
+---
+layout: none
+---
 <!DOCTYPE html>
 <html>
 	<head>
@@ -28,10 +31,10 @@
 	</head>
 	<body>
 		<p>
-			This site (the Netlify site for
-			<a href="https://github.com/remoteintech/remote-jobs">remoteintech/remote-jobs</a>)
+			This site is <strong>deprecated</strong>.
 			<br />
-			is still under construction.  Check back later!
+			Go here instead:
+			<a href="https://remoteintech.company/">remoteintech.company</a>
 		</p>
 	</body>
 </html>
diff --git a/_layouts/home.html b/_layouts/home.html
diff --git a/bin/build-site.js b/bin/build-site.js
@@ -2,18 +2,236 @@
 
 const fs = require( 'fs' );
 const path = require( 'path' );
+const util = require( 'util' );
 
-const siteDir = path.join( __dirname, '..', 'site' );
-const siteBuildDir = path.join( siteDir, 'build' );
+const cheerio = require( 'cheerio' );
+const phin = require( 'phin' );
+const rimraf = require( 'rimraf' );
+const swig = require( 'swig-templates' );
 
-if ( ! fs.existsSync( siteBuildDir ) ) {
-	fs.mkdirSync( siteBuildDir );
+const { parseFromDirectory, headingPropertyNames } = require( '../lib' );
+const contentPath = path.join( __dirname, '..' );
+const sitePath = path.join( __dirname, '..', 'site' );
+const siteBuildPath = path.join( sitePath, 'build' );
+
+// If we are inside the site build path, this is going to cause problems since
+// we blow away this directory before regenerating the site
+// Error message (in node core): path.js:1086 cwd = process.cwd();
+// Error: ENOENT: no such file or directory, uv_cwd
+function checkPath( wd ) {
+	const checkWorkingPath = path.resolve( wd ) + path.sep;
+	const checkBuildPath = siteBuildPath + path.sep;
+	if ( checkWorkingPath.substring( 0, checkBuildPath.length ) === checkBuildPath ) {
+		throw new Error(
+			"Please change out of the 'site/build' directory before running this script"
+		);
+	}
+}
+checkPath( process.cwd() );
+if ( process.env.INIT_CWD ) {
+	// This script was run via npm; check the original working directory
+	// because npm barfs in this situation too
+	checkPath( process.env.INIT_CWD );
+}
+
+// Parse the content from the Markdown files
+console.log( 'Parsing content' );
+const data = parseFromDirectory( contentPath );
+
+// Stop if there were any errors
+if ( data.errors && data.errors.length > 0 ) {
+	data.errors.forEach( err => {
+		err.message.split( '\n' ).forEach( line => {
+			console.log( '%s: %s', err.filename, line );
+		} );
+	} );
+	process.exit( 1 );
+}
+
+// Otherwise, OK to continue building the static site
+
+const assetCacheBuster = Date.now();
+
+// https://github.com/nodejs/node/issues/17871 :(
+process.on( 'unhandledRejection', err => {
+	console.error( 'Unhandled promise rejection:', err );
+	process.exit( 1 );
+} );
+
+/**
+ * Perform an HTTP request to a URL and return the request body.
+ */
+async function request( url ) {
+	console.log(
+		'Requesting URL "%s"',
+		url.length > 70
+			? url.substring( 0, 67 ) + '...'
+			: url
+	);
+	const res = await phin.promisified( url );
+	if ( res.statusCode !== 200 ) {
+		throw new Error(
+			'HTTP response code ' + res.statusCode
+			+ ' for URL: ' + url
+		);
+	}
+	return res.body.toString();
+}
+
+/**
+ * Write a file to site/build/assets/ (from memory or from an existing file in
+ * site/assets/) and include a cache buster in the new name.  Return the URL to
+ * the asset file.
+ */
+function copyAssetToBuild( filename, content = null ) {
+	const destFilename = filename
+		.replace( /(\.[^.]+)$/, '-' + assetCacheBuster + '$1' );
+	const destPath = path.join( siteBuildPath, 'assets', destFilename );
+	if ( ! content ) {
+		const srcPath = path.join( sitePath, 'assets', filename );
+		content = fs.readFileSync( srcPath, 'utf8' );
+	}
+	fs.writeFileSync( destPath, content );
+	return '/assets/' + destFilename;
+}
+
+/**
+ * Write a page's contents to an HTML file.
+ */
+function writePage( filename, pageContent ) {
+	console.log( 'Writing page "%s"', filename );
+	filename = path.join( siteBuildPath, filename );
+	if ( ! fs.existsSync( path.dirname( filename ) ) ) {
+		fs.mkdirSync( path.dirname( filename ) );
+	}
+	fs.writeFileSync( filename, pageContent );
+}
+
+/**
+ * The main function that prepares the static site.
+ */
+async function buildSite() {
+	// Load the HTML from the WP.com blog site
+	const $ = cheerio.load( await request( 'https://blog.remoteintech.company/' ) );
+
+	// Load stylesheets from the WP.com blog site
+	const wpcomStylesheets = $( 'style, link[rel=stylesheet]' ).map( ( i, el ) => {
+		const $el = $( el );
+		const stylesheet = {
+			id: $el.attr( 'id' ) || null,
+			media: $el.attr( 'media' ) || null,
+		};
+		if ( $el.is( 'style' ) ) {
+			stylesheet.content = $el.html();
+		} else {
+			stylesheet.url = $el.attr( 'href' );
+		}
+		return stylesheet;
+	} ).toArray();
+
+	// Fetch the contents of stylesheets included via <link> tags
+	await Promise.all(
+		wpcomStylesheets.filter( s => !! s.url ).map( stylesheet => {
+			return request( stylesheet.url ).then( content => {
+				stylesheet.content = content;
+			} );
+		} )
+	);
+	// TODO: Most URLs that appear inside these CSS files are broken because
+	// they refer to relative URLs against s[012].wp.com
+	const wpcomStylesheetContent = wpcomStylesheets
+		.filter( stylesheet => !! stylesheet.content.trim() )
+		.map( stylesheet => {
+			const lines = [ '/**' ];
+			const idString = (
+				stylesheet.id ? ' (id="' + stylesheet.id + '")' : ''
+			);
+			if ( stylesheet.url ) {
+				lines.push( ' * WP.com external style' + idString );
+				lines.push( ' * ' + stylesheet.url );
+			} else {
+				lines.push( ' * WP.com inline style' + idString );
+			}
+			lines.push( ' */' );
+			if ( stylesheet.media && stylesheet.media !== 'all' ) {
+				lines.push( '@media ' + stylesheet.media + ' {' );
+			}
+			lines.push( stylesheet.content.trim() );
+			if ( stylesheet.media && stylesheet.media !== 'all' ) {
+				lines.push( '} /* @media ' + stylesheet.media + ' */' );
+			}
+			return lines.join( '\n' );
+		} ).join( '\n\n' ) + '\n';
+
+	// Use the emoji code from WP.com
+	// Most platforms will display emoji natively, but e.g. Linux does not
+	let wpcomEmojiScript = null;
+	$( 'script' ).each( ( i, el ) => {
+		const scriptContents = $( el ).html();
+		if ( /\bwindow\._wpemojiSettings\s*=\s*{/.test( scriptContents ) ) {
+			wpcomEmojiScript = scriptContents;
+		}
+	} );
+
+	// Set up the site build directory (start fresh each time)
+	rimraf.sync( siteBuildPath );
+	fs.mkdirSync( siteBuildPath );
+	fs.mkdirSync( path.join( siteBuildPath, 'assets' ) );
+
+	// Set up styles/scripts to be included on all pages
+	const stylesheets = [ {
+		url: copyAssetToBuild( 'wpcom-blog-styles.css', wpcomStylesheetContent ),
+	}, {
+		url: '//fonts.googleapis.com/css?family=Source+Sans+Pro:r%7CSource+Sans+Pro:r,i,b,bi&amp;subset=latin,latin-ext,latin,latin-ext',
+	} ];
+	const scripts = [];
+	if ( wpcomEmojiScript ) {
+		scripts.push( {
+			url: copyAssetToBuild( 'wpcom-emoji.js', wpcomEmojiScript ),
+		} );
+	}
+
+	// Set up styles/scripts for specific pages
+	const indexStylesheets = [ {
+		url: copyAssetToBuild( 'companies-table.css' ),
+	} ];
+	const indexScripts = [ {
+		url: '//cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js',
+	}, {
+		url: copyAssetToBuild( 'companies-table.js' ),
+	} ];
+	const profileStylesheets = [ {
+		url: copyAssetToBuild( 'company-profile.css' ),
+	} ];
+
+	// Generate the index.html file from the main README
+	// TODO: Build this page and its table dynamically; more filters
+	const readmeTemplate = swig.compileFile(
+		path.join( sitePath, 'templates', 'index.html' )
+	);
+	writePage( 'index.html', readmeTemplate( {
+		stylesheets: stylesheets.concat( indexStylesheets ),
+		scripts: scripts.concat( indexScripts ),
+		pageContent: data.readmeContent,
+	} ) );
+
+	// Generate the page for each company
+	const companyTemplate = swig.compileFile(
+		path.join( sitePath, 'templates', 'company.html' )
+	);
+	data.companies.forEach( company => {
+		const dirname = company.linkedFilename.replace( /\.md$/, '' );
+		const missingHeadings = Object.keys( headingPropertyNames )
+			.filter( h => ! company.profileContent[ h ] );
+
+		writePage( path.join( dirname, 'index.html' ), companyTemplate( {
+			stylesheets: stylesheets.concat( profileStylesheets ),
+			scripts,
+			company,
+			headingPropertyNames,
+			missingHeadings,
+		} ) );
+	} );
 }
 
-fs.writeFileSync(
-	path.join( siteBuildDir, 'index.html' ),
-	fs.readFileSync(
-		path.join( siteDir, 'coming-soon.html' ),
-		'utf8'
-	)
-);
+buildSite();
diff --git a/lib/index.js b/lib/index.js
@@ -72,6 +72,15 @@ function stripExtraChars( text ) {
 exports.stripExtraChars = stripExtraChars;
 
 
+/**
+ * Other exports
+ */
+exports.headingPropertyNames = headingsAll.reduce( ( acc, val ) => {
+	acc[ toIdentifierCase( val ) ] = val;
+	return acc;
+}, {} );
+
+
 /**
  * The main exported function
  *
@@ -111,6 +120,8 @@ exports.parseFromDirectory = contentPath => {
 
 	$( 'tr' ).each( ( i, tr ) => {
 		if ( i === 0 ) {
+			// Assign an ID to the table.
+			$( tr ).closest( 'table' ).attr( 'id', 'companies-table' );
 			// Skip the table header row.
 			return;
 		}
@@ -123,13 +134,20 @@ exports.parseFromDirectory = contentPath => {
 			);
 		}
 
+		const websiteUrl = $td.eq( 1 ).text();
+		const websiteText = websiteUrl
+			.replace( /^https?:\/\//, '' )
+			.replace( /^www\./, '' )
+			.replace( /\/$/, '' );
+
 		const readmeEntry = {
 			// Strip out warning emoji indicating that this profile is incomplete
 			name: $td.eq( 0 ).text().replace( /\u26a0/, '' ).trim(),
 			// Detect warning emoji next to company name
 			isIncomplete: /\u26a0/.test( $td.eq( 0 ).text() ),
-			website: $td.eq( 1 ).text(),
-			shortRegion: $td.eq( 2 ).text(),
+			websiteUrl,
+			websiteText,
+			shortRegion: $td.eq( 2 ).text().trim(),
 		};
 
 		if ( ! readmeEntry.name ) {
@@ -206,9 +224,28 @@ exports.parseFromDirectory = contentPath => {
 			);
 		}
 
+		// Rewrite company profile link to the correct URL for the static site
+		if ( $profileLink.length ) {
+			$profileLink.attr(
+				'href',
+				$profileLink.attr( 'href' )
+					.replace( /^\/company-profiles\//, '/' )
+					.replace( /\.md$/, '/' )
+			);
+		}
+
+		// Rewrite external website link (target="_blank" etc, shorter text)
+		const $websiteLink = $td.eq( 1 ).children().eq( 0 );
+		$websiteLink
+			.attr( 'target', '_blank' )
+			.attr( 'rel', 'noopener noreferrer' )
+			.text( websiteText );
+
 		readmeCompanies.push( readmeEntry );
 	} );
 
+	const readmeContent = $( 'body' ).html();
+
 	// Scan the individual Markdown files containing the company profiles.
 
 	const allProfileHeadings = {};
@@ -444,5 +481,6 @@ exports.parseFromDirectory = contentPath => {
 		profileFilenames,
 		profileHeadingCounts,
 		companies: readmeCompanies,
+		readmeContent,
 	};
 };
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,4 +2,4 @@ language: node_js

		before_script: npm install

		script: node bin/validate.js && node node_modules/.bin/mocha
		script: npm run validate && npm test