[FHL] Web clipper next

gulpfile.js

@@ -229,6 +229,13 @@ gulp.task("bundleAppendIsInstalledMarker", function () {
     return merge(tasks);
 });
 
+gulp.task("bundleContentScript", function () {
+    var extensionRoot = PATHS.BUILDROOT + "scripts/extensions/";
+    var files = ["contentScript.js"];
+    var tasks = generateBrowserifyTasks(extensionRoot, files);
+    return merge(tasks);
+});
+
 gulp.task("bundleClipperUI", function () {
     var extensionRoot = PATHS.BUILDROOT + "scripts/clipperUI/";
     var files = ["clipper.js", "pageNav.js", "localeSpecificTasks.js", "unsupportedBrowser.js"];
@@ -299,6 +306,7 @@ gulp.task("bundleTests", function () {
 gulp.task("bundle", function(callback) {
     runSequence(
         "bundleAppendIsInstalledMarker",
+        "bundleContentScript",
         "bundleClipperUI",
         "bundleLogManager",
         "bundleBookmarklet",
@@ -512,6 +520,10 @@ function exportChromeJS() {
         PATHS.BUNDLEROOT + "appendIsInstalledMarker.js"
     ]).pipe(concat("appendIsInstalledMarker.js")).pipe(gulp.dest(targetDir));
 
+    var contentScriptTask = gulp.src([
+        PATHS.BUNDLEROOT + "contentScript.js"
+    ]).pipe(concat("contentScript.js")).pipe(gulp.dest(targetDir));
+
     var chromeExtensionTask = gulp.src([
         targetDir + "logManager.js",
         targetDir + "oneNoteApi.min.js",
@@ -537,9 +549,9 @@ function exportChromeJS() {
     ]).pipe(concat("chromePageNavInject.js")).pipe(gulp.dest(targetDir));
 
     if (commonTask) {
-        return merge(commonTask, appendIsInstalledMarkerTask, chromeExtensionTask, chromeDebugLoggingInjectTask, chromeInjectTask, chromePageNavInjectTask);
+        return merge(commonTask, appendIsInstalledMarkerTask, contentScriptTask, chromeExtensionTask, chromeDebugLoggingInjectTask, chromeInjectTask, chromePageNavInjectTask);
     }
-    return merge(chromeExtensionTask, appendIsInstalledMarkerTask, chromeDebugLoggingInjectTask, chromeInjectTask, chromePageNavInjectTask);
+    return merge(chromeExtensionTask, appendIsInstalledMarkerTask, contentScriptTask, chromeDebugLoggingInjectTask, chromeInjectTask, chromePageNavInjectTask);
 }
 
 function exportChromeCSS() {

src/scripts/extensions/chrome/chromeExtension.ts

@@ -2,6 +2,73 @@ import {ClientType} from "../../clientType";
 
 import {WebExtension} from "../webExtensionBase/webExtension";
 
+function sendMessageToContentScript(tabs, msg) {
+	if (tabs.length > 0) {
+		chrome.tabs.sendMessage(tabs[0].id, { bMessage: msg }, function (response) {
+			if (msg !== "GET_ALL_YOUTUBE_DETAILS") {
+				console.log(response.cMessage);
+			}
+		});
+	}
+}
+
+/****** START CODE TO COMMUNICATE FROM CONTENT TO BACKGROUND SCRIPT ******/
+
+chrome.runtime.onMessage.addListener(
+	function (request, sender, sendResponse) {
+		// Received all details from content script, now send back to native application
+		sendMessageToNativeApplication({
+			youtubeURL: request.youtubeURL,
+			documentTitle: request.documentTitle,
+			videoId: request.videoId,
+			streamPlayer: request.streamPlayer,
+			pictureOfVideo: request.pictureOfVideo,
+			transcript: request.transcript
+		});
+	}
+);
+
+/****** END CODE TO COMMUNICATE FROM CONTENT TO BACKGROUND SCRIPT ******/
+
+/****** START CODE TO COMMUNICATE BETWEEN EXTENSION AND NATIVE APPLICATION ******/
+
+let hostName = 'com.microsoft.onenote.stickynotes';
+let port = chrome.runtime.connectNative(hostName);
+
+function sendMessageToNativeApplication(msg) {
+	port.postMessage(msg);
+}
+
+port.onMessage.addListener((response) => {
+	console.log('Received from native application: ' + JSON.stringify(response));
+	console.log('window.location.href as known to the current background script is: ' + window.location.href);
+	/****** START CODE TO COMMUNICATE FROM BACKGROUND TO CONTENT SCRIPT ******/
+	/**
+	 * This is a background script and cannot access the DOM of the current page.
+	 * To access the DOM of the current page, we need to send a message to the
+	 * content script.
+	 */
+	chrome.tabs.query({ active: true, currentWindow: true }, function (tabs) {
+		/*
+		console.log('Getting youtube url from content script...');
+		sendMessageToContentScript(tabs, "GET_YOUTUBE_URL");
+		console.log('Getting video id from content script...');
+		sendMessageToContentScript(tabs, "GET_VIDEO_ID");
+		console.log('Getting stream player from content script...');
+		sendMessageToContentScript(tabs, "GET_STREAM_PLAYER");
+		*/
+		console.log('Getting all details from content script and sending back to native application...');
+		sendMessageToContentScript(tabs, "GET_ALL_YOUTUBE_DETAILS");
+	});
+	/****** END CODE TO COMMUNICATE FROM BACKGROUND TO CONTENT SCRIPT ******/
+});
+
+port.onDisconnect.addListener(() => {
+  console.log('Disconnected');
+});
+
+/****** END CODE TO COMMUNICATE BETWEEN EXTENSION AND NATIVE APPLICATION ******/
+
 WebExtension.browser = chrome;
 let clipperBackground = new WebExtension(ClientType.ChromeExtension, {
 	debugLoggingInjectUrl: "chromeDebugLoggingInject.js",

src/scripts/extensions/chrome/manifest.json

@@ -13,6 +13,11 @@
         "js": ["appendIsInstalledMarker.js"],
         "run_at": "document_start",
         "all_frames": true
+    }, {
+        "matches": ["<all_urls>"],
+        "js": ["contentScript.js"],
+        "run_at": "document_end",
+        "all_frames": true
     }],
 
     "web_accessible_resources": [
@@ -27,7 +32,8 @@
         "tabs",
         "webRequest",
         "storage",
-        "webNavigation"
+        "webNavigation",
+        "nativeMessaging"
     ],
 
     "content_security_policy": "script-src 'self'; object-src 'self'",

src/scripts/extensions/contentScript.ts

@@ -0,0 +1,184 @@
+console.log("Content script loaded");
+
+/****** START CODE TO COMMUNICATE BETWEEN BACKGROUND AND CONTENT SCRIPTS ******/
+
+// Listen for a message from the background script
+chrome.runtime.onMessage.addListener(
+  function (request, sender, sendResponse) {
+    if (window.self !== window.top) {
+      // We're in an iframe, ignore the message
+      return;
+    }
+    console.log(JSON.stringify(request));
+    if (request === undefined) {
+      sendResponse({ cMessage: "Invalid request received by content script" });
+    }
+    switch (request.bMessage) {
+      /*
+      case "GET_YOUTUBE_URL":
+        sendResponse({ cMessage: !isYoutube() ? "NOT_YOUTUBE" : getYoutubeURL() });
+        break;
+      case "GET_VIDEO_ID":
+        sendResponse({ cMessage: !isYoutube() ? "NOT_YOUTUBE" : getVideoId() });
+        break;
+      case "GET_STREAM_PLAYER":
+        sendResponse({ cMessage: !isYoutube() ? "NOT_YOUTUBE" : (getStreamPlayer() as HTMLElement).outerHTML });
+        break;
+      */
+      case "GET_ALL_YOUTUBE_DETAILS":
+        if (!isYoutube()) {
+          chrome.runtime.sendMessage({
+            youtubeURL: "NOT_YOUTUBE",
+            documentTitle: getDocumentTitle(),
+            pictureOfVideo: "NOT_YOUTUBE",
+            transcript: "NOT_YOUTUBE"
+          });
+        } else {
+          getTranscriptAndReturnAllYoutubeDetails();
+        }
+        break;
+      default:
+        sendResponse({ cMessage: "Invalid message received by content script" });
+    }
+  });
+
+/****** END CODE TO COMMUNICATE BETWEEN BACKGROUND AND CONTENT SCRIPTS ******/
+
+function isYoutube() {
+  return document.location.href.indexOf("youtube.com") !== -1;
+}
+
+function getDocumentTitle() {
+  return document.title;
+}
+
+function getStreamPlayer() {
+  return document.getElementsByClassName("video-stream")[0];
+}
+
+function getPictureOfVideo() {
+  const player = getStreamPlayer() as HTMLVideoElement;
+  var canvas = document.createElement("canvas");
+  canvas.width = player.videoWidth;
+  canvas.height = player.videoHeight;
+  canvas.getContext("2d").drawImage(player, 0, 0, canvas.width, canvas.height);
+
+  /****** START IMAGE COMPRESSION LOGIC ******/
+
+  // Create a new canvas element
+  const newCanvas = document.createElement('canvas');
+
+  // Set the dimensions of the new canvas to the desired size
+  newCanvas.width = canvas.width / 10; // Reduce the width
+  newCanvas.height = canvas.height / 10; // Reduce the height
+
+  // Get the 2D context of the new canvas
+  const ctx = newCanvas.getContext('2d');
+
+  // Draw the image from the original canvas onto the new canvas
+  ctx.drawImage(canvas, 0, 0, canvas.width, canvas.height, 0, 0, newCanvas.width, newCanvas.height);
+
+  /****** END IMAGE COMPRESSION LOGIC ******/
+
+  // Get the data URL of the image from the new canvas
+  const img = newCanvas.toDataURL("image/png");
+  return img;
+}
+
+function getTranscriptAroundCurrentTimestamp(fullTranscript) {
+  var player = getStreamPlayer() as HTMLVideoElement;
+  var time = player.currentTime;
+
+  let result = "";
+
+  for (let i = 0; i < fullTranscript.length; i++) {
+    if (fullTranscript[i][0] < time - 15) {
+      continue;
+    }
+    if (fullTranscript[i][0] > time + 15) {
+      break;
+    }
+    // Include transcripts within 10 seconds of the current timestamp
+    result += fullTranscript[i][1] + " ";
+  }
+
+  return result;
+}
+
+function getTranscriptAndReturnAllYoutubeDetails() {
+  const injectedCode = `
+h2 = document.createElement('h2')
+h2.id = 'ytTranscript'
+h2.innerText = ytInitialPlayerResponse.captions.playerCaptionsTracklistRenderer.captionTracks[0].baseUrl
+document.getElementById('content').appendChild(h2)
+`;
+  var script = document.createElement("script");
+  script.textContent = injectedCode;
+  (document.head).appendChild(script);
+  var ytTranscript = document.getElementById('ytTranscript');
+  var subsUrl;
+  if (ytTranscript) {
+    subsUrl = document.getElementById('ytTranscript').innerText;
+    let xhr = new XMLHttpRequest();
+    xhr.open('GET', subsUrl);
+    xhr.onreadystatechange = function () {
+      if (xhr.readyState === 4 && xhr.status === 200) {
+        // Manipulate this code to get the timestamp as well
+        let xml = new DOMParser().parseFromString(xhr.responseText, 'text/xml');
+        let textNodes = Array.prototype.slice.call(xml.getElementsByTagName('text'));
+        let subsText = []
+        for (const t of textNodes) {
+          var obj = [];
+          const start = t.getAttribute('start');
+          const duration = t.getAttribute('dur');
+          obj.push(start)
+          obj.push(t.textContent)
+          subsText.push(obj)
+        }
+        console.log(JSON.stringify(subsText));
+        chrome.runtime.sendMessage({
+          youtubeURL: getYoutubeURL(),
+          documentTitle: getDocumentTitle(),
+          pictureOfVideo: getPictureOfVideo(),
+          transcript: getTranscriptAroundCurrentTimestamp(subsText)
+        });
+      }
+    }
+    xhr.send();
+  }
+}
+
+function parseParams(href) {
+  const noHash = href.split("#")[0];
+  const paramString = noHash.split("?")[1];
+  const params = {};
+  if (paramString) {
+    const paramsArray = paramString.split("&");
+    for (const kv of paramsArray) {
+      const tmparr = kv.split("=");
+      params[tmparr[0]] = tmparr[1];
+    }
+  }
+  return params;
+}
+
+function getVideoId() {
+  if (window.location.pathname == "/watch") {
+    return parseParams(window.location.href)["v"];
+  } else if (window.location.pathname.indexOf("/embed/") == 0) {
+    return window.location.pathname.substring("/embed/".length);
+  } else {
+    return null;
+  }
+}
+
+function getYoutubeURL() {
+  var player = getStreamPlayer() as HTMLVideoElement;
+  var time = player.currentTime;
+  var path = document.location.href;
+  path = path.split("?")[0];
+  path += "?v=" + getVideoId() + "&t=" + Math.floor(time) + "s";
+  return path;
+}
+
+console.log("Content script finished executing");

winManifest.json

@@ -0,0 +1,9 @@
+{
+	"name": "com.microsoft.onenote.stickynotes",
+	"description": "Native host for OneNote Extension",
+	"path": "C:\\Program Files\\Microsoft Office\\root\\Office16\\onenote.exe",
+	"type": "stdio",
+	"allowed_origins": [
+	  "chrome-extension://gopaeppdaekmbdnampgofgnfdcfojdbn/"
+	]
+  }