Feature: GitHub sync

develop/README.md

@@ -80,6 +80,7 @@ each service:
 | `history-v1`       | 9239 |
 | `project-history`  | 9240 |
 | `linked-url-proxy` | 9241 |
+| `github-sync`      | 9242 |
 
 To attach to a service using Chrome's _remote debugging_, go to
 <chrome://inspect/> and make sure _Discover network targets_ is checked. Next

develop/docker-compose.dev.yml

@@ -65,6 +65,17 @@ services:
       - ../services/filestore/app.js:/overleaf/services/filestore/app.js
       - ../services/filestore/config:/overleaf/services/filestore/config
 
+  github-sync:
+    command: ["node", "--watch", "app.js"]
+    environment:
+      - NODE_OPTIONS=--inspect=0.0.0.0:9229
+    ports:
+      - "127.0.0.1:9242:9229"
+    volumes:
+      - ../services/github-sync/app:/overleaf/services/github-sync/app
+      - ../services/github-sync/app.js:/overleaf/services/github-sync/app.js
+      - ../services/github-sync/config:/overleaf/services/github-sync/config
+
   history-v1:
     command: ["node", "--watch", "app.js"]
     environment:

develop/docker-compose.yml

@@ -71,6 +71,13 @@ services:
       - filestore-uploads:/overleaf/services/filestore/uploads
       - history-v1-buckets:/buckets
 
+  github-sync:
+    build:
+      context: ..
+      dockerfile: services/github-sync/Dockerfile
+    env_file:
+      - dev.env
+
   history-v1:
     build:
       context: ..
@@ -155,6 +162,7 @@ services:
     volumes:
       - sharelatex-data:/var/lib/overleaf
       - web-data:/overleaf/services/web/data
+      - ./data/certs/:/var/lib/overleaf/certs/
     depends_on:
       - mongo
       - redis
@@ -232,4 +240,10 @@ services:
     ports:
       - "8081:8081"
     environment:
-      ME_CONFIG_MONGODB_SERVER: mongo
+      ME_CONFIG_MONGODB_SERVER: mongo
+
+  # For ldap testing
+  # https://github.com/rroemhild/docker-test-openldap
+  ldap:
+    restart: always
+    image: ghcr.io/rroemhild/docker-test-openldap:v2.5.0

services/github-sync/.nvmrc

@@ -0,0 +1 @@
+22.18.0

services/github-sync/Dockerfile

@@ -0,0 +1,46 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/overleaf/internal/
+
+FROM node:22.18.0 AS base
+
+WORKDIR /overleaf/services/github-sync
+
+# Google Cloud Storage needs a writable $HOME/.config for resumable uploads
+# (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream)
+RUN mkdir /home/node/.config && chown node:node /home/node/.config
+
+FROM base AS app
+
+COPY package.json package-lock.json /overleaf/
+COPY libraries/access-token-encryptor/package.json /overleaf/libraries/access-token-encryptor/package.json
+COPY libraries/fetch-utils/package.json /overleaf/libraries/fetch-utils/package.json
+COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
+COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
+COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json
+COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
+COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
+COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
+COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
+COPY services/github-sync/package.json /overleaf/services/github-sync/package.json
+COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
+COPY patches/ /overleaf/patches/
+COPY tools/migrations/ /overleaf/tools/migrations/
+
+RUN cd /overleaf && npm ci --quiet
+COPY libraries/access-token-encryptor/ /overleaf/libraries/access-token-encryptor/
+COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
+COPY libraries/logger/ /overleaf/libraries/logger/
+COPY libraries/metrics/ /overleaf/libraries/metrics/
+COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/
+COPY libraries/o-error/ /overleaf/libraries/o-error/
+COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
+COPY libraries/settings/ /overleaf/libraries/settings/
+COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
+COPY services/github-sync/ /overleaf/services/github-sync/
+COPY tools/migrations/ /overleaf/tools/migrations/
+
+FROM app
+USER node
+
+CMD ["node", "--expose-gc", "app.js"]

services/github-sync/README.md

@@ -0,0 +1,3 @@
+# GitHub Sync Service
+
+Overleaf Github Sync Service, @Ayaka-notes.

services/github-sync/app.js

@@ -0,0 +1,19 @@
+// Metrics must be initialized before importing anything else
+import '@overleaf/metrics/initialize.js'
+
+import logger from '@overleaf/logger'
+import Settings from '@overleaf/settings'
+import { createServer } from './app/js/server.js'
+
+const port = Settings.internal?.githubSync?.port
+const host = Settings.internal?.githubSync?.host
+
+const { server } = createServer()
+server.listen(port, host, err => {
+  if (err) {
+    logger.fatal({ err }, `Cannot bind to ${host}:${port}. Exiting.`)
+    process.exit(1)
+  }
+
+  logger.info({ host, port }, 'GitHub Sync service listening')
+})

services/github-sync/app/js/mongodb.js

@@ -0,0 +1,26 @@
+// @ts-check
+
+import Metrics from '@overleaf/metrics'
+import Settings from '@overleaf/settings'
+import MongoUtils from '@overleaf/mongo-utils'
+import { MongoClient } from 'mongodb'
+
+export { ObjectId } from 'mongodb'
+
+export const mongoClient = new MongoClient(
+  Settings.mongo.url,
+  Settings.mongo.options
+)
+const mongoDb = mongoClient.db()
+
+export const db = {
+  githubSyncEntityVersions: mongoDb.collection('githubSyncEntityVersions'),
+  githubSyncProjectStates: mongoDb.collection('githubSyncProjectStates'),
+  githubSyncUserCredentials: mongoDb.collection('githubSyncUserCredentials'),
+}
+
+Metrics.mongodb.monitor(mongoClient)
+
+export async function cleanupTestDatabase() {
+  await MongoUtils.cleanupTestDatabase(mongoClient)
+}

services/github-sync/app/js/server.js

@@ -0,0 +1,16 @@
+import express from 'express'
+import { db, ObjectId } from './mongodb.js'
+
+export function createServer() {
+  const app = express()
+
+  app.get('/status', (_req, res) => {
+    res.status(200).json({ status: 'ok', service: 'github-sync' })
+  })
+
+  app.get('/healthz', (_req, res) => {
+    res.sendStatus(204)
+  })
+
+  return { app, server: app }
+}

services/github-sync/config/settings.defaults.cjs

@@ -0,0 +1,30 @@
+const http = require('node:http')
+const https = require('node:https')
+
+http.globalAgent.maxSockets = 300
+http.globalAgent.keepAlive = false
+https.globalAgent.keepAlive = false
+
+module.exports = {
+  internal: {
+    githubSync: {
+      host: process.env.GITHUB_SYNC_HOST || process.env.LISTEN_ADDRESS || '127.0.0.1',
+      port: 3022,
+    },
+  },
+
+  mongo: {
+    url:
+      process.env.MONGO_CONNECTION_STRING ||
+      `mongodb://${process.env.MONGO_HOST || '127.0.0.1'}/sharelatex`,
+    options: {
+      monitorCommands: true,
+    },
+  },
+
+  apis: {
+    project_history: {
+      url: `http://${process.env.PROJECT_HISTORY_HOST || '127.0.0.1'}:3054`,
+    }
+  }
+}

services/github-sync/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@overleaf/github-sync",
+  "description": "GitHub synchronization microservice for Overleaf",
+  "private": true,
+  "main": "app.js",
+  "type": "module",
+  "scripts": {
+    "start": "node app.js",
+    "nodemon": "node --watch app.js",
+    "lint": "eslint --max-warnings 0 --format unix .",
+    "format": "prettier --list-different $PWD/'**/{*.*js,*.ts}'",
+    "format:fix": "prettier --write $PWD/'**/{*.*js,*.ts}'",
+    "lint:fix": "eslint --fix .",
+    "types:check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@octokit/request": "^9.2.2",
+    "@overleaf/access-token-encryptor": "*",
+    "@overleaf/fetch-utils": "*",
+    "@overleaf/logger": "*",
+    "@overleaf/metrics": "*",
+    "@overleaf/mongo-utils": "*",
+    "@overleaf/o-error": "*",
+    "@overleaf/promise-utils": "*",
+    "@overleaf/settings": "*",
+    "async": "^3.2.5",
+    "base64-stream": "^0.1.2",
+    "body-parser": "1.20.4",
+    "bunyan": "^1.8.15",
+    "express": "4.22.1",
+    "lodash": "^4.17.21",
+    "mongodb-legacy": "6.1.3",
+    "nock": "^13.5.6",
+    "octonode": "0.9.5",
+    "p-limit": "^2.2.0",
+    "randomstring": "^1.1.5",
+    "request": "2.88.2"
+  },
+  "devDependencies": {
+    "@overleaf/migrations": "*",
+    "@overleaf/stream-utils": "*",
+    "@pollyjs/adapter-node-http": "^6.0.6",
+    "@pollyjs/core": "^6.0.6",
+    "@pollyjs/persister-fs": "^6.0.6",
+    "chai": "^4.3.6",
+    "chai-as-promised": "^7.1.1",
+    "mocha": "^11.1.0",
+    "mocha-junit-reporter": "^2.2.1",
+    "mocha-multi-reporters": "^1.5.1",
+    "sandboxed-module": "^2.0.4",
+    "sinon": "^9.2.4",
+    "sinon-chai": "^3.7.0",
+    "timekeeper": "2.2.0",
+    "typescript": "^5.0.4"
+  },
+  "version": "1.0.0",
+  "license": "AGPL-3.0"
+}

services/web/config/settings.defaults.js

@@ -1011,10 +1011,25 @@ module.exports = {
     mainEditorLayoutPanels: [],
     langFeedbackLinkingWidgets: [],
     labsExperiments: [],
-    integrationLinkingWidgets: [],
+    integrationLinkingWidgets: [
+      Path.resolve(
+        __dirname,
+        '../modules/github-sync/frontend/js/components/github-sync-widget.tsx'
+      ),
+    ],
     referenceLinkingWidgets: [],
-    importProjectFromGithubModalWrapper: [],
-    importProjectFromGithubMenu: [],
+    importProjectFromGithubModalWrapper: [
+      Path.resolve(
+        __dirname,
+        '../modules/github-sync/frontend/js/components/import-from-github-modal-wrapper.tsx'
+      ),
+    ],
+    importProjectFromGithubMenu: [
+      Path.resolve(
+        __dirname,
+        '../modules/github-sync/frontend/js/components/import-from-github-menu.tsx'
+      ),
+    ],
     editorLeftMenuSync: [
       Path.resolve(
         __dirname,
@@ -1105,6 +1120,7 @@ module.exports = {
     'login-register',
     'oauth2-server',
     'git-bridge',
+    'github-sync'
   ],
   viewIncludes: {},
 
@@ -1154,4 +1170,11 @@ module.exports.oauthProviders = {
       linkPath: '/oidc/login',
     },
   }),
+}
+
+module.exports.githubSync = {
+  enabled: process.env.GITHUB_SYNC_ENABLED === 'true',
+  clientID: process.env.GITHUB_SYNC_CLIENT_ID,
+  clientSecret: process.env.GITHUB_SYNC_CLIENT_SECRET,
+  callbackURL: process.env.GITHUB_SYNC_CALLBACK_URL,
 }

services/web/modules/github-sync/app/models/githubSyncUserCredentials.mjs

@@ -0,0 +1,17 @@
+import mongoose from "../../../../app/src/infrastructure/Mongoose.mjs"
+
+const { Schema } = mongoose
+const { ObjectId } = Schema
+
+export const GitHubSyncUserCredentialsSchema = new Schema(
+  {
+    userId: { type: ObjectId, ref: 'User', required: true, unique: true },
+    auth_token_encrypted: { type: String, required: true },
+  },
+  { collection: 'githubSyncUserCredentials', minimize: false }
+)
+
+export const GitHubSyncUserCredentials = mongoose.model(
+  'GitHubSyncUserCredentials',
+  GitHubSyncUserCredentialsSchema,
+)