diff --git a/.github/workflows/build_dev_image.yml b/.github/workflows/build_dev_image.yml index 38b25a6acf..90ac4073c6 100644 --- a/.github/workflows/build_dev_image.yml +++ b/.github/workflows/build_dev_image.yml @@ -36,7 +36,8 @@ jobs: docstore, contacts, chat, - clsi + clsi, + github-sync ] platform: [linux/amd64, linux/arm64] include: @@ -97,7 +98,8 @@ jobs: docstore, contacts, chat, - clsi + clsi, + github-sync ] steps: diff --git a/README.md b/README.md index 524a831262..cad1d86b54 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,29 @@ License

-A screenshot of a project being edited in Overleaf Community Edition +A screenshot of a project being edited in Overleaf Community Edition

- Figure 1: A screenshot of a project being edited in Overleaf Community Edition. + Figure 1: A screenshot of a project being edited in Overleaf Pro Edition.

+## Overleaf Pro Edition +Overleaf Pro is an enhanced version of Overleaf with almost all features and capabilities. For details, please check [Overleaf Pro](https://overleaf-pro.ayaka.space) page. Features in Overleaf Pro include: + +- GitHub Sync in 2-ways (Features in SaaS) +- Git-Bridge Support (Features in Server Pro) +- Admin Panel (User/Project management) +- SSO with LDAP and SAML or OAuth 2.0 +- Unlimited Compile Times (Adjustable in admin panel) +- Self Register (Optional, can be limited by mail domain) +- Sandbox Compile (With [texlive-full](https://github.com/ayaka-notes/texlive-full) Image Support) +- Template System (With Template Gallery) +- Track Changes (With Review and Comment Panel) +- Full Project History +- Symbol Palette +- ARM Support + +Last but not least, Overleaf Pro is open-source, free to use and modify. You can self-host it and contribute to the development of Overleaf Pro. For more details, please check [Developer Documentation](https://overleaf-pro.ayaka.space/dev) page. + ## Community Edition [Overleaf](https://www.overleaf.com) is an open-source online real-time collaborative LaTeX editor. We run a hosted version at [www.overleaf.com](https://www.overleaf.com), but you can also run your own local version, and contribute to the development of Overleaf. @@ -79,3 +97,7 @@ Please see the [CONTRIBUTING](CONTRIBUTING.md) file for information on contribut The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. A copy can be found in the [`LICENSE`](LICENSE) file. Copyright (c) Overleaf, 2014-2025. + +## Star History + +[![Star History Chart](https://api.star-history.com/image?repos=ayaka-notes/overleaf-pro&type=timeline&legend=top-left)](https://www.star-history.com/?repos=ayaka-notes%2Foverleaf-pro&type=timeline&legend=top-left) diff --git a/develop/README.md b/develop/README.md index 2cd6a38c4b..bd1e299a96 100644 --- a/develop/README.md +++ b/develop/README.md @@ -80,6 +80,7 @@ each service: | `history-v1` | 9239 | | `project-history` | 9240 | | `linked-url-proxy` | 9241 | +| `github-sync` | 9242 | To attach to a service using Chrome's _remote debugging_, go to and make sure _Discover network targets_ is checked. Next diff --git a/develop/docker-compose.dev.yml b/develop/docker-compose.dev.yml index d6f20310fd..d9d7796d9b 100644 --- a/develop/docker-compose.dev.yml +++ b/develop/docker-compose.dev.yml @@ -65,6 +65,17 @@ services: - ../services/filestore/app.js:/overleaf/services/filestore/app.js - ../services/filestore/config:/overleaf/services/filestore/config + github-sync: + command: ["node", "--watch", "app.js"] + environment: + - NODE_OPTIONS=--inspect=0.0.0.0:9229 + ports: + - "127.0.0.1:9242:9229" + volumes: + - ../services/github-sync/app:/overleaf/services/github-sync/app + - ../services/github-sync/app.js:/overleaf/services/github-sync/app.js + - ../services/github-sync/config:/overleaf/services/github-sync/config + history-v1: command: ["node", "--watch", "app.js"] environment: diff --git a/develop/docker-compose.yml b/develop/docker-compose.yml index c22ec0129c..6630790d83 100644 --- a/develop/docker-compose.yml +++ b/develop/docker-compose.yml @@ -71,6 +71,13 @@ services: - filestore-uploads:/overleaf/services/filestore/uploads - history-v1-buckets:/buckets + github-sync: + build: + context: .. + dockerfile: services/github-sync/Dockerfile + env_file: + - dev.env + history-v1: build: context: .. @@ -155,6 +162,7 @@ services: volumes: - sharelatex-data:/var/lib/overleaf - web-data:/overleaf/services/web/data + - ./data/certs/:/var/lib/overleaf/certs/ depends_on: - mongo - redis @@ -232,4 +240,10 @@ services: ports: - "8081:8081" environment: - ME_CONFIG_MONGODB_SERVER: mongo \ No newline at end of file + ME_CONFIG_MONGODB_SERVER: mongo + + # For ldap testing + # https://github.com/rroemhild/docker-test-openldap + ldap: + restart: always + image: ghcr.io/rroemhild/docker-test-openldap:v2.5.0 \ No newline at end of file diff --git a/doc/screenshot-pro.png b/doc/screenshot-pro.png new file mode 100644 index 0000000000..8ccb6a9f2e Binary files /dev/null and b/doc/screenshot-pro.png differ diff --git a/server-ce/config/env.sh b/server-ce/config/env.sh index 7c12b7aa30..a0364c65d1 100644 --- a/server-ce/config/env.sh +++ b/server-ce/config/env.sh @@ -12,6 +12,7 @@ export PROJECT_HISTORY_HOST=127.0.0.1 export REALTIME_HOST=127.0.0.1 export WEB_HOST=127.0.0.1 export WEB_API_HOST=127.0.0.1 +export GITHUB_SYNC_HOST=127.0.0.1 # If SANDBOXED_COMPILES_SIBLING_CONTAINERS is set to true, # we need to set the TEXLIVE_IMAGE_USER to www-data so that the diff --git a/server-ce/runit/github-sync/run b/server-ce/runit/github-sync/run new file mode 100755 index 0000000000..1b028cbfec --- /dev/null +++ b/server-ce/runit/github-sync/run @@ -0,0 +1,12 @@ +#!/bin/bash + +NODE_PARAMS="" +if [ "$DEBUG_NODE" == "true" ]; then + echo "running debug - github-sync" + NODE_PARAMS="--inspect=0.0.0.0:30670" +fi + +source /etc/overleaf/env.sh +export LISTEN_ADDRESS=127.0.0.1 + +exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/github-sync/app.js >> /var/log/overleaf/github-sync.log 2>&1 \ No newline at end of file diff --git a/server-ce/services.js b/server-ce/services.js index 546df2bc3a..5cf372e1b6 100644 --- a/server-ce/services.js +++ b/server-ce/services.js @@ -35,6 +35,9 @@ module.exports = [ { name: 'linked-url-proxy', }, + { + name: 'github-sync', + } ] if (require.main === module) { diff --git a/services/github-sync/.nvmrc b/services/github-sync/.nvmrc new file mode 100644 index 0000000000..89b93fd74a --- /dev/null +++ b/services/github-sync/.nvmrc @@ -0,0 +1 @@ +22.18.0 \ No newline at end of file diff --git a/services/github-sync/Dockerfile b/services/github-sync/Dockerfile new file mode 100644 index 0000000000..4e6847ea67 --- /dev/null +++ b/services/github-sync/Dockerfile @@ -0,0 +1,46 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/overleaf/internal/ + +FROM node:22.18.0 AS base + +WORKDIR /overleaf/services/github-sync + +# Google Cloud Storage needs a writable $HOME/.config for resumable uploads +# (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream) +RUN mkdir /home/node/.config && chown node:node /home/node/.config + +FROM base AS app + +COPY package.json package-lock.json /overleaf/ +COPY libraries/access-token-encryptor/package.json /overleaf/libraries/access-token-encryptor/package.json +COPY libraries/fetch-utils/package.json /overleaf/libraries/fetch-utils/package.json +COPY libraries/logger/package.json /overleaf/libraries/logger/package.json +COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json +COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json +COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json +COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json +COPY libraries/settings/package.json /overleaf/libraries/settings/package.json +COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json +COPY services/github-sync/package.json /overleaf/services/github-sync/package.json +COPY tools/migrations/package.json /overleaf/tools/migrations/package.json +COPY patches/ /overleaf/patches/ +COPY tools/migrations/ /overleaf/tools/migrations/ + +RUN cd /overleaf && npm ci --quiet +COPY libraries/access-token-encryptor/ /overleaf/libraries/access-token-encryptor/ +COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/ +COPY libraries/logger/ /overleaf/libraries/logger/ +COPY libraries/metrics/ /overleaf/libraries/metrics/ +COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/ +COPY libraries/o-error/ /overleaf/libraries/o-error/ +COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/ +COPY libraries/settings/ /overleaf/libraries/settings/ +COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/ +COPY services/github-sync/ /overleaf/services/github-sync/ +COPY tools/migrations/ /overleaf/tools/migrations/ + +FROM app +USER node + +CMD ["node", "--expose-gc", "app.js"] \ No newline at end of file diff --git a/services/github-sync/README.md b/services/github-sync/README.md new file mode 100644 index 0000000000..4f67510e31 --- /dev/null +++ b/services/github-sync/README.md @@ -0,0 +1,36 @@ +# GitHub Sync Service + +Overleaf Github Sync Service, @Ayaka-notes. + +## Service Overview +This service is only responsible for 2 things: +- export existing Overleaf projects to GitHub repositories +- export existing overleaf changes and merge changes from GitHub repositories to Overleaf projects + +## How we do sync bewtween Overleaf and GitHub +We use `sync point` to track the last synced commit in GitHub and the last synced version in Overleaf, which is stored in the mongodb. A `sync point` means a version in overleaf and a commit in github, they are totally the same content. + +When we do sync, we will first check the last overleaf version and current overleaf version, if they are different, we will export the changes from overleaf to github, **as a branch**, we call it `overleaf branch` in the later. + +Then we will try to merge this branch to the default branch. However, in GitHub, there are 2 kinds of merge: +- fast forward merge: if there is no new commit in the default branch, we can directly merge the branch to the default branch, and update the sync point. +- normal merge: if there are new commits in the default branch, we need to create a merge commit to merge the branch to the default branch, and update the sync point. + +Now we have the new commits in the default branch (called newSha in the code), we will try to merge the new commits to overleaf. + +Since GitHub stored all the files in the repository, we can get the changed files between the last synced commit and the new commit, we only return changes files with URLs, then web service will download the changed files and update the overleaf project. + +## What if there are conflicts? +If there are conflicts, we will not merge the branch to the default branch, and we will return error to the web service, then web service will get sync status and show the conflict to the user, and ask the user to resolve the conflict in GitHub. + +If user merge that conflict branch to the default branch, then we will update the sync point and merge the changes to overleaf. + +> How we detect if a branch is merged to the default branch? +> +> We will use GitHub API to diff the default branch and the `overleaf branch`, if the default falls behind the branch, it means the branch is not merged, if the default branch is ahead of the branch, it means the branch is merged. +> +> In a corner case, if the overleaf branch is merged but deleted, we will use the latest commit in the default branch as the new sync point, and merge the changes to overleaf. + + +## Copyright +Copyright (C) 2026 Ayaka-notes. \ No newline at end of file diff --git a/services/github-sync/app.js b/services/github-sync/app.js new file mode 100644 index 0000000000..24b9a7afa4 --- /dev/null +++ b/services/github-sync/app.js @@ -0,0 +1,28 @@ +// Metrics must be initialized before importing anything else +import '@overleaf/metrics/initialize.js' +import logger from '@overleaf/logger' +import Settings from '@overleaf/settings' +import { createServer } from './app/js/server.js' +import { mongoClient } from './app/js/mongodb.js' + +const port = Settings.internal?.githubSync?.port +const host = Settings.internal?.githubSync?.host +mongoClient + .connect() + .then(() => { + logger.debug('Connected to MongoDB from GitHub Sync service') + }) + .catch(err => { + logger.fatal({ err }, 'Cannot connect to mongo. Exiting.') + process.exit(1) + }) + +const { server } = createServer() +server.listen(port, host, err => { + if (err) { + logger.fatal({ err }, `Cannot bind to ${host}:${port}. Exiting.`) + process.exit(1) + } + + logger.info({ host, port }, 'GitHub Sync service listening') +}) \ No newline at end of file diff --git a/services/github-sync/app/js/GitHubSyncController.js b/services/github-sync/app/js/GitHubSyncController.js new file mode 100644 index 0000000000..19a98e8c6e --- /dev/null +++ b/services/github-sync/app/js/GitHubSyncController.js @@ -0,0 +1,287 @@ +import { GitHubSyncProjectStates } from './modals/index.js' +import GithubSyncHandler from './GitHubSyncHandler.js' +import { expressify } from '@overleaf/promise-utils' +import logger from '@overleaf/logger' + + +// This function will create a new repo on GitHub, export current project to that repo, +// and link the repo with the project by saving sync status in database. +// body: {name: "xxx", description: "xxx", private: true, org: "github-org-name"} +// name: the name of the repo to be created on GitHub, required +// description: the description of the repo to be created on GitHub, optional +// private: whether the repo is private or not, required +// org: if provided, the repo will be created under the organization, +// otherwise it will be created under user's account. +async function exportProjectToGithub(req, res, next) { + const { Project_id: projectId, user_id: userId } = req.params + const { name, description, private: isPrivate, org } = req.body + // org can be optional + if (!projectId || !name || isPrivate === undefined) { + return res.status(400).json({ error: 'Project_id, name and private are required' }) + } + + try { + const projectStatus = await GithubSyncHandler.promises.getProjectGitHubSyncStatus(projectId) + if (projectStatus) { + return res.status(400).json({ error: 'Project is already linked to a GitHub repository' }) + } + const repoResult = await GithubSyncHandler.promises.createRepositoryOnGitHub( + userId, + name, + description, + isPrivate, + org + ) + + const repoFullName = repoResult.full_name + const defaultBranch = repoResult.default_branch + const statusData = await GithubSyncHandler.promises.initializeRepositoryForProject( + projectId, + userId, + repoFullName, + defaultBranch + ) + + res.json({ statusData }) + } catch (err) { + res.status(500).json({ error: err.message }) + } +} + + +// This funcion will check github sync status. +// 0. No merge_status in db, return error, no linked repo. +// 1. If merge_status is `success` +// a), we will export a changes in overleaf since last sync to github, +// as a branch with name `overleaf-2026-02-26-1528` +// b), we will call api to merge the branch `overleaf-2026-02-26-1528` to default branch in our db. +// c), If merge success, goto step 3, +// if failed, we will set merge_status to `failure`, and +// set unmerged_branch to `overleaf-2026-02-26-1528`, +// and return error to client, [end] + +// 2. If merge_status is `failure` we will check if there is an unmerged_branch on Github +// a), if unmerged_branch deleted, we will choose defeault branch latest commit sha as next +// b), if unmerged_branch still exists, we compare the unmerged_branch with default branch, +// [] if unmerged_branch falls behind, do nothing, remain conflict status. +// [] if unmerged_branch is ahead, we can try to merge unmerged_branch to default +// branch, if success, goto step 3, if failed, return error + +// 3. we need to remember the new merged sha, and compare it with old sha. +// a), list the differences between old sha and new sha +// b), post the changes to web service, give them a [filePath, URL], +// just like what we do in git-bridge, we use an internal API/v0 +// c), web service will download URL to a temp folder, and apply all changes to the project +// this is a realtime API call. + +// 4. we need to update the sync status in our db, +// set merge_status to `success`, unmerged_branch to null +// update last_sync_sha to new merged sha, and last_sync_version to version we just merged. +// [end] + +// What should we return? +// If nothing wrong, we return +// { +// "newSha": string, // the new sha after merge +// "files": [ +// "name": string, // the file path in overleaf project +// "url": string, // the url we can download the file content, +// null if no changes +// ] +// } +// We leave web to to import changes and update project sync infomation. +async function mergeToGitHubAndPushback(req, res, next) { + logger.info('Received request to merge changes to GitHub and push back changes if needed', { params: req.params }) + const { Project_id: projectId, user_id: userId } = req.params + + let message = 'Sync changes from Overleaf' + if (req.body && req.body.message) { + message = req.body.message + } + + try { + // Step 0, check if the project is linked to a GitHub repository + const projectStatus = await GithubSyncHandler.promises.getProjectGitHubSyncStatus(projectId) + if (!projectStatus) { + return res.status(400).json({ error: 'Project is not linked to a GitHub repository' }) + } + let newSha = "" + + // Step 1, last merge success, we can try to merge new changes to github + if (projectStatus.merge_status === 'success') { + const latestVersionData = await GithubSyncHandler.promises.getProjectLatestVersion(projectId) + const latestVersion = latestVersionData.version + const last_sync_version = projectStatus.last_sync_version + const last_sync_sha = projectStatus.last_sync_sha + const branch_latest_sha = await GithubSyncHandler.promises.getBranchHeadCommitSha( + projectStatus.repo, + projectStatus.default_branch, + userId + ) + + // only export changes + if (latestVersion > last_sync_version) { + // create a new branch from last_sync_sha + const now = new Date() + const branchName = `overleaf-${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(now.getDate()).padStart(2, '0')}-${String(now.getHours()).padStart(2, '0')}${String(now.getMinutes()).padStart(2, '0')}` + + const branchCreated = await GithubSyncHandler.promises.createOrUpdateBranchRef( + projectStatus.repo, + branchName, + last_sync_sha, + userId + ) + + // export changes to github + await GithubSyncHandler.promises.exportChangesToGitHub( + projectId, + userId, + projectStatus.repo, + branchName, + last_sync_version, + latestVersion, + last_sync_sha, + message + ) + + // Try merge the branch to default branch, if failed, record. + try { + if (branch_latest_sha !== last_sync_sha) { + const mergeResult = await GithubSyncHandler.promises.mergeBranchToDefaultBranch( + projectStatus.repo, + branchName, + projectStatus.default_branch, + userId + ) + newSha = mergeResult.sha + } else { + // we need to fast-forward the default branch + const mergeResult = await GithubSyncHandler.promises.fastForwardBranchToDefaultBranch( + projectStatus.repo, + branchName, + projectStatus.default_branch, + userId + ) + newSha = mergeResult.sha + } + + + + logger.debug({ projectId, branchName, newSha }, 'Branch merged to default branch successfully') + // delete overleaf branch + await GithubSyncHandler.promises.deleteBranchOnGitHub( + projectStatus.repo, + branchName, + userId + ) + + } catch (err) { + // update merge_status to failure, and save the unmerged_branch + await GithubSyncHandler.promises.updateProjectGitHubSyncStatus(projectId, { + merge_status: 'failure', + unmerged_branch: branchName, + }) + + logger.error('Failed to merge branch to default branch', { err }) + return res.status(500).json({ + error: 'Failed to merge changes to GitHub, please resolve the conflict on GitHub and try again' + }) + } + + } else if (latestVersion === last_sync_version) { + // newSha will be the latest sha in default branch + newSha = await GithubSyncHandler.promises.getBranchHeadCommitSha( + projectStatus.repo, + projectStatus.default_branch, + userId + ) + + } + + } else if (projectStatus.merge_status === 'failure') { + // If the last merge failed, we try to re-merge the unmerged branch + const unmergedBranch = projectStatus.unmerged_branch + if (!unmergedBranch) { + return res.status(500).json({ error: 'Unmerged branch info is missing.' }) + } + + try { + // const mergeResult = await GithubSyncHandler.promises.mergeBranchToDefaultBranch( + // projectStatus.repo, + // unmergedBranch, + // projectStatus.default_branch, + // userId + // ) + const diff = await GithubSyncHandler.promises.diffBranchsOnGitHub( + projectStatus.repo, + projectStatus.default_branch, + unmergedBranch, + userId + ) + + if (diff.length === 0) { + newSha = await GithubSyncHandler.promises.getBranchHeadCommitSha( + projectStatus.repo, + projectStatus.default_branch, + userId + ) + } else { + return res.status(500).json({ error: 'There are still conflicts between unmerged branch and default branch, please resolve the conflict on GitHub and try again' }) + } + } catch (err) { + await GithubSyncHandler.promises.updateProjectGitHubSyncStatus( + projectId, + { + merge_status: 'failure', + unmerged_branch: unmergedBranch, // keep the same unmerged branch + } + ) + logger.error('Failed to re-merge unmerged branch', { err }) + return res.status(500).json({ + error: 'Failed to re-merge changes to GitHub, please resolve the conflict on GitHub and try again' + }) + } + } + + // now we have the new sha after merge. + // Step 3, we need to compare the new sha with last_sync_sha, if they are different, we need to push the changes back to overleaf. + if (newSha && newSha != "" + && newSha !== projectStatus.last_sync_sha) { + // list the differences between newSha and last_sync_sha on GitHub + const diff = await GithubSyncHandler.promises.diffChangesOnGitHub( + projectStatus.repo, + projectStatus.default_branch, + projectStatus.last_sync_sha, + newSha, + userId + ) + + const tree = await GithubSyncHandler.promises.getFileTreeOnCommit( + projectStatus.repo, + newSha, + userId + ) + + let resp = GithubSyncHandler.generateRespURL(diff, tree, projectStatus.repo, newSha) + + return res.json( + { + files: resp, + newSha: newSha + } + ) + } else { + // no changes, just return ok + return res.status(200).json({ message: 'Already up to date, no changes to sync' }) + } + } catch (err) { + return res.status(500).json({ error: err.message }) + } + // Never reach here, but just in case + return res.status(200).json({ message: 'Merge to GitHub and push back process completed' }) +} + +export default { + exportProjectToGithub: expressify(exportProjectToGithub), + mergeToGitHubAndPushback: expressify(mergeToGitHubAndPushback), +} \ No newline at end of file diff --git a/services/github-sync/app/js/GitHubSyncHandler.js b/services/github-sync/app/js/GitHubSyncHandler.js new file mode 100644 index 0000000000..6c56b42d91 --- /dev/null +++ b/services/github-sync/app/js/GitHubSyncHandler.js @@ -0,0 +1,809 @@ +import logger from '@overleaf/logger' +import { GitHubSyncProjectStates, GitHubSyncUserCredentials } from './modals/index.js' +import { ObjectId } from './mongodb.js' +import SecretHelper from './SecretHelper.js' +import Settings from '@overleaf/settings' +import HttpsProxyAgent from 'https-proxy-agent' +import fetch from 'node-fetch' + +const GITHUB_API_BASE = 'https://api.github.com' +const proxyUrl = process.env.GITHUB_SYNC_PROXY_URL +const httpsAgent = proxyUrl ? new HttpsProxyAgent(proxyUrl) : undefined + + +async function getProjectGitHubSyncStatus(projectId) { + return GitHubSyncProjectStates.findByProjectId(projectId) +} + +async function saveProjectGitHubSyncStatus(projectId, status) { + return GitHubSyncProjectStates.saveByProjectId(projectId, status) +} + +async function updateProjectGitHubSyncStatus(projectId, status) { + return GitHubSyncProjectStates.updateByProjectId(projectId, status) +} + + +async function getUserGitHubCredentials(userId) { + const credentials = await GitHubSyncUserCredentials.findByUserId(userId) + if (!credentials) { + return null + } + return await SecretHelper.decryptAccessToken(credentials.auth_token_encrypted) +} + + +// This function will create a repository on GitHub for the project +// If org is provided, it will create the repository under the organization, +// otherwise it will create the repository under the user's account. +// We will initialize the repository with a README file, and then we will +// remove the README file later, because we need to make sure the repository +// is not empty, otherwise GitHub API will reject our commit. +// No other initialization is done in this function. +async function createRepositoryOnGitHub(userId, repoName, repoDescription, isPrivate, org) { + const accessToken = await getUserGitHubCredentials(userId) + if (!accessToken) { + throw new Error('User does not have GitHub credentials') + } + + const githubApiUrl = org + ? `${GITHUB_API_BASE}/orgs/${org}/repos` + : `${GITHUB_API_BASE}/user/repos` + + const response = await fetch(githubApiUrl, { + method: 'POST', + headers: { + 'Authorization': `token ${accessToken}`, + 'Accept': 'application/vnd.github.v3+json', + }, + agent: httpsAgent, + body: JSON.stringify({ + name: repoName, + description: repoDescription, + private: isPrivate, + auto_init: true, // we need this, but will remove later. + }), + }) + + if (!response.ok) { + const errorData = await response.json() + logger.error('Failed to create GitHub repository', { userId, repoName, error: errorData }) + throw new Error(`Repository creation failed.`) + } + + const repoData = await response.json() + return repoData +} + + + + +// Request files list from project history, should return like this +// { +// "projectId": "699fbae90f632055939d7a5d", +// "files": { +// "main.tex": { +// "data": { +// "hash": "fd3c0326302e49486d3ea86c833edf9b88320c41" +// } +// }, +// "sample.bib": { +// "data": { +// "hash": "a0e21c740cf81e868f158e30e88985b5ea1d6c19" +// } +// }, +// "frog.jpg": { +// "data": { +// "hash": "5b889ef3cf71c83a4c027c4e4dc3d1a106b27809" +// } +// }, +// } +// We added version for next step to pull file contents. +async function getProjectLatestVersion(projectId) { + let verURL = `${Settings.apis.project_history.url}/project/${projectId}/version` + const response = await fetch(verURL) + if (!response.ok) { + const errorData = await response.json() + logger.error('Failed to pull project version from Project History', { projectId, error: errorData }) + throw new Error(`Project History API error: ${errorData.message}`) + } + const versionData = await response.json() + const latestVersion = versionData.version + + let URL = `${Settings.apis.project_history.url}/project/${projectId}/version/${latestVersion}` + const fileResponse = await fetch(URL) + if (!fileResponse.ok) { + const errorData = await fileResponse.json() + logger.error('Failed to pull project files from Project History', { projectId, version: latestVersion, error: errorData }) + throw new Error(`Project History API error: ${errorData.message}`) + } + + let result = await fileResponse.json() + result.version = latestVersion + return result +} + + +// Communicate with project history service to get the file tree diff +// between two versions, and return a array +// If operation appeared, it means it has been changed. +// operation can be "added", "removed", "edited". +// [ +// { +// "pathname": "main.tex", +// "editable": true +// }, +// { +// "pathname": "sample.bib", +// "operation": "removed", +// "editable": true, +// "deletedAtV": 5 +// }, +// { +// "pathname": "frog.jpg", +// "operation": "added", +// "editable": false +// }, +// { +// "pathname": "4535345345/3453453.tex", +// "operation": "added", +// "editable": true +// } +// ] +async function getProjectFileTreeDiff(projectId, fromVersion, toVersion) { + let historyURL = `${Settings.apis.project_history.url}/project/${projectId}/filetree/diff?from=${fromVersion}&to=${toVersion}` + const response = await fetch(historyURL) + if (!response.ok) { + const errorData = await response.json() + logger.error('Failed to pull project file tree diff from Project History', { projectId, fromVersion, toVersion, error: errorData }) + throw new Error(`Project History API error: ${errorData.message}`) + } + const diffData = await response.json() + + if (!diffData || !diffData.diff) { + return [] + } + return diffData.diff +} + + +async function uploadBlobToGitHub(repoFullName, filePath, buffer, accessToken) { + const encoding = 'base64' + const content = buffer.toString('base64') + + const response = await fetch(`${GITHUB_API_BASE}/repos/${repoFullName}/git/blobs`, { + method: 'POST', + headers: { + 'Authorization': `token ${accessToken}`, + 'Accept': 'application/vnd.github.v3+json', + }, + agent: httpsAgent, + body: JSON.stringify({ + content, + encoding, + }), + }) + + if (!response.ok) { + const errorData = await response.json() + logger.error('Failed to upload blob to GitHub', { repoFullName, filePath, error: errorData }) + throw new Error(`GitHub API error: ${errorData.message}`) + } + + const blobData = await response.json() + return blobData.sha +} + +async function createTreeOnGitHub(repoFullName, blobShas, accessToken, baseTreeSha = null) { + const body = { + tree: blobShas.map(item => ({ + path: item.path, + sha: item.sha, + mode: '100644', + type: 'blob', + })), + } + if (baseTreeSha) { + body.base_tree = baseTreeSha + } + + const response = await fetch(`${GITHUB_API_BASE}/repos/${repoFullName}/git/trees`, { + method: 'POST', + headers: { + 'Authorization': `token ${accessToken}`, + 'Accept': 'application/vnd.github.v3+json', + }, + agent: httpsAgent, + body: JSON.stringify(body), + }) + + if (!response.ok) { + const errorData = await response.json() + logger.error('Failed to create tree on GitHub', { repoFullName, error: errorData }) + throw new Error(`GitHub API error: ${errorData.message}`) + } + + const treeData = await response.json() + return treeData.sha +} + + +async function createCommitOnGitHub(repoFullName, treeSha, message, accessToken, parents = []) { + const response = await fetch(`${GITHUB_API_BASE}/repos/${repoFullName}/git/commits`, { + method: 'POST', + headers: { + Authorization: `token ${accessToken}`, + Accept: 'application/vnd.github.v3+json', + }, + body: JSON.stringify({ + message, + tree: treeSha, + parents: parents, + }), + agent: httpsAgent, + }) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + logger.error('Failed to create commit on GitHub', { repoFullName, error: errorData }) + throw new Error(`GitHub API error: ${errorData.message || response.statusText}`) + } + + const commitData = await response.json() + return commitData.sha +} + +async function getBranchHeadCommitSha(repoFullName, branch, userId) { + const accessToken = await getUserGitHubCredentials(userId) + + if (!accessToken) { + throw new Error('User does not have GitHub credentials') + } + + const response = await fetch(`${GITHUB_API_BASE}/repos/${repoFullName}/git/ref/heads/${branch}`, { + method: 'GET', + headers: { + Authorization: `token ${accessToken}`, + Accept: 'application/vnd.github.v3+json', + }, + agent: httpsAgent, + }) + + if (response.status === 404) return null + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + throw new Error(`GitHub API error: ${errorData.message || response.statusText}`) + } + + const refData = await response.json() + return refData?.object?.sha || null +} + +// We need to remove init README. +async function updateBranchToCommit( + repoFullName, branch, commitSha, accessToken, ifForce = false +) { + const response = await fetch( + `${GITHUB_API_BASE}/repos/${repoFullName}/git/refs/heads/${branch}`, + { + method: 'PATCH', + headers: { + Authorization: `token ${accessToken}`, + Accept: 'application/vnd.github.v3+json', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + sha: commitSha, + force: ifForce, + }), + agent: httpsAgent, + } + ) + + const text = await response.text().catch(() => '') + if (!response.ok) { + let err = {} + try { err = JSON.parse(text) } catch { } + logger.error({ repoFullName, branch, commitSha, status: response.status, body: text }, 'Failed to force update ref') + throw new Error(`GitHub API error: ${err.message || text || response.statusText}`) + } + + return JSON.parse(text) +} + +// Communicate with project history service to get the file content buffer, and return the buffer. +async function getProjectFileBuffer(projectId, version, filePath) { + const fileURL = `${Settings.apis.project_history.url}/project/${projectId}/version/${version}/${encodeURIComponent(filePath)}` + const response = await fetch(fileURL) + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + logger.error({ projectId, version, filePath, errorData }, 'Failed to fetch file snapshot') + throw new Error(`Project History API error: ${errorData.message || response.statusText}`) + } + return Buffer.from(await response.arrayBuffer()) +} + +// Export a project to GitHub will be a complex process, +// 1. We need to get the latest version of the project, and get the file list with their hashes from project history service. +// 2. Then we need to pull the file contents from project history service, and upload the file blobs to GitHub, and get the blob shas. +// 3. Then, we need to create a tree with all the blobs, and create a commit with the tree, and finally update the ref of the repo to point to the new commit. +// 4. Finally, we need to save the GitHub sync status to the database, so we can show the status on the UI. +async function initializeRepositoryForProject(projectId, userId, repoFullName, defaultBranch) { + const accessToken = await getUserGitHubCredentials(userId) + if (!accessToken) { + throw new Error('User does not have GitHub credentials') + } + + let blobShas = [] + try { + // Get latest version, then ask for file contents. + const latestVersionData = await getProjectLatestVersion(projectId) + const latestVersion = latestVersionData.version + + for (const filePath in latestVersionData.files) { + const buffer = await getProjectFileBuffer(projectId, latestVersion, filePath) + const blobSha = await uploadBlobToGitHub(repoFullName, filePath, buffer, accessToken) + blobShas.push({ path: filePath, sha: blobSha }) + logger.debug({ projectId, filePath, blobSha }, + 'Uploaded file blob to GitHub Successfully') + } + + // Then, we need to create a tree with all the blobs, and + // create a commit with the tree, and finally update the ref + // of the repo to point to the new commit. + const treeSha = await createTreeOnGitHub( + repoFullName, blobShas, accessToken, null) + + const commitSha = await createCommitOnGitHub( + repoFullName, treeSha, `Initial Overleaf Import`, accessToken) + + const updateRefResult = await updateBranchToCommit( + repoFullName, defaultBranch, commitSha, accessToken, true) + + logger.debug({ projectId, repoFullName, treeSha, commitSha, updateRefResult }, + 'Created initial commit on GitHub Successfully') + + // Finally, we need to save the GitHub sync status to the database, + // so we can show the status on the UI. + return await saveProjectGitHubSyncStatus(projectId, { + merge_status: 'success', + default_branch: defaultBranch, + unmerged_branch: null, + last_sync_sha: commitSha, + last_sync_version: latestVersion, + repo: repoFullName, + ownerId: new ObjectId(userId), + }) + } catch (err) { + logger.error({ err, projectId }, 'Error initializing GitHub repository for project') + throw err + } +} + +// This function will export all changes in overleaf to github +async function exportChangesToGitHub( + projectId, userId, repoFullName, branch, + fromV, toV, parentCommitSha, commitMessage +) { + const accessToken = await getUserGitHubCredentials(userId) + if (!accessToken) { + throw new Error('User does not have GitHub credentials') + } + const diff = await getProjectFileTreeDiff(projectId, fromV, toV) + const upsertPaths = new Set() + const deletePaths = new Set() + + for (const item of diff) { + if (!item?.operation) continue + + if (item.operation === 'added' || item.operation === 'edited') { + upsertPaths.add(item.pathname) + continue + } + + if (item.operation === 'removed') { + deletePaths.add(item.pathname) + continue + } + + if (item.operation === 'renamed') { + deletePaths.add(item.pathname) + if (item.newPathname) upsertPaths.add(item.newPathname) + continue + } + } + + // 冲突消解：同路径既删又改，按“保留最终文件”处理 + for (const p of upsertPaths) { + if (deletePaths.has(p)) deletePaths.delete(p) + } + + const changed = Array.from(upsertPaths) + const removed = Array.from(deletePaths) + + // 没变化直接返回 + if (changed.length === 0 && removed.length === 0) { + return { + noChange: true, + commitSha: parentCommitSha, + changed, + removed, + } + } + + const treeEntries = [] + + // 处理新增/修改：拉最新版本文件 -> upload blob + for (const filePath of changed) { + const buffer = await getProjectFileBuffer(projectId, toV, filePath) + const blobSha = await uploadBlobToGitHub(repoFullName, filePath, buffer, accessToken) + treeEntries.push({ path: filePath, sha: blobSha }) + } + + // 处理删除：sha=null + for (const filePath of removed) { + treeEntries.push({ path: filePath, sha: null }) + } + + const baseTreeSha = await getCommitTreeSha(repoFullName, parentCommitSha, accessToken) + const treeSha = await createTreeOnGitHub(repoFullName, treeEntries, accessToken, baseTreeSha) + + const commitSha = await createCommitOnGitHub( + repoFullName, + treeSha, + commitMessage || `Sync Overleaf changes v${fromV}..v${toV}`, + accessToken, + [parentCommitSha] + ) + + await createOrUpdateBranchRef(repoFullName, branch, commitSha, userId) + + return { + noChange: false, + commitSha, + treeSha, + changed, + removed, + } +} + +// create a new branch from commitSha +// if the branch already exists, we will update the branch to point to the new commitSha +// but only if the branch is currently pointing to the commitSha, +// otherwise we consider it as a conflict and throw an error. +async function createOrUpdateBranchRef(repoFullName, branch, commitSha, userId) { + const accessToken = await getUserGitHubCredentials(userId) + if (!accessToken) { + throw new Error('User does not have GitHub credentials') + } + const headSha = await getBranchHeadCommitSha(repoFullName, branch, userId) + + if (!headSha) { + const createResp = await fetch(`${GITHUB_API_BASE}/repos/${repoFullName}/git/refs`, { + method: 'POST', + headers: { + Authorization: `token ${accessToken}`, + Accept: 'application/vnd.github.v3+json', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + ref: `refs/heads/${branch}`, + sha: commitSha, + }), + agent: httpsAgent, + }) + if (!createResp.ok) { + const errorData = await createResp.json().catch(() => ({})) + throw new Error(`GitHub API error: ${errorData.message || createResp.statusText}`) + } + return + } + + const updateResp = await fetch(`${GITHUB_API_BASE}/repos/${repoFullName}/git/refs/heads/${branch}`, { + method: 'PATCH', + headers: { + Authorization: `token ${accessToken}`, + Accept: 'application/vnd.github.v3+json', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + sha: commitSha, + force: false, + }), + agent: httpsAgent, + }) + + if (!updateResp.ok) { + const errorData = await updateResp.json().catch(() => ({})) + throw new Error(`GitHub API error: ${errorData.message || updateResp.statusText}`) + } +} + + +async function deleteBranchOnGitHub(repoFullName, branch, userId) { + logger.debug({ repoFullName, branch }, 'Deleting branch on GitHub') + const accessToken = await getUserGitHubCredentials(userId) + if (!accessToken) { + throw new Error('User does not have GitHub credentials') + } + + const url = `${GITHUB_API_BASE}/repos/${repoFullName}/git/refs/heads/${branch}` + + const response = await fetch(url, { + method: 'DELETE', + headers: { + Authorization: `token ${accessToken}`, + Accept: 'application/vnd.github.v3+json', + }, + agent: httpsAgent, + }) + + if (response.status === 404) { + logger.warn({ repoFullName, branch }, 'Branch not found when trying to delete, ignoring') + return + } + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + logger.warn( + { repoFullName, branch, status: response.status, error: errorData }, + 'Failed to delete branch on GitHub' + ) + } + + logger.debug({ repoFullName, branch, status: response.status }, 'Deleted branch on GitHub') +} + + +async function getCommitTreeSha(repoFullName, commitSha, accessToken) { + const response = await fetch(`${GITHUB_API_BASE}/repos/${repoFullName}/git/commits/${commitSha}`, { + method: 'GET', + headers: { + Authorization: `token ${accessToken}`, + Accept: 'application/vnd.github.v3+json', + }, + agent: httpsAgent, + }) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + throw new Error(`GitHub API error: ${errorData.message || response.statusText}`) + } + + const commitData = await response.json() + return commitData?.tree?.sha +} + +async function mergeBranchToDefaultBranch(repoFullName, sourceBranch, defaultBranch, userId) { + const accessToken = await getUserGitHubCredentials(userId) + if (!accessToken) { + throw new Error('User does not have GitHub credentials') + } + + const response = await fetch(`${GITHUB_API_BASE}/repos/${repoFullName}/merges`, { + method: 'POST', + headers: { + Authorization: `token ${accessToken}`, + Accept: 'application/vnd.github.v3+json', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + base: defaultBranch, + head: sourceBranch, + commit_message: `Merge ${sourceBranch} to ${defaultBranch}`, + }), + agent: httpsAgent, + }) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + throw new Error(`GitHub API error: ${errorData.message || response.statusText}`) + } + + const mergeData = await response.json() + return mergeData +} + +async function fastForwardBranchToDefaultBranch(repoFullName, sourceBranch, defaultBranch, userId) { + const accessToken = await getUserGitHubCredentials(userId) + if (!accessToken) { + throw new Error('User does not have GitHub credentials') + } + + const sourceHeadSha = await getBranchHeadCommitSha(repoFullName, sourceBranch, userId) + if (!sourceHeadSha) { + throw new Error(`Source branch ${sourceBranch} not found`) + } + + + // Fast forward the default branch to the source branch head sha, if possible. + let ffResult = await updateBranchToCommit( + repoFullName, defaultBranch, sourceHeadSha, accessToken, false + ) + + return { + ...ffResult, + sha: ffResult?.object?.sha || sourceHeadSha, + } +} + + +async function diffChangesOnGitHub(repoFullName, baseBranch, + fromSha, toSha, userId) { + const accessToken = await getUserGitHubCredentials(userId) + if (!accessToken) { + throw new Error('User does not have GitHub credentials') + } + + const response = await fetch(`${GITHUB_API_BASE}/repos/${repoFullName}/compare/${fromSha}...${toSha}`, { + method: 'GET', + headers: { + Authorization: `token ${accessToken}`, + Accept: 'application/vnd.github.v3+json', + }, + agent: httpsAgent, + }) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + throw new Error(`GitHub API error: ${errorData.message || response.statusText}`) + } + + const compareData = await response.json() + return compareData.files || [] +} + + +// We do a workarount here +// if user delete that branch on GitHub, we will consider merge successful +// Return [] means no diff. +async function diffBranchsOnGitHub( + repoFullName, baseBranch, compareBranch, userId +){ + const accessToken = await getUserGitHubCredentials(userId) + if (!accessToken) { + throw new Error('User does not have GitHub credentials') + } + + const response = await fetch(`${GITHUB_API_BASE}/repos/${repoFullName}/compare/${baseBranch}...${compareBranch}`, { + method: 'GET', + headers: { + Authorization: `token ${accessToken}`, + Accept: 'application/vnd.github.v3+json', + }, + agent: httpsAgent, + }) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + logger.error('Failed to diff branches on GitHub', { repoFullName, baseBranch, compareBranch, error: errorData }) + + // treat 404 as no diff. + if (errorData.status === '404' || response.status === 404) { + return [] + } + throw new Error(`GitHub API error: ${errorData.message || response.statusText}`) + } + const compareData = await response.json() + return compareData.files || [] +} + +// Return File list on Github on a specific commit. +async function getFileTreeOnCommit(repoFullName, commitSha, userId) { + const accessToken = await getUserGitHubCredentials(userId) + if (!accessToken) { + throw new Error('User does not have GitHub credentials') + } + + const response = await fetch(`${GITHUB_API_BASE}/repos/${repoFullName}/git/commits/${commitSha}`, { + method: 'GET', + headers: { + Authorization: `token ${accessToken}`, + Accept: 'application/vnd.github.v3+json', + }, + agent: httpsAgent, + }) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) + throw new Error(`GitHub API error: ${errorData.message || response.statusText}`) + } + + const commitData = await response.json() + const treeURL = commitData?.tree?.url + `?recursive=1` + if (!treeURL) { + throw new Error('Invalid commit data from GitHub API: missing tree url') + } + + const treeResponse = await fetch(treeURL, { + method: 'GET', + headers: { + Authorization: `token ${accessToken}`, + Accept: 'application/vnd.github.v3+json', + }, + agent: httpsAgent, + }) + + if (!treeResponse.ok) { + const errorData = await treeResponse.json().catch(() => ({})) + throw new Error(`GitHub API error: ${errorData.message || treeResponse.statusText}`) + } + + const treeData = await treeResponse.json() + return treeData.tree || [] +} + +// diff: +// [ +// { +// "sha": "f2e6004eece8a664d1c603ff1a0b6b13400553fb", +// "filename": "develop/dev.env", +// "status": "modified", +// "additions": 1, +// "deletions": 0, +// "changes": 1, +// "blob_url": "https://github.com/overleaf/overleaf/blob/17e01526b48b070a374cca24d779c462336560ae/develop%2Fdev.env", +// "raw_url": "https://github.com/overleaf/overleaf/raw/17e01526b48b070a374cca24d779c462336560ae/develop%2Fdev.env", +// "contents_url": "https://api.github.com/repos/overleaf/overleaf/contents/develop%2Fdev.env?ref=17e01526b48b070a374cca24d779c462336560ae", +// "patch": "@@ -1,5 +1,6 @@\n CHAT_HOST=chat\n CLSI_HOST=clsi\n+DOWNLOAD_HOST=clsi-nginx\n CONTACTS_HOST=contacts\n DOCSTORE_HOST=docstore\n DOCUMENT_UPDATER_HOST=document-updater" +// }, +// ... +// ] + + +// tree +// [ +// { +// "path": "main.tex", +// "mode": "100644", +// "type": "blob", +// "sha": "3d21ec53a331a6f037a91c368710b99387d012c1" +// "size": 30, +// "url": "https://api.github.com/repos/octocat/Hello-World/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1" +// }, +// ... +// ] +function generateRespURL(diff, tree, repoFullName, newSha) { + const resp = [] + const BaseURL = `${GITHUB_API_BASE}/repos/${repoFullName}/contents/` + for (const item of tree) { + if (item.type !== 'blob') continue + let obj = { + name: item.path, + } + + // if item.path exists in diff, we put url in that + const diffItem = diff.find(d => d.filename === item.path) + if (diffItem) { + obj.url = `${BaseURL}${item.path}?ref=${newSha}` + } + resp.push(obj) + } + return resp +} + + + +export default { + promises: { + getProjectGitHubSyncStatus, + getUserGitHubCredentials, + getFileTreeOnCommit, + createRepositoryOnGitHub, + createOrUpdateBranchRef, + getCommitTreeSha, + initializeRepositoryForProject, + getProjectFileTreeDiff, + exportChangesToGitHub, + mergeBranchToDefaultBranch, + diffChangesOnGitHub, + getProjectLatestVersion, + getBranchHeadCommitSha, + updateProjectGitHubSyncStatus, + saveProjectGitHubSyncStatus, + diffBranchsOnGitHub, + deleteBranchOnGitHub, + fastForwardBranchToDefaultBranch + }, + generateRespURL, +} diff --git a/services/github-sync/app/js/GitHubSyncMiddleware.js b/services/github-sync/app/js/GitHubSyncMiddleware.js new file mode 100644 index 0000000000..a34a6e656f --- /dev/null +++ b/services/github-sync/app/js/GitHubSyncMiddleware.js @@ -0,0 +1,33 @@ +import pLimit from 'p-limit' + +const projectLimiters = new Map() + +function getProjectLimiter(projectId) { + if (!projectLimiters.has(projectId)) { + projectLimiters.set(projectId, pLimit(1)) + } + return projectLimiters.get(projectId) +} + +export function projectConcurrencyMiddleware(req, res, next) { + const projectId = req.params.Project_id + if (!projectId) return res.status(400).json({ error: 'Missing Project_id' }) + const limiter = getProjectLimiter(projectId) + + limiter(() => new Promise(resolve => { + let released = false + const releaseOnce = () => { + if (released) return + released = true + resolve() + } + + req._releaseLimiter = releaseOnce + + res.on('finish', releaseOnce) + res.on('close', releaseOnce) + + next() + })) +} + diff --git a/services/github-sync/app/js/SecretHelper.js b/services/github-sync/app/js/SecretHelper.js new file mode 100644 index 0000000000..689cbb6f34 --- /dev/null +++ b/services/github-sync/app/js/SecretHelper.js @@ -0,0 +1,35 @@ +import AccessTokenEncryptor from '@overleaf/access-token-encryptor' +import logger from '@overleaf/logger' + +const accessTokenEncryptor = new AccessTokenEncryptor({ + cipherPasswords: { + [process.env.CIPHER_LABEL || "2042.1-v3"]: process.env.CIPHER_PASSWORD, + }, + cipherLabel: process.env.CIPHER_LABEL || "2042.1-v3", +}) + +const SecretsHelper = { + async encryptAccessToken(accessToken) { + let tokenEncrypted = "" + try { + tokenEncrypted = await accessTokenEncryptor.promises.encryptJson(accessToken) + } catch (err) { + logger.error({ err }, 'Error encrypting GitHub access token') + return "" // Return empty string on encryption failure + } + return tokenEncrypted + }, + + async decryptAccessToken(tokenEncrypted) { + let tokenDecrypted = "" + try { + tokenDecrypted = await accessTokenEncryptor.promises.decryptToJson(tokenEncrypted) + } catch (err) { + logger.error({ err }, 'Error decrypting GitHub access token') + return "" // Return empty string on decryption failure + } + return tokenDecrypted + } +} + +export default SecretsHelper \ No newline at end of file diff --git a/services/github-sync/app/js/modals/githubSyncEntityVersions.js b/services/github-sync/app/js/modals/githubSyncEntityVersions.js new file mode 100644 index 0000000000..3ed2c42595 --- /dev/null +++ b/services/github-sync/app/js/modals/githubSyncEntityVersions.js @@ -0,0 +1,86 @@ +import { db, ObjectId } from '../mongodb.js' + +const ENTITY_VERSION_TTL_MS = 1000 * 60 * 60 * 24 * 30 + +function normalizeObjectId(value) { + if (value instanceof ObjectId) { + return value + } + return new ObjectId(value) +} + +function defaultExpiresAt() { + return new Date(Date.now() + ENTITY_VERSION_TTL_MS) +} + +async function findOne(query = {}, options = {}) { + return await db.githubSyncEntityVersions.findOne(query, options) +} + +async function findByProjectIdAndEntityId(projectId, entityId, options = {}) { + return await findOne( + { + pid: normalizeObjectId(projectId), + eid: normalizeObjectId(entityId), + }, + options + ) +} + +async function saveOrUpdate(projectId, entityId, version, expiresAt) { + const now = new Date() + await db.githubSyncEntityVersions.updateOne( + { + pid: normalizeObjectId(projectId), + eid: normalizeObjectId(entityId), + }, + { + $set: { + pid: normalizeObjectId(projectId), + eid: normalizeObjectId(entityId), + v: version, + c: expiresAt || defaultExpiresAt(), + updated_at: now, + }, + $setOnInsert: { + created_at: now, + }, + }, + { upsert: true } + ) + + return await findByProjectIdAndEntityId(projectId, entityId) +} + +async function updateByProjectIdAndEntityId(projectId, entityId, update = {}) { + const now = new Date() + const nextUpdate = { ...update, updated_at: now } + if (!Object.hasOwn(nextUpdate, 'c')) { + nextUpdate.c = defaultExpiresAt() + } + + await db.githubSyncEntityVersions.updateOne( + { + pid: normalizeObjectId(projectId), + eid: normalizeObjectId(entityId), + }, + { $set: nextUpdate } + ) + + return await findByProjectIdAndEntityId(projectId, entityId) +} + +async function removeByProjectIdAndEntityId(projectId, entityId) { + return await db.githubSyncEntityVersions.deleteOne({ + pid: normalizeObjectId(projectId), + eid: normalizeObjectId(entityId), + }) +} + +export default { + findOne, + findByProjectIdAndEntityId, + saveOrUpdate, + updateByProjectIdAndEntityId, + removeByProjectIdAndEntityId, +} diff --git a/services/github-sync/app/js/modals/githubSyncProjectStates.js b/services/github-sync/app/js/modals/githubSyncProjectStates.js new file mode 100644 index 0000000000..27455d0602 --- /dev/null +++ b/services/github-sync/app/js/modals/githubSyncProjectStates.js @@ -0,0 +1,48 @@ +import { db, ObjectId } from '../mongodb.js' + +function normalizeObjectId(value) { + if (value instanceof ObjectId) { + return value + } + return new ObjectId(value) +} + +async function findOne(query = {}, options = {}) { + return await db.githubSyncProjectStates.findOne(query, options) +} + +async function findByProjectId(projectId, options = {}) { + return await findOne({ projectId: normalizeObjectId(projectId) }, options) +} + +// no upsert, only update if existed, otherwise return null +async function updateByProjectId(projectId, update = {}) { + await db.githubSyncProjectStates.updateOne( + { projectId: normalizeObjectId(projectId) }, + { $set: { ...update } } + ) + return await findByProjectId(projectId) +} + +// with upsert true +async function saveByProjectId(projectId, update = {}) { + await db.githubSyncProjectStates.updateOne( + { projectId: normalizeObjectId(projectId) }, + { $set: { ...update } }, + { upsert: true } + ) + return await findByProjectId(projectId) +} + +async function removeByProjectId(projectId) { + return await db.githubSyncProjectStates.deleteOne({ + projectId: normalizeObjectId(projectId), + }) +} + +export default { + findOne, + findByProjectId, + updateByProjectId, + saveByProjectId, +} diff --git a/services/github-sync/app/js/modals/githubSyncUserCredentials.js b/services/github-sync/app/js/modals/githubSyncUserCredentials.js new file mode 100644 index 0000000000..1518c7f967 --- /dev/null +++ b/services/github-sync/app/js/modals/githubSyncUserCredentials.js @@ -0,0 +1,54 @@ +import { db, ObjectId } from '../mongodb.js' + +function normalizeObjectId(value) { + if (value instanceof ObjectId) { + return value + } + return new ObjectId(value) +} + +async function findOne(query = {}, options = {}) { + return await db.githubSyncUserCredentials.findOne(query, options) +} + +async function findByUserId(userId, options = {}) { + return await findOne({ userId: normalizeObjectId(userId) }, options) +} + +async function saveOrUpdateByUserId(userId, authTokenEncrypted) { + await db.githubSyncUserCredentials.updateOne( + { userId: normalizeObjectId(userId) }, + { + $set: { + auth_token_encrypted: authTokenEncrypted, + }, + $setOnInsert: { + }, + }, + { upsert: true } + ) + + return await findByUserId(userId) +} + +async function updateByUserId(userId, update = {}) { + await db.githubSyncUserCredentials.updateOne( + { userId: normalizeObjectId(userId) }, + { $set: { ...update } } + ) + return await findByUserId(userId) +} + +async function removeByUserId(userId) { + return await db.githubSyncUserCredentials.deleteMany({ + userId: normalizeObjectId(userId), + }) +} + +export default { + findOne, + findByUserId, + saveOrUpdateByUserId, + updateByUserId, + removeByUserId, +} diff --git a/services/github-sync/app/js/modals/index.js b/services/github-sync/app/js/modals/index.js new file mode 100644 index 0000000000..91211d6cd6 --- /dev/null +++ b/services/github-sync/app/js/modals/index.js @@ -0,0 +1,15 @@ +import GitHubSyncUserCredentials from './githubSyncUserCredentials.js' +import GitHubSyncProjectStates from './githubSyncProjectStates.js' +import GitHubSyncEntityVersions from './githubSyncEntityVersions.js' + +export { + GitHubSyncUserCredentials, + GitHubSyncProjectStates, + GitHubSyncEntityVersions, +} + +export default { + GitHubSyncUserCredentials, + GitHubSyncProjectStates, + GitHubSyncEntityVersions, +} diff --git a/services/github-sync/app/js/mongodb.js b/services/github-sync/app/js/mongodb.js new file mode 100644 index 0000000000..25d5e5dc25 --- /dev/null +++ b/services/github-sync/app/js/mongodb.js @@ -0,0 +1,26 @@ +// @ts-check + +import Metrics from '@overleaf/metrics' +import Settings from '@overleaf/settings' +import MongoUtils from '@overleaf/mongo-utils' +import { MongoClient } from 'mongodb' + +export { ObjectId } from 'mongodb' + +export const mongoClient = new MongoClient( + Settings.mongo.url, + Settings.mongo.options +) +const mongoDb = mongoClient.db() + +export const db = { + githubSyncEntityVersions: mongoDb.collection('githubSyncEntityVersions'), + githubSyncProjectStates: mongoDb.collection('githubSyncProjectStates'), + githubSyncUserCredentials: mongoDb.collection('githubSyncUserCredentials'), +} + +Metrics.mongodb.monitor(mongoClient) + +export async function cleanupTestDatabase() { + await MongoUtils.cleanupTestDatabase(mongoClient) +} diff --git a/services/github-sync/app/js/server.js b/services/github-sync/app/js/server.js new file mode 100644 index 0000000000..179a0c8282 --- /dev/null +++ b/services/github-sync/app/js/server.js @@ -0,0 +1,29 @@ +import express from 'express' +import GitHubSyncController from './GitHubSyncController.js' +import { projectConcurrencyMiddleware } from './GitHubSyncMiddleware.js' + +export function createServer() { + const app = express() + app.use(express.json()) + + app.get('/status', (_req, res) => { + res.status(200).json({ status: 'ok', service: 'github-sync' }) + }) + + app.get('/healthz', (_req, res) => { + res.sendStatus(204) + }) + + // Export a existing project to GitHub + app.post('/project/:Project_id/user/:user_id/export', + projectConcurrencyMiddleware, + GitHubSyncController.exportProjectToGithub, + ) + + app.post('/project/:Project_id/user/:user_id/merge', + projectConcurrencyMiddleware, + GitHubSyncController.mergeToGitHubAndPushback, + ) + + return { app, server: app } +} \ No newline at end of file diff --git a/services/github-sync/config/settings.defaults.cjs b/services/github-sync/config/settings.defaults.cjs new file mode 100644 index 0000000000..59e66492e5 --- /dev/null +++ b/services/github-sync/config/settings.defaults.cjs @@ -0,0 +1,30 @@ +const http = require('node:http') +const https = require('node:https') + +http.globalAgent.maxSockets = 300 +http.globalAgent.keepAlive = false +https.globalAgent.keepAlive = false + +module.exports = { + internal: { + githubSync: { + host: process.env.LISTEN_ADDRESS || '127.0.0.1', + port: 3022, + }, + }, + + mongo: { + url: + process.env.MONGO_CONNECTION_STRING || + `mongodb://${process.env.MONGO_HOST || '127.0.0.1'}/sharelatex`, + options: { + monitorCommands: true, + }, + }, + + apis: { + project_history: { + url: `http://${process.env.PROJECT_HISTORY_HOST || '127.0.0.1'}:3054`, + } + } +} \ No newline at end of file diff --git a/services/github-sync/package.json b/services/github-sync/package.json new file mode 100644 index 0000000000..91aa17ed4b --- /dev/null +++ b/services/github-sync/package.json @@ -0,0 +1,59 @@ +{ + "name": "@overleaf/github-sync", + "description": "GitHub synchronization microservice for Overleaf", + "private": true, + "main": "app.js", + "type": "module", + "scripts": { + "start": "node app.js", + "nodemon": "node --watch app.js", + "lint": "eslint --max-warnings 0 --format unix .", + "format": "prettier --list-different $PWD/'**/{*.*js,*.ts}'", + "format:fix": "prettier --write $PWD/'**/{*.*js,*.ts}'", + "lint:fix": "eslint --fix .", + "types:check": "tsc --noEmit" + }, + "dependencies": { + "@octokit/request": "^9.2.2", + "@overleaf/access-token-encryptor": "*", + "@overleaf/fetch-utils": "*", + "@overleaf/logger": "*", + "@overleaf/metrics": "*", + "@overleaf/mongo-utils": "*", + "@overleaf/o-error": "*", + "@overleaf/promise-utils": "*", + "@overleaf/settings": "*", + "async": "^3.2.5", + "base64-stream": "^0.1.2", + "body-parser": "1.20.4", + "bunyan": "^1.8.15", + "express": "4.22.1", + "https-proxy-agent": "^7.0.6", + "lodash": "^4.17.21", + "mongodb-legacy": "6.1.3", + "nock": "^13.5.6", + "octonode": "0.9.5", + "p-limit": "^2.2.0", + "randomstring": "^1.1.5", + "request": "2.88.2" + }, + "devDependencies": { + "@overleaf/migrations": "*", + "@overleaf/stream-utils": "*", + "@pollyjs/adapter-node-http": "^6.0.6", + "@pollyjs/core": "^6.0.6", + "@pollyjs/persister-fs": "^6.0.6", + "chai": "^4.3.6", + "chai-as-promised": "^7.1.1", + "mocha": "^11.1.0", + "mocha-junit-reporter": "^2.2.1", + "mocha-multi-reporters": "^1.5.1", + "sandboxed-module": "^2.0.4", + "sinon": "^9.2.4", + "sinon-chai": "^3.7.0", + "timekeeper": "2.2.0", + "typescript": "^5.0.4" + }, + "version": "1.0.0", + "license": "AGPL-3.0" +} \ No newline at end of file diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index 2473d03cb3..eb58fc21f0 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -303,6 +303,10 @@ module.exports = { // For legacy reasons, we need to populate the below objects. v1: {}, recurly: {}, + + github_sync: { + url: `http://${process.env.GITHUB_SYNC_HOST || '127.0.0.1'}:${process.env.GITHUB_SYNC_PORT || 3022}`, + }, }, // Defines which features are allowed in the @@ -1011,10 +1015,25 @@ module.exports = { mainEditorLayoutPanels: [], langFeedbackLinkingWidgets: [], labsExperiments: [], - integrationLinkingWidgets: [], + integrationLinkingWidgets: [ + Path.resolve( + __dirname, + '../modules/github-sync/frontend/js/components/github-sync-widget.tsx' + ), + ], referenceLinkingWidgets: [], - importProjectFromGithubModalWrapper: [], - importProjectFromGithubMenu: [], + importProjectFromGithubModalWrapper: [ + Path.resolve( + __dirname, + '../modules/github-sync/frontend/js/components/import-from-github-modal-wrapper.tsx' + ), + ], + importProjectFromGithubMenu: [ + Path.resolve( + __dirname, + '../modules/github-sync/frontend/js/components/import-from-github-menu.tsx' + ), + ], editorLeftMenuSync: [ Path.resolve( __dirname, @@ -1080,6 +1099,10 @@ module.exports = { __dirname, '../modules/git-bridge/frontend/js/components/git-bridge-integration-card.tsx' ), + Path.resolve( + __dirname, + '../modules/github-sync/frontend/js/components/github-integration-card.tsx' + ), ], referenceSearchSetting: [], errorLogsComponents: [], @@ -1105,6 +1128,7 @@ module.exports = { 'login-register', 'oauth2-server', 'git-bridge', + 'github-sync' ], viewIncludes: {}, @@ -1154,4 +1178,11 @@ module.exports.oauthProviders = { linkPath: '/oidc/login', }, }), +} + +module.exports.githubSync = { + enabled: process.env.GITHUB_SYNC_ENABLED === 'true', + clientID: process.env.GITHUB_SYNC_CLIENT_ID, + clientSecret: process.env.GITHUB_SYNC_CLIENT_SECRET, + callbackURL: process.env.GITHUB_SYNC_CALLBACK_URL, } \ No newline at end of file diff --git a/services/web/modules/github-sync/Readme.md b/services/web/modules/github-sync/Readme.md new file mode 100644 index 0000000000..0b8f5fb469 --- /dev/null +++ b/services/web/modules/github-sync/Readme.md @@ -0,0 +1,171 @@ +# GitHub Sync API Document + +### API End Point: + +#### 01. Link and Unlink Github Account (Finished in web/modules) +Appeared in user setting page, user can link or unlink github account here. +``` +GET `/github-sync/beginAuth` +``` +It will redirect to github auth and fetch token. + + +Appear in user setting page too, user can unlink github account here. It will revoke token and remove github account info in our database. +``` +POST /github-sync/unlink +{"_csrf": "xxxx" } +``` +Result: just refresh page after success. + + +#### 02. Import Project from Github (Finished in web/modules) + +In project list page, we can select import from github, it should shows all repos. +``` +GET: /user/github-sync/repos +{ + "repos": [ + { + "name": "testRepoName", + "full_name": "user/testRepoName" + }, + { + "name": "testRepoName2", + "full_name": "user/testRepoName2" + }, + ] +} +``` + +After select a repo, we can create a new project with that repo, it will import all files in that repo to the new project, and link that repo to the project. +``` +POST: `/project/new/github-sync` +{"projectName":"auto-overleaf","repo":"ayaka-notes/auto-overleaf"} +``` + +API return: +``` +{"project_id":"699b0d628a0bdc986b68f21a"} +``` + +#### 03. Publish new project to github +In a created project, we can export a project to github(create a new repo, and export current project to that repo). +``` +POST: /project/699b0ea46161d1787ce2329b/github-sync/export +{name: "internal-test", description: "internal-test", private: true, org: "ayaka-notes"} +``` + +Check if user has github sync feature, if user has, we will return github sync status. +- enabled: if user hase linked github account +- available: if paid user(we set to true currently) + +``` +GET: /user/github-sync/status # check if paid user +{available: true, enabled: true} +``` + + +#### 04. Sync between overleaf and github +When user open project github sync modal, we will fetch github sync status and show it in modal, user can choose to pull github's change or merge overleaf's change to github. + +There are no difference between pull and merge, the only difference is push need a `message` for commit, but pull not. + +Internally, we will export overleaf's changes since last sync point to github (export as a branch), and then we will try to merge the exported branch to default branch (main/master/etc). +- if there is no conflict, we will just merge it, delete the exported branch. +- if there is conflict, we will keep the exported branch and show the unmerged branch info in modal, user can choose to merge or not. + +When there are no conflict, we will just merge the change, delete the exported branch and update sync point. + +Once sync point is determined, we will fetch all changed files since last sync point, and then we will replace all of those files in overleaf with the content in github, and then we will update sync point to latest commit. + +Get github sync status, including if github sync enabled, merge status, repo info, unmerged branch info and owner info. + +Check project github sync status, including if github sync enabled, merge status, repo info, unmerged branch info and owner info. +``` +GET /project/699b0ea46161d1787ce2329b/github-sync/status +{ + + "enabled": true, + + + "merge_status": "success", + + "repo": "ayaka-notes/internal-test", + + "unmerged_branch": null, + + "owner_id": "698bf0400fb804ce63648e1a", + + "owner": { + "_id": "698bf0400fb804ce63648e1a", + "email": "xxxx@outlook.in", + "githubFeature": { + "available": true, + "enabled": true + } + } +} +``` + +Pull github's change (maybe some changes Since last sync point) +Authorization: shared user with read/write can commit changes. +``` +GET /project/699b0ea46161d1787ce2329b/github-sync/commits/unmerged +{diverged: false, commits: []} +{ + + "diverged": false, + + "commits": [ + { + "message": "Update main.tex", + "author": { + "name": "xxxx", + "email": "xxxx@xxx.xxx.cn", + "date": "2026-02-22T14:24:30Z" + }, + "sha": "94bf4029733794b9b68fb2692ec06d6a75b5c2b6" + } + ] +} +// or another example: +{ + "diverged": false, + "commits": [ + { + "message": "Update introduction section content", + "author": { + "name": "xxx", + "email": "xxx@xx.edu.cn", + "date": "2026-02-22T16:07:32Z" + }, + "sha": "6bf04e90903975d1b197b0626d9578dbc599176d" + }, + { + "message": "Update main.tex", + "author": { + "name": "xxx", + "email": "xxx@xx.edu.cn", + "date": "2026-02-22T16:07:40Z" + }, + "sha": "1d3f5b1811de8ba25e1407d8f08c0126660cafaf" + }, + { + "message": "Update main.tex", + "author": { + "name": "xxx", + "email": "xxx@xx.edu.cn", + "date": "2026-02-22T16:07:57Z" + }, + "sha": "34d4c2506fbff5cf4a66e4ca63687e64e290e097" + } + ] +} +``` + +User commit overleaf's change to github. +Authorization: shared user with read/write can commit changes. +``` +POST /project/699c54c33e4bb0e9c15e00c4/github-sync/merge +{message: "123123123"} +``` \ No newline at end of file diff --git a/services/web/modules/github-sync/app/models/githubSyncProjectStates.mjs b/services/web/modules/github-sync/app/models/githubSyncProjectStates.mjs new file mode 100644 index 0000000000..91d9890832 --- /dev/null +++ b/services/web/modules/github-sync/app/models/githubSyncProjectStates.mjs @@ -0,0 +1,33 @@ +import mongoose from "../../../../app/src/infrastructure/Mongoose.mjs" + +const { Schema } = mongoose +const { ObjectId } = Schema + +export const GitHubSyncProjectStatesSchema = new Schema( + { + // the project we sync to github + projectId: { type: ObjectId, ref: 'Project', required: true, unique: true }, + // the user who syncs the project to github + // may not be the project owner, but must have write access to the project + // he can connect the project to github, and do sync operation. + ownerId: { type: ObjectId, ref: 'User', required: true }, + // the repo we sync to, in format "owner/repoName" + repo: { type: String, required: true }, + // if last merge is success + merge_status: { type: String, enum: ['success', 'failure', 'pending'], default: 'pending' }, + // sync branch + default_branch: { type: String, default: null }, + // if merge_status is failure, this field will be the branch name we pushed to. + unmerged_branch: { type: String, default: null }, + // the sha of last commit we synced to github + last_sync_sha: { type: String, default: null }, + // the version in overleaf project when we do last sync. + last_sync_version: { type: Number, default: null }, + }, + { collection: 'githubSyncProjectStates', minimize: false } +) + +export const GitHubSyncProjectStates = mongoose.model( + 'GitHubSyncProjectStates', + GitHubSyncProjectStatesSchema, +) \ No newline at end of file diff --git a/services/web/modules/github-sync/app/models/githubSyncUserCredentials.mjs b/services/web/modules/github-sync/app/models/githubSyncUserCredentials.mjs new file mode 100644 index 0000000000..b143eea3e4 --- /dev/null +++ b/services/web/modules/github-sync/app/models/githubSyncUserCredentials.mjs @@ -0,0 +1,17 @@ +import mongoose from "../../../../app/src/infrastructure/Mongoose.mjs" + +const { Schema } = mongoose +const { ObjectId } = Schema + +export const GitHubSyncUserCredentialsSchema = new Schema( + { + userId: { type: ObjectId, ref: 'User', required: true, unique: true }, + auth_token_encrypted: { type: String, required: true }, + }, + { collection: 'githubSyncUserCredentials', minimize: false } +) + +export const GitHubSyncUserCredentials = mongoose.model( + 'GitHubSyncUserCredentials', + GitHubSyncUserCredentialsSchema, +) \ No newline at end of file diff --git a/services/web/modules/github-sync/app/src/GitHubApiClient.mjs b/services/web/modules/github-sync/app/src/GitHubApiClient.mjs new file mode 100644 index 0000000000..57809a7193 --- /dev/null +++ b/services/web/modules/github-sync/app/src/GitHubApiClient.mjs @@ -0,0 +1,298 @@ +import logger from '@overleaf/logger' +import fetch from 'node-fetch' +import Settings from '@overleaf/settings' +import { HttpsProxyAgent } from 'https-proxy-agent' + +const GITHUB_API_BASE = 'https://api.github.com' + +// For example: 'http://127.0.0.1:1080' +const proxyUrl = process.env.GITHUB_SYNC_PROXY_URL +const httpsAgent = proxyUrl ? new HttpsProxyAgent(proxyUrl) : undefined + + +/** + * Create headers for GitHub API requests + * @param {string} pat - Personal Access Token + * @returns {Object} + */ +function getHeaders(pat) { + return { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${pat}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'Overleaf-GitHub-Sync', + } +} + +/** + * Verify PAT and get user info + * @param {string} pat - Personal Access Token + * @returns {Promise<{login: string, id: number, name: string}>} + */ +async function verifyPat(pat) { + const response = await fetch(`${GITHUB_API_BASE}/user`, { + headers: getHeaders(pat), + agent: httpsAgent, + }) + + if (!response.ok) { + if (response.status === 401) { + throw new Error('Invalid GitHub Personal Access Token') + } + throw new Error(`GitHub API error: ${response.status}`) + } + + const user = await response.json() + return { + login: user.login, + id: user.id, + name: user.name, + } +} + +/** + * List 100 repositories for the authenticated user + */ +async function listRepos(pat, page = 1, perPage = 100) { + const params = new URLSearchParams({ + page: page.toString(), + per_page: perPage.toString(), + sort: 'updated', + direction: 'desc', + }) + + const response = await fetch( + `${GITHUB_API_BASE}/user/repos?${params.toString()}`, + { + headers: getHeaders(pat), + agent: httpsAgent, + } + ) + + if (!response.ok) { + throw new Error(`GitHub API error: ${response.status}`) + } + + let repos = await response.json() + + return repos.map(repo => ({ + name: repo.name, + fullName: repo.full_name, + })) +} + +/** + * List All repositories for the authenticated user + */ +async function listAllRepos(pat) { + let page = 1 + const perPage = 100 + let allRepos = [] + while (true) { + const repos = await listRepos(pat, page, perPage) + allRepos = allRepos.concat(repos) + if (repos.length < perPage) break + page++ + } + return allRepos +} + +/** + * Get repository info + * @param {string} pat - Personal Access Token + * @param {string} fullName - Full repository name (e.g. "owner/repo") + * @returns {Promise<{fullName: string, defaultBranch: string, latestCommitSha: string}>} + */ +async function getRepoInfo(pat, fullName) { + const response = await fetch(`${GITHUB_API_BASE}/repos/${fullName}`, { + headers: getHeaders(pat), + agent: httpsAgent, + }) + + if (!response.ok) { + if (response.status === 404) { + throw new Error('Repository not found or access denied') + } + throw new Error(`GitHub API error: ${response.status}`) + } + + const repo = await response.json() + const defaultBranch = repo.default_branch + + const branchResp = await fetch( + `${GITHUB_API_BASE}/repos/${fullName}/branches/${encodeURIComponent(defaultBranch)}`, + { headers: getHeaders(pat), agent: httpsAgent } + ) + + if (!branchResp.ok) { + throw new Error(`GitHub API error: ${branchResp.status}`) + } + + const branch = await branchResp.json() + + return { + fullName: repo.full_name, + defaultBranch: repo.default_branch, + latestCommitSha: branch.commit?.sha, + } +} + +async function listOrgs(pat) { + const response = await fetch(`${GITHUB_API_BASE}/user/orgs`, { + headers: getHeaders(pat), + agent: httpsAgent, + }) + + if (!response.ok) { + throw new Error(`GitHub API error: ${response.status}`) + } + + const orgs = await response.json() + return orgs.map(org => ({ + login: org.login, + })) +} + +async function listUser(pat) { + const response = await fetch(`${GITHUB_API_BASE}/user`, { + headers: getHeaders(pat), + agent: httpsAgent, + }) + + if (!response.ok) { + throw new Error(`GitHub API error: ${response.status}`) + } + + const user = await response.json() + + return { + login: user.login + } +} + +async function revokePat(token) { + const ULR = `${GITHUB_API_BASE}/applications/${Settings.githubSync.clientID}/token` + const clientId = Settings.githubSync.clientID + const clientSecret = Settings.githubSync.clientSecret + + if (!clientId || !clientSecret) { + logger.warn('GitHub client ID or secret not configured, skipping token revocation') + return + } + + const Authorization = `Basic ${Buffer.from(`${clientId}:${clientSecret}`).toString('base64')}` + const resp = await fetch(ULR, { + method: 'DELETE', + agent: httpsAgent, + headers: { + 'Accept': 'application/vnd.github+json', + 'Authorization': Authorization, + }, + body: JSON.stringify({ access_token: token }), + }) + + if (!resp.ok) { + logger.warn(`Failed to revoke GitHub token: ${resp.status} ${await resp.text()}`) + } +} + + + +// This function would exchange the OAuth code for an access token with GitHub +// For security, this should be done server-side and not exposed to the client +// The implementation would involve making a POST request to GitHub's token endpoint +// with the client ID, client secret, and the code received from the OAuth callback +async function exchangeCodeForPat(code) { + const resp = await fetch('https://github.com/login/oauth/access_token', { + method: 'POST', + agent: httpsAgent, + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + client_id: Settings.githubSync.clientID, + client_secret: Settings.githubSync.clientSecret, + code, + redirect_uri: Settings.githubSync.callbackURL, + }), + }) + + const data = await resp.json() + if (!resp.ok || data.error) { + throw new Error( + `GitHub token exchange failed: ${data.error || resp.status} ${data.error_description || ''}`.trim() + ) + } + + return data +} + +async function listCommitsSince(pat, fullName, branch, sinceCommitSha) { + logger.info({ fullName, branch, sinceCommitSha }, 'Listing commits since last sync') + + if (!sinceCommitSha) { + return [] + } + + const url = `${GITHUB_API_BASE}/repos/${fullName}/compare/` + + `${encodeURIComponent(sinceCommitSha)}...${encodeURIComponent(branch)}` + + const response = await fetch(url, { + headers: getHeaders(pat), + agent: httpsAgent, + }) + + if (!response.ok) { + if (response.status === 404) { + throw new Error('Repository not found or access denied') + } + throw new Error(`GitHub API error: ${response.status}`) + } + + const data = await response.json() + + const commits = (data.commits || []).map(c => ({ + message: c.commit?.message || '', + author: { + name: c.commit?.author?.name || '', + email: c.commit?.author?.email || '', + date: c.commit?.author?.date || '', + }, + sha: c.sha, + })) + + return commits +} + +async function getRepoZipball(pat, repoFullName, latestCommitSha) { + const url = `${GITHUB_API_BASE}/repos/${repoFullName}/zipball/${latestCommitSha}` + const headers = { + Authorization: `token ${pat}`, + Accept: 'application/vnd.github.v3+json', + } + + const response = await fetch(url, { + headers, + agent: httpsAgent, + }) + + if (!response.ok) { + throw new Error(`GitHub API error: ${response.status} - ${response.statusText}`) + } + + return response +} + +export default { + exchangeCodeForPat, + verifyPat, + revokePat, + listRepos, + listAllRepos, + listOrgs, + listUser, + getRepoInfo, + getRepoZipball, + listCommitsSince, +} \ No newline at end of file diff --git a/services/web/modules/github-sync/app/src/GitHubSyncController.mjs b/services/web/modules/github-sync/app/src/GitHubSyncController.mjs new file mode 100644 index 0000000000..e20f3dfeae --- /dev/null +++ b/services/web/modules/github-sync/app/src/GitHubSyncController.mjs @@ -0,0 +1,343 @@ +import { expressify } from '@overleaf/promise-utils' +import SessionManager from '../../../../app/src/Features/Authentication/SessionManager.mjs' +import Csrf from '../../../../app/src/infrastructure/Csrf.mjs' +import GitHubSyncHandler from './GitHubSyncHandler.mjs' +import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' +import Path from 'path' +import fs from 'fs' +import crypto from 'crypto' +import ProjectUploadManager from '../../../../app/src/Features/Uploads/ProjectUploadManager.mjs' +import ProjectGetter from '../../../../app/src/Features/Project/ProjectGetter.mjs' +import { fetchJson } from '@overleaf/fetch-utils' +import UserGetter from '../../../../app/src/Features/User/UserGetter.mjs' + + +/** + * Get user's GitHub connection status + */ +async function getStatus(req, res) { + const userId = SessionManager.getLoggedInUserId(req.session) + + const status = await GitHubSyncHandler.promises.getUserGitHubStatus(userId) + if (!status) { + return res.json({ enabled: false }) + } + res.json(status) +} + +/** + * List user's GitHub repositories + */ +async function listRepos(req, res) { + const userId = SessionManager.getLoggedInUserId(req.session) + + try { + const repos = await GitHubSyncHandler.promises.listUserRepos(userId) + res.json({ repos }) + } catch (error) { + res.status(400).json({ error: error.message }) + } +} + + + + +/** + * Get project's GitHub sync status + */ +async function getProjectStatus(req, res) { + const { Project_id: projectId } = req.params + + const status = await GitHubSyncHandler.promises.getProjectGitHubSyncStatus(projectId) + + if (status && status.enabled) { + const ownerId = status.ownerId + const owner = await UserGetter.promises.getUser(ownerId, { + _id: 1, + email: 1, + }) + if (owner) { + status.owner = owner + } + + // check if owner's GitHub credentials are still valid. If not, return enabled: false to trigger re-auth flow in frontend + const credentials = await GitHubSyncHandler.promises.getGitHubAccessTokenForUser(ownerId) + if (!credentials) { + status.enabled = false + return res.json({ + enabled: false + } + ) + } + + // remove status. last_sync_sha and .last_sync_version + if (status.last_sync_sha) + delete status.last_sync_sha + if (status.last_sync_version) + delete status.last_sync_version + } + res.json(status) +} + + + + +/** + * Import a GitHub repository as a new project + */ +async function importRepo(req, res) { + const userId = SessionManager.getLoggedInUserId(req.session) + const { projectName, repo } = req.body + + try { + // Get the latest sha1, branch name of a repo + const { defaultBranch, latestCommitSha } = await GitHubSyncHandler.promises.getRepoInfo(userId, repo) + + // Then download the zipball from GitHub and create a new project with that zipball + const response = await GitHubSyncHandler.promises.getRepoZipball(userId, repo, latestCommitSha) + + const fsPath = Path.join( + Settings.path.dumpFolder, + `github_import_${crypto.randomUUID()}` + ) + + const ab = await response.arrayBuffer() + fs.writeFileSync(fsPath, Buffer.from(ab)) + + // Upload zip to create a new project + const { project } = await ProjectUploadManager.promises.createProjectFromZipArchiveWithName( + userId, + projectName, + fsPath, + {} + ) + const projectId = project._id.toString() + + // Clean up temp file + fs.unlinkSync(fsPath) + + // Re get projectID and version + // We need get from project history, because that's more accurate. + const snapshot = await fetchJson( + `${Settings.apis.project_history.url}/project/${projectId}/version` + ) + const projectVersion = snapshot.version + await GitHubSyncHandler.promises.saveNewlySyncedProjectState( + project._id, + userId, + repo, + latestCommitSha, + defaultBranch, + projectVersion + ) + + res.json({ projectId: project._id}) + } catch (error) { + logger.error({ err: error instanceof Error ? error : new Error(String(error)) }, 'Error importing GitHub repository') + res.status(400).json({ error: error instanceof Error ? error.message : 'Unknown error' }) + } +} + + + +/** + * Redirect user to GitHub OAuth authorization URL + * to begin linking process + */ +async function beginAuth(req, res) { + // build GitHub OAuth URL with required query parameters + let authUrl = new URL('https://github.com/login/oauth/authorize') + authUrl.searchParams.append('client_id', Settings.githubSync.clientID) + authUrl.searchParams.append('redirect_uri', Settings.githubSync.callbackURL) + authUrl.searchParams.append('scope', 'read:org,repo,workflow') + let state = req.csrfToken() + authUrl.searchParams.append('state', state) + + res.redirect(authUrl.toString()) +} + + +/** + * Handle GitHub OAuth callback and complete registration + * 1. Validate CSRF token + * 2. Exchange code for access token + * 3. Save access token for user + * 4. Redirect to user settings with success message + */ +async function completeRegistration(req, res) { + const userId = SessionManager.getLoggedInUserId(req.session) + const { code, state } = req.query + try { + await Csrf.promises.validateToken(state, req.session) + } catch (error) { + return res.status(403).json({ error: 'Invalid CSRF token' }) + } + + // fetch access token from GitHub using the code + let data + try { + data = await GitHubSyncHandler.promises.exchangeCodeForToken(code) + } catch (error) { + return res.status(400).json({ error: error.message }) + } + + if (!data.access_token) { + return res.status(400).json({ error: 'Failed to obtain access token from GitHub' }) + } + + await GitHubSyncHandler.promises.saveGitHubAccessTokenForUser(userId, data.access_token) + + // Save success message in session to display on redirect + req.session.projectSyncSuccessMessage = req.i18n.translate('github_successfully_linked_description') + // redirect to /user/settings + res.redirect('/user/settings?oauth-complete=github#project-sync') +} + + +/** + * Disconnect user's GitHub account + */ +async function unlink(req, res) { + const userId = SessionManager.getLoggedInUserId(req.session) + await GitHubSyncHandler.promises.removeGitHubAccessTokenForUser(userId) + res.json({ success: true }) +} + +/** + * Export changes to Github. + * This will create a new repository on GitHub, and link the project to that repository. + * Since this operation is invoked by loggenin user, we will use this to sync. + */ +async function exportProject(req, res){ + const userId = SessionManager.getLoggedInUserId(req.session) + const { Project_id: projectId } = req.params + const { name, description, private: isPrivate, org } = req.body + + logger.debug({ userId, projectId, name, isPrivate, org }, 'Received request to export project to GitHub') + if (!name || isPrivate === undefined || !projectId) { + return res.status(400).json({ error: 'Name, private and projectId are required' }) + } + + + try { + const repoResult = await GitHubSyncHandler.promises.exportProjectToGitHub( + userId, + projectId, + name, + description, + isPrivate, + org + ) + res.json(repoResult) + } catch (error) { + logger.error({ err: error instanceof Error ? error : new Error(String(error)) }, 'Error exporting project to GitHub') + res.status(500).json({ error: error instanceof Error ? error.message : 'Unknown error' }) + } +} + +/** + * Get unmerged commits from GitHub and show in Overleaf, + * so user can choose to merge or not. + * Since this operation is invoked by any editor of the project, we will + * use githubSyncStatus's owner to get + */ +async function getUnmergedCommits(req, res){ + const { Project_id: projectId } = req.params + const projectStatus = await GitHubSyncHandler.promises.getProjectGitHubSyncStatus(projectId) + + if (!projectStatus?.enabled) { + return res.status(400).json({ error: 'Project is not linked to a GitHub repository' }) + } + + const ownerId = projectStatus.ownerId + const lastSyncSha = projectStatus.last_sync_sha + const repo = projectStatus.repo + const defaultBranch = projectStatus.default_branch + + if (!ownerId || !repo || !defaultBranch) { + return res.status(400).json({ error: 'Project GitHub sync state is invalid' }) + } + const credentials = await GitHubSyncHandler.promises.getGitHubAccessTokenForUser(ownerId) + if (!credentials) { + return res.status(400).json({ error: 'GitHub credentials not found for project owner' }) + } + + try { + logger.debug({ lastSyncSha }, 'Getting commits since last sync') + let commits = await GitHubSyncHandler.promises.listCommitsSince( + ownerId, repo, defaultBranch, lastSyncSha + ) + res.json({ + diverged: projectStatus.merge_status === 'failure', + commits: commits + }) + } catch (error) { + logger.error({ err: error instanceof Error ? error : new Error(String(error)) }, 'Error listing commits since last sync') + res.status(500).json({ error: error instanceof Error ? error.message : 'Unknown error' }) + } +} + +async function mergeFromGitHub(req, res){ + const projectId = req.params.Project_id + if (!projectId) { + return res.status(400).json({ error: 'Project ID is required' }) + } + + let message = 'Merge changes from overleaf' + if (req.body && req.body.message) { + message = req.body.message + } + + const projectStatus = await GitHubSyncHandler.promises.getProjectGitHubSyncStatus(projectId) + if (!projectStatus?.enabled) { + return res.status(400).json({ error: 'Project is not linked to a GitHub repository' }) + } + const { ownerId, repo, default_branch: defaultBranch } = projectStatus + if (!ownerId || !repo || !defaultBranch) { + return res.status(400).json({ error: 'Project GitHub sync state is invalid' }) + } + + try { + const result = await GitHubSyncHandler.promises.syncProjectToGitHub( + ownerId, projectId, message + ) + logger.debug({ projectId, result }, 'Merge from GitHub result') + + if (result.newSha && result.files) { + const credentials = await GitHubSyncHandler.promises.getGitHubAccessTokenForUser(ownerId) + if (!credentials) { + throw new Error('GitHub credentials not found for project owner') + } + + await GitHubSyncHandler.promises.applyChangesToOverleaf( + projectId, result.newSha, result.files, ownerId) + } + + res.status(200).json({ success: true }) + } catch (error) { + logger.error({ err: error instanceof Error ? error : new Error(String(error)) }, 'Error syncing project to GitHub') + res.status(500).json({ error: error instanceof Error ? error.message : 'Unknown error' }) + } +} + +// List user and user's orgs. +async function listOrgs(req, res) { + const userId = SessionManager.getLoggedInUserId(req.session) + const result = await GitHubSyncHandler.promises.getGitHubOrgsForUser(userId) + res.json(result) +} + +export default { + getStatus: expressify(getStatus), + beginAuth: expressify(beginAuth), + unlink: expressify(unlink), + listOrgs: expressify(listOrgs), + completeRegistration: expressify(completeRegistration), + listRepos: expressify(listRepos), + getProjectStatus: expressify(getProjectStatus), + importRepo: expressify(importRepo), + exportProject: expressify(exportProject), + getUnmergedCommits: expressify(getUnmergedCommits), + mergeFromGitHub: expressify(mergeFromGitHub), +} \ No newline at end of file diff --git a/services/web/modules/github-sync/app/src/GitHubSyncHandler.mjs b/services/web/modules/github-sync/app/src/GitHubSyncHandler.mjs new file mode 100644 index 0000000000..b70cead5b0 --- /dev/null +++ b/services/web/modules/github-sync/app/src/GitHubSyncHandler.mjs @@ -0,0 +1,308 @@ +import { Project } from '../../../../app/src/models/Project.mjs' +import GitHubApiClient from './GitHubApiClient.mjs' +import { GitHubSyncUserCredentials } from '../models/githubSyncUserCredentials.mjs' +import { GitHubSyncProjectStates } from '../models/githubSyncProjectStates.mjs' +import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' +import SecretsHelper from './SecretsHelper.mjs' +import GitHubSyncUpdater from './GitHubSyncUpdater.mjs' +import { fetchJson } from '@overleaf/fetch-utils' + +/** + * Get user's GitHub sync status + */ +async function getUserGitHubStatus(userId) { + const credentials = await GitHubSyncUserCredentials.findOne({ userId }).lean() + if (!credentials) { + return { available: true, enabled: false } + } + + // test if the token is still valid by making an API call to GitHub + const token = await SecretsHelper.decryptAccessToken(credentials.auth_token_encrypted) + try { + await GitHubApiClient.listRepos(token, 1, 1) // just list 1 repo to check token validity + } catch (err) { + logger.warn({ userId, err }, 'GitHub token invalid, treating as not connected') + return { available: true, enabled: false } + } + + return { + available: true, + enabled: true + } +} + +/** + * Get project's GitHub sync status + */ +async function getProjectGitHubSyncStatus(projectId) { + const projectStatus = await GitHubSyncProjectStates.findOne({ projectId }, + { + _id: 0, __v: 0, + // last_sync_sha: 0, + // last_sync_version: 0, + } + ).lean() + if (!projectStatus) { + return { enabled: false } + } + projectStatus.enabled = true + return projectStatus +} + +/** + * Save project's GitHub sync status + */ +async function updateProjectGitHubSyncStatus(projectId, updateFields) { + logger.debug({ projectId, updateFields }, 'Updating project GitHub sync status') + const projectStatus = await GitHubSyncProjectStates.findOneAndUpdate( + { projectId }, + { $set: updateFields }, + { new: true, upsert: false, fields: { _id: 0, __v: 0 } } + ).lean() + + if (!projectStatus) { + throw new Error('Project GitHub sync status not found') + } + return projectStatus +} + +/** + * Delete project's GitHub sync status + */ +async function deleteProjectGitHubSyncStatus(projectId) { + await GitHubSyncProjectStates.deleteOne({ projectId }) +} + +/** + * List user's GitHub repositories + * @param {string} userId - User ID + * @returns {Promise} + */ +async function listUserRepos(userId) { + const pat = await getGitHubAccessTokenForUser(userId) + if (!pat) { + throw new Error('GitHub not connected') + } + + return await GitHubApiClient.listAllRepos(pat) +} + + +/** + * Get project's GitHub sync status, directly from db. + */ +async function getProjectSyncStatus(projectId) { + const projectStatus = await GitHubSyncProjectStates.findOne({ projectId }, { _id: 0, __v: 0 }).lean() + if (!projectStatus) { + return { enabled: false } + } + return projectStatus +} + + +// This function would exchange the OAuth code for an access token with GitHub +// For security, this should be done server-side and not exposed to the client +// The implementation would involve making a POST request to GitHub's token endpoint +// with the client ID, client secret, and the code received from the OAuth callback +async function exchangeCodeForToken(code) { + return await GitHubApiClient.exchangeCodeForPat(code) +} + +// Save the GitHub access token for a user, encrypted in the database +async function saveGitHubAccessTokenForUser(userId, accessToken) { + const tokenEncrypted = await SecretsHelper.encryptAccessToken(accessToken) + // save tp database + await GitHubSyncUserCredentials.findOneAndUpdate( + { userId }, + { + $set: { auth_token_encrypted: tokenEncrypted }, + $setOnInsert: { userId }, + }, + { + upsert: true, + new: true, + setDefaultsOnInsert: true, + } + ) +} + +// Save githubSyncProjectStates for a project +async function saveNewlySyncedProjectState(projectId, ownerId, repo, sha, branch, ver) { + let gitHubSyncProjectStates = new GitHubSyncProjectStates() + gitHubSyncProjectStates.projectId = projectId + gitHubSyncProjectStates.ownerId = ownerId + gitHubSyncProjectStates.repo = repo + gitHubSyncProjectStates.merge_status = 'success' + gitHubSyncProjectStates.last_sync_sha = sha + gitHubSyncProjectStates.default_branch = branch + gitHubSyncProjectStates.last_sync_sha = sha + gitHubSyncProjectStates.last_sync_version = ver + await gitHubSyncProjectStates.save() +} + + + +/** + * Remove a user's GitHub access token from the database. + * Revokes the token with GitHub before deleting it locally.(try) + * @param {string} userId - User ID + */ +async function removeGitHubAccessTokenForUser(userId) { + let token = await getGitHubAccessTokenForUser(userId) + if (token) { + await GitHubApiClient.revokePat(token) + } + await GitHubSyncUserCredentials.deleteMany({ userId }) +} + +/** + * Get a user's GitHub token + * @param {string} userId - User ID + */ +async function getGitHubAccessTokenForUser(userId) { + const credentials = await GitHubSyncUserCredentials.findOne({ userId }).lean() + if (!credentials) { + return null + } + return await SecretsHelper.decryptAccessToken(credentials.auth_token_encrypted) +} + +/** + * Get a repo's basic info + * @param {string} userId - User ID + */ +async function getRepoInfo(userId, repoFullName) { + const pat = await getGitHubAccessTokenForUser(userId) + if (!pat) { + throw new Error('GitHub not connected') + } + + return await GitHubApiClient.getRepoInfo(pat, repoFullName) +} + +async function getGitHubOrgsForUser(userId) { + const pat = await getGitHubAccessTokenForUser(userId) + if (!pat) { + throw new Error('GitHub not connected') + } + + const orgs = await GitHubApiClient.listOrgs(pat) + const user = await GitHubApiClient.listUser(pat) + return { user: user, orgs: orgs } +} + +async function exportProjectToGitHub(userId, projectId, name, description, isPrivate, org) { + const url = `${Settings.apis.github_sync.url}/project/${projectId}/user/${userId}/export` + + logger.debug({ userId, projectId, url }, 'Exporting project to GitHub') + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ name, description, private: isPrivate, org }), + }) + + if (!response.ok) { + const error = await response.json() + throw new Error(`GitHub Sync Service error: ${response.status} - ${error.error || error}`) + } + + return await response.json() +} + + +async function listCommitsSince(userId, repoFullName, branch, since) { + const pat = await getGitHubAccessTokenForUser(userId) + if (!pat) { + throw new Error('GitHub not connected') + } + return await GitHubApiClient.listCommitsSince(pat, repoFullName, branch, since) +} + +async function syncProjectToGitHub(userId, projectId, message) { + const url = `${Settings.apis.github_sync.url}/project/${projectId}/user/${userId}/merge` + logger.debug({ userId, projectId, url }, 'Syncing project to GitHub') + + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ message }), + }) + + if (!response.ok) { + const error = await response.json() + throw new Error(`GitHub Sync Service error: ${response.status} - ${error.error || error}`) + } + + return await response.json() +} + + +// 1. apply changes +// 2. update project sync status +// 3. return ok, client will fetch status again. +async function applyChangesToOverleaf(projectId, newSha, files, userId) { + const token = await getGitHubAccessTokenForUser(userId) + + if (!token) { + throw new Error('GitHub not connected') + } + + try { + await GitHubSyncUpdater.promises.postSnapshot(projectId, files, userId, token) + // Re get projectID and version + const snapshot = await fetchJson( + `${Settings.apis.project_history.url}/project/${projectId}/version` + ) + const projectVersion = snapshot.version + + + // get latest version in overleaf + await updateProjectGitHubSyncStatus(projectId, + { + merge_status: 'success', + last_sync_version: projectVersion, + last_sync_sha: newSha, + unmerged_branch: null, // clear unmerged branch if any + }) + } catch (err) { + logger.error({ err, projectId, newSha }, 'Failed to apply changes to Overleaf') + throw err + } + +} + +async function getRepoZipball(userId, repoFullName, latestCommitSha) { + const pat = await getGitHubAccessTokenForUser(userId) + if (!pat) { + throw new Error('GitHub not connected') + } + + return await GitHubApiClient.getRepoZipball(pat, repoFullName, latestCommitSha) +} + +export default { + promises: { + getUserGitHubStatus, + getProjectGitHubSyncStatus, + listUserRepos, + getProjectSyncStatus, + exchangeCodeForToken, + saveGitHubAccessTokenForUser, + removeGitHubAccessTokenForUser, + getGitHubAccessTokenForUser, + getRepoInfo, + getRepoZipball, + saveNewlySyncedProjectState, + getGitHubOrgsForUser, + exportProjectToGitHub, + listCommitsSince, + syncProjectToGitHub, + applyChangesToOverleaf, + deleteProjectGitHubSyncStatus, + }, +} \ No newline at end of file diff --git a/services/web/modules/github-sync/app/src/GitHubSyncRouter.mjs b/services/web/modules/github-sync/app/src/GitHubSyncRouter.mjs new file mode 100644 index 0000000000..090a4f0a4e --- /dev/null +++ b/services/web/modules/github-sync/app/src/GitHubSyncRouter.mjs @@ -0,0 +1,94 @@ +import logger from '@overleaf/logger' + +import GitHubSyncController from './GitHubSyncController.mjs' +import AuthenticationController from '../../../../app/src/Features/Authentication/AuthenticationController.mjs' +import AuthorizationMiddleware from '../../../../app/src/Features/Authorization/AuthorizationMiddleware.mjs' + +export default { + apply(webRouter) { + logger.debug({}, 'Init github-sync router') + // Check user's GitHub Auth Status + webRouter.get( + '/user/github-sync/status', + AuthenticationController.requireLogin(), + GitHubSyncController.getStatus + ) + + // OAuth redirect endpoint for GitHub Auth flow + webRouter.get( + '/github-sync/beginAuth', + AuthenticationController.requireLogin(), + GitHubSyncController.beginAuth + ) + + // Get user's Github org + webRouter.get( + '/user/github-sync/orgs', + AuthenticationController.requireLogin(), + GitHubSyncController.listOrgs + ) + + + // OAuth callback for GitHub registration flow + webRouter.get( + '/github-sync/completeRegistration', + AuthenticationController.requireLogin(), + GitHubSyncController.completeRegistration + ) + + // Unlink GitHub account + webRouter.post( + '/github-sync/unlink', + AuthenticationController.requireLogin(), + GitHubSyncController.unlink + ) + + + // Repository listing (import github project) + webRouter.get( + '/user/github-sync/repos', + AuthenticationController.requireLogin(), + GitHubSyncController.listRepos + ) + + // Need to be owner of that project to configure GitHub Sync + // Export a existing project to GitHub + webRouter.post( + '/project/:Project_id/github-sync/export', + AuthenticationController.requireLogin(), + AuthorizationMiddleware.ensureUserCanWriteProjectContent, + GitHubSyncController.exportProject + ) + + + webRouter.get( + '/project/:Project_id/github-sync/status', + AuthenticationController.requireLogin(), + AuthorizationMiddleware.ensureUserCanReadProject, + GitHubSyncController.getProjectStatus + ) + + webRouter.get( + '/project/:Project_id/github-sync/commits/unmerged', + AuthenticationController.requireLogin(), + AuthorizationMiddleware.ensureUserCanWriteProjectContent, + GitHubSyncController.getUnmergedCommits + ) + + + + // + webRouter.post( + '/project/:Project_id/github-sync/merge', + AuthenticationController.requireLogin(), + AuthorizationMiddleware.ensureUserCanWriteProjectContent, + GitHubSyncController.mergeFromGitHub + ) + + webRouter.post( + '/project/new/github-sync', + AuthenticationController.requireLogin(), + GitHubSyncController.importRepo + ) + }, +} \ No newline at end of file diff --git a/services/web/modules/github-sync/app/src/GitHubSyncUpdater.mjs b/services/web/modules/github-sync/app/src/GitHubSyncUpdater.mjs new file mode 100644 index 0000000000..75b8725cbe --- /dev/null +++ b/services/web/modules/github-sync/app/src/GitHubSyncUpdater.mjs @@ -0,0 +1,243 @@ +import logger from "@overleaf/logger"; +import settings from "@overleaf/settings"; +import ProjectEntityHandler from "../../../../app/src/Features/Project/ProjectEntityHandler.mjs"; +import ProjectGetter from "../../../../app/src/Features/Project/ProjectGetter.mjs"; +import Path from 'path' +import crypto from 'crypto' +import fs from 'fs' +import FileTypeManager from "../../../../app/src/Features/Uploads/FileTypeManager.mjs"; +import EditorController from "../../../../app/src/Features/Editor/EditorController.mjs"; +import { fetchJson } from '@overleaf/fetch-utils' +import { promises as fsPromises } from 'fs' +import { Snapshot } from 'overleaf-editor-core' +import { pipeline } from 'stream/promises' +import { fetchStream } from '@overleaf/fetch-utils' +import { HttpsProxyAgent } from 'https-proxy-agent' +import fetch from 'node-fetch' + +const proxyUrl = process.env.GITHUB_SYNC_PROXY_URL +const httpsAgent = proxyUrl ? new HttpsProxyAgent(proxyUrl) : undefined + + +/** + * Validate a file path + */ +function validateFilePath(path) { + // Check for invalid characters or patterns + // Git-bridge already handles most validation, but we do basic checks + + if (!path || path.length === 0) { + return { valid: false, state: 'error' } + } + + // Check for null bytes + if (path.includes('\0')) { + return { valid: false, state: 'error' } + } + + // Check for suspicious patterns + if (path.includes('..') || path.startsWith('/')) { + return { valid: false, state: 'error' } + } + + // Check for .git directory + if (path.startsWith('.git/') || path === '.git') { + return { valid: false, state: 'disallowed' } + } + + return { valid: true } +} + +async function postSnapshot(projectId, files, userId, token) { + const source = 'github' + const project = await ProjectGetter.promises.getProject(projectId, { + name: 1, + rootFolder: 1, + }) + + if (!project) { + throw new Error('Project not found') + } + + const { docs, files: existingFiles } = + await ProjectEntityHandler.promises.getAllEntities(projectId) + + logger.debug( + { projectId, numFiles: files.length, userId }, + 'Processing snapshot push' + ) + + const existingPaths = new Set() + // docs is editable docs + docs.forEach(doc => existingPaths.add(doc.path)) + // existingFiles is blob files + existingFiles.forEach(file => existingPaths.add(file.path)) + + // Track which paths are in the new snapshot + const newPaths = new Set(files.map(f => "/" + f.name)) + + // validate files first + const invalidFiles = [] + for (const file of files) { + const validation = validateFilePath(file.name) + if (!validation.valid) { + invalidFiles.push({ + file: file.name, + state: validation.state, + cleanFile: validation.cleanPath, + }) + } + } + logger.debug( + { invalidFiles }, 'File validation completed successfully' + ) + + if (invalidFiles.length > 0) { + logger.warn( + { projectId, invalidFiles }, 'Invalid file paths detected in snapshot' + ) + } + + const fsPath = Path.join( + settings.path.dumpFolder, + `${projectId}_${crypto.randomUUID()}` + ) + + for (const file of files) { + if (file.url) { + // File has been modified - download and update it + await downloadFile(file.url, fsPath, file.name, token) + } + } + + for (const file of files) { + const filePath = file.name + const elementPath = "/" + file.name + const localPath = Path.join(fsPath, file.name) + + if (file.url) { + const fileType = await determineFileType(projectId, filePath, localPath, docs, existingFiles) + + // File has been modified - update it + if (fileType === 'doc') { + const docLines = await readFileIntoTextArray(localPath) + + await EditorController.promises.upsertDocWithPath( + projectId, + elementPath, + docLines, + source, + userId + ) + } else { + await EditorController.promises.upsertFileWithPath( + projectId, elementPath, localPath, null, source, userId) + } + } + } + + // Now handle deletions - any existing path not in newPaths should be deleted + const pathsToDelete = [...existingPaths].filter(path => !newPaths.has(path)) + for (const path of pathsToDelete) { + try { + await EditorController.promises.deleteEntityWithPath( + projectId, + path, + source, + userId + ) + logger.debug({ projectId, path, source }, 'Deleted file from project') + } catch (err) { + logger.warn({ err, projectId, path, source }, 'Failed to delete file') + } + } + + // Clean up temp files + fs.rm(fsPath, { recursive: true, force: true }, (err) => { + if (err) { + logger.warn({ err, projectId, fsPath }, 'Failed to clean up temp files') + } else { + logger.debug({ projectId, fsPath }, 'Cleaned up temp files successfully') + } + }) +} + + +/** + * Read a file into an array of lines + */ +async function readFileIntoTextArray(fsPath) { + let content = await fsPromises.readFile(fsPath, 'utf8') + if (content === null || content === undefined) { + content = '' + } + const lines = content.split(/\r\n|\n|\r/) + return lines +} + +/** + * Determine if a file should be treated as a doc or binary file + * - path: relateive path to project + * - fsPath: the local file path we downloaded to + */ +async function determineFileType(projectId, path, fsPath, docs, files) { + // Check if there is an existing file with the same path + const existingDoc = docs.find(d => d.path === path) + const existingFile = files.find(f => f.path === path) + const existingFileType = existingDoc ? 'doc' : existingFile ? 'file' : null + + // Determine whether the update should create a doc or binary file + const { binary, encoding } = await FileTypeManager.promises.getType( + path, + fsPath, + existingFileType + ) + + // If we receive a non-utf8 encoding, treat as binary + const isBinary = binary || encoding !== 'utf-8' + + // If a binary file already exists, always keep it as a binary file + if (existingFileType === 'file') { + return 'file' + } else { + return isBinary ? 'file' : 'doc' + } +} + +// fsPath: dumpFolder for our temp download +async function downloadFile(url, fsPath, fileName, token) { + const headers = { + 'Authorization': `Bearer ${token}`, + 'Accept': 'application/vnd.github.v3.raw' + } + + const response = await fetch(url, { headers: headers, agent: httpsAgent }) + + if (!response.ok) { + throw new Error(`Failed to download file from ${url}: ${response.statusText}`) + } + + const filePath = Path.join(fsPath, fileName) + await fs.promises.mkdir(Path.dirname(filePath), { recursive: true }) + const writeStream = fs.createWriteStream(filePath) + + try { + const readStream = await fetchStream(url, { headers, agent: httpsAgent }) + await pipeline(readStream, writeStream) + return fsPath + } catch (err) { + // Clean up on error + try { + await fsPromises.unlink(filePath) + } catch (unlinkErr) { + logger.warn({ err: unlinkErr, fsPath }, 'Failed to delete file after download error') + } + throw err + } +} + +export default { + promises: { + postSnapshot, + } +} diff --git a/services/web/modules/github-sync/app/src/SecretsHelper.mjs b/services/web/modules/github-sync/app/src/SecretsHelper.mjs new file mode 100644 index 0000000000..689cbb6f34 --- /dev/null +++ b/services/web/modules/github-sync/app/src/SecretsHelper.mjs @@ -0,0 +1,35 @@ +import AccessTokenEncryptor from '@overleaf/access-token-encryptor' +import logger from '@overleaf/logger' + +const accessTokenEncryptor = new AccessTokenEncryptor({ + cipherPasswords: { + [process.env.CIPHER_LABEL || "2042.1-v3"]: process.env.CIPHER_PASSWORD, + }, + cipherLabel: process.env.CIPHER_LABEL || "2042.1-v3", +}) + +const SecretsHelper = { + async encryptAccessToken(accessToken) { + let tokenEncrypted = "" + try { + tokenEncrypted = await accessTokenEncryptor.promises.encryptJson(accessToken) + } catch (err) { + logger.error({ err }, 'Error encrypting GitHub access token') + return "" // Return empty string on encryption failure + } + return tokenEncrypted + }, + + async decryptAccessToken(tokenEncrypted) { + let tokenDecrypted = "" + try { + tokenDecrypted = await accessTokenEncryptor.promises.decryptToJson(tokenEncrypted) + } catch (err) { + logger.error({ err }, 'Error decrypting GitHub access token') + return "" // Return empty string on decryption failure + } + return tokenDecrypted + } +} + +export default SecretsHelper \ No newline at end of file diff --git a/services/web/modules/github-sync/frontend/js/components/github-integration-card.tsx b/services/web/modules/github-sync/frontend/js/components/github-integration-card.tsx new file mode 100644 index 0000000000..6584c4271f --- /dev/null +++ b/services/web/modules/github-sync/frontend/js/components/github-integration-card.tsx @@ -0,0 +1,776 @@ + +import { useTranslation } from 'react-i18next' +import { useState, useEffect } from 'react' +import GithubLogo from '@/shared/svgs/github-logo' +import { useProjectContext } from '@/shared/context/project-context' +import IntegrationCard from '@/features/ide-redesign/components/integrations-panel/integration-card' +import { + OLModalBody, + OLModalFooter, + OLModalHeader, + OLModalTitle, + OLModal, +} from '@/shared/components/ol/ol-modal' +import OLButton from '@/shared/components/ol/ol-button' +import OLForm from '@/shared/components/ol/ol-form' +import OLFormGroup from '@/shared/components/ol/ol-form-group' +import OLFormControl from '@/shared/components/ol/ol-form-control' +import OLFormLabel from '@/shared/components/ol/ol-form-label' +import OLFormCheckbox from '@/shared/components/ol/ol-form-checkbox' +import OLFormSelect from '@/shared/components/ol/ol-form-select' + +import OLRow from '@/shared/components/ol/ol-row' +import OLCol from '@/shared/components/ol/ol-col' +import { + getJSON, + postJSON +} from '../../../../../frontend/js/infrastructure/fetch-json' +import getMeta from '@/utils/meta' +import OLNotification from '@/shared/components/ol/ol-notification' +import { useEditorManagerContext } from '@/features/ide-react/context/editor-manager-context' +import { Trans } from 'react-i18next' + + +type GitHubSyncModalStatus = 'loading' | 'export' | 'merge' | 'pushSubmit' | 'syncing' | 'conflict' | 'need-auth' + +type GitHubSyncModalNeedAuthProps = { + handleHide: () => void +} + +const GitHubSyncModalNeedAuth = ({ handleHide }: GitHubSyncModalNeedAuthProps) => { + const { t } = useTranslation() + const { appName } = getMeta('ol-ExposedSettings') + return ( + <> + +

{t('link_to_github_description', { appName })}

+ + + + {t('close')} + + + { + window.open( + '/github-sync/beginAuth', + 'githubAuth', + 'width=600,height=700' + ) + }} + > + {t('link_to_github')} + + + + ) +} + +type GitHubSyncModalSyncingProps = { + handleHide: () => void + modalStatus: GitHubSyncModalStatus + setModalStatus: (modalStatus: GitHubSyncModalStatus) => void + commitMessage: string + projectId: string +} + +const GitHubSyncModalSyncing = ({ + handleHide, modalStatus, setModalStatus, + commitMessage, projectId +}: GitHubSyncModalSyncingProps) => { + const { t } = useTranslation() + const { setIgnoringExternalUpdates } = useEditorManagerContext() + + // launch syncing when this component is rendered + useEffect(() => { + let cancelled = false + + const syncProjectWithGitHub = async () => { + setIgnoringExternalUpdates(true) + try { + // call backend to start syncing process, endpoint: /project//github-sync/sync + await postJSON(`/project/${projectId}/github-sync/merge`, { + body: { + message: commitMessage, + }, + }) + // after syncing is done, we can set modal status to loading to fetch latest status and show merge table if needed. + if (!cancelled) + setModalStatus('loading') + } catch (err) { + console.error('Failed to sync project with GitHub', err) + // if error occurs, we can set modal status to merge to show pull/push table, user can decide how to resolve the conflict. + if (!cancelled) + setModalStatus('loading') + } finally { + setIgnoringExternalUpdates(false) + } + } + + if (modalStatus === 'syncing') syncProjectWithGitHub() + + return () => { + cancelled = true + setIgnoringExternalUpdates(false) + } + }, [modalStatus, projectId, commitMessage, setModalStatus, setIgnoringExternalUpdates]) + + + return ( + <> + +
+ + {t('importing_and_merging_changes_in_github')} +
+ + + + {t('cancel')} + + + + ) +} + +type GithubSyncModalLoadingProps = { + handleHide: () => void +} +const GithubSyncModalLoading = ({ handleHide }: GithubSyncModalLoadingProps) => { + const { t } = useTranslation() + return ( + <> + +
+ + {t('checking_project_github_status')} +
+ + + + {t('cancel')} + + + + ) +} + +type GithubSyncModalExportingProps = { + handleHide: () => void + handleSetModalState: (modalStatus: GitHubSyncModalStatus) => void +} +const GithubSyncModalExporting = ({ handleHide, handleSetModalState }: GithubSyncModalExportingProps) => { + const { t } = useTranslation() + const [orgs, setOrgs] = useState([]) + const [user, setUser] = useState('') + const [selectedOwner, setSelectedOwner] = useState('') + const [repoName, setRepoName] = useState('') + const [description, setDescription] = useState('') + const [visibility, setVisibility] = useState<'public' | 'private'>('private') + const [submitLoading, setSubmitLoading] = useState(false) + const [isSubmitError, setIsSubmitError] = useState(false) + const { project } = useProjectContext() + + useEffect(() => { + const fetchOrgs = async () => { + try { + const data = await getJSON('/user/github-sync/orgs') + if (data.user) { + setUser(data.user.login) + setSelectedOwner(data.user.login) + } + if (data.orgs) { + setOrgs(data.orgs.map((org: any) => org.login)) + } + } catch (err: any) { + console.error('Failed to fetch GitHub orgs', err) + } + } + + fetchOrgs() + }, []) + + const handlerSubmit = (e: React.FormEvent) => { + e.preventDefault() + setSubmitLoading(true) + // Interface with backend + // Endpoint: /project//github-sync/export + + // description: "repoDescription" + // name: "repoName" + // private: true + // org: "test-org" (Optional, if not set, use user's) + const exportRepo = async () => { + try { + await postJSON(`/project/${project?._id}/github-sync/export`, { + body: { + name: repoName, + description, + private: visibility === 'private', + org: selectedOwner === user ? undefined : selectedOwner, + }, + }) + // After successful export, we should set modal status to loading. + + setSubmitLoading(false) + handleSetModalState('loading') + } catch (err: any) { + console.error('Failed to export project to GitHub', err) + setSubmitLoading(false) + setIsSubmitError(true) + } + } + + exportRepo() + } + + return ( + <> + +

{t('export_project_to_github')}

+

{t('project_not_linked_to_github')}

+ { + isSubmitError && ( + + ) + } + + + + + + {t('owner')} + + ) => + setSelectedOwner(e.target.value) + } + > + + {orgs.map(org => ( + + ))} + + + + + + + + {t('repository_name')} + + ) => + setRepoName(e.target.value) + } + /> + + + + + + + + + {t('description')} ({t('optional')}) + + ) => + setDescription(e.target.value) + } + /> + + + + +
+ +
+ + {t('repository_visibility')} + + + + + + setVisibility('public')} + label={t('public', { defaultValue: 'Public' })} + description={t('github_public_description')} + /> + + + + + + + + setVisibility('private')} + label={t('private', { defaultValue: 'Private' })} + description={t('github_private_description')} + /> + + + +
+ + + + + {t('cancel')} + + + {t('create_project_in_github')} + + + + ) +} + + +type GithubSyncModalMergingProps = { + handleHide: () => void + setModalStatus: (modalStatus: GitHubSyncModalStatus) => void + projectSyncStatus: any + projectId: string + // other props to show conflict and allow user to resolve conflict +} + +const GithubSyncModalMerging = ({ + handleHide, setModalStatus, projectSyncStatus, projectId +}: GithubSyncModalMergingProps) => { + const { t } = useTranslation() + const { appName } = getMeta('ol-ExposedSettings') + const [unmergedCommits, setUnmergedCommits] = useState([]) + const [isLoadingCommits, setIsLoadingCommits] = useState(true) + + useEffect(() => { + const fetchUnmergedCommits = async () => { + try { + const data = await getJSON(`/project/${projectId}/github-sync/commits/unmerged`) + setUnmergedCommits(data.commits) + setIsLoadingCommits(false) + } catch (err) { + console.error('Failed to fetch unmerged commits', err) + setIsLoadingCommits(false) + } + } + fetchUnmergedCommits() + }, []) + + if (isLoadingCommits) { + return ( + <> + +
+ + {t('checking_project_github_status')} +
+ + + + {t('close')} + + + + ) + } + + + return ( + <> + +

{t('project_linked_to')}: + + {projectSyncStatus.repo} + +

+
+ + {unmergedCommits.length === 0 && +
+

{t('no_new_commits_in_github')}

+
+ } + + {unmergedCommits.length > 0 && + <> +

+ {t('recent_commits_in_github')} +

+
+ {unmergedCommits.map((commit: any) => ( +
+ + + {commit.sha.substring(0, 7)} + + + + {commit.message} + +
by {commit.author.name} <{commit.author.email}>
+
+ //

{commit.message}

+ ))} +
+ + + } +

+ setModalStatus('syncing')} + > + {t('pull_github_changes_into_sharelatex', { appName })} + +

+
+

+ setModalStatus('pushSubmit')} + > + {t('push_sharelatex_changes_to_github', { appName })} + +

+ + + + + + {t('close')} + + + + ) +} + + +type GitHubSyncModalPushSubmitProps = { + handleHide: () => void + // other props to show commit message input and submit button + commitMessage: string + setCommitMessage: (message: string) => void + setModalStatus: (modalStatus: GitHubSyncModalStatus) => void +} + +const GitHubSyncModalPushSubmit = ({ + handleHide, commitMessage, setCommitMessage, setModalStatus +}: GitHubSyncModalPushSubmitProps) => { + const { t } = useTranslation() + const { appName } = getMeta('ol-ExposedSettings') + + return ( + <> + + +

{t('sync_project_to_github_explanation', { appName })}

+ + setCommitMessage(e.target.value)} + /> + + + + + + {t('cancel')} + + { + setModalStatus('syncing') + }} + > + {t('sync')} + + + + ) +} + +type GitHubSyncModalConflictProps = { + // props to show conflict and allow user to resolve conflict + projectSyncStatus: any + handleHide: () => void + setModalStatus: (modalStatus: GitHubSyncModalStatus) => void +} + +const GitHubSyncModalConflict = ({ projectSyncStatus, handleHide, setModalStatus }: GitHubSyncModalConflictProps) => { + const { t } = useTranslation() + const { appName } = getMeta('ol-ExposedSettings') + + return ( + <> + + +

+ ]} + /> +

+ + + + {t('close')} + + setModalStatus('pushSubmit') + } + > + {t('continue_github_merge')} + + + + ) +} + +type GitHubSyncModalProps = { + show: boolean + handleHide: () => void + projectId: string + modalStatus: GitHubSyncModalStatus + setModalStatus: (modalStatus: GitHubSyncModalStatus) => void +} + +// 0. Check project github sync status +// Show spinner while loading, show error message if error occurs +// 1. If /project//github-sync/status +// returns {enabled: false} then show export Github table +// a) export Github table will check /user/github-sync/orgs +// b) once user submits export, spinner in button +// c) if export is successful, return to step 0, to reload status. +// 2. If /project//github-sync/status +// returns {enabled: true, merge_status: 'success'} +// then show pull/push table +// a) check /project//github-sync/commits/unmerged +// b) if there are unmerged commits, show pull button +// c) push button should always be shown +// 3. If /project//github-sync/status +// returns {enabled: true, merge_status: 'conflict'} +// then show conflict resolution contents. +// a) user can choose to merge confict in github, and submit +// remerge form overleaf. +function GitHubSyncModal({ + show, handleHide, projectId, modalStatus, setModalStatus +}: GitHubSyncModalProps) { + const { t } = useTranslation() + const { project } = useProjectContext() + + const [projectSyncStatus, setProjectSyncStatus] = useState(null) + + + // since we need to switch to syncing component when user clicks pull or push, + // we need to pass this var + const [commitMessage, setCommitMessage] = useState('') + + // If modalStatus is loading, we will fetch status + useEffect(() => { + if (!show || !project || modalStatus !== 'loading') { + return + } + + const fetchUserGitHubSyncStatus = async () => { + try { + const data = await getJSON('/user/github-sync/status') + if (data.enabled) { + // fetch project github sync status + const projectStatus = await getJSON(`/project/${projectId}/github-sync/status`) + if (projectStatus.enabled && projectStatus.merge_status === 'success') { + setModalStatus('merge') + setProjectSyncStatus(projectStatus) + } else if (projectStatus.enabled && projectStatus.merge_status === 'failure') { + setModalStatus('conflict') + setProjectSyncStatus(projectStatus) + } else { + setModalStatus('export') + } + } else { + setModalStatus('need-auth') + } + } catch (err: any) { + console.error('Failed to fetch GitHub sync status', err) + } + } + + // const fetchGitHubSyncStatus = async () => { + // try { + // const data = await getJSON(`/project/${projectId}/github-sync/status`) + // if (data.enabled && data.merge_status === 'success') { + // setModalStatus('merge') + // setProjectSyncStatus(data) + // } else if (data.enabled && data.merge_status === 'failure') { + // setModalStatus('conflict') + // setProjectSyncStatus(data) + // } + // else { + // setModalStatus('export') + // } + // } catch (err: any) { + // console.error('Failed to fetch GitHub sync status', err) + // } + // } + + fetchUserGitHubSyncStatus() + }, [show, modalStatus]) + + + + useEffect(() => { + if (!show) + setCommitMessage('') + }, [show]) + + return ( + + + {t('github_sync')} + + {modalStatus === 'loading' && } + {modalStatus === 'export' && } + {modalStatus === 'merge' && + } + {modalStatus === 'pushSubmit' && } + {modalStatus === 'syncing' && } + { + modalStatus === 'conflict' && + } + { + modalStatus === 'need-auth' && + } + + + ) +} + + + + +const GitHubSyncCard = () => { + const { t } = useTranslation() + + const [showGithubSyncModal, setShowGithubSyncModal] = useState(false) + const { project, tags: projectTags } = useProjectContext() + + // loading: checking github sync status + // export: show export table to link github repo + // merge: show remote changes from github and allow user to pull/push + // pushSubmit: allow user to fill submit message + // syncing: show syncing spinner while request is being processed + const [modalStatus, setModalStatus] = useState('loading') + + return ( + <> + } + showPaywallBadge={false} + onClick={() => setShowGithubSyncModal(true)} + > + + { + setShowGithubSyncModal(false) + setModalStatus('loading') + }} + projectId={project?._id || ''} + /> + + ) +} + +export default GitHubSyncCard diff --git a/services/web/modules/github-sync/frontend/js/components/github-sync-button.tsx b/services/web/modules/github-sync/frontend/js/components/github-sync-button.tsx new file mode 100644 index 0000000000..de6c46dd3a --- /dev/null +++ b/services/web/modules/github-sync/frontend/js/components/github-sync-button.tsx @@ -0,0 +1,391 @@ +import { useCallback, useState, useEffect } from 'react' +import { useTranslation } from 'react-i18next' +import { + postJSON, + deleteJSON, + getJSON, +} from '../../../../../frontend/js/infrastructure/fetch-json' +import LeftMenuButton from '../../../../../frontend/js/features/editor-left-menu/components/left-menu-button' +import { + OLModal, + OLModalBody, + OLModalFooter, + OLModalHeader, + OLModalTitle, +} from '@/shared/components/ol/ol-modal' +import OLButton from '@/shared/components/ol/ol-button' +import OLFormGroup from '@/shared/components/ol/ol-form-group' +import OLFormControl from '@/shared/components/ol/ol-form-control' +import OLFormLabel from '@/shared/components/ol/ol-form-label' +import OLNotification from '@/shared/components/ol/ol-notification' +import { useProjectContext } from '@/shared/context/project-context' +import getMeta from '@/utils/meta' + +type GitHubStatus = { + connected: boolean + username?: string +} + +type ProjectSyncStatus = { + configured: boolean + repoOwner?: string + repoName?: string + branch?: string + lastSyncedAt?: string +} + +type Repo = { + owner: string + name: string + fullName: string + private: boolean + defaultBranch: string +} + +type Branch = { + name: string + protected: boolean +} + +export default function GitHubSyncButton() { + const { t } = useTranslation() + const { _id: projectId } = useProjectContext() + const [showModal, setShowModal] = useState(false) + const [userStatus, setUserStatus] = useState({ connected: false }) + const [projectStatus, setProjectStatus] = useState({ + configured: false, + }) + const [repos, setRepos] = useState([]) + const [branches, setBranches] = useState([]) + const [selectedRepo, setSelectedRepo] = useState('') + const [selectedBranch, setSelectedBranch] = useState('') + const [loading, setLoading] = useState(false) + const [pushing, setPushing] = useState(false) + const [configuring, setConfiguring] = useState(false) + const [error, setError] = useState('') + const [success, setSuccess] = useState('') + + const anonymous = getMeta('ol-anonymous') + + const fetchData = useCallback(async () => { + if (anonymous) return + + setLoading(true) + setError('') + + try { + const [userStatusData, projectStatusData] = await Promise.all([ + getJSON('/user/github-sync/status'), + getJSON(`/project/${projectId}/github-sync/status`), + ]) + + setUserStatus(userStatusData) + setProjectStatus(projectStatusData) + + if (userStatusData.connected) { + const reposData = await getJSON<{ repos: Repo[] }>( + '/user/github-sync/repos' + ) + setRepos(reposData.repos) + + if (projectStatusData.configured) { + setSelectedRepo( + `${projectStatusData.repoOwner}/${projectStatusData.repoName}` + ) + setSelectedBranch(projectStatusData.branch || '') + + // Load branches for configured repo + const branchesData = await getJSON<{ branches: Branch[] }>( + `/user/github-sync/repos/${projectStatusData.repoOwner}/${projectStatusData.repoName}/branches` + ) + setBranches(branchesData.branches) + } + } + } catch (err: any) { + setError(err.message || 'Failed to fetch GitHub data') + } finally { + setLoading(false) + } + }, [projectId, anonymous]) + + useEffect(() => { + if (showModal) { + fetchData() + } + }, [showModal, fetchData]) + + const handleRepoChange = useCallback( + async (repoFullName: string) => { + setSelectedRepo(repoFullName) + setSelectedBranch('') + setBranches([]) + setError('') + + if (!repoFullName) return + + const [owner, repo] = repoFullName.split('/') + + try { + const branchesData = await getJSON<{ branches: Branch[] }>( + `/user/github-sync/repos/${owner}/${repo}/branches` + ) + setBranches(branchesData.branches) + + // Auto-select main or master branch if available + const mainBranch = branchesData.branches.find( + b => b.name === 'main' || b.name === 'master' + ) + if (mainBranch) { + setSelectedBranch(mainBranch.name) + } else if (branchesData.branches.length > 0) { + setSelectedBranch(branchesData.branches[0].name) + } + } catch (err: any) { + setError(err.message || 'Failed to fetch branches') + } + }, + [] + ) + + const handleConfigure = useCallback(async () => { + if (!selectedRepo || !selectedBranch) { + setError('Please select a repository and branch') + return + } + + setConfiguring(true) + setError('') + + const [owner, repo] = selectedRepo.split('/') + + try { + await postJSON(`/project/${projectId}/github-sync/configure`, { + body: { owner, repo, branch: selectedBranch }, + }) + setProjectStatus({ + configured: true, + repoOwner: owner, + repoName: repo, + branch: selectedBranch, + }) + setSuccess('GitHub sync configured successfully') + } catch (err: any) { + setError(err.message || 'Failed to configure GitHub sync') + } finally { + setConfiguring(false) + } + }, [projectId, selectedRepo, selectedBranch]) + + const handleUnconfigure = useCallback(async () => { + setConfiguring(true) + setError('') + + try { + await deleteJSON(`/project/${projectId}/github-sync/configure`) + setProjectStatus({ configured: false }) + setSelectedRepo('') + setSelectedBranch('') + setBranches([]) + setSuccess('GitHub sync removed') + } catch (err: any) { + setError(err.message || 'Failed to remove GitHub sync') + } finally { + setConfiguring(false) + } + }, [projectId]) + + const handlePush = useCallback(async () => { + setPushing(true) + setError('') + setSuccess('') + + try { + await postJSON(`/project/${projectId}/github-sync/push`) + setSuccess('Successfully pushed to GitHub!') + // Refresh status to get updated lastSyncedAt + const statusData = await getJSON( + `/project/${projectId}/github-sync/status` + ) + setProjectStatus(statusData) + } catch (err: any) { + setError(err.message || 'Failed to push to GitHub') + } finally { + setPushing(false) + } + }, [projectId]) + + if (anonymous) { + return null + } + + return ( + <> + setShowModal(true)} icon="cloud_upload"> + {t('github_sync', { defaultValue: 'GitHub' })} + + + setShowModal(false)} size="lg" backdrop="static" initialFocus={false}> + + + {t('github_sync_title', { defaultValue: 'GitHub Sync' })} + + + + {loading ? ( +

{t('loading')}...

+ ) : !userStatus.connected ? ( +
+

+ {t('github_not_connected', { + defaultValue: + 'Your GitHub account is not connected. Please connect it in your account settings.', + })} +

+ + {t('go_to_settings', { defaultValue: 'Go to Settings' })} + +
+ ) : ( +
+

+ {t('connected_as')}:{' '} + {userStatus.username} +

+ + {!projectStatus.configured ? ( + <> +
+ {t('configure_repository', { + defaultValue: 'Configure Repository', + })} +
+ + + {t('repository')} + + ) => + handleRepoChange(e.target.value) + } + > + + {repos.map(repo => ( + + ))} + + + + {branches.length > 0 && ( + + + {t('branch')} + + ) => + setSelectedBranch(e.target.value) + } + > + + {branches.map(branch => ( + + ))} + + + )} + + ) : ( +
+
+ {t('linked_repository', { + defaultValue: 'Linked Repository', + })} +
+

+ + {projectStatus.repoOwner}/{projectStatus.repoName} + {' '} + ({projectStatus.branch}) +

+ {projectStatus.lastSyncedAt && ( +

+ {t('last_synced', { defaultValue: 'Last synced' })}:{' '} + {new Date(projectStatus.lastSyncedAt).toLocaleString()} +

+ )} + +
+ )} + + {error && } + {success && } +
+ )} + + + setShowModal(false)}> + {t('close')} + + {userStatus.connected && ( + <> + {projectStatus.configured ? ( + <> + + {t('unlink_repository', { defaultValue: 'Unlink' })} + + + {pushing + ? t('pushing', { defaultValue: 'Pushing...' }) + : t('push_to_github', { defaultValue: 'Push to GitHub' })} + + + ) : ( + + {configuring + ? t('configuring', { defaultValue: 'Configuring...' }) + : t('link_repository', { defaultValue: 'Link Repository' })} + + )} + + )} + + + + ) +} \ No newline at end of file diff --git a/services/web/modules/github-sync/frontend/js/components/github-sync-widget.tsx b/services/web/modules/github-sync/frontend/js/components/github-sync-widget.tsx new file mode 100644 index 0000000000..19b32b47bf --- /dev/null +++ b/services/web/modules/github-sync/frontend/js/components/github-sync-widget.tsx @@ -0,0 +1,149 @@ +import { useCallback, useState, useEffect } from 'react' +import { useTranslation } from 'react-i18next' +import { + postJSON, + getJSON, +} from '../../../../../frontend/js/infrastructure/fetch-json' +import OLButton from '@/shared/components/ol/ol-button' +import { + OLModal, + OLModalBody, + OLModalFooter, + OLModalHeader, + OLModalTitle, +} from '@/shared/components/ol/ol-modal' +import OLNotification from '@/shared/components/ol/ol-notification' +import GithubLogo from '@/shared/svgs/github-logo' + +type GitHubStatus = { + available: boolean + enabled: boolean +} + +export default function GitHubSyncWidget() { + const { t } = useTranslation() + const [status, setStatus] = useState({ available: false, enabled: false }) + const [loading, setLoading] = useState(true) + const [error, setError] = useState('') + const [showDisconnectModal, setShowDisconnectModal] = useState(false) + const [disconnecting, setDisconnecting] = useState(false) + + const fetchStatus = useCallback(async () => { + try { + const data = await getJSON('/user/github-sync/status') + setStatus(data) + setError('') + } catch (err: any) { + setError(err.message || 'Failed to fetch GitHub status') + } finally { + setLoading(false) + } + }, []) + + useEffect(() => { + fetchStatus() + }, [fetchStatus]) + + const handleDisconnect = useCallback(async () => { + setDisconnecting(true) + setError('') + + try { + await postJSON('/github-sync/unlink') + setStatus({ available: true, enabled: false }) + setShowDisconnectModal(false) + } catch (err: any) { + setError(err.message || 'Failed to disconnect GitHub account') + } finally { + setDisconnecting(false) + } + }, []) + + if (loading) { + return ( +
+
+ +
+
+
+

GitHub

+
+

{t('loading')}...

+
+
+ ) + } + + return ( + <> +
+
+ +
+
+
+

GitHub

+
+

+ {t('github_sync_description', { + defaultValue: + 'Sync your Overleaf projects with GitHub repositories.', + })} +

+ {error && } +
+
+ {status.enabled ? ( + setShowDisconnectModal(true)} + disabled={disconnecting} + > + {disconnecting ? t('unlinking') : t('unlink')} + + ) : ( + + {t('link')} + + )} +
+
+ + + {/* Disconnect Confirmation Modal */} + setShowDisconnectModal(false)} + backdrop="static" initialFocus={false} enforceFocus={false} + > + + + {t('unlink_provider_account_title', { provider: 'GitHub' })} + + + +

{t('unlink_github_warning', { provider: 'GitHub' })}

+ + + setShowDisconnectModal(false)} + > + {t('cancel')} + + + {disconnecting ? t('unlinking') : t('unlink')} + + + + + ) +} \ No newline at end of file diff --git a/services/web/modules/github-sync/frontend/js/components/import-from-github-menu.tsx b/services/web/modules/github-sync/frontend/js/components/import-from-github-menu.tsx new file mode 100644 index 0000000000..47f6afb647 --- /dev/null +++ b/services/web/modules/github-sync/frontend/js/components/import-from-github-menu.tsx @@ -0,0 +1,12 @@ +import React from 'react' +import { useTranslation } from 'react-i18next' +import { DropdownItem } from '@/shared/components/dropdown/dropdown-menu' + +export default function ImportFromGitHubMenu({ onClick }: { onClick: (e: React.MouseEvent) => void }) { + const { t } = useTranslation() + return ( + + {t('import_from_github')} + + ) +} \ No newline at end of file diff --git a/services/web/modules/github-sync/frontend/js/components/import-from-github-modal-wrapper.tsx b/services/web/modules/github-sync/frontend/js/components/import-from-github-modal-wrapper.tsx new file mode 100644 index 0000000000..0064fca9a6 --- /dev/null +++ b/services/web/modules/github-sync/frontend/js/components/import-from-github-modal-wrapper.tsx @@ -0,0 +1,182 @@ +import React from 'react' +import { useTranslation } from 'react-i18next' +import getMeta from '@/utils/meta' +import { + OLModal, + OLModalBody, + OLModalFooter, + OLModalHeader, + OLModalTitle, +} from '@/shared/components/ol/ol-modal' +import OLButton from '@/shared/components/ol/ol-button' + + +const ModalContent = () => { + const { t } = useTranslation() + const { appName } = getMeta('ol-ExposedSettings') + const [isLoading, setIsLoading] = React.useState(true) + const [isLoadingImport, setIsLoadingImport] = React.useState(false) + const [isEnabled, setIsEnabled] = React.useState(false) + const [error, setError] = React.useState(null) + const [repos, setRepos] = React.useState>([]) + + + const handleImport = (name: string, fullName: string) => { + setIsLoading(true) + setIsLoadingImport(true) + fetch('/project/new/github-sync', { + method: 'POST', + headers: { + 'X-CSRF-Token': getMeta('ol-csrfToken'), + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + projectName: name, + repo: fullName, + }), + }) + .then(response => response.json()) + .then(data => { + if (data.projectId) { + window.location.href = `/project/${data.projectId}` + } else { + throw new Error('Failed to import project') + } + }) + .catch(error => { + console.error('Error importing GitHub repository:', error) + setError(error instanceof Error ? error.message : 'Unknown error') + }) + } + + // Fetch data from '/user/github-sync/repos', + // and set isLoading to false once data is fetched + React.useEffect(() => { + async function checkIfEnabled() { + try { + const response = await fetch('/user/github-sync/status') + if (!response.ok) { + throw new Error('Network response was not ok') + } + const statusData = await response.json() + setIsEnabled(statusData.enabled) + } catch (error) { + console.error('Error checking GitHub sync status:', error) + setError(error instanceof Error ? error.message : 'Unknown error') + } + } + + async function fetchRepos() { + try { + const response = await fetch('/user/github-sync/repos') + if (!response.ok) { + throw new Error('Network response was not ok') + } + // Assuming the response is a JSON array of repositories + const reposData = await response.json() + setRepos(reposData.repos) + } catch (error) { + console.error('Error fetching GitHub repositories:', error) + setError(error instanceof Error ? error.message : 'Unknown error') + } finally { + setIsLoading(false) + } + } + checkIfEnabled() + fetchRepos() + }, []) + + return ( + <> + {isLoading ? ( +
+
+ + { + isLoadingImport ? t('importing') : t('loading_github_repositories') + } +
+
+ ) : isEnabled ? ( + <> + { + error &&

{error}

+ } +

+ {t('select_github_repository', { appName })} +

+
+ + + {repos.map((repo) => ( + + + + + ))} + +
+ {repo.name} + + + handleImport(repo.name, repo.fullName)} + > + {t('import_to_sharelatex', { appName })} + +
+
+ + ) : +
+

{ + t('link_to_github_description', { appName }) + }

+ + {t('link_to_github')} + +
+ + } + + ) +} + +type ImportFromGitHubModalProps = { + onHide: () => void +} + +export default function ImportFromGitHubModal({ onHide }: ImportFromGitHubModalProps) { + const { t } = useTranslation() + // backdrop="static" can prevent the modal from being closed by + // clicking outside of it + return ( + + + + {t('import_from_github')} + + + + + + + + {t('cancel')} + + + + ) +} \ No newline at end of file diff --git a/services/web/modules/github-sync/frontend/js/components/tsconfig.json b/services/web/modules/github-sync/frontend/js/components/tsconfig.json new file mode 100644 index 0000000000..7159377670 --- /dev/null +++ b/services/web/modules/github-sync/frontend/js/components/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES6", + "module": "ESNext", + "jsx": "react-jsx", + "lib": [ + "ESNext", + "DOM" + ], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "." + ] +} \ No newline at end of file diff --git a/services/web/modules/github-sync/index.mjs b/services/web/modules/github-sync/index.mjs new file mode 100644 index 0000000000..80a728ecb3 --- /dev/null +++ b/services/web/modules/github-sync/index.mjs @@ -0,0 +1,31 @@ +import Modules from '../../app/src/infrastructure/Modules.mjs' +import logger from '@overleaf/logger' +/** @import { WebModule } from "../../types/web-module" */ + + +let GitHubSyncModule = {} +if (process.env.GITHUB_SYNC_ENABLED === 'true') { + logger.debug({}, 'Enabling GitHub Sync module') + const [{ default: GitHubSyncRouter }, { default: GitHubSyncHandler }] = + await Promise.all([ + import('./app/src/GitHubSyncRouter.mjs'), + import('./app/src/GitHubSyncHandler.mjs'), + ]) + + // Notes, this is only triggered in hard delete, no soft delete. + // Hard delete have some time delay, usually 30 or more days after soft delete. + Modules.hooks.attach('projectExpired', async projectId => { + try { + await GitHubSyncHandler.promises.deleteProjectGitHubSyncStatus(projectId) + logger.debug({ projectId }, 'Deleted GitHub sync status for expired project') + } catch (err) { + logger.warn({ projectId, err }, 'failed deleting project in github sync.') + } + }) + + GitHubSyncModule = { + router: GitHubSyncRouter, + } +} + +export default GitHubSyncModule diff --git a/services/web/package.json b/services/web/package.json index 6d6f1c939f..a7c29072d1 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -171,7 +171,7 @@ "parse-data-url": "^2.0.0", "passport": "^0.6.0", "passport-google-oauth20": "^2.0.0", - "passport-ldapauth": "^2.1.4", + "passport-ldapauth": "^3.0.1", "passport-local": "^1.0.0", "passport-oauth2": "^1.5.0", "passport-openidconnect": "^0.1.2",