From 903e443f2e0875f0b020e921c07e29bbb6ac5bdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Denis=20=C4=86ori=C4=87?= Date: Mon, 10 Mar 2025 12:56:22 +0100 Subject: [PATCH 1/2] feat: implements config loader to enable remote or external configs fix: config loader clone command issue fix: adds input validation, uses array arguments, prevented shell spawn fix: adds failsafe checking for directory location and structure fix: env-paths change to v2.2.1 which support require and minor code fix fix: improves test coverage Adds additional tests for better cove fix: fixed creating cache directory --- .gitignore | 6 + config.schema.json | 12 + package-lock.json | 2 +- package.json | 1 + packages/git-proxy-cli/index.js | 28 ++ proxy.config.json | 41 ++- src/config/ConfigLoader.js | 259 ++++++++++++++ src/config/index.ts | 105 +++++- src/proxy/index.js | 107 ++++++ src/service/index.js | 38 +++ test/ConfigLoader.test.js | 414 +++++++++++++++++++++++ test/chain.test.js | 92 ++--- website/docs/configuration/overview.mdx | 75 +++- website/docs/configuration/reference.mdx | 168 +++++++-- 14 files changed, 1251 insertions(+), 97 deletions(-) create mode 100644 src/config/ConfigLoader.js create mode 100644 src/proxy/index.js create mode 100644 test/ConfigLoader.test.js diff --git a/.gitignore b/.gitignore index 1849589c4..cefc84b7c 100644 --- a/.gitignore +++ b/.gitignore @@ -117,6 +117,9 @@ dist # Stores VSCode versions used for testing VSCode extensions .vscode-test +# Jetbrains +.idea + # yarn v2 .yarn/cache @@ -212,6 +215,9 @@ dist # https://nextjs.org/blog/next-9-1#public-directory-support # public +# git-config-cache +.git-config-cache + # vuepress build output .vuepress/dist diff --git a/config.schema.json b/config.schema.json index 771e83d0c..54b6aa230 100644 --- a/config.schema.json +++ b/config.schema.json @@ -5,6 +5,18 @@ "description": "Configuration for customizing git-proxy", "type": "object", "properties": { + "configurationSources": { + "enabled": { "type": "boolean" }, + "reloadIntervalSeconds": { "type": "number" }, + "merge": { "type": "boolean" }, + "sources": { + "type": "array", + "items": { + "type": "object", + "description": "Configuration source" + } + } + }, "proxyUrl": { "type": "string" }, "cookieSecret": { "type": "string" }, "sessionMaxAgeHours": { "type": "number" }, diff --git a/package-lock.json b/package-lock.json index 3052eaddc..04e182bfa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,7 @@ "connect-mongo": "^5.1.0", "cors": "^2.8.5", "diff2html": "^3.4.33", + "env-paths": "^2.2.1", "express": "^4.18.2", "express-http-proxy": "^2.0.0", "express-rate-limit": "^7.1.5", @@ -6167,7 +6168,6 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" diff --git a/package.json b/package.json index 757dfbd92..f924619ed 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "connect-mongo": "^5.1.0", "cors": "^2.8.5", "diff2html": "^3.4.33", + "env-paths": "^2.2.1", "express": "^4.18.2", "express-http-proxy": "^2.0.0", "express-rate-limit": "^7.1.5", diff --git a/packages/git-proxy-cli/index.js b/packages/git-proxy-cli/index.js index b0090a4bf..ab6810c4e 100755 --- a/packages/git-proxy-cli/index.js +++ b/packages/git-proxy-cli/index.js @@ -306,6 +306,29 @@ async function logout() { console.log('Logout: OK'); } +/** + * Reloads the GitProxy configuration without restarting the process + */ +async function reloadConfig() { + if (!fs.existsSync(GIT_PROXY_COOKIE_FILE)) { + console.error('Error: Reload config: Authentication required'); + process.exitCode = 1; + return; + } + + try { + const cookies = JSON.parse(fs.readFileSync(GIT_PROXY_COOKIE_FILE, 'utf8')); + + await axios.post(`${baseUrl}/api/v1/admin/reload-config`, {}, { headers: { Cookie: cookies } }); + + console.log('Configuration reloaded successfully'); + } catch (error) { + const errorMessage = `Error: Reload config: '${error.message}'`; + process.exitCode = 2; + console.error(errorMessage); + } +} + // Parsing command line arguments yargs(hideBin(process.argv)) // eslint-disable-line @typescript-eslint/no-unused-expressions .command({ @@ -436,6 +459,11 @@ yargs(hideBin(process.argv)) // eslint-disable-line @typescript-eslint/no-unused rejectGitPush(argv.id); }, }) + .command({ + command: 'reload-config', + description: 'Reload GitProxy configuration without restarting', + action: reloadConfig, + }) .demandCommand(1, 'You need at least one command before moving on') .strict() .help().argv; diff --git a/proxy.config.json b/proxy.config.json index 14d016e4d..db0d4f772 100644 --- a/proxy.config.json +++ b/proxy.config.json @@ -2,17 +2,44 @@ "proxyUrl": "https://github.com", "cookieSecret": "cookie secret", "sessionMaxAgeHours": 12, + "configurationSources": { + "enabled": false, + "reloadIntervalSeconds": 60, + "merge": false, + "sources": [ + { + "type": "file", + "enabled": false, + "path": "./external-config.json" + }, + { + "type": "http", + "enabled": false, + "url": "http://config-service/git-proxy-config", + "headers": {}, + "auth": { + "type": "bearer", + "token": "" + } + }, + { + "type": "git", + "enabled": false, + "repository": "https://git-server.com/project/git-proxy-config", + "branch": "main", + "path": "git-proxy/config.json", + "auth": { + "type": "ssh", + "privateKeyPath": "/path/to/.ssh/id_rsa" + } + } + ] + }, "tempPassword": { "sendEmail": false, "emailConfig": {} }, - "authorisedList": [ - { - "project": "finos", - "name": "git-proxy", - "url": "https://github.com/finos/git-proxy.git" - } - ], + "authorisedList": [], "sink": [ { "type": "fs", diff --git a/src/config/ConfigLoader.js b/src/config/ConfigLoader.js new file mode 100644 index 000000000..38816decb --- /dev/null +++ b/src/config/ConfigLoader.js @@ -0,0 +1,259 @@ +const fs = require('fs'); +const path = require('path'); +const axios = require('axios'); +const { execFile } = require('child_process'); +const { promisify } = require('util'); +const execFileAsync = promisify(execFile); +const EventEmitter = require('events'); +const envPaths = require('env-paths'); + +// Add path validation helper +function isValidPath(filePath) { + if (!filePath || typeof filePath !== 'string') return false; + + // Check for null bytes and other control characters + if (/[\0]/.test(filePath)) return false; + + try { + path.resolve(filePath); + return true; + } catch (error) { + return false; + } +} + +// Add URL validation helper +function isValidGitUrl(url) { + // Allow git://, https://, or ssh:// URLs + // Also allow scp-style URLs (user@host:path) + const validUrlPattern = + /^(git:\/\/|https:\/\/|ssh:\/\/|[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}:)/; + return typeof url === 'string' && validUrlPattern.test(url); +} + +// Add branch name validation helper +function isValidBranchName(branch) { + if (typeof branch !== 'string') return false; + + // Check for consecutive dots + if (branch.includes('..')) return false; + + // Check other branch name rules + // Branch names can contain alphanumeric, -, _, /, and . + // Cannot start with - or . + // Cannot contain consecutive dots + // Cannot contain control characters or spaces + const validBranchPattern = /^[a-zA-Z0-9][a-zA-Z0-9_/.-]*$/; + return validBranchPattern.test(branch); +} + +class ConfigLoader extends EventEmitter { + constructor(initialConfig) { + super(); + this.config = initialConfig; + this.reloadTimer = null; + this.isReloading = false; + this.cacheDir = null; + } + + async initialize() { + // Get cache directory path + const paths = envPaths('git-proxy'); + this.cacheDir = paths.cache; + + // Create cache directory if it doesn't exist + if (!fs.existsSync(this.cacheDir)) { + try { + fs.mkdirSync(this.cacheDir, { recursive: true }); + return true; + } catch (err) { + console.error('Failed to create cache directory:', err); + return false; + } + } + return true; + } + + async start() { + const { configurationSources } = this.config; + if (!configurationSources?.enabled) { + return; + } + + // Clear any existing interval before starting a new one + if (this.reloadTimer) { + clearInterval(this.reloadTimer); + this.reloadTimer = null; + } + + // Start periodic reload if interval is set + if (configurationSources.reloadIntervalSeconds > 0) { + this.reloadTimer = setInterval( + () => this.reloadConfiguration(), + configurationSources.reloadIntervalSeconds * 1000, + ); + } + + // Do initial load + await this.reloadConfiguration(); + } + + stop() { + if (this.reloadTimer) { + clearInterval(this.reloadTimer); + this.reloadTimer = null; + } + } + + async reloadConfiguration() { + if (this.isReloading) return; + this.isReloading = true; + + try { + const { configurationSources } = this.config; + if (!configurationSources?.enabled) return; + + const configs = await Promise.all( + configurationSources.sources + .filter((source) => source.enabled) + .map((source) => this.loadFromSource(source)), + ); + + // Use merge strategy based on configuration + const shouldMerge = configurationSources.merge ?? true; // Default to true for backward compatibility + const newConfig = shouldMerge + ? configs.reduce( + (acc, curr) => { + return this.deepMerge(acc, curr); + }, + { ...this.config }, + ) + : { ...this.config, ...configs[configs.length - 1] }; // Use last config for override + + // Emit change event if config changed + if (JSON.stringify(newConfig) !== JSON.stringify(this.config)) { + this.config = newConfig; + this.emit('configurationChanged', this.config); + } + } catch (error) { + console.error('Error reloading configuration:', error); + this.emit('configurationError', error); + } finally { + this.isReloading = false; + } + } + + async loadFromSource(source) { + switch (source.type) { + case 'file': + return this.loadFromFile(source); + case 'http': + return this.loadFromHttp(source); + case 'git': + return this.loadFromGit(source); + default: + throw new Error(`Unsupported configuration source type: ${source.type}`); + } + } + + async loadFromFile(source) { + const configPath = path.resolve(process.cwd(), source.path); + if (!isValidPath(configPath)) { + throw new Error('Invalid configuration file path'); + } + const content = await fs.promises.readFile(configPath, 'utf8'); + return JSON.parse(content); + } + + async loadFromHttp(source) { + const headers = { + ...source.headers, + ...(source.auth?.type === 'bearer' ? { Authorization: `Bearer ${source.auth.token}` } : {}), + }; + + const response = await axios.get(source.url, { headers }); + return response.data; + } + + async loadFromGit(source) { + // Validate inputs + if (!source.repository || !isValidGitUrl(source.repository)) { + throw new Error('Invalid repository URL format'); + } + if (source.branch && !isValidBranchName(source.branch)) { + throw new Error('Invalid branch name format'); + } + + // Use OS-specific cache directory + const paths = envPaths('git-proxy', { suffix: '' }); + const tempDir = path.join(paths.cache, 'git-config-cache'); + + if (!isValidPath(tempDir)) { + throw new Error('Invalid temporary directory path'); + } + await fs.promises.mkdir(tempDir, { recursive: true }); + + const repoDir = path.join(tempDir, Buffer.from(source.repository).toString('base64')); + if (!isValidPath(repoDir)) { + throw new Error('Invalid repository directory path'); + } + + // Clone or pull repository + if (!fs.existsSync(repoDir)) { + const execOptions = { + cwd: process.cwd(), + env: { + ...process.env, + ...(source.auth?.type === 'ssh' + ? { + GIT_SSH_COMMAND: `ssh -i ${source.auth.privateKeyPath}`, + } + : {}), + }, + }; + await execFileAsync('git', ['clone', source.repository, repoDir], execOptions); + } else { + await execFileAsync('git', ['pull'], { cwd: repoDir }); + } + + // Checkout specific branch if specified + if (source.branch) { + await execFileAsync('git', ['checkout', source.branch], { cwd: repoDir }); + } + + // Read and parse config file + const configPath = path.join(repoDir, source.path); + if (!isValidPath(configPath)) { + throw new Error('Invalid configuration file path in repository'); + } + const content = await fs.promises.readFile(configPath, 'utf8'); + return JSON.parse(content); + } + + deepMerge(target, source) { + const output = { ...target }; + if (isObject(target) && isObject(source)) { + Object.keys(source).forEach((key) => { + if (isObject(source[key])) { + if (!(key in target)) { + Object.assign(output, { [key]: source[key] }); + } else { + output[key] = this.deepMerge(target[key], source[key]); + } + } else { + Object.assign(output, { [key]: source[key] }); + } + }); + } + return output; + } +} + +function isObject(item) { + return item && typeof item === 'object' && !Array.isArray(item); +} + +module.exports = ConfigLoader; +module.exports.isValidGitUrl = isValidGitUrl; +module.exports.isValidPath = isValidPath; +module.exports.isValidBranchName = isValidBranchName; diff --git a/src/config/index.ts b/src/config/index.ts index a1779ed9d..a3a6cf471 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -1,4 +1,7 @@ import { existsSync, readFileSync } from 'fs'; +const fs = require('fs'); +const ConfigLoader = require('./ConfigLoader'); +const { validate } = require('./file'); // Import the validate function import defaultSettings from '../../proxy.config.json'; import { configFile } from './file'; @@ -34,9 +37,11 @@ export const getProxyUrl = () => { if (_userSettings !== null && _userSettings.proxyUrl) { _proxyUrl = _userSettings.proxyUrl; } +// Initialize configuration with defaults and user settings +let _config = { ...defaultSettings, ...(_userSettings || {}) }; - return _proxyUrl; -}; +// Create config loader instance +const configLoader = new ConfigLoader(_config); // Gets a list of authorised repositories export const getAuthorisedList = () => { @@ -44,6 +49,9 @@ export const getAuthorisedList = () => { _authorisedList = _userSettings.authorisedList; } return _authorisedList; +// Helper function to get current config value +const getConfig = (key) => { + return _config[key]; }; // Gets a list of authorised repositories @@ -61,15 +69,21 @@ export const getDatabase = () => { _database = _userSettings.sink; } for (const ix in _database) { +// Update existing getter functions to use the new config object +const getProxyUrl = () => getConfig('proxyUrl'); +const getAuthorisedList = () => getConfig('authorisedList'); +const getTempPasswordConfig = () => getConfig('tempPassword'); +const getDatabase = () => { + const sinks = getConfig('sink'); + for (const ix in sinks) { if (ix) { - const db = _database[ix]; + const db = sinks[ix]; if (db.enabled) { return db; } } } - - throw Error('No database cofigured!'); + throw Error('No database configured!'); }; // Gets the configured authentication method, defaults to local @@ -78,16 +92,32 @@ export const getAuthentication = () => { _authentication = _userSettings.authentication; } for (const ix in _authentication) { +const getAuthentication = () => { + const auths = getConfig('authentication'); + for (const ix in auths) { if (!ix) continue; - const auth = _authentication[ix]; + const auth = auths[ix]; if (auth.enabled) { return auth; } } - - throw Error('No authentication cofigured!'); + throw Error('No authentication configured!'); }; +const getAPIs = () => getConfig('api'); +const getCookieSecret = () => getConfig('cookieSecret'); +const getSessionMaxAgeHours = () => getConfig('sessionMaxAgeHours'); +const getCommitConfig = () => getConfig('commitConfig'); +const getAttestationConfig = () => getConfig('attestationConfig'); +const getPrivateOrganizations = () => getConfig('privateOrganizations'); +const getURLShortener = () => getConfig('urlShortener'); +const getContactEmail = () => getConfig('contactEmail'); +const getCSRFProtection = () => getConfig('csrfProtection'); +const getPlugins = () => getConfig('plugins'); +const getSSLKeyPath = () => getConfig('sslKeyPemPath') || '../../certs/key.pem'; +const getSSLCertPath = () => getConfig('sslCertPemPath') || '../../certs/cert.pem'; +const getDomains = () => getConfig('domains'); + // Log configuration to console export const logConfiguration = () => { console.log(`authorisedList = ${JSON.stringify(getAuthorisedList())}`); @@ -101,6 +131,12 @@ export const getAPIs = () => { } return _api; }; +// Function to handle configuration updates +const handleConfigUpdate = async (newConfig) => { + console.log('Configuration updated from external source'); + try { + // 1. Get proxy module dynamically to avoid circular dependency + const proxy = require('../proxy'); export const getCookieSecret = () => { if (_userSettings && _userSettings.cookieSecret) { @@ -108,6 +144,8 @@ export const getCookieSecret = () => { } return _cookieSecret; }; + // 2. Stop existing services + await proxy.stop(); export const getSessionMaxAgeHours = () => { if (_userSettings && _userSettings.sessionMaxAgeHours) { @@ -115,6 +153,8 @@ export const getSessionMaxAgeHours = () => { } return _sessionMaxAgeHours; }; + // 3. Update config + _config = newConfig; // Get commit related configuration export const getCommitConfig = () => { @@ -123,6 +163,8 @@ export const getCommitConfig = () => { } return _commitConfig; }; + // 4. Validate new configuration + validate(); // Get attestation related configuration export const getAttestationConfig = () => { @@ -131,13 +173,24 @@ export const getAttestationConfig = () => { } return _attestationConfig; }; + // 5. Restart services with new config + await proxy.start(); // Get private organizations related configuration export const getPrivateOrganizations = () => { if (_userSettings && _userSettings.privateOrganizations) { _privateOrganizations = _userSettings.privateOrganizations; + console.log('Services restarted with new configuration'); + } catch (error) { + console.error('Failed to apply new configuration:', error); + // Attempt to restart with previous config + try { + const proxy = require('../proxy'); + await proxy.start(); + } catch (startError) { + console.error('Failed to restart services:', startError); + } } - return _privateOrganizations; }; // Get URL shortener @@ -147,6 +200,8 @@ export const getURLShortener = () => { } return _urlShortener; }; +// Handle configuration updates +configLoader.on('configurationChanged', handleConfigUpdate); // Get contact e-mail address export const getContactEmail = () => { @@ -155,6 +210,9 @@ export const getContactEmail = () => { } return _contactEmail; }; +configLoader.on('configurationError', (error) => { + console.error('Error loading external configuration:', error); +}); // Get CSRF protection flag export const getCSRFProtection = () => { @@ -163,6 +221,10 @@ export const getCSRFProtection = () => { } return _csrfProtection; }; +// Start the config loader if external sources are enabled +configLoader.start().catch((error) => { + console.error('Failed to start configuration loader:', error); +}); // Get loadable push plugins export const getPlugins = () => { @@ -180,6 +242,9 @@ export const getSSLKeyPath = () => { return '../../certs/key.pem'; } return _sslKeyPath; +// Force reload of configuration +const reloadConfiguration = async () => { + await configLoader.reloadConfiguration(); }; export const getSSLCertPath = () => { @@ -198,3 +263,25 @@ export const getDomains = () => { } return _domains; }; + +// Export all the functions +exports.getAPIs = getAPIs; +exports.getProxyUrl = getProxyUrl; +exports.getAuthorisedList = getAuthorisedList; +exports.getDatabase = getDatabase; +exports.logConfiguration = logConfiguration; +exports.getAuthentication = getAuthentication; +exports.getTempPasswordConfig = getTempPasswordConfig; +exports.getCookieSecret = getCookieSecret; +exports.getSessionMaxAgeHours = getSessionMaxAgeHours; +exports.getCommitConfig = getCommitConfig; +exports.getAttestationConfig = getAttestationConfig; +exports.getPrivateOrganizations = getPrivateOrganizations; +exports.getURLShortener = getURLShortener; +exports.getContactEmail = getContactEmail; +exports.getCSRFProtection = getCSRFProtection; +exports.getPlugins = getPlugins; +exports.getSSLKeyPath = getSSLKeyPath; +exports.getSSLCertPath = getSSLCertPath; +exports.getDomains = getDomains; +exports.reloadConfiguration = reloadConfiguration; diff --git a/src/proxy/index.js b/src/proxy/index.js new file mode 100644 index 000000000..47540b3da --- /dev/null +++ b/src/proxy/index.js @@ -0,0 +1,107 @@ +const express = require('express'); +const bodyParser = require('body-parser'); +const http = require('http'); +const https = require('https'); +const fs = require('fs'); +const path = require('path'); +const router = require('./routes').router; +const config = require('../config'); +const db = require('../db'); +const { PluginLoader } = require('../plugin'); +const chain = require('./chain'); +const { GIT_PROXY_SERVER_PORT: proxyHttpPort } = require('../config/env').Vars; +const { GIT_PROXY_HTTPS_SERVER_PORT: proxyHttpsPort } = require('../config/env').Vars; + +const options = { + inflate: true, + limit: '100000kb', + type: '*/*', + key: fs.readFileSync(path.join(__dirname, config.getSSLKeyPath())), + cert: fs.readFileSync(path.join(__dirname, config.getSSLCertPath())), +}; + +const proxyPreparations = async () => { + const plugins = config.getPlugins(); + const pluginLoader = new PluginLoader(plugins); + await pluginLoader.load(); + chain.chainPluginLoader = pluginLoader; + // Check to see if the default repos are in the repo list + const defaultAuthorisedRepoList = config.getAuthorisedList(); + const allowedList = await db.getRepos(); + + defaultAuthorisedRepoList.forEach(async (x) => { + const found = allowedList.find((y) => y.project === x.project && x.name === y.name); + if (!found) { + await db.createRepo(x); + await db.addUserCanPush(x.name, 'admin'); + await db.addUserCanAuthorise(x.name, 'admin'); + } + }); +}; + +// just keep this async incase it needs async stuff in the future +const createApp = async () => { + const app = express(); + // Setup the proxy middleware + app.use(bodyParser.raw(options)); + app.use('/', router); + return app; +}; + +let httpServer = null; +let httpsServer = null; + +const start = async () => { + const app = await createApp(); + await proxyPreparations(); + + // Start HTTP server + httpServer = http.createServer(options, app); + httpServer.listen(proxyHttpPort, () => { + console.log(`HTTP Proxy Listening on ${proxyHttpPort}`); + }); + + // Start HTTPS server if SSL certificates exist + const sslKeyPath = config.getSSLKeyPath(); + const sslCertPath = config.getSSLCertPath(); + + if (fs.existsSync(sslKeyPath) && fs.existsSync(sslCertPath)) { + httpsServer = https.createServer(options, app); + httpsServer.listen(proxyHttpsPort, () => { + console.log(`HTTPS Proxy Listening on ${proxyHttpsPort}`); + }); + } + + return app; +}; + +const stop = () => { + return new Promise((resolve, reject) => { + try { + // Close HTTP server if it exists + if (httpServer) { + httpServer.close(() => { + console.log('HTTP server closed'); + httpServer = null; + }); + } + + // Close HTTPS server if it exists + if (httpsServer) { + httpsServer.close(() => { + console.log('HTTPS server closed'); + httpsServer = null; + }); + } + + resolve(); + } catch (error) { + reject(error); + } + }); +}; + +module.exports.proxyPreparations = proxyPreparations; +module.exports.createApp = createApp; +module.exports.start = start; +module.exports.stop = stop; diff --git a/src/service/index.js b/src/service/index.js index d384fcd6e..8f6589d47 100644 --- a/src/service/index.js +++ b/src/service/index.js @@ -8,6 +8,8 @@ const config = require('../config'); const db = require('../db'); const rateLimit = require('express-rate-limit'); const lusca = require('lusca'); +const configLoader = require('../config/ConfigLoader'); +const proxy = require('../proxy'); const limiter = rateLimit({ windowMs: 15 * 60 * 1000, // 15 minutes @@ -32,6 +34,42 @@ const createApp = async () => { app.use(cors(corsOptions)); app.set('trust proxy', 1); app.use(limiter); + + // Add new admin-only endpoint to reload config + app.post('/api/v1/admin/reload-config', async (req, res) => { + if (!req.isAuthenticated() || !req.user.admin) { + return res.status(403).json({ error: 'Unauthorized' }); + } + + try { + // 1. Reload configuration + await configLoader.loadConfiguration(); + + // 2. Stop existing services + await proxy.stop(); + + // 3. Apply new configuration + config.validate(); + + // 4. Restart services with new config + await proxy.start(); + + console.log('Configuration reloaded and services restarted successfully'); + res.json({ status: 'success', message: 'Configuration reloaded and services restarted' }); + } catch (error) { + console.error('Failed to reload configuration and restart services:', error); + + // Attempt to restart with existing config if reload fails + try { + await proxy.start(); + } catch (startError) { + console.error('Failed to restart services:', startError); + } + + res.status(500).json({ error: 'Failed to reload configuration' }); + } + }); + app.use( session({ store: config.getDatabase().type === 'mongo' ? db.getSessionStore(session) : null, diff --git a/test/ConfigLoader.test.js b/test/ConfigLoader.test.js new file mode 100644 index 000000000..fe82766c2 --- /dev/null +++ b/test/ConfigLoader.test.js @@ -0,0 +1,414 @@ +const chai = require('chai'); +const fs = require('fs'); +const path = require('path'); +const { expect } = chai; +const ConfigLoader = require('../src/config/ConfigLoader'); +const { isValidGitUrl, isValidPath, isValidBranchName } = ConfigLoader; +const sinon = require('sinon'); +const axios = require('axios'); + +describe('ConfigLoader', () => { + let configLoader; + let tempDir; + let tempConfigFile; + + beforeEach(() => { + // Create temp directory for test files + tempDir = fs.mkdtempSync('gitproxy-configloader-test-'); + tempConfigFile = path.join(tempDir, 'test-config.json'); + }); + + afterEach(() => { + // Clean up temp files + if (fs.existsSync(tempDir)) { + fs.rmSync(tempDir, { recursive: true }); + } + sinon.restore(); + }); + + describe('loadFromFile', () => { + it('should load configuration from file', async () => { + const testConfig = { + proxyUrl: 'https://test.com', + cookieSecret: 'test-secret', + }; + fs.writeFileSync(tempConfigFile, JSON.stringify(testConfig)); + + configLoader = new ConfigLoader({}); + const result = await configLoader.loadFromFile({ path: tempConfigFile }); + + expect(result).to.deep.equal(testConfig); + }); + }); + + describe('loadFromHttp', () => { + it('should load configuration from HTTP endpoint', async () => { + const testConfig = { + proxyUrl: 'https://test.com', + cookieSecret: 'test-secret', + }; + + sinon.stub(axios, 'get').resolves({ data: testConfig }); + + configLoader = new ConfigLoader({}); + const result = await configLoader.loadFromHttp({ + url: 'http://config-service/config', + headers: {}, + }); + + expect(result).to.deep.equal(testConfig); + }); + + it('should include bearer token if provided', async () => { + const axiosStub = sinon.stub(axios, 'get').resolves({ data: {} }); + + configLoader = new ConfigLoader({}); + await configLoader.loadFromHttp({ + url: 'http://config-service/config', + auth: { + type: 'bearer', + token: 'test-token', + }, + }); + + expect( + axiosStub.calledWith('http://config-service/config', { + headers: { Authorization: 'Bearer test-token' }, + }), + ).to.be.true; + }); + }); + + describe('reloadConfiguration', () => { + it('should emit configurationChanged event when config changes', async () => { + const initialConfig = { + configurationSources: { + enabled: true, + sources: [ + { + type: 'file', + enabled: true, + path: tempConfigFile, + }, + ], + }, + }; + + const newConfig = { + proxyUrl: 'https://new-test.com', + }; + + fs.writeFileSync(tempConfigFile, JSON.stringify(newConfig)); + + configLoader = new ConfigLoader(initialConfig); + const spy = sinon.spy(); + configLoader.on('configurationChanged', spy); + + await configLoader.reloadConfiguration(); + + expect(spy.calledOnce).to.be.true; + expect(spy.firstCall.args[0]).to.deep.include(newConfig); + }); + + it('should not emit event if config has not changed', async () => { + const testConfig = { + proxyUrl: 'https://test.com', + }; + + const config = { + configurationSources: { + enabled: true, + sources: [ + { + type: 'file', + enabled: true, + path: tempConfigFile, + }, + ], + }, + }; + + fs.writeFileSync(tempConfigFile, JSON.stringify(testConfig)); + + configLoader = new ConfigLoader(config); + const spy = sinon.spy(); + configLoader.on('configurationChanged', spy); + + await configLoader.reloadConfiguration(); // First reload should emit + await configLoader.reloadConfiguration(); // Second reload should not emit since config hasn't changed + + expect(spy.calledOnce).to.be.true; // Should only emit once + }); + }); + + describe('initialize', () => { + it('should initialize cache directory using env-paths', async () => { + const configLoader = new ConfigLoader({}); + await configLoader.initialize(); + + // Check that cacheDir is set and is a string + expect(configLoader.cacheDir).to.be.a('string'); + + // Check that it contains 'git-proxy' in the path + expect(configLoader.cacheDir).to.include('git-proxy'); + + // On macOS, it should be in the Library/Caches directory + // On Linux, it should be in the ~/.cache directory + // On Windows, it should be in the AppData/Local directory + if (process.platform === 'darwin') { + expect(configLoader.cacheDir).to.include('Library/Caches'); + } else if (process.platform === 'linux') { + expect(configLoader.cacheDir).to.include('.cache'); + } else if (process.platform === 'win32') { + expect(configLoader.cacheDir).to.include('AppData/Local'); + } + }); + + it('should create cache directory if it does not exist', async () => { + const configLoader = new ConfigLoader({}); + await configLoader.initialize(); + + // Check if directory exists + expect(fs.existsSync(configLoader.cacheDir)).to.be.true; + }); + }); + + describe('loadRemoteConfig', () => { + let configLoader; + beforeEach(async () => { + const configFilePath = path.join(__dirname, '..', 'proxy.config.json'); + const config = JSON.parse(fs.readFileSync(configFilePath)); + + config.configurationSources.enabled = true; + configLoader = new ConfigLoader(config); + await configLoader.initialize(); + }); + + it('should load configuration from git repository', async function () { + // eslint-disable-next-line no-invalid-this + this.timeout(10000); + + const source = { + type: 'git', + repository: 'https://github.com/finos/git-proxy.git', + path: 'proxy.config.json', + branch: 'main', + }; + + const config = await configLoader.loadFromGit(source); + + // Verify the loaded config has expected structure + expect(config).to.be.an('object'); + expect(config).to.have.property('proxyUrl'); + expect(config).to.have.property('cookieSecret'); + }); + + it('should throw error for invalid configuration file path', async function () { + const source = { + type: 'git', + repository: 'https://github.com/finos/git-proxy.git', + path: '\0', // Invalid path + branch: 'main', + }; + + try { + await configLoader.loadFromGit(source); + throw new Error('Expected error was not thrown'); + } catch (error) { + expect(error.message).to.equal('Invalid configuration file path in repository'); + } + }); + + it('should load configuration from http', async function () { + // eslint-disable-next-line no-invalid-this + this.timeout(10000); + + const source = { + type: 'http', + url: 'https://raw.githubusercontent.com/finos/git-proxy/refs/heads/main/proxy.config.json', + }; + + const config = await configLoader.loadFromHttp(source); + + // Verify the loaded config has expected structure + expect(config).to.be.an('object'); + expect(config).to.have.property('proxyUrl'); + expect(config).to.have.property('cookieSecret'); + }); + }); + + describe('deepMerge', () => { + let configLoader; + + beforeEach(() => { + configLoader = new ConfigLoader({}); + }); + + it('should merge simple objects', () => { + const target = { a: 1, b: 2 }; + const source = { b: 3, c: 4 }; + + const result = configLoader.deepMerge(target, source); + + expect(result).to.deep.equal({ a: 1, b: 3, c: 4 }); + }); + + it('should merge nested objects', () => { + const target = { + a: 1, + b: { x: 1, y: 2 }, + c: { z: 3 }, + }; + const source = { + b: { y: 4, w: 5 }, + c: { z: 6 }, + }; + + const result = configLoader.deepMerge(target, source); + + expect(result).to.deep.equal({ + a: 1, + b: { x: 1, y: 4, w: 5 }, + c: { z: 6 }, + }); + }); + + it('should handle arrays by replacing them', () => { + const target = { + a: [1, 2, 3], + b: { items: [4, 5] }, + }; + const source = { + a: [7, 8], + b: { items: [9] }, + }; + + const result = configLoader.deepMerge(target, source); + + expect(result).to.deep.equal({ + a: [7, 8], + b: { items: [9] }, + }); + }); + + it('should handle null and undefined values', () => { + const target = { + a: 1, + b: null, + c: undefined, + }; + const source = { + a: null, + b: 2, + c: 3, + }; + + const result = configLoader.deepMerge(target, source); + + expect(result).to.deep.equal({ + a: null, + b: 2, + c: 3, + }); + }); + + it('should handle empty objects', () => { + const target = {}; + const source = { a: 1, b: { c: 2 } }; + + const result = configLoader.deepMerge(target, source); + + expect(result).to.deep.equal({ a: 1, b: { c: 2 } }); + }); + + it('should not modify the original objects', () => { + const target = { a: 1, b: { c: 2 } }; + const source = { b: { c: 3 } }; + const originalTarget = { ...target }; + const originalSource = { ...source }; + + configLoader.deepMerge(target, source); + + expect(target).to.deep.equal(originalTarget); + expect(source).to.deep.equal(originalSource); + }); + }); +}); + +describe('Validation Helpers', () => { + describe('isValidGitUrl', () => { + it('should validate git URLs correctly', () => { + // Valid URLs + expect(isValidGitUrl('git://github.com/user/repo.git')).to.be.true; + expect(isValidGitUrl('https://github.com/user/repo.git')).to.be.true; + expect(isValidGitUrl('ssh://git@github.com/user/repo.git')).to.be.true; + expect(isValidGitUrl('user@github.com:user/repo.git')).to.be.true; + + // Invalid URLs + expect(isValidGitUrl('not-a-git-url')).to.be.false; + expect(isValidGitUrl('http://github.com/user/repo')).to.be.false; + expect(isValidGitUrl('')).to.be.false; + expect(isValidGitUrl(null)).to.be.false; + expect(isValidGitUrl(undefined)).to.be.false; + expect(isValidGitUrl(123)).to.be.false; + }); + }); + + describe('isValidPath', () => { + it('should validate file paths correctly', () => { + const cwd = process.cwd(); + + // Valid paths + expect(isValidPath(path.join(cwd, 'config.json'))).to.be.true; + expect(isValidPath(path.join(cwd, 'subfolder/config.json'))).to.be.true; + expect(isValidPath('/etc/passwd')).to.be.true; + expect(isValidPath('../config.json')).to.be.true; + + // Invalid paths + expect(isValidPath('')).to.be.false; + expect(isValidPath(null)).to.be.false; + expect(isValidPath(undefined)).to.be.false; + + // Additional edge cases + expect(isValidPath({})).to.be.false; + expect(isValidPath([])).to.be.false; + expect(isValidPath(123)).to.be.false; + expect(isValidPath(true)).to.be.false; + expect(isValidPath('\0invalid')).to.be.false; + expect(isValidPath('\u0000')).to.be.false; + }); + + it('should handle path resolution errors', () => { + // Mock path.resolve to throw an error + const originalResolve = path.resolve; + path.resolve = () => { + throw new Error('Mock path resolution error'); + }; + + expect(isValidPath('some/path')).to.be.false; + + // Restore original path.resolve + path.resolve = originalResolve; + }); + }); + + describe('isValidBranchName', () => { + it('should validate git branch names correctly', () => { + // Valid branch names + expect(isValidBranchName('main')).to.be.true; + expect(isValidBranchName('feature/new-feature')).to.be.true; + expect(isValidBranchName('release-1.0')).to.be.true; + expect(isValidBranchName('fix_123')).to.be.true; + expect(isValidBranchName('user/feature/branch')).to.be.true; + + // // Invalid branch names + expect(isValidBranchName('.invalid')).to.be.false; + expect(isValidBranchName('-invalid')).to.be.false; + expect(isValidBranchName('branch with spaces')).to.be.false; + expect(isValidBranchName('')).to.be.false; + expect(isValidBranchName(null)).to.be.false; + expect(isValidBranchName(undefined)).to.be.false; + expect(isValidBranchName('branch..name')).to.be.false; + }); + }); +}); diff --git a/test/chain.test.js b/test/chain.test.js index d646b9dc7..9913ac1ed 100644 --- a/test/chain.test.js +++ b/test/chain.test.js @@ -15,36 +15,45 @@ const mockLoader = { ], }; -const mockPushProcessors = { - parsePush: sinon.stub(), - audit: sinon.stub(), - checkRepoInAuthorisedList: sinon.stub(), - checkCommitMessages: sinon.stub(), - checkAuthorEmails: sinon.stub(), - checkUserPushPermission: sinon.stub(), - checkIfWaitingAuth: sinon.stub(), - pullRemote: sinon.stub(), - writePack: sinon.stub(), - preReceive: sinon.stub(), - getDiff: sinon.stub(), - clearBareClone: sinon.stub(), - scanDiff: sinon.stub(), - blockForAuth: sinon.stub(), +const clearCache = (sinon) => { + delete require.cache[require.resolve('../src/proxy/processors')]; + delete require.cache[require.resolve('../src/proxy/chain')]; + sinon.restore(); +}; + +const initMockPushProcessors = (sinon) => { + const mockPushProcessors = { + parsePush: sinon.stub(), + audit: sinon.stub(), + checkRepoInAuthorisedList: sinon.stub(), + checkCommitMessages: sinon.stub(), + checkAuthorEmails: sinon.stub(), + checkUserPushPermission: sinon.stub(), + checkIfWaitingAuth: sinon.stub(), + pullRemote: sinon.stub(), + writePack: sinon.stub(), + getDiff: sinon.stub(), + clearBareClone: sinon.stub(), + scanDiff: sinon.stub(), + blockForAuth: sinon.stub(), + preReceive: sinon.stub(), + }; + mockPushProcessors.parsePush.displayName = 'parsePush'; + mockPushProcessors.audit.displayName = 'audit'; + mockPushProcessors.checkRepoInAuthorisedList.displayName = 'checkRepoInAuthorisedList'; + mockPushProcessors.checkCommitMessages.displayName = 'checkCommitMessages'; + mockPushProcessors.checkAuthorEmails.displayName = 'checkAuthorEmails'; + mockPushProcessors.checkUserPushPermission.displayName = 'checkUserPushPermission'; + mockPushProcessors.checkIfWaitingAuth.displayName = 'checkIfWaitingAuth'; + mockPushProcessors.pullRemote.displayName = 'pullRemote'; + mockPushProcessors.writePack.displayName = 'writePack'; + mockPushProcessors.getDiff.displayName = 'getDiff'; + mockPushProcessors.clearBareClone.displayName = 'clearBareClone'; + mockPushProcessors.scanDiff.displayName = 'scanDiff'; + mockPushProcessors.blockForAuth.displayName = 'blockForAuth'; + mockPushProcessors.preReceive.displayName = 'preReceive'; + return mockPushProcessors; }; -mockPushProcessors.parsePush.displayName = 'parsePush'; -mockPushProcessors.audit.displayName = 'audit'; -mockPushProcessors.checkRepoInAuthorisedList.displayName = 'checkRepoInAuthorisedList'; -mockPushProcessors.checkCommitMessages.displayName = 'checkCommitMessages'; -mockPushProcessors.checkAuthorEmails.displayName = 'checkAuthorEmails'; -mockPushProcessors.checkUserPushPermission.displayName = 'checkUserPushPermission'; -mockPushProcessors.checkIfWaitingAuth.displayName = 'checkIfWaitingAuth'; -mockPushProcessors.pullRemote.displayName = 'pullRemote'; -mockPushProcessors.writePack.displayName = 'writePack'; -mockPushProcessors.preReceive.displayName = 'preReceive'; -mockPushProcessors.getDiff.displayName = 'getDiff'; -mockPushProcessors.clearBareClone.displayName = 'clearBareClone'; -mockPushProcessors.scanDiff.displayName = 'scanDiff'; -mockPushProcessors.blockForAuth.displayName = 'blockForAuth'; const mockPreProcessors = { parseAction: sinon.stub(), @@ -53,27 +62,30 @@ const mockPreProcessors = { describe('proxy chain', function () { let processors; let chain; + let sandboxSinon; + let mockPushProcessors; beforeEach(async () => { - // Re-import the processors module after clearing the cache - processors = await import('../src/proxy/processors'); + sandboxSinon = sinon.createSandbox(); - // Mock the processors module - sinon.stub(processors, 'pre').value(mockPreProcessors); + // Init mock processors + mockPushProcessors = initMockPushProcessors(sandboxSinon); - sinon.stub(processors, 'push').value(mockPushProcessors); + // Re-require the processors module after clearing the cache + processors = require('../src/proxy/processors'); + + // Mock the processors module + sandboxSinon.stub(processors, 'pre').value(mockPreProcessors); + sandboxSinon.stub(processors, 'push').value(mockPushProcessors); - // Re-import the chain module after stubbing processors - chain = (await import('../src/proxy/chain')).default; + // Re-require the chain module after stubbing processors + chain = require('../src/proxy/chain'); chain.chainPluginLoader = new PluginLoader([]); }); afterEach(() => { - // Clear the module from the cache after each test - delete require.cache[require.resolve('../src/proxy/processors')]; - delete require.cache[require.resolve('../src/proxy/chain')]; - sinon.reset(); + clearCache(sandboxSinon); }); it('getChain should set pluginLoaded if loader is undefined', async function () { diff --git a/website/docs/configuration/overview.mdx b/website/docs/configuration/overview.mdx index 5493d54f6..fe5820566 100644 --- a/website/docs/configuration/overview.mdx +++ b/website/docs/configuration/overview.mdx @@ -7,6 +7,71 @@ description: How to customise push protections and policies On installation, GitProxy ships with an [out-of-the-box configuration](https://github.com/finos/git-proxy/blob/main/proxy.config.json). This is fine for demonstration purposes but is likely not what you want to deploy into your environment. + +### Configuration Sources + +GitProxy supports dynamic configuration loading from multiple sources. This feature allows you to manage your configuration from external sources and update it without restarting the service. Configuration sources can be files, HTTP endpoints, or Git repositories. + +To enable configuration sources, add the `configurationSources` section to your configuration: + +```json +{ + "configurationSources": { + "enabled": true, + "reloadIntervalSeconds": 60, + "merge": false, + "sources": [ + { + "type": "file", + "enabled": true, + "path": "./external-config.json" + }, + { + "type": "http", + "enabled": true, + "url": "http://config-service/git-proxy-config", + "headers": {}, + "auth": { + "type": "bearer", + "token": "your-token" + } + }, + { + "type": "git", + "enabled": true, + "repository": "https://git-server.com/project/git-proxy-config", + "branch": "main", + "path": "git-proxy/config.json", + "auth": { + "type": "ssh", + "privateKeyPath": "/path/to/.ssh/id_rsa" + } + } + ] + } +} +``` + +The configuration options for `configurationSources` are: + +- `enabled`: Enable/disable dynamic configuration loading +- `reloadIntervalSeconds`: How often to check for configuration updates (in seconds) +- `merge`: When true, merges configurations from all enabled sources. When false, uses the last successful configuration load. This can be used to upload only partial configuration to external source +- `sources`: Array of configuration sources to load from + +Each source can be one of three types: + +1. `file`: Load from a local JSON file +2. `http`: Load from an HTTP endpoint +3. `git`: Load from a Git repository + +When configuration changes are detected, GitProxy will: + +1. Validate the new configuration +2. Stop existing services +3. Apply the new configuration +4. Restart services with the updated configuration + ### Customise configuration To customise your GitProxy configuration, create a `proxy.config.json` in your current @@ -44,8 +109,9 @@ npx -- @finos/git-proxy --config ./config.json ``` ### Set ports with ENV variables + By default, GitProxy uses port 8000 to expose the Git Server and 8080 for the frontend application. -The ports can be changed by setting the `GIT_PROXY_SERVER_PORT`, `GIT_PROXY_HTTPS_SERVER_PORT` (optional) and `GIT_PROXY_UI_PORT` +The ports can be changed by setting the `GIT_PROXY_SERVER_PORT`, `GIT_PROXY_HTTPS_SERVER_PORT` (optional) and `GIT_PROXY_UI_PORT` environment variables: ``` @@ -54,10 +120,10 @@ export GIT_PROXY_SERVER_PORT="9090" export GIT_PROXY_HTTPS_SERVER_PORT="9443" ``` -Note that `GIT_PROXY_UI_PORT` is needed for both server and UI Node processes, +Note that `GIT_PROXY_UI_PORT` is needed for both server and UI Node processes, whereas `GIT_PROXY_SERVER_PORT` (and `GIT_PROXY_HTTPS_SERVER_PORT`) is only needed by the server process. -By default, GitProxy CLI connects to GitProxy running on localhost and default port. This can be +By default, GitProxy CLI connects to GitProxy running on localhost and default port. This can be changed by setting the `GIT_PROXY_UI_HOST` and `GIT_PROXY_UI_PORT` environment variables: ``` @@ -78,6 +144,3 @@ To validate your configuration at a custom file location, run: ```bash git-proxy --validate --config ./config.json ``` - - - diff --git a/website/docs/configuration/reference.mdx b/website/docs/configuration/reference.mdx index 6a7eceedf..b95b76109 100644 --- a/website/docs/configuration/reference.mdx +++ b/website/docs/configuration/reference.mdx @@ -17,7 +17,106 @@ description: JSON schema reference documentation for GitProxy
- 1. [Optional] Property GitProxy configuration file > proxyUrl + 1. [Optional] Property GitProxy configuration file > configurationSources + +
+ +| | | +| ------------------------- | ------------------------------------------------------- | +| **Type** | `object` | +| **Required** | No | +| **Additional properties** | [[Not allowed]](# "Additional Properties not allowed.") | + +**Description:** Configuration for dynamic loading from external sources + +
+ + 1.1. [Optional] Property configurationSources > enabled + +
+ +| | | +| ------------ | --------- | +| **Type** | `boolean` | +| **Required** | No | + +**Description:** Enable/disable dynamic configuration loading + +
+
+ +
+ + 1.2. [Optional] Property configurationSources > reloadIntervalSeconds + +
+ +| | | +| ------------ | -------- | +| **Type** | `number` | +| **Required** | No | + +**Description:** How often to check for configuration updates (in seconds) + +
+
+ +
+ + 1.3. [Optional] Property configurationSources > merge + +
+ +| | | +| ------------ | --------- | +| **Type** | `boolean` | +| **Required** | No | + +**Description:** When true, merges configurations from all enabled sources. When false, uses the last successful configuration load + +
+
+ +
+ + 1.4. [Optional] Property configurationSources > sources + +
+ +| | | +| ------------ | ------- | +| **Type** | `array` | +| **Required** | No | + +**Description:** Array of configuration sources to load from + +Each item in the array must be an object with the following properties: + +- `type`: (Required) Type of configuration source (`"file"`, `"http"`, or `"git"`) +- `enabled`: (Required) Whether this source is enabled +- `path`: (Required for `file` type) Path to the configuration file +- `url`: (Required for `http` type) URL of the configuration endpoint +- `repository`: (Required for `git` type) Git repository URL +- `branch`: (Optional for `git` type) Branch to use +- `path`: (Required for `git` type) Path to configuration file in repository +- `headers`: (Optional for `http` type) HTTP headers to include +- `auth`: (Optional) Authentication configuration + - For `http` type: + - `type`: `"bearer"` + - `token`: Bearer token value + - For `git` type: + - `type`: `"ssh"` + - `privateKeyPath`: Path to SSH private key + +
+
+ +
+
+ +
+ + 2. [Optional] Property GitProxy configuration file > proxyUrl
@@ -31,7 +130,7 @@ description: JSON schema reference documentation for GitProxy
- 2. [Optional] Property GitProxy configuration file > cookieSecret + 3. [Optional] Property GitProxy configuration file > cookieSecret
@@ -45,7 +144,7 @@ description: JSON schema reference documentation for GitProxy
- 3. [Optional] Property GitProxy configuration file > sessionMaxAgeHours + 4. [Optional] Property GitProxy configuration file > sessionMaxAgeHours
@@ -59,7 +158,7 @@ description: JSON schema reference documentation for GitProxy
- 4. [Optional] Property GitProxy configuration file > api + 5. [Optional] Property GitProxy configuration file > api
@@ -76,7 +175,7 @@ description: JSON schema reference documentation for GitProxy
- 5. [Optional] Property GitProxy configuration file > commitConfig + 6. [Optional] Property GitProxy configuration file > commitConfig
@@ -93,7 +192,7 @@ description: JSON schema reference documentation for GitProxy
- 6. [Optional] Property GitProxy configuration file > attestationConfig + 7. [Optional] Property GitProxy configuration file > attestationConfig
@@ -110,7 +209,7 @@ description: JSON schema reference documentation for GitProxy
- 7. [Optional] Property GitProxy configuration file > domains + 8. [Optional] Property GitProxy configuration file > domains
@@ -127,7 +226,7 @@ description: JSON schema reference documentation for GitProxy
- 8. [Optional] Property GitProxy configuration file > privateOrganizations + 9. [Optional] Property GitProxy configuration file > privateOrganizations
@@ -143,7 +242,7 @@ description: JSON schema reference documentation for GitProxy
- 9. [Optional] Property GitProxy configuration file > urlShortener + 10. [Optional] Property GitProxy configuration file > urlShortener
@@ -159,7 +258,7 @@ description: JSON schema reference documentation for GitProxy
- 10. [Optional] Property GitProxy configuration file > contactEmail + 11. [Optional] Property GitProxy configuration file > contactEmail
@@ -175,7 +274,7 @@ description: JSON schema reference documentation for GitProxy
- 11. [Optional] Property GitProxy configuration file > csrfProtection + 12. [Optional] Property GitProxy configuration file > csrfProtection
@@ -191,7 +290,7 @@ description: JSON schema reference documentation for GitProxy
- 12. [Optional] Property GitProxy configuration file > plugins + 13. [Optional] Property GitProxy configuration file > plugins
@@ -206,7 +305,7 @@ description: JSON schema reference documentation for GitProxy | ------------------------------- | ----------- | | [plugins items](#plugins_items) | - | -### 12.1. GitProxy configuration file > plugins > plugins items +### 13.1. GitProxy configuration file > plugins > plugins items | | | | ------------ | -------- | @@ -218,7 +317,7 @@ description: JSON schema reference documentation for GitProxy
- 13. [Optional] Property GitProxy configuration file > authorisedList + 14. [Optional] Property GitProxy configuration file > authorisedList
@@ -233,7 +332,7 @@ description: JSON schema reference documentation for GitProxy | --------------------------------------- | ----------- | | [authorisedRepo](#authorisedList_items) | - | -### 13.1. GitProxy configuration file > authorisedList > authorisedRepo +### 14.1. GitProxy configuration file > authorisedList > authorisedRepo | | | | ------------------------- | ------------------------------------------------------------------------- | @@ -244,7 +343,7 @@ description: JSON schema reference documentation for GitProxy
- 13.1.1. [Required] Property GitProxy configuration file > authorisedList > authorisedList items > project + 14.1.1. [Required] Property GitProxy configuration file > authorisedList > authorisedList items > project
@@ -258,7 +357,7 @@ description: JSON schema reference documentation for GitProxy
- 13.1.2. [Required] Property GitProxy configuration file > authorisedList > authorisedList items > name + 14.1.2. [Required] Property GitProxy configuration file > authorisedList > authorisedList items > name
@@ -272,7 +371,7 @@ description: JSON schema reference documentation for GitProxy
- 13.1.3. [Required] Property GitProxy configuration file > authorisedList > authorisedList items > url + 14.1.3. [Required] Property GitProxy configuration file > authorisedList > authorisedList items > url
@@ -289,7 +388,7 @@ description: JSON schema reference documentation for GitProxy
- 14. [Optional] Property GitProxy configuration file > sink + 15. [Optional] Property GitProxy configuration file > sink
@@ -304,7 +403,7 @@ description: JSON schema reference documentation for GitProxy | ------------------------------- | ----------- | | [database](#sink_items) | - | -### 14.1. GitProxy configuration file > sink > database +### 15.1. GitProxy configuration file > sink > database | | | | ------------------------- | ------------------------------------------------------------------------- | @@ -315,7 +414,7 @@ description: JSON schema reference documentation for GitProxy
- 14.1.1. [Required] Property GitProxy configuration file > sink > sink items > type + 15.1.1. [Required] Property GitProxy configuration file > sink > sink items > type
@@ -329,7 +428,7 @@ description: JSON schema reference documentation for GitProxy
- 14.1.2. [Required] Property GitProxy configuration file > sink > sink items > enabled + 15.1.2. [Required] Property GitProxy configuration file > sink > sink items > enabled
@@ -343,7 +442,7 @@ description: JSON schema reference documentation for GitProxy
- 14.1.3. [Optional] Property GitProxy configuration file > sink > sink items > connectionString + 15.1.3. [Optional] Property GitProxy configuration file > sink > sink items > connectionString
@@ -357,7 +456,7 @@ description: JSON schema reference documentation for GitProxy
- 14.1.4. [Optional] Property GitProxy configuration file > sink > sink items > options + 15.1.4. [Optional] Property GitProxy configuration file > sink > sink items > options
@@ -372,7 +471,7 @@ description: JSON schema reference documentation for GitProxy
- 14.1.5. [Optional] Property GitProxy configuration file > sink > sink items > params + 15.1.5. [Optional] Property GitProxy configuration file > sink > sink items > params
@@ -390,7 +489,7 @@ description: JSON schema reference documentation for GitProxy
- 15. [Optional] Property GitProxy configuration file > authentication + 16. [Optional] Property GitProxy configuration file > authentication
@@ -405,7 +504,7 @@ description: JSON schema reference documentation for GitProxy | --------------------------------------- | ----------- | | [authentication](#authentication_items) | - | -### 15.1. GitProxy configuration file > authentication > authentication +### 16.1. GitProxy configuration file > authentication > authentication | | | | ------------------------- | ------------------------------------------------------------------------- | @@ -416,7 +515,7 @@ description: JSON schema reference documentation for GitProxy
- 15.1.1. [Required] Property GitProxy configuration file > authentication > authentication items > type + 16.1.1. [Required] Property GitProxy configuration file > authentication > authentication items > type
@@ -430,7 +529,7 @@ description: JSON schema reference documentation for GitProxy
- 15.1.2. [Required] Property GitProxy configuration file > authentication > authentication items > enabled + 16.1.2. [Required] Property GitProxy configuration file > authentication > authentication items > enabled
@@ -444,7 +543,7 @@ description: JSON schema reference documentation for GitProxy
- 15.1.3. [Optional] Property GitProxy configuration file > authentication > authentication items > options + 16.1.3. [Optional] Property GitProxy configuration file > authentication > authentication items > options
@@ -462,7 +561,7 @@ description: JSON schema reference documentation for GitProxy
- 16. [Optional] Property GitProxy configuration file > tempPassword + 17. [Optional] Property GitProxy configuration file > tempPassword
@@ -476,7 +575,7 @@ description: JSON schema reference documentation for GitProxy
- 16.1. [Optional] Property GitProxy configuration file > tempPassword > sendEmail + 17.1. [Optional] Property GitProxy configuration file > tempPassword > sendEmail
@@ -490,7 +589,7 @@ description: JSON schema reference documentation for GitProxy
- 16.2. [Optional] Property GitProxy configuration file > tempPassword > emailConfig + 17.2. [Optional] Property GitProxy configuration file > tempPassword > emailConfig
@@ -508,5 +607,6 @@ description: JSON schema reference documentation for GitProxy
----------------------------------------------------------------------------------------------------------------------------- +--- + Generated using [json-schema-for-humans](https://github.com/coveooss/json-schema-for-humans) on 2024-10-22 at 16:45:32 +0100 From a73338d4e51f5343c984c7aa558d66a755892ef1 Mon Sep 17 00:00:00 2001 From: Denis Coric Date: Tue, 8 Apr 2025 21:32:36 +0200 Subject: [PATCH 2/2] fix: rebased to latest main --- index.ts | 4 +- src/config/ConfigLoader.js | 115 ++++++++++++++++--- src/config/index.ts | 187 ++++++++++++++----------------- src/proxy/actions/autoActions.ts | 5 +- src/proxy/chain.ts | 14 ++- src/proxy/index.js | 107 ------------------ src/proxy/index.ts | 109 ++++++++++++------ test/chain.test.js | 2 +- 8 files changed, 272 insertions(+), 271 deletions(-) delete mode 100644 src/proxy/index.js diff --git a/index.ts b/index.ts index 880ccfe02..2d458cbad 100755 --- a/index.ts +++ b/index.ts @@ -4,7 +4,7 @@ import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; import * as fs from 'fs'; import { configFile, setConfigFile, validate } from './src/config/file'; -import proxy from './src/proxy'; +import * as proxy from './src/proxy'; import service from './src/service'; const argv = yargs(hideBin(process.argv)) @@ -28,7 +28,7 @@ const argv = yargs(hideBin(process.argv)) .strict() .parseSync(); -setConfigFile(argv.c as string || ""); +setConfigFile((argv.c as string) || ''); if (argv.v) { if (!fs.existsSync(configFile)) { diff --git a/src/config/ConfigLoader.js b/src/config/ConfigLoader.js index 38816decb..df297fe7b 100644 --- a/src/config/ConfigLoader.js +++ b/src/config/ConfigLoader.js @@ -65,21 +65,29 @@ class ConfigLoader extends EventEmitter { if (!fs.existsSync(this.cacheDir)) { try { fs.mkdirSync(this.cacheDir, { recursive: true }); + console.log(`Created cache directory at ${this.cacheDir}`); return true; } catch (err) { console.error('Failed to create cache directory:', err); return false; } } + console.log(`Using cache directory at ${this.cacheDir}`); return true; } async start() { const { configurationSources } = this.config; if (!configurationSources?.enabled) { + console.log('Configuration sources are disabled'); return; } + console.log('Configuration sources are enabled'); + console.log( + `Sources: ${JSON.stringify(configurationSources.sources.filter((s) => s.enabled).map((s) => s.type))}`, + ); + // Clear any existing interval before starting a new one if (this.reloadTimer) { clearInterval(this.reloadTimer); @@ -88,6 +96,9 @@ class ConfigLoader extends EventEmitter { // Start periodic reload if interval is set if (configurationSources.reloadIntervalSeconds > 0) { + console.log( + `Setting reload interval to ${configurationSources.reloadIntervalSeconds} seconds`, + ); this.reloadTimer = setInterval( () => this.reloadConfiguration(), configurationSources.reloadIntervalSeconds * 1000, @@ -106,34 +117,63 @@ class ConfigLoader extends EventEmitter { } async reloadConfiguration() { - if (this.isReloading) return; + if (this.isReloading) { + console.log('Configuration reload already in progress, skipping'); + return; + } this.isReloading = true; + console.log('Starting configuration reload'); try { const { configurationSources } = this.config; - if (!configurationSources?.enabled) return; + if (!configurationSources?.enabled) { + console.log('Configuration sources are disabled, skipping reload'); + return; + } + + const enabledSources = configurationSources.sources.filter((source) => source.enabled); + console.log(`Found ${enabledSources.length} enabled configuration sources`); const configs = await Promise.all( - configurationSources.sources - .filter((source) => source.enabled) - .map((source) => this.loadFromSource(source)), + enabledSources.map(async (source) => { + try { + console.log(`Loading configuration from ${source.type} source`); + return await this.loadFromSource(source); + } catch (error) { + console.error(`Error loading from ${source.type} source:`, error.message); + return null; + } + }), ); + // Filter out null results from failed loads + const validConfigs = configs.filter((config) => config !== null); + + if (validConfigs.length === 0) { + console.log('No valid configurations loaded from any source'); + return; + } + // Use merge strategy based on configuration const shouldMerge = configurationSources.merge ?? true; // Default to true for backward compatibility + console.log(`Using ${shouldMerge ? 'merge' : 'override'} strategy for configuration`); + const newConfig = shouldMerge - ? configs.reduce( + ? validConfigs.reduce( (acc, curr) => { return this.deepMerge(acc, curr); }, { ...this.config }, ) - : { ...this.config, ...configs[configs.length - 1] }; // Use last config for override + : { ...this.config, ...validConfigs[validConfigs.length - 1] }; // Use last config for override // Emit change event if config changed if (JSON.stringify(newConfig) !== JSON.stringify(this.config)) { + console.log('Configuration has changed, updating and emitting change event'); this.config = newConfig; this.emit('configurationChanged', this.config); + } else { + console.log('Configuration has not changed, no update needed'); } } catch (error) { console.error('Error reloading configuration:', error); @@ -161,11 +201,13 @@ class ConfigLoader extends EventEmitter { if (!isValidPath(configPath)) { throw new Error('Invalid configuration file path'); } + console.log(`Loading configuration from file: ${configPath}`); const content = await fs.promises.readFile(configPath, 'utf8'); return JSON.parse(content); } async loadFromHttp(source) { + console.log(`Loading configuration from HTTP: ${source.url}`); const headers = { ...source.headers, ...(source.auth?.type === 'bearer' ? { Authorization: `Bearer ${source.auth.token}` } : {}), @@ -176,6 +218,8 @@ class ConfigLoader extends EventEmitter { } async loadFromGit(source) { + console.log(`Loading configuration from Git: ${source.repository}`); + // Validate inputs if (!source.repository || !isValidGitUrl(source.repository)) { throw new Error('Invalid repository URL format'); @@ -191,15 +235,25 @@ class ConfigLoader extends EventEmitter { if (!isValidPath(tempDir)) { throw new Error('Invalid temporary directory path'); } + + console.log(`Creating git cache directory at ${tempDir}`); await fs.promises.mkdir(tempDir, { recursive: true }); - const repoDir = path.join(tempDir, Buffer.from(source.repository).toString('base64')); + // Create a safe directory name from the repository URL + const repoDirName = Buffer.from(source.repository) + .toString('base64') + .replace(/[^a-zA-Z0-9]/g, '_'); + const repoDir = path.join(tempDir, repoDirName); + if (!isValidPath(repoDir)) { throw new Error('Invalid repository directory path'); } + console.log(`Using repository directory: ${repoDir}`); + // Clone or pull repository if (!fs.existsSync(repoDir)) { + console.log(`Cloning repository ${source.repository} to ${repoDir}`); const execOptions = { cwd: process.cwd(), env: { @@ -211,14 +265,35 @@ class ConfigLoader extends EventEmitter { : {}), }, }; - await execFileAsync('git', ['clone', source.repository, repoDir], execOptions); + + try { + await execFileAsync('git', ['clone', source.repository, repoDir], execOptions); + console.log('Repository cloned successfully'); + } catch (error) { + console.error('Failed to clone repository:', error.message); + throw new Error(`Failed to clone repository: ${error.message}`); + } } else { - await execFileAsync('git', ['pull'], { cwd: repoDir }); + console.log(`Pulling latest changes from ${source.repository}`); + try { + await execFileAsync('git', ['pull'], { cwd: repoDir }); + console.log('Repository pulled successfully'); + } catch (error) { + console.error('Failed to pull repository:', error.message); + throw new Error(`Failed to pull repository: ${error.message}`); + } } // Checkout specific branch if specified if (source.branch) { - await execFileAsync('git', ['checkout', source.branch], { cwd: repoDir }); + console.log(`Checking out branch: ${source.branch}`); + try { + await execFileAsync('git', ['checkout', source.branch], { cwd: repoDir }); + console.log(`Branch ${source.branch} checked out successfully`); + } catch (error) { + console.error(`Failed to checkout branch ${source.branch}:`, error.message); + throw new Error(`Failed to checkout branch ${source.branch}: ${error.message}`); + } } // Read and parse config file @@ -226,8 +301,21 @@ class ConfigLoader extends EventEmitter { if (!isValidPath(configPath)) { throw new Error('Invalid configuration file path in repository'); } - const content = await fs.promises.readFile(configPath, 'utf8'); - return JSON.parse(content); + + console.log(`Reading configuration file: ${configPath}`); + if (!fs.existsSync(configPath)) { + throw new Error(`Configuration file not found at ${configPath}`); + } + + try { + const content = await fs.promises.readFile(configPath, 'utf8'); + const config = JSON.parse(content); + console.log('Configuration loaded successfully from Git'); + return config; + } catch (error) { + console.error('Failed to read or parse configuration file:', error.message); + throw new Error(`Failed to read or parse configuration file: ${error.message}`); + } } deepMerge(target, source) { @@ -249,6 +337,7 @@ class ConfigLoader extends EventEmitter { } } +// Helper function to check if a value is an object function isObject(item) { return item && typeof item === 'object' && !Array.isArray(item); } diff --git a/src/config/index.ts b/src/config/index.ts index a3a6cf471..d0a3ddab9 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -1,12 +1,16 @@ import { existsSync, readFileSync } from 'fs'; -const fs = require('fs'); const ConfigLoader = require('./ConfigLoader'); const { validate } = require('./file'); // Import the validate function import defaultSettings from '../../proxy.config.json'; import { configFile } from './file'; -import { Authentication, AuthorisedRepo, Database, TempPasswordConfig, UserSettings } from './types'; - +import { + Authentication, + AuthorisedRepo, + Database, + TempPasswordConfig, + UserSettings, +} from './types'; let _userSettings: UserSettings | null = null; if (existsSync(configFile)) { @@ -37,6 +41,9 @@ export const getProxyUrl = () => { if (_userSettings !== null && _userSettings.proxyUrl) { _proxyUrl = _userSettings.proxyUrl; } + return _proxyUrl; +}; + // Initialize configuration with defaults and user settings let _config = { ...defaultSettings, ...(_userSettings || {}) }; @@ -49,9 +56,6 @@ export const getAuthorisedList = () => { _authorisedList = _userSettings.authorisedList; } return _authorisedList; -// Helper function to get current config value -const getConfig = (key) => { - return _config[key]; }; // Gets a list of authorised repositories @@ -69,15 +73,8 @@ export const getDatabase = () => { _database = _userSettings.sink; } for (const ix in _database) { -// Update existing getter functions to use the new config object -const getProxyUrl = () => getConfig('proxyUrl'); -const getAuthorisedList = () => getConfig('authorisedList'); -const getTempPasswordConfig = () => getConfig('tempPassword'); -const getDatabase = () => { - const sinks = getConfig('sink'); - for (const ix in sinks) { if (ix) { - const db = sinks[ix]; + const db = _database[ix]; if (db.enabled) { return db; } @@ -92,11 +89,8 @@ export const getAuthentication = () => { _authentication = _userSettings.authentication; } for (const ix in _authentication) { -const getAuthentication = () => { - const auths = getConfig('authentication'); - for (const ix in auths) { if (!ix) continue; - const auth = auths[ix]; + const auth = _authentication[ix]; if (auth.enabled) { return auth; } @@ -104,93 +98,52 @@ const getAuthentication = () => { throw Error('No authentication configured!'); }; -const getAPIs = () => getConfig('api'); -const getCookieSecret = () => getConfig('cookieSecret'); -const getSessionMaxAgeHours = () => getConfig('sessionMaxAgeHours'); -const getCommitConfig = () => getConfig('commitConfig'); -const getAttestationConfig = () => getConfig('attestationConfig'); -const getPrivateOrganizations = () => getConfig('privateOrganizations'); -const getURLShortener = () => getConfig('urlShortener'); -const getContactEmail = () => getConfig('contactEmail'); -const getCSRFProtection = () => getConfig('csrfProtection'); -const getPlugins = () => getConfig('plugins'); -const getSSLKeyPath = () => getConfig('sslKeyPemPath') || '../../certs/key.pem'; -const getSSLCertPath = () => getConfig('sslCertPemPath') || '../../certs/cert.pem'; -const getDomains = () => getConfig('domains'); - -// Log configuration to console -export const logConfiguration = () => { - console.log(`authorisedList = ${JSON.stringify(getAuthorisedList())}`); - console.log(`data sink = ${JSON.stringify(getDatabase())}`); - console.log(`authentication = ${JSON.stringify(getAuthentication())}`); -}; - +// Get API configuration export const getAPIs = () => { if (_userSettings && _userSettings.api) { _api = _userSettings.api; } return _api; }; -// Function to handle configuration updates -const handleConfigUpdate = async (newConfig) => { - console.log('Configuration updated from external source'); - try { - // 1. Get proxy module dynamically to avoid circular dependency - const proxy = require('../proxy'); +// Get cookie secret export const getCookieSecret = () => { if (_userSettings && _userSettings.cookieSecret) { _cookieSecret = _userSettings.cookieSecret; } return _cookieSecret; }; - // 2. Stop existing services - await proxy.stop(); +// Get session max age hours export const getSessionMaxAgeHours = () => { if (_userSettings && _userSettings.sessionMaxAgeHours) { _sessionMaxAgeHours = _userSettings.sessionMaxAgeHours; } return _sessionMaxAgeHours; }; - // 3. Update config - _config = newConfig; -// Get commit related configuration +// Get commit config export const getCommitConfig = () => { if (_userSettings && _userSettings.commitConfig) { _commitConfig = _userSettings.commitConfig; } return _commitConfig; }; - // 4. Validate new configuration - validate(); -// Get attestation related configuration +// Get attestation config export const getAttestationConfig = () => { if (_userSettings && _userSettings.attestationConfig) { _attestationConfig = _userSettings.attestationConfig; } return _attestationConfig; }; - // 5. Restart services with new config - await proxy.start(); -// Get private organizations related configuration +// Get private organizations export const getPrivateOrganizations = () => { if (_userSettings && _userSettings.privateOrganizations) { _privateOrganizations = _userSettings.privateOrganizations; - console.log('Services restarted with new configuration'); - } catch (error) { - console.error('Failed to apply new configuration:', error); - // Attempt to restart with previous config - try { - const proxy = require('../proxy'); - await proxy.start(); - } catch (startError) { - console.error('Failed to restart services:', startError); - } } + return _privateOrganizations; }; // Get URL shortener @@ -200,40 +153,32 @@ export const getURLShortener = () => { } return _urlShortener; }; -// Handle configuration updates -configLoader.on('configurationChanged', handleConfigUpdate); -// Get contact e-mail address +// Get contact email export const getContactEmail = () => { if (_userSettings && _userSettings.contactEmail) { _contactEmail = _userSettings.contactEmail; } return _contactEmail; }; -configLoader.on('configurationError', (error) => { - console.error('Error loading external configuration:', error); -}); -// Get CSRF protection flag +// Get CSRF protection export const getCSRFProtection = () => { if (_userSettings && _userSettings.csrfProtection) { _csrfProtection = _userSettings.csrfProtection; } return _csrfProtection; }; -// Start the config loader if external sources are enabled -configLoader.start().catch((error) => { - console.error('Failed to start configuration loader:', error); -}); -// Get loadable push plugins +// Get plugins export const getPlugins = () => { if (_userSettings && _userSettings.plugins) { _plugins = _userSettings.plugins; } return _plugins; -} +}; +// Get SSL key path export const getSSLKeyPath = () => { if (_userSettings && _userSettings.sslKeyPemPath) { _sslKeyPath = _userSettings.sslKeyPemPath; @@ -242,11 +187,9 @@ export const getSSLKeyPath = () => { return '../../certs/key.pem'; } return _sslKeyPath; -// Force reload of configuration -const reloadConfiguration = async () => { - await configLoader.reloadConfiguration(); }; +// Get SSL cert path export const getSSLCertPath = () => { if (_userSettings && _userSettings.sslCertPemPath) { _sslCertPath = _userSettings.sslCertPemPath; @@ -257,6 +200,7 @@ export const getSSLCertPath = () => { return _sslCertPath; }; +// Get domains export const getDomains = () => { if (_userSettings && _userSettings.domains) { _domains = _userSettings.domains; @@ -264,24 +208,61 @@ export const getDomains = () => { return _domains; }; -// Export all the functions -exports.getAPIs = getAPIs; -exports.getProxyUrl = getProxyUrl; -exports.getAuthorisedList = getAuthorisedList; -exports.getDatabase = getDatabase; -exports.logConfiguration = logConfiguration; -exports.getAuthentication = getAuthentication; -exports.getTempPasswordConfig = getTempPasswordConfig; -exports.getCookieSecret = getCookieSecret; -exports.getSessionMaxAgeHours = getSessionMaxAgeHours; -exports.getCommitConfig = getCommitConfig; -exports.getAttestationConfig = getAttestationConfig; -exports.getPrivateOrganizations = getPrivateOrganizations; -exports.getURLShortener = getURLShortener; -exports.getContactEmail = getContactEmail; -exports.getCSRFProtection = getCSRFProtection; -exports.getPlugins = getPlugins; -exports.getSSLKeyPath = getSSLKeyPath; -exports.getSSLCertPath = getSSLCertPath; -exports.getDomains = getDomains; -exports.reloadConfiguration = reloadConfiguration; +// Log configuration to console +export const logConfiguration = () => { + console.log(`authorisedList = ${JSON.stringify(getAuthorisedList())}`); + console.log(`data sink = ${JSON.stringify(getDatabase())}`); + console.log(`authentication = ${JSON.stringify(getAuthentication())}`); +}; + +// Function to handle configuration updates +const handleConfigUpdate = async (newConfig: typeof _config) => { + console.log('Configuration updated from external source'); + try { + // 1. Get proxy module dynamically to avoid circular dependency + const proxy = require('../proxy'); + + // 2. Stop existing services + await proxy.stop(); + + // 3. Update config + _config = newConfig; + + // 4. Validate new configuration + validate(); + + // 5. Restart services with new config + await proxy.start(); + + console.log('Services restarted with new configuration'); + } catch (error) { + console.error('Failed to apply new configuration:', error); + // Attempt to restart with previous config + try { + const proxy = require('../proxy'); + await proxy.start(); + } catch (startError) { + console.error('Failed to restart services:', startError); + } + } +}; + +// Handle configuration updates +configLoader.on('configurationChanged', handleConfigUpdate); + +configLoader.on('configurationError', (error: Error) => { + console.error('Error loading external configuration:', error); +}); + +// Start the config loader if external sources are enabled +configLoader.start().catch((error: Error) => { + console.error('Failed to start configuration loader:', error); +}); + +// Force reload of configuration +const reloadConfiguration = async () => { + await configLoader.reloadConfiguration(); +}; + +// Export reloadConfiguration +export { reloadConfiguration }; diff --git a/src/proxy/actions/autoActions.ts b/src/proxy/actions/autoActions.ts index 03ed0529a..450c97d80 100644 --- a/src/proxy/actions/autoActions.ts +++ b/src/proxy/actions/autoActions.ts @@ -33,7 +33,4 @@ const attemptAutoRejection = async (action: Action) => { } }; -export { - attemptAutoApproval, - attemptAutoRejection, -}; +export { attemptAutoApproval, attemptAutoRejection }; diff --git a/src/proxy/chain.ts b/src/proxy/chain.ts index 41a7cc495..53ca3827b 100644 --- a/src/proxy/chain.ts +++ b/src/proxy/chain.ts @@ -19,11 +19,13 @@ const pushActionChain: ((req: any, action: Action) => Promise)[] = [ proc.push.blockForAuth, ]; -const pullActionChain: ((req: any, action: Action) => Promise)[] = [proc.push.checkRepoInAuthorisedList]; +const pullActionChain: ((req: any, action: Action) => Promise)[] = [ + proc.push.checkRepoInAuthorisedList, +]; let pluginsInserted = false; -export const executeChain = async (req: any, res: any): Promise => { +const executeChain = async (req: any, res: any): Promise => { let action: Action = {} as Action; try { action = await proc.pre.parseAction(req); @@ -52,12 +54,14 @@ export const executeChain = async (req: any, res: any): Promise => { }; /** - * The plugin loader used for the GitProxy chain. + * The plugin loader used for GitProxy chain. * @type {import('../plugin').PluginLoader} */ let chainPluginLoader: PluginLoader; -const getChain = async (action: Action): Promise<((req: any, action: Action) => Promise)[]> => { +const getChain = async ( + action: Action, +): Promise<((req: any, action: Action) => Promise)[]> => { if (chainPluginLoader === undefined) { console.error( 'Plugin loader was not initialized! This is an application error. Please report it to the GitProxy maintainers. Skipping plugins...', @@ -105,3 +109,5 @@ export default { executeChain, getChain, }; + +export { executeChain, getChain, chainPluginLoader, pushActionChain, pullActionChain }; diff --git a/src/proxy/index.js b/src/proxy/index.js deleted file mode 100644 index 47540b3da..000000000 --- a/src/proxy/index.js +++ /dev/null @@ -1,107 +0,0 @@ -const express = require('express'); -const bodyParser = require('body-parser'); -const http = require('http'); -const https = require('https'); -const fs = require('fs'); -const path = require('path'); -const router = require('./routes').router; -const config = require('../config'); -const db = require('../db'); -const { PluginLoader } = require('../plugin'); -const chain = require('./chain'); -const { GIT_PROXY_SERVER_PORT: proxyHttpPort } = require('../config/env').Vars; -const { GIT_PROXY_HTTPS_SERVER_PORT: proxyHttpsPort } = require('../config/env').Vars; - -const options = { - inflate: true, - limit: '100000kb', - type: '*/*', - key: fs.readFileSync(path.join(__dirname, config.getSSLKeyPath())), - cert: fs.readFileSync(path.join(__dirname, config.getSSLCertPath())), -}; - -const proxyPreparations = async () => { - const plugins = config.getPlugins(); - const pluginLoader = new PluginLoader(plugins); - await pluginLoader.load(); - chain.chainPluginLoader = pluginLoader; - // Check to see if the default repos are in the repo list - const defaultAuthorisedRepoList = config.getAuthorisedList(); - const allowedList = await db.getRepos(); - - defaultAuthorisedRepoList.forEach(async (x) => { - const found = allowedList.find((y) => y.project === x.project && x.name === y.name); - if (!found) { - await db.createRepo(x); - await db.addUserCanPush(x.name, 'admin'); - await db.addUserCanAuthorise(x.name, 'admin'); - } - }); -}; - -// just keep this async incase it needs async stuff in the future -const createApp = async () => { - const app = express(); - // Setup the proxy middleware - app.use(bodyParser.raw(options)); - app.use('/', router); - return app; -}; - -let httpServer = null; -let httpsServer = null; - -const start = async () => { - const app = await createApp(); - await proxyPreparations(); - - // Start HTTP server - httpServer = http.createServer(options, app); - httpServer.listen(proxyHttpPort, () => { - console.log(`HTTP Proxy Listening on ${proxyHttpPort}`); - }); - - // Start HTTPS server if SSL certificates exist - const sslKeyPath = config.getSSLKeyPath(); - const sslCertPath = config.getSSLCertPath(); - - if (fs.existsSync(sslKeyPath) && fs.existsSync(sslCertPath)) { - httpsServer = https.createServer(options, app); - httpsServer.listen(proxyHttpsPort, () => { - console.log(`HTTPS Proxy Listening on ${proxyHttpsPort}`); - }); - } - - return app; -}; - -const stop = () => { - return new Promise((resolve, reject) => { - try { - // Close HTTP server if it exists - if (httpServer) { - httpServer.close(() => { - console.log('HTTP server closed'); - httpServer = null; - }); - } - - // Close HTTPS server if it exists - if (httpsServer) { - httpsServer.close(() => { - console.log('HTTPS server closed'); - httpsServer = null; - }); - } - - resolve(); - } catch (error) { - reject(error); - } - }); -}; - -module.exports.proxyPreparations = proxyPreparations; -module.exports.createApp = createApp; -module.exports.start = start; -module.exports.stop = stop; diff --git a/src/proxy/index.ts b/src/proxy/index.ts index 89a0977b3..b32678ceb 100644 --- a/src/proxy/index.ts +++ b/src/proxy/index.ts @@ -1,58 +1,56 @@ -import express from 'express'; +import express, { Application } from 'express'; import bodyParser from 'body-parser'; import http from 'http'; import https from 'https'; import fs from 'fs'; import path from 'path'; import { router } from './routes'; -import { - getAuthorisedList, - getPlugins, - getSSLCertPath, - getSSLKeyPath -} from '../config'; -import { - addUserCanAuthorise, - addUserCanPush, - createRepo, - getRepos -} from '../db'; +import * as config from '../config'; +import * as db from '../db'; import { PluginLoader } from '../plugin'; import chain from './chain'; -import { Repo } from '../db/types'; +import { serverConfig } from '../config/env'; -const { GIT_PROXY_SERVER_PORT: proxyHttpPort, GIT_PROXY_HTTPS_SERVER_PORT: proxyHttpsPort } = - require('../config/env').serverConfig; +const proxyHttpPort = serverConfig.GIT_PROXY_SERVER_PORT; +const proxyHttpsPort = serverConfig.GIT_PROXY_HTTPS_SERVER_PORT; -const options = { +interface ServerOptions { + inflate: boolean; + limit: string; + type: string; + key: Buffer; + cert: Buffer; +} + +const options: ServerOptions = { inflate: true, limit: '100000kb', type: '*/*', - key: fs.readFileSync(path.join(__dirname, getSSLKeyPath())), - cert: fs.readFileSync(path.join(__dirname, getSSLCertPath())), + key: fs.readFileSync(path.join(__dirname, config.getSSLKeyPath())), + cert: fs.readFileSync(path.join(__dirname, config.getSSLCertPath())), }; -const proxyPreparations = async () => { - const plugins = getPlugins(); +const proxyPreparations = async (): Promise => { + const plugins = config.getPlugins(); const pluginLoader = new PluginLoader(plugins); await pluginLoader.load(); chain.chainPluginLoader = pluginLoader; // Check to see if the default repos are in the repo list - const defaultAuthorisedRepoList = getAuthorisedList(); - const allowedList: Repo[] = await getRepos(); + const defaultAuthorisedRepoList = config.getAuthorisedList(); + const allowedList = await db.getRepos(); defaultAuthorisedRepoList.forEach(async (x) => { - const found = allowedList.find((y) => y.project === x.project && x.name === y.name); + const found = allowedList.find((y: any) => y.project === x.project && x.name === y.name); if (!found) { - await createRepo(x); - await addUserCanPush(x.name, 'admin'); - await addUserCanAuthorise(x.name, 'admin'); + await db.createRepo(x); + await db.addUserCanPush(x.name, 'admin'); + await db.addUserCanAuthorise(x.name, 'admin'); } }); }; // just keep this async incase it needs async stuff in the future -const createApp = async () => { +const createApp = async (): Promise => { const app = express(); // Setup the proxy middleware app.use(bodyParser.raw(options)); @@ -60,21 +58,58 @@ const createApp = async () => { return app; }; -const start = async () => { +let httpServer: http.Server | null = null; +let httpsServer: https.Server | null = null; + +const start = async (): Promise => { const app = await createApp(); await proxyPreparations(); - http.createServer(options as any, app).listen(proxyHttpPort, () => { + + // Start HTTP server + // @ts-expect-error - The options type is compatible with what http.createServer expects + httpServer = http.createServer(options, app); + httpServer.listen(proxyHttpPort, () => { console.log(`HTTP Proxy Listening on ${proxyHttpPort}`); }); - https.createServer(options, app).listen(proxyHttpsPort, () => { - console.log(`HTTPS Proxy Listening on ${proxyHttpsPort}`); - }); + + // Start HTTPS server if SSL certificates exist + const sslKeyPath = config.getSSLKeyPath(); + const sslCertPath = config.getSSLCertPath(); + + if (fs.existsSync(sslKeyPath) && fs.existsSync(sslCertPath)) { + httpsServer = https.createServer(options, app); + httpsServer.listen(proxyHttpsPort, () => { + console.log(`HTTPS Proxy Listening on ${proxyHttpsPort}`); + }); + } return app; }; -export default { - proxyPreparations, - createApp, - start +const stop = (): Promise => { + return new Promise((resolve, reject) => { + try { + // Close HTTP server if it exists + if (httpServer) { + httpServer.close(() => { + console.log('HTTP server closed'); + httpServer = null; + }); + } + + // Close HTTPS server if it exists + if (httpsServer) { + httpsServer.close(() => { + console.log('HTTPS server closed'); + httpsServer = null; + }); + } + + resolve(); + } catch (error) { + reject(error); + } + }); }; + +export { proxyPreparations, createApp, start, stop }; diff --git a/test/chain.test.js b/test/chain.test.js index 9913ac1ed..0315c3c3b 100644 --- a/test/chain.test.js +++ b/test/chain.test.js @@ -79,7 +79,7 @@ describe('proxy chain', function () { sandboxSinon.stub(processors, 'push').value(mockPushProcessors); // Re-require the chain module after stubbing processors - chain = require('../src/proxy/chain'); + chain = require('../src/proxy/chain').default; chain.chainPluginLoader = new PluginLoader([]); });