From b1a87fb4d0b65bcc720a3531459f1acc750a70ca Mon Sep 17 00:00:00 2001 From: jim Date: Sun, 25 Aug 2019 16:02:26 +0200 Subject: [PATCH] initial commit --- .env.example | 5 + .gitignore | 6 + .php_cs.dist | 18 ++ README.md | 32 ++++ cache/.gitignore | 2 + composer.json | 14 ++ docs | 1 + public/.htaccess | 6 + public/functions.php | 380 +++++++++++++++++++++++++++++++++++++++++++ public/index.php | 45 +++++ 10 files changed, 509 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 .php_cs.dist create mode 100644 README.md create mode 100644 cache/.gitignore create mode 100644 composer.json create mode 120000 docs create mode 100644 public/.htaccess create mode 100644 public/functions.php create mode 100644 public/index.php diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..c92aec9 --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +ENDPOINT=https://gitlab.example.com/api/v4/ +API_KEY=123456789ABCDEFGHIJKL +METHOD=ssh +PORT=22 +INCLUDE_BRANCHES=true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..11a0c21 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/confs/static-repos.json +/vendor +/composer.lock +/.idea +/.php_cs.cache +/.env \ No newline at end of file diff --git a/.php_cs.dist b/.php_cs.dist new file mode 100644 index 0000000..3f2608b --- /dev/null +++ b/.php_cs.dist @@ -0,0 +1,18 @@ +in(__DIR__) + ->name('*.php') + ->exclude('vendor') + ->ignoreDotFiles(true) + ->ignoreVCS(true); + +return Config::create() + ->setRules([ + '@PSR2' => true, + 'array_syntax' => ['syntax' => 'short'], + ]) + ->setFinder($finder); diff --git a/README.md b/README.md new file mode 100644 index 0000000..6c4700c --- /dev/null +++ b/README.md @@ -0,0 +1,32 @@ +# Gitlab NPM repository + +Small script that loops through all branches and tags of all projects in a Gitlab installation +and if it contains a `package.json`, and has a non empty `name` it is added. + +This script is not a full repository like npmjs.com, but only implements the install command. + +## Installation + + 1. Run `composer install` + 2. Copy `.env.example` to `.env` + 3. Get a "personal access token" with scopes: `api,read_repository` + 4. For some reason npm sends `/` like `%2F`, so you need to set `AllowEncodedSlashes NoDecode` inside your `` + +## Usage + +Simply include a package.json in your project, all branches and tags using +[semver](https://semver.org/) will be detected. "v1.2.3" will be converted to "1.2.3" + +To use your repository, use this command to install the packages: +``` +npm install project-name --registry http://npm.gitlab.localhost/ +``` + +## Warning + +This script could allow access to private repositories. Because I haven't figured out the +correct way for npm authorization, the current .htaccess restricts to local usage. +So be sure to update the .htaccess to match your requirements. + +## Author + * [Maglr](https://github.com/maglr) diff --git a/cache/.gitignore b/cache/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/cache/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..759c23c --- /dev/null +++ b/composer.json @@ -0,0 +1,14 @@ +{ + "require": { + "m4tthumphrey/php-gitlab-api": "^9.0", + "php-http/guzzle6-adapter": "^1.1", + "vlucas/phpdotenv": "^3.4", + "ext-json": "*", + "ext-curl": "*" + }, + "scripts": { + "post-install-cmd": [ + "chmod 777 cache || true" + ] + } +} diff --git a/docs b/docs new file mode 120000 index 0000000..d298be1 --- /dev/null +++ b/docs @@ -0,0 +1 @@ +public/ \ No newline at end of file diff --git a/public/.htaccess b/public/.htaccess new file mode 100644 index 0000000..d59a037 --- /dev/null +++ b/public/.htaccess @@ -0,0 +1,6 @@ +DirectoryIndex /index.php +FallbackResource /index.php + +Require ip ::1 +Require ip 127.0.0.1 +Require ip 192.168.1 diff --git a/public/functions.php b/public/functions.php new file mode 100644 index 0000000..6bc1582 --- /dev/null +++ b/public/functions.php @@ -0,0 +1,380 @@ + 'registry', + 'doc_count' => 0, + ]; + outputJson($data); +} + +function handleSinglePackage($packageName) +{ + $client = getClient(); + $repos = $client->repositories(); + $allProjects = getAllProjects($client); + foreach ($allProjects as $project) { + if (($package = loadData($project, $repos)) && ($package_name = getPackageName($project, $repos))) { + if ($packageName === $package_name) { + $latestTag = reset($package); // TODO: newest should not always be the "latest" + $return = [ + 'name' => $packageName, + 'description' => @$latestTag['description'], + 'dist-tags' => ['latest' => $latestTag['version']], + 'versions' => $package, + ]; + outputJson($return); + break; + } + } + } + + header('Location: https://registry.npmjs.org/'.$packageName); // redirect everything i dont know to npmjs.org + die(); +} + +/** + * @return Gitlab\Client + */ +function getClient() +{ + return Gitlab\Client::create(getenv('ENDPOINT'))->authenticate(getenv('API_KEY'), Gitlab\Client::AUTH_URL_TOKEN); +} + +function getAllProjects($client) +{ + $projects = $client->projects(); + + // Load projects + $all_projects = []; + + // We have to get all accessible projects + for ($page = 1; count($p = $projects->all(['page' => $page, 'per_page' => 100])); $page++) { + foreach ($p as $project) { + $all_projects[] = $project; + } + } + return $all_projects; +} + +function getMaxMtime($allProjects) +{ + $mtime = 0; + foreach ($allProjects as $project) { + $mtime = max($mtime, strtotime($project['last_activity_at'])); + } + + return $mtime; +} + +function handlePackageList() +{ + $client = getClient(); + $allProjects = getAllProjects($client); + $repos = $client->repositories(); + + // Regenerate packages_file is needed + if (!file_exists(PACKAGES_FILE) || filemtime(PACKAGES_FILE) < getMaxMtime($allProjects)) { + $packages = []; + foreach ($allProjects as $project) { + if (($package = loadData($project, $repos)) && ($package_name = getPackageName($project, $repos))) { + $packages[$package_name] = $package; + } + } + + $data = json_encode([ + 'packages' => array_filter($packages), + ], JSON_PRETTY_PRINT); + + file_put_contents(PACKAGES_FILE, $data); + @chmod(0777, PACKAGES_FILE); + } + + outputFile(PACKAGES_FILE); +} + + + +/** + * @param $file + * Output a json file, sending max-age header, then dies + */ +function outputFile($file) +{ + $mtime = filemtime($file); + + header('Content-Type: application/json'); + header('Last-Modified: ' . gmdate('r', $mtime)); + header('Cache-Control: max-age=0'); + + if (!empty($_SERVER['HTTP_IF_MODIFIED_SINCE']) && ($since = strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE'])) && $since >= $mtime) { + header('HTTP/1.0 304 Not Modified'); + } else { + readfile($file); + } + die(); +} + +function outputJson($data) +{ + header('Content-Type: application/json'); + header('Cache-Control: max-age=0'); + echo json_encode($data, JSON_PRETTY_PRINT); + die(); +} + +/** + * Retrieves some information about a project's package.json + * + * @param array $project + * @param string $ref commit id + * @param Gitlab\Api\Repositories $repos + * @return array|false + */ +function fetchPackage($project, $ref, $repos) +{ + try { + $c = $repos->getFile($project['id'], 'package.json', $ref); + + if (!isset($c['content'])) { + return false; + } + + return json_decode(base64_decode($c['content']), true); + } catch (Gitlab\Exception\RuntimeException $e) { + return false; + } +} + +/** + * Retrieves some information about a project for a specific ref + * + * @param array $project + * @param array $ref + * @param Gitlab\Api\Repositories $repos + * @return array [$version => ['name' => $name, 'version' => $version, 'source' => [...]]] + */ +function fetchRef($project, $ref, $repos) +{ + static $ref_cache = []; + + $ref_key = md5(serialize($project) . serialize($ref)); + + if (isset($ref_cache[$ref_key])) { + return $ref_cache[$ref_key]; + } + $version = checkVersion($ref['name']); + $data = fetchPackage($project, $ref['commit']['id'], $repos); + + if ($data !== false) { + $data['version'] = $version; + $url = $project[METHOD . '_url_to_repo']; + if (METHOD == 'ssh' && PORT != '') { + $url = 'ssh://' . strstr($project['ssh_url_to_repo'], ':', true); + $url .= ':' . PORT . '/' . $project['path_with_namespace']; + } + $data['repository'] = [ + 'type' => 'git', + 'url' => $url, + 'reference' => $ref['commit']['id'], + ]; + + $path = '/projects/' . $project['id'] . '/repository/archive.tgz?sha=' . $ref['commit']['id']; + $data['dist']['tarball'] = $_SERVER['REQUEST_SCHEME'] . '://' . $_SERVER['HTTP_HOST'] . '/gitlab/proxy?path=' . urlencode($path); + + $ref_cache[$ref_key] = [$version => $data]; + return $ref_cache[$ref_key]; + } + $ref_cache[$ref_key] = []; + + return $ref_cache[$ref_key]; +} + +/** + * Transforms v1.0.0 -> 1.0.0 + * @param $tagOrBranch + * @return string + */ +function checkVersion($tagOrBranch) +{ + if (validateVersion($tagOrBranch)) { + return $tagOrBranch; + } + $trimmed = ltrim($tagOrBranch, 'v'); + if (validateVersion($trimmed)) { + return $trimmed; + } + + return $tagOrBranch; +} + +/** + * Validates if we are using semver + * @param $tagOrBranch + * @return false|int + */ +function validateVersion($tagOrBranch) +{ + // official regex from: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + $regex = '/^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/'; + return preg_match($regex, $tagOrBranch); +} + + +/** + * Retrieves some information about a project for all refs + * @param array $project + * @param Gitlab\Api\Repositories $repos + * @return array Same as $fetch_ref, but for all refs + */ +function fetchRefs($project, $repos) +{ + $return = []; + try { + $refs = $repos->tags($project['id']); + + if (getenv('INCLUDE_BRANCHES')) { // branches are optional + $branches = $repos->branches($project['id']); + $refs = array_merge($refs, $branches); + } + + foreach ($refs as $ref) { + foreach (fetchRef($project, $ref, $repos) as $version => $data) { + if (validateVersion($version)) { + $return[$version] = $data; + } + } + } + } catch (Gitlab\Exception\RuntimeException $e) { + // The repo has no commits — skipping it. + } + + return $return; +}; + +/** + * Caching layer on top of $fetch_refs + * Uses last_activity_at from the $project array, so no invalidation is needed + * + * @param array $project + * @param Gitlab\Api\Repositories $repos + * @return array Same as $fetch_refs + */ +function loadData($project, $repos) +{ + $file = __DIR__ . "/../cache/{$project['path_with_namespace']}.json"; + $mtime = strtotime($project['last_activity_at']); + + if (!is_dir(dirname($file))) { + mkdir(dirname($file), 0777, true); + } + + if (file_exists($file) && filemtime($file) >= $mtime) { + if (filesize($file) > 0) { + return json_decode(file_get_contents($file), true); + } else { + return false; + } + } elseif ($data = fetchRefs($project, $repos)) { + file_put_contents($file, json_encode($data, JSON_PRETTY_PRINT)); + touch($file, $mtime); + @chmod(0777, $file); + + return $data; + } else { + $f = fopen($file, 'w'); + fclose($f); + touch($file, $mtime); + @chmod(0777, $file); + + return false; + } +} + +/** + * Determine the name to use for the package. + * + * @param array $project + * @param Gitlab\Api\Repositories $repos + * @return string The name of the project + */ +function getPackageName($project, $repos) +{ + $ref = fetchRef($project, $repos->branch($project['id'], $project['default_branch']), $repos); + $first = reset($ref); + if (!empty($first['name'])) { + return $first['name']; + } + + return false; + //return $project['path_with_namespace']; +} + +/** + * Clear the cache folder if the .env is newer than the packages.json file + * @param string $cache_folder + * @param string $config_file + * @param string $packages_file + * @return bool + */ +function clearCacheOnConfigChange($cache_folder, $config_file, $packages_file) +{ + if (!is_dir($cache_folder)) { + die('cache folder: '.$cache_folder.' does not exist'); + } + if (!is_writable($cache_folder)) { + die('cache folder: '.$cache_folder.' is not writable'); + } + if (!file_exists($packages_file)) { + return false; + } + if (filemtime($config_file) < filemtime($packages_file)) { + return false; + } + + return clearCache($cache_folder); +} + +function clearCache($cacheFolder) +{ + if (!is_dir($cacheFolder) || strlen($cacheFolder) < 20) { + die('clear_cache_on_config_change safety check failed'); + } + shell_exec('rm -rf '.$cacheFolder.'/*'); + return true; +} diff --git a/public/index.php b/public/index.php new file mode 100644 index 0000000..acbea66 --- /dev/null +++ b/public/index.php @@ -0,0 +1,45 @@ +load(); + +define('PORT', intval(getenv('PORT'))); + +$validMethods = ['ssh', 'http']; +if (getenv('METHOD') && in_array(getenv('METHOD'), $validMethods)) { + define('METHOD', getenv('METHOD')); +} else { + define('METHOD', 'ssh'); +} + +clearCacheOnConfigChange($cacheFolder, __DIR__.'/../.env', PACKAGES_FILE); +//clearCache($cacheFolder); // when debugging always clear cache + +$parts = array_filter(explode('/', trim($_SERVER['REQUEST_URI'], '/'))); + +if (empty($_SERVER['REQUEST_URI'])) { + die('something went wrong'); +} +if ($_SERVER['REQUEST_URI'] === '/') { + handleOverview(); +} +if ($parts[0] === 'packages.json') { + handlePackageList(); +} + +if (!empty($parts[1])) { + if (empty($_GET['path'])) { + http_response_code(500); + die('invalid request'); + } + handleProxy(getenv('ENDPOINT').$_GET['path']); +} else { + handleSinglePackage(trim($_SERVER['REQUEST_URI'], '/')); +}