Skip to content

Commit 4a49edb

Browse files
reggireggi
authored
feat: preparation for v11 & gh api rate limiting (#1402)
1 parent 110d75c commit 4a49edb

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

101 files changed

+1202
-190
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -40,5 +40,6 @@
4040
!/test/
4141
!/tsconfig.json
4242
!/webpack.config.js
43+
!cli-cache.json
4344
tap-testdir*/
4445
!/cli/

‎.prettierignore

+1
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@
55
/.reuse/
66
.nyc_output/
77
coverage/
8+
cli-cache.json

‎cli-cache.json

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"v11": "cf52b8be2645ee098ee83ea9981fc32a11932fad",
3+
"v9": "64763a341e7aa5b456e696f956759bf9b3440dc1",
4+
"v10": "a3041941586b6fb8ed7403fe3c24d81138a96005",
5+
"v8": "aa8fff11cdab94fff1a2160ee5241f5f4632e96b"
6+
}

‎cli/bin/build.js

+15-12
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ const {resolve, relative, join} = require('path')
22
const {spawnSync} = require('child_process')
33
const build = require('../lib/build.js')
44
const {nwo} = require('../lib/gh')
5+
const {CacheVersionSha} = require('../lib/cache.js')
56

67
// check only build with the current versions instead of checking the registry
78
// and also fails if any changes are detected. this is used in CI to make sure
@@ -24,21 +25,23 @@ const checkContent = () => {
2425
}
2526
}
2627

27-
build({
28-
releases: require('../releases.json'),
29-
loglevel: process.argv.includes('--debug') || process.env.CI ? 'verbose' : 'info',
30-
prerelease: false,
31-
useCurrent: checkOnly,
32-
contentPath,
33-
navPath,
34-
})
35-
.then(() => {
28+
;(async () => {
29+
try {
30+
await build({
31+
cache: await CacheVersionSha.load(join(ROOT, 'cli-cache.json')),
32+
releases: require('../releases.json'),
33+
loglevel: process.argv.includes('--debug') || process.env.CI ? 'verbose' : 'info',
34+
prerelease: false,
35+
useCurrent: checkOnly,
36+
contentPath,
37+
navPath,
38+
})
3639
if (checkOnly) {
3740
checkContent()
3841
}
3942
return console.log('DONE')
40-
})
41-
.catch(e => {
43+
} catch (e) {
4244
console.error(e)
4345
process.exit(1)
44-
})
46+
}
47+
})()

‎cli/lib/build.js

+4-2
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ const getCurrentVersions = nav => {
5555
}
5656
}
5757

58-
const main = async ({loglevel, releases: rawReleases, useCurrent, navPath, contentPath, prerelease}) => {
58+
const main = async ({loglevel, releases: rawReleases, useCurrent, navPath, contentPath, prerelease, cache}) => {
5959
/* istanbul ignore next */
6060
if (loglevel) {
6161
log.on(loglevel)
@@ -114,9 +114,11 @@ const main = async ({loglevel, releases: rawReleases, useCurrent, navPath, conte
114114
})
115115

116116
const updates = await Promise.all(
117-
releases.map(r => extractRelease(r, {contentPath, baseNav: navData, prerelease})),
117+
releases.map(r => extractRelease(r, {cache, contentPath, baseNav: navData, prerelease})),
118118
).then(r => r.filter(Boolean))
119119

120+
await cache?.save()
121+
120122
await updateNav(updates, {nav: navDoc, path: navPath})
121123
}
122124

‎cli/lib/cache.js

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
const fs = require('fs/promises')
2+
3+
/** cache npm cli version shas to NOT pull down changes we already have */
4+
class CacheVersionSha {
5+
constructor(cache, path) {
6+
this.cache = cache
7+
this.path = path
8+
}
9+
10+
static async load(path) {
11+
return new CacheVersionSha(JSON.parse(await fs.readFile(path, 'utf-8')), path)
12+
}
13+
14+
async save() {
15+
await fs.writeFile(this.path, JSON.stringify(this.cache, null, 2))
16+
return this
17+
}
18+
19+
set(id, sha) {
20+
this.cache[id] = sha
21+
return this
22+
}
23+
24+
same(id, value) {
25+
return this.cache[id] === value
26+
}
27+
}
28+
29+
module.exports = {
30+
CacheVersionSha,
31+
}

‎cli/lib/extract.js

+10-1
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,16 @@ const writeChangelog = async ({release, nav, cwd, srcPath, contentPath}) => {
138138
})
139139
}
140140

141-
const unpackRelease = async (release, {contentPath, baseNav, prerelease = false}) => {
141+
const unpackRelease = async (release, {cache, contentPath, baseNav, prerelease = false}) => {
142+
if (cache) {
143+
const sha = await gh.getCurrentSha(release.branch)
144+
if (cache.same(release.id, sha)) {
145+
log.info(`Skipping ${release.id} due to cache`)
146+
return
147+
}
148+
cache.set(release.id, sha)
149+
}
150+
142151
if (release.prerelease && !prerelease) {
143152
log.info(`Skipping ${release.id} due to prerelease ${release.version}`)
144153
return

‎cli/lib/gh.js

+20-1
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,32 @@
11
const {posix, sep} = require('node:path')
2+
const {execSync} = require('node:child_process')
23

34
if (!process.env.GITHUB_TOKEN) {
4-
throw new Error('GITHUB_TOKEN env var is required to build CLI docs')
5+
try {
6+
// this allows people to run this locally
7+
process.env.GITHUB_TOKEN = execSync('gh auth token', {encoding: 'utf8'}).trim()
8+
} catch (err) {
9+
throw new Error('GITHUB_TOKEN env var is required to build CLI docs')
10+
}
511
}
612

713
let octokit
814
const owner = 'npm'
915
const repo = 'cli'
1016
const opts = {owner, repo}
1117

18+
const getCurrentSha = async branch => {
19+
if (!octokit) {
20+
const {Octokit} = await import('@octokit/rest')
21+
octokit = new Octokit({auth: process.env.GITHUB_TOKEN})
22+
}
23+
const {data} = await octokit.repos.getBranch({
24+
...opts,
25+
branch,
26+
})
27+
return data.commit.sha
28+
}
29+
1230
const getFile = async ({sha, ref, path}) => {
1331
if (!octokit) {
1432
const {Octokit} = await import('@octokit/rest')
@@ -51,5 +69,6 @@ const pathExists = async (ref, path) => {
5169
module.exports = {
5270
getFile,
5371
pathExists,
72+
getCurrentSha,
5473
nwo: `${owner}/${repo}`,
5574
}

‎cli/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
},
4141
"templateOSS": {
4242
"//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.",
43-
"version": "4.23.6",
43+
"version": "4.23.5",
4444
"content": "./scripts/template-oss"
4545
},
4646
"files": [

‎cli/releases.json

+4-8
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,4 @@
11
[
2-
{
3-
"id": "v6",
4-
"branch": "release/v6"
5-
},
6-
{
7-
"id": "v7",
8-
"branch": "release/v7"
9-
},
102
{
113
"id": "v8",
124
"branch": "release/v8"
@@ -17,6 +9,10 @@
179
},
1810
{
1911
"id": "v10",
12+
"branch": "release/v10"
13+
},
14+
{
15+
"id": "v11",
2016
"branch": "latest"
2117
}
2218
]

‎cli/test/index.js

+5
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ const mockBuild = async (t, {releases = getReleases(), packument = {}, testdir:
6666
return yaml.stringify(children).replace(new RegExp(`/cli/${id}/`, 'g'), '/')
6767
}
6868

69+
let shaCounter = 0
6970
const build = t.mockRequire('../lib/build', {
7071
pacote: {
7172
...pacote,
@@ -83,6 +84,10 @@ const mockBuild = async (t, {releases = getReleases(), packument = {}, testdir:
8384
},
8485
'@prettier/sync': {format: s => s},
8586
'../lib/gh.js': {
87+
getCurrentSha: async () => {
88+
shaCounter = shaCounter + 1
89+
return 'abc' + shaCounter
90+
},
8691
getFile: async ({ref}) => navSection(ref),
8792
pathExists: async (ref, p) => {
8893
if (ref.includes('v6') && p.includes('docs/lib/content')) {

‎content/cli/v10/commands/index.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
title: CLI Commands
33
shortName: Commands
44
github_repo: npm/cli
5-
github_branch: latest
5+
github_branch: release/v10
66
github_path: docs/lib/content/nav.yml
77
redirect_from:
88
- /cli-commands

‎content/cli/v10/commands/npm-access.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-access
33
section: 1
44
description: Set access level on published packages
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-access.md
88
redirect_from:
99
- /cli-commands/access

‎content/cli/v10/commands/npm-adduser.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-adduser
33
section: 1
44
description: Add a registry user account
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-adduser.md
88
redirect_from:
99
- /cli-commands/adduser

‎content/cli/v10/commands/npm-audit.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-audit
33
section: 1
44
description: Run a security audit
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-audit.md
88
redirect_from:
99
- /cli-commands/audit

‎content/cli/v10/commands/npm-bugs.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-bugs
33
section: 1
44
description: Report bugs for a package in a web browser
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-bugs.md
88
redirect_from:
99
- /cli-commands/bugs

‎content/cli/v10/commands/npm-cache.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-cache
33
section: 1
44
description: Manipulates packages cache
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-cache.md
88
redirect_from:
99
- /cli-commands/cache

‎content/cli/v10/commands/npm-ci.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-ci
33
section: 1
44
description: Clean install a project
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-ci.md
88
redirect_from:
99
- /cli-commands/ci

‎content/cli/v10/commands/npm-completion.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-completion
33
section: 1
44
description: Tab Completion for npm
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-completion.md
88
redirect_from:
99
- /cli-commands/completion

‎content/cli/v10/commands/npm-config.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-config
33
section: 1
44
description: Manage the npm configuration files
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-config.md
88
redirect_from:
99
- /cli-commands/config

‎content/cli/v10/commands/npm-dedupe.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-dedupe
33
section: 1
44
description: Reduce duplication in the package tree
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-dedupe.md
88
redirect_from:
99
- /cli-commands/dedupe

‎content/cli/v10/commands/npm-deprecate.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-deprecate
33
section: 1
44
description: Deprecate a version of a package
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-deprecate.md
88
redirect_from:
99
- /cli-commands/deprecate

‎content/cli/v10/commands/npm-diff.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-diff
33
section: 1
44
description: The registry diff command
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-diff.md
88
redirect_from:
99
- /cli-commands/diff

‎content/cli/v10/commands/npm-dist-tag.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-dist-tag
33
section: 1
44
description: Modify package distribution tags
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-dist-tag.md
88
redirect_from:
99
- /cli-commands/dist-tag

‎content/cli/v10/commands/npm-docs.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-docs
33
section: 1
44
description: Open documentation for a package in a web browser
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-docs.md
88
redirect_from:
99
- /cli-commands/docs

‎content/cli/v10/commands/npm-doctor.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-doctor
33
section: 1
44
description: Check the health of your npm environment
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-doctor.md
88
redirect_from:
99
- /cli-commands/doctor

‎content/cli/v10/commands/npm-edit.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-edit
33
section: 1
44
description: Edit an installed package
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-edit.md
88
redirect_from:
99
- /cli-commands/edit

‎content/cli/v10/commands/npm-exec.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-exec
33
section: 1
44
description: Run a command from a local or remote npm package
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-exec.md
88
redirect_from:
99
- /cli-commands/exec

‎content/cli/v10/commands/npm-explain.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-explain
33
section: 1
44
description: Explain installed packages
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-explain.md
88
redirect_from:
99
- /cli-commands/explain

‎content/cli/v10/commands/npm-explore.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ title: npm-explore
33
section: 1
44
description: Browse an installed package
55
github_repo: npm/cli
6-
github_branch: latest
6+
github_branch: release/v10
77
github_path: docs/lib/content/commands/npm-explore.md
88
redirect_from:
99
- /cli-commands/explore

0 commit comments

Comments
 (0)