From f7be7f0129573742033745cbc8c71b28a3cd20aa Mon Sep 17 00:00:00 2001 From: Peter Bengtsson Date: Fri, 13 Jan 2023 18:52:48 +0100 Subject: [PATCH] Revert "Remove ENABLED_LANGUAGES" (#33953) --- .github/workflows/link-check-on-pr.yml | 5 +++++ .github/workflows/test.yml | 4 ++++ Dockerfile | 6 ++++++ azure-preview-env-template.json | 4 ++++ contributing/development.md | 2 +- docker-compose.staging.tmpl.yaml | 1 + lib/languages.js | 25 ++++++++++++------------- package.json | 4 ++-- script/search/index-elasticsearch.js | 18 +++++++++--------- tests/translations/api-search.js | 7 ++----- 10 files changed, 46 insertions(+), 30 deletions(-) diff --git a/.github/workflows/link-check-on-pr.yml b/.github/workflows/link-check-on-pr.yml index eff427361d27..8efb2ca526e6 100644 --- a/.github/workflows/link-check-on-pr.yml +++ b/.github/workflows/link-check-on-pr.yml @@ -86,6 +86,11 @@ jobs: SHOULD_COMMENT: ${{ secrets.DOCS_BOT_FR != '' }} CHECK_EXTERNAL_LINKS: false CREATE_REPORT: false + # Not strictly necessary bit it makes warmServer() a bit faster + # because it only bothers with English to begin with, which + # we're filtering on anyway once the list of all pages has + # been loaded. + ENABLED_LANGUAGES: en run: node .github/actions-scripts/rendered-content-link-checker.js - name: Upload artifact(s) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 925039d04853..277265806d4f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -168,4 +168,8 @@ jobs: env: DIFF_FILE: get_diff_files.txt CHANGELOG_CACHE_FILE_PATH: tests/fixtures/changelog-feed.json + # By default, when `process.env.NODE_ENV === 'test'` it forces the + # tests run only in English. The exception is the + # `tests/translations/` suite which needs all languages to be set up. + ENABLED_LANGUAGES: ${{ matrix.test-group == 'translations' && 'all' || '' }} run: npm test -- tests/${{ matrix.test-group }}/ diff --git a/Dockerfile b/Dockerfile index 24fe8f2f9911..214c05df996d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,6 +74,8 @@ ENV NODE_ENV production # Preferred port for server.js ENV PORT 4000 +ENV ENABLED_LANGUAGES "en" + # This makes it possible to set `--build-arg BUILD_SHA=abc123` # and it then becomes available as an environment variable in the docker run. ARG BUILD_SHA @@ -99,5 +101,9 @@ CMD ["node", "server.js"] # -------------------------------------------------------------------------------- FROM preview as production +# Override what was set for previews +# Make this match the default of `Object.keys(languages)` in lib/languages.js +ENV ENABLED_LANGUAGES "en,zh,ja,es,pt,de,fr,ru,ko" + # Copy in all translations COPY --chown=node:node translations ./translations diff --git a/azure-preview-env-template.json b/azure-preview-env-template.json index a57aa01de73e..f63f318358de 100644 --- a/azure-preview-env-template.json +++ b/azure-preview-env-template.json @@ -61,6 +61,10 @@ { "name": "WEB_CONCURRENCY", "value": "1" + }, + { + "name": "ENABLED_LANGUAGES", + "value": "en" } ], "resources": { diff --git a/contributing/development.md b/contributing/development.md index b872a7bd0c4c..ee88816545ce 100644 --- a/contributing/development.md +++ b/contributing/development.md @@ -42,7 +42,7 @@ The [`script/bookmarklets`](../script/bookmarklets) directory contains some brow ### Enabling different languages -By default the local server won't run with all supported languages enabled. If you need to run the server with a particular language, you'll need to clone that language into your translations directory. +By default the local server won't run with all supported languages enabled. If you need to run the server with a particular language, you can temporarily edit the `start` script in `package.json` and update the `ENABLED_LANGUAGES` variable. For example, to enable Japanese and Portuguese, you can set it to `ENABLED_LANGUAGES='en,ja,pt'` and then you need to restart the server for the change to take effect. The supported language codes are defined in [lib/languages.js](../lib/languages.js). diff --git a/docker-compose.staging.tmpl.yaml b/docker-compose.staging.tmpl.yaml index 42167ffe0ff7..0ca9b2d0fae1 100644 --- a/docker-compose.staging.tmpl.yaml +++ b/docker-compose.staging.tmpl.yaml @@ -14,6 +14,7 @@ services: HYDRO_SECRET: ${HYDRO_SECRET} HAYSTACK_URL: ${HAYSTACK_URL} HEROKU_APP_NAME: ${HEROKU_APP_NAME} + ENABLED_LANGUAGES: ${ENABLED_LANGUAGES} DEPLOYMENT_ENV: ${DEPLOYMENT_ENV} HEROKU_PRODUCTION_APP: true PORT: 4000 diff --git a/lib/languages.js b/lib/languages.js index b41089a47f49..6d9168ac2824 100644 --- a/lib/languages.js +++ b/lib/languages.js @@ -6,7 +6,6 @@ import dotenv from 'dotenv' import { TRANSLATIONS_ROOT } from './constants.js' import path from 'path' -import fs from 'fs/promises' dotenv.config() @@ -37,7 +36,7 @@ function getRoot(languageCode) { // Languages in order of accept-language header frequency // 92BD1212-61B8-4E7A: Remove `wip: Boolean` for the public ship of ko, fr, de, ru -let languages = { +const languages = { en: { name: 'English', code: 'en', @@ -116,18 +115,18 @@ let languages = { } if (process.env.ENABLED_LANGUAGES) { - languages = Object.fromEntries( - process.env.ENABLED_LANGUAGES.split(',').map((code) => [code, languages[code]]) - ) -} - -for (const language of Object.values(languages)) { - if (language.code === 'en') continue - try { - await fs.readdir(language.dir) - } catch (err) { - delete languages[language.code] + if (process.env.ENABLED_LANGUAGES.toLowerCase() !== 'all') { + Object.keys(languages).forEach((code) => { + if (!process.env.ENABLED_LANGUAGES.includes(code)) delete languages[code] + }) + // This makes the translation health report not valid JSON + // console.log(`ENABLED_LANGUAGES: ${process.env.ENABLED_LANGUAGES}`) } +} else if (process.env.NODE_ENV === 'test') { + // Unless explicitly set, when running tests default to just English + Object.keys(languages).forEach((code) => { + if (code !== 'en') delete languages[code] + }) } export const languageKeys = Object.keys(languages) diff --git a/package.json b/package.json index 43e874506514..7398bf619628 100644 --- a/package.json +++ b/package.json @@ -180,7 +180,7 @@ "scripts": { "browser-test": "cross-env BROWSER=1 NODE_OPTIONS=--experimental-vm-modules JEST_PUPPETEER_CONFIG=jest-puppeteer.config.cjs jest tests/browser/browser.js", "build": "next build", - "debug": "cross-env NODE_ENV=development nodemon --inspect server.js", + "debug": "cross-env NODE_ENV=development ENABLED_LANGUAGES=en nodemon --inspect server.js", "dev": "cross-env npm start", "index-test-fixtures": "node script/search/index-elasticsearch.js -l en -l ja -V ghae -V dotcom --index-prefix tests -- tests/content/fixtures/search-indexes", "lint": "eslint '**/*.{js,mjs,ts,tsx}'", @@ -191,7 +191,7 @@ "prevent-pushes-to-main": "node script/prevent-pushes-to-main.js", "rest-dev": "script/rest/update-files.js && npm run dev", "show-action-deps": "echo 'Action Dependencies:' && rg '^[\\s|-]*(uses:.*)$' .github -I -N --no-heading -r '$1$2' | sort | uniq | cut -c 7-", - "start": "cross-env NODE_ENV=development nodemon server.js", + "start": "cross-env NODE_ENV=development ENABLED_LANGUAGES=en nodemon server.js", "start-all-languages": "cross-env NODE_ENV=development nodemon server.js", "sync-search": "cross-env NODE_OPTIONS='--max_old_space_size=8192' start-server-and-test sync-search-server 4002 sync-search-indices", "sync-search-ghes-release": "cross-env GHES_RELEASE=1 start-server-and-test sync-search-server 4002 sync-search-indices", diff --git a/script/search/index-elasticsearch.js b/script/search/index-elasticsearch.js index 5e7e503574ad..e289961a4d15 100755 --- a/script/search/index-elasticsearch.js +++ b/script/search/index-elasticsearch.js @@ -53,8 +53,12 @@ program .description('Creates Elasticsearch index from records') .option('-v, --verbose', 'Verbose outputs') .addOption(new Option('-V, --version [VERSION...]', 'Specific versions').choices(allVersionKeys)) - .addOption(new Option('-l, --language ', 'Which languages to focus on')) - .addOption(new Option('--not-language ', 'Specific language to omit')) + .addOption( + new Option('-l, --language ', 'Which languages to focus on').choices(languageKeys) + ) + .addOption( + new Option('--not-language ', 'Specific language to omit').choices(languageKeys) + ) .option('-u, --elasticsearch-url ', 'If different from $ELASTICSEARCH_URL') .option('-p, --index-prefix ', 'Index string to put before index name') .argument('', 'where the indexable files are') @@ -409,13 +413,9 @@ function escapeHTML(content) { } async function loadRecords(indexName, sourceDirectory) { - try { - const filePath = path.join(sourceDirectory, `${indexName}-records.json`) - const payload = await fs.readFile(filePath) - return JSON.parse(payload) - } catch (err) { - throw new Error(`No records named ${indexName}-records.json, or not valid format.`) - } + const filePath = path.join(sourceDirectory, `${indexName}-records.json`) + const payload = await fs.readFile(filePath) + return JSON.parse(payload) } function getSnowballLanguage(language) { diff --git a/tests/translations/api-search.js b/tests/translations/api-search.js index efbeaf97317f..7d7ec393669c 100644 --- a/tests/translations/api-search.js +++ b/tests/translations/api-search.js @@ -2,21 +2,18 @@ import { jest, test, expect } from '@jest/globals' import { describeIfElasticsearchURL } from '../helpers/conditional-runs.js' import { get } from '../helpers/e2etest.js' -import { languageKeys } from '../../lib/languages.js' - -const langs = languageKeys.filter((lang) => lang !== 'en').filter((lang) => lang === 'ja') // temporary: only has japanese fixture so far // This suite only runs if $ELASTICSEARCH_URL is set. describeIfElasticsearchURL('search v1 middleware in non-English', () => { jest.setTimeout(60 * 1000) - test.each(langs)('basic search in %s', async (lang) => { + test('basic search in Japanese', async () => { const sp = new URLSearchParams() // To see why this will work, // see tests/content/fixtures/search-indexes/github-docs-dotcom-en-records.json // which clearly has a record with the title "Foo" sp.set('query', 'foo') - sp.set('language', lang) + sp.set('language', 'ja') const res = await get('/api/search/v1?' + sp) expect(res.statusCode).toBe(200) const results = JSON.parse(res.text)