From cdf1123135145c14ebcf1679c90814a7b6b81376 Mon Sep 17 00:00:00 2001 From: Kevin Heis Date: Fri, 13 Jan 2023 08:42:00 -0800 Subject: [PATCH] Remove ENABLED_LANGUAGES (#33766) --- .github/workflows/link-check-on-pr.yml | 5 ----- .github/workflows/test.yml | 4 ---- Dockerfile | 6 ------ azure-preview-env-template.json | 4 ---- contributing/development.md | 2 +- docker-compose.staging.tmpl.yaml | 1 - lib/languages.js | 25 +++++++++++++------------ package.json | 4 ++-- script/search/index-elasticsearch.js | 18 +++++++++--------- tests/translations/api-search.js | 7 +++++-- 10 files changed, 30 insertions(+), 46 deletions(-) diff --git a/.github/workflows/link-check-on-pr.yml b/.github/workflows/link-check-on-pr.yml index 8efb2ca526e6..eff427361d27 100644 --- a/.github/workflows/link-check-on-pr.yml +++ b/.github/workflows/link-check-on-pr.yml @@ -86,11 +86,6 @@ jobs: SHOULD_COMMENT: ${{ secrets.DOCS_BOT_FR != '' }} CHECK_EXTERNAL_LINKS: false CREATE_REPORT: false - # Not strictly necessary bit it makes warmServer() a bit faster - # because it only bothers with English to begin with, which - # we're filtering on anyway once the list of all pages has - # been loaded. - ENABLED_LANGUAGES: en run: node .github/actions-scripts/rendered-content-link-checker.js - name: Upload artifact(s) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 277265806d4f..925039d04853 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -168,8 +168,4 @@ jobs: env: DIFF_FILE: get_diff_files.txt CHANGELOG_CACHE_FILE_PATH: tests/fixtures/changelog-feed.json - # By default, when `process.env.NODE_ENV === 'test'` it forces the - # tests run only in English. The exception is the - # `tests/translations/` suite which needs all languages to be set up. - ENABLED_LANGUAGES: ${{ matrix.test-group == 'translations' && 'all' || '' }} run: npm test -- tests/${{ matrix.test-group }}/ diff --git a/Dockerfile b/Dockerfile index 214c05df996d..24fe8f2f9911 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,8 +74,6 @@ ENV NODE_ENV production # Preferred port for server.js ENV PORT 4000 -ENV ENABLED_LANGUAGES "en" - # This makes it possible to set `--build-arg BUILD_SHA=abc123` # and it then becomes available as an environment variable in the docker run. ARG BUILD_SHA @@ -101,9 +99,5 @@ CMD ["node", "server.js"] # -------------------------------------------------------------------------------- FROM preview as production -# Override what was set for previews -# Make this match the default of `Object.keys(languages)` in lib/languages.js -ENV ENABLED_LANGUAGES "en,zh,ja,es,pt,de,fr,ru,ko" - # Copy in all translations COPY --chown=node:node translations ./translations diff --git a/azure-preview-env-template.json b/azure-preview-env-template.json index f63f318358de..a57aa01de73e 100644 --- a/azure-preview-env-template.json +++ b/azure-preview-env-template.json @@ -61,10 +61,6 @@ { "name": "WEB_CONCURRENCY", "value": "1" - }, - { - "name": "ENABLED_LANGUAGES", - "value": "en" } ], "resources": { diff --git a/contributing/development.md b/contributing/development.md index ee88816545ce..b872a7bd0c4c 100644 --- a/contributing/development.md +++ b/contributing/development.md @@ -42,7 +42,7 @@ The [`script/bookmarklets`](../script/bookmarklets) directory contains some brow ### Enabling different languages -By default the local server won't run with all supported languages enabled. If you need to run the server with a particular language, you can temporarily edit the `start` script in `package.json` and update the `ENABLED_LANGUAGES` variable. For example, to enable Japanese and Portuguese, you can set it to `ENABLED_LANGUAGES='en,ja,pt'` and then you need to restart the server for the change to take effect. +By default the local server won't run with all supported languages enabled. If you need to run the server with a particular language, you'll need to clone that language into your translations directory. The supported language codes are defined in [lib/languages.js](../lib/languages.js). diff --git a/docker-compose.staging.tmpl.yaml b/docker-compose.staging.tmpl.yaml index 0ca9b2d0fae1..42167ffe0ff7 100644 --- a/docker-compose.staging.tmpl.yaml +++ b/docker-compose.staging.tmpl.yaml @@ -14,7 +14,6 @@ services: HYDRO_SECRET: ${HYDRO_SECRET} HAYSTACK_URL: ${HAYSTACK_URL} HEROKU_APP_NAME: ${HEROKU_APP_NAME} - ENABLED_LANGUAGES: ${ENABLED_LANGUAGES} DEPLOYMENT_ENV: ${DEPLOYMENT_ENV} HEROKU_PRODUCTION_APP: true PORT: 4000 diff --git a/lib/languages.js b/lib/languages.js index 6d9168ac2824..b41089a47f49 100644 --- a/lib/languages.js +++ b/lib/languages.js @@ -6,6 +6,7 @@ import dotenv from 'dotenv' import { TRANSLATIONS_ROOT } from './constants.js' import path from 'path' +import fs from 'fs/promises' dotenv.config() @@ -36,7 +37,7 @@ function getRoot(languageCode) { // Languages in order of accept-language header frequency // 92BD1212-61B8-4E7A: Remove `wip: Boolean` for the public ship of ko, fr, de, ru -const languages = { +let languages = { en: { name: 'English', code: 'en', @@ -115,18 +116,18 @@ const languages = { } if (process.env.ENABLED_LANGUAGES) { - if (process.env.ENABLED_LANGUAGES.toLowerCase() !== 'all') { - Object.keys(languages).forEach((code) => { - if (!process.env.ENABLED_LANGUAGES.includes(code)) delete languages[code] - }) - // This makes the translation health report not valid JSON - // console.log(`ENABLED_LANGUAGES: ${process.env.ENABLED_LANGUAGES}`) + languages = Object.fromEntries( + process.env.ENABLED_LANGUAGES.split(',').map((code) => [code, languages[code]]) + ) +} + +for (const language of Object.values(languages)) { + if (language.code === 'en') continue + try { + await fs.readdir(language.dir) + } catch (err) { + delete languages[language.code] } -} else if (process.env.NODE_ENV === 'test') { - // Unless explicitly set, when running tests default to just English - Object.keys(languages).forEach((code) => { - if (code !== 'en') delete languages[code] - }) } export const languageKeys = Object.keys(languages) diff --git a/package.json b/package.json index 7398bf619628..43e874506514 100644 --- a/package.json +++ b/package.json @@ -180,7 +180,7 @@ "scripts": { "browser-test": "cross-env BROWSER=1 NODE_OPTIONS=--experimental-vm-modules JEST_PUPPETEER_CONFIG=jest-puppeteer.config.cjs jest tests/browser/browser.js", "build": "next build", - "debug": "cross-env NODE_ENV=development ENABLED_LANGUAGES=en nodemon --inspect server.js", + "debug": "cross-env NODE_ENV=development nodemon --inspect server.js", "dev": "cross-env npm start", "index-test-fixtures": "node script/search/index-elasticsearch.js -l en -l ja -V ghae -V dotcom --index-prefix tests -- tests/content/fixtures/search-indexes", "lint": "eslint '**/*.{js,mjs,ts,tsx}'", @@ -191,7 +191,7 @@ "prevent-pushes-to-main": "node script/prevent-pushes-to-main.js", "rest-dev": "script/rest/update-files.js && npm run dev", "show-action-deps": "echo 'Action Dependencies:' && rg '^[\\s|-]*(uses:.*)$' .github -I -N --no-heading -r '$1$2' | sort | uniq | cut -c 7-", - "start": "cross-env NODE_ENV=development ENABLED_LANGUAGES=en nodemon server.js", + "start": "cross-env NODE_ENV=development nodemon server.js", "start-all-languages": "cross-env NODE_ENV=development nodemon server.js", "sync-search": "cross-env NODE_OPTIONS='--max_old_space_size=8192' start-server-and-test sync-search-server 4002 sync-search-indices", "sync-search-ghes-release": "cross-env GHES_RELEASE=1 start-server-and-test sync-search-server 4002 sync-search-indices", diff --git a/script/search/index-elasticsearch.js b/script/search/index-elasticsearch.js index e289961a4d15..5e7e503574ad 100755 --- a/script/search/index-elasticsearch.js +++ b/script/search/index-elasticsearch.js @@ -53,12 +53,8 @@ program .description('Creates Elasticsearch index from records') .option('-v, --verbose', 'Verbose outputs') .addOption(new Option('-V, --version [VERSION...]', 'Specific versions').choices(allVersionKeys)) - .addOption( - new Option('-l, --language ', 'Which languages to focus on').choices(languageKeys) - ) - .addOption( - new Option('--not-language ', 'Specific language to omit').choices(languageKeys) - ) + .addOption(new Option('-l, --language ', 'Which languages to focus on')) + .addOption(new Option('--not-language ', 'Specific language to omit')) .option('-u, --elasticsearch-url ', 'If different from $ELASTICSEARCH_URL') .option('-p, --index-prefix ', 'Index string to put before index name') .argument('', 'where the indexable files are') @@ -413,9 +409,13 @@ function escapeHTML(content) { } async function loadRecords(indexName, sourceDirectory) { - const filePath = path.join(sourceDirectory, `${indexName}-records.json`) - const payload = await fs.readFile(filePath) - return JSON.parse(payload) + try { + const filePath = path.join(sourceDirectory, `${indexName}-records.json`) + const payload = await fs.readFile(filePath) + return JSON.parse(payload) + } catch (err) { + throw new Error(`No records named ${indexName}-records.json, or not valid format.`) + } } function getSnowballLanguage(language) { diff --git a/tests/translations/api-search.js b/tests/translations/api-search.js index 7d7ec393669c..efbeaf97317f 100644 --- a/tests/translations/api-search.js +++ b/tests/translations/api-search.js @@ -2,18 +2,21 @@ import { jest, test, expect } from '@jest/globals' import { describeIfElasticsearchURL } from '../helpers/conditional-runs.js' import { get } from '../helpers/e2etest.js' +import { languageKeys } from '../../lib/languages.js' + +const langs = languageKeys.filter((lang) => lang !== 'en').filter((lang) => lang === 'ja') // temporary: only has japanese fixture so far // This suite only runs if $ELASTICSEARCH_URL is set. describeIfElasticsearchURL('search v1 middleware in non-English', () => { jest.setTimeout(60 * 1000) - test('basic search in Japanese', async () => { + test.each(langs)('basic search in %s', async (lang) => { const sp = new URLSearchParams() // To see why this will work, // see tests/content/fixtures/search-indexes/github-docs-dotcom-en-records.json // which clearly has a record with the title "Foo" sp.set('query', 'foo') - sp.set('language', 'ja') + sp.set('language', lang) const res = await get('/api/search/v1?' + sp) expect(res.statusCode).toBe(200) const results = JSON.parse(res.text)