diff --git a/apps/webapp/.eslintrc b/apps/webapp/.eslintrc index 9731fbdd6..96ee8ee86 100644 --- a/apps/webapp/.eslintrc +++ b/apps/webapp/.eslintrc @@ -18,7 +18,7 @@ "no-await-in-loop": "warn", "import/no-extraneous-dependencies": [ "error", - { "devDependencies": ["**/*.test.tsx", "**/*.spec.tsx", "**/*.test.ts", "**/*.spec.ts", "**/*.config.ts"] } + { "devDependencies": ["**/*.test.tsx", "**/*.spec.tsx", "**/*.test.ts", "**/*.spec.ts", "**/*.config.ts","**/fixtures.ts"] } ], "jsx-a11y/label-has-associated-control": [ "error", diff --git a/apps/webapp/package-lock.json b/apps/webapp/package-lock.json index 4d96c8261..707eed8c7 100644 --- a/apps/webapp/package-lock.json +++ b/apps/webapp/package-lock.json @@ -122,9 +122,9 @@ "@aws-sdk/types": "^3.425.0", "@next/bundle-analyzer": "^14.1.3", "@next/env": "^15.1.6", - "@playwright/test": "^1.50.0", - "@tailwindcss/forms": "^0.5.3", - "@tailwindcss/typography": "^0.5.9", + "@playwright/test": "^1.52.0", + "@tailwindcss/forms": "^0.5.10", + "@tailwindcss/typography": "^0.5.16", "@testing-library/dom": "^10.4.0", "@testing-library/react": "^16.2.0", "@types/d3": "^7.4.3", @@ -132,7 +132,7 @@ "@types/lodash": "^4.6.9", "@types/mjml": "^4.7.4", "@types/ms": "^0.7.31", - "@types/node": "^22.13.1", + "@types/node": "^22.14.0", "@types/node-gzip": "^1.1.3", "@types/nodemailer": "^6.4.9", "@types/pg": "^8.11.11", @@ -5636,13 +5636,13 @@ } }, "node_modules/@playwright/test": { - "version": "1.50.0", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.50.0.tgz", - "integrity": "sha512-ZGNXbt+d65EGjBORQHuYKj+XhCewlwpnSd/EDuLPZGSiEWmgOJB5RmMCCYGy5aMfTs9wx61RivfDKi8H/hcMvw==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.52.0.tgz", + "integrity": "sha512-uh6W7sb55hl7D6vsAeA+V2p5JnlAqzhqFyF0VcJkKZXkgnFcVG9PziERRHQfPLfNGx1C292a4JqbWzhR8L4R1g==", "devOptional": true, "license": "Apache-2.0", "dependencies": { - "playwright": "1.50.0" + "playwright": "1.52.0" }, "bin": { "playwright": "cli.js" @@ -9412,12 +9412,12 @@ } }, "node_modules/@types/node": { - "version": "22.13.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.1.tgz", - "integrity": "sha512-jK8uzQlrvXqEU91UxiK5J7pKHyzgnI1Qnl0QDHIgVGuolJhRb9EEl28Cj9b3rGR8B2lhFCtvIm5os8lFnO/1Ew==", + "version": "22.14.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.14.0.tgz", + "integrity": "sha512-Kmpl+z84ILoG+3T/zQFyAJsU6EPTmOCj8/2+83fSN6djd6I4o7uOuGIH6vq3PrjY5BGitSbFuMN18j3iknubbA==", "license": "MIT", "dependencies": { - "undici-types": "~6.20.0" + "undici-types": "~6.21.0" } }, "node_modules/@types/node-fetch": { @@ -21427,13 +21427,13 @@ } }, "node_modules/playwright": { - "version": "1.50.0", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.50.0.tgz", - "integrity": "sha512-+GinGfGTrd2IfX1TA4N2gNmeIksSb+IAe589ZH+FlmpV3MYTx6+buChGIuDLQwrGNCw2lWibqV50fU510N7S+w==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.52.0.tgz", + "integrity": "sha512-JAwMNMBlxJ2oD1kce4KPtMkDeKGHQstdpFPcPH3maElAXon/QZeTvtsfXmTMRyO9TslfoYOXkSsvao2nE1ilTw==", "devOptional": true, "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.50.0" + "playwright-core": "1.52.0" }, "bin": { "playwright": "cli.js" @@ -21446,9 +21446,9 @@ } }, "node_modules/playwright-core": { - "version": "1.50.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.50.0.tgz", - "integrity": "sha512-CXkSSlr4JaZs2tZHI40DsZUN/NIwgaUPsyLuOAaIZp2CyF2sN5MM5NJsyB188lFSSozFxQ5fPT4qM+f0tH/6wQ==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.52.0.tgz", + "integrity": "sha512-l2osTgLXSMeuLZOML9qYODUQoPPnUsKsb5/P6LJ2e6uPKXUdPK5WYhN4z03G+YNbWmGDY4YENauNu4ZKczreHg==", "devOptional": true, "license": "Apache-2.0", "bin": { @@ -26010,9 +26010,9 @@ } }, "node_modules/undici-types": { - "version": "6.20.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", - "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "license": "MIT" }, "node_modules/unified": { diff --git a/apps/webapp/package.json b/apps/webapp/package.json index b90dc537d..8accbdeff 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -152,9 +152,9 @@ "@aws-sdk/types": "^3.425.0", "@next/bundle-analyzer": "^14.1.3", "@next/env": "^15.1.6", - "@playwright/test": "^1.50.0", - "@tailwindcss/forms": "^0.5.3", - "@tailwindcss/typography": "^0.5.9", + "@playwright/test": "^1.52.0", + "@tailwindcss/forms": "^0.5.10", + "@tailwindcss/typography": "^0.5.16", "@testing-library/dom": "^10.4.0", "@testing-library/react": "^16.2.0", "@types/d3": "^7.4.3", @@ -162,7 +162,7 @@ "@types/lodash": "^4.6.9", "@types/mjml": "^4.7.4", "@types/ms": "^0.7.31", - "@types/node": "^22.13.1", + "@types/node": "^22.14.0", "@types/node-gzip": "^1.1.3", "@types/nodemailer": "^6.4.9", "@types/pg": "^8.11.11", diff --git a/apps/webapp/tests-playwright/fixtures.ts b/apps/webapp/tests-playwright/fixtures.ts new file mode 100644 index 000000000..69be593dd --- /dev/null +++ b/apps/webapp/tests-playwright/fixtures.ts @@ -0,0 +1,21 @@ +import { test as base } from '@playwright/test'; + +type MyFixtures = { + baseUrl: string; + gemmaUrl: string; + steerUrl: string; + searchUrl: string; +}; + +export const test = base.extend({ + baseUrl: ['http://localhost:3000', { option: true }], + gemmaUrl: async ({ baseUrl }, use) => { + await use(`${baseUrl}/gemma-scope`); + }, + steerUrl: async ({ baseUrl }, use) => { + await use(`${baseUrl}/steer`) + }, + searchUrl: async ({ baseUrl }, use) => { + await use(`${baseUrl}/search`); + } +}); \ No newline at end of file diff --git a/apps/webapp/tests-playwright/local/gemmascope.test.ts b/apps/webapp/tests-playwright/local/gemmascope.test.ts new file mode 100644 index 000000000..6148f2a85 --- /dev/null +++ b/apps/webapp/tests-playwright/local/gemmascope.test.ts @@ -0,0 +1,724 @@ +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; + +// http://localhost:3000/gemma-scope#main + +test('main page', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#main`); + + await page.locator('textarea[name="The inner workings"]').isVisible(); +}); + +test('microscope button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#main`); + + await page.getByText("scan Gemma 2's brain to see what it's thinking").click(); + await expect(page).toHaveURL(`${gemmaUrl}#microscope`); +}); + +test('analyze features button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#main`); + + await page.getByText('Make features fire and figure out what they do').click(); + await expect(page).toHaveURL(`${gemmaUrl}#analyze`); +}); + +test('steer gemma button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#main`); + + await page.getByText("Change Gemma's behavior by manipulating features").click(); + await expect(page).toHaveURL(`${gemmaUrl}#steer`); +}); + +test('do more button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#main`); + + await page.getByText('Dive deeper into the exciting world of AI interpretability').click(); + await expect(page).toHaveURL(`${gemmaUrl}#learn`); +}); + +test('browse & search saes button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#main`); + + await page.getByText('Directly explore the SAEs in Gemma Scope').click(); + await expect(page).toHaveURL(`${gemmaUrl}#browse`); +}); + +test('microscope page', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); + + await page.locator('textarea[name="To understand what AI is thinking"]').isVisible(); +}); + +test('demo button feeling lucky', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); + + await page.getByText("I'm Feeling Lucky").click(); + await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); +}); + +test('demo button food', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); + + await page.getByText('Food').click(); + await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); +}); + +test('demo button News', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); + + await page.getByText('News').click(); + await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); +}); + +test('demo button Literary', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); + + await page.getByText('Literary').click(); + await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); +}); + +test('demo button Personal', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); + + await page.getByText('Personal').click(); + await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); +}); + +test('demo button Programming', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); + + await page.getByText('Programming').click(); + await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); +}); + +test('challenge gemma', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); + + // challenge is unlocked after doing the demo, or pressing skip demo + await page.getByText('skip demo').first().click(); + + const gemmaSearch = page.getByPlaceholder('Hint: Try a short sentence instead of a word or phrase.'); + await gemmaSearch.fill('testing inputs'); + await gemmaSearch.press('Enter'); + await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); +}); + +test('next analyze features button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#microscope`); + + // next is unlocked after doing the demo, or pressing skip demo + await page.getByText('skip demo').nth(1).click(); + await page.getByText(' Next - Analyze Features').click(); + await expect(page).toHaveURL(`${gemmaUrl}#analyze`); +}); + +test('analyze features page', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); + + await page.locator('textarea[name="The features in Gemma Scope have labels"]').isVisible(); +}); + +test('olympic sports button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); + await page.getByText('Olympic sports').click(); + await expect(page.getByText('What do these activations (in green) have in common?')).toBeVisible(); +}); + +test('famous cities button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); + await page.getByText('famous cities').click(); + await expect(page.getByText('What do these activations (in green) have in common?')).toBeVisible(); +}); + +test('reference to animals button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); + await page.getByText('references to animals').click(); + await expect(page.getByText('Nice! This was tricky, because there are two possibly correct answers.')).toBeVisible(); +}); + +test('none of these button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); + await page.getByText('none of these').click(); + await expect(page.getByText('Nice! This was tricky, because there are two possibly correct answers.')).toBeVisible(); +}); + +test('analyze steer', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); + + // unlocked after doing the demo, or pressing skip demo + await page.getByText('skip example').first().click(); + const analyzeSteer = page.getByPlaceholder('Can you write a text snippet that activates it?'); + await analyzeSteer.fill('test'); + await analyzeSteer.press('Enter'); + + await expect(page.getByText('0.00')).toBeVisible(); +}); + + +test('next steer gemma button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); + + // unlocked after puzzles, or pressing skip analyze + await page.getByText('skip analyze').nth(1).click(); + await page.getByText(' Next - Steer Gemma').click(); + await expect(page).toHaveURL(`${gemmaUrl}#steer`); +}); + +// test('steer page', async ({ page }) => { +// await page.goto('https://neuronpedia.org/gemma-scope#steer'); + +// await page.locator('textarea[name="Let\'s put these features to use"]').isVisible(); +// }); + +// test('steer demo', async ({ page }) => { +// await page.goto('https://neuronpedia.org/gemma-scope#steer'); + +// // check for demo question +// await expect(page.getByText('What are you?').first()).toBeVisible(); +// }); + +// test('next do more button', async ({ page }) => { +// await page.goto('https://neuronpedia.org/gemma-scope#steer'); + +// await page.getByText('Next - Do More').click(); +// await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#learn'); +// }); + +test('do more page', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + await page.locator('textarea[name="This demo of Gemma Scope"]').isVisible(); +}); + +test('advanced steer button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + await page.locator('a[target="_blank"][href*="https://www.neuronpedia.org/steer"]').click(), + ]); + await expect(newPage).toHaveURL(/\/steer$/); +}); + +test('browse saes button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + await page.getByText('Browse SAEs').click(); + await expect(page).toHaveURL(`${gemmaUrl}#browse`); +}); + +// test('Contact Us', async({ page }) => { +// await page.goto(`${gemmaUrl}#learn`); + +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page.locator('a[target="_blank"][href*="mailto:johnny@neuronpedia\.org"]').click() +// ]); + +// await expect(newPage).toHaveURL(/mailto:johnny@neuronpedia\.org*/); +// }); + +test('deepmind blog post button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('DeepMind Blog Post').click()]); + + await expect(newPage).toHaveURL( + 'https://deepmind.google/discover/blog/gemma-scope-helping-the-safety-community-shed-light-on-the-inner-workings-of-language-models/', + ); +}); + +test('coding tutorial button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('Coding Tutorial').click()]); + + await expect(newPage).toHaveURL( + 'https://colab.research.google.com/drive/17dQFYUYnuKnP6OwQPH9v_GSYUW5aj-Rp?usp=sharing', + ); +}); + +// testing it the default method with expect().toHaveURL doesnt work, url ends up being blank +// only tests for the correct link being on the page +test('technical report button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + const href = await page.getByText('Technical Report').getAttribute('href'); + + expect(href).toContain('storage.googleapis.com/gemma-scope/gemma-scope-report.pdf'); +}); + +test('huggingface button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('HuggingFace').click()]); + + await expect(newPage).toHaveURL('https://huggingface.co/google/gemma-scope'); +}); + +test('get started with mech interp button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + await page.getByText('Get Started with Mech Interp').click(), + ]); + + await expect(newPage).toHaveURL('https://www.neelnanda.io/mechanistic-interpretability/getting-started'); +}); + +test('favourite mech interp papers button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + await page.getByText('Favourite Mech Interp Papers').click(), + ]); + + await expect(newPage).toHaveURL( + 'https://www.alignmentforum.org/posts/NfFST5Mio7BCAQHPA/an-extremely-opinionated-annotated-list-of-my-favourite-1', + ); +}); + +test('toward monosemanticity button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + await page.getByText('Toward Monosemanticity').click(), + ]); + + await expect(newPage).toHaveURL('https://transformer-circuits.pub/2023/monosemantic-features'); +}); + +test('saelens button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('SAELens').click()]); + + await expect(newPage).toHaveURL('https://github.com/jbloomAus/SAELens'); +}); + +test('transformerlens button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('TransformerLens').click()]); + + await expect(newPage).toHaveURL('https://github.com/TransformerLensOrg/TransformerLens'); +}); + +test('nnsight button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('NNsight').click()]); + + await expect(newPage).toHaveURL('https://github.com/ndif-team/nnsight'); +}); + +test('mats program button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('MATS Program').click()]); + + await expect(newPage).toHaveURL('https://www.matsprogram.org/'); +}); + +test('arena education button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('ARENA Education').click()]); + + await expect(newPage).toHaveURL('https://www.arena.education/'); +}); + +test('twitter button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('Twitter').first().click()]); + + await expect(newPage).toHaveURL('https://x.com/neuronpedia'); +}); + +// second contact us button on same page +// test('Contact Us', async({ page }) => { +// await page.goto(`${gemmaUrl}#learn`); + +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page.locator('a[target="_blank"][href*="mailto:johnny@neuronpedia\.org"]').click() +// ]); + +// await expect(newPage).toHaveURL(/mailto:johnny@neuronpedia\.org*/); +// }); + +test('open problems button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + await page.getByRole('button', { name: 'Open Problems' }).click(); + await expect(page).toHaveURL(`${gemmaUrl}#openproblems`); +}); + +test('playground button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#learn`); + + await page.getByRole('button', { name: 'Playground' }).click(); + await expect(page).toHaveURL(`${gemmaUrl}#playground`); +}); + +test('open problems page', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + await page.locator('textarea[name="Interpretability has many unsolved problems"]').isVisible(); +}); + +test('slack link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Slack').nth(1).click()]); + + await expect(newPage).toHaveURL( + 'https://opensourcemechanistic.slack.com/join/shared_invite/zt-2o756ku1c-_yKBeUQMVfS_p_qcK6QLeA#/shared-invite/email', + ); +}); + +// test('lesswrong link', async ({ page }) => { +// await page.goto(`${gemmaUrl}#openproblems`); + +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page.getByText('LessWrong').click() +// ]); + +// await expect(newPage).toHaveURL('https://www.lesswrong.com'); +// }); + +test('wattenberg link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Wattenberg et al').click()]); + + await expect(newPage).toHaveURL('https://arxiv.org/abs/2407.14662'); +}); + +test('Ziegler link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Ziegler et al').click()]); + + await expect(newPage).toHaveURL('https://arxiv.org/abs/2205.01663'); +}); + +test('Steering Vectors link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('steering vectors').click()]); + + await expect(newPage).toHaveURL('https://arxiv.org/abs/2308.10248'); +}); + +test('SAE feature steering link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('SAE feature steering').click()]); + + await expect(newPage).toHaveURL( + 'https://www.alignmentforum.org/posts/C5KAZQib3bzzpeyrg/full-post-progress-update-1-from-the-gdm-mech-interp-team#Activation_Steering_with_SAEs', + ); +}); + +test('clamping link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('clamping').click()]); + + await expect(newPage).toHaveURL('https://transformer-circuits.pub/2024/scaling-monosemanticity/index.html'); +}); + +test('huang et al link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Huang et al').click()]); + + await expect(newPage).toHaveURL('https://arxiv.org/abs/2309.10312'); +}); + +test('bills et al link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Bills et al').click()]); + + await expect(newPage).toHaveURL('https://openaipublic.blob.core.windows.net/neuron-explainer/paper/index.html'); +}); + +// same link appears twice +test('engels et al link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [firstPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Engels et al').first().click()]); + + await expect(firstPage).toHaveURL('https://arxiv.org/abs/2405.14860'); + await firstPage.close(); + + const [secondNewPage] = await Promise.all([ + page.waitForEvent('popup'), + page.getByText('Engels et al').nth(1).click(), + ]); + await expect(secondNewPage).toHaveURL('https://arxiv.org/abs/2405.14860'); +}); + +test('toy models link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + page.getByText('as shown to happen in toy models').click(), + ]); + + await expect(newPage).toHaveURL( + 'https://www.lesswrong.com/posts/a5wwqza2cY3W7L9cj/sparse-autoencoders-find-composed-features-in-small-toy', + ); +}); + +test('stolfo et al link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Stolfo et al').click()]); + + await expect(newPage).toHaveURL('https://arxiv.org/abs/2305.15054'); +}); + +test('marks et al link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Marks et al').click()]); + + await expect(newPage).toHaveURL('https://arxiv.org/abs/2403.19647'); +}); + +test('mlp transcoders link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('MLP transcoders').click()]); + + await expect(newPage).toHaveURL('https://arxiv.org/abs/2406.11944v1'); +}); + +test('jain et al link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Jain et al').click()]); + + await expect(newPage).toHaveURL('https://arxiv.org/abs/2407.10264'); +}); + +test('hendel et al link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Hendel et al').click()]); + + await expect(newPage).toHaveURL('https://arxiv.org/abs/2310.15916'); +}); + +test('Todd et al link', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Todd et al').click()]); + + await expect(newPage).toHaveURL('https://functions.baulab.info/'); +}); + +test('do more button open problems', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + await page.getByRole('button', { name: 'Do More' }).click(); + await expect(page).toHaveURL(`${gemmaUrl}#learn`); +}); + +test('home button open problems', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#openproblems`); + + await page.getByRole('button', { name: 'Home' }).click(); + await expect(page).toHaveURL(`${gemmaUrl}#main`); +}); + +test('playground page', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#playground`); + + await page.locator('textarea[name="Enter something below to see"]').isVisible(); +}); + +test('im feeling lucky button', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#playground`); + + await page.getByRole('button', { name: "I'm Feeling Lucky" }).click(); + await expect( + page.getByText("Enter something below to see what features activate, or try I'm Feeling Lucky."), + ).not.toBeVisible(); +}); + +test('model selector', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#playground`); + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + + const modelNames = ['GEMMA-2-2B']; + + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); +}); + +test('source set selector', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#playground`); + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + + const modelNames = ['gemmascope-res-16k']; + + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); +}); + +test('playground steering', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#playground`); + + const gemmaSearch = page.getByPlaceholder('Make gemma-2-2b think about'); + await gemmaSearch.fill('testing inputs'); + await page.getByRole('button', { name: 'Go' }).click(); + await expect(page.getByRole('button', { name: 'testing' })).toBeVisible(); +}); + +test('browse & search page', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#browse`); + + await page.locator('textarea[name="Search 50,000,000+ features"]').isVisible(); +}); + +test('jump to appears', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#browse`); + + // Look for the h3 element with the specific classes and text + const jumpToHeading = page.locator( + 'h3.cursor-default.select-none.text-xl.font-semibold.leading-none.tracking-tight[data-sentry-element="CardTitle"][data-sentry-source-file="jump-to-pane.tsx"]', + ); + + await expect(jumpToHeading).toBeVisible(); + await expect(jumpToHeading).toHaveText('Jump To'); +}); + +test('jump to sae/source models', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#browse`); + + // 3 occurences of this combobox on this page + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').first().click(); + const modelNames = ['GEMMA-2-2B']; + + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); +}); + +test('jump to sae/source source/sae', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#browse`); + + // 2 occurences of this combobox on this page + await page.locator('[data-state="closed"][data-sentry-source-file="source-selector.tsx"]').first().click(); + const modelNames = [ + 'gemmascope-res-16k' + ]; + + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); +}); + +test('jump to feature models', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#browse`); + + // 3 occurences of this combobox on this page + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').nth(1).click(); + const modelNames = ['GEMMA-2-2B']; + + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); +}); + +test('jump to feature source/sae', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#browse`); + + // 2 occurences of this combobox on this page + await page.locator('[data-state="closed"][data-sentry-source-file="source-selector.tsx"]').nth(1).click(); + const modelNames = [ + 'gemmascope-res-16k', + ]; + + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); +}); + +test('search explanations appears', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#browse`); + + // Look for the h3 element with the specific classes and text + const jumpToHeading = page.locator( + 'h3.cursor-default.select-none.text-xl.font-semibold.leading-none.tracking-tight[data-sentry-element="CardTitle"][data-sentry-source-file="search-explanations-pane.tsx"]', + ); + + await expect(jumpToHeading).toBeVisible(); + await expect(jumpToHeading).toHaveText('Search Explanations'); +}); + +test('search via interference appears', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#browse`); + + const jumpToHeading = page.locator( + 'h3.cursor-default.select-none.text-xl.font-semibold.leading-none.tracking-tight[data-sentry-element="CardTitle"][data-sentry-source-file="search-inference-release-pane.tsx"]', + ); + + await expect(jumpToHeading).toBeVisible(); + await expect(jumpToHeading).toHaveText('Search via Inference'); +}); + +test('search via inference models', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#browse`); + + // 3 occurences of this combobox on this page + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').nth(2).click(); + const modelNames = ['GEMMA-2-2B']; + + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); +}); + +test('search via inference source/sae', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#browse`); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').first().click(); + const modelNames = [ + // 'gemmascope-res-131k', + 'gemmascope-res-16k', + ]; + + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); +}); diff --git a/apps/webapp/tests-playwright/local/homepage.test.ts b/apps/webapp/tests-playwright/local/homepage.test.ts new file mode 100644 index 000000000..aecc38369 --- /dev/null +++ b/apps/webapp/tests-playwright/local/homepage.test.ts @@ -0,0 +1,293 @@ +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; + +test('homepage has expected title', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + await expect(page).toHaveTitle(/Neuronpedia/); +}); + +test('API navigation link', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + // Find and click the API link + const apiLink = page.getByRole('link', { name: 'API', exact: true }); + await apiLink.waitFor({ state: 'visible' }); + await apiLink.click(); + + // Wait for navigation and check exact URL + await page.waitForLoadState('networkidle'); + await expect(page).toHaveURL(`${baseUrl}/api-doc`); +}); + +test('Steer', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + // Find and click the Steer link + const steerLink = page.getByRole('link', { name: 'Steer', exact: true }); + await steerLink.waitFor({ state: 'visible' }); + await steerLink.click(); + + await expect(page).toHaveURL(/\/steer$/); +}); + +// Must be logged in to test this +// test('My lists', async({ page }) => { +// await page.goto('https://neuronpedia.org'); + +// // Find and click the My lists link +// const myListsLink = page.getByRole('link', { name: 'My lists', exact: true }); +// await myListsLink.waitFor({ state: 'visible' }); +// await myListsLink.click(); + +// await expect(page).toHaveURL(/.*neuronpedia\.org\/user\/.*\/lists/); +// }) + +// Must be logged in to test this +// /* test('My vectors', async({ page }) => { +// await page.goto('https://neuronpedia.org'); +// await expect(page).toHaveURL(/.*neuronpedia\.org\/user\/.*\/vectors/); +// }) */ + +// no local docs currently +// test('Getting Started', async ({ page }) => { +// await page.goto(baseUrl); + +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page.getByRole('link', { name: 'Getting Started', exact: true }).click(), +// ]); + +// await expect(newPage).toHaveURL('https://docs.neuronpedia.org/'); +// }); + +test('MIT Technology Review link', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + page.locator('a[target="_blank"][href*="technologyreview.com"]').click(), + ]); + + await expect(newPage).toHaveURL(/.*technologyreview\.com/); +}); + +test('Google Deepmind', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + page.locator('a[target="_blank"][href*="gemma-scope"]').click(), + ]); + + await expect(newPage).toHaveURL(/.*gemma-scope/); +}); + +test('Fudan University', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + page.locator('a[target="_blank"][href*="llama-scope"]').click(), + ]); + + await expect(newPage).toHaveURL(/.*llama-scope/); +}); + +test('Apollo Research', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + page.locator('a[target="_blank"][href*="gpt2sm-apollojt"]').click(), + ]); + + await expect(newPage).toHaveURL(/.*gpt2sm-apollojt/); +}); + +// no link +// test('ML Alignment & Theory Scholars', async({ page }) => { +// await page.goto('https://neuronpedia.org'); + +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page.getByRole('link', { name: 'MATS' }).click() +// ]); + +// await expect(newPage).toHaveURL('#mats'); +// }); + +test('EleutherAI', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + page.locator('a[target="_blank"][href*="llama3.1-8b-eleuther_gp"]').click(), + ]); + + await expect(newPage).toHaveURL(/.*llama3\.1-8b-eleuther_gp/); +}); + +// no local docs +// test('latents/features', async ({ page }) => { +// await page.goto(baseUrl); + +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page.getByRole('link', { name: 'latents/features', exact: true }).click(), +// ]); + +// await expect(newPage).toHaveURL('https://docs.neuronpedia.org/features'); +// }); + +test('concepts', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + page.getByRole('link', { name: 'concepts', exact: true }).click(), + ]); + + await expect(newPage).toHaveURL(`${baseUrl}/axbench`); +}); + +test('releases display', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + await expect(page.getByText('Llama Scope R1: SAEs for DeepSeek-R1-Distill-Llama-8B')).toBeVisible(); +}); + +test('models display', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + await expect(page.getByText('DeepSeek-R1-Llama-8B').nth(1)).toBeVisible(); +}); + +test('jump to display', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + await expect(page.getByText('jump to source/sae')).toBeVisible(); +}); + +test('cat steering', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + page.getByRole('link', { name: 'Try It: Gemma 2 - Cat Steering', exact: true }).click(), + ]); + + await expect(newPage).toHaveURL(`https://www.neuronpedia.org/gemma-2-9b-it/steer?saved=cm7cp63af00jx1q952neqg6e5`); +}); + +test('search by explanation', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + page.getByRole('link', { name: 'Try It: Search by Explanation', exact: true }).click(), + ]); + + await expect(newPage).toHaveURL( `${baseUrl}/search-explanations`); +}); + +// no local docs +// test('search via interference', async ({ page }) => { +// await page.goto(baseUrl); + +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page.getByRole('link', { name: 'Docs: Search via Inference', exact: true }).click(), +// ]); + +// await expect(newPage).toHaveURL('https://docs.neuronpedia.org/search'); +// }); + +test('api + libraries', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + page.getByRole('link', { name: 'API Playground', exact: true }).click(), + ]); + + await expect(newPage).toHaveURL(`${baseUrl}/api-doc`); +}); + +test('searcher is embedded in the page', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + await page.locator('textarea[name="searchQuery"]').isVisible(); +}); + +test('searcher example', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + await page.locator('textarea[name="Test activation with custom text./"]').isVisible(); +}); + +// no local docs +// test('Docs: Lists', async ({ page }) => { +// await page.goto(baseUrl); + +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page.getByRole('link', { name: 'Docs: Lists', exact: true }).click(), +// ]); + +// await expect(newPage).toHaveURL('https://docs.neuronpedia.org/lists'); +// }); + +// test('Docs: Embed', async ({ page }) => { +// await page.goto(baseUrl); + +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page.getByRole('link', { name: 'Docs: Embed', exact: true }).click(), +// ]); + +// await expect(newPage).toHaveURL('https://docs.neuronpedia.org/embed-iframe'); +// }); + +test('Slack', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + page.getByRole('button', { name: 'Slack' }).click(), + ]); + + await expect(newPage).toHaveURL(/.*slack\.com*/); +}); + +test('Donate', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + page.getByRole('button', { name: 'Donate' }).click(), + ]); + + await expect(newPage).toHaveURL(/.*every\.org*/); +}); + +test('Upskill', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + const [newPage] = await Promise.all([ + page.waitForEvent('popup'), + page.getByRole('button', { name: 'Upskill' }).click(), + ]); + + await expect(newPage).toHaveURL('https://www.arena.education/'); +}); + +// online service +// test('all services are online', async ({ page }) => { +// await page.goto(baseUrl); + +// const [newPage] = await Promise.all([ +// page.waitForEvent('popup'), +// page +// .locator('iframe[title="Neuronpedia Status"]') +// .contentFrame() +// .getByRole('link', { name: 'All services are online' }) +// .click(), +// ]); + +// await expect(newPage).toHaveURL('https://status.neuronpedia.org/'); +// }); diff --git a/apps/webapp/tests-playwright/local/sae-bench.spec.ts b/apps/webapp/tests-playwright/local/sae-bench.spec.ts index 645880131..464e96330 100644 --- a/apps/webapp/tests-playwright/local/sae-bench.spec.ts +++ b/apps/webapp/tests-playwright/local/sae-bench.spec.ts @@ -1,6 +1,6 @@ import { expect, test } from '@playwright/test'; -test('should navigate to the SAE Evals page', async ({ page }) => { +test.skip('should navigate to the SAE Evals page', async ({ page }) => { await page.goto('/'); await page.click('text=SAE Evals'); await expect(page).toHaveURL('/sae-bench'); diff --git a/apps/webapp/tests-playwright/local/search.test.ts b/apps/webapp/tests-playwright/local/search.test.ts new file mode 100644 index 000000000..ec88844ef --- /dev/null +++ b/apps/webapp/tests-playwright/local/search.test.ts @@ -0,0 +1,149 @@ +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; + +// http:localhost:3000/search +test('search models', async ({ page, searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + const modelNames = [ + 'DeepSeek-R1-Llama-8B', + 'GEMMA-2-2B', + 'GPT2-SM', + ]; + + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); +}); + +test('deepseek sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + await page.getByText('DeepSeek-R1-Llama-8B').first().click(); + + await expect(page.getByText('llamascope-slimpj-res-32k')).toBeVisible(); +}); + +test('gemma-2-2b sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + // currently default search model + // await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + // await page.getByText('GEMMA-2-2B').click(); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + const saeNames = ['gemmascope-res-16k']; + + await Promise.all( + saeNames.map(saeset => + expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() + ) + ); +}); + +test('gpt2-sm sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + await page.getByText('GPT2-SM').first().click(); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + const saeNames = [ + 'res-jb', + ]; + + await Promise.all( + saeNames.map(saeset => + expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() + ) + ); +}); + +// checks for new url in the case that the default model is changed +test('random button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Random').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('food button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Food').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('news button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('News').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('literary button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Literary').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('personal button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Personal').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('Programming button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Programming').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('Techinical button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Technical').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('academic button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Academic').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('business button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Business').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('legal button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Legal').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('educational button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Educational').click(); + await expect(page).not.toHaveURL(searchUrl); +}); + +test('Cultural button', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.getByText('Cultural').click(); + await expect(page).not.toHaveURL(searchUrl); +}); diff --git a/apps/webapp/tests-playwright/local/steer.test.ts b/apps/webapp/tests-playwright/local/steer.test.ts new file mode 100644 index 000000000..c24222373 --- /dev/null +++ b/apps/webapp/tests-playwright/local/steer.test.ts @@ -0,0 +1,102 @@ +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; + +// http://localhost:3000/steer defaults to localhost:3000/gemma-2-2b-it/steer + +test('steer page loads', async ({ page, steerUrl }) => { + await page.goto(steerUrl); + await expect(page.locator('text="Steer Models"')).toBeVisible(); +}); + +test('model selector', async ({ page, steerUrl }) => { + await page.goto(steerUrl); + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + + const modelNames = [ + 'DeepSeek-R1-Llama-8B', + 'GEMMA-2-2B', + 'GEMMA-2-2B-IT', + // 'GEMMA-2-9B', + // 'GEMMA-2-9B-IT', + 'GPT2-SM', + // 'LLAMA3.1-8B', + ]; + + await Promise.all( + modelNames.map(model => + expect(page.getByText(model, { exact: true }).first()).toBeVisible() + ) + ); +}); + +// tests designed for production +// test('deepseek-r1-llama-8b steer', async ({ page }) => { +// await page.goto('https://neuronpedia.org/deepseek-r1-distill-llama-8b/steer'); + +// // user first occurring demo to search +// await page.getByRole('button', { name: 'Demo' }).first().click(); + +// // check to see if text changed due to a new response +// await expect(page.getByText("Hey, I'm steered deepseek-r1-distill-llama-8b")).not.toBeVisible({ timeout: 30000 }); +// }); + +// test('gemma-2-2b steer', async ({ page }) => { +// await page.goto('https://neuronpedia.org/gemma-2-2b/steer'); + +// // user first occurring demo to search +// await page.getByRole('button', { name: 'Demo' }).first().click(); + +// // check to see if text changed due to a new response +// await expect(page.getByText("Hey, I'm steered gemma-2-2b!")).not.toBeVisible({ timeout: 30000 }); +// }); + +// test('gemma-2-2b-it steer', async ({ page }) => { +// await page.goto('https://neuronpedia.org/gemma-2-2b-it/steer'); + +// // user first occurring demo to search +// await page.getByRole('button', { name: 'Demo' }).first().click(); + +// // check to see if text changed due to a new response +// await expect(page.getByText("Hey, I'm steered gemma-2-2b-it!")).not.toBeVisible({ timeout: 30000 }); +// }); + +// // No presets +// // test('gemma-2-9b steer', async({ page}) => { +// // await page.goto('https://neuronpedia.org/gemma-2-9b/steer'); + +// // //user first occurring demo to search +// // await page.getByRole('button', { name: 'Demo' }).first().click(); + +// // //check to see if text changed due to a new response +// // await expect(page.getByText("Hey, I'm steered gemma-2-9b!")).not.toBeVisible({ timeout: 30000 }); +// // }); + +// test('gemma-2-9b-it steer', async ({ page }) => { +// await page.goto('https://neuronpedia.org/gemma-2-9b-it/steer'); + +// // user first occurring demo to search +// await page.getByRole('button', { name: 'Demo' }).first().click(); + +// // check to see if text changed due to a new response +// await expect(page.getByText("Hey, I'm steered gemma-2-9b-it!")).not.toBeVisible({ timeout: 30000 }); +// }); + +// test('gpt2-small steer', async ({ page }) => { +// await page.goto('https://neuronpedia.org/gpt2-small/steer'); + +// // user first occurring demo to search +// await page.getByRole('button', { name: 'Demo' }).first().click(); + +// // check to see if text changed due to a new response +// await expect(page.getByText("Hey, I'm steered gpt2-small!")).not.toBeVisible({ timeout: 30000 }); +// }); + +// test('llama3.1-8b steer', async ({ page }) => { +// await page.goto('https://neuronpedia.org/llama3.1-8b/steer'); + +// // user first occurring demo to search +// await page.getByRole('button', { name: 'Demo' }).first().click(); + +// // check to see if text changed due to a new response +// await expect(page.getByText("Hey, I'm steered llama3.1-8b!")).not.toBeVisible({ timeout: 30000 }); +// }); diff --git a/apps/webapp/tests-playwright/production/gemmascope.test.ts b/apps/webapp/tests-playwright/production/gemmascope.test.ts index 175223563..6c6c4edbe 100644 --- a/apps/webapp/tests-playwright/production/gemmascope.test.ts +++ b/apps/webapp/tests-playwright/production/gemmascope.test.ts @@ -1,741 +1,30 @@ -import { expect, test } from '@playwright/test'; - -test('main page', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#main'); - - await page.locator('textarea[name="The inner workings"]').isVisible(); -}); - -test('microscope button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#main'); - - await page.getByText("scan Gemma 2's brain to see what it's thinking").click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#microscope'); -}); - -test('analyze features button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#main'); - - await page.getByText('Make features fire and figure out what they do').click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#analyze'); -}); - -test('steer gemma button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#main'); - - await page.getByText("Change Gemma's behavior by manipulating features").click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#steer'); -}); - -test('do more button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#main'); - - await page.getByText('Dive deeper into the exciting world of AI interpretability').click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#learn'); -}); - -test('browse & search saes button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#main'); - - await page.getByText('Directly explore the SAEs in Gemma Scope').click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#browse'); -}); - -test('microscope page', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); - - await page.locator('textarea[name="To understand what AI is thinking"]').isVisible(); -}); - -test('demo button feeling lucky', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); - - await page.getByText("I'm Feeling Lucky").click(); - await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); -}); - -test('demo button food', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); - - await page.getByText('Food').click(); - await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); -}); - -test('demo button News', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); - - await page.getByText('News').click(); - await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); -}); - -test('demo button Literary', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); - - await page.getByText('Literary').click(); - await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); -}); - -test('demo button Personal', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); - - await page.getByText('Personal').click(); - await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); -}); - -test('demo button Programming', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); - - await page.getByText('Programming').click(); - await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); -}); - -test('challenge gemma', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); - - // challenge is unlocked after doing the demo, or pressing skip demo - await page.getByText('skip demo').first().click(); - - const gemmaSearch = page.getByPlaceholder('Hint: Try a short sentence instead of a word or phrase.'); - await gemmaSearch.fill('testing inputs'); - await gemmaSearch.press('Enter'); - await expect(page.getByText('Click a preset above to send a text to Gemma')).not.toBeVisible({ timeout: 30000 }); -}); - -test('next analyze features button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#microscope'); - - // next is unlocked after doing the demo, or pressing skip demo - await page.getByText('skip demo').nth(1).click(); - await page.getByText(' Next - Analyze Features').click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#analyze'); -}); - -test('analyze features page', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); - - await page.locator('textarea[name="The features in Gemma Scope have labels"]').isVisible(); -}); - -test('olympic sports button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); - await page.getByText('Olympic sports').click(); - await expect(page.getByText('What do these activations (in green) have in common?')).toBeVisible(); -}); - -test('famous cities button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); - await page.getByText('famous cities').click(); - await expect(page.getByText('What do these activations (in green) have in common?')).toBeVisible(); -}); - -test('reference to animals button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); - await page.getByText('references to animals').click(); - await expect(page.getByText('Nice! This was tricky, because there are two possibly correct answers.')).toBeVisible(); -}); - -test('none of these button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); - await page.getByText('none of these').click(); - await expect(page.getByText('Nice! This was tricky, because there are two possibly correct answers.')).toBeVisible(); -}); - -test('analyze steer', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); - - // unlocked after doing the demo, or pressing skip demo - await page.getByText('skip example').first().click(); - const analyzeSteer = page.getByPlaceholder('Can you write a text snippet that activates it?'); - await analyzeSteer.fill('test'); - await analyzeSteer.press('Enter'); - - await expect(page.getByText('0.00')).toBeVisible(); -}); - -test('puzzles labels', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); - - // unlocked after doing analyze steer, or pressing skip analyze - await page.getByText('skip analyze').first().click(); - - // test all 3 buttons - const labelButtons = await page.getByRole('button', { name: 'Reveal Our Label' }).all(); - // click each button - for (const button of labelButtons) { - await button.click(); - } - - // check labels - await expect(page.getByText('Lies and falsehoods')).toBeVisible(); - await expect(page.getByText('Misspellings or typos')).toBeVisible(); - await expect(page.getByText('Bad/cringe stories')).toBeVisible(); -}); - -test('next steer gemma button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#analyze'); - - // unlocked after puzzles, or pressing skip analyze - await page.getByText('skip analyze').nth(1).click(); - await page.getByText(' Next - Steer Gemma').click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#steer'); -}); - -test('steer page', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#steer'); - - await page.locator('textarea[name="Let\'s put these features to use"]').isVisible(); -}); - -test('steer demo', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#steer'); - - // check for demo question - await expect(page.getByText('What are you?').first()).toBeVisible(); -}); - -test('next do more button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#steer'); - - await page.getByText('Next - Do More').click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#learn'); -}); - -test('do more page', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - await page.locator('textarea[name="This demo of Gemma Scope"]').isVisible(); -}); - -test('advanced steer button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - await page.locator('a[target="_blank"][href*="https://www.neuronpedia.org/steer"]').click(), - ]); - await expect(newPage).toHaveURL(/https:\/\/www\.neuronpedia\.org\/.*\/steer/); -}); - -test('browse saes button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - await page.getByText('Browse SAEs').click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#browse'); -}); - -// test('Contact Us', async({ page }) => { -// await page.goto('https://neuronpedia.org/gemma-scope#learn'); - -// const [newPage] = await Promise.all([ -// page.waitForEvent('popup'), -// page.locator('a[target="_blank"][href*="mailto:johnny@neuronpedia\.org"]').click() -// ]); - -// await expect(newPage).toHaveURL(/mailto:johnny@neuronpedia\.org*/); -// }); - -test('deepmind blog post button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('DeepMind Blog Post').click()]); - - await expect(newPage).toHaveURL( - 'https://deepmind.google/discover/blog/gemma-scope-helping-the-safety-community-shed-light-on-the-inner-workings-of-language-models/', - ); -}); - -test('coding tutorial button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('Coding Tutorial').click()]); - - await expect(newPage).toHaveURL( - 'https://colab.research.google.com/drive/17dQFYUYnuKnP6OwQPH9v_GSYUW5aj-Rp?usp=sharing', - ); -}); - -// testing it the default method with expect().toHaveURL doesnt work, url ends up being blank -// only tests for the correct link being on the page -test('technical report button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - const href = await page.getByText('Technical Report').getAttribute('href'); - - expect(href).toContain('storage.googleapis.com/gemma-scope/gemma-scope-report.pdf'); -}); - -test('huggingface button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('HuggingFace').click()]); - - await expect(newPage).toHaveURL('https://huggingface.co/google/gemma-scope'); -}); - -test('get started with mech interp button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - await page.getByText('Get Started with Mech Interp').click(), - ]); - - await expect(newPage).toHaveURL('https://www.neelnanda.io/mechanistic-interpretability/getting-started'); -}); - -test('favourite mech interp papers button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - await page.getByText('Favourite Mech Interp Papers').click(), - ]); - - await expect(newPage).toHaveURL( - 'https://www.alignmentforum.org/posts/NfFST5Mio7BCAQHPA/an-extremely-opinionated-annotated-list-of-my-favourite-1', - ); -}); - -test('toward monosemanticity button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - await page.getByText('Toward Monosemanticity').click(), - ]); - - await expect(newPage).toHaveURL('https://transformer-circuits.pub/2023/monosemantic-features'); -}); - -test('saelens button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('SAELens').click()]); - - await expect(newPage).toHaveURL('https://github.com/jbloomAus/SAELens'); -}); - -test('transformerlens button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('TransformerLens').click()]); - - await expect(newPage).toHaveURL('https://github.com/TransformerLensOrg/TransformerLens'); -}); - -test('nnsight button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('NNsight').click()]); - - await expect(newPage).toHaveURL('https://github.com/ndif-team/nnsight'); -}); - -test('mats program button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('MATS Program').click()]); - - await expect(newPage).toHaveURL('https://www.matsprogram.org/'); -}); - -test('arena education button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('ARENA Education').click()]); - - await expect(newPage).toHaveURL('https://www.arena.education/'); -}); - -test('twitter button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), await page.getByText('Twitter').first().click()]); - - await expect(newPage).toHaveURL('https://x.com/neuronpedia'); -}); - -// second contact us button on same page -// test('Contact Us', async({ page }) => { -// await page.goto('https://neuronpedia.org/gemma-scope#learn'); - -// const [newPage] = await Promise.all([ -// page.waitForEvent('popup'), -// page.locator('a[target="_blank"][href*="mailto:johnny@neuronpedia\.org"]').click() -// ]); - -// await expect(newPage).toHaveURL(/mailto:johnny@neuronpedia\.org*/); -// }); - -test('open problems button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - await page.getByRole('button', { name: 'Open Problems' }).click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#openproblems'); -}); - -test('playground button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#learn'); - - await page.getByRole('button', { name: 'Playground' }).click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#playground'); -}); - -test('open problems page', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - await page.locator('textarea[name="Interpretability has many unsolved problems"]').isVisible(); -}); - -test('slack link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Slack').click()]); - - await expect(newPage).toHaveURL( - 'https://opensourcemechanistic.slack.com/join/shared_invite/zt-2o756ku1c-_yKBeUQMVfS_p_qcK6QLeA#/shared-invite/email', - ); -}); - -// test('lesswrong link', async ({ page }) => { -// await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - -// const [newPage] = await Promise.all([ -// page.waitForEvent('popup'), -// page.getByText('LessWrong').click() -// ]); - -// await expect(newPage).toHaveURL('https://www.lesswrong.com'); -// }); - -test('wattenberg link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Wattenberg et al').click()]); - - await expect(newPage).toHaveURL('https://arxiv.org/abs/2407.14662'); -}); - -test('Ziegler link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Ziegler et al').click()]); - - await expect(newPage).toHaveURL('https://arxiv.org/abs/2205.01663'); -}); - -test('Steering Vectors link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('steering vectors').click()]); - - await expect(newPage).toHaveURL('https://arxiv.org/abs/2308.10248'); -}); - -test('SAE feature steering link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('SAE feature steering').click()]); - - await expect(newPage).toHaveURL( - 'https://www.alignmentforum.org/posts/C5KAZQib3bzzpeyrg/full-post-progress-update-1-from-the-gdm-mech-interp-team#Activation_Steering_with_SAEs', - ); -}); - -test('clamping link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('clamping').click()]); - - await expect(newPage).toHaveURL('https://transformer-circuits.pub/2024/scaling-monosemanticity/index.html'); -}); - -test('huang et al link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Huang et al').click()]); - - await expect(newPage).toHaveURL('https://arxiv.org/abs/2309.10312'); -}); - -test('bills et al link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Bills et al').click()]); - - await expect(newPage).toHaveURL('https://openaipublic.blob.core.windows.net/neuron-explainer/paper/index.html'); -}); - -// same link appears twice -test('engels et al link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [firstPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Engels et al').first().click()]); - - await expect(firstPage).toHaveURL('https://arxiv.org/abs/2405.14860'); - await firstPage.close(); - - const [secondNewPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByText('Engels et al').nth(1).click(), - ]); - await expect(secondNewPage).toHaveURL('https://arxiv.org/abs/2405.14860'); -}); - -test('toy models link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByText('as shown to happen in toy models').click(), - ]); - - await expect(newPage).toHaveURL( - 'https://www.lesswrong.com/posts/a5wwqza2cY3W7L9cj/sparse-autoencoders-find-composed-features-in-small-toy', - ); -}); - -test('stolfo et al link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Stolfo et al').click()]); - - await expect(newPage).toHaveURL('https://arxiv.org/abs/2305.15054'); -}); - -test('marks et al link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Marks et al').click()]); - - await expect(newPage).toHaveURL('https://arxiv.org/abs/2403.19647'); -}); - -test('mlp transcoders link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('MLP transcoders').click()]); - - await expect(newPage).toHaveURL('https://arxiv.org/abs/2406.11944v1'); -}); - -test('jain et al link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Jain et al').click()]); - - await expect(newPage).toHaveURL('https://arxiv.org/abs/2407.10264'); -}); - -test('hendel et al link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Hendel et al').click()]); - - await expect(newPage).toHaveURL('https://arxiv.org/abs/2310.15916'); -}); - -test('Todd et al link', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - const [newPage] = await Promise.all([page.waitForEvent('popup'), page.getByText('Todd et al').click()]); - - await expect(newPage).toHaveURL('https://functions.baulab.info/'); -}); - -test('do more button open problems', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - await page.getByRole('button', { name: 'Do More' }).click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#learn'); -}); - -test('home button open problems', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#openproblems'); - - await page.getByRole('button', { name: 'Home' }).click(); - await expect(page).toHaveURL('https://www.neuronpedia.org/gemma-scope#main'); -}); - -test('playground page', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#playground'); - - await page.locator('textarea[name="Enter something below to see"]').isVisible(); -}); - -test('im feeling lucky button', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#playground'); - - await page.getByRole('button', { name: "I'm Feeling Lucky" }).click(); - await expect( - page.getByText("Enter something below to see what features activate, or try I'm Feeling Lucky."), - ).not.toBeVisible(); -}); - -test('model selector', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#playground'); - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - - const modelNames = ['GEMMA-2-2B', 'GEMMA-2-2B-IT', 'GEMMA-2-9B', 'GEMMA-2-9B-IT']; - - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } -}); - -test('source set selector', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#playground'); - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); - - const modelNames = ['gemmascope-att-16k', 'gemmascope-res-16k', 'gemmascope-res-65k']; - - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } -}); - -test('playground steering', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#playground'); - - const gemmaSearch = page.getByPlaceholder('Make gemma-2-2b think about'); - await gemmaSearch.fill('testing inputs'); - await page.getByRole('button', { name: 'Go' }).click(); - await expect(page.getByRole('button', { name: 'testing' })).toBeVisible(); -}); - -test('browse & search page', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); - - await page.locator('textarea[name="Search 50,000,000+ features"]').isVisible(); -}); - -test('jump to appears', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); - - // Look for the h3 element with the specific classes and text - const jumpToHeading = page.locator( - 'h3.cursor-default.select-none.text-xl.font-semibold.leading-none.tracking-tight[data-sentry-element="CardTitle"][data-sentry-source-file="jump-to-pane.tsx"]', - ); - - await expect(jumpToHeading).toBeVisible(); - await expect(jumpToHeading).toHaveText('Jump To'); -}); - -test('jump to sae/source models', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); - - // 3 occurences of this combobox on this page - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').first().click(); - const modelNames = ['GEMMA-2-2B', 'GEMMA-2-2B-IT', 'GEMMA-2-9B', 'GEMMA-2-9B-IT']; - - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } -}); - -test('jump to sae/source source/sae', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); - - // 2 occurences of this combobox on this page - await page.locator('[data-state="closed"][data-sentry-source-file="source-selector.tsx"]').first().click(); - const modelNames = [ - 'gemmascope-att-16k', - 'gemmascope-att-65k', - 'gemmascope-mlp-16k', - 'gemmascope-mlp-65k', - 'gemmascope-res-16k', - 'gemmascope-res-1m', - 'gemmascope-res-262k', - 'gemmascope-res-32k', - 'gemmascope-res-524k', - 'gemmascope-res-65k', - ]; - - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } -}); - -test('jump to feature models', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); - - // 3 occurences of this combobox on this page - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').nth(1).click(); - const modelNames = ['GEMMA-2-2B', 'GEMMA-2-2B-IT', 'GEMMA-2-9B', 'GEMMA-2-9B-IT']; - - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } -}); - -test('jump to feature source/sae', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); - - // 2 occurences of this combobox on this page - await page.locator('[data-state="closed"][data-sentry-source-file="source-selector.tsx"]').nth(1).click(); - const modelNames = [ - 'gemmascope-att-16k', - 'gemmascope-att-65k', - 'gemmascope-mlp-16k', - 'gemmascope-mlp-65k', - 'gemmascope-res-16k', - 'gemmascope-res-1m', - 'gemmascope-res-262k', - 'gemmascope-res-32k', - 'gemmascope-res-524k', - 'gemmascope-res-65k', - ]; - - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } -}); - -test('search explanations appears', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); - - // Look for the h3 element with the specific classes and text - const jumpToHeading = page.locator( - 'h3.cursor-default.select-none.text-xl.font-semibold.leading-none.tracking-tight[data-sentry-element="CardTitle"][data-sentry-source-file="search-explanations-pane.tsx"]', - ); - - await expect(jumpToHeading).toBeVisible(); - await expect(jumpToHeading).toHaveText('Search Explanations'); -}); - -test('search via interference appears', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); - - const jumpToHeading = page.locator( - 'h3.cursor-default.select-none.text-xl.font-semibold.leading-none.tracking-tight[data-sentry-element="CardTitle"][data-sentry-source-file="search-inference-release-pane.tsx"]', - ); - - await expect(jumpToHeading).toBeVisible(); - await expect(jumpToHeading).toHaveText('Search via Inference'); -}); - -test('search via inference models', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); - - // 3 occurences of this combobox on this page - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').nth(2).click(); - const modelNames = ['GEMMA-2-2B', 'GEMMA-2-2B-IT', 'GEMMA-2-9B', 'GEMMA-2-9B-IT']; - - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } -}); - -test('search via inference source/sae', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-scope#browse'); - - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').first().click(); - const modelNames = [ - // 'gemmascope-res-131k', - 'gemmascope-res-16k', - ]; - - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } -}); +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; + +test('gemma steer demo', async ({ page }) => { + await page.goto('https://neuronpedia.org/gemma-scope#steer'); + + await page.click('text='); + await page.getByRole('button', { name: 'Tell me about yourself.' }).first().click(); + await page.waitForTimeout(3000); + await expect(page.getByText("Sorry, your message could not be sent at this time. Please try again later.")).not.toBeVisible({ timeout: 30000 }); + }); + + test('puzzles labels', async ({ page, gemmaUrl }) => { + await page.goto(`${gemmaUrl}#analyze`); + + // unlocked after doing analyze steer, or pressing skip analyze + await page.getByText('skip analyze').first().click(); + + // test all 3 buttons + const labelButtons = await page.getByRole('button', { name: 'Reveal Our Label' }).all(); + // click each button + await Promise.all( + labelButtons.map(button => button.click()) + ); + + // check labels + await expect(page.getByText('Lies and falsehoods')).toBeVisible(); + await expect(page.getByText('Misspellings or typos')).toBeVisible(); + await expect(page.getByText('Bad/cringe stories')).toBeVisible(); + }); \ No newline at end of file diff --git a/apps/webapp/tests-playwright/production/homepage.test.ts b/apps/webapp/tests-playwright/production/homepage.test.ts index 419de6321..1d0f446cc 100644 --- a/apps/webapp/tests-playwright/production/homepage.test.ts +++ b/apps/webapp/tests-playwright/production/homepage.test.ts @@ -1,311 +1,15 @@ -import { expect, test } from '@playwright/test'; - -test('homepage has expected title', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - await expect(page).toHaveTitle(/Neuronpedia/); -}); - -test('API navigation link', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - // Find and click the API link - const apiLink = page.getByRole('link', { name: 'API', exact: true }); - await apiLink.waitFor({ state: 'visible' }); - await apiLink.click(); - - // Wait for navigation and check exact URL - await page.waitForLoadState('networkidle'); - await expect(page).toHaveURL('https://www.neuronpedia.org/api-doc'); -}); - -test('SAE Evals link', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - // Find and click the SAE Evals link - const saeEvalsLink = page.getByRole('link', { name: 'SAE Evals', exact: true }); - await saeEvalsLink.waitFor({ state: 'visible' }); - await saeEvalsLink.click(); - - // Wait for navigation and check exact URL - await page.waitForLoadState('networkidle'); - await expect(page).toHaveURL('https://www.neuronpedia.org/sae-bench'); -}); - -test('Steer', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - // Find and click the Steer link - const steerLink = page.getByRole('link', { name: 'Steer', exact: true }); - await steerLink.waitFor({ state: 'visible' }); - await steerLink.click(); - - await expect(page).toHaveURL(/https:\/\/www\.neuronpedia\.org\/.*\/steer/); -}); - -// Must be logged in to test this -// test('My lists', async({ page }) => { -// await page.goto('https://neuronpedia.org'); - -// // Find and click the My lists link -// const myListsLink = page.getByRole('link', { name: 'My lists', exact: true }); -// await myListsLink.waitFor({ state: 'visible' }); -// await myListsLink.click(); - -// await expect(page).toHaveURL(/.*neuronpedia\.org\/user\/.*\/lists/); -// }) - -// Must be logged in to test this -// /* test('My vectors', async({ page }) => { -// await page.goto('https://neuronpedia.org'); -// await expect(page).toHaveURL(/.*neuronpedia\.org\/user\/.*\/vectors/); -// }) */ - -test('Getting Started', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('link', { name: 'Getting Started', exact: true }).click(), - ]); - - await expect(newPage).toHaveURL('https://docs.neuronpedia.org/'); -}); - -test('MIT Technology Review link', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.locator('a[target="_blank"][href*="technologyreview.com"]').click(), - ]); - - await expect(newPage).toHaveURL(/.*technologyreview\.com/); -}); - -test('Google Deepmind', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.locator('a[target="_blank"][href*="gemma-scope"]').click(), - ]); - - await expect(newPage).toHaveURL(/.*gemma-scope/); -}); - -test('Fudan University', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.locator('a[target="_blank"][href*="llama-scope"]').click(), - ]); - - await expect(newPage).toHaveURL(/.*llama-scope/); -}); - -test('Apollo Research', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.locator('a[target="_blank"][href*="gpt2sm-apollojt"]').click(), - ]); - - await expect(newPage).toHaveURL(/.*gpt2sm-apollojt/); -}); - -// no link -// test('ML Alignment & Theory Scholars', async({ page }) => { -// await page.goto('https://neuronpedia.org'); - -// const [newPage] = await Promise.all([ -// page.waitForEvent('popup'), -// page.getByRole('link', { name: 'MATS' }).click() -// ]); - -// await expect(newPage).toHaveURL('#mats'); -// }); - -test('EleutherAI', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.locator('a[target="_blank"][href*="llama3.1-8b-eleuther_gp"]').click(), - ]); - - await expect(newPage).toHaveURL(/.*llama3\.1-8b-eleuther_gp/); -}); - -test('latents/features', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('link', { name: 'latents/features', exact: true }).click(), - ]); - - await expect(newPage).toHaveURL('https://docs.neuronpedia.org/features'); -}); - -test('concepts', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('link', { name: 'concepts', exact: true }).click(), - ]); - - await expect(newPage).toHaveURL('https://www.neuronpedia.org/axbench'); -}); - -test('releases display', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - await expect(page.getByText('Llama Scope R1: SAEs for DeepSeek-R1-Distill-Llama-8B')).toBeVisible(); -}); - -test('models display', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - await expect(page.getByText('DeepSeek-R1-Llama-8B')).toBeVisible(); -}); - -test('jump to display', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - await expect(page.getByText('jump to source/sae')).toBeVisible(); -}); - -test('cat steering', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('link', { name: 'Try It: Gemma 2 - Cat Steering', exact: true }).click(), - ]); - - await expect(newPage).toHaveURL('https://www.neuronpedia.org/gemma-2-9b-it/steer?saved=cm7cp63af00jx1q952neqg6e5'); -}); - -test('search by explanation', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('link', { name: 'Try It: Search by Explanation', exact: true }).click(), - ]); - - await expect(newPage).toHaveURL('https://www.neuronpedia.org/search-explanations'); -}); - -test('search via interference', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('link', { name: 'Docs: Search via Inference', exact: true }).click(), - ]); - - await expect(newPage).toHaveURL('https://docs.neuronpedia.org/search'); -}); - -test('api + libraries', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('link', { name: 'API Playground', exact: true }).click(), - ]); - - await expect(newPage).toHaveURL('https://www.neuronpedia.org/api-doc'); -}); - -test('searcher is embedded in the page', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - await page.locator('textarea[name="searchQuery"]').isVisible(); -}); - -test('searcher example', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - await page.locator('textarea[name="Test activation with custom text./"]').isVisible(); -}); - -test('Docs: Lists', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('link', { name: 'Docs: Lists', exact: true }).click(), - ]); - - await expect(newPage).toHaveURL('https://docs.neuronpedia.org/lists'); -}); - -test('Docs: Embed', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('link', { name: 'Docs: Embed', exact: true }).click(), - ]); - - await expect(newPage).toHaveURL('https://docs.neuronpedia.org/embed-iframe'); -}); - -test('Slack', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('button', { name: 'Slack' }).click(), - ]); - - await expect(newPage).toHaveURL(/.*slack\.com*/); -}); - -test('Donate', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('button', { name: 'Donate' }).click(), - ]); - - await expect(newPage).toHaveURL(/.*every\.org*/); -}); - -// test('Feedback', async({ page }) => { -// await page.goto('https://neuronpedia.org'); - -// const [newPage] = await Promise.all([ -// page.waitForEvent('popup'), -// page.getByRole('button', { name: 'Feedback' }).click() -// ]); - -// await expect(newPage).toHaveURL(/mailto:johnny@neuronpedia\.org*/); -// }); - -test('Upskill', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page.getByRole('button', { name: 'Upskill' }).click(), - ]); - - await expect(newPage).toHaveURL('https://www.arena.education/'); -}); - -test('all services are online', async ({ page }) => { - await page.goto('https://neuronpedia.org'); - - const [newPage] = await Promise.all([ - page.waitForEvent('popup'), - page - .locator('iframe[title="Neuronpedia Status"]') - .contentFrame() - .getByRole('link', { name: 'All services are online' }) - .click(), - ]); - - await expect(newPage).toHaveURL('https://status.neuronpedia.org/'); -}); +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; + +test('SAE Evals link', async ({ page, baseUrl }) => { + await page.goto(baseUrl); + + // Find and click the SAE Evals link + const saeEvalsLink = page.getByRole('link', { name: 'SAE Evals', exact: true }); + await saeEvalsLink.waitFor({ state: 'visible' }); + await saeEvalsLink.click(); + + // Wait for navigation and check exact URL + await page.waitForLoadState('networkidle'); + await expect(page).toHaveURL(`${baseUrl}/sae-bench`); + }); \ No newline at end of file diff --git a/apps/webapp/tests-playwright/production/search.test.ts b/apps/webapp/tests-playwright/production/search.test.ts index a5d9fa5de..2aa5fdb26 100644 --- a/apps/webapp/tests-playwright/production/search.test.ts +++ b/apps/webapp/tests-playwright/production/search.test.ts @@ -1,206 +1,87 @@ -import { expect, test } from '@playwright/test'; - -test('search models', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - const modelNames = [ - 'DEEPSEEK-R1-LLAMA-8B', - 'GEMMA-2-2B', - 'GEMMA-2-2B-IT', - 'GEMMA-2-9B', - 'GEMMA-2-9B-IT', - 'GPT2-SM', - 'LLAMA3.1-8B', - ]; - - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } -}); - -test('deepseek sae', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - await page.getByText('DEEPSEEK-R1-LLAMA-8B').click(); - - await expect(page.getByText('llamascope-openr1-res-32k')).toBeVisible(); -}); - -test('gemma-2-2b sae', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - // currently default search model - // await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - // await page.getByText('GEMMA-2-2B').click(); - - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); - const saeNames = ['gemmascope-att-16k', 'gemmascope-res-16k', 'gemmascope-res-65k']; - - for (const saeset of saeNames) { - await expect(page.getByText(saeset, { exact: true }).first()).toBeVisible(); - } -}); - -test('gemma-2-9b sae', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - await page.getByText('GEMMA-2-9B').first().click(); - - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); - const saeNames = ['gemmascope-res-16k']; - - for (const saeset of saeNames) { - await expect(page.getByText(saeset, { exact: true }).first()).toBeVisible(); - } -}); - -test('gemma-2-9b-it sae', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - await page.getByText('GEMMA-2-9B-IT').first().click(); - - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); - const saeNames = ['gemmascope-res-131k', 'gemmascope-res-16k']; - - for (const saeset of saeNames) { - await expect(page.getByText(saeset, { exact: true }).first()).toBeVisible(); - } -}); - -test('gpt2-sm sae', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - await page.getByText('GPT2-SM').first().click(); - - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); - const saeNames = [ - 'att_32k-oai', - 'att-kk', - 'mlp_32k-oai', - 'res_fs12288-jb', - 'res_fs1536-jb', - 'res_fs24576-jb', - 'res_fs3072-jb', - 'res_fs49152-jb', - 'res_fs6144-jb', - 'res_fs768-jb', - 'res_fs98304-jb', - 'res_mid_32k-oai', - 'res_post_32k-oai', - 'res_sce-ajt', - 'res_scefr-ajt', - 'res_scl-ajt', - 'res_sle-ajt', - 'res_slefr-ajt', - 'res_sll-ajt', - 'res-jb', - ]; - - for (const saeset of saeNames) { - await expect(page.getByText(saeset, { exact: true }).first()).toBeVisible(); - } -}); - -test('llama3.1-8b sae', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - await page.getByText('LLAMA3.1-8B').first().click(); - - await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); - const saeNames = ['llamascope-res-32k']; - - for (const saeset of saeNames) { - await expect(page.getByText(saeset, { exact: true }).first()).toBeVisible(); - } -}); - -// checks for new url in the case that the default model is changed -test('random button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Random').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('food button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Food').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('news button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('News').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('literary button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Literary').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('personal button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Personal').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('Programming button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Programming').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('Techinical button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Technical').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('academic button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Academic').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('business button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Business').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('legal button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Legal').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('educational button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Educational').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); - -test('Cultural button', async ({ page }) => { - await page.goto('https://neuronpedia.org/search'); - - await page.getByText('Cultural').click(); - await expect(page).not.toHaveURL('https://www.neuronpedia.org/search'); -}); +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; + +test('gemma-2-9b sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + await page.getByText('GEMMA-2-9B').first().click(); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + const saeNames = ['gemmascope-res-16k']; + + await Promise.all( + saeNames.map(saeset => + expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() + ) + ); + }); + + test('gpt2-sm sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + await page.getByText('GPT2-SM').first().click(); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + const saeNames = [ + 'att_32k-oai', + 'att-kk', + 'mlp_32k-oai', + 'res_fs12288-jb', + 'res_fs1536-jb', + 'res_fs24576-jb', + 'res_fs3072-jb', + 'res_fs49152-jb', + 'res_fs6144-jb', + 'res_fs768-jb', + 'res_fs98304-jb', + 'res_mid_32k-oai', + 'res_post_32k-oai', + 'res_sce-ajt', + 'res_scefr-ajt', + 'res_scl-ajt', + 'res_sle-ajt', + 'res_slefr-ajt', + 'res_sll-ajt', + 'res-jb', + ]; + await Promise.all( + saeNames.map(saeset => + expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() + ) + ); + }); + + +test('gemma-2-9b-it sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + await page.getByText('GEMMA-2-9B-IT').first().click(); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + const saeNames = ['gemmascope-res-131k', 'gemmascope-res-16k']; + + await Promise.all( + saeNames.map(saeset => + expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() + ) + ); + }); + + test('llama3.1-8b sae', async ({ page,searchUrl }) => { + await page.goto(searchUrl); + + await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); + await page.getByText('LLAMA3.1-8B').first().click(); + + await page.locator('[data-state="closed"][data-sentry-source-file="sourceset-selector.tsx"]').click(); + const saeNames = ['llamascope-res-32k']; + + await Promise.all( + saeNames.map(saeset => + expect(page.getByText(saeset, { exact: true }).first()).toBeVisible() + ) + ); + }); diff --git a/apps/webapp/tests-playwright/production/steer.test.ts b/apps/webapp/tests-playwright/production/steer.test.ts deleted file mode 100644 index 9d43e43b2..000000000 --- a/apps/webapp/tests-playwright/production/steer.test.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { expect, test } from '@playwright/test'; - -test('steer page loads', async ({ page }) => { - await page.goto('https://neuronpedia.org/steer'); - await expect(page.locator('text="Steer Models"')).toBeVisible(); -}); - -test('model selector', async ({ page }) => { - await page.goto('https://neuronpedia.org/steer'); - await page.locator('[data-state="closed"][data-sentry-source-file="model-selector.tsx"]').click(); - - const modelNames = [ - 'DEEPSEEK-R1-LLAMA-8B', - 'GEMMA-2-2B', - 'GEMMA-2-2B-IT', - 'GEMMA-2-9B', - 'GEMMA-2-9B-IT', - 'GPT2-SM', - 'LLAMA3.1-8B', - ]; - - for (const model of modelNames) { - await expect(page.getByText(model, { exact: true }).first()).toBeVisible(); - } -}); - -test('deepseek-r1-llama-8b steer', async ({ page }) => { - await page.goto('https://neuronpedia.org/deepseek-r1-distill-llama-8b/steer'); - - // user first occurring demo to search - await page.getByRole('button', { name: 'Demo' }).first().click(); - - // check to see if text changed due to a new response - await expect(page.getByText("Hey, I'm steered deepseek-r1-distill-llama-8b")).not.toBeVisible({ timeout: 30000 }); -}); - -test('gemma-2-2b steer', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-2-2b/steer'); - - // user first occurring demo to search - await page.getByRole('button', { name: 'Demo' }).first().click(); - - // check to see if text changed due to a new response - await expect(page.getByText("Hey, I'm steered gemma-2-2b!")).not.toBeVisible({ timeout: 30000 }); -}); - -test('gemma-2-2b-it steer', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-2-2b-it/steer'); - - // user first occurring demo to search - await page.getByRole('button', { name: 'Demo' }).first().click(); - - // check to see if text changed due to a new response - await expect(page.getByText("Hey, I'm steered gemma-2-2b-it!")).not.toBeVisible({ timeout: 30000 }); -}); - -// No presets -// test('gemma-2-9b steer', async({ page}) => { -// await page.goto('https://neuronpedia.org/gemma-2-9b/steer'); - -// //user first occurring demo to search -// await page.getByRole('button', { name: 'Demo' }).first().click(); - -// //check to see if text changed due to a new response -// await expect(page.getByText("Hey, I'm steered gemma-2-9b!")).not.toBeVisible({ timeout: 30000 }); -// }); - -test('gemma-2-9b-it steer', async ({ page }) => { - await page.goto('https://neuronpedia.org/gemma-2-9b-it/steer'); - - // user first occurring demo to search - await page.getByRole('button', { name: 'Demo' }).first().click(); - - // check to see if text changed due to a new response - await expect(page.getByText("Hey, I'm steered gemma-2-9b-it!")).not.toBeVisible({ timeout: 30000 }); -}); - -test('gpt2-small steer', async ({ page }) => { - await page.goto('https://neuronpedia.org/gpt2-small/steer'); - - // user first occurring demo to search - await page.getByRole('button', { name: 'Demo' }).first().click(); - - // check to see if text changed due to a new response - await expect(page.getByText("Hey, I'm steered gpt2-small!")).not.toBeVisible({ timeout: 30000 }); -}); - -test('llama3.1-8b steer', async ({ page }) => { - await page.goto('https://neuronpedia.org/llama3.1-8b/steer'); - - // user first occurring demo to search - await page.getByRole('button', { name: 'Demo' }).first().click(); - - // check to see if text changed due to a new response - await expect(page.getByText("Hey, I'm steered llama3.1-8b!")).not.toBeVisible({ timeout: 30000 }); -}); diff --git a/apps/webapp/tsconfig.json b/apps/webapp/tsconfig.json index 728abf399..fcc4016ef 100644 --- a/apps/webapp/tsconfig.json +++ b/apps/webapp/tsconfig.json @@ -30,7 +30,8 @@ } ], "downlevelIteration": true, - "typeRoots": ["./types", "./node_modules/@types"] + "typeRoots": ["./types", "./node_modules/@types"], + "types": ["node", "@playwright/test"] }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts", "middleware.ts"], "exclude": ["node_modules"] diff --git a/package-lock.json b/package-lock.json index 45583f92f..c18c37af8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,107 @@ { - "name": "neuronpedia-monorepo", + "name": "neuronpedia", "lockfileVersion": 3, "requires": true, - "packages": {} + "packages": { + "": { + "devDependencies": { + "@playwright/test": "^1.51.1", + "@tailwindcss/forms": "^0.5.10" + } + }, + "node_modules/@playwright/test": { + "version": "1.51.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.51.1.tgz", + "integrity": "sha512-nM+kEaTSAoVlXmMPH10017vn3FSiFqr/bh4fKg9vmAdMfd9SDqRZNvPSiAHADc/itWak+qPvMPZQOPwCBW7k7Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.51.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@tailwindcss/forms": { + "version": "0.5.10", + "resolved": "https://registry.npmjs.org/@tailwindcss/forms/-/forms-0.5.10.tgz", + "integrity": "sha512-utI1ONF6uf/pPNO68kmN1b8rEwNXv3czukalo8VtJH8ksIkZXr3Q3VYudZLkCsDd4Wku120uF02hYK25XGPorw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mini-svg-data-uri": "^1.2.3" + }, + "peerDependencies": { + "tailwindcss": ">=3.0.0 || >= 3.0.0-alpha.1 || >= 4.0.0-alpha.20 || >= 4.0.0-beta.1" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/mini-svg-data-uri": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/mini-svg-data-uri/-/mini-svg-data-uri-1.4.4.tgz", + "integrity": "sha512-r9deDe9p5FJUPZAk3A59wGH7Ii9YrjjWw0jmw/liSbHl2CHiyXj6FcDXDu2K3TjVAXqiJdaw3xxwlZZr9E6nHg==", + "dev": true, + "license": "MIT", + "bin": { + "mini-svg-data-uri": "cli.js" + } + }, + "node_modules/playwright": { + "version": "1.51.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.51.1.tgz", + "integrity": "sha512-kkx+MB2KQRkyxjYPc3a0wLZZoDczmppyGJIvQ43l+aZihkaVvmu/21kiyaHeHjiFxjxNNFnUncKmcGIyOojsaw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.51.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.51.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.51.1.tgz", + "integrity": "sha512-/crRMj8+j/Nq5s8QcvegseuyeZPxpQCZb6HNk3Sos3BlZyAknRjoyJPFWkpNn8v0+P3WiwqFF8P+zQo4eqiNuw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/tailwindcss": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.1.5.tgz", + "integrity": "sha512-nYtSPfWGDiWgCkwQG/m+aX83XCwf62sBgg3bIlNiiOcggnS1x3uVRDAuyelBFL+vJdOPPCGElxv9DjHJjRHiVA==", + "dev": true, + "license": "MIT", + "peer": true + } + } } diff --git a/package.json b/package.json new file mode 100644 index 000000000..daf1520ab --- /dev/null +++ b/package.json @@ -0,0 +1,6 @@ +{ + "devDependencies": { + "@playwright/test": "^1.51.1", + "@tailwindcss/forms": "^0.5.10" + } +}