From 9eb78d380ad6d2ad84279d7cf7a42c684bdd4e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=BE=E3=81=A3=E3=81=A1=E3=82=83=E3=81=A8=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=82=85?= <17376330+u1-liquid@users.noreply.github.com> Date: Tue, 18 Jul 2023 19:37:26 +0900 Subject: [PATCH 1/3] =?UTF-8?q?spec(misskey-host):=20=E3=83=AF=E3=83=BC?= =?UTF-8?q?=E3=82=AF=E3=83=95=E3=83=AD=E3=83=BC=E3=81=AE=E3=82=BB=E3=83=83?= =?UTF-8?q?=E3=83=88=E3=82=A2=E3=83=83=E3=83=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker-host.yml | 50 +++++++++++ .github/workflows/reviewer_lottery.yml | 13 --- .github/workflows/storybook.yml | 112 ------------------------- 3 files changed, 50 insertions(+), 125 deletions(-) create mode 100644 .github/workflows/docker-host.yml delete mode 100644 .github/workflows/reviewer_lottery.yml delete mode 100644 .github/workflows/storybook.yml diff --git a/.github/workflows/docker-host.yml b/.github/workflows/docker-host.yml new file mode 100644 index 000000000000..7f8171715e2d --- /dev/null +++ b/.github/workflows/docker-host.yml @@ -0,0 +1,50 @@ +name: Publish Docker image (host) + +on: + push: + tags: + - "**" + + workflow_dispatch: + +jobs: + push_to_registry: + name: Push Docker image to GitHub Container Registry + runs-on: ubuntu-22.04 + if: github.repository == 'MisskeyIO/misskey' + steps: + - name: Check out the repo + uses: actions/checkout@v3 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + platforms: linux/amd64 + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/misskeyio/misskey + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Prepare image tags + run: | + echo "FORMATTED_BRANCH_NAME=$(echo ${{ github.ref_name }} | sed -e 's/\//-/g' )" >> $GITHUB_ENV + - name: Build and Push to GitHub Container Registry + uses: docker/build-push-action@v4 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + push: true + platforms: ${{ steps.buildx.outputs.platforms }} + provenance: false + labels: ${{ env.FORMATTED_BRANCH_NAME }} + cache-from: type=gha + cache-to: type=gha,mode=max + tags: | + ghcr.io/misskeyio/misskey:host + ghcr.io/misskeyio/misskey:${{ env.FORMATTED_BRANCH_NAME }} diff --git a/.github/workflows/reviewer_lottery.yml b/.github/workflows/reviewer_lottery.yml deleted file mode 100644 index 33228d7465a6..000000000000 --- a/.github/workflows/reviewer_lottery.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: "Reviewer lottery" -on: - pull_request_target: - types: [opened, ready_for_review, reopened] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: uesteibar/reviewer-lottery@v2 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/storybook.yml b/.github/workflows/storybook.yml deleted file mode 100644 index 1aea8b54595e..000000000000 --- a/.github/workflows/storybook.yml +++ /dev/null @@ -1,112 +0,0 @@ -name: Storybook - -on: - push: - branches: - - master - - develop - pull_request_target: - -jobs: - build: - runs-on: ubuntu-latest - - env: - NODE_OPTIONS: "--max_old_space_size=7168" - - steps: - - uses: actions/checkout@v3.3.0 - if: github.event_name != 'pull_request_target' - with: - fetch-depth: 0 - submodules: true - - uses: actions/checkout@v3.3.0 - if: github.event_name == 'pull_request_target' - with: - fetch-depth: 0 - submodules: true - ref: "refs/pull/${{ github.event.number }}/merge" - - name: Checkout actual HEAD - if: github.event_name == 'pull_request_target' - id: rev - run: | - echo "base=$(git rev-list --parents -n1 HEAD | cut -d" " -f2)" >> $GITHUB_OUTPUT - git checkout $(git rev-list --parents -n1 HEAD | cut -d" " -f3) - - name: Install pnpm - uses: pnpm/action-setup@v2 - with: - version: 8 - run_install: false - - name: Use Node.js 20.x - uses: actions/setup-node@v3.6.0 - with: - node-version-file: '.node-version' - cache: 'pnpm' - - run: corepack enable - - run: pnpm i --frozen-lockfile - - name: Check pnpm-lock.yaml - run: git diff --exit-code pnpm-lock.yaml - - name: Build misskey-js - run: pnpm --filter misskey-js build - - name: Build storybook - run: pnpm --filter frontend build-storybook - - name: Publish to Chromatic - if: github.event_name != 'pull_request_target' && github.ref == 'refs/heads/master' - run: pnpm --filter frontend chromatic --exit-once-uploaded -d storybook-static - env: - CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} - - name: Publish to Chromatic - if: github.event_name != 'pull_request_target' && github.ref != 'refs/heads/master' - id: chromatic_push - run: | - DIFF="${{ github.event.before }} HEAD" - if [ "$DIFF" = "0000000000000000000000000000000000000000 HEAD" ]; then - DIFF="HEAD" - fi - CHROMATIC_PARAMETER="$(node packages/frontend/.storybook/changes.js $(git diff-tree --no-commit-id --name-only -r $(echo "$DIFF") | xargs))" - if [ "$CHROMATIC_PARAMETER" = " --skip" ]; then - echo "skip=true" >> $GITHUB_OUTPUT - fi - if pnpm --filter frontend chromatic -d storybook-static $(echo "$CHROMATIC_PARAMETER"); then - echo "success=true" >> $GITHUB_OUTPUT - else - echo "success=false" >> $GITHUB_OUTPUT - fi - env: - CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} - - name: Publish to Chromatic - if: github.event_name == 'pull_request_target' - id: chromatic_pull_request - run: | - DIFF="${{ steps.rev.outputs.base }} HEAD" - if [ "$DIFF" = "0000000000000000000000000000000000000000 HEAD" ]; then - DIFF="HEAD" - fi - CHROMATIC_PARAMETER="$(node packages/frontend/.storybook/changes.js $(git diff-tree --no-commit-id --name-only -r $(echo "$DIFF") | xargs))" - if [ "$CHROMATIC_PARAMETER" = " --skip" ]; then - echo "skip=true" >> $GITHUB_OUTPUT - fi - BRANCH="${{ github.event.pull_request.head.user.login }}:${{ github.event.pull_request.head.ref }}" - if [ "$BRANCH" = "misskey-dev:${{ github.event.pull_request.head.ref }}" ]; then - BRANCH="${{ github.event.pull_request.head.ref }}" - fi - pnpm --filter frontend chromatic --exit-once-uploaded -d storybook-static --branch-name $BRANCH $(echo "$CHROMATIC_PARAMETER") - env: - CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} - - name: Notify that Chromatic detects changes - uses: actions/github-script@v6.4.0 - if: github.event_name != 'pull_request_target' && steps.chromatic_push.outputs.success == 'false' - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - github.rest.repos.createCommitComment({ - owner: context.repo.owner, - repo: context.repo.repo, - commit_sha: context.sha, - body: 'Chromatic detects changes. Please [review the changes on Chromatic](https://www.chromatic.com/builds?appId=6428f7d7b962f0b79f97d6e4).' - }) - - name: Upload Artifacts - uses: actions/upload-artifact@v3 - with: - name: storybook - path: packages/frontend/storybook-static From c27e3bd72cc2360b1aacaea0d61650a2e25dbae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=BE=E3=81=A3=E3=81=A1=E3=82=83=E3=81=A8=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=82=85?= <17376330+u1-liquid@users.noreply.github.com> Date: Wed, 19 Jul 2023 04:46:03 +0900 Subject: [PATCH 2/3] =?UTF-8?q?spec(misskey-host):=20=E3=82=AA=E3=83=96?= =?UTF-8?q?=E3=82=B8=E3=82=A7=E3=82=AF=E3=83=88=E3=82=B9=E3=83=88=E3=83=AC?= =?UTF-8?q?=E3=83=BC=E3=82=B8=E3=82=92=E8=A8=AD=E5=AE=9A=E3=83=95=E3=82=A1?= =?UTF-8?q?=E3=82=A4=E3=83=AB=E3=81=A7=E7=AE=A1=E7=90=86=E3=81=99=E3=82=8B?= =?UTF-8?q?=E6=A7=98=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 元コミット: ff1bd68ebdfc9581e3b84a07b0843a791187e54c --- .config/example.yml | 17 ++ packages/backend/src/config.ts | 15 ++ packages/backend/src/core/DriveService.ts | 27 ++- packages/backend/src/core/S3Service.ts | 33 ++-- packages/backend/src/models/entities/Meta.ts | 72 -------- .../src/server/api/endpoints/admin/meta.ts | 61 ------- .../server/api/endpoints/admin/update-meta.ts | 65 -------- .../backend/src/server/api/endpoints/meta.ts | 2 +- packages/backend/test/unit/S3Service.ts | 9 +- packages/frontend/src/pages/admin/index.vue | 5 - .../src/pages/admin/object-storage.vue | 157 ------------------ packages/frontend/src/router.ts | 4 - 12 files changed, 63 insertions(+), 404 deletions(-) delete mode 100644 packages/frontend/src/pages/admin/object-storage.vue diff --git a/.config/example.yml b/.config/example.yml index b996a83fb50a..e83b71ac6b0f 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -116,6 +116,23 @@ redis: # index: '' # scope: local +# ┌────────────────────────────────────┐ +#───┘ File storage (Drive) configuration └────────────────────── + +#s3: +# baseUrl: s3.example.com +# bucket: example-bucket +# prefix: example-prefix +# endpoint: s3.example.com +# region: us-east-1 +# useSSL: true +# accessKey: example-access-key +# secretKey: example-secret-key +# options: +# setPublicRead: true +# forcePathStyle: false +# useProxy: false + # ┌───────────────┐ #───┘ ID generation └─────────────────────────────────────────── diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 253975096e2d..bc86e59980d3 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -56,6 +56,21 @@ export type Source = { index: string; scope?: 'local' | 'global' | string[]; }; + s3?: { + baseUrl: string; + bucket: string; + prefix: string; + endpoint: string; + region?: string; + useSSL: boolean; + accessKey: string; + secretKey: string; + options?: { + setPublicRead?: boolean; + forcePathStyle?: boolean; + useProxy?: boolean; + } + }; proxy?: string; proxySmtp?: string; diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 355e5e8c0dd5..1819c6032184 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -137,9 +137,7 @@ export class DriveService { // thunbnail, webpublic を必要なら生成 const alts = await this.generateAlts(path, type, !file.uri); - const meta = await this.metaService.fetch(); - - if (meta.useObjectStorage) { + if (this.config.s3) { //#region ObjectStorage params let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) ?? ['']); @@ -158,11 +156,11 @@ export class DriveService { ext = ''; } - const baseUrl = meta.objectStorageBaseUrl - ?? `${ meta.objectStorageUseSSL ? 'https' : 'http' }://${ meta.objectStorageEndpoint }${ meta.objectStoragePort ? `:${meta.objectStoragePort}` : '' }/${ meta.objectStorageBucket }`; + const baseUrl = this.config.s3.baseUrl + ?? `${ this.config.s3.useSSL ? 'https' : 'http' }://${ this.config.s3.endpoint }/${ this.config.s3.bucket }`; // for original - const key = `${meta.objectStoragePrefix}/${randomUUID()}${ext}`; + const key = `${this.config.s3.prefix}/${randomUUID()}${ext}`; const url = `${ baseUrl }/${ key }`; // for alts @@ -179,7 +177,7 @@ export class DriveService { ]; if (alts.webpublic) { - webpublicKey = `${meta.objectStoragePrefix}/webpublic-${randomUUID()}.${alts.webpublic.ext}`; + webpublicKey = `${this.config.s3.prefix}/webpublic-${randomUUID()}.${alts.webpublic.ext}`; webpublicUrl = `${ baseUrl }/${ webpublicKey }`; this.registerLogger.info(`uploading webpublic: ${webpublicKey}`); @@ -187,7 +185,7 @@ export class DriveService { } if (alts.thumbnail) { - thumbnailKey = `${meta.objectStoragePrefix}/thumbnail-${randomUUID()}.${alts.thumbnail.ext}`; + thumbnailKey = `${this.config.s3.prefix}/thumbnail-${randomUUID()}.${alts.thumbnail.ext}`; thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`; this.registerLogger.info(`uploading thumbnail: ${thumbnailKey}`); @@ -364,10 +362,8 @@ export class DriveService { if (type === 'image/apng') type = 'image/png'; if (!FILE_TYPE_BROWSERSAFE.includes(type)) type = 'application/octet-stream'; - const meta = await this.metaService.fetch(); - const params = { - Bucket: meta.objectStorageBucket, + Bucket: this.config.s3?.bucket, Key: key, Body: stream, ContentType: type, @@ -380,9 +376,9 @@ export class DriveService { // 許可されているファイル形式でしか拡張子をつけない ext ? correctFilename(filename, ext) : filename, ); - if (meta.objectStorageSetPublicRead) params.ACL = 'public-read'; + if (this.config.s3?.options?.setPublicRead) params.ACL = 'public-read'; - await this.s3Service.upload(meta, params) + await this.s3Service.upload(params) .then( result => { if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput @@ -734,14 +730,13 @@ export class DriveService { @bindThis public async deleteObjectStorageFile(key: string) { - const meta = await this.metaService.fetch(); try { const param = { - Bucket: meta.objectStorageBucket, + Bucket: this.config.s3?.bucket, Key: key, } as DeleteObjectCommandInput; - await this.s3Service.delete(meta, param); + await this.s3Service.delete(param); } catch (err: any) { if (err.name === 'NoSuchKey') { this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error); diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts index 01ce12ffdd9c..d9c0fb13dd73 100644 --- a/packages/backend/src/core/S3Service.ts +++ b/packages/backend/src/core/S3Service.ts @@ -7,7 +7,6 @@ import { Upload } from '@aws-sdk/lib-storage'; import { NodeHttpHandler, NodeHttpHandlerOptions } from '@aws-sdk/node-http-handler'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; -import type { Meta } from '@/models/entities/Meta.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import type { DeleteObjectCommandInput, PutObjectCommandInput } from '@aws-sdk/client-s3'; @@ -23,35 +22,33 @@ export class S3Service { } @bindThis - public getS3Client(meta: Meta): S3Client { - const u = meta.objectStorageEndpoint - ? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}` - : `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent + public getS3Client(): S3Client { + const u = `${this.config.s3?.useSSL ? 'https' : 'http'}://${this.config.s3?.endpoint ?? 'example.net'}`; // dummy url to select http(s) agent - const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy); + const agent = this.httpRequestService.getAgentByUrl(new URL(u), !this.config.s3?.options?.useProxy); const handlerOption: NodeHttpHandlerOptions = {}; - if (meta.objectStorageUseSSL) { + if (this.config.s3?.useSSL) { handlerOption.httpsAgent = agent as https.Agent; } else { handlerOption.httpAgent = agent as http.Agent; } return new S3Client({ - endpoint: meta.objectStorageEndpoint ? u : undefined, - credentials: (meta.objectStorageAccessKey !== null && meta.objectStorageSecretKey !== null) ? { - accessKeyId: meta.objectStorageAccessKey, - secretAccessKey: meta.objectStorageSecretKey, + endpoint: this.config.s3?.endpoint ? u : undefined, + credentials: (this.config.s3 && (this.config.s3.accessKey !== null && this.config.s3.secretKey !== null)) ? { + accessKeyId: this.config.s3.accessKey, + secretAccessKey: this.config.s3.secretKey, } : undefined, - region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, // 空文字列もundefinedにするため ?? は使わない - tls: meta.objectStorageUseSSL, - forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted + region: this.config.s3?.region ?? 'us-east-1', + tls: this.config.s3?.useSSL, + forcePathStyle: this.config.s3?.options?.forcePathStyle ?? false, // AWS with endPoint omitted requestHandler: new NodeHttpHandler(handlerOption), }); } @bindThis - public async upload(meta: Meta, input: PutObjectCommandInput) { - const client = this.getS3Client(meta); + public async upload(input: PutObjectCommandInput) { + const client = this.getS3Client(); return new Upload({ client, params: input, @@ -62,8 +59,8 @@ export class S3Service { } @bindThis - public delete(meta: Meta, input: DeleteObjectCommandInput) { - const client = this.getS3Client(meta); + public delete(input: DeleteObjectCommandInput) { + const client = this.getS3Client(); return client.send(new DeleteObjectCommand(input)); } } diff --git a/packages/backend/src/models/entities/Meta.ts b/packages/backend/src/models/entities/Meta.ts index 7bb1b677123f..755f8bccb875 100644 --- a/packages/backend/src/models/entities/Meta.ts +++ b/packages/backend/src/models/entities/Meta.ts @@ -325,78 +325,6 @@ export class Meta { }) public defaultDarkTheme: string | null; - @Column('boolean', { - default: false, - }) - public useObjectStorage: boolean; - - @Column('varchar', { - length: 1024, - nullable: true, - }) - public objectStorageBucket: string | null; - - @Column('varchar', { - length: 1024, - nullable: true, - }) - public objectStoragePrefix: string | null; - - @Column('varchar', { - length: 1024, - nullable: true, - }) - public objectStorageBaseUrl: string | null; - - @Column('varchar', { - length: 1024, - nullable: true, - }) - public objectStorageEndpoint: string | null; - - @Column('varchar', { - length: 1024, - nullable: true, - }) - public objectStorageRegion: string | null; - - @Column('varchar', { - length: 1024, - nullable: true, - }) - public objectStorageAccessKey: string | null; - - @Column('varchar', { - length: 1024, - nullable: true, - }) - public objectStorageSecretKey: string | null; - - @Column('integer', { - nullable: true, - }) - public objectStoragePort: number | null; - - @Column('boolean', { - default: true, - }) - public objectStorageUseSSL: boolean; - - @Column('boolean', { - default: true, - }) - public objectStorageUseProxy: boolean; - - @Column('boolean', { - default: false, - }) - public objectStorageSetPublicRead: boolean; - - @Column('boolean', { - default: true, - }) - public objectStorageS3ForcePathStyle: boolean; - @Column('boolean', { default: false, }) diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 084bdb598be6..5daa8db5ac32 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -201,54 +201,6 @@ export const meta = { type: 'string', optional: true, nullable: true, }, - useObjectStorage: { - type: 'boolean', - optional: true, nullable: false, - }, - objectStorageBaseUrl: { - type: 'string', - optional: true, nullable: true, - }, - objectStorageBucket: { - type: 'string', - optional: true, nullable: true, - }, - objectStoragePrefix: { - type: 'string', - optional: true, nullable: true, - }, - objectStorageEndpoint: { - type: 'string', - optional: true, nullable: true, - }, - objectStorageRegion: { - type: 'string', - optional: true, nullable: true, - }, - objectStoragePort: { - type: 'number', - optional: true, nullable: true, - }, - objectStorageAccessKey: { - type: 'string', - optional: true, nullable: true, - }, - objectStorageSecretKey: { - type: 'string', - optional: true, nullable: true, - }, - objectStorageUseSSL: { - type: 'boolean', - optional: true, nullable: false, - }, - objectStorageUseProxy: { - type: 'boolean', - optional: true, nullable: false, - }, - objectStorageSetPublicRead: { - type: 'boolean', - optional: true, nullable: false, - }, enableIpLogging: { type: 'boolean', optional: true, nullable: false, @@ -357,19 +309,6 @@ export default class extends Endpoint { smtpUser: instance.smtpUser, smtpPass: instance.smtpPass, swPrivateKey: instance.swPrivateKey, - useObjectStorage: instance.useObjectStorage, - objectStorageBaseUrl: instance.objectStorageBaseUrl, - objectStorageBucket: instance.objectStorageBucket, - objectStoragePrefix: instance.objectStoragePrefix, - objectStorageEndpoint: instance.objectStorageEndpoint, - objectStorageRegion: instance.objectStorageRegion, - objectStoragePort: instance.objectStoragePort, - objectStorageAccessKey: instance.objectStorageAccessKey, - objectStorageSecretKey: instance.objectStorageSecretKey, - objectStorageUseSSL: instance.objectStorageUseSSL, - objectStorageUseProxy: instance.objectStorageUseProxy, - objectStorageSetPublicRead: instance.objectStorageSetPublicRead, - objectStorageS3ForcePathStyle: instance.objectStorageS3ForcePathStyle, deeplAuthKey: instance.deeplAuthKey, deeplIsPro: instance.deeplIsPro, enableIpLogging: instance.enableIpLogging, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 144360a9218c..463597489e09 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -80,19 +80,6 @@ export const paramDef = { tosUrl: { type: 'string', nullable: true }, repositoryUrl: { type: 'string' }, feedbackUrl: { type: 'string' }, - useObjectStorage: { type: 'boolean' }, - objectStorageBaseUrl: { type: 'string', nullable: true }, - objectStorageBucket: { type: 'string', nullable: true }, - objectStoragePrefix: { type: 'string', nullable: true }, - objectStorageEndpoint: { type: 'string', nullable: true }, - objectStorageRegion: { type: 'string', nullable: true }, - objectStoragePort: { type: 'integer', nullable: true }, - objectStorageAccessKey: { type: 'string', nullable: true }, - objectStorageSecretKey: { type: 'string', nullable: true }, - objectStorageUseSSL: { type: 'boolean' }, - objectStorageUseProxy: { type: 'boolean' }, - objectStorageSetPublicRead: { type: 'boolean' }, - objectStorageS3ForcePathStyle: { type: 'boolean' }, enableIpLogging: { type: 'boolean' }, enableActiveEmailValidation: { type: 'boolean' }, enableChartsForRemoteUser: { type: 'boolean' }, @@ -326,58 +313,6 @@ export default class extends Endpoint { set.feedbackUrl = ps.feedbackUrl; } - if (ps.useObjectStorage !== undefined) { - set.useObjectStorage = ps.useObjectStorage; - } - - if (ps.objectStorageBaseUrl !== undefined) { - set.objectStorageBaseUrl = ps.objectStorageBaseUrl; - } - - if (ps.objectStorageBucket !== undefined) { - set.objectStorageBucket = ps.objectStorageBucket; - } - - if (ps.objectStoragePrefix !== undefined) { - set.objectStoragePrefix = ps.objectStoragePrefix; - } - - if (ps.objectStorageEndpoint !== undefined) { - set.objectStorageEndpoint = ps.objectStorageEndpoint; - } - - if (ps.objectStorageRegion !== undefined) { - set.objectStorageRegion = ps.objectStorageRegion; - } - - if (ps.objectStoragePort !== undefined) { - set.objectStoragePort = ps.objectStoragePort; - } - - if (ps.objectStorageAccessKey !== undefined) { - set.objectStorageAccessKey = ps.objectStorageAccessKey; - } - - if (ps.objectStorageSecretKey !== undefined) { - set.objectStorageSecretKey = ps.objectStorageSecretKey; - } - - if (ps.objectStorageUseSSL !== undefined) { - set.objectStorageUseSSL = ps.objectStorageUseSSL; - } - - if (ps.objectStorageUseProxy !== undefined) { - set.objectStorageUseProxy = ps.objectStorageUseProxy; - } - - if (ps.objectStorageSetPublicRead !== undefined) { - set.objectStorageSetPublicRead = ps.objectStorageSetPublicRead; - } - - if (ps.objectStorageS3ForcePathStyle !== undefined) { - set.objectStorageS3ForcePathStyle = ps.objectStorageS3ForcePathStyle; - } - if (ps.deeplAuthKey !== undefined) { if (ps.deeplAuthKey === '') { set.deeplAuthKey = null; diff --git a/packages/backend/src/server/api/endpoints/meta.ts b/packages/backend/src/server/api/endpoints/meta.ts index adfa579558aa..8245ed65aa3e 100644 --- a/packages/backend/src/server/api/endpoints/meta.ts +++ b/packages/backend/src/server/api/endpoints/meta.ts @@ -350,7 +350,7 @@ export default class extends Endpoint { hcaptcha: instance.enableHcaptcha, recaptcha: instance.enableRecaptcha, turnstile: instance.enableTurnstile, - objectStorage: instance.useObjectStorage, + objectStorage: !!config.s3, serviceWorker: instance.enableServiceWorker, miauth: true, }; diff --git a/packages/backend/test/unit/S3Service.ts b/packages/backend/test/unit/S3Service.ts index 1dfa22afd290..86dd96c13b34 100644 --- a/packages/backend/test/unit/S3Service.ts +++ b/packages/backend/test/unit/S3Service.ts @@ -6,7 +6,6 @@ import { mockClient } from 'aws-sdk-client-mock'; import { GlobalModule } from '@/GlobalModule.js'; import { CoreModule } from '@/core/CoreModule.js'; import { S3Service } from '@/core/S3Service'; -import { Meta } from '@/models'; import type { TestingModule } from '@nestjs/testing'; describe('S3Service', () => { @@ -35,7 +34,7 @@ describe('S3Service', () => { test('upload a file', async () => { s3Mock.on(PutObjectCommand).resolves({}); - await s3Service.upload({ objectStorageRegion: 'us-east-1' } as Meta, { + await s3Service.upload({ Bucket: 'fake', Key: 'fake', Body: 'x', @@ -47,7 +46,7 @@ describe('S3Service', () => { s3Mock.on(UploadPartCommand).resolves({ ETag: '1' }); s3Mock.on(CompleteMultipartUploadCommand).resolves({ Bucket: 'fake', Key: 'fake' }); - await s3Service.upload({} as Meta, { + await s3Service.upload({ Bucket: 'fake', Key: 'fake', Body: 'x'.repeat(8 * 1024 * 1024 + 1), // デフォルトpartSizeにしている 8 * 1024 * 1024 を越えるサイズ @@ -57,7 +56,7 @@ describe('S3Service', () => { test('upload a file error', async () => { s3Mock.on(PutObjectCommand).rejects({ name: 'Fake Error' }); - await expect(s3Service.upload({ objectStorageRegion: 'us-east-1' } as Meta, { + await expect(s3Service.upload({ Bucket: 'fake', Key: 'fake', Body: 'x', @@ -67,7 +66,7 @@ describe('S3Service', () => { test('upload a large file error', async () => { s3Mock.on(UploadPartCommand).rejects(); - await expect(s3Service.upload({} as Meta, { + await expect(s3Service.upload({ Bucket: 'fake', Key: 'fake', Body: 'x'.repeat(8 * 1024 * 1024 + 1), // デフォルトpartSizeにしている 8 * 1024 * 1024 を越えるサイズ diff --git a/packages/frontend/src/pages/admin/index.vue b/packages/frontend/src/pages/admin/index.vue index e91f65b5d57e..3945b290c70f 100644 --- a/packages/frontend/src/pages/admin/index.vue +++ b/packages/frontend/src/pages/admin/index.vue @@ -163,11 +163,6 @@ const menuDef = $computed(() => [{ text: i18n.ts.emailServer, to: '/admin/email-settings', active: currentPage?.route.name === 'email-settings', - }, { - icon: 'ti ti-cloud', - text: i18n.ts.objectStorage, - to: '/admin/object-storage', - active: currentPage?.route.name === 'object-storage', }, { icon: 'ti ti-lock', text: i18n.ts.security, diff --git a/packages/frontend/src/pages/admin/object-storage.vue b/packages/frontend/src/pages/admin/object-storage.vue deleted file mode 100644 index e569aad1b88c..000000000000 --- a/packages/frontend/src/pages/admin/object-storage.vue +++ /dev/null @@ -1,157 +0,0 @@ - - - - - diff --git a/packages/frontend/src/router.ts b/packages/frontend/src/router.ts index a4276ff4c0f7..d3a7d011861c 100644 --- a/packages/frontend/src/router.ts +++ b/packages/frontend/src/router.ts @@ -404,10 +404,6 @@ export const routes = [{ path: '/email-settings', name: 'email-settings', component: page(() => import('./pages/admin/email-settings.vue')), - }, { - path: '/object-storage', - name: 'object-storage', - component: page(() => import('./pages/admin/object-storage.vue')), }, { path: '/security', name: 'security', From becfdc2b7fdabd2840e2184d42f21c2b31fa26f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=BE=E3=81=A3=E3=81=A1=E3=82=83=E3=81=A8=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=82=85?= <17376330+u1-liquid@users.noreply.github.com> Date: Sat, 22 Jul 2023 03:01:49 +0900 Subject: [PATCH 3/3] =?UTF-8?q?spec(misskey-host):=20=E3=83=AA=E3=83=A2?= =?UTF-8?q?=E3=83=BC=E3=83=88=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=92?= =?UTF-8?q?=E3=82=AD=E3=83=A3=E3=83=83=E3=82=B7=E3=83=A5=E3=81=97=E3=81=AA?= =?UTF-8?q?=E3=81=84=E6=A7=98=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 元コミット: b47d7e6be2cefcaf2a12e43f67f9db19245b0b0a --- .../core/activitypub/models/ApImageService.ts | 11 +--- packages/backend/src/models/entities/Meta.ts | 10 ---- .../src/server/api/endpoints/admin/meta.ts | 10 ---- .../server/api/endpoints/admin/update-meta.ts | 10 ---- .../backend/src/server/api/endpoints/meta.ts | 4 +- packages/backend/test/unit/activitypub.ts | 60 ------------------- .../frontend/src/pages/admin/settings.vue | 24 -------- 7 files changed, 3 insertions(+), 126 deletions(-) diff --git a/packages/backend/src/core/activitypub/models/ApImageService.ts b/packages/backend/src/core/activitypub/models/ApImageService.ts index 1f2984894c32..05c83cc9594b 100644 --- a/packages/backend/src/core/activitypub/models/ApImageService.ts +++ b/packages/backend/src/core/activitypub/models/ApImageService.ts @@ -3,7 +3,6 @@ import { DI } from '@/di-symbols.js'; import type { DriveFilesRepository } from '@/models/index.js'; import type { RemoteUser } from '@/models/entities/User.js'; import type { DriveFile } from '@/models/entities/DriveFile.js'; -import { MetaService } from '@/core/MetaService.js'; import { truncate } from '@/misc/truncate.js'; import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/const.js'; import { DriveService } from '@/core/DriveService.js'; @@ -22,7 +21,6 @@ export class ApImageService { @Inject(DI.driveFilesRepository) private driveFilesRepository: DriveFilesRepository, - private metaService: MetaService, private apResolverService: ApResolverService, private driveService: DriveService, private apLoggerService: ApLoggerService, @@ -56,19 +54,12 @@ export class ApImageService { this.logger.info(`Creating the Image: ${image.url}`); - const instance = await this.metaService.fetch(); - - // Cache if remote file cache is on AND either - // 1. remote sensitive file is also on - // 2. or the image is not sensitive - const shouldBeCached = instance.cacheRemoteFiles && (instance.cacheRemoteSensitiveFiles || !image.sensitive); - const file = await this.driveService.uploadFromUrl({ url: image.url, user: actor, uri: image.url, sensitive: image.sensitive, - isLink: !shouldBeCached, + isLink: true, comment: truncate(image.name ?? undefined, DB_MAX_IMAGE_COMMENT_LENGTH), }); if (!file.isLink || file.url === image.url) return file; diff --git a/packages/backend/src/models/entities/Meta.ts b/packages/backend/src/models/entities/Meta.ts index 755f8bccb875..e46a6a456835 100644 --- a/packages/backend/src/models/entities/Meta.ts +++ b/packages/backend/src/models/entities/Meta.ts @@ -120,16 +120,6 @@ export class Meta { }) public infoImageUrl: string | null; - @Column('boolean', { - default: true, - }) - public cacheRemoteFiles: boolean; - - @Column('boolean', { - default: true, - }) - public cacheRemoteSensitiveFiles: boolean; - @Column({ ...id(), nullable: true, diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 5daa8db5ac32..45402ed42d1c 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -15,14 +15,6 @@ export const meta = { type: 'object', optional: false, nullable: false, properties: { - cacheRemoteFiles: { - type: 'boolean', - optional: false, nullable: false, - }, - cacheRemoteSensitiveFiles: { - type: 'boolean', - optional: false, nullable: false, - }, emailRequiredForSignup: { type: 'boolean', optional: false, nullable: false, @@ -286,8 +278,6 @@ export default class extends Endpoint { enableEmail: instance.enableEmail, enableServiceWorker: instance.enableServiceWorker, translatorAvailable: instance.deeplAuthKey != null, - cacheRemoteFiles: instance.cacheRemoteFiles, - cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles, pinnedUsers: instance.pinnedUsers, hiddenTags: instance.hiddenTags, blockedHosts: instance.blockedHosts, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 463597489e09..33b2e2eb716c 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -42,8 +42,6 @@ export const paramDef = { description: { type: 'string', nullable: true }, defaultLightTheme: { type: 'string', nullable: true }, defaultDarkTheme: { type: 'string', nullable: true }, - cacheRemoteFiles: { type: 'boolean' }, - cacheRemoteSensitiveFiles: { type: 'boolean' }, emailRequiredForSignup: { type: 'boolean' }, enableHcaptcha: { type: 'boolean' }, hcaptchaSiteKey: { type: 'string', nullable: true }, @@ -177,14 +175,6 @@ export default class extends Endpoint { set.defaultDarkTheme = ps.defaultDarkTheme; } - if (ps.cacheRemoteFiles !== undefined) { - set.cacheRemoteFiles = ps.cacheRemoteFiles; - } - - if (ps.cacheRemoteSensitiveFiles !== undefined) { - set.cacheRemoteSensitiveFiles = ps.cacheRemoteSensitiveFiles; - } - if (ps.emailRequiredForSignup !== undefined) { set.emailRequiredForSignup = ps.emailRequiredForSignup; } diff --git a/packages/backend/src/server/api/endpoints/meta.ts b/packages/backend/src/server/api/endpoints/meta.ts index 8245ed65aa3e..5ab821041a7e 100644 --- a/packages/backend/src/server/api/endpoints/meta.ts +++ b/packages/backend/src/server/api/endpoints/meta.ts @@ -332,8 +332,8 @@ export default class extends Endpoint { mediaProxy: this.config.mediaProxy, ...(ps.detail ? { - cacheRemoteFiles: instance.cacheRemoteFiles, - cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles, + cacheRemoteFiles: false, + cacheRemoteSensitiveFiles: false, requireSetup: (await this.usersRepository.countBy({ host: IsNull(), })) === 0, diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index 78b916c11281..12d45c09c0f9 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -85,8 +85,6 @@ describe('ActivityPub', () => { let resolver: MockResolver; const metaInitial = { - cacheRemoteFiles: true, - cacheRemoteSensitiveFiles: true, blockedHosts: [] as string[], sensitiveWords: [] as string[], } as Meta; @@ -258,35 +256,6 @@ describe('ActivityPub', () => { describe('Images', () => { test('Create images', async () => { - const imageObject: IApDocument = { - type: 'Document', - mediaType: 'image/png', - url: 'http://host1.test/foo.png', - name: '', - }; - const driveFile = await imageService.createImage( - await createRandomRemoteUser(resolver, personService), - imageObject, - ); - assert.ok(!driveFile.isLink); - - const sensitiveImageObject: IApDocument = { - type: 'Document', - mediaType: 'image/png', - url: 'http://host1.test/bar.png', - name: '', - sensitive: true, - }; - const sensitiveDriveFile = await imageService.createImage( - await createRandomRemoteUser(resolver, personService), - sensitiveImageObject, - ); - assert.ok(!sensitiveDriveFile.isLink); - }); - - test('cacheRemoteFiles=false disables caching', async () => { - meta = { ...metaInitial, cacheRemoteFiles: false }; - const imageObject: IApDocument = { type: 'Document', mediaType: 'image/png', @@ -312,34 +281,5 @@ describe('ActivityPub', () => { ); assert.ok(sensitiveDriveFile.isLink); }); - - test('cacheRemoteSensitiveFiles=false only affects sensitive files', async () => { - meta = { ...metaInitial, cacheRemoteSensitiveFiles: false }; - - const imageObject: IApDocument = { - type: 'Document', - mediaType: 'image/png', - url: 'http://host1.test/foo.png', - name: '', - }; - const driveFile = await imageService.createImage( - await createRandomRemoteUser(resolver, personService), - imageObject, - ); - assert.ok(!driveFile.isLink); - - const sensitiveImageObject: IApDocument = { - type: 'Document', - mediaType: 'image/png', - url: 'http://host1.test/bar.png', - name: '', - sensitive: true, - }; - const sensitiveDriveFile = await imageService.createImage( - await createRandomRemoteUser(resolver, personService), - sensitiveImageObject, - ); - assert.ok(sensitiveDriveFile.isLink); - }); }); }); diff --git a/packages/frontend/src/pages/admin/settings.vue b/packages/frontend/src/pages/admin/settings.vue index bd57c061812f..430260bd8b34 100644 --- a/packages/frontend/src/pages/admin/settings.vue +++ b/packages/frontend/src/pages/admin/settings.vue @@ -29,24 +29,6 @@ - - - -
- - - - - - -
- - @@ -117,8 +99,6 @@ let description: string | null = $ref(null); let maintainerName: string | null = $ref(null); let maintainerEmail: string | null = $ref(null); let pinnedUsers: string = $ref(''); -let cacheRemoteFiles: boolean = $ref(false); -let cacheRemoteSensitiveFiles: boolean = $ref(false); let enableServiceWorker: boolean = $ref(false); let swPublicKey: any = $ref(null); let swPrivateKey: any = $ref(null); @@ -132,8 +112,6 @@ async function init(): Promise { maintainerName = meta.maintainerName; maintainerEmail = meta.maintainerEmail; pinnedUsers = meta.pinnedUsers.join('\n'); - cacheRemoteFiles = meta.cacheRemoteFiles; - cacheRemoteSensitiveFiles = meta.cacheRemoteSensitiveFiles; enableServiceWorker = meta.enableServiceWorker; swPublicKey = meta.swPublickey; swPrivateKey = meta.swPrivateKey; @@ -148,8 +126,6 @@ function save(): void { maintainerName, maintainerEmail, pinnedUsers: pinnedUsers.split('\n'), - cacheRemoteFiles, - cacheRemoteSensitiveFiles, enableServiceWorker, swPublicKey, swPrivateKey,