From 276a4150db66f564deb75153e754a8427c5b6b37 Mon Sep 17 00:00:00 2001 From: krishna Date: Wed, 2 Jul 2025 11:42:27 -0500 Subject: [PATCH 01/14] Create a configuration for perf in gen3-gitops under ci --- .github/workflows/integration-tests.yaml | 31 + .secrets.baseline | 11 +- ci/default/values/etl.yaml | 4 - ci/default/values/fence.yaml | 641 ++++++++++++++-- ci/default/values/guppy.yaml | 20 +- ci/default/values/portal.yaml | 9 +- ci/default/values/values.yaml | 456 ++++++----- ci/perf/Chart.yaml | 23 + ci/perf/templates/app.yaml | 33 + ci/perf/values.yaml | 0 ci/perf/values/fence.yaml | 605 +++++++++++++++ ci/perf/values/guppy.yaml | 32 + ci/perf/values/portal.yaml | 924 +++++++++++++++++++++++ ci/perf/values/values.yaml | 303 ++++++++ 14 files changed, 2814 insertions(+), 278 deletions(-) create mode 100644 .github/workflows/integration-tests.yaml create mode 100644 ci/perf/Chart.yaml create mode 100644 ci/perf/templates/app.yaml create mode 100644 ci/perf/values.yaml create mode 100644 ci/perf/values/fence.yaml create mode 100644 ci/perf/values/guppy.yaml create mode 100644 ci/perf/values/portal.yaml create mode 100644 ci/perf/values/values.yaml diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml new file mode 100644 index 000000000..0e6c46e1e --- /dev/null +++ b/.github/workflows/integration-tests.yaml @@ -0,0 +1,31 @@ +name: Integration Tests + +on: pull_request + +permissions: + id-token: write + contents: read + pull-requests: write + issues: write + +jobs: + integration_tests: + name: Integration tests + uses: uc-cdis/gen3-code-vigil/.github/workflows/shared_integration_tests.yaml@feat/GPE-1747 + with: + TEST_REPO_BRANCH: chore/tweak_helm_ci + GEN3_INSTANCE_TYPE: HELM_LOCAL + secrets: + CI_AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }} + CI_AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }} + QA_DASHBOARD_S3_PATH: ${{ secrets.QA_DASHBOARD_S3_PATH }} + JENKINS_API_TOKEN: ${{ secrets.JENKINS_API_TOKEN }} + CI_TEST_ORCID_USERID: ${{ secrets.CI_TEST_ORCID_USERID }} + CI_TEST_ORCID_PASSWORD: ${{ secrets.CI_TEST_ORCID_PASSWORD }} + CI_TEST_RAS_EMAIL: ${{ secrets.CI_TEST_RAS_EMAIL }} + CI_TEST_RAS_PASSWORD: ${{ secrets.CI_TEST_RAS_PASSWORD }} + CI_TEST_RAS_2_EMAIL: ${{ secrets.CI_TEST_RAS_2_EMAIL }} + CI_TEST_RAS_2_PASSWORD: ${{ secrets.CI_TEST_RAS_2_PASSWORD }} + CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }} + CI_SLACK_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID }} + EKS_CLUSTER_NAME: ${{ secrets.EKS_CLUSTER_NAME }} diff --git a/.secrets.baseline b/.secrets.baseline index 7927cc06f..23a886622 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -255,15 +255,6 @@ "line_number": 15 } ], - "ci/default/values/values.yaml": [ - { - "type": "Secret Keyword", - "filename": "ci/default/values/values.yaml", - "hashed_secret": "9b5925ea817163740dfb287a9894e8ab3aba2c18", - "is_verified": false, - "line_number": 348 - } - ], "covid19prod/chicagoland.pandemicresponsecommons.org/values/values.yaml": [ { "type": "Base64 High Entropy String", @@ -1847,5 +1838,5 @@ } ] }, - "generated_at": "2025-07-01T20:31:40Z" + "generated_at": "2025-07-02T16:42:22Z" } diff --git a/ci/default/values/etl.yaml b/ci/default/values/etl.yaml index 34f49c683..989f3751c 100644 --- a/ci/default/values/etl.yaml +++ b/ci/default/values/etl.yaml @@ -1,13 +1,9 @@ - etl: enabled: true image: spark: repository: "quay.io/cdis/gen3-spark" tag: "master" - tube: - repository: "quay.io/cdis/tube" - tag: "2024.12" etlMapping: mappings: - name: ci.planx-ci.io_subject diff --git a/ci/default/values/fence.yaml b/ci/default/values/fence.yaml index 739c1f4a6..80b00a329 100644 --- a/ci/default/values/fence.yaml +++ b/ci/default/values/fence.yaml @@ -1,80 +1,603 @@ fence: - # Lower cost - resources: - requests: - memory: "105Mi" - cpu: "15m" + # -- (map) External Secrets settings. + externalSecrets: + createK8sJwtKeysSecret: false + fenceJwtKeys: ci-fence-jwt + fenceGoogleAppCredsSecret: ci-fence-google-app-creds-secret + fenceGoogleStorageCredsSecret: ci-fence-google-storage-creds-secret + createK8sGoogleAppSecrets: false + fenceConfig: ci-fence-config-priv + fenceSshKeys: ci-fence-ssh-keys + fenceDeleteExpiredClients: + schedule: "0 0 * * *" + replicaCount: 2 image: repository: quay.io/cdis/fence pullPolicy: IfNotPresent - tag: '2025.01' - - # -- (map) Annotations to add to the pod - ## TODO ## Do we need this? - # podAnnotations: - # prometheus.io/path: /metrics - # prometheus.io/scrape: "true" - + tag: 'master' usersync: usersync: true - userYamlS3Path: s3://local-helm-testing/user.yaml + userYamlS3Path: s3://cdis-gen3-users/ci/user.yaml # we do not want usersync to interrupt test flow # so it is set to run on the 1st of January if it is a Monday schedule: '0 0 1 1 1' - syncFromDbgap: false - env: - - name: AWS_ACCESS_KEY_ID - value: test - - name: AWS_SECRET_ACCESS_KEY - value: test - - name: AWS_ENDPOINT_URL - value: 'http://localstack.localstack.svc.cluster.local:4566' - - name: AWS_DEFAULT_REGION - value: 'us-east-1' - - # # -- (map) Public configuration settings for Fence app - # FENCE_CONFIG_PUBLIC: - # ENABLE_PROMETHEUS_METRICS: true - # ENABLE_DB_MIGRATION: true - - FENCE_CONFIG: - ENABLE_AUDIT_LOGS: - presigned_url: true - login: true + syncFromDbgap: true + FENCE_CONFIG_PUBLIC: + # Prefix to namespace Google Groups on a single Cloud Identity (see cirrus + # setup for more info on Cloud Identity) + # + # NOTE: Make this short! Less than 8 characters if possible. Google has + # length restrictions on group names. + # This will be dynamically replaced by env_setup.sh script. + GOOGLE_GROUP_PREFIX: + + # Prefix to namespace Google Service Accounts in a single Google Cloud Platform Project. + # This is primarily to support multiple instances of fence references the same Google + # project. If that is not something you need to support, then you can leave this blank. + # + # NOTE: Make this short! Less than 8 characters if possible. Google has + # length restrictions on service account names. + # This will be dynamically replaced by env_setup.sh script. + GOOGLE_SERVICE_ACCOUNT_PREFIX: + + # This will be dynamically replaced by env_setup.sh script. + BASE_URL: https://HOSTNAME/user + + # -- (string) Name of the Fence app + APP_NAME: "Gen3 Data Commons" + + # -- (map) Debug and security settings + # Modify based on whether you're in a dev environment or in production + DEBUG: false + # -- (bool) if true, will automatically login a user with username "test" + # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) + MOCK_AUTH: false + + # -- (bool) if true, will fake a successful login response from Google in /login/google + # NOTE: this will also modify the behavior of /link/google endpoints + # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) + # will login as the username set in cookie DEV_LOGIN_COOKIE_NAME + MOCK_GOOGLE_AUTH: true + + # -- (str) the name of the cookie set by mock authentication (used for testing only) + DEV_LOGIN_COOKIE_NAME: "dev_login" + + # -- (bool) if true, will ignore anything configured in STORAGE_CREDENTIALS + MOCK_STORAGE: false + + # -- (bool) allow OIDC traffic on http for development. By default it requires https. + # + # WARNING: ONLY set to true when fence will be deployed in such a way that it will + # ONLY receive traffic from internal clients and can safely use HTTP. + AUTHLIB_INSECURE_TRANSPORT: true + + # -- (bool) enable Prometheus Metrics for observability purposes + # + # WARNING: Any counters, gauges, histograms, etc. should be carefully + # reviewed to make sure its labels do not contain any PII / PHI + ENABLE_PROMETHEUS_METRICS: false + + # -- (bool) set if you want browsers to only send cookies with requests over HTTPS + SESSION_COOKIE_SECURE: true + + # -- (bool) enable CSRF protection + ENABLE_CSRF_PROTECTION: true + + # -- (str) signing key for WTForms to sign CSRF tokens with + WTF_CSRF_SECRET_KEY: "{{ENCRYPTION_KEY}}" + + # -- (bool) fence (at the moment) attempts a migration on startup. setting this to false will disable that + # WARNING: ONLY set to false if you do NOT want to automatically migrate your database. + # You should be careful about incompatible versions of your db schema with what + # fence expects. In other words, things could be broken if you update to a later + # fence that expects a schema your database isn't migrated to. + # NOTE: We are working to improve the migration process in the near future + ENABLE_DB_MIGRATION: true + + # these are the *possible* scopes a client can be given, NOT scopes that are + # given to all clients. You can be more restrictive during client creation + # -- (list) These are the *possible* scopes a client can be given, NOT scopes that are given to all clients. You can be more restrictive during client creation + CLIENT_ALLOWED_SCOPES: + - "openid" + - "user" + - "data" + - "google_credentials" + - "google_service_account" + - "google_link" + - "ga4gh_passport_v1" + + # -- (list) these are the scopes that CAN be included in a user's own access_token + USER_ALLOWED_SCOPES: + - "fence" + - "openid" + - "user" + - "data" + - "admin" + - "google_credentials" + - "google_service_account" + - "google_link" + - "ga4gh_passport_v1" + + # -- (list) these are the scopes that a browser session can create for a user (very similar to USER_ALLOWED_SCOPES, as the session will actually create access_tokens for an actively logged in user) + SESSION_ALLOWED_SCOPES: + - "openid" + - "user" + - "credentials" + - "data" + - "admin" + - "google_credentials" + - "google_service_account" + - "google_link" + - "ga4gh_passport_v1" + + # ////////////////////////////////////////////////////////////////////////////////////// + # LOGIN + # - Modify based on which OIDC provider(s) you configured above + # - NOTE: You can have multiple IDPs for users to login with, but one has to be set + # as the default + # ////////////////////////////////////////////////////////////////////////////////////// + + # List of enabled login options (used by data-portal to display login buttons). + # Each option must be configured with a "name" and an "idp". + # - "idp" must be a configured provider in OPENID_CONNECT section. + # Multiple options can be configured with the same idp. + # - if provider_id is "fence", "fence_idp" can be any of the providers + # supported by the other Fence. If not specified, will default to NIH login. + # - if provider_id is "fence" and fence_idp is "shibboleth", a list of + # "shib_idps" can be configured for InCommon login. If not specified, will + # default to NIH login. + # - Optional parameters: "desc" (description) and "secondary" (boolean - can + # be used by the frontend to display secondary buttons differently). + # -- (list) List of enabled login options (used by data-portal to display login buttons). + LOGIN_OPTIONS: + - name: 'Login from Google' + idp: google + - name: 'ORCID Login' + idp: fence + fence_idp: orcid + - name: 'Login from RAS' + idp: ras + + + # -- (string) Default login provider. - must be configured in LOGIN_OPTIONS and OPENID_CONNECT - - if several options in LOGIN_OPTIONS are defined for this IDP, will default to the first one + DEFAULT_LOGIN_IDP: google + + # -- (string) Default login URL: DEPRECATED and replaced by LOGIN_OPTIONS + DEFAULT_LOGIN_IDP configs + DEFAULT_LOGIN_URL: "{{BASE_URL}}/login/google" + + # `LOGIN_REDIRECT_WHITELIST` is a list of extra whitelisted URLs which can be redirected + # to by the `/login/*` endpoints. Fence automatically populates this with the redirect + # URLs for any registered OAuth clients, and its own URL. When validating the redirects, + # fence chesk whether the domain for the redirect matches a domain in the whitelist (so + # only the domains for the additional desired redirects are necessary here). LOGIN_REDIRECT_WHITELIST: [".", "/login"] - PUSH_AUDIT_LOGS_CONFIG: - type: aws_sqs - aws_sqs_config: - sqs_url: http://localstack.localstack.svc.cluster.local:4566/000000000000/audit-service-sqs - region: us-east-1 - aws_cred: 'cdistest' - AWS_CREDENTIALS: - cdistest: - aws_access_key_id: test - aws_secret_access_key: test #pragma: allowlist secret + + ### DEPRECATED and replaced by OPENID_CONNECT + LOGIN_OPTIONS configs + ENABLED_IDENTITY_PROVIDERS: + default: google + providers: + google: + name: Login from Google + + # ////////////////////////////////////////////////////////////////////////////////////// + # LIBRARY CONFIGURATION (authlib & flask) + # - Already contains reasonable defaults + # ////////////////////////////////////////////////////////////////////////////////////// + # authlib-specific configs for OIDC flow and JWTs + # NOTE: the OAUTH2_JWT_KEY cfg gets set automatically by fence if keys are setup + # correctly + OAUTH2_JWT_ALG: "RS256" + OAUTH2_JWT_ENABLED: true + OAUTH2_JWT_ISS: "{{BASE_URL}}" + OAUTH2_PROVIDER_ERROR_URI: "/api/oauth2/errors" + + # used for flask, "path mounted under by the application / web server" + # since we deploy as microservices, fence is typically under {{base}}/user + # this is also why our BASE_URL default ends in /user + APPLICATION_ROOT: "/user" + + # ////////////////////////////////////////////////////////////////////////////////////// + # Tokens, Lifetimes, & Expirations + # - Already contains reasonable defaults + # ////////////////////////////////////////////////////////////////////////////////////// + # The name of the browser cookie in which the access token will be stored. + ACCESS_TOKEN_COOKIE_NAME: "access_token" + + # The name of the browser cookie in which the session token will be stored. + # Note that the session token also stores information for the + # ``flask.session`` in the ``context`` field of the token. + SESSION_COOKIE_NAME: "fence" + + # The domain of the browser cookie in which the session token will be stored. + # Leave unset (not empty string!) for normal single-site deployment. + SESSION_COOKIE_DOMAIN: + + OAUTH2_TOKEN_EXPIRES_IN: + "authorization_code": 1200 + "implicit": 1200 + + # The number of seconds after an access token is issued until it expires. + ACCESS_TOKEN_EXPIRES_IN: 1200 + + # The number of seconds after a refresh token is issued until it expires. + REFRESH_TOKEN_EXPIRES_IN: 2592000 + + # The number of seconds after which a browser session is considered stale. + SESSION_TIMEOUT: 1800 + + # The maximum session lifetime in seconds. + SESSION_LIFETIME: 28800 + + # The number of seconds the user's Google service account key used for + # url signing will last before being expired/rotated + # 30 days: 2592000 seconds + GOOGLE_SERVICE_ACCOUNT_KEY_FOR_URL_SIGNING_EXPIRES_IN: 2592000 + + # The number of seconds after a User's Google Service account is added to bucket + # access until it expires. + # 7 days: 604800 seconds + GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN: 604800 + + # The number of seconds after a User's Google account is added to bucket + # access until it expires. + GOOGLE_ACCOUNT_ACCESS_EXPIRES_IN: 86400 + + # The number of seconds after a pre-signed url is issued until it expires. + MAX_PRESIGNED_URL_TTL: 3600 + + # The number of seconds after an API KEY is issued until it expires. + MAX_API_KEY_TTL: 2592000 + + # The number of seconds after an access token is issued until it expires. + MAX_ACCESS_TOKEN_TTL: 3600 + + # TEMPORARY: The maximum number of projects allowed in token claims. + # This config var should be removed after sheepdog and peregrine support + # auth checks against Arborist, and no longer check the token. + TOKEN_PROJECTS_CUTOFF: 10 + + # If set to true, will generate an new access token each time when a browser session update happens + RENEW_ACCESS_TOKEN_BEFORE_EXPIRATION: false + + # The maximum lifetime of a Gen3 passport in seconds + GEN3_PASSPORT_EXPIRES_IN: 43200 + + ######################################################################################## + # OPTIONAL CONFIGURATIONS # + ######################################################################################## + + # For displaying a privacy policy to users, we can either link to the URL specified by + # PRIVACY_POLICY_URL, or default to the `static/privacy_policy.md` file in fence. + PRIVACY_POLICY_URL: null + + # ////////////////////////////////////////////////////////////////////////////////////// + # RELIABILITY OPTS + # ////////////////////////////////////////////////////////////////////////////////////// + # Configurations related to resiliency, fault-tolerance and availability + # This is the number of requests per second that the Nginx proxy will accept before reaching fence + # The value defined in fence-config-public.yaml takes precedence over this one + # In the absence of this OVERRIDE prefixed config, the legacy NGINX_RATE_LIMIT from the k8s deployment yaml is applied + OVERRIDE_NGINX_RATE_LIMIT: 18 + + # ////////////////////////////////////////////////////////////////////////////////////// + # SUPPORT INFO + # ////////////////////////////////////////////////////////////////////////////////////// + # If you want an email address to show up when an unhandled error occurs, provide one + # here. Something like: support@example.com + SUPPORT_EMAIL_FOR_ERRORS: null + + # ////////////////////////////////////////////////////////////////////////////////////// + # SHIBBOLETH + # - Support using `shibboleth` in LOGIN_OPTIONS + # - Contains defaults for using NIH's Login. + # ////////////////////////////////////////////////////////////////////////////////////// + # assumes shibboleth is deployed under {{BASE_URL}}/shibboleth + SHIBBOLETH_HEADER: "persistent_id" + SSO_URL: "https://auth.nih.gov/affwebservices/public/saml2sso?SPID={{BASE_URL}}/shibboleth&RelayState=" + ITRUST_GLOBAL_LOGOUT: "https://auth.nih.gov/siteminderagent/smlogout.asp?mode=nih&AppReturnUrl=" + + DBGAP_ACCESSION_WITH_CONSENT_REGEX: "(?Pphs[0-9]+)(.(?Pv[0-9]+)){0,1}(.(?Pp[0-9]+)){0,1}.(?Pc[0-9]+)" + + # ////////////////////////////////////////////////////////////////////////////////////// + # STORAGE BACKENDS AND CREDENTIALS + # - Optional: Used for `/admin` & `/credentials` endpoints for user management. + # Also used during User Syncing process to automate managing Storage + # access for users. + # ////////////////////////////////////////////////////////////////////////////////////// + # When true, this modifies usersync (not fence service itself) such that when syncing user + # access to a Google storage backend happens in "bulk" by doing a diff *per google group* + # between what's in Google and what's expected. Then it adds, removes only as necessary. + # This is in contrast to the default logic which does blind updates per user and ignores + # 409s from Google. + # NOTE: This reduces the number of API calls to Google in the general case, but increases + # memory usages by usersync (as it has to track all the Google groups and user access) + GOOGLE_BULK_UPDATES: true + + # Configuration for various storage systems for the backend + # NOTE: Remove the {} and supply backends if needed. Example in comments below + # STORAGE_CREDENTIALS: {} + STORAGE_CREDENTIALS: + 'google': + backend: google + # this should be the project id where the Google Groups for data access are managed + google_project_id: dcf-integration + + # Cleversafe data storage backend + # + # 'cleversafe-server-a': + # backend: 'cleversafe' + # aws_access_key_id: '' + # aws_secret_access_key: '' + # host: 'somemanager.osdc.io' + # public_host: 'someobjstore.example.com' + # port: 443 + # is_secure: true + # username: 'someone' + # password: 'somepass' + # is_mocked: true + + # ////////////////////////////////////////////////////////////////////////////////////// + # AWS BUCKETS AND CREDENTIALS + # - Support `/data` endpoints + # ////////////////////////////////////////////////////////////////////////////////////// + + # NOTE: the region is optonal for s3_buckets, however it should be specified to avoid a + # call to GetBucketLocation which you make lack the AWS ACLs for. + # public buckets do not need the region field. + # the cred values should be keys in section `AWS_CREDENTIALS`. S3_BUCKETS: cdis-presigned-url-test: region: 'us-east-1' cred: 'cdistest' - endpoint_url: 'http://localstack.localstack.svc.cluster.local:4566' gen3-helm-data-upload-bucket: region: 'us-east-1' cred: 'cdistest' - endpoint_url: 'http://localstack.localstack.svc.cluster.local:4566' DATA_UPLOAD_BUCKET: 'gen3-helm-data-upload-bucket' - LOGIN_OPTIONS: - - name: 'Login from Google' - desc: 'description' - idp: google - #secondary: True - - name: 'RAS Login' - idp: ras - MOCK_GOOGLE_AUTH: true + + # ////////////////////////////////////////////////////////////////////////////////////// + # PROXY + # - Optional: If the api is behind firewall that needs to set http proxy + # ////////////////////////////////////////////////////////////////////////////////////// + # NOTE: leave as-is to not use proxy + # this is only used by the Google Oauth2Client at the moment if provided HTTP_PROXY: host: 'cloud-proxy.internal.io' port: 3128 - ENABLED_IDENTITY_PROVIDERS: - default: google - providers: - google: - name: Login from Google + + # ////////////////////////////////////////////////////////////////////////////////////// + # MICROSERVICE PATHS + # - Support `/data` endpoints & authz functionality + # ////////////////////////////////////////////////////////////////////////////////////// + # url where indexd microservice is running (for signed urls primarily) + # NOTE: Leaving as null will force fence to default to {{BASE_URL}}/index + # example value: 'https://example.com/index' + INDEXD: http://indexd-service + + # this is the username which fence uses to make authenticated requests to indexd + INDEXD_USERNAME: "fence" + # this is the password which fence uses to make authenticated requests to indexd + INDEXD_PASSWORD: "" + + # ////////////////////////////////////////////////////////////////////////////////////// + # AZURE STORAGE BLOB CONFIGURATION + # - Support Azure Blob Data Access Methods + # ////////////////////////////////////////////////////////////////////////////////////// + + # https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?toc=%2Fazure%2Fstorage%2Fblobs%2Ftoc.json&tabs=azure-portal#view-account-access-keys + # AZ_BLOB_CREDENTIALS: 'fake connection string' + AZ_BLOB_CREDENTIALS: + + # AZ_BLOB_CONTAINER_URL: 'https://storageaccount.blob.core.windows.net/container/' + # this is the container used for uploading, and should match the storage account + # used in the connection string for AZ_BLOB_CREDENTIALS + AZ_BLOB_CONTAINER_URL: "https://myfakeblob.blob.core.windows.net/my-fake-container/" + + # url where authz microservice is running + ARBORIST: http://arborist-service + + # url where the audit-service is running + # This will be dynamically replaced by env_setup.sh script. + AUDIT_SERVICE: "http://audit-service" + ENABLE_AUDIT_LOGS: + presigned_url: true + login: true + # `PUSH_AUDIT_LOGS_CONFIG.type` is one of: [api, aws_sqs]. + # - if type == api: logs are created by hitting the log creation endpoint. + # - if type == aws_sqs: logs are pushed to an SQS and `aws_sqs_config` fields + # `sqs_url` and `region` are required. Field `aws_cred` is optional and it + # should be a key in section `AWS_CREDENTIALS`. + PUSH_AUDIT_LOGS_CONFIG: + type: aws_sqs + aws_sqs_config: + # This will be dynamically replaced by env_setup.sh script. + sqs_url: https://sqs.us-east-1.amazonaws.com/707767160287/audit-service-sqs + region: us-east-1 + aws_cred: 'cdistest' + + # A Google Project identitifier representing the default project to bill to for + # accessing Google Requester Pays buckets (for signed urls and/or temporary service account + # credentials). If this is provided and the API call for + # Google access does not include a `userProject`, this will be used instead. + # + # WARNING: Setting this WITHOUT setting "ENABLE_AUTOMATIC_BILLING_*" to `true` below, + # means that clients and end-users will be responsible for making sure that + # the service account used in either of these methods actually has billing + # permission in the specified project. + BILLING_PROJECT_FOR_SIGNED_URLS: + BILLING_PROJECT_FOR_SA_CREDS: + + # Setting this to `true` will make Fence automatically attempt to create a Custom Role + # in the billing project and give the necessary Google Service Account that role + # (which will allow it to bill to the project). + # + # NOTE: The Fence SA will need the necessary permissions in the specified project to + # both create a custom role and update the Project's IAM Policy to include the + # necessary SA. At the time of writing, there are pre-defined roles in Google's + # IAM that provide the necessary permissions. Those are "Project IAM Admin" and + # "Role Administrator" + # + # NOTE2: It may be possible to further restrict the permissions in the future to + # be more fine-grained. + # + ENABLE_AUTOMATIC_BILLING_PERMISSION_SIGNED_URLS: false + ENABLE_AUTOMATIC_BILLING_PERMISSION_SA_CREDS: false + + # ////////////////////////////////////////////////////////////////////////////////////// + # EMAIL + # - Support for sending emails from fence. Used for user certificates + # and `/google/service_accounts` endpoints + # ////////////////////////////////////////////////////////////////////////////////////// + # Gun Mail Service (for sending emails from fence) + # + # NOTE: Example in comments below + GUN_MAIL: + "datacommons.io": + smtp_hostname: "smtp.mailgun.org" + api_key: "" + default_login: "postmaster@mailgun.example.com" + api_url: "https://api.mailgun.net/v3/mailgun.example.com" + smtp_password: "" + + # For emails regarding users certificates + EMAIL_SERVER: "localhost" + SEND_FROM: "example@gmail.com" + SEND_TO: "example@gmail.com" + + # ////////////////////////////////////////////////////////////////////////////////////// + # DATA ACCESS: GOOGLE LINKING & SERVICE ACCOUNT REGISTRATION + # - Support `/google/service_accounts` endpoints + # ////////////////////////////////////////////////////////////////////////////////////// + # whether or not to allow access to the /link/google endpoints + ALLOW_GOOGLE_LINKING: true + + # A Google Project with controlled data access will be determined INVALID if + # if it has a parent organization UNLESS that parent organization's ID is in this + # whitelist. + # + # NOTE: Remove the [] and Google Organization IDs if needed. Example in comments below + WHITE_LISTED_GOOGLE_PARENT_ORGS: [] + # - '12345678910' + + # when service accounts or google projects are determined invalid, an email is sent + # to the project owners. These settings are for that email + REMOVE_SERVICE_ACCOUNT_EMAIL_NOTIFICATION: + enable: false + # this domain MUST exist in GUN_MAIL config + domain: "example.com" + from: "do-not-reply@example.com" + subject: "User service account removal notification" + # the {} gets replaced dynamically in the Python code to be the Project ID + content: > + Service accounts were removed from access control data because some users or + service accounts of GCP Project {} are not authorized to access the data sets + associated to the service accounts, or do not adhere to the security policies. + # this admin email will be included as a recipient to *any* email to anyone about + # service account removal. + # + # WARNING: This is NOT a bcc so the email is visible to the end-user + admin: + - "admin@example.edu" + + PROBLEM_USER_EMAIL_NOTIFICATION: + # this domain MUST exist in GUN_MAIL config + domain: "example.com" + from: "do-not-reply@example.com" + subject: "Account access error notification" + # the {} gets replaced dynamically in the Python code to be the Project ID + content: > + The Data Commons Framework utilizes dbGaP for data access authorization. + Another member of a Google project you belong to ({}) is attempting to + register a service account to the following additional datasets ({}). + Please contact dbGaP to request access. + # this admin email will be included as a recipient to *any* email to anyone about + # service account removal. + # + # WARNING: This is NOT a bcc so the email is visible to the end-user + admin: + - "admin@example.edu" + + # Service account email domains that represent a service account that Google owns. + # These are usually created when a sepcific GCP service is enabled. + # This is used for Service Account Validation for Data Access. + GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS: + - "dataflow-service-producer-prod.iam.gserviceaccount.com" + - "cloudbuild.gserviceaccount.com" + - "cloud-ml.google.com.iam.gserviceaccount.com" + - "container-engine-robot.iam.gserviceaccount.com" + - "dataflow-service-producer-prod.iam.gserviceaccount.com" + - "sourcerepo-service-accounts.iam.gserviceaccount.com" + - "dataproc-accounts.iam.gserviceaccount.com" + - "gae-api-prod.google.com.iam.gserviceaccount.com" + - "genomics-api.google.com.iam.gserviceaccount.com" + - "containerregistry.iam.gserviceaccount.com" + - "container-analysis.iam.gserviceaccount.com" + - "cloudservices.gserviceaccount.com" + - "stackdriver-service.iam.gserviceaccount.com" + - "appspot.gserviceaccount.com" + - "partnercontent.gserviceaccount.com" + - "trifacta-gcloud-prod.iam.gserviceaccount.com" + - "gcf-admin-robot.iam.gserviceaccount.com" + - "compute-system.iam.gserviceaccount.com" + - "gcp-sa-websecurityscanner.iam.gserviceaccount.com" + - "storage-transfer-service.iam.gserviceaccount.com" + - "firebase-sa-management.iam.gserviceaccount.com" + - "firebase-rules.iam.gserviceaccount.com" + - "gcp-sa-cloudbuild.iam.gserviceaccount.com" + - "gcp-sa-automl.iam.gserviceaccount.com" + - "gcp-sa-datalabeling.iam.gserviceaccount.com" + - "gcp-sa-cloudscheduler.iam.gserviceaccount.com" + + # The types of service accounts that are allowed to be registered at + # /google/service_accounts endpoints + ALLOWED_USER_SERVICE_ACCOUNT_DOMAINS: + # compute engine default service account + - "developer.gserviceaccount.com" + # app engine default service account + - "appspot.gserviceaccount.com" + # user-managed service account + - "iam.gserviceaccount.com" + + # Synapse integration and DREAM challenge mapping. Team is from Synapse, and group is + # providing the actual permission in Arborist. User will be added to the group for TTL + # seconds if the team matches. + DREAM_CHALLENGE_TEAM: "DREAM" + DREAM_CHALLENGE_GROUP: "DREAM" + SYNAPSE_URI: "https://repo-prod.prod.sagebase.org/auth/v1" + SYNAPSE_JWKS_URI: + # deprecated, use the discovery_url in the OPENID_CONNECT block for the synapse client + SYNAPSE_DISCOVERY_URL: + SYNAPSE_AUTHZ_TTL: 86400 + + # Role caching for generating presigned urls if max role session increase is true + # then we can increase the amount of time that a session is valid for + MAX_ROLE_SESSION_INCREASE: false + ASSUME_ROLE_CACHE_SECONDS: 1800 + + # Optional user registration feature: Ask users to register (provide firstname/lastname/org/email) on login. + # If user registers, add them to configured Arborist group; idea is that the Arborist group + # will have access to download data. + REGISTER_USERS_ON: true + REGISTERED_USERS_GROUP: "" + # RAS refresh_tokens expire in 15 days + RAS_REFRESH_EXPIRATION: 1296000 + # List of JWT issuers from which Fence will accept GA4GH visas + GA4GH_VISA_ISSUER_ALLOWLIST: + - "{{BASE_URL}}" + - "https://sts.nih.gov" + - "https://stsstg.nih.gov" + # Number of projects that can be registered to a Google Service Accont + SERVICE_ACCOUNT_LIMIT: 6 + + # Global sync visas during login + # None(Default): Allow per client i.e. a fence client can pick whether or not to sync their visas during login with parse_visas param in /authorization endpoint + # True: Parse for all clients i.e. a fence client will always sync their visas during login + # False: Parse for no clients i.e. a fence client will not be able to sync visas during login even with parse_visas param + GLOBAL_PARSE_VISAS_ON_LOGIN: false + # Settings for usersync with visas + USERSYNC: + sync_from_visas: false + # fallback to dbgap sftp when there are no valid visas for a user i.e. if they're expired or if they're malformed + fallback_to_dbgap_sftp: false + visa_types: + ras: ["https://ras.nih.gov/visas/v1", "https://ras.nih.gov/visas/v1.1"] + RAS_USERINFO_ENDPOINT: "/openid/connect/v1.1/userinfo" diff --git a/ci/default/values/guppy.yaml b/ci/default/values/guppy.yaml index 5faace68f..36d80a84c 100644 --- a/ci/default/values/guppy.yaml +++ b/ci/default/values/guppy.yaml @@ -1,10 +1,12 @@ guppy: + replicaCount: 2 # Lower cost resources: requests: memory: "105Mi" cpu: "15m" enabled: true + #TO DO Re-enable once ES index restore is added to modify values script dbRestore: false image: repository: quay.io/cdis/guppy @@ -14,15 +16,15 @@ guppy: type: subject - index: ci_file_alias type: file - - index: ci_cmc_alias - type: clinical_trials - - index: ci_midrc_dataset_alias - type: dataset - - index: ci_midrc_case_alias - type: case - - index: ci_midrc_imaging_data_file_alias - type: imaging_data_file - - index: ci_midrc_imaging_study_alias + # - index: ci_cmc_alias + # type: clinical_trials + # - index: ci_dataset_alias + # type: dataset + # - index: ci_case_alias + # type: case + # - index: ci_imaging_data_file_alias + # type: imaging_data_file + - index: ci_imaging_study_alias type: imaging_study configIndex: ci_configs_alias authFilterField: auth_resource_path diff --git a/ci/default/values/portal.yaml b/ci/default/values/portal.yaml index 33bcc4090..4128704a5 100644 --- a/ci/default/values/portal.yaml +++ b/ci/default/values/portal.yaml @@ -1,8 +1,11 @@ portal: + # replicaCount: 2 resources: + limits: + memory: 6Gi requests: - cpu: "0.2" - memory: 100Mi + cpu: 4 + memory: 6Gi image: repository: quay.io/cdis/data-portal tag: master @@ -651,6 +654,7 @@ portal: "rightIcon": "download" } ], + "loginForDownload": true, "guppyConfig": { "dataType": "subject", "nodeCountTitle": "Subjects", @@ -716,6 +720,7 @@ portal: "file_size", "object_id" ] }, + "loginForDownload": true, "guppyConfig": { "dataType": "file", "fieldMapping": [ diff --git a/ci/default/values/values.yaml b/ci/default/values/values.yaml index 7bfa34dea..32e18e701 100644 --- a/ci/default/values/values.yaml +++ b/ci/default/values/values.yaml @@ -1,117 +1,133 @@ global: + # This will be dynamically replaced by env_setup.sh script. hostname: dictionaryUrl: https://s3.amazonaws.com/dictionary-artifacts/anvil/master/schema.json + pdb: true aws: - awsAccessKeyId: test - awsSecretAccessKey: test #pragma: allowlist secret - region: us-east-1 + externalSecrets: + # -- (bool) Whether to use External Secrets for aws config. + enabled: true + # -- (String) Name of Secrets Manager secret. + externalSecretAwsCreds: ci-aws-config + secretStoreServiceAccount: + # -- (bool) Set true if deploying to AWS and want to use service account and IAM role instead of aws keys. Must provide role-arn. + enabled: true + # -- (string) Name of the service account to create + name: secret-store-sa + # -- (string) AWS Role ARN for Secret Store to use + roleArn: arn:aws:iam::707767160287:role/devplanetv2-external-secrets-sa + sower_role: sower-jobs-role + externalSecrets: + deploy: true + dbCreate: true + manifestGlobalExtraValues: + # This will be dynamically replaced by env_setup.sh script. + fence_url: https://hostname/user + google_enabled: true + tierAccessLevel: libre -secrets: - awsAccessKeyId: test - awsSecretAccessKey: test #pragma: allowlist secret +mutatingWebhook: + # -- (bool) Whether to deploy the mutating webhook service. + enabled: false -arborist: - resources: - requests: - memory: "105Mi" - cpu: "15m" - enabled: true +postgresql: + primary: + persistence: + enabled: true + size: 1Gi image: - repository: "quay.io/cdis/arborist" - tag: "master" + registry: quay.io + repository: cdis/postgresql + tag: 14.5.0-debian-11-r35 + +hatchery: + replicaCount: 2 + image: + tag: master + hatchery: + containers: + - target-port: 8888 + cpu-limit: '0.5' + memory-limit: 1Gi + name: "(Tutorial) Bacpac Synthetic Data Analysis Notebook" + image: "quay.io/cdis/heal-notebooks:bacpac__10670ad79b4488cfa9f1b3681ce52bc3f1139b5a" + env: + # This will be dynamically replaced by env_setup.sh script. + FRAME_ANCESTORS: https:// + args: + - "--NotebookApp.base_url=/lw-workspace/proxy/" + - "--NotebookApp.password=''" + - "--NotebookApp.token=''" + - "--NotebookApp.quit_button=False" + command: + - start-notebook.sh + path-rewrite: "/lw-workspace/proxy/" + use-tls: 'false' + ready-probe: "/lw-workspace/proxy/" + lifecycle-post-start: + - "/bin/sh" + - "-c" + - export IAM=`whoami`; rm -rf /home/$IAM/pd/dockerHome; rm -rf /home/$IAM/pd/lost+found; ln -s /data /home/$IAM/pd/; true + user-uid: 1000 + fs-gid: 100 indexd: + replicaCount: 2 resources: requests: memory: "105Mi" cpu: "15m" + # This will be dynamically replaced by env_setup.sh script. defaultPrefix: "" enabled: true image: repository: "quay.io/cdis/indexd" tag: "master" + externalSecrets: + createK8sServiceCredsSecret: true + secrets: + userdb: + # This will be dynamically replaced by env_setup.sh script. + ssj: + # This will be dynamically replaced by env_setup.sh script. + gateway: -peregrine: - resources: - requests: - memory: "105Mi" - cpu: "15m" - enabled: true - image: - repository: "quay.io/cdis/peregrine" - tag: "master" - pullPolicy: Always - env: - - name: AWS_ACCESS_KEY_ID - value: test - - name: AWS_SECRET_ACCESS_KEY - value: test - - name: AWS_ENDPOINT_URL - value: 'http://localstack.localstack.svc.cluster.local:4566' - - name: AWS_DEFAULT_REGION - value: 'us-east-1' -revproxy: - ingress: - enabled: true - className: cloudflare-tunnel - hosts: - - host: - paths: - - path: / - pathType: Prefix - resources: - requests: - memory: "105Mi" - cpu: "15m" +metadata: + replicaCount: 2 image: - repository: "quay.io/cdis/nginx" - tag: "master" - -sheepdog: + repository: quay.io/cdis/metadata-service + tag: 'master' resources: + # -- (map) The amount of resources that the container requests requests: - memory: "105Mi" - cpu: "15m" + memory: 25Mi enabled: true - authNamespace: '' - image: - repository: "quay.io/cdis/sheepdog" - tag: "master" + externalSecrets: + createK8sMetadataSecret: true + useAggMds: "True" + aggMdsNamespace: default + aggMdsConfig: | + { + "gen3_commons": { + "HEAL": { + "mds_url": "http://revproxy-service", + "commons_url": "http://revproxy-service" + } + }, + "adapter_commons": {} + } -manifestservice: - resources: - requests: - memory: "105Mi" - cpu: "15m" - enabled: true - image: - repository: "quay.io/cdis/manifestservice" - tag: "master" - manifestserviceG3auto: - bucketName: gen3-helm-manifest-bucket - prefix: 'test' - awsaccesskey: test - awssecretkey: test #pragma: allowlist secret - env: - - name: AWS_ACCESS_KEY_ID - value: test - - name: AWS_SECRET_ACCESS_KEY - value: test - - name: AWS_ENDPOINT_URL - value: 'http://localstack.localstack.svc.cluster.local:4566' - - name: AWS_DEFAULT_REGION - value: 'us-east-1' - - name: REQUESTS_CA_BUNDLE - value: /etc/ssl/certs/ca-certificates.crt - - name: MANIFEST_SERVICE_CONFIG_PATH - value: /var/gen3/config/config.json - - name: GEN3_DEBUG - value: "False" +maxUnavailable: "" +elasticsearch: + maxUnavailable: "" + volumeClaimTemplate: + resources: + requests: + storage: 1Gi sower: - pelican: - bucket: 'sower-bucket' + replicaCount: 2 image: repository: quay.io/cdis/sower tag: 'master' @@ -119,12 +135,13 @@ sower: serviceAccount: # -- (bool) Specifies whether a service account should be created. create: true - # -- (map) Annotations to add to the service account. - annotations: - eks.amazonaws.com/role-arn: arn:aws:iam::000000000000:role/sower-jobs-role # -- (string) The name of the service account to use. # If not set and create is true, a name is generated using the fullname template name: "sower-service-account" + externalSecrets: + createK8sPelicanServiceSecret: false + pelicanserviceG3auto: ci-pelicanservice-g3auto + createK8sSowerJobsSecret: true sowerConfig: - name: pelican-export action: export @@ -170,21 +187,13 @@ sower: secretKeyRef: name: indexd-service-creds key: sheepdog - - name: AWS_ENDPOINT_URL - value: "http://localstack.localstack.svc.cluster.local:4566" - - name: AWS_DEFAULT_REGION - value: us-east-1 - - name: AWS_ACCESS_KEY_ID - value: test - - name: AWS_SECRET_ACCESS_KEY - value: test volumeMounts: - name: pelican-creds-volume readOnly: true mountPath: "/pelican-creds.json" subPath: config.json - cpu-limit: "1" - memory-limit: 12Gi + cpu-limit: "0.5" + memory-limit: 1Gi volumes: - name: pelican-creds-volume secret: @@ -236,21 +245,13 @@ sower: secretKeyRef: name: indexd-service-creds key: sheepdog - - name: AWS_ENDPOINT_URL - value: "http://localstack.localstack.svc.cluster.local:4566" - - name: AWS_DEFAULT_REGION - value: us-east-1 - - name: AWS_ACCESS_KEY_ID - value: test - - name: AWS_SECRET_ACCESS_KEY - value: test volumeMounts: - name: pelican-creds-volume readOnly: true mountPath: "/pelican-creds.json" subPath: config.json - cpu-limit: "1" - memory-limit: 12Gi + cpu-limit: "0.5" + memory-limit: 1Gi volumes: - name: pelican-creds-volume secret: @@ -266,142 +267,209 @@ sower: pull_policy: Always env: - name: BUCKET - value: sower-bucket + value: sower-jobs-bucket - name: GEN3_HOSTNAME valueFrom: configMapKeyRef: name: manifest-global key: hostname - - name: AWS_ENDPOINT_URL - value: "http://localstack.localstack.svc.cluster.local:4566" - - name: AWS_DEFAULT_REGION - value: us-east-1 - - name: AWS_ACCESS_KEY_ID - value: test - - name: AWS_SECRET_ACCESS_KEY - value: test cpu-limit: "1" memory-limit: 1Gi restart_policy: Never ssjdispatcher: + replicaCount: 2 image: repository: quay.io/cdis/ssjdispatcher tag: 'master' enabled: true indexing: quay.io/cdis/indexs3client:master ssjcreds: - sqsUrl: http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/gen3-helm-data-upload-bucket - jobPattern: s3://test-12345678901234-upload/* - jobUser: sheepdog_gen3 - metadataserviceUsername: metadata_gen3 + # This will be dynamically replaced by env_setup.sh script. + sqsUrl: https://sqs.us-east-1.amazonaws.com/707767160287/gen3-helm-data-upload-bucket + # This will be dynamically replaced by env_setup.sh script. + jobPattern: "s3://gen3-helm-data-upload-bucket//*" + # This will be dynamically replaced by env_setup.sh script. + jobPassword: "replace_with_password" + # This will be dynamically replaced by env_setup.sh script. + metadataservicePassword: "replace_with_password" serviceAccount: create: true annotations: - eks.amazonaws.com/role-arn: arn:aws:iam::000000000000:role/ssjdispatcher-role + eks.amazonaws.com/role-arn: arn:aws:iam::707767160287:role/ssjdispatcher-role name: "ssjdispatcher-service-account" + jobServiceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::707767160287:role/ssjdispatcher-role-job + externalSecrets: + createK8sSsjdispatcherSecret: true + # This will be dynamically replaced by env_setup.sh script. + gen3Namespace: default -metadata: - image: - repository: quay.io/cdis/metadata-service - tag: 'master' - enabled: true - useAggMds: "True" - aggMdsNamespace: default - aggMdsConfig: | - { - "gen3_commons": { - "HEAL": { - "mds_url": "http://revproxy-service.default.svc.cluster.local", - "commons_url": "http://revproxy-service.default.svc.cluster.local" - } - }, - "adapter_commons": {} - } - -ambassador: +sheepdog: + replicaCount: 2 + resources: + requests: + memory: "105Mi" + cpu: "15m" enabled: true + authNamespace: '' image: - repository: "quay.io/datawire/ambassador" - tag: "1.4.2" + repository: "quay.io/cdis/sheepdog" + tag: "master" + # This will be dynamically replaced by env_setup.sh script. + fenceUrl: https://hostname/user -wts: +peregrine: + replicaCount: 2 + resources: + requests: + memory: "105Mi" + cpu: "15m" enabled: true image: - repository: "quay.io/cdis/workspace-token-service" + repository: "quay.io/cdis/peregrine" tag: "master" + pullPolicy: Always -hatchery: +requestor: + replicaCount: 2 + enabled: true image: - tag: master - hatchery: - containers: - - target-port: 8888 - cpu-limit: '0.5' - memory-limit: 1Gi - name: "(Tutorial) Bacpac Synthetic Data Analysis Notebook" - image: "quay.io/cdis/heal-notebooks:bacpac__10670ad79b4488cfa9f1b3681ce52bc3f1139b5a" - env: - FRAME_ANCESTORS: https://{{ .Values.global.hostname }} - args: - - "--NotebookApp.base_url=/lw-workspace/proxy/" - - "--NotebookApp.password=''" - - "--NotebookApp.token=''" - - "--NotebookApp.quit_button=False" - command: - - start-notebook.sh - path-rewrite: "/lw-workspace/proxy/" - use-tls: 'false' - ready-probe: "/lw-workspace/proxy/" - lifecycle-post-start: - - "/bin/sh" - - "-c" - - export IAM=`whoami`; rm -rf /home/$IAM/pd/dockerHome; rm -rf /home/$IAM/pd/lost+found; - ln -s /data /home/$IAM/pd/; true - user-uid: 1000 - fs-gid: 100 + repository: "quay.io/cdis/requestor" + tag: "master" audit: + replicaCount: 2 env: - name: ARBORIST_URL valueFrom: - configMapKeyRef: - name: manifest-global - key: arborist_url - optional: true - - name: AWS_ACCESS_KEY_ID - value: test - - name: AWS_SECRET_ACCESS_KEY - value: test - - name: AWS_ENDPOINT_URL - value: 'http://localstack.localstack.svc.cluster.local:4566' - - name: AWS_DEFAULT_REGION - value: 'us-east-1' + configMapKeyRef: + name: manifest-global + key: arborist_url + optional: true + externalSecrets: + createK8sAuditSecret: true image: repository: quay.io/cdis/audit-service - tag: '2025.01' + tag: 'master' server: pull_from_queue: true debug: true sqs: region: us-east-1 - url: http://localstack.localstack.svc.cluster.local:4566/000000000000/audit-service-sqs + # This will be dynamically replaced by env_setup.sh script. + url: https://sqs.us-east-1.amazonaws.com/707767160287/audit-service-sqs enabled: true -requestor: +ambassador: + replicaCount: 2 enabled: true image: - repository: "quay.io/cdis/requestor" + repository: "quay.io/datawire/ambassador" + tag: "1.4.2" + +arborist: + replicaCount: 2 + resources: + requests: + memory: "105Mi" + cpu: "15m" + enabled: true + image: + repository: "quay.io/cdis/arborist" tag: "master" -dicom-viewer: +revproxy: + replicaCount: 2 + ingress: + enabled: true + annotations: + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/tags: Environment=devplanetv2 + alb.ingress.kubernetes.io/certificate-arn: 'arn:aws:acm:us-east-1:707767160287:certificate/f427a646-94fd-4b91-8bcd-80f5920c1e46' + alb.ingress.kubernetes.io/group.name: devplanetv2 + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' + alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' + alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-FIPS-2023-04 + className: alb + hosts: + # This will be dynamically replaced by env_setup.sh script. + - host: HOSTNAME + paths: + - path: / + pathType: Prefix + resources: + requests: + memory: "105Mi" + cpu: "15m" + + +wts: + replicaCount: 2 enabled: true image: - repository: "quay.io/cdis/ohif-viewer" + repository: "quay.io/cdis/workspace-token-service" tag: "master" + externalSecrets: + createWtsOidcClientSecret: true + createK8sWtsSecret: true -dicom-server: +manifestservice: + replicaCount: 2 + image: + repository: quay.io/cdis/manifestservice + tag: "krishnaa05-patch-1" + env: + - name: REQUESTS_CA_BUNDLE + value: /etc/ssl/certs/ca-certificates.crt + - name: MANIFEST_SERVICE_CONFIG_PATH + value: /var/gen3/config/config.json + - name: GEN3_DEBUG + value: "False" + manifestserviceG3auto: + # This will be dynamically replaced by env_setup.sh script. + hostname: + bucketName: gen3-helm-manifest-bucket + prefix: 'test' + externalSecrets: + # -- (string) Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. + createK8sManifestServiceSecret: true + + +ohif-viewer: + replicaCount: 2 enabled: true + image: + repository: "quay.io/cdis/ohif-viewer" + tag: "gen3-v3.8.0" + +dicom-server: + replicaCount: 2 + enabled: false image: repository: "quay.io/cdis/gen3-orthanc" tag: "master" + +gen3-user-data-library: + replicaCount: 2 + enabled: true + image: + repository: "quay.io/cdis/gen3-user-data-library" + tag: "main" + externalSecrets: + createK8sGen3UserDataLibrarySecret: true + +access-backend: + enabled: false + +orthanc: + replicaCount: 2 + enabled: true + image: + repository: "quay.io/cdis/gen3-orthanc" + tag: "orthancteam-master-gen3-24.3.5" + # -- (map) External Secrets settings. + externalSecrets: + # -- (string) Will override the name of the aws secrets manager secret. Default is "orthanc-s3-g3auto" + orthancS3G3Auto: ci-orthanc-s3-g3auto diff --git a/ci/perf/Chart.yaml b/ci/perf/Chart.yaml new file mode 100644 index 000000000..b6fa1679f --- /dev/null +++ b/ci/perf/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: perf +description: ci perf setup + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: "1.0" diff --git a/ci/perf/templates/app.yaml b/ci/perf/templates/app.yaml new file mode 100644 index 000000000..29cf34821 --- /dev/null +++ b/ci/perf/templates/app.yaml @@ -0,0 +1,33 @@ +# Note- Won't deploy via argocd in CI +# apiVersion: argoproj.io/v1alpha1 +# kind: Application +# metadata: +# name: gen3-ci +# namespace: argocd +# finalizers: +# - resources-finalizer.argocd.argoproj.io +# spec: +# project: default +# sources: +# - path: helm/gen3 +# repoURL: https://github.com/uc-cdis/gen3-helm +# targetRevision: master +# helm: +# releaseName: oadc +# valueFiles: +# - $values/ci/values/values.yaml +# - $values/ci/values/fence.yaml +# - $values/ci/values/portal.yaml +# - $values/ci/values/guppy.yaml +# - $values/ci/values/etl.yaml +# - repoURL: 'https://github.com/uc-cdis/gen3-gitops.git' +# targetRevision: master +# ref: values +# destination: +# server: "https://kubernetes.default.svc" +# namespace: <> +# syncPolicy: +# syncOptions: +# - CreateNamespace=true +# automated: +# selfHeal: true diff --git a/ci/perf/values.yaml b/ci/perf/values.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/ci/perf/values/fence.yaml b/ci/perf/values/fence.yaml new file mode 100644 index 000000000..9b1d150cb --- /dev/null +++ b/ci/perf/values/fence.yaml @@ -0,0 +1,605 @@ +fence: + # -- (map) External Secrets settings. + externalSecrets: + createK8sJwtKeysSecret: false + fenceJwtKeys: ci-fence-jwt + fenceGoogleAppCredsSecret: ci-fence-google-app-creds-secret + fenceGoogleStorageCredsSecret: ci-fence-google-storage-creds-secret + createK8sGoogleAppSecrets: false + fenceConfig: ci-fence-config-priv + fenceSshKeys: ci-fence-ssh-keys + fenceDeleteExpiredClients: + schedule: "0 0 * * *" + replicaCount: 2 + autoscaling: + enabled: true + image: + repository: quay.io/cdis/fence + pullPolicy: IfNotPresent + tag: 'master' + usersync: + usersync: true + userYamlS3Path: s3://cdis-gen3-users/ci/user.yaml + # we do not want usersync to interrupt test flow + # so it is set to run on the 1st of January if it is a Monday + schedule: '0 0 1 1 1' + syncFromDbgap: true + FENCE_CONFIG_PUBLIC: + # Prefix to namespace Google Groups on a single Cloud Identity (see cirrus + # setup for more info on Cloud Identity) + # + # NOTE: Make this short! Less than 8 characters if possible. Google has + # length restrictions on group names. + # This will be dynamically replaced by env_setup.sh script. + GOOGLE_GROUP_PREFIX: + + # Prefix to namespace Google Service Accounts in a single Google Cloud Platform Project. + # This is primarily to support multiple instances of fence references the same Google + # project. If that is not something you need to support, then you can leave this blank. + # + # NOTE: Make this short! Less than 8 characters if possible. Google has + # length restrictions on service account names. + # This will be dynamically replaced by env_setup.sh script. + GOOGLE_SERVICE_ACCOUNT_PREFIX: + + # This will be dynamically replaced by env_setup.sh script. + BASE_URL: https://HOSTNAME/user + + # -- (string) Name of the Fence app + APP_NAME: "Gen3 Data Commons" + + # -- (map) Debug and security settings + # Modify based on whether you're in a dev environment or in production + DEBUG: false + # -- (bool) if true, will automatically login a user with username "test" + # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) + MOCK_AUTH: false + + # -- (bool) if true, will fake a successful login response from Google in /login/google + # NOTE: this will also modify the behavior of /link/google endpoints + # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) + # will login as the username set in cookie DEV_LOGIN_COOKIE_NAME + MOCK_GOOGLE_AUTH: true + + # -- (str) the name of the cookie set by mock authentication (used for testing only) + DEV_LOGIN_COOKIE_NAME: "dev_login" + + # -- (bool) if true, will ignore anything configured in STORAGE_CREDENTIALS + MOCK_STORAGE: false + + # -- (bool) allow OIDC traffic on http for development. By default it requires https. + # + # WARNING: ONLY set to true when fence will be deployed in such a way that it will + # ONLY receive traffic from internal clients and can safely use HTTP. + AUTHLIB_INSECURE_TRANSPORT: true + + # -- (bool) enable Prometheus Metrics for observability purposes + # + # WARNING: Any counters, gauges, histograms, etc. should be carefully + # reviewed to make sure its labels do not contain any PII / PHI + ENABLE_PROMETHEUS_METRICS: false + + # -- (bool) set if you want browsers to only send cookies with requests over HTTPS + SESSION_COOKIE_SECURE: true + + # -- (bool) enable CSRF protection + ENABLE_CSRF_PROTECTION: true + + # -- (str) signing key for WTForms to sign CSRF tokens with + WTF_CSRF_SECRET_KEY: "{{ENCRYPTION_KEY}}" + + # -- (bool) fence (at the moment) attempts a migration on startup. setting this to false will disable that + # WARNING: ONLY set to false if you do NOT want to automatically migrate your database. + # You should be careful about incompatible versions of your db schema with what + # fence expects. In other words, things could be broken if you update to a later + # fence that expects a schema your database isn't migrated to. + # NOTE: We are working to improve the migration process in the near future + ENABLE_DB_MIGRATION: true + + # these are the *possible* scopes a client can be given, NOT scopes that are + # given to all clients. You can be more restrictive during client creation + # -- (list) These are the *possible* scopes a client can be given, NOT scopes that are given to all clients. You can be more restrictive during client creation + CLIENT_ALLOWED_SCOPES: + - "openid" + - "user" + - "data" + - "google_credentials" + - "google_service_account" + - "google_link" + - "ga4gh_passport_v1" + + # -- (list) these are the scopes that CAN be included in a user's own access_token + USER_ALLOWED_SCOPES: + - "fence" + - "openid" + - "user" + - "data" + - "admin" + - "google_credentials" + - "google_service_account" + - "google_link" + - "ga4gh_passport_v1" + + # -- (list) these are the scopes that a browser session can create for a user (very similar to USER_ALLOWED_SCOPES, as the session will actually create access_tokens for an actively logged in user) + SESSION_ALLOWED_SCOPES: + - "openid" + - "user" + - "credentials" + - "data" + - "admin" + - "google_credentials" + - "google_service_account" + - "google_link" + - "ga4gh_passport_v1" + + # ////////////////////////////////////////////////////////////////////////////////////// + # LOGIN + # - Modify based on which OIDC provider(s) you configured above + # - NOTE: You can have multiple IDPs for users to login with, but one has to be set + # as the default + # ////////////////////////////////////////////////////////////////////////////////////// + + # List of enabled login options (used by data-portal to display login buttons). + # Each option must be configured with a "name" and an "idp". + # - "idp" must be a configured provider in OPENID_CONNECT section. + # Multiple options can be configured with the same idp. + # - if provider_id is "fence", "fence_idp" can be any of the providers + # supported by the other Fence. If not specified, will default to NIH login. + # - if provider_id is "fence" and fence_idp is "shibboleth", a list of + # "shib_idps" can be configured for InCommon login. If not specified, will + # default to NIH login. + # - Optional parameters: "desc" (description) and "secondary" (boolean - can + # be used by the frontend to display secondary buttons differently). + # -- (list) List of enabled login options (used by data-portal to display login buttons). + LOGIN_OPTIONS: + - name: 'Login from Google' + idp: google + - name: 'ORCID Login' + idp: fence + fence_idp: orcid + - name: 'Login from RAS' + idp: ras + + + # -- (string) Default login provider. - must be configured in LOGIN_OPTIONS and OPENID_CONNECT - - if several options in LOGIN_OPTIONS are defined for this IDP, will default to the first one + DEFAULT_LOGIN_IDP: google + + # -- (string) Default login URL: DEPRECATED and replaced by LOGIN_OPTIONS + DEFAULT_LOGIN_IDP configs + DEFAULT_LOGIN_URL: "{{BASE_URL}}/login/google" + + # `LOGIN_REDIRECT_WHITELIST` is a list of extra whitelisted URLs which can be redirected + # to by the `/login/*` endpoints. Fence automatically populates this with the redirect + # URLs for any registered OAuth clients, and its own URL. When validating the redirects, + # fence chesk whether the domain for the redirect matches a domain in the whitelist (so + # only the domains for the additional desired redirects are necessary here). + LOGIN_REDIRECT_WHITELIST: [".", "/login"] + + ### DEPRECATED and replaced by OPENID_CONNECT + LOGIN_OPTIONS configs + ENABLED_IDENTITY_PROVIDERS: + default: google + providers: + google: + name: Login from Google + + # ////////////////////////////////////////////////////////////////////////////////////// + # LIBRARY CONFIGURATION (authlib & flask) + # - Already contains reasonable defaults + # ////////////////////////////////////////////////////////////////////////////////////// + # authlib-specific configs for OIDC flow and JWTs + # NOTE: the OAUTH2_JWT_KEY cfg gets set automatically by fence if keys are setup + # correctly + OAUTH2_JWT_ALG: "RS256" + OAUTH2_JWT_ENABLED: true + OAUTH2_JWT_ISS: "{{BASE_URL}}" + OAUTH2_PROVIDER_ERROR_URI: "/api/oauth2/errors" + + # used for flask, "path mounted under by the application / web server" + # since we deploy as microservices, fence is typically under {{base}}/user + # this is also why our BASE_URL default ends in /user + APPLICATION_ROOT: "/user" + + # ////////////////////////////////////////////////////////////////////////////////////// + # Tokens, Lifetimes, & Expirations + # - Already contains reasonable defaults + # ////////////////////////////////////////////////////////////////////////////////////// + # The name of the browser cookie in which the access token will be stored. + ACCESS_TOKEN_COOKIE_NAME: "access_token" + + # The name of the browser cookie in which the session token will be stored. + # Note that the session token also stores information for the + # ``flask.session`` in the ``context`` field of the token. + SESSION_COOKIE_NAME: "fence" + + # The domain of the browser cookie in which the session token will be stored. + # Leave unset (not empty string!) for normal single-site deployment. + SESSION_COOKIE_DOMAIN: + + OAUTH2_TOKEN_EXPIRES_IN: + "authorization_code": 1200 + "implicit": 1200 + + # The number of seconds after an access token is issued until it expires. + ACCESS_TOKEN_EXPIRES_IN: 1200 + + # The number of seconds after a refresh token is issued until it expires. + REFRESH_TOKEN_EXPIRES_IN: 2592000 + + # The number of seconds after which a browser session is considered stale. + SESSION_TIMEOUT: 1800 + + # The maximum session lifetime in seconds. + SESSION_LIFETIME: 28800 + + # The number of seconds the user's Google service account key used for + # url signing will last before being expired/rotated + # 30 days: 2592000 seconds + GOOGLE_SERVICE_ACCOUNT_KEY_FOR_URL_SIGNING_EXPIRES_IN: 2592000 + + # The number of seconds after a User's Google Service account is added to bucket + # access until it expires. + # 7 days: 604800 seconds + GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN: 604800 + + # The number of seconds after a User's Google account is added to bucket + # access until it expires. + GOOGLE_ACCOUNT_ACCESS_EXPIRES_IN: 86400 + + # The number of seconds after a pre-signed url is issued until it expires. + MAX_PRESIGNED_URL_TTL: 3600 + + # The number of seconds after an API KEY is issued until it expires. + MAX_API_KEY_TTL: 2592000 + + # The number of seconds after an access token is issued until it expires. + MAX_ACCESS_TOKEN_TTL: 3600 + + # TEMPORARY: The maximum number of projects allowed in token claims. + # This config var should be removed after sheepdog and peregrine support + # auth checks against Arborist, and no longer check the token. + TOKEN_PROJECTS_CUTOFF: 10 + + # If set to true, will generate an new access token each time when a browser session update happens + RENEW_ACCESS_TOKEN_BEFORE_EXPIRATION: false + + # The maximum lifetime of a Gen3 passport in seconds + GEN3_PASSPORT_EXPIRES_IN: 43200 + + ######################################################################################## + # OPTIONAL CONFIGURATIONS # + ######################################################################################## + + # For displaying a privacy policy to users, we can either link to the URL specified by + # PRIVACY_POLICY_URL, or default to the `static/privacy_policy.md` file in fence. + PRIVACY_POLICY_URL: null + + # ////////////////////////////////////////////////////////////////////////////////////// + # RELIABILITY OPTS + # ////////////////////////////////////////////////////////////////////////////////////// + # Configurations related to resiliency, fault-tolerance and availability + # This is the number of requests per second that the Nginx proxy will accept before reaching fence + # The value defined in fence-config-public.yaml takes precedence over this one + # In the absence of this OVERRIDE prefixed config, the legacy NGINX_RATE_LIMIT from the k8s deployment yaml is applied + OVERRIDE_NGINX_RATE_LIMIT: 18 + + # ////////////////////////////////////////////////////////////////////////////////////// + # SUPPORT INFO + # ////////////////////////////////////////////////////////////////////////////////////// + # If you want an email address to show up when an unhandled error occurs, provide one + # here. Something like: support@example.com + SUPPORT_EMAIL_FOR_ERRORS: null + + # ////////////////////////////////////////////////////////////////////////////////////// + # SHIBBOLETH + # - Support using `shibboleth` in LOGIN_OPTIONS + # - Contains defaults for using NIH's Login. + # ////////////////////////////////////////////////////////////////////////////////////// + # assumes shibboleth is deployed under {{BASE_URL}}/shibboleth + SHIBBOLETH_HEADER: "persistent_id" + SSO_URL: "https://auth.nih.gov/affwebservices/public/saml2sso?SPID={{BASE_URL}}/shibboleth&RelayState=" + ITRUST_GLOBAL_LOGOUT: "https://auth.nih.gov/siteminderagent/smlogout.asp?mode=nih&AppReturnUrl=" + + DBGAP_ACCESSION_WITH_CONSENT_REGEX: "(?Pphs[0-9]+)(.(?Pv[0-9]+)){0,1}(.(?Pp[0-9]+)){0,1}.(?Pc[0-9]+)" + + # ////////////////////////////////////////////////////////////////////////////////////// + # STORAGE BACKENDS AND CREDENTIALS + # - Optional: Used for `/admin` & `/credentials` endpoints for user management. + # Also used during User Syncing process to automate managing Storage + # access for users. + # ////////////////////////////////////////////////////////////////////////////////////// + # When true, this modifies usersync (not fence service itself) such that when syncing user + # access to a Google storage backend happens in "bulk" by doing a diff *per google group* + # between what's in Google and what's expected. Then it adds, removes only as necessary. + # This is in contrast to the default logic which does blind updates per user and ignores + # 409s from Google. + # NOTE: This reduces the number of API calls to Google in the general case, but increases + # memory usages by usersync (as it has to track all the Google groups and user access) + GOOGLE_BULK_UPDATES: true + + # Configuration for various storage systems for the backend + # NOTE: Remove the {} and supply backends if needed. Example in comments below + # STORAGE_CREDENTIALS: {} + STORAGE_CREDENTIALS: + 'google': + backend: google + # this should be the project id where the Google Groups for data access are managed + google_project_id: dcf-integration + + # Cleversafe data storage backend + # + # 'cleversafe-server-a': + # backend: 'cleversafe' + # aws_access_key_id: '' + # aws_secret_access_key: '' + # host: 'somemanager.osdc.io' + # public_host: 'someobjstore.example.com' + # port: 443 + # is_secure: true + # username: 'someone' + # password: 'somepass' + # is_mocked: true + + # ////////////////////////////////////////////////////////////////////////////////////// + # AWS BUCKETS AND CREDENTIALS + # - Support `/data` endpoints + # ////////////////////////////////////////////////////////////////////////////////////// + + # NOTE: the region is optonal for s3_buckets, however it should be specified to avoid a + # call to GetBucketLocation which you make lack the AWS ACLs for. + # public buckets do not need the region field. + # the cred values should be keys in section `AWS_CREDENTIALS`. + S3_BUCKETS: + cdis-presigned-url-test: + region: 'us-east-1' + cred: 'cdistest' + gen3-helm-data-upload-bucket: + region: 'us-east-1' + cred: 'cdistest' + DATA_UPLOAD_BUCKET: 'gen3-helm-data-upload-bucket' + + # ////////////////////////////////////////////////////////////////////////////////////// + # PROXY + # - Optional: If the api is behind firewall that needs to set http proxy + # ////////////////////////////////////////////////////////////////////////////////////// + # NOTE: leave as-is to not use proxy + # this is only used by the Google Oauth2Client at the moment if provided + HTTP_PROXY: + host: 'cloud-proxy.internal.io' + port: 3128 + + # ////////////////////////////////////////////////////////////////////////////////////// + # MICROSERVICE PATHS + # - Support `/data` endpoints & authz functionality + # ////////////////////////////////////////////////////////////////////////////////////// + # url where indexd microservice is running (for signed urls primarily) + # NOTE: Leaving as null will force fence to default to {{BASE_URL}}/index + # example value: 'https://example.com/index' + INDEXD: http://indexd-service + + # this is the username which fence uses to make authenticated requests to indexd + INDEXD_USERNAME: "fence" + # this is the password which fence uses to make authenticated requests to indexd + INDEXD_PASSWORD: "" + + # ////////////////////////////////////////////////////////////////////////////////////// + # AZURE STORAGE BLOB CONFIGURATION + # - Support Azure Blob Data Access Methods + # ////////////////////////////////////////////////////////////////////////////////////// + + # https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?toc=%2Fazure%2Fstorage%2Fblobs%2Ftoc.json&tabs=azure-portal#view-account-access-keys + # AZ_BLOB_CREDENTIALS: 'fake connection string' + AZ_BLOB_CREDENTIALS: + + # AZ_BLOB_CONTAINER_URL: 'https://storageaccount.blob.core.windows.net/container/' + # this is the container used for uploading, and should match the storage account + # used in the connection string for AZ_BLOB_CREDENTIALS + AZ_BLOB_CONTAINER_URL: "https://myfakeblob.blob.core.windows.net/my-fake-container/" + + # url where authz microservice is running + ARBORIST: http://arborist-service + + # url where the audit-service is running + # This will be dynamically replaced by env_setup.sh script. + AUDIT_SERVICE: "http://audit-service" + ENABLE_AUDIT_LOGS: + presigned_url: true + login: true + # `PUSH_AUDIT_LOGS_CONFIG.type` is one of: [api, aws_sqs]. + # - if type == api: logs are created by hitting the log creation endpoint. + # - if type == aws_sqs: logs are pushed to an SQS and `aws_sqs_config` fields + # `sqs_url` and `region` are required. Field `aws_cred` is optional and it + # should be a key in section `AWS_CREDENTIALS`. + PUSH_AUDIT_LOGS_CONFIG: + type: aws_sqs + aws_sqs_config: + # This will be dynamically replaced by env_setup.sh script. + sqs_url: https://sqs.us-east-1.amazonaws.com/707767160287/audit-service-sqs + region: us-east-1 + aws_cred: 'cdistest' + + # A Google Project identitifier representing the default project to bill to for + # accessing Google Requester Pays buckets (for signed urls and/or temporary service account + # credentials). If this is provided and the API call for + # Google access does not include a `userProject`, this will be used instead. + # + # WARNING: Setting this WITHOUT setting "ENABLE_AUTOMATIC_BILLING_*" to `true` below, + # means that clients and end-users will be responsible for making sure that + # the service account used in either of these methods actually has billing + # permission in the specified project. + BILLING_PROJECT_FOR_SIGNED_URLS: + BILLING_PROJECT_FOR_SA_CREDS: + + # Setting this to `true` will make Fence automatically attempt to create a Custom Role + # in the billing project and give the necessary Google Service Account that role + # (which will allow it to bill to the project). + # + # NOTE: The Fence SA will need the necessary permissions in the specified project to + # both create a custom role and update the Project's IAM Policy to include the + # necessary SA. At the time of writing, there are pre-defined roles in Google's + # IAM that provide the necessary permissions. Those are "Project IAM Admin" and + # "Role Administrator" + # + # NOTE2: It may be possible to further restrict the permissions in the future to + # be more fine-grained. + # + ENABLE_AUTOMATIC_BILLING_PERMISSION_SIGNED_URLS: false + ENABLE_AUTOMATIC_BILLING_PERMISSION_SA_CREDS: false + + # ////////////////////////////////////////////////////////////////////////////////////// + # EMAIL + # - Support for sending emails from fence. Used for user certificates + # and `/google/service_accounts` endpoints + # ////////////////////////////////////////////////////////////////////////////////////// + # Gun Mail Service (for sending emails from fence) + # + # NOTE: Example in comments below + GUN_MAIL: + "datacommons.io": + smtp_hostname: "smtp.mailgun.org" + api_key: "" + default_login: "postmaster@mailgun.example.com" + api_url: "https://api.mailgun.net/v3/mailgun.example.com" + smtp_password: "" + + # For emails regarding users certificates + EMAIL_SERVER: "localhost" + SEND_FROM: "example@gmail.com" + SEND_TO: "example@gmail.com" + + # ////////////////////////////////////////////////////////////////////////////////////// + # DATA ACCESS: GOOGLE LINKING & SERVICE ACCOUNT REGISTRATION + # - Support `/google/service_accounts` endpoints + # ////////////////////////////////////////////////////////////////////////////////////// + # whether or not to allow access to the /link/google endpoints + ALLOW_GOOGLE_LINKING: true + + # A Google Project with controlled data access will be determined INVALID if + # if it has a parent organization UNLESS that parent organization's ID is in this + # whitelist. + # + # NOTE: Remove the [] and Google Organization IDs if needed. Example in comments below + WHITE_LISTED_GOOGLE_PARENT_ORGS: [] + # - '12345678910' + + # when service accounts or google projects are determined invalid, an email is sent + # to the project owners. These settings are for that email + REMOVE_SERVICE_ACCOUNT_EMAIL_NOTIFICATION: + enable: false + # this domain MUST exist in GUN_MAIL config + domain: "example.com" + from: "do-not-reply@example.com" + subject: "User service account removal notification" + # the {} gets replaced dynamically in the Python code to be the Project ID + content: > + Service accounts were removed from access control data because some users or + service accounts of GCP Project {} are not authorized to access the data sets + associated to the service accounts, or do not adhere to the security policies. + # this admin email will be included as a recipient to *any* email to anyone about + # service account removal. + # + # WARNING: This is NOT a bcc so the email is visible to the end-user + admin: + - "admin@example.edu" + + PROBLEM_USER_EMAIL_NOTIFICATION: + # this domain MUST exist in GUN_MAIL config + domain: "example.com" + from: "do-not-reply@example.com" + subject: "Account access error notification" + # the {} gets replaced dynamically in the Python code to be the Project ID + content: > + The Data Commons Framework utilizes dbGaP for data access authorization. + Another member of a Google project you belong to ({}) is attempting to + register a service account to the following additional datasets ({}). + Please contact dbGaP to request access. + # this admin email will be included as a recipient to *any* email to anyone about + # service account removal. + # + # WARNING: This is NOT a bcc so the email is visible to the end-user + admin: + - "admin@example.edu" + + # Service account email domains that represent a service account that Google owns. + # These are usually created when a sepcific GCP service is enabled. + # This is used for Service Account Validation for Data Access. + GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS: + - "dataflow-service-producer-prod.iam.gserviceaccount.com" + - "cloudbuild.gserviceaccount.com" + - "cloud-ml.google.com.iam.gserviceaccount.com" + - "container-engine-robot.iam.gserviceaccount.com" + - "dataflow-service-producer-prod.iam.gserviceaccount.com" + - "sourcerepo-service-accounts.iam.gserviceaccount.com" + - "dataproc-accounts.iam.gserviceaccount.com" + - "gae-api-prod.google.com.iam.gserviceaccount.com" + - "genomics-api.google.com.iam.gserviceaccount.com" + - "containerregistry.iam.gserviceaccount.com" + - "container-analysis.iam.gserviceaccount.com" + - "cloudservices.gserviceaccount.com" + - "stackdriver-service.iam.gserviceaccount.com" + - "appspot.gserviceaccount.com" + - "partnercontent.gserviceaccount.com" + - "trifacta-gcloud-prod.iam.gserviceaccount.com" + - "gcf-admin-robot.iam.gserviceaccount.com" + - "compute-system.iam.gserviceaccount.com" + - "gcp-sa-websecurityscanner.iam.gserviceaccount.com" + - "storage-transfer-service.iam.gserviceaccount.com" + - "firebase-sa-management.iam.gserviceaccount.com" + - "firebase-rules.iam.gserviceaccount.com" + - "gcp-sa-cloudbuild.iam.gserviceaccount.com" + - "gcp-sa-automl.iam.gserviceaccount.com" + - "gcp-sa-datalabeling.iam.gserviceaccount.com" + - "gcp-sa-cloudscheduler.iam.gserviceaccount.com" + + # The types of service accounts that are allowed to be registered at + # /google/service_accounts endpoints + ALLOWED_USER_SERVICE_ACCOUNT_DOMAINS: + # compute engine default service account + - "developer.gserviceaccount.com" + # app engine default service account + - "appspot.gserviceaccount.com" + # user-managed service account + - "iam.gserviceaccount.com" + + # Synapse integration and DREAM challenge mapping. Team is from Synapse, and group is + # providing the actual permission in Arborist. User will be added to the group for TTL + # seconds if the team matches. + DREAM_CHALLENGE_TEAM: "DREAM" + DREAM_CHALLENGE_GROUP: "DREAM" + SYNAPSE_URI: "https://repo-prod.prod.sagebase.org/auth/v1" + SYNAPSE_JWKS_URI: + # deprecated, use the discovery_url in the OPENID_CONNECT block for the synapse client + SYNAPSE_DISCOVERY_URL: + SYNAPSE_AUTHZ_TTL: 86400 + + # Role caching for generating presigned urls if max role session increase is true + # then we can increase the amount of time that a session is valid for + MAX_ROLE_SESSION_INCREASE: false + ASSUME_ROLE_CACHE_SECONDS: 1800 + + # Optional user registration feature: Ask users to register (provide firstname/lastname/org/email) on login. + # If user registers, add them to configured Arborist group; idea is that the Arborist group + # will have access to download data. + REGISTER_USERS_ON: true + REGISTERED_USERS_GROUP: "" + # RAS refresh_tokens expire in 15 days + RAS_REFRESH_EXPIRATION: 1296000 + # List of JWT issuers from which Fence will accept GA4GH visas + GA4GH_VISA_ISSUER_ALLOWLIST: + - "{{BASE_URL}}" + - "https://sts.nih.gov" + - "https://stsstg.nih.gov" + # Number of projects that can be registered to a Google Service Accont + SERVICE_ACCOUNT_LIMIT: 6 + + # Global sync visas during login + # None(Default): Allow per client i.e. a fence client can pick whether or not to sync their visas during login with parse_visas param in /authorization endpoint + # True: Parse for all clients i.e. a fence client will always sync their visas during login + # False: Parse for no clients i.e. a fence client will not be able to sync visas during login even with parse_visas param + GLOBAL_PARSE_VISAS_ON_LOGIN: false + # Settings for usersync with visas + USERSYNC: + sync_from_visas: false + # fallback to dbgap sftp when there are no valid visas for a user i.e. if they're expired or if they're malformed + fallback_to_dbgap_sftp: false + visa_types: + ras: ["https://ras.nih.gov/visas/v1", "https://ras.nih.gov/visas/v1.1"] + RAS_USERINFO_ENDPOINT: "/openid/connect/v1.1/userinfo" diff --git a/ci/perf/values/guppy.yaml b/ci/perf/values/guppy.yaml new file mode 100644 index 000000000..3c5638d00 --- /dev/null +++ b/ci/perf/values/guppy.yaml @@ -0,0 +1,32 @@ +guppy: + replicaCount: 2 + # Lower cost + resources: + requests: + memory: "105Mi" + cpu: "15m" + enabled: true + #TO DO Re-enable once ES index restore is added to modify values script + dbRestore: false + image: + repository: quay.io/cdis/guppy + tag: master + indices: + - index: ci_subject_alias + type: subject + - index: ci_file_alias + type: file + # - index: ci_cmc_alias + # type: clinical_trials + # - index: ci_dataset_alias + # type: dataset + # - index: ci_case_alias + # type: case + # - index: ci_imaging_data_file_alias + # type: imaging_data_file + - index: ci_imaging_study_alias + type: imaging_study + configIndex: ci_configs_alias + authFilterField: auth_resource_path + autoscaling: + enabled: true diff --git a/ci/perf/values/portal.yaml b/ci/perf/values/portal.yaml new file mode 100644 index 000000000..94d2ea9f2 --- /dev/null +++ b/ci/perf/values/portal.yaml @@ -0,0 +1,924 @@ +portal: + # replicaCount: 2 + resources: + limits: + memory: 6Gi + requests: + cpu: 4 + memory: 6Gi + image: + repository: quay.io/cdis/data-portal + tag: master + autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 2 + gitops: + json: | + { + "gaTrackingId": "UA-119127212-2", + "argoTemplate": "gwas-template", + "graphql": { + "boardCounts": [ + ], + "chartCounts": [ + ], + "projectDetails": "boardCounts" + }, + "useArboristUI": true, + "showArboristAuthzOnProfile": true, + "showFenceAuthzOnProfile": false, + "componentToResourceMapping": { + "Workspace": { + "resource": "/workspace", + "method": "access", + "service": "jupyterhub" + }, + "Query": { + "resource": "/query_page", + "method": "access", + "service": "query_page" + }, + "Send Queries": { + "resource": "/query_page", + "method": "access", + "service": "query_page" + }, + "Data Dictionary": { + "resource": "/dictionary_page", + "method": "access", + "service": "dictionary_page" + }, + "Dictionary": { + "resource": "/dictionary_page", + "method": "access", + "service": "dictionary_page" + } + }, + "components": { + "appName": "BloodPAC Metadata Submission Portal", + "index": { + "introduction": { + "heading": "BloodPAC Data Commons", + "text": "The Blood Profiling Atlas in Cancer (BloodPAC) supports the management, analysis and sharing of liquid biopsy data for the oncology research community and aims to accelerate discovery and development of therapies, diagnostic tests, and other technologies for cancer treatment and prevention. The data commons supports cross-project analyses by harmonizing data from different projects through the collaborative development of a data dictionary, providing an API for data queries and download, and providing a cloud-based analysis workspace with rich tools and resources.", + "link": "/submission" + }, + "homepageChartNodes": [{ + "node": "subject", + "name": "Subjects" + } + ], + "buttons": [{ + "name": "Data Dictionary", + "icon": "data-field-define", + "body": "The BloodPAC Data Commons employs a data model to power queries and data submissions. Study the dictionary to send GraphQL queries or format your data submissions.", + "link": "/DD", + "label": "Learn more" + }, + { + "name": "Submit Data", + "icon": "data-submit", + "body": "Browse the list of submitted data projects, or find your project and submit data.", + "link": "/submission", + "label": "Submit data" + }, + { + "name": "Explore Data", + "icon": "data-files", + "body": "The Exploration app enables faceted search for cohort building, getting file download manifests, and browsing data across projects.", + "link": "/explorer", + "label": "Exploration" + }, + { + "name": "Send Queries", + "icon": "data-access", + "body": "Build and send your own custom GraphQL queries to the database to pinpoint specific data.", + "link": "/query", + "label": "Send Queries" + }, + { + "icon": "analysis", + "link": "/analysis", + "color": "#a2a2a2", + "name": "Apps" + } + ] + }, + "navigation": { + "title": "BloodPAC Data Commons", + "items": [ + { + "icon": "query", + "link": "/discovery", + "name": "Discovery", + "tooltip": "Use free-text search and tags to rapidly find relevant studies, discover new datasets across multiple resources, and easily export selected data files to the analysis workspace." + }, + { + "icon": "exploration", + "link": "/explorer", + "color": "#a2a2a2", + "name": "Exploration" + }, + { + "icon": "query", + "link": "/study-viewer/clinical_trials", + "color": "#a2a2a2", + "name": "Study Viewer" + }, + { + "icon": "workspace", + "link": "/workspace", + "color": "#a2a2a2", + "name": "Workspace" + }, + { + "icon": "dictionary", + "link": "/DD", + "color": "#a2a2a2", + "name": "Dictionary" + }, + { + "icon": "analysis", + "link": "/analysis", + "color": "#a2a2a2", + "name": "Apps" + }, + { + "icon": "query", + "link": "/query", + "color": "#a2a2a2", + "name": "Query" + }, + { + "icon": "profile", + "link": "/identity", + "color": "#a2a2a2", + "name": "Profile" + } + ] + }, + "topBar": { + "items": [{ + "link": "https://bloodpac.org/", + "name": "About" + }, + { + "icon": "upload", + "link": "/submission", + "name": "Submit Data" + }, + { + "link": "https://www.synapse.org/#!Synapse:syn8011461/wiki/411591", + "name": "Documentation" + }, + { + "link": "bpa-support@gen3.org", + "name": "Email Support" + } + ], + "useProfileDropdown": false + }, + "login": { + "title": "BloodPAC Data Commons", + "subTitle": "Explore, Analyze, and Share Data", + "text": "This website combines liquid biopsy data from academic, government, and industry partners and aims to accelerate discovery and development of therapies, diagnostic tests, and other technologies for the treatment and prevention of cancer.", + "contact": "If you have any questions about access or the registration process, please contact ", + "email": "bpa-support@gen3.org" + }, + "certs": { + "security_quiz": { + "hasCorrectAnswers": true, + "title": "BloodPAC User agreement", + "description": "", + "questions": [{ + "name": "", + "question": "NOTICE: To protect the system from unauthorized use and to ensure that the system is functioning, activities on this system are monitored and recorded and subject to audit. Use of this system is express consent to such monitoring and recording. Any unauthorized access or use of this system is prohibited. You may only use this system if you have been designated as an Authenticated Individual by your company or institution. You may not use the system in any way that is inconsistent with the agreements between CCSR and your company or institution. Please pay particular attention to the restrictions on controlled access data and transfer it in an approved secure manner.", + "options": ["I agree"], + "answer": 0, + "hint": "Click on I agree" + }] + } + } + }, + "featureFlags": { + "explorer": true, + "analysis": true, + "explorerPublic": true, + "discovery": true, + "discoveryUseAggMDS": true, + "studyRegistration": true + }, + "analysisTools": [ + { + "title": "OHDSI Atlas", + "description": "Use this App for cohort creation. These will be automatically populated in the Gen3 GWAS App", + "applicationUrl": "https://atlas.qa-mickey.planx-pla.net/atlas/", + "image": "/custom/sponsors/gitops-sponsors/atlas-logo.png" + }, + { + "appId": "GWASUIApp", + "title": "Gen3 GWAS", + "description": "Use this App to perform high throughput GWAS on Million Veteran Program (MVP) data, using the University of Washington Genesis pipeline", + "image": "/src/img/analysis-icons/gwas.svg", + "needsTeamProject": true + }, + { + "appId": "GWASResults", + "title": "GWAS Results", + "description": "Use this App to view status & results of submitted workflows", + "image": "/src/img/analysis-icons/gwasResults.svg", + "needsTeamProject": true + } + ], + "discoveryConfig": { + "features": { + "advSearchFilters": { + "enabled": false, + "field": "advSearchFilters", + "filters": [ + { "key": "Research Focus Area" }, + { "key": "Administering IC(s)" }, + { "key": "Simulated vs. Real Data" }, + { "key": "Year Grant Awarded" }, + { "key": "Study Description" } + ] + }, + "exportToWorkspace": { + "enabled": true, + "manifestFieldName": "__manifest", + "downloadManifestButtonText": "Download", + "enableDownloadManifest": false, + "downloadZipButtonText": "Download Zip", + "enableDownloadZip": true + }, + "pageTitle": { + "enabled": false + }, + "search": { + "searchBar": { + "enabled": true, + "searchableTextFields": [ + "summary", + "location", + "study_name", + "institutions", + "investigators", + "project_title", + "project_number", + "dataset_1_title", + "dataset_2_title", + "dataset_3_title", + "dataset_4_title", + "administering_ic", + "research_program", + "research_question", + "dataset_description", + "dataset_1_description", + "dataset_2_description", + "dataset_3_description", + "dataset_4_description", + "data_dictionary" + ] + } + }, + "authorization": { + "enabled": true, + "supportedValues": { + "accessible": { + "enabled": true, + "menuText": "Accessible" + }, + "unaccessible": { + "enabled": true, + "menuText": "Not Accessible" + }, + "pending": { + "enabled": false + }, + "notAvailable": { + "enabled": false + } + } + } + }, + "aggregations": [ + { + "name": "Studies", + "field": "project_number", + "type": "count" + } + ], + "tagSelector": { + "title": "Tags by category", + "showTagCategoryNames": true + }, + "studies": [], + "studyColumns": [ + { + "name": "Project Title", + "field": "project_title", + "errorIfNotAvailable": false, + "valueIfNotAvailable": "n/a" + }, + { + "name": "Research Focus Area", + "field": "research_focus_area", + "errorIfNotAvailable": false, + "valueIfNotAvailable": "n/a" + }, + { + "name": "Investigator(s)", + "field": "investigators", + "errorIfNotAvailable": false, + "valueIfNotAvailable": "n/a" + } + ], + "studyPreviewField": { + "name": "Summary", + "field": "summary", + "contentType": "string", + "includeName": false, + "includeIfNotAvailable": true, + "valueIfNotAvailable": "No summary has been provided for this study." + }, + "studyPageFields": { + "header": { + "field": "project_title" + }, + "fieldsToShow": [ + { + "groupName": "HEAL Research Initiatives", + "groupWidth": "full", + "includeName": false, + "fields": [ + { + "name": "Project Number", + "field": "project_number" + }, + { + "name": "Research Focus Area", + "field": "research_focus_area" + }, + { + "name": "Research Program", + "field": "research_program" + }, + { + "name": "Research Question", + "field": "research_question" + } + ] + }, + { + "groupName": "Host Institution", + "groupWidth": "half", + "includeName": false, + "fields": [ + { + "name": "Institution(s)", + "field": "institutions" + }, + { + "name": "Location", + "field": "location" + }, + { + "name": "Investigator(s)", + "field": "investigators" + }, + { + "name": "Administering IC(s)", + "field": "administering_ic" + }, + { + "name": "Year Grant Awarded", + "field": "year_awarded" + } + ] + }, + { + "groupName": "Clinical Trial", + "groupWidth": "half", + "includeName": false, + "fields": [ + { + "name": "Study Type", + "field": "study_type" + }, + { + "name": "Study Name", + "field": "study_name" + }, + { + "name": "Study Description", + "field": "study_description" + }, + { + "name": "Number of Subjects", + "field": "subjects" + }, + { + "name": "Number of Sites", + "field": "sites" + }, + { + "name": "Clinical Trial Link", + "field": "clinical_trial_link" + }, + { + "name": "Protocol Name", + "field": "protocol_name" + }, + { + "name": "Dataset Description", + "field": "dataset_description" + }, + { + "name": "Dataset Category", + "field": "dataset_category" + } + ] + }, + { + "groupName": "Tags", + "includeName": false, + "groupWidth": "full", + "fields": [ + { + "name": "Tags", + "field": "tags", + "contentType": "tags", + "includeIfNotAvailable": false + } + ] + }, + { + "groupWidth": "full", + "fields": [ + { + "name": "Summary", + "field": "summary", + "contentType": "paragraphs", + "includeName": true, + "includeIfNotAvailable": true, + "valueIfNotAvailable": "No summary has been provided for this study." + } + ] + }, + { + "groupName": "Datasets", + "groupWidth": "full", + "includeName": false, + "fields": [ + { + "name": "Dataset 1 - Title", + "field": "dataset_1_title" + }, + { + "name": "Dataset 1 - Description", + "field": "dataset_1_description" + }, + { + "name": "Dataset 1 - Type", + "field": "dataset_1_type" + }, + { + "name": "Dataset 2 - Title", + "field": "dataset_2_title" + }, + { + "name": "Dataset 2 - Description", + "field": "dataset_2_description" + }, + { + "name": "Dataset 2 - Type", + "field": "dataset_2_type" + }, + { + "name": "Dataset 3 - Title", + "field": "dataset_3_title" + }, + { + "name": "Dataset 3 - Description", + "field": "dataset_3_description" + }, + { + "name": "Dataset 3 - Type", + "field": "dataset_3_type" + }, + { + "name": "Dataset 4 - Title", + "field": "dataset_4_title" + }, + { + "name": "Dataset 4 - Description", + "field": "dataset_4_description" + }, + { + "name": "Dataset 4 - Type", + "field": "dataset_4_type" + }, + { + "name": "Dataset 5 - Title", + "field": "dataset_5_title" + }, + { + "name": "Dataset 5 - Description", + "field": "dataset_5_description" + }, + { + "name": "Dataset 5 - Type", + "field": "dataset_5_type" + } + ] + } + ] + }, + "minimalFieldMapping": { + "authzField": "authz", + "tagsListFieldName": "tags", + "dataAvailabilityField": "data_availability", + "uid": "project_number" + }, + "tagCategories": [ + { + "name": "Demographic Info", + "color": "rgba(64, 34, 100, 1)", + "display": true + }, + { + "name": "Study Design", + "color": "rgba(153, 40, 107, 1)", + "display": true + }, + { + "name": "HEAL Specific", + "color": "rgba(161, 36, 46, 1)", + "display": true + }, + { + "name": "Other", + "color": "rgba(207, 54, 38, 1)", + "display": true + }, + { + "name": "Commons", + "displayName": "Data Resources", + "color": "rgba(21, 6, 76, 1)" + } + ] + }, + "studyViewerConfig": [ + { + "dataType": "subject", + "title": "Studies", + "titleField": "project_id", + "rowAccessor": "submitter_id", + "listItemConfig": { + "blockFields": ["project_id"], + "tableFields": ["data_type"] + }, + "fieldMapping": [ + { "field": "category", "name": "Study Type"}, + { "field": "project_id", "name": "Project_ID"}, + { "field": "data_type", "name": "Data_Type"}, + { "field": "condition", "name": "Condition"} + ], + "openMode": "close-all", + "buttons": [ + { + "type": "download" + }, + { + "type": "request_access", + "resourceDisplayNameField": "title", + "accessRequestedText": "DAR In Progress", + "accessRequestedTooltipText": "Your recently submitted DAR is being reviewed by NIAID" + }] + } + ], + "explorerConfig": [ + { + "tabTitle": "Data", + "charts": { + "project_id": { + "chartType": "count", + "title": "Projects" + } + }, + "filters": { + "tabs": [ + { + "title": "Project", + "fields": [ + "project_id", + "data_type", + "data_format" + ] + } + ] + }, + "table": { + "enabled": true, + "fields": [ + "project_id", + "data_format" + ] + }, + "dropdowns": { + "download": { + "title": "Download" + } + }, + "buttons": [ + { + "enabled": true, + "type": "export", + "title": "Export All to Terra", + "rightIcon": "external-link" + }, + { + "enabled": true, + "type": "export-to-seven-bridges", + "title": "Export All to Seven Bridges", + "rightIcon": "external-link" + }, + { + "enabled": true, + "type": "export-to-pfb", + "title": "Export to PFB", + "leftIcon": "datafile", + "rightIcon": "download" + }, + { + "enabled": true, + "type": "export-to-workspace", + "title": "Export to Workspace", + "leftIcon": "datafile", + "rightIcon": "download" + } + ], + "loginForDownload": true, + "guppyConfig": { + "dataType": "subject", + "nodeCountTitle": "Subjects", + "fieldMapping": [ + { "field": "disease_id", "name": "Disease ID" }, + { "field": "age_of_onset", "name": "Age of Onset" }, + { "field": "project_dbgap_accession_number", "name": "Project dbGaP Accession Number" }, + { "field": "project_dbgap_consent_text", "name":"Project dbGaP Consent Text"}, + { "field": "project_dbgap_phs", "name":"Project dbGaP Phs"} + ], + "manifestMapping": { + "resourceIndexType": "file", + "resourceIdField": "object_id", + "referenceIdFieldInResourceIndex": "_subject_id", + "referenceIdFieldInDataIndex": "_subject_id" + }, + "accessibleFieldCheckList": [ + "project_id" + ], + "accessibleValidationField": "project_id" + }, + "getAccessButtonLink": "https://dbgap.ncbi.nlm.nih.gov/", + "terraExportURL": "https://terra.biodatacatalyst.nhlbi.nih.gov/#import-data", + "terraTemplate": [ + "bdc", + "gen3" + ], + "sevenBridgesExportURL": "https://platform.sb.biodatacatalyst.nhlbi.nih.gov/import/windmill" + }, + { + "tabTitle": "File", + "charts": { + "data_type": { + "chartType": "stackedBar", + "title": "File Type" + }, + "data_format": { + "chartType": "stackedBar", + "title": "File Format" + } + }, + "filters": { + "tabs": [ + { + "title": "File", + "fields": [ + "project_id", + "data_category", + "data_type", + "data_format" + ], + "searchFields": [ + "file_name" + ] + } + ] + }, + "table": { + "enabled": true, + "fields": [ + "project_id", + "file_name", + "file_size", + "object_id" ] + }, + "loginForDownload": true, + "guppyConfig": { + "dataType": "file", + "fieldMapping": [ + { + "field": "object_id", + "name": "GUID" + }, + { + "field": "programs_name", + "name": "Program" + } + ], + "nodeCountTitle": "Files", + "manifestMapping": { + "resourceIndexType": "subject", + "resourceIdField": "_subject_id", + "referenceIdFieldInResourceIndex": "object_id", + "referenceIdFieldInDataIndex": "object_id" + }, + "accessibleFieldCheckList": [ + "project_id" + ], + "accessibleValidationField": "project_id", + "downloadAccessor": "object_id" + }, + "buttons": [ + { + "enabled": true, + "type": "file-manifest", + "title": "Download Manifest", + "leftIcon": "datafile", + "rightIcon": "download", + "fileName": "file-manifest.json", + "dropdownId": "download" + }, + { + "enabled": true, + "type": "export-files-to-workspace", + "title": "Export to Workspace", + "leftIcon": "datafile", + "rightIcon": "download" + }, + { + "enabled": true, + "type": "export-files-to-pfb", + "title": "Export All to PFB", + "rightIcon": "external-link" + }, + { + "enabled": true, + "type": "export-files", + "title": "Export All to Terra", + "rightIcon": "external-link" + }, + { + "enabled": true, + "type": "export-files-to-seven-bridges", + "title": "Export All to Seven Bridges", + "rightIcon": "external-link" + } + ], + "terraExportURL": "https://terra.biodatacatalyst.nhlbi.nih.gov/#import-data", + "terraTemplate": [ + "bdc", + "gen3" + ], + "sevenBridgesExportURL": "https://platform.sb.biodatacatalyst.nhlbi.nih.gov/import/windmill", + "enableLimitedFilePFBExport": { + "sourceNodeField": "source_node" + }, + "dropdowns": {} + }, + { + "tabTitle": "Imaging Studies", + "charts": {}, + "filters": { + "tabs": [ + { + "title": "Study Properties", + "searchFields": [ + "study_uid" + ], + "fields": [ + "study_modality", + "study_description", + "age_at_imaging", + "study_year", + "study_year_shifted", + "body_part_examined", + "days_from_study_to_pos_covid_test", + "days_from_study_to_neg_covid_test", + "project_id" + ] + } + ] + }, + "table": { + "enabled": true, + "fields": [ + "submitter_id", + "study_uid", + "case_ids", + "object_id", + "project_id" + ], + "dicomViewerId": "submitter_id", + "dicomViewerUrl": "ohif-viewer", + "linkFields": [ + "submitter_id" + ] + }, + "dropdowns": { + "download-table": { + "title": "Download Table" + } + }, + "buttons": [ + { + "enabled": true, + "type": "data-json", + "title": "JSON", + "dropdownId": "download-table", + "leftIcon": "datafile", + "rightIcon": "download", + "fileName": "Imaging_Studies_table.json" + }, + { + "enabled": true, + "type": "data-tsv", + "title": "TSV", + "dropdownId": "download-table", + "leftIcon": "datafile", + "rightIcon": "download", + "fileName": "Imaging_Studies_table.tsv" + }, + { + "enabled": true, + "type": "data-csv", + "title": "CSV", + "dropdownId": "download-table", + "leftIcon": "datafile", + "rightIcon": "download", + "fileName": "Imaging_Studies_table.csv" + }, + { + "enabled": true, + "type": "manifest", + "title": "Download File Manifest for Imaging Studies", + "leftIcon": "datafile", + "rightIcon": "download", + "fileName": "imaging_study_manifest.json" + }, + { + "enabled": false, + "type": "export-to-workspace", + "title": "Export to Workspace", + "leftIcon": "datafile", + "rightIcon": "download" + } + ], + "loginForDownload": true, + "guppyConfig": { + "dataType": "imaging_study", + "nodeCountTitle": "Imaging Studies", + "fieldMapping": [ + { + "field": "project_id", + "name": "Project ID" + }, + { + "field": "study_uid", + "name": "Study UID" + }, + { + "field": "submitter_id", + "name": "Browse in DICOM viewer" + }, + { + "field": "case_ids", + "name": "Patient ID" + } + ], + "manifestMapping": { + "resourceIndexType": "data_file", + "resourceIdField": "object_id", + "referenceIdFieldInResourceIndex": "_imaging_study_id", + "referenceIdFieldInDataIndex": "_imaging_study_id", + "useFilterForCounts": false + }, + "accessibleFieldCheckList": [ + "project_id" + ], + "accessibleValidationField": "project_id" + } + } + ] + } diff --git a/ci/perf/values/values.yaml b/ci/perf/values/values.yaml new file mode 100644 index 000000000..4e82f91ea --- /dev/null +++ b/ci/perf/values/values.yaml @@ -0,0 +1,303 @@ +global: + # This will be dynamically replaced by env_setup.sh script. + hostname: + dictionaryUrl: https://s3.amazonaws.com/dictionary-artifacts/anvil/master/schema.json + pdb: true + aws: + externalSecrets: + # -- (bool) Whether to use External Secrets for aws config. + enabled: true + # -- (String) Name of Secrets Manager secret. + externalSecretAwsCreds: ci-aws-config + secretStoreServiceAccount: + # -- (bool) Set true if deploying to AWS and want to use service account and IAM role instead of aws keys. Must provide role-arn. + enabled: true + # -- (string) Name of the service account to create + name: secret-store-sa + # -- (string) AWS Role ARN for Secret Store to use + roleArn: arn:aws:iam::707767160287:role/devplanetv2-external-secrets-sa + sower_role: sower-jobs-role + externalSecrets: + deploy: true + dbCreate: true + manifestGlobalExtraValues: + # This will be dynamically replaced by env_setup.sh script. + fence_url: https://hostname/user + google_enabled: true + tierAccessLevel: libre + +mutatingWebhook: + # -- (bool) Whether to deploy the mutating webhook service. + enabled: false + +postgresql: + primary: + persistence: + enabled: true + size: 1Gi + image: + registry: quay.io + repository: cdis/postgresql + tag: 14.5.0-debian-11-r35 + +maxUnavailable: "" + +elasticsearch: + maxUnavailable: "" + volumeClaimTemplate: + resources: + requests: + storage: 1Gi + +arborist: + replicaCount: 2 + resources: + requests: + memory: "105Mi" + cpu: "15m" + enabled: true + image: + repository: "quay.io/cdis/arborist" + tag: "master" + autoscaling: + enabled: true + +indexd: + replicaCount: 2 + resources: + requests: + memory: "105Mi" + cpu: "15m" + # This will be dynamically replaced by env_setup.sh script. + defaultPrefix: "" + enabled: true + image: + repository: "quay.io/cdis/indexd" + tag: "master" + externalSecrets: + createK8sServiceCredsSecret: true + secrets: + userdb: + # This will be dynamically replaced by env_setup.sh script. + ssj: + # This will be dynamically replaced by env_setup.sh script. + gateway: + autoscaling: + enabled: true + +peregrine: + replicaCount: 2 + resources: + requests: + memory: "105Mi" + cpu: "15m" + enabled: true + image: + repository: "quay.io/cdis/peregrine" + tag: "master" + pullPolicy: Always + autoscaling: + enabled: true + +revproxy: + replicaCount: 2 + ingress: + enabled: true + annotations: + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/tags: Environment=devplanetv2 + alb.ingress.kubernetes.io/certificate-arn: 'arn:aws:acm:us-east-1:707767160287:certificate/f427a646-94fd-4b91-8bcd-80f5920c1e46' + alb.ingress.kubernetes.io/group.name: devplanetv2 + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' + alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' + alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-FIPS-2023-04 + className: alb + hosts: + # This will be dynamically replaced by env_setup.sh script. + - host: HOSTNAME + paths: + - path: / + pathType: Prefix + resources: + requests: + memory: "105Mi" + cpu: "15m" + autoscaling: + enabled: true + +sheepdog: + replicaCount: 2 + resources: + requests: + memory: "105Mi" + cpu: "15m" + enabled: true + authNamespace: '' + image: + repository: "quay.io/cdis/sheepdog" + tag: "master" + # This will be dynamically replaced by env_setup.sh script. + fenceUrl: https://hostname/user + +manifestservice: + replicaCount: 2 + image: + repository: quay.io/cdis/manifestservice + tag: "master" + env: + - name: REQUESTS_CA_BUNDLE + value: /etc/ssl/certs/ca-certificates.crt + - name: MANIFEST_SERVICE_CONFIG_PATH + value: /var/gen3/config/config.json + - name: GEN3_DEBUG + value: "False" + manifestserviceG3auto: + # This will be dynamically replaced by env_setup.sh script. + hostname: + bucketName: gen3-helm-manifest-bucket + prefix: 'test' + externalSecrets: + # -- (string) Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. + createK8sManifestServiceSecret: true + +wts: + replicaCount: 2 + enabled: true + image: + repository: "quay.io/cdis/workspace-token-service" + tag: "master" + externalSecrets: + createWtsOidcClientSecret: true + createK8sWtsSecret: true + +hatchery: + replicaCount: 2 + image: + tag: master + hatchery: + containers: + - target-port: 8888 + cpu-limit: '0.5' + memory-limit: 1Gi + name: "(Tutorial) Bacpac Synthetic Data Analysis Notebook" + image: "quay.io/cdis/heal-notebooks:bacpac__10670ad79b4488cfa9f1b3681ce52bc3f1139b5a" + env: + # This will be dynamically replaced by env_setup.sh script. + FRAME_ANCESTORS: https:// + args: + - "--NotebookApp.base_url=/lw-workspace/proxy/" + - "--NotebookApp.password=''" + - "--NotebookApp.token=''" + - "--NotebookApp.quit_button=False" + command: + - start-notebook.sh + path-rewrite: "/lw-workspace/proxy/" + use-tls: 'false' + ready-probe: "/lw-workspace/proxy/" + lifecycle-post-start: + - "/bin/sh" + - "-c" + - export IAM=`whoami`; rm -rf /home/$IAM/pd/dockerHome; rm -rf /home/$IAM/pd/lost+found; ln -s /data /home/$IAM/pd/; true + user-uid: 1000 + fs-gid: 100 + +metadata: + replicaCount: 2 + image: + repository: quay.io/cdis/metadata-service + tag: 'master' + resources: + # -- (map) The amount of resources that the container requests + requests: + memory: 25Mi + enabled: true + externalSecrets: + createK8sMetadataSecret: true + useAggMds: "True" + aggMdsNamespace: default + aggMdsConfig: | + { + "gen3_commons": { + "HEAL": { + "mds_url": "http://revproxy-service", + "commons_url": "http://revproxy-service" + } + }, + "adapter_commons": {} + } + autoscaling: + enabled: true + +sower: + replicaCount: 2 + image: + repository: quay.io/cdis/sower + tag: 'master' + enabled: true + serviceAccount: + # -- (bool) Specifies whether a service account should be created. + create: true + # -- (string) The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "sower-service-account" + externalSecrets: + createK8sPelicanServiceSecret: false + pelicanserviceG3auto: ci-pelicanservice-g3auto + createK8sSowerJobsSecret: true + sowerConfig: + - name: pelican-export + action: export + container: + name: job-task + image: quay.io/cdis/pelican-export:master + pull_policy: Always + env: + - name: DICTIONARY_URL + valueFrom: + configMapKeyRef: + name: manifest-global + key: dictionary_url + - name: GEN3_HOSTNAME + valueFrom: + configMapKeyRef: + name: manifest-global + key: hostname + - name: ROOT_NODE + value: subject + - name: DB_HOST + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: host + - name: DB_DATABASE + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: database + - name: DB_USER + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: username + - name: DB_PASS + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: password + - name: SHEEPDOG + valueFrom: + secretKeyRef: + name: indexd-service-creds + key: sheepdog + volumeMounts: + - name: pelican-creds-volume + readOnly: true + mountPath: "/pelican-creds.json" + subPath: config.json + cpu-limit: "0.5" + memory-limit: 1Gi + volumes: + - name: pelican-creds-volume + secret: + secretName: pelicanservice-g3auto #pragma: allowlist secret + restart_policy: Never From 100fd455719b29cad02fb6eb46b34735ce632f85 Mon Sep 17 00:00:00 2001 From: krishna Date: Wed, 2 Jul 2025 11:43:55 -0500 Subject: [PATCH 02/14] Create a configuration for perf in gen3-gitops under ci --- ci/default/values/fence.yaml | 641 ++++------------------------------ ci/default/values/guppy.yaml | 20 +- ci/default/values/portal.yaml | 9 +- ci/default/values/values.yaml | 456 ++++++++++-------------- 4 files changed, 264 insertions(+), 862 deletions(-) diff --git a/ci/default/values/fence.yaml b/ci/default/values/fence.yaml index 80b00a329..739c1f4a6 100644 --- a/ci/default/values/fence.yaml +++ b/ci/default/values/fence.yaml @@ -1,603 +1,80 @@ fence: - # -- (map) External Secrets settings. - externalSecrets: - createK8sJwtKeysSecret: false - fenceJwtKeys: ci-fence-jwt - fenceGoogleAppCredsSecret: ci-fence-google-app-creds-secret - fenceGoogleStorageCredsSecret: ci-fence-google-storage-creds-secret - createK8sGoogleAppSecrets: false - fenceConfig: ci-fence-config-priv - fenceSshKeys: ci-fence-ssh-keys - fenceDeleteExpiredClients: - schedule: "0 0 * * *" - replicaCount: 2 + # Lower cost + resources: + requests: + memory: "105Mi" + cpu: "15m" image: repository: quay.io/cdis/fence pullPolicy: IfNotPresent - tag: 'master' + tag: '2025.01' + + # -- (map) Annotations to add to the pod + ## TODO ## Do we need this? + # podAnnotations: + # prometheus.io/path: /metrics + # prometheus.io/scrape: "true" + usersync: usersync: true - userYamlS3Path: s3://cdis-gen3-users/ci/user.yaml + userYamlS3Path: s3://local-helm-testing/user.yaml # we do not want usersync to interrupt test flow # so it is set to run on the 1st of January if it is a Monday schedule: '0 0 1 1 1' - syncFromDbgap: true - FENCE_CONFIG_PUBLIC: - # Prefix to namespace Google Groups on a single Cloud Identity (see cirrus - # setup for more info on Cloud Identity) - # - # NOTE: Make this short! Less than 8 characters if possible. Google has - # length restrictions on group names. - # This will be dynamically replaced by env_setup.sh script. - GOOGLE_GROUP_PREFIX: - - # Prefix to namespace Google Service Accounts in a single Google Cloud Platform Project. - # This is primarily to support multiple instances of fence references the same Google - # project. If that is not something you need to support, then you can leave this blank. - # - # NOTE: Make this short! Less than 8 characters if possible. Google has - # length restrictions on service account names. - # This will be dynamically replaced by env_setup.sh script. - GOOGLE_SERVICE_ACCOUNT_PREFIX: - - # This will be dynamically replaced by env_setup.sh script. - BASE_URL: https://HOSTNAME/user - - # -- (string) Name of the Fence app - APP_NAME: "Gen3 Data Commons" - - # -- (map) Debug and security settings - # Modify based on whether you're in a dev environment or in production - DEBUG: false - # -- (bool) if true, will automatically login a user with username "test" - # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) - MOCK_AUTH: false - - # -- (bool) if true, will fake a successful login response from Google in /login/google - # NOTE: this will also modify the behavior of /link/google endpoints - # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) - # will login as the username set in cookie DEV_LOGIN_COOKIE_NAME - MOCK_GOOGLE_AUTH: true - - # -- (str) the name of the cookie set by mock authentication (used for testing only) - DEV_LOGIN_COOKIE_NAME: "dev_login" - - # -- (bool) if true, will ignore anything configured in STORAGE_CREDENTIALS - MOCK_STORAGE: false - - # -- (bool) allow OIDC traffic on http for development. By default it requires https. - # - # WARNING: ONLY set to true when fence will be deployed in such a way that it will - # ONLY receive traffic from internal clients and can safely use HTTP. - AUTHLIB_INSECURE_TRANSPORT: true - - # -- (bool) enable Prometheus Metrics for observability purposes - # - # WARNING: Any counters, gauges, histograms, etc. should be carefully - # reviewed to make sure its labels do not contain any PII / PHI - ENABLE_PROMETHEUS_METRICS: false - - # -- (bool) set if you want browsers to only send cookies with requests over HTTPS - SESSION_COOKIE_SECURE: true - - # -- (bool) enable CSRF protection - ENABLE_CSRF_PROTECTION: true - - # -- (str) signing key for WTForms to sign CSRF tokens with - WTF_CSRF_SECRET_KEY: "{{ENCRYPTION_KEY}}" - - # -- (bool) fence (at the moment) attempts a migration on startup. setting this to false will disable that - # WARNING: ONLY set to false if you do NOT want to automatically migrate your database. - # You should be careful about incompatible versions of your db schema with what - # fence expects. In other words, things could be broken if you update to a later - # fence that expects a schema your database isn't migrated to. - # NOTE: We are working to improve the migration process in the near future - ENABLE_DB_MIGRATION: true - - # these are the *possible* scopes a client can be given, NOT scopes that are - # given to all clients. You can be more restrictive during client creation - # -- (list) These are the *possible* scopes a client can be given, NOT scopes that are given to all clients. You can be more restrictive during client creation - CLIENT_ALLOWED_SCOPES: - - "openid" - - "user" - - "data" - - "google_credentials" - - "google_service_account" - - "google_link" - - "ga4gh_passport_v1" - - # -- (list) these are the scopes that CAN be included in a user's own access_token - USER_ALLOWED_SCOPES: - - "fence" - - "openid" - - "user" - - "data" - - "admin" - - "google_credentials" - - "google_service_account" - - "google_link" - - "ga4gh_passport_v1" - - # -- (list) these are the scopes that a browser session can create for a user (very similar to USER_ALLOWED_SCOPES, as the session will actually create access_tokens for an actively logged in user) - SESSION_ALLOWED_SCOPES: - - "openid" - - "user" - - "credentials" - - "data" - - "admin" - - "google_credentials" - - "google_service_account" - - "google_link" - - "ga4gh_passport_v1" - - # ////////////////////////////////////////////////////////////////////////////////////// - # LOGIN - # - Modify based on which OIDC provider(s) you configured above - # - NOTE: You can have multiple IDPs for users to login with, but one has to be set - # as the default - # ////////////////////////////////////////////////////////////////////////////////////// - - # List of enabled login options (used by data-portal to display login buttons). - # Each option must be configured with a "name" and an "idp". - # - "idp" must be a configured provider in OPENID_CONNECT section. - # Multiple options can be configured with the same idp. - # - if provider_id is "fence", "fence_idp" can be any of the providers - # supported by the other Fence. If not specified, will default to NIH login. - # - if provider_id is "fence" and fence_idp is "shibboleth", a list of - # "shib_idps" can be configured for InCommon login. If not specified, will - # default to NIH login. - # - Optional parameters: "desc" (description) and "secondary" (boolean - can - # be used by the frontend to display secondary buttons differently). - # -- (list) List of enabled login options (used by data-portal to display login buttons). - LOGIN_OPTIONS: - - name: 'Login from Google' - idp: google - - name: 'ORCID Login' - idp: fence - fence_idp: orcid - - name: 'Login from RAS' - idp: ras - - - # -- (string) Default login provider. - must be configured in LOGIN_OPTIONS and OPENID_CONNECT - - if several options in LOGIN_OPTIONS are defined for this IDP, will default to the first one - DEFAULT_LOGIN_IDP: google - - # -- (string) Default login URL: DEPRECATED and replaced by LOGIN_OPTIONS + DEFAULT_LOGIN_IDP configs - DEFAULT_LOGIN_URL: "{{BASE_URL}}/login/google" - - # `LOGIN_REDIRECT_WHITELIST` is a list of extra whitelisted URLs which can be redirected - # to by the `/login/*` endpoints. Fence automatically populates this with the redirect - # URLs for any registered OAuth clients, and its own URL. When validating the redirects, - # fence chesk whether the domain for the redirect matches a domain in the whitelist (so - # only the domains for the additional desired redirects are necessary here). + syncFromDbgap: false + env: + - name: AWS_ACCESS_KEY_ID + value: test + - name: AWS_SECRET_ACCESS_KEY + value: test + - name: AWS_ENDPOINT_URL + value: 'http://localstack.localstack.svc.cluster.local:4566' + - name: AWS_DEFAULT_REGION + value: 'us-east-1' + + # # -- (map) Public configuration settings for Fence app + # FENCE_CONFIG_PUBLIC: + # ENABLE_PROMETHEUS_METRICS: true + # ENABLE_DB_MIGRATION: true + + FENCE_CONFIG: + ENABLE_AUDIT_LOGS: + presigned_url: true + login: true LOGIN_REDIRECT_WHITELIST: [".", "/login"] - - ### DEPRECATED and replaced by OPENID_CONNECT + LOGIN_OPTIONS configs - ENABLED_IDENTITY_PROVIDERS: - default: google - providers: - google: - name: Login from Google - - # ////////////////////////////////////////////////////////////////////////////////////// - # LIBRARY CONFIGURATION (authlib & flask) - # - Already contains reasonable defaults - # ////////////////////////////////////////////////////////////////////////////////////// - # authlib-specific configs for OIDC flow and JWTs - # NOTE: the OAUTH2_JWT_KEY cfg gets set automatically by fence if keys are setup - # correctly - OAUTH2_JWT_ALG: "RS256" - OAUTH2_JWT_ENABLED: true - OAUTH2_JWT_ISS: "{{BASE_URL}}" - OAUTH2_PROVIDER_ERROR_URI: "/api/oauth2/errors" - - # used for flask, "path mounted under by the application / web server" - # since we deploy as microservices, fence is typically under {{base}}/user - # this is also why our BASE_URL default ends in /user - APPLICATION_ROOT: "/user" - - # ////////////////////////////////////////////////////////////////////////////////////// - # Tokens, Lifetimes, & Expirations - # - Already contains reasonable defaults - # ////////////////////////////////////////////////////////////////////////////////////// - # The name of the browser cookie in which the access token will be stored. - ACCESS_TOKEN_COOKIE_NAME: "access_token" - - # The name of the browser cookie in which the session token will be stored. - # Note that the session token also stores information for the - # ``flask.session`` in the ``context`` field of the token. - SESSION_COOKIE_NAME: "fence" - - # The domain of the browser cookie in which the session token will be stored. - # Leave unset (not empty string!) for normal single-site deployment. - SESSION_COOKIE_DOMAIN: - - OAUTH2_TOKEN_EXPIRES_IN: - "authorization_code": 1200 - "implicit": 1200 - - # The number of seconds after an access token is issued until it expires. - ACCESS_TOKEN_EXPIRES_IN: 1200 - - # The number of seconds after a refresh token is issued until it expires. - REFRESH_TOKEN_EXPIRES_IN: 2592000 - - # The number of seconds after which a browser session is considered stale. - SESSION_TIMEOUT: 1800 - - # The maximum session lifetime in seconds. - SESSION_LIFETIME: 28800 - - # The number of seconds the user's Google service account key used for - # url signing will last before being expired/rotated - # 30 days: 2592000 seconds - GOOGLE_SERVICE_ACCOUNT_KEY_FOR_URL_SIGNING_EXPIRES_IN: 2592000 - - # The number of seconds after a User's Google Service account is added to bucket - # access until it expires. - # 7 days: 604800 seconds - GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN: 604800 - - # The number of seconds after a User's Google account is added to bucket - # access until it expires. - GOOGLE_ACCOUNT_ACCESS_EXPIRES_IN: 86400 - - # The number of seconds after a pre-signed url is issued until it expires. - MAX_PRESIGNED_URL_TTL: 3600 - - # The number of seconds after an API KEY is issued until it expires. - MAX_API_KEY_TTL: 2592000 - - # The number of seconds after an access token is issued until it expires. - MAX_ACCESS_TOKEN_TTL: 3600 - - # TEMPORARY: The maximum number of projects allowed in token claims. - # This config var should be removed after sheepdog and peregrine support - # auth checks against Arborist, and no longer check the token. - TOKEN_PROJECTS_CUTOFF: 10 - - # If set to true, will generate an new access token each time when a browser session update happens - RENEW_ACCESS_TOKEN_BEFORE_EXPIRATION: false - - # The maximum lifetime of a Gen3 passport in seconds - GEN3_PASSPORT_EXPIRES_IN: 43200 - - ######################################################################################## - # OPTIONAL CONFIGURATIONS # - ######################################################################################## - - # For displaying a privacy policy to users, we can either link to the URL specified by - # PRIVACY_POLICY_URL, or default to the `static/privacy_policy.md` file in fence. - PRIVACY_POLICY_URL: null - - # ////////////////////////////////////////////////////////////////////////////////////// - # RELIABILITY OPTS - # ////////////////////////////////////////////////////////////////////////////////////// - # Configurations related to resiliency, fault-tolerance and availability - # This is the number of requests per second that the Nginx proxy will accept before reaching fence - # The value defined in fence-config-public.yaml takes precedence over this one - # In the absence of this OVERRIDE prefixed config, the legacy NGINX_RATE_LIMIT from the k8s deployment yaml is applied - OVERRIDE_NGINX_RATE_LIMIT: 18 - - # ////////////////////////////////////////////////////////////////////////////////////// - # SUPPORT INFO - # ////////////////////////////////////////////////////////////////////////////////////// - # If you want an email address to show up when an unhandled error occurs, provide one - # here. Something like: support@example.com - SUPPORT_EMAIL_FOR_ERRORS: null - - # ////////////////////////////////////////////////////////////////////////////////////// - # SHIBBOLETH - # - Support using `shibboleth` in LOGIN_OPTIONS - # - Contains defaults for using NIH's Login. - # ////////////////////////////////////////////////////////////////////////////////////// - # assumes shibboleth is deployed under {{BASE_URL}}/shibboleth - SHIBBOLETH_HEADER: "persistent_id" - SSO_URL: "https://auth.nih.gov/affwebservices/public/saml2sso?SPID={{BASE_URL}}/shibboleth&RelayState=" - ITRUST_GLOBAL_LOGOUT: "https://auth.nih.gov/siteminderagent/smlogout.asp?mode=nih&AppReturnUrl=" - - DBGAP_ACCESSION_WITH_CONSENT_REGEX: "(?Pphs[0-9]+)(.(?Pv[0-9]+)){0,1}(.(?Pp[0-9]+)){0,1}.(?Pc[0-9]+)" - - # ////////////////////////////////////////////////////////////////////////////////////// - # STORAGE BACKENDS AND CREDENTIALS - # - Optional: Used for `/admin` & `/credentials` endpoints for user management. - # Also used during User Syncing process to automate managing Storage - # access for users. - # ////////////////////////////////////////////////////////////////////////////////////// - # When true, this modifies usersync (not fence service itself) such that when syncing user - # access to a Google storage backend happens in "bulk" by doing a diff *per google group* - # between what's in Google and what's expected. Then it adds, removes only as necessary. - # This is in contrast to the default logic which does blind updates per user and ignores - # 409s from Google. - # NOTE: This reduces the number of API calls to Google in the general case, but increases - # memory usages by usersync (as it has to track all the Google groups and user access) - GOOGLE_BULK_UPDATES: true - - # Configuration for various storage systems for the backend - # NOTE: Remove the {} and supply backends if needed. Example in comments below - # STORAGE_CREDENTIALS: {} - STORAGE_CREDENTIALS: - 'google': - backend: google - # this should be the project id where the Google Groups for data access are managed - google_project_id: dcf-integration - - # Cleversafe data storage backend - # - # 'cleversafe-server-a': - # backend: 'cleversafe' - # aws_access_key_id: '' - # aws_secret_access_key: '' - # host: 'somemanager.osdc.io' - # public_host: 'someobjstore.example.com' - # port: 443 - # is_secure: true - # username: 'someone' - # password: 'somepass' - # is_mocked: true - - # ////////////////////////////////////////////////////////////////////////////////////// - # AWS BUCKETS AND CREDENTIALS - # - Support `/data` endpoints - # ////////////////////////////////////////////////////////////////////////////////////// - - # NOTE: the region is optonal for s3_buckets, however it should be specified to avoid a - # call to GetBucketLocation which you make lack the AWS ACLs for. - # public buckets do not need the region field. - # the cred values should be keys in section `AWS_CREDENTIALS`. + PUSH_AUDIT_LOGS_CONFIG: + type: aws_sqs + aws_sqs_config: + sqs_url: http://localstack.localstack.svc.cluster.local:4566/000000000000/audit-service-sqs + region: us-east-1 + aws_cred: 'cdistest' + AWS_CREDENTIALS: + cdistest: + aws_access_key_id: test + aws_secret_access_key: test #pragma: allowlist secret S3_BUCKETS: cdis-presigned-url-test: region: 'us-east-1' cred: 'cdistest' + endpoint_url: 'http://localstack.localstack.svc.cluster.local:4566' gen3-helm-data-upload-bucket: region: 'us-east-1' cred: 'cdistest' + endpoint_url: 'http://localstack.localstack.svc.cluster.local:4566' DATA_UPLOAD_BUCKET: 'gen3-helm-data-upload-bucket' - - # ////////////////////////////////////////////////////////////////////////////////////// - # PROXY - # - Optional: If the api is behind firewall that needs to set http proxy - # ////////////////////////////////////////////////////////////////////////////////////// - # NOTE: leave as-is to not use proxy - # this is only used by the Google Oauth2Client at the moment if provided + LOGIN_OPTIONS: + - name: 'Login from Google' + desc: 'description' + idp: google + #secondary: True + - name: 'RAS Login' + idp: ras + MOCK_GOOGLE_AUTH: true HTTP_PROXY: host: 'cloud-proxy.internal.io' port: 3128 - - # ////////////////////////////////////////////////////////////////////////////////////// - # MICROSERVICE PATHS - # - Support `/data` endpoints & authz functionality - # ////////////////////////////////////////////////////////////////////////////////////// - # url where indexd microservice is running (for signed urls primarily) - # NOTE: Leaving as null will force fence to default to {{BASE_URL}}/index - # example value: 'https://example.com/index' - INDEXD: http://indexd-service - - # this is the username which fence uses to make authenticated requests to indexd - INDEXD_USERNAME: "fence" - # this is the password which fence uses to make authenticated requests to indexd - INDEXD_PASSWORD: "" - - # ////////////////////////////////////////////////////////////////////////////////////// - # AZURE STORAGE BLOB CONFIGURATION - # - Support Azure Blob Data Access Methods - # ////////////////////////////////////////////////////////////////////////////////////// - - # https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?toc=%2Fazure%2Fstorage%2Fblobs%2Ftoc.json&tabs=azure-portal#view-account-access-keys - # AZ_BLOB_CREDENTIALS: 'fake connection string' - AZ_BLOB_CREDENTIALS: - - # AZ_BLOB_CONTAINER_URL: 'https://storageaccount.blob.core.windows.net/container/' - # this is the container used for uploading, and should match the storage account - # used in the connection string for AZ_BLOB_CREDENTIALS - AZ_BLOB_CONTAINER_URL: "https://myfakeblob.blob.core.windows.net/my-fake-container/" - - # url where authz microservice is running - ARBORIST: http://arborist-service - - # url where the audit-service is running - # This will be dynamically replaced by env_setup.sh script. - AUDIT_SERVICE: "http://audit-service" - ENABLE_AUDIT_LOGS: - presigned_url: true - login: true - # `PUSH_AUDIT_LOGS_CONFIG.type` is one of: [api, aws_sqs]. - # - if type == api: logs are created by hitting the log creation endpoint. - # - if type == aws_sqs: logs are pushed to an SQS and `aws_sqs_config` fields - # `sqs_url` and `region` are required. Field `aws_cred` is optional and it - # should be a key in section `AWS_CREDENTIALS`. - PUSH_AUDIT_LOGS_CONFIG: - type: aws_sqs - aws_sqs_config: - # This will be dynamically replaced by env_setup.sh script. - sqs_url: https://sqs.us-east-1.amazonaws.com/707767160287/audit-service-sqs - region: us-east-1 - aws_cred: 'cdistest' - - # A Google Project identitifier representing the default project to bill to for - # accessing Google Requester Pays buckets (for signed urls and/or temporary service account - # credentials). If this is provided and the API call for - # Google access does not include a `userProject`, this will be used instead. - # - # WARNING: Setting this WITHOUT setting "ENABLE_AUTOMATIC_BILLING_*" to `true` below, - # means that clients and end-users will be responsible for making sure that - # the service account used in either of these methods actually has billing - # permission in the specified project. - BILLING_PROJECT_FOR_SIGNED_URLS: - BILLING_PROJECT_FOR_SA_CREDS: - - # Setting this to `true` will make Fence automatically attempt to create a Custom Role - # in the billing project and give the necessary Google Service Account that role - # (which will allow it to bill to the project). - # - # NOTE: The Fence SA will need the necessary permissions in the specified project to - # both create a custom role and update the Project's IAM Policy to include the - # necessary SA. At the time of writing, there are pre-defined roles in Google's - # IAM that provide the necessary permissions. Those are "Project IAM Admin" and - # "Role Administrator" - # - # NOTE2: It may be possible to further restrict the permissions in the future to - # be more fine-grained. - # - ENABLE_AUTOMATIC_BILLING_PERMISSION_SIGNED_URLS: false - ENABLE_AUTOMATIC_BILLING_PERMISSION_SA_CREDS: false - - # ////////////////////////////////////////////////////////////////////////////////////// - # EMAIL - # - Support for sending emails from fence. Used for user certificates - # and `/google/service_accounts` endpoints - # ////////////////////////////////////////////////////////////////////////////////////// - # Gun Mail Service (for sending emails from fence) - # - # NOTE: Example in comments below - GUN_MAIL: - "datacommons.io": - smtp_hostname: "smtp.mailgun.org" - api_key: "" - default_login: "postmaster@mailgun.example.com" - api_url: "https://api.mailgun.net/v3/mailgun.example.com" - smtp_password: "" - - # For emails regarding users certificates - EMAIL_SERVER: "localhost" - SEND_FROM: "example@gmail.com" - SEND_TO: "example@gmail.com" - - # ////////////////////////////////////////////////////////////////////////////////////// - # DATA ACCESS: GOOGLE LINKING & SERVICE ACCOUNT REGISTRATION - # - Support `/google/service_accounts` endpoints - # ////////////////////////////////////////////////////////////////////////////////////// - # whether or not to allow access to the /link/google endpoints - ALLOW_GOOGLE_LINKING: true - - # A Google Project with controlled data access will be determined INVALID if - # if it has a parent organization UNLESS that parent organization's ID is in this - # whitelist. - # - # NOTE: Remove the [] and Google Organization IDs if needed. Example in comments below - WHITE_LISTED_GOOGLE_PARENT_ORGS: [] - # - '12345678910' - - # when service accounts or google projects are determined invalid, an email is sent - # to the project owners. These settings are for that email - REMOVE_SERVICE_ACCOUNT_EMAIL_NOTIFICATION: - enable: false - # this domain MUST exist in GUN_MAIL config - domain: "example.com" - from: "do-not-reply@example.com" - subject: "User service account removal notification" - # the {} gets replaced dynamically in the Python code to be the Project ID - content: > - Service accounts were removed from access control data because some users or - service accounts of GCP Project {} are not authorized to access the data sets - associated to the service accounts, or do not adhere to the security policies. - # this admin email will be included as a recipient to *any* email to anyone about - # service account removal. - # - # WARNING: This is NOT a bcc so the email is visible to the end-user - admin: - - "admin@example.edu" - - PROBLEM_USER_EMAIL_NOTIFICATION: - # this domain MUST exist in GUN_MAIL config - domain: "example.com" - from: "do-not-reply@example.com" - subject: "Account access error notification" - # the {} gets replaced dynamically in the Python code to be the Project ID - content: > - The Data Commons Framework utilizes dbGaP for data access authorization. - Another member of a Google project you belong to ({}) is attempting to - register a service account to the following additional datasets ({}). - Please contact dbGaP to request access. - # this admin email will be included as a recipient to *any* email to anyone about - # service account removal. - # - # WARNING: This is NOT a bcc so the email is visible to the end-user - admin: - - "admin@example.edu" - - # Service account email domains that represent a service account that Google owns. - # These are usually created when a sepcific GCP service is enabled. - # This is used for Service Account Validation for Data Access. - GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS: - - "dataflow-service-producer-prod.iam.gserviceaccount.com" - - "cloudbuild.gserviceaccount.com" - - "cloud-ml.google.com.iam.gserviceaccount.com" - - "container-engine-robot.iam.gserviceaccount.com" - - "dataflow-service-producer-prod.iam.gserviceaccount.com" - - "sourcerepo-service-accounts.iam.gserviceaccount.com" - - "dataproc-accounts.iam.gserviceaccount.com" - - "gae-api-prod.google.com.iam.gserviceaccount.com" - - "genomics-api.google.com.iam.gserviceaccount.com" - - "containerregistry.iam.gserviceaccount.com" - - "container-analysis.iam.gserviceaccount.com" - - "cloudservices.gserviceaccount.com" - - "stackdriver-service.iam.gserviceaccount.com" - - "appspot.gserviceaccount.com" - - "partnercontent.gserviceaccount.com" - - "trifacta-gcloud-prod.iam.gserviceaccount.com" - - "gcf-admin-robot.iam.gserviceaccount.com" - - "compute-system.iam.gserviceaccount.com" - - "gcp-sa-websecurityscanner.iam.gserviceaccount.com" - - "storage-transfer-service.iam.gserviceaccount.com" - - "firebase-sa-management.iam.gserviceaccount.com" - - "firebase-rules.iam.gserviceaccount.com" - - "gcp-sa-cloudbuild.iam.gserviceaccount.com" - - "gcp-sa-automl.iam.gserviceaccount.com" - - "gcp-sa-datalabeling.iam.gserviceaccount.com" - - "gcp-sa-cloudscheduler.iam.gserviceaccount.com" - - # The types of service accounts that are allowed to be registered at - # /google/service_accounts endpoints - ALLOWED_USER_SERVICE_ACCOUNT_DOMAINS: - # compute engine default service account - - "developer.gserviceaccount.com" - # app engine default service account - - "appspot.gserviceaccount.com" - # user-managed service account - - "iam.gserviceaccount.com" - - # Synapse integration and DREAM challenge mapping. Team is from Synapse, and group is - # providing the actual permission in Arborist. User will be added to the group for TTL - # seconds if the team matches. - DREAM_CHALLENGE_TEAM: "DREAM" - DREAM_CHALLENGE_GROUP: "DREAM" - SYNAPSE_URI: "https://repo-prod.prod.sagebase.org/auth/v1" - SYNAPSE_JWKS_URI: - # deprecated, use the discovery_url in the OPENID_CONNECT block for the synapse client - SYNAPSE_DISCOVERY_URL: - SYNAPSE_AUTHZ_TTL: 86400 - - # Role caching for generating presigned urls if max role session increase is true - # then we can increase the amount of time that a session is valid for - MAX_ROLE_SESSION_INCREASE: false - ASSUME_ROLE_CACHE_SECONDS: 1800 - - # Optional user registration feature: Ask users to register (provide firstname/lastname/org/email) on login. - # If user registers, add them to configured Arborist group; idea is that the Arborist group - # will have access to download data. - REGISTER_USERS_ON: true - REGISTERED_USERS_GROUP: "" - # RAS refresh_tokens expire in 15 days - RAS_REFRESH_EXPIRATION: 1296000 - # List of JWT issuers from which Fence will accept GA4GH visas - GA4GH_VISA_ISSUER_ALLOWLIST: - - "{{BASE_URL}}" - - "https://sts.nih.gov" - - "https://stsstg.nih.gov" - # Number of projects that can be registered to a Google Service Accont - SERVICE_ACCOUNT_LIMIT: 6 - - # Global sync visas during login - # None(Default): Allow per client i.e. a fence client can pick whether or not to sync their visas during login with parse_visas param in /authorization endpoint - # True: Parse for all clients i.e. a fence client will always sync their visas during login - # False: Parse for no clients i.e. a fence client will not be able to sync visas during login even with parse_visas param - GLOBAL_PARSE_VISAS_ON_LOGIN: false - # Settings for usersync with visas - USERSYNC: - sync_from_visas: false - # fallback to dbgap sftp when there are no valid visas for a user i.e. if they're expired or if they're malformed - fallback_to_dbgap_sftp: false - visa_types: - ras: ["https://ras.nih.gov/visas/v1", "https://ras.nih.gov/visas/v1.1"] - RAS_USERINFO_ENDPOINT: "/openid/connect/v1.1/userinfo" + ENABLED_IDENTITY_PROVIDERS: + default: google + providers: + google: + name: Login from Google diff --git a/ci/default/values/guppy.yaml b/ci/default/values/guppy.yaml index 36d80a84c..5faace68f 100644 --- a/ci/default/values/guppy.yaml +++ b/ci/default/values/guppy.yaml @@ -1,12 +1,10 @@ guppy: - replicaCount: 2 # Lower cost resources: requests: memory: "105Mi" cpu: "15m" enabled: true - #TO DO Re-enable once ES index restore is added to modify values script dbRestore: false image: repository: quay.io/cdis/guppy @@ -16,15 +14,15 @@ guppy: type: subject - index: ci_file_alias type: file - # - index: ci_cmc_alias - # type: clinical_trials - # - index: ci_dataset_alias - # type: dataset - # - index: ci_case_alias - # type: case - # - index: ci_imaging_data_file_alias - # type: imaging_data_file - - index: ci_imaging_study_alias + - index: ci_cmc_alias + type: clinical_trials + - index: ci_midrc_dataset_alias + type: dataset + - index: ci_midrc_case_alias + type: case + - index: ci_midrc_imaging_data_file_alias + type: imaging_data_file + - index: ci_midrc_imaging_study_alias type: imaging_study configIndex: ci_configs_alias authFilterField: auth_resource_path diff --git a/ci/default/values/portal.yaml b/ci/default/values/portal.yaml index 4128704a5..33bcc4090 100644 --- a/ci/default/values/portal.yaml +++ b/ci/default/values/portal.yaml @@ -1,11 +1,8 @@ portal: - # replicaCount: 2 resources: - limits: - memory: 6Gi requests: - cpu: 4 - memory: 6Gi + cpu: "0.2" + memory: 100Mi image: repository: quay.io/cdis/data-portal tag: master @@ -654,7 +651,6 @@ portal: "rightIcon": "download" } ], - "loginForDownload": true, "guppyConfig": { "dataType": "subject", "nodeCountTitle": "Subjects", @@ -720,7 +716,6 @@ portal: "file_size", "object_id" ] }, - "loginForDownload": true, "guppyConfig": { "dataType": "file", "fieldMapping": [ diff --git a/ci/default/values/values.yaml b/ci/default/values/values.yaml index 32e18e701..7bfa34dea 100644 --- a/ci/default/values/values.yaml +++ b/ci/default/values/values.yaml @@ -1,133 +1,117 @@ global: - # This will be dynamically replaced by env_setup.sh script. hostname: dictionaryUrl: https://s3.amazonaws.com/dictionary-artifacts/anvil/master/schema.json - pdb: true aws: - externalSecrets: - # -- (bool) Whether to use External Secrets for aws config. - enabled: true - # -- (String) Name of Secrets Manager secret. - externalSecretAwsCreds: ci-aws-config - secretStoreServiceAccount: - # -- (bool) Set true if deploying to AWS and want to use service account and IAM role instead of aws keys. Must provide role-arn. - enabled: true - # -- (string) Name of the service account to create - name: secret-store-sa - # -- (string) AWS Role ARN for Secret Store to use - roleArn: arn:aws:iam::707767160287:role/devplanetv2-external-secrets-sa - sower_role: sower-jobs-role - externalSecrets: - deploy: true - dbCreate: true - manifestGlobalExtraValues: - # This will be dynamically replaced by env_setup.sh script. - fence_url: https://hostname/user - google_enabled: true - tierAccessLevel: libre + awsAccessKeyId: test + awsSecretAccessKey: test #pragma: allowlist secret + region: us-east-1 -mutatingWebhook: - # -- (bool) Whether to deploy the mutating webhook service. - enabled: false +secrets: + awsAccessKeyId: test + awsSecretAccessKey: test #pragma: allowlist secret -postgresql: - primary: - persistence: - enabled: true - size: 1Gi - image: - registry: quay.io - repository: cdis/postgresql - tag: 14.5.0-debian-11-r35 - -hatchery: - replicaCount: 2 +arborist: + resources: + requests: + memory: "105Mi" + cpu: "15m" + enabled: true image: - tag: master - hatchery: - containers: - - target-port: 8888 - cpu-limit: '0.5' - memory-limit: 1Gi - name: "(Tutorial) Bacpac Synthetic Data Analysis Notebook" - image: "quay.io/cdis/heal-notebooks:bacpac__10670ad79b4488cfa9f1b3681ce52bc3f1139b5a" - env: - # This will be dynamically replaced by env_setup.sh script. - FRAME_ANCESTORS: https:// - args: - - "--NotebookApp.base_url=/lw-workspace/proxy/" - - "--NotebookApp.password=''" - - "--NotebookApp.token=''" - - "--NotebookApp.quit_button=False" - command: - - start-notebook.sh - path-rewrite: "/lw-workspace/proxy/" - use-tls: 'false' - ready-probe: "/lw-workspace/proxy/" - lifecycle-post-start: - - "/bin/sh" - - "-c" - - export IAM=`whoami`; rm -rf /home/$IAM/pd/dockerHome; rm -rf /home/$IAM/pd/lost+found; ln -s /data /home/$IAM/pd/; true - user-uid: 1000 - fs-gid: 100 + repository: "quay.io/cdis/arborist" + tag: "master" indexd: - replicaCount: 2 resources: requests: memory: "105Mi" cpu: "15m" - # This will be dynamically replaced by env_setup.sh script. defaultPrefix: "" enabled: true image: repository: "quay.io/cdis/indexd" tag: "master" - externalSecrets: - createK8sServiceCredsSecret: true - secrets: - userdb: - # This will be dynamically replaced by env_setup.sh script. - ssj: - # This will be dynamically replaced by env_setup.sh script. - gateway: - -metadata: - replicaCount: 2 - image: - repository: quay.io/cdis/metadata-service - tag: 'master' +peregrine: resources: - # -- (map) The amount of resources that the container requests requests: - memory: 25Mi + memory: "105Mi" + cpu: "15m" enabled: true - externalSecrets: - createK8sMetadataSecret: true - useAggMds: "True" - aggMdsNamespace: default - aggMdsConfig: | - { - "gen3_commons": { - "HEAL": { - "mds_url": "http://revproxy-service", - "commons_url": "http://revproxy-service" - } - }, - "adapter_commons": {} - } + image: + repository: "quay.io/cdis/peregrine" + tag: "master" + pullPolicy: Always + env: + - name: AWS_ACCESS_KEY_ID + value: test + - name: AWS_SECRET_ACCESS_KEY + value: test + - name: AWS_ENDPOINT_URL + value: 'http://localstack.localstack.svc.cluster.local:4566' + - name: AWS_DEFAULT_REGION + value: 'us-east-1' -maxUnavailable: "" -elasticsearch: - maxUnavailable: "" - volumeClaimTemplate: +revproxy: + ingress: + enabled: true + className: cloudflare-tunnel + hosts: + - host: + paths: + - path: / + pathType: Prefix resources: requests: - storage: 1Gi + memory: "105Mi" + cpu: "15m" + image: + repository: "quay.io/cdis/nginx" + tag: "master" + +sheepdog: + resources: + requests: + memory: "105Mi" + cpu: "15m" + enabled: true + authNamespace: '' + image: + repository: "quay.io/cdis/sheepdog" + tag: "master" + +manifestservice: + resources: + requests: + memory: "105Mi" + cpu: "15m" + enabled: true + image: + repository: "quay.io/cdis/manifestservice" + tag: "master" + manifestserviceG3auto: + bucketName: gen3-helm-manifest-bucket + prefix: 'test' + awsaccesskey: test + awssecretkey: test #pragma: allowlist secret + env: + - name: AWS_ACCESS_KEY_ID + value: test + - name: AWS_SECRET_ACCESS_KEY + value: test + - name: AWS_ENDPOINT_URL + value: 'http://localstack.localstack.svc.cluster.local:4566' + - name: AWS_DEFAULT_REGION + value: 'us-east-1' + - name: REQUESTS_CA_BUNDLE + value: /etc/ssl/certs/ca-certificates.crt + - name: MANIFEST_SERVICE_CONFIG_PATH + value: /var/gen3/config/config.json + - name: GEN3_DEBUG + value: "False" sower: - replicaCount: 2 + pelican: + bucket: 'sower-bucket' image: repository: quay.io/cdis/sower tag: 'master' @@ -135,13 +119,12 @@ sower: serviceAccount: # -- (bool) Specifies whether a service account should be created. create: true + # -- (map) Annotations to add to the service account. + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::000000000000:role/sower-jobs-role # -- (string) The name of the service account to use. # If not set and create is true, a name is generated using the fullname template name: "sower-service-account" - externalSecrets: - createK8sPelicanServiceSecret: false - pelicanserviceG3auto: ci-pelicanservice-g3auto - createK8sSowerJobsSecret: true sowerConfig: - name: pelican-export action: export @@ -187,13 +170,21 @@ sower: secretKeyRef: name: indexd-service-creds key: sheepdog + - name: AWS_ENDPOINT_URL + value: "http://localstack.localstack.svc.cluster.local:4566" + - name: AWS_DEFAULT_REGION + value: us-east-1 + - name: AWS_ACCESS_KEY_ID + value: test + - name: AWS_SECRET_ACCESS_KEY + value: test volumeMounts: - name: pelican-creds-volume readOnly: true mountPath: "/pelican-creds.json" subPath: config.json - cpu-limit: "0.5" - memory-limit: 1Gi + cpu-limit: "1" + memory-limit: 12Gi volumes: - name: pelican-creds-volume secret: @@ -245,13 +236,21 @@ sower: secretKeyRef: name: indexd-service-creds key: sheepdog + - name: AWS_ENDPOINT_URL + value: "http://localstack.localstack.svc.cluster.local:4566" + - name: AWS_DEFAULT_REGION + value: us-east-1 + - name: AWS_ACCESS_KEY_ID + value: test + - name: AWS_SECRET_ACCESS_KEY + value: test volumeMounts: - name: pelican-creds-volume readOnly: true mountPath: "/pelican-creds.json" subPath: config.json - cpu-limit: "0.5" - memory-limit: 1Gi + cpu-limit: "1" + memory-limit: 12Gi volumes: - name: pelican-creds-volume secret: @@ -267,209 +266,142 @@ sower: pull_policy: Always env: - name: BUCKET - value: sower-jobs-bucket + value: sower-bucket - name: GEN3_HOSTNAME valueFrom: configMapKeyRef: name: manifest-global key: hostname + - name: AWS_ENDPOINT_URL + value: "http://localstack.localstack.svc.cluster.local:4566" + - name: AWS_DEFAULT_REGION + value: us-east-1 + - name: AWS_ACCESS_KEY_ID + value: test + - name: AWS_SECRET_ACCESS_KEY + value: test cpu-limit: "1" memory-limit: 1Gi restart_policy: Never ssjdispatcher: - replicaCount: 2 image: repository: quay.io/cdis/ssjdispatcher tag: 'master' enabled: true indexing: quay.io/cdis/indexs3client:master ssjcreds: - # This will be dynamically replaced by env_setup.sh script. - sqsUrl: https://sqs.us-east-1.amazonaws.com/707767160287/gen3-helm-data-upload-bucket - # This will be dynamically replaced by env_setup.sh script. - jobPattern: "s3://gen3-helm-data-upload-bucket//*" - # This will be dynamically replaced by env_setup.sh script. - jobPassword: "replace_with_password" - # This will be dynamically replaced by env_setup.sh script. - metadataservicePassword: "replace_with_password" + sqsUrl: http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/gen3-helm-data-upload-bucket + jobPattern: s3://test-12345678901234-upload/* + jobUser: sheepdog_gen3 + metadataserviceUsername: metadata_gen3 serviceAccount: create: true annotations: - eks.amazonaws.com/role-arn: arn:aws:iam::707767160287:role/ssjdispatcher-role + eks.amazonaws.com/role-arn: arn:aws:iam::000000000000:role/ssjdispatcher-role name: "ssjdispatcher-service-account" - jobServiceAccount: - annotations: - eks.amazonaws.com/role-arn: arn:aws:iam::707767160287:role/ssjdispatcher-role-job - externalSecrets: - createK8sSsjdispatcherSecret: true - # This will be dynamically replaced by env_setup.sh script. - gen3Namespace: default -sheepdog: - replicaCount: 2 - resources: - requests: - memory: "105Mi" - cpu: "15m" - enabled: true - authNamespace: '' +metadata: image: - repository: "quay.io/cdis/sheepdog" - tag: "master" - # This will be dynamically replaced by env_setup.sh script. - fenceUrl: https://hostname/user + repository: quay.io/cdis/metadata-service + tag: 'master' + enabled: true + useAggMds: "True" + aggMdsNamespace: default + aggMdsConfig: | + { + "gen3_commons": { + "HEAL": { + "mds_url": "http://revproxy-service.default.svc.cluster.local", + "commons_url": "http://revproxy-service.default.svc.cluster.local" + } + }, + "adapter_commons": {} + } -peregrine: - replicaCount: 2 - resources: - requests: - memory: "105Mi" - cpu: "15m" +ambassador: enabled: true image: - repository: "quay.io/cdis/peregrine" - tag: "master" - pullPolicy: Always + repository: "quay.io/datawire/ambassador" + tag: "1.4.2" -requestor: - replicaCount: 2 +wts: enabled: true image: - repository: "quay.io/cdis/requestor" + repository: "quay.io/cdis/workspace-token-service" tag: "master" +hatchery: + image: + tag: master + hatchery: + containers: + - target-port: 8888 + cpu-limit: '0.5' + memory-limit: 1Gi + name: "(Tutorial) Bacpac Synthetic Data Analysis Notebook" + image: "quay.io/cdis/heal-notebooks:bacpac__10670ad79b4488cfa9f1b3681ce52bc3f1139b5a" + env: + FRAME_ANCESTORS: https://{{ .Values.global.hostname }} + args: + - "--NotebookApp.base_url=/lw-workspace/proxy/" + - "--NotebookApp.password=''" + - "--NotebookApp.token=''" + - "--NotebookApp.quit_button=False" + command: + - start-notebook.sh + path-rewrite: "/lw-workspace/proxy/" + use-tls: 'false' + ready-probe: "/lw-workspace/proxy/" + lifecycle-post-start: + - "/bin/sh" + - "-c" + - export IAM=`whoami`; rm -rf /home/$IAM/pd/dockerHome; rm -rf /home/$IAM/pd/lost+found; + ln -s /data /home/$IAM/pd/; true + user-uid: 1000 + fs-gid: 100 + audit: - replicaCount: 2 env: - name: ARBORIST_URL valueFrom: - configMapKeyRef: - name: manifest-global - key: arborist_url - optional: true - externalSecrets: - createK8sAuditSecret: true + configMapKeyRef: + name: manifest-global + key: arborist_url + optional: true + - name: AWS_ACCESS_KEY_ID + value: test + - name: AWS_SECRET_ACCESS_KEY + value: test + - name: AWS_ENDPOINT_URL + value: 'http://localstack.localstack.svc.cluster.local:4566' + - name: AWS_DEFAULT_REGION + value: 'us-east-1' image: repository: quay.io/cdis/audit-service - tag: 'master' + tag: '2025.01' server: pull_from_queue: true debug: true sqs: region: us-east-1 - # This will be dynamically replaced by env_setup.sh script. - url: https://sqs.us-east-1.amazonaws.com/707767160287/audit-service-sqs - enabled: true - -ambassador: - replicaCount: 2 - enabled: true - image: - repository: "quay.io/datawire/ambassador" - tag: "1.4.2" - -arborist: - replicaCount: 2 - resources: - requests: - memory: "105Mi" - cpu: "15m" + url: http://localstack.localstack.svc.cluster.local:4566/000000000000/audit-service-sqs enabled: true - image: - repository: "quay.io/cdis/arborist" - tag: "master" - -revproxy: - replicaCount: 2 - ingress: - enabled: true - annotations: - alb.ingress.kubernetes.io/scheme: internet-facing - alb.ingress.kubernetes.io/tags: Environment=devplanetv2 - alb.ingress.kubernetes.io/certificate-arn: 'arn:aws:acm:us-east-1:707767160287:certificate/f427a646-94fd-4b91-8bcd-80f5920c1e46' - alb.ingress.kubernetes.io/group.name: devplanetv2 - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' - alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' - alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-FIPS-2023-04 - className: alb - hosts: - # This will be dynamically replaced by env_setup.sh script. - - host: HOSTNAME - paths: - - path: / - pathType: Prefix - resources: - requests: - memory: "105Mi" - cpu: "15m" - -wts: - replicaCount: 2 +requestor: enabled: true image: - repository: "quay.io/cdis/workspace-token-service" + repository: "quay.io/cdis/requestor" tag: "master" - externalSecrets: - createWtsOidcClientSecret: true - createK8sWtsSecret: true -manifestservice: - replicaCount: 2 - image: - repository: quay.io/cdis/manifestservice - tag: "krishnaa05-patch-1" - env: - - name: REQUESTS_CA_BUNDLE - value: /etc/ssl/certs/ca-certificates.crt - - name: MANIFEST_SERVICE_CONFIG_PATH - value: /var/gen3/config/config.json - - name: GEN3_DEBUG - value: "False" - manifestserviceG3auto: - # This will be dynamically replaced by env_setup.sh script. - hostname: - bucketName: gen3-helm-manifest-bucket - prefix: 'test' - externalSecrets: - # -- (string) Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. - createK8sManifestServiceSecret: true - - -ohif-viewer: - replicaCount: 2 +dicom-viewer: enabled: true image: repository: "quay.io/cdis/ohif-viewer" - tag: "gen3-v3.8.0" + tag: "master" dicom-server: - replicaCount: 2 - enabled: false + enabled: true image: repository: "quay.io/cdis/gen3-orthanc" tag: "master" - -gen3-user-data-library: - replicaCount: 2 - enabled: true - image: - repository: "quay.io/cdis/gen3-user-data-library" - tag: "main" - externalSecrets: - createK8sGen3UserDataLibrarySecret: true - -access-backend: - enabled: false - -orthanc: - replicaCount: 2 - enabled: true - image: - repository: "quay.io/cdis/gen3-orthanc" - tag: "orthancteam-master-gen3-24.3.5" - # -- (map) External Secrets settings. - externalSecrets: - # -- (string) Will override the name of the aws secrets manager secret. Default is "orthanc-s3-g3auto" - orthancS3G3Auto: ci-orthanc-s3-g3auto From 1011c4f1a1f04fbf4f264af9942c9c6248118d70 Mon Sep 17 00:00:00 2001 From: Krishna Agarwal <159047652+krishnaa05@users.noreply.github.com> Date: Wed, 2 Jul 2025 11:45:53 -0500 Subject: [PATCH 03/14] Update etl.yaml --- ci/default/values/etl.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/default/values/etl.yaml b/ci/default/values/etl.yaml index 989f3751c..34f49c683 100644 --- a/ci/default/values/etl.yaml +++ b/ci/default/values/etl.yaml @@ -1,9 +1,13 @@ + etl: enabled: true image: spark: repository: "quay.io/cdis/gen3-spark" tag: "master" + tube: + repository: "quay.io/cdis/tube" + tag: "2024.12" etlMapping: mappings: - name: ci.planx-ci.io_subject From 9e67061252af064eaebfd513dbefec15a19941be Mon Sep 17 00:00:00 2001 From: Krishna Agarwal <159047652+krishnaa05@users.noreply.github.com> Date: Wed, 2 Jul 2025 12:47:26 -0500 Subject: [PATCH 04/14] Dummy commit From 083795c5bb2a2918c6ad229c30716ae68a54e983 Mon Sep 17 00:00:00 2001 From: Krishna Agarwal <159047652+krishnaa05@users.noreply.github.com> Date: Wed, 2 Jul 2025 12:59:04 -0500 Subject: [PATCH 05/14] Delete .github/workflows/integration-tests.yaml --- .github/workflows/integration-tests.yaml | 31 ------------------------ 1 file changed, 31 deletions(-) delete mode 100644 .github/workflows/integration-tests.yaml diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml deleted file mode 100644 index 0e6c46e1e..000000000 --- a/.github/workflows/integration-tests.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: Integration Tests - -on: pull_request - -permissions: - id-token: write - contents: read - pull-requests: write - issues: write - -jobs: - integration_tests: - name: Integration tests - uses: uc-cdis/gen3-code-vigil/.github/workflows/shared_integration_tests.yaml@feat/GPE-1747 - with: - TEST_REPO_BRANCH: chore/tweak_helm_ci - GEN3_INSTANCE_TYPE: HELM_LOCAL - secrets: - CI_AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }} - CI_AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }} - QA_DASHBOARD_S3_PATH: ${{ secrets.QA_DASHBOARD_S3_PATH }} - JENKINS_API_TOKEN: ${{ secrets.JENKINS_API_TOKEN }} - CI_TEST_ORCID_USERID: ${{ secrets.CI_TEST_ORCID_USERID }} - CI_TEST_ORCID_PASSWORD: ${{ secrets.CI_TEST_ORCID_PASSWORD }} - CI_TEST_RAS_EMAIL: ${{ secrets.CI_TEST_RAS_EMAIL }} - CI_TEST_RAS_PASSWORD: ${{ secrets.CI_TEST_RAS_PASSWORD }} - CI_TEST_RAS_2_EMAIL: ${{ secrets.CI_TEST_RAS_2_EMAIL }} - CI_TEST_RAS_2_PASSWORD: ${{ secrets.CI_TEST_RAS_2_PASSWORD }} - CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }} - CI_SLACK_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID }} - EKS_CLUSTER_NAME: ${{ secrets.EKS_CLUSTER_NAME }} From edef6616ba44c42f0494dfc76a61f54439dc43b1 Mon Sep 17 00:00:00 2001 From: krishna Date: Wed, 2 Jul 2025 13:52:48 -0500 Subject: [PATCH 06/14] Remove audit fence-config reference --- ci/perf/values/fence.yaml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/ci/perf/values/fence.yaml b/ci/perf/values/fence.yaml index 9b1d150cb..a475d4924 100644 --- a/ci/perf/values/fence.yaml +++ b/ci/perf/values/fence.yaml @@ -397,25 +397,6 @@ fence: # url where authz microservice is running ARBORIST: http://arborist-service - # url where the audit-service is running - # This will be dynamically replaced by env_setup.sh script. - AUDIT_SERVICE: "http://audit-service" - ENABLE_AUDIT_LOGS: - presigned_url: true - login: true - # `PUSH_AUDIT_LOGS_CONFIG.type` is one of: [api, aws_sqs]. - # - if type == api: logs are created by hitting the log creation endpoint. - # - if type == aws_sqs: logs are pushed to an SQS and `aws_sqs_config` fields - # `sqs_url` and `region` are required. Field `aws_cred` is optional and it - # should be a key in section `AWS_CREDENTIALS`. - PUSH_AUDIT_LOGS_CONFIG: - type: aws_sqs - aws_sqs_config: - # This will be dynamically replaced by env_setup.sh script. - sqs_url: https://sqs.us-east-1.amazonaws.com/707767160287/audit-service-sqs - region: us-east-1 - aws_cred: 'cdistest' - # A Google Project identitifier representing the default project to bill to for # accessing Google Requester Pays buckets (for signed urls and/or temporary service account # credentials). If this is provided and the API call for From 79f2a57e0d3d03ba2fd4d76e4e515cfdc04362a8 Mon Sep 17 00:00:00 2001 From: Krishna Agarwal <159047652+krishnaa05@users.noreply.github.com> Date: Wed, 2 Jul 2025 13:54:56 -0500 Subject: [PATCH 07/14] Dummy commit From b27ed8994fe19a806de74fffcd8c04c30f5a82cb Mon Sep 17 00:00:00 2001 From: krishna Date: Wed, 2 Jul 2025 14:28:59 -0500 Subject: [PATCH 08/14] Disable audit --- ci/perf/values/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/perf/values/values.yaml b/ci/perf/values/values.yaml index 4e82f91ea..09fa388b9 100644 --- a/ci/perf/values/values.yaml +++ b/ci/perf/values/values.yaml @@ -62,6 +62,9 @@ arborist: autoscaling: enabled: true +audit: + enabled: false + indexd: replicaCount: 2 resources: From efa99496724cdc6f5f677996b91bf0566170e39a Mon Sep 17 00:00:00 2001 From: krishna Date: Wed, 2 Jul 2025 16:53:41 -0500 Subject: [PATCH 09/14] Remove export-to-pfb --- ci/perf/values/portal.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ci/perf/values/portal.yaml b/ci/perf/values/portal.yaml index 94d2ea9f2..5bfa80d15 100644 --- a/ci/perf/values/portal.yaml +++ b/ci/perf/values/portal.yaml @@ -643,13 +643,6 @@ portal: "title": "Export All to Seven Bridges", "rightIcon": "external-link" }, - { - "enabled": true, - "type": "export-to-pfb", - "title": "Export to PFB", - "leftIcon": "datafile", - "rightIcon": "download" - }, { "enabled": true, "type": "export-to-workspace", From 1e988138a3d430bf89ef58e671c4caf152fa3532 Mon Sep 17 00:00:00 2001 From: krishna Date: Thu, 3 Jul 2025 06:03:42 -0500 Subject: [PATCH 10/14] Disable Etl --- ci/perf/values/etl.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 ci/perf/values/etl.yaml diff --git a/ci/perf/values/etl.yaml b/ci/perf/values/etl.yaml new file mode 100644 index 000000000..91bbfa05a --- /dev/null +++ b/ci/perf/values/etl.yaml @@ -0,0 +1,2 @@ +etl: + enabled: false From e0cc06a53f46dd20eaab6b9e00537ffecc848784 Mon Sep 17 00:00:00 2001 From: krishna Date: Thu, 3 Jul 2025 06:41:41 -0500 Subject: [PATCH 11/14] Minor fix --- ci/perf/values/etl.yaml | 174 ++++++++++++++++++++++++++++++++++++- ci/perf/values/portal.yaml | 29 ------- 2 files changed, 173 insertions(+), 30 deletions(-) diff --git a/ci/perf/values/etl.yaml b/ci/perf/values/etl.yaml index 91bbfa05a..34f49c683 100644 --- a/ci/perf/values/etl.yaml +++ b/ci/perf/values/etl.yaml @@ -1,2 +1,174 @@ + etl: - enabled: false + enabled: true + image: + spark: + repository: "quay.io/cdis/gen3-spark" + tag: "master" + tube: + repository: "quay.io/cdis/tube" + tag: "2024.12" + etlMapping: + mappings: + - name: ci.planx-ci.io_subject + doc_type: subject + type: aggregator + root: subject + props: + - name: submitter_id + - name: project_id + - name: sex + - name: age_value + - name: ancestry + - name: disease_description + - name: phenotype_present + - name: phenotype_absent + - name: disease_id + - name: solve_state + - name: congenital_status + - name: age_of_onset + - name: phenotype_group + - name: anvil_project_id + # flatten_props: + aggregated_props: + - name: sample_provider + path: samples + src: sample_provider + fn: set + - name: tissue_affected_status + path: samples + src: tissue_affected_status + fn: set + - name: tissue_type + path: samples + src: tissue_type + fn: set + - name: sample_type + path: samples + src: sample_type + fn: set + - name: original_material_type + path: samples + src: original_material_type + fn: set + - name: exome_capture_platform + path: samples.sequencings + src: exome_capture_platform + fn: set + - name: library_prep_kit_method + path: samples.sequencings + src: library_prep_kit_method + fn: set + - name: capture_region_bed_file + path: samples.sequencings + src: capture_region_bed_file + fn: set + - name: reference_genome_build + path: samples.sequencings + src: reference_genome_build + fn: set + - name: sequencing_assay + path: samples.sequencings + src: sequencing_assay + fn: set + - name: alignment_method + path: samples.sequencings + src: alignment_method + fn: set + - name: data_processing_pipeline + path: samples.sequencings + src: data_processing_pipeline + fn: set + - name: _samples_count + path: samples + fn: count + - name: _sequencings_count + path: samples.sequencings + fn: count + joining_props: + - index: file + join_on: _subject_id + props: + - name: data_format + src: data_format + fn: set + - name: data_type + src: data_type + fn: set + - name: data_category + src: data_category + fn: set + - name: file_count + src: _file_id + fn: count + parent_props: + - path: projects[project_code:code] + - path: projects[project_name:name] + - path: projects[project_dbgap_phs:dbgap_phs] + - path: projects[project_dbgap_consent_text:dbgap_consent_text] + - path: projects[project_short_name:short_name] + - path: projects[project_dbgap_accession_number:dbgap_accession_number] + - name: ci.planx-ci.io_file + doc_type: file + type: collector + root: None + category: data_file + props: + - name: project_id + - name: submitter_id + - name: object_id + - name: md5sum + - name: file_name + - name: file_size + - name: data_format + - name: data_type + - name: state + - name: data_category + - name: analyte_type + - name: sequencing_assay + - name: source_node + injecting_props: + subject: + props: + - name: _subject_id + src: id + fn: set + - name: subject_submitter_id + src: submitter_id + fn: set + - name: anvil_project_id + src: anvil_project_id + fn: set + - name: sex + src: sex + fn: set + - name: age_value + src: age_value + fn: set + - name: ancestry + src: ancestry + fn: set + - name: disease_description + src: disease_description + fn: set + - name: phenotype_present + src: phenotype_present + fn: set + - name: phenotype_absent + src: phenotype_absent + fn: set + - name: disease_id + src: disease_id + fn: set + - name: solve_state + src: solve_state + fn: set + - name: congenital_status + src: congenital_status + fn: set + - name: age_of_onset + src: age_of_onset + fn: set + - name: phenotype_group + src: phenotype_group + fn: set diff --git a/ci/perf/values/portal.yaml b/ci/perf/values/portal.yaml index 5bfa80d15..c65dd1ffb 100644 --- a/ci/perf/values/portal.yaml +++ b/ci/perf/values/portal.yaml @@ -568,35 +568,6 @@ portal: } ] }, - "studyViewerConfig": [ - { - "dataType": "subject", - "title": "Studies", - "titleField": "project_id", - "rowAccessor": "submitter_id", - "listItemConfig": { - "blockFields": ["project_id"], - "tableFields": ["data_type"] - }, - "fieldMapping": [ - { "field": "category", "name": "Study Type"}, - { "field": "project_id", "name": "Project_ID"}, - { "field": "data_type", "name": "Data_Type"}, - { "field": "condition", "name": "Condition"} - ], - "openMode": "close-all", - "buttons": [ - { - "type": "download" - }, - { - "type": "request_access", - "resourceDisplayNameField": "title", - "accessRequestedText": "DAR In Progress", - "accessRequestedTooltipText": "Your recently submitted DAR is being reviewed by NIAID" - }] - } - ], "explorerConfig": [ { "tabTitle": "Data", From 3132c0b0e949d65ba6a4bc7f9905e85959204a07 Mon Sep 17 00:00:00 2001 From: krishna Date: Thu, 3 Jul 2025 08:12:53 -0500 Subject: [PATCH 12/14] Use jenkins-perf gitops --- ci/perf/values/portal.yaml | 1018 ++++++++++++++---------------------- 1 file changed, 392 insertions(+), 626 deletions(-) diff --git a/ci/perf/values/portal.yaml b/ci/perf/values/portal.yaml index c65dd1ffb..dfd163870 100644 --- a/ci/perf/values/portal.yaml +++ b/ci/perf/values/portal.yaml @@ -16,102 +16,88 @@ portal: gitops: json: | { - "gaTrackingId": "UA-119127212-2", - "argoTemplate": "gwas-template", + "gaTrackingId": "UA-119127212-3", "graphql": { - "boardCounts": [ + "boardCounts": [{ + "graphql": "_subject_count", + "name": "Subject", + "plural": "Subjects" + }, + { + "graphql": "_study_count", + "name": "Study", + "plural": "Studies" + }, + { + "graphql": "_aliquot_count", + "name": "Aliquot", + "plural": "Aliquots" + } ], - "chartCounts": [ + "chartCounts": [{ + "graphql": "_subject_count", + "name": "Subject" + }, + { + "graphql": "_study_count", + "name": "Study" + } ], "projectDetails": "boardCounts" }, - "useArboristUI": true, - "showArboristAuthzOnProfile": true, - "showFenceAuthzOnProfile": false, - "componentToResourceMapping": { - "Workspace": { - "resource": "/workspace", - "method": "access", - "service": "jupyterhub" - }, - "Query": { - "resource": "/query_page", - "method": "access", - "service": "query_page" - }, - "Send Queries": { - "resource": "/query_page", - "method": "access", - "service": "query_page" - }, - "Data Dictionary": { - "resource": "/dictionary_page", - "method": "access", - "service": "dictionary_page" - }, - "Dictionary": { - "resource": "/dictionary_page", - "method": "access", - "service": "dictionary_page" - } - }, "components": { - "appName": "BloodPAC Metadata Submission Portal", + "appName": "NHLBI BioData Catalyst", "index": { "introduction": { - "heading": "BloodPAC Data Commons", - "text": "The Blood Profiling Atlas in Cancer (BloodPAC) supports the management, analysis and sharing of liquid biopsy data for the oncology research community and aims to accelerate discovery and development of therapies, diagnostic tests, and other technologies for cancer treatment and prevention. The data commons supports cross-project analyses by harmonizing data from different projects through the collaborative development of a data dictionary, providing an API for data queries and download, and providing a cloud-based analysis workspace with rich tools and resources.", + "heading": "", + "text": "NHLBI BioData Catalyst supports the management, analysis and sharing of human disease data for the research community and aims to advance basic understanding of the genetic basis of complex traits and accelerate discovery and development of therapies, diagnostic tests, and other technologies for diseases like cancer. The data commons supports cross-project analyses by harmonizing data from different projects through the collaborative development of a data dictionary, providing an API for data queries and download, and providing a cloud-based analysis workspace with rich tools and resources.", "link": "/submission" }, - "homepageChartNodes": [{ - "node": "subject", - "name": "Subjects" - } - ], "buttons": [{ - "name": "Data Dictionary", - "icon": "data-field-define", - "body": "The BloodPAC Data Commons employs a data model to power queries and data submissions. Study the dictionary to send GraphQL queries or format your data submissions.", + "name": "Define Data Field", + "icon": "stage-planning", + "body": "NHLBI BioData Catalyst define the data in a general way. Please study the dictionary before you start browsing.", "link": "/DD", "label": "Learn more" }, - { - "name": "Submit Data", - "icon": "data-submit", - "body": "Browse the list of submitted data projects, or find your project and submit data.", - "link": "/submission", - "label": "Submit data" - }, { "name": "Explore Data", - "icon": "data-files", - "body": "The Exploration app enables faceted search for cohort building, getting file download manifests, and browsing data across projects.", + "icon": "stage-explore", + "body": "The Exploration Page gives you insights and a clear overview under selected factors.", "link": "/explorer", - "label": "Exploration" + "label": "Explore data" }, { - "name": "Send Queries", - "icon": "data-access", - "body": "Build and send your own custom GraphQL queries to the database to pinpoint specific data.", + "name": "Access Data", + "icon": "stage-access", + "body": "Use our selected tool to filter out the data you need.", "link": "/query", - "label": "Send Queries" + "label": "Query data" }, { - "icon": "analysis", - "link": "/analysis", - "color": "#a2a2a2", - "name": "Apps" + "name": "Analyze Data", + "icon": "stage-analyze", + "body": "Analyze your selected cases using Jupyter Notebooks in our secure cloud environment", + "link": "/workspace", + "label": "Run analysis" + } + ], + "homepageChartNodes": [{ + "node": "subject", + "name": "Subjects" + }, + { + "node": "study", + "name": "Studies" } ] }, "navigation": { - "title": "BloodPAC Data Commons", - "items": [ - { - "icon": "query", - "link": "/discovery", - "name": "Discovery", - "tooltip": "Use free-text search and tags to rapidly find relevant studies, discover new datasets across multiple resources, and easily export selected data files to the analysis workspace." + "items": [{ + "icon": "dictionary", + "link": "/DD", + "color": "#a2a2a2", + "name": "Dictionary" }, { "icon": "exploration", @@ -121,9 +107,9 @@ portal: }, { "icon": "query", - "link": "/study-viewer/clinical_trials", + "link": "/query", "color": "#a2a2a2", - "name": "Study Viewer" + "name": "Query" }, { "icon": "workspace", @@ -131,24 +117,6 @@ portal: "color": "#a2a2a2", "name": "Workspace" }, - { - "icon": "dictionary", - "link": "/DD", - "color": "#a2a2a2", - "name": "Dictionary" - }, - { - "icon": "analysis", - "link": "/analysis", - "color": "#a2a2a2", - "name": "Apps" - }, - { - "icon": "query", - "link": "/query", - "color": "#a2a2a2", - "name": "Query" - }, { "icon": "profile", "link": "/identity", @@ -159,433 +127,184 @@ portal: }, "topBar": { "items": [{ - "link": "https://bloodpac.org/", - "name": "About" - }, - { "icon": "upload", "link": "/submission", "name": "Submit Data" }, { - "link": "https://www.synapse.org/#!Synapse:syn8011461/wiki/411591", + "link": "https://bdcatalyst.gitbook.io/biodata-catalyst-documentation/explore_data/gen3-discovering-data", "name": "Documentation" - }, - { - "link": "bpa-support@gen3.org", - "name": "Email Support" } - ], - "useProfileDropdown": false + ] }, "login": { - "title": "BloodPAC Data Commons", + "title": "NHLBI BioData Catalyst", "subTitle": "Explore, Analyze, and Share Data", - "text": "This website combines liquid biopsy data from academic, government, and industry partners and aims to accelerate discovery and development of therapies, diagnostic tests, and other technologies for the treatment and prevention of cancer.", + "text": "This website supports the management, analysis and sharing of human disease data for the research community and aims to advance basic understanding of the genetic basis of complex traits and accelerate discovery and development of therapies, diagnostic tests, and other technologies for diseases like cancer.", "contact": "If you have any questions about access or the registration process, please contact ", - "email": "bpa-support@gen3.org" - }, - "certs": { - "security_quiz": { - "hasCorrectAnswers": true, - "title": "BloodPAC User agreement", - "description": "", - "questions": [{ - "name": "", - "question": "NOTICE: To protect the system from unauthorized use and to ensure that the system is functioning, activities on this system are monitored and recorded and subject to audit. Use of this system is express consent to such monitoring and recording. Any unauthorized access or use of this system is prohibited. You may only use this system if you have been designated as an Authenticated Individual by your company or institution. You may not use the system in any way that is inconsistent with the agreements between CCSR and your company or institution. Please pay particular attention to the restrictions on controlled access data and transfer it in an approved secure manner.", - "options": ["I agree"], - "answer": 0, - "hint": "Click on I agree" - }] - } - } - }, - "featureFlags": { - "explorer": true, - "analysis": true, - "explorerPublic": true, - "discovery": true, - "discoveryUseAggMDS": true, - "studyRegistration": true - }, - "analysisTools": [ - { - "title": "OHDSI Atlas", - "description": "Use this App for cohort creation. These will be automatically populated in the Gen3 GWAS App", - "applicationUrl": "https://atlas.qa-mickey.planx-pla.net/atlas/", - "image": "/custom/sponsors/gitops-sponsors/atlas-logo.png" - }, - { - "appId": "GWASUIApp", - "title": "Gen3 GWAS", - "description": "Use this App to perform high throughput GWAS on Million Veteran Program (MVP) data, using the University of Washington Genesis pipeline", - "image": "/src/img/analysis-icons/gwas.svg", - "needsTeamProject": true - }, - { - "appId": "GWASResults", - "title": "GWAS Results", - "description": "Use this App to view status & results of submitted workflows", - "image": "/src/img/analysis-icons/gwasResults.svg", - "needsTeamProject": true - } - ], - "discoveryConfig": { - "features": { - "advSearchFilters": { - "enabled": false, - "field": "advSearchFilters", - "filters": [ - { "key": "Research Focus Area" }, - { "key": "Administering IC(s)" }, - { "key": "Simulated vs. Real Data" }, - { "key": "Year Grant Awarded" }, - { "key": "Study Description" } - ] + "contact_link": { + "href": "https://biodatacatalyst.nhlbi.nih.gov/contact" }, - "exportToWorkspace": { - "enabled": true, - "manifestFieldName": "__manifest", - "downloadManifestButtonText": "Download", - "enableDownloadManifest": false, - "downloadZipButtonText": "Download Zip", - "enableDownloadZip": true - }, - "pageTitle": { - "enabled": false - }, - "search": { - "searchBar": { - "enabled": true, - "searchableTextFields": [ - "summary", - "location", - "study_name", - "institutions", - "investigators", - "project_title", - "project_number", - "dataset_1_title", - "dataset_2_title", - "dataset_3_title", - "dataset_4_title", - "administering_ic", - "research_program", - "research_question", - "dataset_description", - "dataset_1_description", - "dataset_2_description", - "dataset_3_description", - "dataset_4_description", - "data_dictionary" - ] - } - }, - "authorization": { - "enabled": true, - "supportedValues": { - "accessible": { - "enabled": true, - "menuText": "Accessible" - }, - "unaccessible": { - "enabled": true, - "menuText": "Not Accessible" - }, - "pending": { - "enabled": false - }, - "notAvailable": { - "enabled": false - } - } - } - }, - "aggregations": [ - { - "name": "Studies", - "field": "project_number", - "type": "count" - } - ], - "tagSelector": { - "title": "Tags by category", - "showTagCategoryNames": true + "image": "stage-icons/stage-gene" }, - "studies": [], - "studyColumns": [ - { - "name": "Project Title", - "field": "project_title", - "errorIfNotAvailable": false, - "valueIfNotAvailable": "n/a" + "footerLogos": [{ + "src": "/src/img/gen3.png", + "href": "https://ctds.uchicago.edu/gen3", + "alt": "Gen3 Data Commons" }, { - "name": "Research Focus Area", - "field": "research_focus_area", - "errorIfNotAvailable": false, - "valueIfNotAvailable": "n/a" + "src": "/src/img/createdby.png", + "href": "https://ctds.uchicago.edu/", + "alt": "Center for Translational Data Science at the University of Chicago" }, { - "name": "Investigator(s)", - "field": "investigators", - "errorIfNotAvailable": false, - "valueIfNotAvailable": "n/a" + "src": "/src/img/sponsors/nhlbi.png", + "href": "https://www.nhlbi.nih.gov", + "alt": "NHLBI" } ], - "studyPreviewField": { - "name": "Summary", - "field": "summary", - "contentType": "string", - "includeName": false, - "includeIfNotAvailable": true, - "valueIfNotAvailable": "No summary has been provided for this study." - }, - "studyPageFields": { - "header": { - "field": "project_title" - }, - "fieldsToShow": [ + "footer": { + "links": [{ + "href": "https://biodatacatalyst.nhlbi.nih.gov/privacy/", + "text": "Privacy Policy" + }, { - "groupName": "HEAL Research Initiatives", - "groupWidth": "full", - "includeName": false, - "fields": [ - { - "name": "Project Number", - "field": "project_number" - }, - { - "name": "Research Focus Area", - "field": "research_focus_area" - }, - { - "name": "Research Program", - "field": "research_program" - }, - { - "name": "Research Question", - "field": "research_question" - } - ] + "href": "https://osp.od.nih.gov/scientific-sharing/policies/", + "text": "Data Sharing Policy" }, { - "groupName": "Host Institution", - "groupWidth": "half", - "includeName": false, - "fields": [ - { - "name": "Institution(s)", - "field": "institutions" - }, - { - "name": "Location", - "field": "location" - }, - { - "name": "Investigator(s)", - "field": "investigators" - }, - { - "name": "Administering IC(s)", - "field": "administering_ic" - }, - { - "name": "Year Grant Awarded", - "field": "year_awarded" - } - ] + "text": "Freedom of Information Act (FOIA)", + "href": "https://www.nhlbi.nih.gov/about/foia-fee-for-service-office" }, { - "groupName": "Clinical Trial", - "groupWidth": "half", - "includeName": false, - "fields": [ - { - "name": "Study Type", - "field": "study_type" - }, - { - "name": "Study Name", - "field": "study_name" - }, - { - "name": "Study Description", - "field": "study_description" - }, - { - "name": "Number of Subjects", - "field": "subjects" - }, - { - "name": "Number of Sites", - "field": "sites" - }, - { - "name": "Clinical Trial Link", - "field": "clinical_trial_link" - }, - { - "name": "Protocol Name", - "field": "protocol_name" - }, - { - "name": "Dataset Description", - "field": "dataset_description" - }, - { - "name": "Dataset Category", - "field": "dataset_category" - } - ] + "text": "Accessibility Information", + "href": "https://biodatacatalyst.nhlbi.nih.gov/accessibility" }, { - "groupName": "Tags", - "includeName": false, - "groupWidth": "full", - "fields": [ - { - "name": "Tags", - "field": "tags", - "contentType": "tags", - "includeIfNotAvailable": false - } - ] + "text": "U.S. Department of Health & Human Services", + "href": "https://www.hhs.gov" }, { - "groupWidth": "full", - "fields": [ - { - "name": "Summary", - "field": "summary", - "contentType": "paragraphs", - "includeName": true, - "includeIfNotAvailable": true, - "valueIfNotAvailable": "No summary has been provided for this study." - } - ] + "text": "National Institutes of Health", + "href": "https://www.nih.gov" }, { - "groupName": "Datasets", - "groupWidth": "full", - "includeName": false, - "fields": [ - { - "name": "Dataset 1 - Title", - "field": "dataset_1_title" - }, - { - "name": "Dataset 1 - Description", - "field": "dataset_1_description" - }, - { - "name": "Dataset 1 - Type", - "field": "dataset_1_type" - }, - { - "name": "Dataset 2 - Title", - "field": "dataset_2_title" - }, - { - "name": "Dataset 2 - Description", - "field": "dataset_2_description" - }, - { - "name": "Dataset 2 - Type", - "field": "dataset_2_type" - }, - { - "name": "Dataset 3 - Title", - "field": "dataset_3_title" - }, - { - "name": "Dataset 3 - Description", - "field": "dataset_3_description" - }, - { - "name": "Dataset 3 - Type", - "field": "dataset_3_type" - }, - { - "name": "Dataset 4 - Title", - "field": "dataset_4_title" - }, - { - "name": "Dataset 4 - Description", - "field": "dataset_4_description" - }, - { - "name": "Dataset 4 - Type", - "field": "dataset_4_type" - }, - { - "name": "Dataset 5 - Title", - "field": "dataset_5_title" - }, - { - "name": "Dataset 5 - Description", - "field": "dataset_5_description" - }, - { - "name": "Dataset 5 - Type", - "field": "dataset_5_type" - } - ] + "text": "National Heart, Lung, and Blood Institute", + "href": "https://www.nhlbi.nih.gov" + }, + { + "text": "USA.gov", + "href": "https://www.usa.gov" } ] }, - "minimalFieldMapping": { - "authzField": "authz", - "tagsListFieldName": "tags", - "dataAvailabilityField": "data_availability", - "uid": "project_number" - }, - "tagCategories": [ - { - "name": "Demographic Info", - "color": "rgba(64, 34, 100, 1)", - "display": true - }, - { - "name": "Study Design", - "color": "rgba(153, 40, 107, 1)", - "display": true - }, - { - "name": "HEAL Specific", - "color": "rgba(161, 36, 46, 1)", - "display": true - }, - { - "name": "Other", - "color": "rgba(207, 54, 38, 1)", - "display": true - }, - { - "name": "Commons", - "displayName": "Data Resources", - "color": "rgba(21, 6, 76, 1)" - } + "categorical9Colors": [ + "#c0143c", + "#616265", + "#59CD90", + "#F2DC5D", + "#40476D", + "#FFA630", + "#AE8799", + "#1A535C", + "#462255" + ], + "categorical2Colors": [ + "#616265", + "#c0143c" ] }, - "explorerConfig": [ - { - "tabTitle": "Data", + "requiredCerts": [], + "featureFlags": { + "explorer": true + }, + "dataExplorerConfig": { "charts": { "project_id": { "chartType": "count", "title": "Projects" + }, + "_subject_id": { + "chartType": "count", + "title": "Subjects" + }, + "annotated_sex": { + "chartType": "pie", + "title": "Annotated Sex" + }, + "race": { + "chartType": "bar", + "title": "Race" } }, "filters": { - "tabs": [ - { + "tabs": [{ "title": "Project", "fields": [ + "programs_name", "project_id", + "consent_codes", + "studies_submitter_id", + "geographic_site", "data_type", "data_format" ] + }, + { + "title": "Subject", + "searchFields": [ + "submitter_id" + ], + "fields": [ + "annotated_sex", + "race", + "ethnicity", + "hispanic_subgroup", + "subcohort", + "weight_baseline", + "bmi_baseline", + "height_baseline", + "current_smoker_baseline", + "ever_smoker_baseline" + ] + }, + { + "title": "Harmonized Variables", + "fields": [ + "total_cholesterol", + "hdl", + "ldl", + "triglycerides", + "fasting_lipids", + "lipid_lowering_medication", + "cac_score", + "cac_volume", + "carotid_plaque", + "carotid_stenosis", + "cimt_1", + "cimt_2", + "bp_diastolic", + "bp_systolic", + "antihypertensive_meds", + "vte_case_status", + "vte_followup_start_age", + "vte_prior_history", + "basophil_ncnc_bld", + "eosinophil_ncnc_bld", + "hematocrit_vfr_bld", + "hemoglobin_mcnc_bld", + "lymphocyte_ncnc_bld", + "mch_entmass_rbc", + "mchc_mcnc_rbc", + "mcv_entvol_rbc", + "monocyte_ncnc_bld", + "neutrophil_ncnc_bld", + "platelet_ncnc_bld", + "pmv_entvol_bld", + "rbc_ncnc_bld", + "rdw_ratio_rbc", + "wbc_ncnc_bld" + ] } ] }, @@ -593,7 +312,13 @@ portal: "enabled": true, "fields": [ "project_id", - "data_format" + "data_format", + "race", + "annotated_sex", + "ethnicity", + "bp_diastolic", + "hdl", + "ldl" ] }, "dropdowns": { @@ -601,8 +326,7 @@ portal: "title": "Download" } }, - "buttons": [ - { + "buttons": [{ "enabled": true, "type": "export", "title": "Export All to Terra", @@ -614,6 +338,13 @@ portal: "title": "Export All to Seven Bridges", "rightIcon": "external-link" }, + { + "enabled": true, + "type": "export-to-pfb", + "title": "Export to PFB", + "leftIcon": "datafile", + "rightIcon": "download" + }, { "enabled": true, "type": "export-to-workspace", @@ -622,16 +353,170 @@ portal: "rightIcon": "download" } ], - "loginForDownload": true, + "arrangerConfig": { + "projectId": "search", + "graphqlField": "subject", + "index": "", + "manifestMapping": { + "resourceIndexType": "file", + "resourceIdField": "object_id", + "referenceIdFieldInResourceIndex": "_subject_id", + "referenceIdFieldInDataIndex": "_subject_id" + }, + "nodeCountField": "_subject_id" + }, "guppyConfig": { "dataType": "subject", "nodeCountTitle": "Subjects", - "fieldMapping": [ - { "field": "disease_id", "name": "Disease ID" }, - { "field": "age_of_onset", "name": "Age of Onset" }, - { "field": "project_dbgap_accession_number", "name": "Project dbGaP Accession Number" }, - { "field": "project_dbgap_consent_text", "name":"Project dbGaP Consent Text"}, - { "field": "project_dbgap_phs", "name":"Project dbGaP Phs"} + "fileCountField": "file_count", + "fieldMapping": [{ + "field": "_subject_id", + "name": "Subject ID" + }, + { + "field": "consent_codes", + "name": "data use restriction" + }, + { + "field": "cac_score", + "name": "CAC Score" + }, + { + "field": "cac_volume", + "name": "CAC Volume" + }, + { + "field": "cimt_1", + "name": "CIMT 1" + }, + { + "field": "cimt_2", + "name": "CIMT 2" + }, + { + "field": "vte_case_status", + "name": "VTE Case Status" + }, + { + "field": "vte_followup_start_age", + "name": "VTE Followup Start Age" + }, + { + "field": "vte_prior_history", + "name": "VTE Prior History" + }, + { + "field": "bp_diastolic", + "name": "BP Diastolic" + }, + { + "field": "bp_systolic", + "name": "BP Systolic" + }, + { + "field": "basophil_ncnc_bld", + "name": "Basophils Number Concentration Bld" + }, + { + "field": "eosinophil_ncnc_bld", + "name": "Eosinophils Number Concentration Bld" + }, + { + "field": "hdl", + "name": "HDL" + }, + { + "field": "hematocrit_vfr_bld", + "name": "Hematocrit Fraction of Volume Bld" + }, + { + "field": "hemoglobin_mcnc_bld", + "name": "Hemoglobin Mass Concentration Bld" + }, + { + "field": "ldl", + "name": "LDL" + }, + { + "field": "lymphocyte_ncnc_bld", + "name": "Lymphocytes Number Concentration Bld" + }, + { + "field": "mch_entmass_rbc", + "name": "MCH Average Mass Rbc" + }, + { + "field": "mchc_mcnc_rbc", + "name": "MCHC Mass Concentration Rbc" + }, + { + "field": "mcv_entvol_rbc", + "name": "MCV Average Volume Rbc" + }, + { + "field": "monocyte_ncnc_bld", + "name": "Monocytes Number Concentration Bld" + }, + { + "field": "neutrophil_ncnc_bld", + "name": "Neutrophils Number Concentration Bld" + }, + { + "field": "platelet_ncnc_bld", + "name": "Platelets Number Concentration Bld" + }, + { + "field": "pmv_entvol_bld", + "name": "PMV Mean Volume Bld" + }, + { + "field": "rbc_ncnc_bld", + "name": "Red Blood Cells Number Concentration Bld" + }, + { + "field": "rdw_ratio_rbc", + "name": "Red Cell Distribution Width Ratio Rbc" + }, + { + "field": "wbc_ncnc_bld", + "name": "White Blood Cells Number Concentration Bld" + }, + { + "field": "bmi_baseline", + "name": "BMI Baseline" + }, + { + "field": "current_smoker_baseline", + "name": "Current Smoker Baseline" + }, + { + "field": "ever_smoker_baseline", + "name": "Ever Smoker Baseline" + }, + { + "field": "geographic_site", + "name": "Geographic Site" + }, + { + "field": "height_baseline", + "name": "Height Baseline" + }, + { + "field": "total_cholesterol", + "name": "Total Cholesterol" + }, + { + "field": "triglycerides", + "name": "Triglycerides" + }, + { + "field": "studies_submitter_id", + "name": "Study" + }, + { + "field": "programs_name", + "name": "Program" + } ], "manifestMapping": { "resourceIndexType": "file", @@ -652,8 +537,7 @@ portal: ], "sevenBridgesExportURL": "https://platform.sb.biodatacatalyst.nhlbi.nih.gov/import/windmill" }, - { - "tabTitle": "File", + "fileExplorerConfig": { "charts": { "data_type": { "chartType": "stackedBar", @@ -665,20 +549,21 @@ portal: } }, "filters": { - "tabs": [ - { - "title": "File", - "fields": [ - "project_id", - "data_category", - "data_type", - "data_format" - ], - "searchFields": [ - "file_name" - ] - } - ] + "tabs": [{ + "title": "File", + "fields": [ + "programs_name", + "project_id", + "data_category", + "data_type", + "data_format", + "callset", + "bucket_path" + ], + "searchFields": [ + "file_name" + ] + }] }, "table": { "enabled": true, @@ -686,9 +571,10 @@ portal: "project_id", "file_name", "file_size", - "object_id" ] + "object_id", + "callset" + ] }, - "loginForDownload": true, "guppyConfig": { "dataType": "file", "fieldMapping": [ @@ -697,10 +583,10 @@ portal: "name": "GUID" }, { - "field": "programs_name", - "name": "Program" - } - ], + "field": "programs_name", + "name": "Program" + } + ], "nodeCountTitle": "Files", "manifestMapping": { "resourceIndexType": "subject", @@ -714,8 +600,7 @@ portal: "accessibleValidationField": "project_id", "downloadAccessor": "object_id" }, - "buttons": [ - { + "buttons": [{ "enabled": true, "type": "file-manifest", "title": "Download Manifest", @@ -761,128 +646,9 @@ portal: }, "dropdowns": {} }, - { - "tabTitle": "Imaging Studies", - "charts": {}, - "filters": { - "tabs": [ - { - "title": "Study Properties", - "searchFields": [ - "study_uid" - ], - "fields": [ - "study_modality", - "study_description", - "age_at_imaging", - "study_year", - "study_year_shifted", - "body_part_examined", - "days_from_study_to_pos_covid_test", - "days_from_study_to_neg_covid_test", - "project_id" - ] - } - ] - }, - "table": { - "enabled": true, - "fields": [ - "submitter_id", - "study_uid", - "case_ids", - "object_id", - "project_id" - ], - "dicomViewerId": "submitter_id", - "dicomViewerUrl": "ohif-viewer", - "linkFields": [ - "submitter_id" - ] - }, - "dropdowns": { - "download-table": { - "title": "Download Table" - } - }, - "buttons": [ - { - "enabled": true, - "type": "data-json", - "title": "JSON", - "dropdownId": "download-table", - "leftIcon": "datafile", - "rightIcon": "download", - "fileName": "Imaging_Studies_table.json" - }, - { - "enabled": true, - "type": "data-tsv", - "title": "TSV", - "dropdownId": "download-table", - "leftIcon": "datafile", - "rightIcon": "download", - "fileName": "Imaging_Studies_table.tsv" - }, - { - "enabled": true, - "type": "data-csv", - "title": "CSV", - "dropdownId": "download-table", - "leftIcon": "datafile", - "rightIcon": "download", - "fileName": "Imaging_Studies_table.csv" - }, - { - "enabled": true, - "type": "manifest", - "title": "Download File Manifest for Imaging Studies", - "leftIcon": "datafile", - "rightIcon": "download", - "fileName": "imaging_study_manifest.json" - }, - { - "enabled": false, - "type": "export-to-workspace", - "title": "Export to Workspace", - "leftIcon": "datafile", - "rightIcon": "download" - } - ], - "loginForDownload": true, - "guppyConfig": { - "dataType": "imaging_study", - "nodeCountTitle": "Imaging Studies", - "fieldMapping": [ - { - "field": "project_id", - "name": "Project ID" - }, - { - "field": "study_uid", - "name": "Study UID" - }, - { - "field": "submitter_id", - "name": "Browse in DICOM viewer" - }, - { - "field": "case_ids", - "name": "Patient ID" - } - ], - "manifestMapping": { - "resourceIndexType": "data_file", - "resourceIdField": "object_id", - "referenceIdFieldInResourceIndex": "_imaging_study_id", - "referenceIdFieldInDataIndex": "_imaging_study_id", - "useFilterForCounts": false - }, - "accessibleFieldCheckList": [ - "project_id" - ], - "accessibleValidationField": "project_id" - } + "terraExportWarning": { + "subjectThreshold": 120000, + "title": "Warning: Data Transfer May Fail in Terra", + "message": "Warning: Your selected cohort contains more subjects than are currently supported for import by Terra. It is recommended that you slice your data into segments of no more than 120,000 subjects and then export each segment separately. Would you like to continue anyway?" } - ] } From b2734e4b4c2f936febf819ecb6885791a5d0d935 Mon Sep 17 00:00:00 2001 From: krishna Date: Wed, 16 Jul 2025 10:52:50 -0500 Subject: [PATCH 13/14] Revert portal changes --- ci/perf/values/portal.yaml | 1018 ++++++++++++++++++++++-------------- 1 file changed, 626 insertions(+), 392 deletions(-) diff --git a/ci/perf/values/portal.yaml b/ci/perf/values/portal.yaml index dfd163870..c65dd1ffb 100644 --- a/ci/perf/values/portal.yaml +++ b/ci/perf/values/portal.yaml @@ -16,88 +16,102 @@ portal: gitops: json: | { - "gaTrackingId": "UA-119127212-3", + "gaTrackingId": "UA-119127212-2", + "argoTemplate": "gwas-template", "graphql": { - "boardCounts": [{ - "graphql": "_subject_count", - "name": "Subject", - "plural": "Subjects" - }, - { - "graphql": "_study_count", - "name": "Study", - "plural": "Studies" - }, - { - "graphql": "_aliquot_count", - "name": "Aliquot", - "plural": "Aliquots" - } + "boardCounts": [ ], - "chartCounts": [{ - "graphql": "_subject_count", - "name": "Subject" - }, - { - "graphql": "_study_count", - "name": "Study" - } + "chartCounts": [ ], "projectDetails": "boardCounts" }, + "useArboristUI": true, + "showArboristAuthzOnProfile": true, + "showFenceAuthzOnProfile": false, + "componentToResourceMapping": { + "Workspace": { + "resource": "/workspace", + "method": "access", + "service": "jupyterhub" + }, + "Query": { + "resource": "/query_page", + "method": "access", + "service": "query_page" + }, + "Send Queries": { + "resource": "/query_page", + "method": "access", + "service": "query_page" + }, + "Data Dictionary": { + "resource": "/dictionary_page", + "method": "access", + "service": "dictionary_page" + }, + "Dictionary": { + "resource": "/dictionary_page", + "method": "access", + "service": "dictionary_page" + } + }, "components": { - "appName": "NHLBI BioData Catalyst", + "appName": "BloodPAC Metadata Submission Portal", "index": { "introduction": { - "heading": "", - "text": "NHLBI BioData Catalyst supports the management, analysis and sharing of human disease data for the research community and aims to advance basic understanding of the genetic basis of complex traits and accelerate discovery and development of therapies, diagnostic tests, and other technologies for diseases like cancer. The data commons supports cross-project analyses by harmonizing data from different projects through the collaborative development of a data dictionary, providing an API for data queries and download, and providing a cloud-based analysis workspace with rich tools and resources.", + "heading": "BloodPAC Data Commons", + "text": "The Blood Profiling Atlas in Cancer (BloodPAC) supports the management, analysis and sharing of liquid biopsy data for the oncology research community and aims to accelerate discovery and development of therapies, diagnostic tests, and other technologies for cancer treatment and prevention. The data commons supports cross-project analyses by harmonizing data from different projects through the collaborative development of a data dictionary, providing an API for data queries and download, and providing a cloud-based analysis workspace with rich tools and resources.", "link": "/submission" }, + "homepageChartNodes": [{ + "node": "subject", + "name": "Subjects" + } + ], "buttons": [{ - "name": "Define Data Field", - "icon": "stage-planning", - "body": "NHLBI BioData Catalyst define the data in a general way. Please study the dictionary before you start browsing.", + "name": "Data Dictionary", + "icon": "data-field-define", + "body": "The BloodPAC Data Commons employs a data model to power queries and data submissions. Study the dictionary to send GraphQL queries or format your data submissions.", "link": "/DD", "label": "Learn more" }, + { + "name": "Submit Data", + "icon": "data-submit", + "body": "Browse the list of submitted data projects, or find your project and submit data.", + "link": "/submission", + "label": "Submit data" + }, { "name": "Explore Data", - "icon": "stage-explore", - "body": "The Exploration Page gives you insights and a clear overview under selected factors.", + "icon": "data-files", + "body": "The Exploration app enables faceted search for cohort building, getting file download manifests, and browsing data across projects.", "link": "/explorer", - "label": "Explore data" + "label": "Exploration" }, { - "name": "Access Data", - "icon": "stage-access", - "body": "Use our selected tool to filter out the data you need.", + "name": "Send Queries", + "icon": "data-access", + "body": "Build and send your own custom GraphQL queries to the database to pinpoint specific data.", "link": "/query", - "label": "Query data" + "label": "Send Queries" }, { - "name": "Analyze Data", - "icon": "stage-analyze", - "body": "Analyze your selected cases using Jupyter Notebooks in our secure cloud environment", - "link": "/workspace", - "label": "Run analysis" - } - ], - "homepageChartNodes": [{ - "node": "subject", - "name": "Subjects" - }, - { - "node": "study", - "name": "Studies" + "icon": "analysis", + "link": "/analysis", + "color": "#a2a2a2", + "name": "Apps" } ] }, "navigation": { - "items": [{ - "icon": "dictionary", - "link": "/DD", - "color": "#a2a2a2", - "name": "Dictionary" + "title": "BloodPAC Data Commons", + "items": [ + { + "icon": "query", + "link": "/discovery", + "name": "Discovery", + "tooltip": "Use free-text search and tags to rapidly find relevant studies, discover new datasets across multiple resources, and easily export selected data files to the analysis workspace." }, { "icon": "exploration", @@ -107,9 +121,9 @@ portal: }, { "icon": "query", - "link": "/query", + "link": "/study-viewer/clinical_trials", "color": "#a2a2a2", - "name": "Query" + "name": "Study Viewer" }, { "icon": "workspace", @@ -117,6 +131,24 @@ portal: "color": "#a2a2a2", "name": "Workspace" }, + { + "icon": "dictionary", + "link": "/DD", + "color": "#a2a2a2", + "name": "Dictionary" + }, + { + "icon": "analysis", + "link": "/analysis", + "color": "#a2a2a2", + "name": "Apps" + }, + { + "icon": "query", + "link": "/query", + "color": "#a2a2a2", + "name": "Query" + }, { "icon": "profile", "link": "/identity", @@ -127,184 +159,433 @@ portal: }, "topBar": { "items": [{ + "link": "https://bloodpac.org/", + "name": "About" + }, + { "icon": "upload", "link": "/submission", "name": "Submit Data" }, { - "link": "https://bdcatalyst.gitbook.io/biodata-catalyst-documentation/explore_data/gen3-discovering-data", + "link": "https://www.synapse.org/#!Synapse:syn8011461/wiki/411591", "name": "Documentation" + }, + { + "link": "bpa-support@gen3.org", + "name": "Email Support" } - ] + ], + "useProfileDropdown": false }, "login": { - "title": "NHLBI BioData Catalyst", + "title": "BloodPAC Data Commons", "subTitle": "Explore, Analyze, and Share Data", - "text": "This website supports the management, analysis and sharing of human disease data for the research community and aims to advance basic understanding of the genetic basis of complex traits and accelerate discovery and development of therapies, diagnostic tests, and other technologies for diseases like cancer.", + "text": "This website combines liquid biopsy data from academic, government, and industry partners and aims to accelerate discovery and development of therapies, diagnostic tests, and other technologies for the treatment and prevention of cancer.", "contact": "If you have any questions about access or the registration process, please contact ", - "contact_link": { - "href": "https://biodatacatalyst.nhlbi.nih.gov/contact" + "email": "bpa-support@gen3.org" + }, + "certs": { + "security_quiz": { + "hasCorrectAnswers": true, + "title": "BloodPAC User agreement", + "description": "", + "questions": [{ + "name": "", + "question": "NOTICE: To protect the system from unauthorized use and to ensure that the system is functioning, activities on this system are monitored and recorded and subject to audit. Use of this system is express consent to such monitoring and recording. Any unauthorized access or use of this system is prohibited. You may only use this system if you have been designated as an Authenticated Individual by your company or institution. You may not use the system in any way that is inconsistent with the agreements between CCSR and your company or institution. Please pay particular attention to the restrictions on controlled access data and transfer it in an approved secure manner.", + "options": ["I agree"], + "answer": 0, + "hint": "Click on I agree" + }] + } + } + }, + "featureFlags": { + "explorer": true, + "analysis": true, + "explorerPublic": true, + "discovery": true, + "discoveryUseAggMDS": true, + "studyRegistration": true + }, + "analysisTools": [ + { + "title": "OHDSI Atlas", + "description": "Use this App for cohort creation. These will be automatically populated in the Gen3 GWAS App", + "applicationUrl": "https://atlas.qa-mickey.planx-pla.net/atlas/", + "image": "/custom/sponsors/gitops-sponsors/atlas-logo.png" + }, + { + "appId": "GWASUIApp", + "title": "Gen3 GWAS", + "description": "Use this App to perform high throughput GWAS on Million Veteran Program (MVP) data, using the University of Washington Genesis pipeline", + "image": "/src/img/analysis-icons/gwas.svg", + "needsTeamProject": true + }, + { + "appId": "GWASResults", + "title": "GWAS Results", + "description": "Use this App to view status & results of submitted workflows", + "image": "/src/img/analysis-icons/gwasResults.svg", + "needsTeamProject": true + } + ], + "discoveryConfig": { + "features": { + "advSearchFilters": { + "enabled": false, + "field": "advSearchFilters", + "filters": [ + { "key": "Research Focus Area" }, + { "key": "Administering IC(s)" }, + { "key": "Simulated vs. Real Data" }, + { "key": "Year Grant Awarded" }, + { "key": "Study Description" } + ] }, - "image": "stage-icons/stage-gene" + "exportToWorkspace": { + "enabled": true, + "manifestFieldName": "__manifest", + "downloadManifestButtonText": "Download", + "enableDownloadManifest": false, + "downloadZipButtonText": "Download Zip", + "enableDownloadZip": true + }, + "pageTitle": { + "enabled": false + }, + "search": { + "searchBar": { + "enabled": true, + "searchableTextFields": [ + "summary", + "location", + "study_name", + "institutions", + "investigators", + "project_title", + "project_number", + "dataset_1_title", + "dataset_2_title", + "dataset_3_title", + "dataset_4_title", + "administering_ic", + "research_program", + "research_question", + "dataset_description", + "dataset_1_description", + "dataset_2_description", + "dataset_3_description", + "dataset_4_description", + "data_dictionary" + ] + } + }, + "authorization": { + "enabled": true, + "supportedValues": { + "accessible": { + "enabled": true, + "menuText": "Accessible" + }, + "unaccessible": { + "enabled": true, + "menuText": "Not Accessible" + }, + "pending": { + "enabled": false + }, + "notAvailable": { + "enabled": false + } + } + } + }, + "aggregations": [ + { + "name": "Studies", + "field": "project_number", + "type": "count" + } + ], + "tagSelector": { + "title": "Tags by category", + "showTagCategoryNames": true }, - "footerLogos": [{ - "src": "/src/img/gen3.png", - "href": "https://ctds.uchicago.edu/gen3", - "alt": "Gen3 Data Commons" + "studies": [], + "studyColumns": [ + { + "name": "Project Title", + "field": "project_title", + "errorIfNotAvailable": false, + "valueIfNotAvailable": "n/a" }, { - "src": "/src/img/createdby.png", - "href": "https://ctds.uchicago.edu/", - "alt": "Center for Translational Data Science at the University of Chicago" + "name": "Research Focus Area", + "field": "research_focus_area", + "errorIfNotAvailable": false, + "valueIfNotAvailable": "n/a" }, { - "src": "/src/img/sponsors/nhlbi.png", - "href": "https://www.nhlbi.nih.gov", - "alt": "NHLBI" + "name": "Investigator(s)", + "field": "investigators", + "errorIfNotAvailable": false, + "valueIfNotAvailable": "n/a" } ], - "footer": { - "links": [{ - "href": "https://biodatacatalyst.nhlbi.nih.gov/privacy/", - "text": "Privacy Policy" - }, - { - "href": "https://osp.od.nih.gov/scientific-sharing/policies/", - "text": "Data Sharing Policy" - }, + "studyPreviewField": { + "name": "Summary", + "field": "summary", + "contentType": "string", + "includeName": false, + "includeIfNotAvailable": true, + "valueIfNotAvailable": "No summary has been provided for this study." + }, + "studyPageFields": { + "header": { + "field": "project_title" + }, + "fieldsToShow": [ { - "text": "Freedom of Information Act (FOIA)", - "href": "https://www.nhlbi.nih.gov/about/foia-fee-for-service-office" + "groupName": "HEAL Research Initiatives", + "groupWidth": "full", + "includeName": false, + "fields": [ + { + "name": "Project Number", + "field": "project_number" + }, + { + "name": "Research Focus Area", + "field": "research_focus_area" + }, + { + "name": "Research Program", + "field": "research_program" + }, + { + "name": "Research Question", + "field": "research_question" + } + ] }, { - "text": "Accessibility Information", - "href": "https://biodatacatalyst.nhlbi.nih.gov/accessibility" + "groupName": "Host Institution", + "groupWidth": "half", + "includeName": false, + "fields": [ + { + "name": "Institution(s)", + "field": "institutions" + }, + { + "name": "Location", + "field": "location" + }, + { + "name": "Investigator(s)", + "field": "investigators" + }, + { + "name": "Administering IC(s)", + "field": "administering_ic" + }, + { + "name": "Year Grant Awarded", + "field": "year_awarded" + } + ] }, { - "text": "U.S. Department of Health & Human Services", - "href": "https://www.hhs.gov" + "groupName": "Clinical Trial", + "groupWidth": "half", + "includeName": false, + "fields": [ + { + "name": "Study Type", + "field": "study_type" + }, + { + "name": "Study Name", + "field": "study_name" + }, + { + "name": "Study Description", + "field": "study_description" + }, + { + "name": "Number of Subjects", + "field": "subjects" + }, + { + "name": "Number of Sites", + "field": "sites" + }, + { + "name": "Clinical Trial Link", + "field": "clinical_trial_link" + }, + { + "name": "Protocol Name", + "field": "protocol_name" + }, + { + "name": "Dataset Description", + "field": "dataset_description" + }, + { + "name": "Dataset Category", + "field": "dataset_category" + } + ] }, { - "text": "National Institutes of Health", - "href": "https://www.nih.gov" + "groupName": "Tags", + "includeName": false, + "groupWidth": "full", + "fields": [ + { + "name": "Tags", + "field": "tags", + "contentType": "tags", + "includeIfNotAvailable": false + } + ] }, { - "text": "National Heart, Lung, and Blood Institute", - "href": "https://www.nhlbi.nih.gov" + "groupWidth": "full", + "fields": [ + { + "name": "Summary", + "field": "summary", + "contentType": "paragraphs", + "includeName": true, + "includeIfNotAvailable": true, + "valueIfNotAvailable": "No summary has been provided for this study." + } + ] }, { - "text": "USA.gov", - "href": "https://www.usa.gov" + "groupName": "Datasets", + "groupWidth": "full", + "includeName": false, + "fields": [ + { + "name": "Dataset 1 - Title", + "field": "dataset_1_title" + }, + { + "name": "Dataset 1 - Description", + "field": "dataset_1_description" + }, + { + "name": "Dataset 1 - Type", + "field": "dataset_1_type" + }, + { + "name": "Dataset 2 - Title", + "field": "dataset_2_title" + }, + { + "name": "Dataset 2 - Description", + "field": "dataset_2_description" + }, + { + "name": "Dataset 2 - Type", + "field": "dataset_2_type" + }, + { + "name": "Dataset 3 - Title", + "field": "dataset_3_title" + }, + { + "name": "Dataset 3 - Description", + "field": "dataset_3_description" + }, + { + "name": "Dataset 3 - Type", + "field": "dataset_3_type" + }, + { + "name": "Dataset 4 - Title", + "field": "dataset_4_title" + }, + { + "name": "Dataset 4 - Description", + "field": "dataset_4_description" + }, + { + "name": "Dataset 4 - Type", + "field": "dataset_4_type" + }, + { + "name": "Dataset 5 - Title", + "field": "dataset_5_title" + }, + { + "name": "Dataset 5 - Description", + "field": "dataset_5_description" + }, + { + "name": "Dataset 5 - Type", + "field": "dataset_5_type" + } + ] } ] }, - "categorical9Colors": [ - "#c0143c", - "#616265", - "#59CD90", - "#F2DC5D", - "#40476D", - "#FFA630", - "#AE8799", - "#1A535C", - "#462255" - ], - "categorical2Colors": [ - "#616265", - "#c0143c" + "minimalFieldMapping": { + "authzField": "authz", + "tagsListFieldName": "tags", + "dataAvailabilityField": "data_availability", + "uid": "project_number" + }, + "tagCategories": [ + { + "name": "Demographic Info", + "color": "rgba(64, 34, 100, 1)", + "display": true + }, + { + "name": "Study Design", + "color": "rgba(153, 40, 107, 1)", + "display": true + }, + { + "name": "HEAL Specific", + "color": "rgba(161, 36, 46, 1)", + "display": true + }, + { + "name": "Other", + "color": "rgba(207, 54, 38, 1)", + "display": true + }, + { + "name": "Commons", + "displayName": "Data Resources", + "color": "rgba(21, 6, 76, 1)" + } ] }, - "requiredCerts": [], - "featureFlags": { - "explorer": true - }, - "dataExplorerConfig": { + "explorerConfig": [ + { + "tabTitle": "Data", "charts": { "project_id": { "chartType": "count", "title": "Projects" - }, - "_subject_id": { - "chartType": "count", - "title": "Subjects" - }, - "annotated_sex": { - "chartType": "pie", - "title": "Annotated Sex" - }, - "race": { - "chartType": "bar", - "title": "Race" } }, "filters": { - "tabs": [{ + "tabs": [ + { "title": "Project", "fields": [ - "programs_name", "project_id", - "consent_codes", - "studies_submitter_id", - "geographic_site", "data_type", "data_format" ] - }, - { - "title": "Subject", - "searchFields": [ - "submitter_id" - ], - "fields": [ - "annotated_sex", - "race", - "ethnicity", - "hispanic_subgroup", - "subcohort", - "weight_baseline", - "bmi_baseline", - "height_baseline", - "current_smoker_baseline", - "ever_smoker_baseline" - ] - }, - { - "title": "Harmonized Variables", - "fields": [ - "total_cholesterol", - "hdl", - "ldl", - "triglycerides", - "fasting_lipids", - "lipid_lowering_medication", - "cac_score", - "cac_volume", - "carotid_plaque", - "carotid_stenosis", - "cimt_1", - "cimt_2", - "bp_diastolic", - "bp_systolic", - "antihypertensive_meds", - "vte_case_status", - "vte_followup_start_age", - "vte_prior_history", - "basophil_ncnc_bld", - "eosinophil_ncnc_bld", - "hematocrit_vfr_bld", - "hemoglobin_mcnc_bld", - "lymphocyte_ncnc_bld", - "mch_entmass_rbc", - "mchc_mcnc_rbc", - "mcv_entvol_rbc", - "monocyte_ncnc_bld", - "neutrophil_ncnc_bld", - "platelet_ncnc_bld", - "pmv_entvol_bld", - "rbc_ncnc_bld", - "rdw_ratio_rbc", - "wbc_ncnc_bld" - ] } ] }, @@ -312,13 +593,7 @@ portal: "enabled": true, "fields": [ "project_id", - "data_format", - "race", - "annotated_sex", - "ethnicity", - "bp_diastolic", - "hdl", - "ldl" + "data_format" ] }, "dropdowns": { @@ -326,7 +601,8 @@ portal: "title": "Download" } }, - "buttons": [{ + "buttons": [ + { "enabled": true, "type": "export", "title": "Export All to Terra", @@ -338,13 +614,6 @@ portal: "title": "Export All to Seven Bridges", "rightIcon": "external-link" }, - { - "enabled": true, - "type": "export-to-pfb", - "title": "Export to PFB", - "leftIcon": "datafile", - "rightIcon": "download" - }, { "enabled": true, "type": "export-to-workspace", @@ -353,170 +622,16 @@ portal: "rightIcon": "download" } ], - "arrangerConfig": { - "projectId": "search", - "graphqlField": "subject", - "index": "", - "manifestMapping": { - "resourceIndexType": "file", - "resourceIdField": "object_id", - "referenceIdFieldInResourceIndex": "_subject_id", - "referenceIdFieldInDataIndex": "_subject_id" - }, - "nodeCountField": "_subject_id" - }, + "loginForDownload": true, "guppyConfig": { "dataType": "subject", "nodeCountTitle": "Subjects", - "fileCountField": "file_count", - "fieldMapping": [{ - "field": "_subject_id", - "name": "Subject ID" - }, - { - "field": "consent_codes", - "name": "data use restriction" - }, - { - "field": "cac_score", - "name": "CAC Score" - }, - { - "field": "cac_volume", - "name": "CAC Volume" - }, - { - "field": "cimt_1", - "name": "CIMT 1" - }, - { - "field": "cimt_2", - "name": "CIMT 2" - }, - { - "field": "vte_case_status", - "name": "VTE Case Status" - }, - { - "field": "vte_followup_start_age", - "name": "VTE Followup Start Age" - }, - { - "field": "vte_prior_history", - "name": "VTE Prior History" - }, - { - "field": "bp_diastolic", - "name": "BP Diastolic" - }, - { - "field": "bp_systolic", - "name": "BP Systolic" - }, - { - "field": "basophil_ncnc_bld", - "name": "Basophils Number Concentration Bld" - }, - { - "field": "eosinophil_ncnc_bld", - "name": "Eosinophils Number Concentration Bld" - }, - { - "field": "hdl", - "name": "HDL" - }, - { - "field": "hematocrit_vfr_bld", - "name": "Hematocrit Fraction of Volume Bld" - }, - { - "field": "hemoglobin_mcnc_bld", - "name": "Hemoglobin Mass Concentration Bld" - }, - { - "field": "ldl", - "name": "LDL" - }, - { - "field": "lymphocyte_ncnc_bld", - "name": "Lymphocytes Number Concentration Bld" - }, - { - "field": "mch_entmass_rbc", - "name": "MCH Average Mass Rbc" - }, - { - "field": "mchc_mcnc_rbc", - "name": "MCHC Mass Concentration Rbc" - }, - { - "field": "mcv_entvol_rbc", - "name": "MCV Average Volume Rbc" - }, - { - "field": "monocyte_ncnc_bld", - "name": "Monocytes Number Concentration Bld" - }, - { - "field": "neutrophil_ncnc_bld", - "name": "Neutrophils Number Concentration Bld" - }, - { - "field": "platelet_ncnc_bld", - "name": "Platelets Number Concentration Bld" - }, - { - "field": "pmv_entvol_bld", - "name": "PMV Mean Volume Bld" - }, - { - "field": "rbc_ncnc_bld", - "name": "Red Blood Cells Number Concentration Bld" - }, - { - "field": "rdw_ratio_rbc", - "name": "Red Cell Distribution Width Ratio Rbc" - }, - { - "field": "wbc_ncnc_bld", - "name": "White Blood Cells Number Concentration Bld" - }, - { - "field": "bmi_baseline", - "name": "BMI Baseline" - }, - { - "field": "current_smoker_baseline", - "name": "Current Smoker Baseline" - }, - { - "field": "ever_smoker_baseline", - "name": "Ever Smoker Baseline" - }, - { - "field": "geographic_site", - "name": "Geographic Site" - }, - { - "field": "height_baseline", - "name": "Height Baseline" - }, - { - "field": "total_cholesterol", - "name": "Total Cholesterol" - }, - { - "field": "triglycerides", - "name": "Triglycerides" - }, - { - "field": "studies_submitter_id", - "name": "Study" - }, - { - "field": "programs_name", - "name": "Program" - } + "fieldMapping": [ + { "field": "disease_id", "name": "Disease ID" }, + { "field": "age_of_onset", "name": "Age of Onset" }, + { "field": "project_dbgap_accession_number", "name": "Project dbGaP Accession Number" }, + { "field": "project_dbgap_consent_text", "name":"Project dbGaP Consent Text"}, + { "field": "project_dbgap_phs", "name":"Project dbGaP Phs"} ], "manifestMapping": { "resourceIndexType": "file", @@ -537,7 +652,8 @@ portal: ], "sevenBridgesExportURL": "https://platform.sb.biodatacatalyst.nhlbi.nih.gov/import/windmill" }, - "fileExplorerConfig": { + { + "tabTitle": "File", "charts": { "data_type": { "chartType": "stackedBar", @@ -549,21 +665,20 @@ portal: } }, "filters": { - "tabs": [{ - "title": "File", - "fields": [ - "programs_name", - "project_id", - "data_category", - "data_type", - "data_format", - "callset", - "bucket_path" - ], - "searchFields": [ - "file_name" - ] - }] + "tabs": [ + { + "title": "File", + "fields": [ + "project_id", + "data_category", + "data_type", + "data_format" + ], + "searchFields": [ + "file_name" + ] + } + ] }, "table": { "enabled": true, @@ -571,10 +686,9 @@ portal: "project_id", "file_name", "file_size", - "object_id", - "callset" - ] + "object_id" ] }, + "loginForDownload": true, "guppyConfig": { "dataType": "file", "fieldMapping": [ @@ -583,10 +697,10 @@ portal: "name": "GUID" }, { - "field": "programs_name", - "name": "Program" - } - ], + "field": "programs_name", + "name": "Program" + } + ], "nodeCountTitle": "Files", "manifestMapping": { "resourceIndexType": "subject", @@ -600,7 +714,8 @@ portal: "accessibleValidationField": "project_id", "downloadAccessor": "object_id" }, - "buttons": [{ + "buttons": [ + { "enabled": true, "type": "file-manifest", "title": "Download Manifest", @@ -646,9 +761,128 @@ portal: }, "dropdowns": {} }, - "terraExportWarning": { - "subjectThreshold": 120000, - "title": "Warning: Data Transfer May Fail in Terra", - "message": "Warning: Your selected cohort contains more subjects than are currently supported for import by Terra. It is recommended that you slice your data into segments of no more than 120,000 subjects and then export each segment separately. Would you like to continue anyway?" + { + "tabTitle": "Imaging Studies", + "charts": {}, + "filters": { + "tabs": [ + { + "title": "Study Properties", + "searchFields": [ + "study_uid" + ], + "fields": [ + "study_modality", + "study_description", + "age_at_imaging", + "study_year", + "study_year_shifted", + "body_part_examined", + "days_from_study_to_pos_covid_test", + "days_from_study_to_neg_covid_test", + "project_id" + ] + } + ] + }, + "table": { + "enabled": true, + "fields": [ + "submitter_id", + "study_uid", + "case_ids", + "object_id", + "project_id" + ], + "dicomViewerId": "submitter_id", + "dicomViewerUrl": "ohif-viewer", + "linkFields": [ + "submitter_id" + ] + }, + "dropdowns": { + "download-table": { + "title": "Download Table" + } + }, + "buttons": [ + { + "enabled": true, + "type": "data-json", + "title": "JSON", + "dropdownId": "download-table", + "leftIcon": "datafile", + "rightIcon": "download", + "fileName": "Imaging_Studies_table.json" + }, + { + "enabled": true, + "type": "data-tsv", + "title": "TSV", + "dropdownId": "download-table", + "leftIcon": "datafile", + "rightIcon": "download", + "fileName": "Imaging_Studies_table.tsv" + }, + { + "enabled": true, + "type": "data-csv", + "title": "CSV", + "dropdownId": "download-table", + "leftIcon": "datafile", + "rightIcon": "download", + "fileName": "Imaging_Studies_table.csv" + }, + { + "enabled": true, + "type": "manifest", + "title": "Download File Manifest for Imaging Studies", + "leftIcon": "datafile", + "rightIcon": "download", + "fileName": "imaging_study_manifest.json" + }, + { + "enabled": false, + "type": "export-to-workspace", + "title": "Export to Workspace", + "leftIcon": "datafile", + "rightIcon": "download" + } + ], + "loginForDownload": true, + "guppyConfig": { + "dataType": "imaging_study", + "nodeCountTitle": "Imaging Studies", + "fieldMapping": [ + { + "field": "project_id", + "name": "Project ID" + }, + { + "field": "study_uid", + "name": "Study UID" + }, + { + "field": "submitter_id", + "name": "Browse in DICOM viewer" + }, + { + "field": "case_ids", + "name": "Patient ID" + } + ], + "manifestMapping": { + "resourceIndexType": "data_file", + "resourceIdField": "object_id", + "referenceIdFieldInResourceIndex": "_imaging_study_id", + "referenceIdFieldInDataIndex": "_imaging_study_id", + "useFilterForCounts": false + }, + "accessibleFieldCheckList": [ + "project_id" + ], + "accessibleValidationField": "project_id" + } } + ] } From be18bc44a39f10fda97f3842e2a86925dd2d74b1 Mon Sep 17 00:00:00 2001 From: krishna Date: Wed, 16 Jul 2025 12:53:56 -0500 Subject: [PATCH 14/14] Sync with current CI services --- ci/perf/values/fence.yaml | 19 ++ ci/perf/values/guppy.yaml | 2 + ci/perf/values/portal.yaml | 42 +++- ci/perf/values/values.yaml | 485 +++++++++++++++++++++++++++---------- 4 files changed, 410 insertions(+), 138 deletions(-) diff --git a/ci/perf/values/fence.yaml b/ci/perf/values/fence.yaml index a475d4924..9b1d150cb 100644 --- a/ci/perf/values/fence.yaml +++ b/ci/perf/values/fence.yaml @@ -397,6 +397,25 @@ fence: # url where authz microservice is running ARBORIST: http://arborist-service + # url where the audit-service is running + # This will be dynamically replaced by env_setup.sh script. + AUDIT_SERVICE: "http://audit-service" + ENABLE_AUDIT_LOGS: + presigned_url: true + login: true + # `PUSH_AUDIT_LOGS_CONFIG.type` is one of: [api, aws_sqs]. + # - if type == api: logs are created by hitting the log creation endpoint. + # - if type == aws_sqs: logs are pushed to an SQS and `aws_sqs_config` fields + # `sqs_url` and `region` are required. Field `aws_cred` is optional and it + # should be a key in section `AWS_CREDENTIALS`. + PUSH_AUDIT_LOGS_CONFIG: + type: aws_sqs + aws_sqs_config: + # This will be dynamically replaced by env_setup.sh script. + sqs_url: https://sqs.us-east-1.amazonaws.com/707767160287/audit-service-sqs + region: us-east-1 + aws_cred: 'cdistest' + # A Google Project identitifier representing the default project to bill to for # accessing Google Requester Pays buckets (for signed urls and/or temporary service account # credentials). If this is provided and the API call for diff --git a/ci/perf/values/guppy.yaml b/ci/perf/values/guppy.yaml index 3c5638d00..cb98a7890 100644 --- a/ci/perf/values/guppy.yaml +++ b/ci/perf/values/guppy.yaml @@ -1,5 +1,7 @@ guppy: replicaCount: 2 + autoscaling: + enabled: true # Lower cost resources: requests: diff --git a/ci/perf/values/portal.yaml b/ci/perf/values/portal.yaml index c65dd1ffb..51633cfe7 100644 --- a/ci/perf/values/portal.yaml +++ b/ci/perf/values/portal.yaml @@ -1,5 +1,7 @@ portal: # replicaCount: 2 + autoscaling: + enabled: true resources: limits: memory: 6Gi @@ -9,10 +11,6 @@ portal: image: repository: quay.io/cdis/data-portal tag: master - autoscaling: - enabled: true - minReplicas: 1 - maxReplicas: 2 gitops: json: | { @@ -568,6 +566,35 @@ portal: } ] }, + "studyViewerConfig": [ + { + "dataType": "subject", + "title": "Studies", + "titleField": "project_id", + "rowAccessor": "submitter_id", + "listItemConfig": { + "blockFields": ["project_id"], + "tableFields": ["data_type"] + }, + "fieldMapping": [ + { "field": "category", "name": "Study Type"}, + { "field": "project_id", "name": "Project_ID"}, + { "field": "data_type", "name": "Data_Type"}, + { "field": "condition", "name": "Condition"} + ], + "openMode": "close-all", + "buttons": [ + { + "type": "download" + }, + { + "type": "request_access", + "resourceDisplayNameField": "title", + "accessRequestedText": "DAR In Progress", + "accessRequestedTooltipText": "Your recently submitted DAR is being reviewed by NIAID" + }] + } + ], "explorerConfig": [ { "tabTitle": "Data", @@ -614,6 +641,13 @@ portal: "title": "Export All to Seven Bridges", "rightIcon": "external-link" }, + { + "enabled": true, + "type": "export-to-pfb", + "title": "Export to PFB", + "leftIcon": "datafile", + "rightIcon": "download" + }, { "enabled": true, "type": "export-to-workspace", diff --git a/ci/perf/values/values.yaml b/ci/perf/values/values.yaml index 09fa388b9..198ac58ad 100644 --- a/ci/perf/values/values.yaml +++ b/ci/perf/values/values.yaml @@ -40,141 +40,10 @@ postgresql: repository: cdis/postgresql tag: 14.5.0-debian-11-r35 -maxUnavailable: "" - -elasticsearch: - maxUnavailable: "" - volumeClaimTemplate: - resources: - requests: - storage: 1Gi - -arborist: - replicaCount: 2 - resources: - requests: - memory: "105Mi" - cpu: "15m" - enabled: true - image: - repository: "quay.io/cdis/arborist" - tag: "master" - autoscaling: - enabled: true - -audit: - enabled: false - -indexd: - replicaCount: 2 - resources: - requests: - memory: "105Mi" - cpu: "15m" - # This will be dynamically replaced by env_setup.sh script. - defaultPrefix: "" - enabled: true - image: - repository: "quay.io/cdis/indexd" - tag: "master" - externalSecrets: - createK8sServiceCredsSecret: true - secrets: - userdb: - # This will be dynamically replaced by env_setup.sh script. - ssj: - # This will be dynamically replaced by env_setup.sh script. - gateway: - autoscaling: - enabled: true - -peregrine: - replicaCount: 2 - resources: - requests: - memory: "105Mi" - cpu: "15m" - enabled: true - image: - repository: "quay.io/cdis/peregrine" - tag: "master" - pullPolicy: Always - autoscaling: - enabled: true - -revproxy: +hatchery: replicaCount: 2 - ingress: - enabled: true - annotations: - alb.ingress.kubernetes.io/scheme: internet-facing - alb.ingress.kubernetes.io/tags: Environment=devplanetv2 - alb.ingress.kubernetes.io/certificate-arn: 'arn:aws:acm:us-east-1:707767160287:certificate/f427a646-94fd-4b91-8bcd-80f5920c1e46' - alb.ingress.kubernetes.io/group.name: devplanetv2 - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' - alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' - alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-FIPS-2023-04 - className: alb - hosts: - # This will be dynamically replaced by env_setup.sh script. - - host: HOSTNAME - paths: - - path: / - pathType: Prefix - resources: - requests: - memory: "105Mi" - cpu: "15m" autoscaling: enabled: true - -sheepdog: - replicaCount: 2 - resources: - requests: - memory: "105Mi" - cpu: "15m" - enabled: true - authNamespace: '' - image: - repository: "quay.io/cdis/sheepdog" - tag: "master" - # This will be dynamically replaced by env_setup.sh script. - fenceUrl: https://hostname/user - -manifestservice: - replicaCount: 2 - image: - repository: quay.io/cdis/manifestservice - tag: "master" - env: - - name: REQUESTS_CA_BUNDLE - value: /etc/ssl/certs/ca-certificates.crt - - name: MANIFEST_SERVICE_CONFIG_PATH - value: /var/gen3/config/config.json - - name: GEN3_DEBUG - value: "False" - manifestserviceG3auto: - # This will be dynamically replaced by env_setup.sh script. - hostname: - bucketName: gen3-helm-manifest-bucket - prefix: 'test' - externalSecrets: - # -- (string) Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. - createK8sManifestServiceSecret: true - -wts: - replicaCount: 2 - enabled: true - image: - repository: "quay.io/cdis/workspace-token-service" - tag: "master" - externalSecrets: - createWtsOidcClientSecret: true - createK8sWtsSecret: true - -hatchery: - replicaCount: 2 image: tag: master hatchery: @@ -204,8 +73,34 @@ hatchery: user-uid: 1000 fs-gid: 100 +indexd: + replicaCount: 2 + autoscaling: + enabled: true + resources: + requests: + memory: "105Mi" + cpu: "15m" + # This will be dynamically replaced by env_setup.sh script. + defaultPrefix: "" + enabled: true + image: + repository: "quay.io/cdis/indexd" + tag: "master" + externalSecrets: + createK8sServiceCredsSecret: true + secrets: + userdb: + # This will be dynamically replaced by env_setup.sh script. + ssj: + # This will be dynamically replaced by env_setup.sh script. + gateway: + + metadata: replicaCount: 2 + autoscaling: + enabled: true image: repository: quay.io/cdis/metadata-service tag: 'master' @@ -228,11 +123,19 @@ metadata: }, "adapter_commons": {} } - autoscaling: - enabled: true + +maxUnavailable: "" +elasticsearch: + maxUnavailable: "" + volumeClaimTemplate: + resources: + requests: + storage: 1Gi sower: replicaCount: 2 + autoscaling: + enabled: true image: repository: quay.io/cdis/sower tag: 'master' @@ -304,3 +207,317 @@ sower: secret: secretName: pelicanservice-g3auto #pragma: allowlist secret restart_policy: Never + - name: pelican-export-files + action: export-files + container: + name: job-task + image: quay.io/cdis/pelican-export:master + pull_policy: Always + env: + - name: DICTIONARY_URL + valueFrom: + configMapKeyRef: + name: manifest-global + key: dictionary_url + - name: GEN3_HOSTNAME + valueFrom: + configMapKeyRef: + name: manifest-global + key: hostname + - name: ROOT_NODE + value: file + - name: EXTRA_NODES + value: "" + - name: DB_HOST + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: host + - name: DB_DATABASE + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: database + - name: DB_USER + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: username + - name: DB_PASS + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: password + - name: SHEEPDOG + valueFrom: + secretKeyRef: + name: indexd-service-creds + key: sheepdog + volumeMounts: + - name: pelican-creds-volume + readOnly: true + mountPath: "/pelican-creds.json" + subPath: config.json + cpu-limit: "0.5" + memory-limit: 1Gi + volumes: + - name: pelican-creds-volume + secret: + secretName: pelicanservice-g3auto #pragma: allowlist secret + restart_policy: Never + - name: batch-export + action: batch-export + activeDealineSeconds: 600 + serviceAccountName: sower-service-account + container: + name: job-task + image: quay.io/cdis/batch-export:master + pull_policy: Always + env: + - name: BUCKET + value: sower-jobs-bucket + - name: GEN3_HOSTNAME + valueFrom: + configMapKeyRef: + name: manifest-global + key: hostname + cpu-limit: "1" + memory-limit: 1Gi + restart_policy: Never + +ssjdispatcher: + replicaCount: 2 + autoscaling: + enabled: true + image: + repository: quay.io/cdis/ssjdispatcher + tag: 'master' + enabled: true + indexing: quay.io/cdis/indexs3client:master + ssjcreds: + # This will be dynamically replaced by env_setup.sh script. + sqsUrl: https://sqs.us-east-1.amazonaws.com/707767160287/gen3-helm-data-upload-bucket + # This will be dynamically replaced by env_setup.sh script. + jobPattern: "s3://gen3-helm-data-upload-bucket//*" + # This will be dynamically replaced by env_setup.sh script. + jobPassword: "replace_with_password" + # This will be dynamically replaced by env_setup.sh script. + metadataservicePassword: "replace_with_password" + serviceAccount: + create: true + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::707767160287:role/ssjdispatcher-role + name: "ssjdispatcher-service-account" + jobServiceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::707767160287:role/ssjdispatcher-role-job + externalSecrets: + createK8sSsjdispatcherSecret: true + # This will be dynamically replaced by env_setup.sh script. + gen3Namespace: default + +sheepdog: + replicaCount: 2 + autoscaling: + enabled: true + resources: + requests: + memory: "105Mi" + cpu: "15m" + enabled: true + authNamespace: '' + image: + repository: "quay.io/cdis/sheepdog" + tag: "master" + # This will be dynamically replaced by env_setup.sh script. + fenceUrl: https://hostname/user + +peregrine: + replicaCount: 2 + autoscaling: + enabled: true + resources: + requests: + memory: "105Mi" + cpu: "15m" + enabled: true + image: + repository: "quay.io/cdis/peregrine" + tag: "master" + pullPolicy: Always + +requestor: + replicaCount: 2 + autoscaling: + enabled: true + enabled: true + image: + repository: "quay.io/cdis/requestor" + tag: "master" + +audit: + replicaCount: 2 + autoscaling: + enabled: true + env: + - name: ARBORIST_URL + valueFrom: + configMapKeyRef: + name: manifest-global + key: arborist_url + optional: true + externalSecrets: + createK8sAuditSecret: true + image: + repository: quay.io/cdis/audit-service + tag: 'master' + server: + pull_from_queue: true + debug: true + sqs: + region: us-east-1 + # This will be dynamically replaced by env_setup.sh script. + url: https://sqs.us-east-1.amazonaws.com/707767160287/audit-service-sqs + enabled: true + +cedar: + enabled: true + autoscaling: + enabled: true + externalSecrets: + cedarG3auto: ci-cedar-g3auto + cedarAPIKey: ci-cedar-directory-id + cedarDirectoryId: ci-cedar-api-key + image: + repository: "quay.io/cdis/cedar-wrapper-service" + tag: "master" + +ambassador: + replicaCount: 2 + autoscaling: + enabled: true + enabled: true + image: + repository: "quay.io/datawire/ambassador" + tag: "1.4.2" + +arborist: + replicaCount: 2 + autoscaling: + enabled: true + resources: + requests: + memory: "105Mi" + cpu: "15m" + enabled: true + image: + repository: "quay.io/cdis/arborist" + tag: "master" + +revproxy: + replicaCount: 2 + autoscaling: + enabled: true + ingress: + enabled: true + annotations: + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/tags: Environment=devplanetv2 + alb.ingress.kubernetes.io/certificate-arn: 'arn:aws:acm:us-east-1:707767160287:certificate/f427a646-94fd-4b91-8bcd-80f5920c1e46' + alb.ingress.kubernetes.io/group.name: devplanetv2 + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' + alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' + alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-FIPS-2023-04 + className: alb + hosts: + # This will be dynamically replaced by env_setup.sh script. + - host: HOSTNAME + paths: + - path: / + pathType: Prefix + resources: + requests: + memory: "105Mi" + cpu: "15m" + + +wts: + replicaCount: 2 + autoscaling: + enabled: true + enabled: true + image: + repository: "quay.io/cdis/workspace-token-service" + tag: "master" + externalSecrets: + createWtsOidcClientSecret: true + createK8sWtsSecret: true + +manifestservice: + replicaCount: 2 + autoscaling: + enabled: true + image: + repository: quay.io/cdis/manifestservice + tag: "krishnaa05-patch-1" + env: + - name: REQUESTS_CA_BUNDLE + value: /etc/ssl/certs/ca-certificates.crt + - name: MANIFEST_SERVICE_CONFIG_PATH + value: /var/gen3/config/config.json + - name: GEN3_DEBUG + value: "False" + manifestserviceG3auto: + # This will be dynamically replaced by env_setup.sh script. + hostname: + bucketName: gen3-helm-manifest-bucket + prefix: 'test' + externalSecrets: + # -- (string) Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. + createK8sManifestServiceSecret: true + + +ohif-viewer: + replicaCount: 2 + autoscaling: + enabled: true + enabled: true + image: + repository: "quay.io/cdis/ohif-viewer" + tag: "gen3-v3.8.0" + +dicom-server: + replicaCount: 2 + autoscaling: + enabled: true + enabled: false + image: + repository: "quay.io/cdis/gen3-orthanc" + tag: "master" + +gen3-user-data-library: + replicaCount: 2 + autoscaling: + enabled: true + enabled: true + image: + repository: "quay.io/cdis/gen3-user-data-library" + tag: "main" + externalSecrets: + createK8sGen3UserDataLibrarySecret: true + +access-backend: + enabled: false + +orthanc: + replicaCount: 2 + autoscaling: + enabled: true + enabled: true + image: + repository: "quay.io/cdis/gen3-orthanc" + tag: "orthancteam-master-gen3-24.3.5" + # -- (map) External Secrets settings. + externalSecrets: + # -- (string) Will override the name of the aws secrets manager secret. Default is "orthanc-s3-g3auto" + orthancS3G3Auto: ci-orthanc-s3-g3auto