diff --git a/ci/perf/Chart.yaml b/ci/perf/Chart.yaml new file mode 100644 index 000000000..b6fa1679f --- /dev/null +++ b/ci/perf/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: perf +description: ci perf setup + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: "1.0" diff --git a/ci/perf/templates/app.yaml b/ci/perf/templates/app.yaml new file mode 100644 index 000000000..29cf34821 --- /dev/null +++ b/ci/perf/templates/app.yaml @@ -0,0 +1,33 @@ +# Note- Won't deploy via argocd in CI +# apiVersion: argoproj.io/v1alpha1 +# kind: Application +# metadata: +# name: gen3-ci +# namespace: argocd +# finalizers: +# - resources-finalizer.argocd.argoproj.io +# spec: +# project: default +# sources: +# - path: helm/gen3 +# repoURL: https://github.com/uc-cdis/gen3-helm +# targetRevision: master +# helm: +# releaseName: oadc +# valueFiles: +# - $values/ci/values/values.yaml +# - $values/ci/values/fence.yaml +# - $values/ci/values/portal.yaml +# - $values/ci/values/guppy.yaml +# - $values/ci/values/etl.yaml +# - repoURL: 'https://github.com/uc-cdis/gen3-gitops.git' +# targetRevision: master +# ref: values +# destination: +# server: "https://kubernetes.default.svc" +# namespace: <> +# syncPolicy: +# syncOptions: +# - CreateNamespace=true +# automated: +# selfHeal: true diff --git a/ci/perf/values.yaml b/ci/perf/values.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/ci/perf/values/etl.yaml b/ci/perf/values/etl.yaml new file mode 100644 index 000000000..34f49c683 --- /dev/null +++ b/ci/perf/values/etl.yaml @@ -0,0 +1,174 @@ + +etl: + enabled: true + image: + spark: + repository: "quay.io/cdis/gen3-spark" + tag: "master" + tube: + repository: "quay.io/cdis/tube" + tag: "2024.12" + etlMapping: + mappings: + - name: ci.planx-ci.io_subject + doc_type: subject + type: aggregator + root: subject + props: + - name: submitter_id + - name: project_id + - name: sex + - name: age_value + - name: ancestry + - name: disease_description + - name: phenotype_present + - name: phenotype_absent + - name: disease_id + - name: solve_state + - name: congenital_status + - name: age_of_onset + - name: phenotype_group + - name: anvil_project_id + # flatten_props: + aggregated_props: + - name: sample_provider + path: samples + src: sample_provider + fn: set + - name: tissue_affected_status + path: samples + src: tissue_affected_status + fn: set + - name: tissue_type + path: samples + src: tissue_type + fn: set + - name: sample_type + path: samples + src: sample_type + fn: set + - name: original_material_type + path: samples + src: original_material_type + fn: set + - name: exome_capture_platform + path: samples.sequencings + src: exome_capture_platform + fn: set + - name: library_prep_kit_method + path: samples.sequencings + src: library_prep_kit_method + fn: set + - name: capture_region_bed_file + path: samples.sequencings + src: capture_region_bed_file + fn: set + - name: reference_genome_build + path: samples.sequencings + src: reference_genome_build + fn: set + - name: sequencing_assay + path: samples.sequencings + src: sequencing_assay + fn: set + - name: alignment_method + path: samples.sequencings + src: alignment_method + fn: set + - name: data_processing_pipeline + path: samples.sequencings + src: data_processing_pipeline + fn: set + - name: _samples_count + path: samples + fn: count + - name: _sequencings_count + path: samples.sequencings + fn: count + joining_props: + - index: file + join_on: _subject_id + props: + - name: data_format + src: data_format + fn: set + - name: data_type + src: data_type + fn: set + - name: data_category + src: data_category + fn: set + - name: file_count + src: _file_id + fn: count + parent_props: + - path: projects[project_code:code] + - path: projects[project_name:name] + - path: projects[project_dbgap_phs:dbgap_phs] + - path: projects[project_dbgap_consent_text:dbgap_consent_text] + - path: projects[project_short_name:short_name] + - path: projects[project_dbgap_accession_number:dbgap_accession_number] + - name: ci.planx-ci.io_file + doc_type: file + type: collector + root: None + category: data_file + props: + - name: project_id + - name: submitter_id + - name: object_id + - name: md5sum + - name: file_name + - name: file_size + - name: data_format + - name: data_type + - name: state + - name: data_category + - name: analyte_type + - name: sequencing_assay + - name: source_node + injecting_props: + subject: + props: + - name: _subject_id + src: id + fn: set + - name: subject_submitter_id + src: submitter_id + fn: set + - name: anvil_project_id + src: anvil_project_id + fn: set + - name: sex + src: sex + fn: set + - name: age_value + src: age_value + fn: set + - name: ancestry + src: ancestry + fn: set + - name: disease_description + src: disease_description + fn: set + - name: phenotype_present + src: phenotype_present + fn: set + - name: phenotype_absent + src: phenotype_absent + fn: set + - name: disease_id + src: disease_id + fn: set + - name: solve_state + src: solve_state + fn: set + - name: congenital_status + src: congenital_status + fn: set + - name: age_of_onset + src: age_of_onset + fn: set + - name: phenotype_group + src: phenotype_group + fn: set diff --git a/ci/perf/values/fence.yaml b/ci/perf/values/fence.yaml new file mode 100644 index 000000000..9b1d150cb --- /dev/null +++ b/ci/perf/values/fence.yaml @@ -0,0 +1,605 @@ +fence: + # -- (map) External Secrets settings. + externalSecrets: + createK8sJwtKeysSecret: false + fenceJwtKeys: ci-fence-jwt + fenceGoogleAppCredsSecret: ci-fence-google-app-creds-secret + fenceGoogleStorageCredsSecret: ci-fence-google-storage-creds-secret + createK8sGoogleAppSecrets: false + fenceConfig: ci-fence-config-priv + fenceSshKeys: ci-fence-ssh-keys + fenceDeleteExpiredClients: + schedule: "0 0 * * *" + replicaCount: 2 + autoscaling: + enabled: true + image: + repository: quay.io/cdis/fence + pullPolicy: IfNotPresent + tag: 'master' + usersync: + usersync: true + userYamlS3Path: s3://cdis-gen3-users/ci/user.yaml + # we do not want usersync to interrupt test flow + # so it is set to run on the 1st of January if it is a Monday + schedule: '0 0 1 1 1' + syncFromDbgap: true + FENCE_CONFIG_PUBLIC: + # Prefix to namespace Google Groups on a single Cloud Identity (see cirrus + # setup for more info on Cloud Identity) + # + # NOTE: Make this short! Less than 8 characters if possible. Google has + # length restrictions on group names. + # This will be dynamically replaced by env_setup.sh script. + GOOGLE_GROUP_PREFIX: + + # Prefix to namespace Google Service Accounts in a single Google Cloud Platform Project. + # This is primarily to support multiple instances of fence references the same Google + # project. If that is not something you need to support, then you can leave this blank. + # + # NOTE: Make this short! Less than 8 characters if possible. Google has + # length restrictions on service account names. + # This will be dynamically replaced by env_setup.sh script. + GOOGLE_SERVICE_ACCOUNT_PREFIX: + + # This will be dynamically replaced by env_setup.sh script. + BASE_URL: https://HOSTNAME/user + + # -- (string) Name of the Fence app + APP_NAME: "Gen3 Data Commons" + + # -- (map) Debug and security settings + # Modify based on whether you're in a dev environment or in production + DEBUG: false + # -- (bool) if true, will automatically login a user with username "test" + # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) + MOCK_AUTH: false + + # -- (bool) if true, will fake a successful login response from Google in /login/google + # NOTE: this will also modify the behavior of /link/google endpoints + # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only) + # will login as the username set in cookie DEV_LOGIN_COOKIE_NAME + MOCK_GOOGLE_AUTH: true + + # -- (str) the name of the cookie set by mock authentication (used for testing only) + DEV_LOGIN_COOKIE_NAME: "dev_login" + + # -- (bool) if true, will ignore anything configured in STORAGE_CREDENTIALS + MOCK_STORAGE: false + + # -- (bool) allow OIDC traffic on http for development. By default it requires https. + # + # WARNING: ONLY set to true when fence will be deployed in such a way that it will + # ONLY receive traffic from internal clients and can safely use HTTP. + AUTHLIB_INSECURE_TRANSPORT: true + + # -- (bool) enable Prometheus Metrics for observability purposes + # + # WARNING: Any counters, gauges, histograms, etc. should be carefully + # reviewed to make sure its labels do not contain any PII / PHI + ENABLE_PROMETHEUS_METRICS: false + + # -- (bool) set if you want browsers to only send cookies with requests over HTTPS + SESSION_COOKIE_SECURE: true + + # -- (bool) enable CSRF protection + ENABLE_CSRF_PROTECTION: true + + # -- (str) signing key for WTForms to sign CSRF tokens with + WTF_CSRF_SECRET_KEY: "{{ENCRYPTION_KEY}}" + + # -- (bool) fence (at the moment) attempts a migration on startup. setting this to false will disable that + # WARNING: ONLY set to false if you do NOT want to automatically migrate your database. + # You should be careful about incompatible versions of your db schema with what + # fence expects. In other words, things could be broken if you update to a later + # fence that expects a schema your database isn't migrated to. + # NOTE: We are working to improve the migration process in the near future + ENABLE_DB_MIGRATION: true + + # these are the *possible* scopes a client can be given, NOT scopes that are + # given to all clients. You can be more restrictive during client creation + # -- (list) These are the *possible* scopes a client can be given, NOT scopes that are given to all clients. You can be more restrictive during client creation + CLIENT_ALLOWED_SCOPES: + - "openid" + - "user" + - "data" + - "google_credentials" + - "google_service_account" + - "google_link" + - "ga4gh_passport_v1" + + # -- (list) these are the scopes that CAN be included in a user's own access_token + USER_ALLOWED_SCOPES: + - "fence" + - "openid" + - "user" + - "data" + - "admin" + - "google_credentials" + - "google_service_account" + - "google_link" + - "ga4gh_passport_v1" + + # -- (list) these are the scopes that a browser session can create for a user (very similar to USER_ALLOWED_SCOPES, as the session will actually create access_tokens for an actively logged in user) + SESSION_ALLOWED_SCOPES: + - "openid" + - "user" + - "credentials" + - "data" + - "admin" + - "google_credentials" + - "google_service_account" + - "google_link" + - "ga4gh_passport_v1" + + # ////////////////////////////////////////////////////////////////////////////////////// + # LOGIN + # - Modify based on which OIDC provider(s) you configured above + # - NOTE: You can have multiple IDPs for users to login with, but one has to be set + # as the default + # ////////////////////////////////////////////////////////////////////////////////////// + + # List of enabled login options (used by data-portal to display login buttons). + # Each option must be configured with a "name" and an "idp". + # - "idp" must be a configured provider in OPENID_CONNECT section. + # Multiple options can be configured with the same idp. + # - if provider_id is "fence", "fence_idp" can be any of the providers + # supported by the other Fence. If not specified, will default to NIH login. + # - if provider_id is "fence" and fence_idp is "shibboleth", a list of + # "shib_idps" can be configured for InCommon login. If not specified, will + # default to NIH login. + # - Optional parameters: "desc" (description) and "secondary" (boolean - can + # be used by the frontend to display secondary buttons differently). + # -- (list) List of enabled login options (used by data-portal to display login buttons). + LOGIN_OPTIONS: + - name: 'Login from Google' + idp: google + - name: 'ORCID Login' + idp: fence + fence_idp: orcid + - name: 'Login from RAS' + idp: ras + + + # -- (string) Default login provider. - must be configured in LOGIN_OPTIONS and OPENID_CONNECT - - if several options in LOGIN_OPTIONS are defined for this IDP, will default to the first one + DEFAULT_LOGIN_IDP: google + + # -- (string) Default login URL: DEPRECATED and replaced by LOGIN_OPTIONS + DEFAULT_LOGIN_IDP configs + DEFAULT_LOGIN_URL: "{{BASE_URL}}/login/google" + + # `LOGIN_REDIRECT_WHITELIST` is a list of extra whitelisted URLs which can be redirected + # to by the `/login/*` endpoints. Fence automatically populates this with the redirect + # URLs for any registered OAuth clients, and its own URL. When validating the redirects, + # fence chesk whether the domain for the redirect matches a domain in the whitelist (so + # only the domains for the additional desired redirects are necessary here). + LOGIN_REDIRECT_WHITELIST: [".", "/login"] + + ### DEPRECATED and replaced by OPENID_CONNECT + LOGIN_OPTIONS configs + ENABLED_IDENTITY_PROVIDERS: + default: google + providers: + google: + name: Login from Google + + # ////////////////////////////////////////////////////////////////////////////////////// + # LIBRARY CONFIGURATION (authlib & flask) + # - Already contains reasonable defaults + # ////////////////////////////////////////////////////////////////////////////////////// + # authlib-specific configs for OIDC flow and JWTs + # NOTE: the OAUTH2_JWT_KEY cfg gets set automatically by fence if keys are setup + # correctly + OAUTH2_JWT_ALG: "RS256" + OAUTH2_JWT_ENABLED: true + OAUTH2_JWT_ISS: "{{BASE_URL}}" + OAUTH2_PROVIDER_ERROR_URI: "/api/oauth2/errors" + + # used for flask, "path mounted under by the application / web server" + # since we deploy as microservices, fence is typically under {{base}}/user + # this is also why our BASE_URL default ends in /user + APPLICATION_ROOT: "/user" + + # ////////////////////////////////////////////////////////////////////////////////////// + # Tokens, Lifetimes, & Expirations + # - Already contains reasonable defaults + # ////////////////////////////////////////////////////////////////////////////////////// + # The name of the browser cookie in which the access token will be stored. + ACCESS_TOKEN_COOKIE_NAME: "access_token" + + # The name of the browser cookie in which the session token will be stored. + # Note that the session token also stores information for the + # ``flask.session`` in the ``context`` field of the token. + SESSION_COOKIE_NAME: "fence" + + # The domain of the browser cookie in which the session token will be stored. + # Leave unset (not empty string!) for normal single-site deployment. + SESSION_COOKIE_DOMAIN: + + OAUTH2_TOKEN_EXPIRES_IN: + "authorization_code": 1200 + "implicit": 1200 + + # The number of seconds after an access token is issued until it expires. + ACCESS_TOKEN_EXPIRES_IN: 1200 + + # The number of seconds after a refresh token is issued until it expires. + REFRESH_TOKEN_EXPIRES_IN: 2592000 + + # The number of seconds after which a browser session is considered stale. + SESSION_TIMEOUT: 1800 + + # The maximum session lifetime in seconds. + SESSION_LIFETIME: 28800 + + # The number of seconds the user's Google service account key used for + # url signing will last before being expired/rotated + # 30 days: 2592000 seconds + GOOGLE_SERVICE_ACCOUNT_KEY_FOR_URL_SIGNING_EXPIRES_IN: 2592000 + + # The number of seconds after a User's Google Service account is added to bucket + # access until it expires. + # 7 days: 604800 seconds + GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN: 604800 + + # The number of seconds after a User's Google account is added to bucket + # access until it expires. + GOOGLE_ACCOUNT_ACCESS_EXPIRES_IN: 86400 + + # The number of seconds after a pre-signed url is issued until it expires. + MAX_PRESIGNED_URL_TTL: 3600 + + # The number of seconds after an API KEY is issued until it expires. + MAX_API_KEY_TTL: 2592000 + + # The number of seconds after an access token is issued until it expires. + MAX_ACCESS_TOKEN_TTL: 3600 + + # TEMPORARY: The maximum number of projects allowed in token claims. + # This config var should be removed after sheepdog and peregrine support + # auth checks against Arborist, and no longer check the token. + TOKEN_PROJECTS_CUTOFF: 10 + + # If set to true, will generate an new access token each time when a browser session update happens + RENEW_ACCESS_TOKEN_BEFORE_EXPIRATION: false + + # The maximum lifetime of a Gen3 passport in seconds + GEN3_PASSPORT_EXPIRES_IN: 43200 + + ######################################################################################## + # OPTIONAL CONFIGURATIONS # + ######################################################################################## + + # For displaying a privacy policy to users, we can either link to the URL specified by + # PRIVACY_POLICY_URL, or default to the `static/privacy_policy.md` file in fence. + PRIVACY_POLICY_URL: null + + # ////////////////////////////////////////////////////////////////////////////////////// + # RELIABILITY OPTS + # ////////////////////////////////////////////////////////////////////////////////////// + # Configurations related to resiliency, fault-tolerance and availability + # This is the number of requests per second that the Nginx proxy will accept before reaching fence + # The value defined in fence-config-public.yaml takes precedence over this one + # In the absence of this OVERRIDE prefixed config, the legacy NGINX_RATE_LIMIT from the k8s deployment yaml is applied + OVERRIDE_NGINX_RATE_LIMIT: 18 + + # ////////////////////////////////////////////////////////////////////////////////////// + # SUPPORT INFO + # ////////////////////////////////////////////////////////////////////////////////////// + # If you want an email address to show up when an unhandled error occurs, provide one + # here. Something like: support@example.com + SUPPORT_EMAIL_FOR_ERRORS: null + + # ////////////////////////////////////////////////////////////////////////////////////// + # SHIBBOLETH + # - Support using `shibboleth` in LOGIN_OPTIONS + # - Contains defaults for using NIH's Login. + # ////////////////////////////////////////////////////////////////////////////////////// + # assumes shibboleth is deployed under {{BASE_URL}}/shibboleth + SHIBBOLETH_HEADER: "persistent_id" + SSO_URL: "https://auth.nih.gov/affwebservices/public/saml2sso?SPID={{BASE_URL}}/shibboleth&RelayState=" + ITRUST_GLOBAL_LOGOUT: "https://auth.nih.gov/siteminderagent/smlogout.asp?mode=nih&AppReturnUrl=" + + DBGAP_ACCESSION_WITH_CONSENT_REGEX: "(?Pphs[0-9]+)(.(?Pv[0-9]+)){0,1}(.(?Pp[0-9]+)){0,1}.(?Pc[0-9]+)" + + # ////////////////////////////////////////////////////////////////////////////////////// + # STORAGE BACKENDS AND CREDENTIALS + # - Optional: Used for `/admin` & `/credentials` endpoints for user management. + # Also used during User Syncing process to automate managing Storage + # access for users. + # ////////////////////////////////////////////////////////////////////////////////////// + # When true, this modifies usersync (not fence service itself) such that when syncing user + # access to a Google storage backend happens in "bulk" by doing a diff *per google group* + # between what's in Google and what's expected. Then it adds, removes only as necessary. + # This is in contrast to the default logic which does blind updates per user and ignores + # 409s from Google. + # NOTE: This reduces the number of API calls to Google in the general case, but increases + # memory usages by usersync (as it has to track all the Google groups and user access) + GOOGLE_BULK_UPDATES: true + + # Configuration for various storage systems for the backend + # NOTE: Remove the {} and supply backends if needed. Example in comments below + # STORAGE_CREDENTIALS: {} + STORAGE_CREDENTIALS: + 'google': + backend: google + # this should be the project id where the Google Groups for data access are managed + google_project_id: dcf-integration + + # Cleversafe data storage backend + # + # 'cleversafe-server-a': + # backend: 'cleversafe' + # aws_access_key_id: '' + # aws_secret_access_key: '' + # host: 'somemanager.osdc.io' + # public_host: 'someobjstore.example.com' + # port: 443 + # is_secure: true + # username: 'someone' + # password: 'somepass' + # is_mocked: true + + # ////////////////////////////////////////////////////////////////////////////////////// + # AWS BUCKETS AND CREDENTIALS + # - Support `/data` endpoints + # ////////////////////////////////////////////////////////////////////////////////////// + + # NOTE: the region is optonal for s3_buckets, however it should be specified to avoid a + # call to GetBucketLocation which you make lack the AWS ACLs for. + # public buckets do not need the region field. + # the cred values should be keys in section `AWS_CREDENTIALS`. + S3_BUCKETS: + cdis-presigned-url-test: + region: 'us-east-1' + cred: 'cdistest' + gen3-helm-data-upload-bucket: + region: 'us-east-1' + cred: 'cdistest' + DATA_UPLOAD_BUCKET: 'gen3-helm-data-upload-bucket' + + # ////////////////////////////////////////////////////////////////////////////////////// + # PROXY + # - Optional: If the api is behind firewall that needs to set http proxy + # ////////////////////////////////////////////////////////////////////////////////////// + # NOTE: leave as-is to not use proxy + # this is only used by the Google Oauth2Client at the moment if provided + HTTP_PROXY: + host: 'cloud-proxy.internal.io' + port: 3128 + + # ////////////////////////////////////////////////////////////////////////////////////// + # MICROSERVICE PATHS + # - Support `/data` endpoints & authz functionality + # ////////////////////////////////////////////////////////////////////////////////////// + # url where indexd microservice is running (for signed urls primarily) + # NOTE: Leaving as null will force fence to default to {{BASE_URL}}/index + # example value: 'https://example.com/index' + INDEXD: http://indexd-service + + # this is the username which fence uses to make authenticated requests to indexd + INDEXD_USERNAME: "fence" + # this is the password which fence uses to make authenticated requests to indexd + INDEXD_PASSWORD: "" + + # ////////////////////////////////////////////////////////////////////////////////////// + # AZURE STORAGE BLOB CONFIGURATION + # - Support Azure Blob Data Access Methods + # ////////////////////////////////////////////////////////////////////////////////////// + + # https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?toc=%2Fazure%2Fstorage%2Fblobs%2Ftoc.json&tabs=azure-portal#view-account-access-keys + # AZ_BLOB_CREDENTIALS: 'fake connection string' + AZ_BLOB_CREDENTIALS: + + # AZ_BLOB_CONTAINER_URL: 'https://storageaccount.blob.core.windows.net/container/' + # this is the container used for uploading, and should match the storage account + # used in the connection string for AZ_BLOB_CREDENTIALS + AZ_BLOB_CONTAINER_URL: "https://myfakeblob.blob.core.windows.net/my-fake-container/" + + # url where authz microservice is running + ARBORIST: http://arborist-service + + # url where the audit-service is running + # This will be dynamically replaced by env_setup.sh script. + AUDIT_SERVICE: "http://audit-service" + ENABLE_AUDIT_LOGS: + presigned_url: true + login: true + # `PUSH_AUDIT_LOGS_CONFIG.type` is one of: [api, aws_sqs]. + # - if type == api: logs are created by hitting the log creation endpoint. + # - if type == aws_sqs: logs are pushed to an SQS and `aws_sqs_config` fields + # `sqs_url` and `region` are required. Field `aws_cred` is optional and it + # should be a key in section `AWS_CREDENTIALS`. + PUSH_AUDIT_LOGS_CONFIG: + type: aws_sqs + aws_sqs_config: + # This will be dynamically replaced by env_setup.sh script. + sqs_url: https://sqs.us-east-1.amazonaws.com/707767160287/audit-service-sqs + region: us-east-1 + aws_cred: 'cdistest' + + # A Google Project identitifier representing the default project to bill to for + # accessing Google Requester Pays buckets (for signed urls and/or temporary service account + # credentials). If this is provided and the API call for + # Google access does not include a `userProject`, this will be used instead. + # + # WARNING: Setting this WITHOUT setting "ENABLE_AUTOMATIC_BILLING_*" to `true` below, + # means that clients and end-users will be responsible for making sure that + # the service account used in either of these methods actually has billing + # permission in the specified project. + BILLING_PROJECT_FOR_SIGNED_URLS: + BILLING_PROJECT_FOR_SA_CREDS: + + # Setting this to `true` will make Fence automatically attempt to create a Custom Role + # in the billing project and give the necessary Google Service Account that role + # (which will allow it to bill to the project). + # + # NOTE: The Fence SA will need the necessary permissions in the specified project to + # both create a custom role and update the Project's IAM Policy to include the + # necessary SA. At the time of writing, there are pre-defined roles in Google's + # IAM that provide the necessary permissions. Those are "Project IAM Admin" and + # "Role Administrator" + # + # NOTE2: It may be possible to further restrict the permissions in the future to + # be more fine-grained. + # + ENABLE_AUTOMATIC_BILLING_PERMISSION_SIGNED_URLS: false + ENABLE_AUTOMATIC_BILLING_PERMISSION_SA_CREDS: false + + # ////////////////////////////////////////////////////////////////////////////////////// + # EMAIL + # - Support for sending emails from fence. Used for user certificates + # and `/google/service_accounts` endpoints + # ////////////////////////////////////////////////////////////////////////////////////// + # Gun Mail Service (for sending emails from fence) + # + # NOTE: Example in comments below + GUN_MAIL: + "datacommons.io": + smtp_hostname: "smtp.mailgun.org" + api_key: "" + default_login: "postmaster@mailgun.example.com" + api_url: "https://api.mailgun.net/v3/mailgun.example.com" + smtp_password: "" + + # For emails regarding users certificates + EMAIL_SERVER: "localhost" + SEND_FROM: "example@gmail.com" + SEND_TO: "example@gmail.com" + + # ////////////////////////////////////////////////////////////////////////////////////// + # DATA ACCESS: GOOGLE LINKING & SERVICE ACCOUNT REGISTRATION + # - Support `/google/service_accounts` endpoints + # ////////////////////////////////////////////////////////////////////////////////////// + # whether or not to allow access to the /link/google endpoints + ALLOW_GOOGLE_LINKING: true + + # A Google Project with controlled data access will be determined INVALID if + # if it has a parent organization UNLESS that parent organization's ID is in this + # whitelist. + # + # NOTE: Remove the [] and Google Organization IDs if needed. Example in comments below + WHITE_LISTED_GOOGLE_PARENT_ORGS: [] + # - '12345678910' + + # when service accounts or google projects are determined invalid, an email is sent + # to the project owners. These settings are for that email + REMOVE_SERVICE_ACCOUNT_EMAIL_NOTIFICATION: + enable: false + # this domain MUST exist in GUN_MAIL config + domain: "example.com" + from: "do-not-reply@example.com" + subject: "User service account removal notification" + # the {} gets replaced dynamically in the Python code to be the Project ID + content: > + Service accounts were removed from access control data because some users or + service accounts of GCP Project {} are not authorized to access the data sets + associated to the service accounts, or do not adhere to the security policies. + # this admin email will be included as a recipient to *any* email to anyone about + # service account removal. + # + # WARNING: This is NOT a bcc so the email is visible to the end-user + admin: + - "admin@example.edu" + + PROBLEM_USER_EMAIL_NOTIFICATION: + # this domain MUST exist in GUN_MAIL config + domain: "example.com" + from: "do-not-reply@example.com" + subject: "Account access error notification" + # the {} gets replaced dynamically in the Python code to be the Project ID + content: > + The Data Commons Framework utilizes dbGaP for data access authorization. + Another member of a Google project you belong to ({}) is attempting to + register a service account to the following additional datasets ({}). + Please contact dbGaP to request access. + # this admin email will be included as a recipient to *any* email to anyone about + # service account removal. + # + # WARNING: This is NOT a bcc so the email is visible to the end-user + admin: + - "admin@example.edu" + + # Service account email domains that represent a service account that Google owns. + # These are usually created when a sepcific GCP service is enabled. + # This is used for Service Account Validation for Data Access. + GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS: + - "dataflow-service-producer-prod.iam.gserviceaccount.com" + - "cloudbuild.gserviceaccount.com" + - "cloud-ml.google.com.iam.gserviceaccount.com" + - "container-engine-robot.iam.gserviceaccount.com" + - "dataflow-service-producer-prod.iam.gserviceaccount.com" + - "sourcerepo-service-accounts.iam.gserviceaccount.com" + - "dataproc-accounts.iam.gserviceaccount.com" + - "gae-api-prod.google.com.iam.gserviceaccount.com" + - "genomics-api.google.com.iam.gserviceaccount.com" + - "containerregistry.iam.gserviceaccount.com" + - "container-analysis.iam.gserviceaccount.com" + - "cloudservices.gserviceaccount.com" + - "stackdriver-service.iam.gserviceaccount.com" + - "appspot.gserviceaccount.com" + - "partnercontent.gserviceaccount.com" + - "trifacta-gcloud-prod.iam.gserviceaccount.com" + - "gcf-admin-robot.iam.gserviceaccount.com" + - "compute-system.iam.gserviceaccount.com" + - "gcp-sa-websecurityscanner.iam.gserviceaccount.com" + - "storage-transfer-service.iam.gserviceaccount.com" + - "firebase-sa-management.iam.gserviceaccount.com" + - "firebase-rules.iam.gserviceaccount.com" + - "gcp-sa-cloudbuild.iam.gserviceaccount.com" + - "gcp-sa-automl.iam.gserviceaccount.com" + - "gcp-sa-datalabeling.iam.gserviceaccount.com" + - "gcp-sa-cloudscheduler.iam.gserviceaccount.com" + + # The types of service accounts that are allowed to be registered at + # /google/service_accounts endpoints + ALLOWED_USER_SERVICE_ACCOUNT_DOMAINS: + # compute engine default service account + - "developer.gserviceaccount.com" + # app engine default service account + - "appspot.gserviceaccount.com" + # user-managed service account + - "iam.gserviceaccount.com" + + # Synapse integration and DREAM challenge mapping. Team is from Synapse, and group is + # providing the actual permission in Arborist. User will be added to the group for TTL + # seconds if the team matches. + DREAM_CHALLENGE_TEAM: "DREAM" + DREAM_CHALLENGE_GROUP: "DREAM" + SYNAPSE_URI: "https://repo-prod.prod.sagebase.org/auth/v1" + SYNAPSE_JWKS_URI: + # deprecated, use the discovery_url in the OPENID_CONNECT block for the synapse client + SYNAPSE_DISCOVERY_URL: + SYNAPSE_AUTHZ_TTL: 86400 + + # Role caching for generating presigned urls if max role session increase is true + # then we can increase the amount of time that a session is valid for + MAX_ROLE_SESSION_INCREASE: false + ASSUME_ROLE_CACHE_SECONDS: 1800 + + # Optional user registration feature: Ask users to register (provide firstname/lastname/org/email) on login. + # If user registers, add them to configured Arborist group; idea is that the Arborist group + # will have access to download data. + REGISTER_USERS_ON: true + REGISTERED_USERS_GROUP: "" + # RAS refresh_tokens expire in 15 days + RAS_REFRESH_EXPIRATION: 1296000 + # List of JWT issuers from which Fence will accept GA4GH visas + GA4GH_VISA_ISSUER_ALLOWLIST: + - "{{BASE_URL}}" + - "https://sts.nih.gov" + - "https://stsstg.nih.gov" + # Number of projects that can be registered to a Google Service Accont + SERVICE_ACCOUNT_LIMIT: 6 + + # Global sync visas during login + # None(Default): Allow per client i.e. a fence client can pick whether or not to sync their visas during login with parse_visas param in /authorization endpoint + # True: Parse for all clients i.e. a fence client will always sync their visas during login + # False: Parse for no clients i.e. a fence client will not be able to sync visas during login even with parse_visas param + GLOBAL_PARSE_VISAS_ON_LOGIN: false + # Settings for usersync with visas + USERSYNC: + sync_from_visas: false + # fallback to dbgap sftp when there are no valid visas for a user i.e. if they're expired or if they're malformed + fallback_to_dbgap_sftp: false + visa_types: + ras: ["https://ras.nih.gov/visas/v1", "https://ras.nih.gov/visas/v1.1"] + RAS_USERINFO_ENDPOINT: "/openid/connect/v1.1/userinfo" diff --git a/ci/perf/values/guppy.yaml b/ci/perf/values/guppy.yaml new file mode 100644 index 000000000..cb98a7890 --- /dev/null +++ b/ci/perf/values/guppy.yaml @@ -0,0 +1,34 @@ +guppy: + replicaCount: 2 + autoscaling: + enabled: true + # Lower cost + resources: + requests: + memory: "105Mi" + cpu: "15m" + enabled: true + #TO DO Re-enable once ES index restore is added to modify values script + dbRestore: false + image: + repository: quay.io/cdis/guppy + tag: master + indices: + - index: ci_subject_alias + type: subject + - index: ci_file_alias + type: file + # - index: ci_cmc_alias + # type: clinical_trials + # - index: ci_dataset_alias + # type: dataset + # - index: ci_case_alias + # type: case + # - index: ci_imaging_data_file_alias + # type: imaging_data_file + - index: ci_imaging_study_alias + type: imaging_study + configIndex: ci_configs_alias + authFilterField: auth_resource_path + autoscaling: + enabled: true diff --git a/ci/perf/values/portal.yaml b/ci/perf/values/portal.yaml new file mode 100644 index 000000000..51633cfe7 --- /dev/null +++ b/ci/perf/values/portal.yaml @@ -0,0 +1,922 @@ +portal: + # replicaCount: 2 + autoscaling: + enabled: true + resources: + limits: + memory: 6Gi + requests: + cpu: 4 + memory: 6Gi + image: + repository: quay.io/cdis/data-portal + tag: master + gitops: + json: | + { + "gaTrackingId": "UA-119127212-2", + "argoTemplate": "gwas-template", + "graphql": { + "boardCounts": [ + ], + "chartCounts": [ + ], + "projectDetails": "boardCounts" + }, + "useArboristUI": true, + "showArboristAuthzOnProfile": true, + "showFenceAuthzOnProfile": false, + "componentToResourceMapping": { + "Workspace": { + "resource": "/workspace", + "method": "access", + "service": "jupyterhub" + }, + "Query": { + "resource": "/query_page", + "method": "access", + "service": "query_page" + }, + "Send Queries": { + "resource": "/query_page", + "method": "access", + "service": "query_page" + }, + "Data Dictionary": { + "resource": "/dictionary_page", + "method": "access", + "service": "dictionary_page" + }, + "Dictionary": { + "resource": "/dictionary_page", + "method": "access", + "service": "dictionary_page" + } + }, + "components": { + "appName": "BloodPAC Metadata Submission Portal", + "index": { + "introduction": { + "heading": "BloodPAC Data Commons", + "text": "The Blood Profiling Atlas in Cancer (BloodPAC) supports the management, analysis and sharing of liquid biopsy data for the oncology research community and aims to accelerate discovery and development of therapies, diagnostic tests, and other technologies for cancer treatment and prevention. The data commons supports cross-project analyses by harmonizing data from different projects through the collaborative development of a data dictionary, providing an API for data queries and download, and providing a cloud-based analysis workspace with rich tools and resources.", + "link": "/submission" + }, + "homepageChartNodes": [{ + "node": "subject", + "name": "Subjects" + } + ], + "buttons": [{ + "name": "Data Dictionary", + "icon": "data-field-define", + "body": "The BloodPAC Data Commons employs a data model to power queries and data submissions. Study the dictionary to send GraphQL queries or format your data submissions.", + "link": "/DD", + "label": "Learn more" + }, + { + "name": "Submit Data", + "icon": "data-submit", + "body": "Browse the list of submitted data projects, or find your project and submit data.", + "link": "/submission", + "label": "Submit data" + }, + { + "name": "Explore Data", + "icon": "data-files", + "body": "The Exploration app enables faceted search for cohort building, getting file download manifests, and browsing data across projects.", + "link": "/explorer", + "label": "Exploration" + }, + { + "name": "Send Queries", + "icon": "data-access", + "body": "Build and send your own custom GraphQL queries to the database to pinpoint specific data.", + "link": "/query", + "label": "Send Queries" + }, + { + "icon": "analysis", + "link": "/analysis", + "color": "#a2a2a2", + "name": "Apps" + } + ] + }, + "navigation": { + "title": "BloodPAC Data Commons", + "items": [ + { + "icon": "query", + "link": "/discovery", + "name": "Discovery", + "tooltip": "Use free-text search and tags to rapidly find relevant studies, discover new datasets across multiple resources, and easily export selected data files to the analysis workspace." + }, + { + "icon": "exploration", + "link": "/explorer", + "color": "#a2a2a2", + "name": "Exploration" + }, + { + "icon": "query", + "link": "/study-viewer/clinical_trials", + "color": "#a2a2a2", + "name": "Study Viewer" + }, + { + "icon": "workspace", + "link": "/workspace", + "color": "#a2a2a2", + "name": "Workspace" + }, + { + "icon": "dictionary", + "link": "/DD", + "color": "#a2a2a2", + "name": "Dictionary" + }, + { + "icon": "analysis", + "link": "/analysis", + "color": "#a2a2a2", + "name": "Apps" + }, + { + "icon": "query", + "link": "/query", + "color": "#a2a2a2", + "name": "Query" + }, + { + "icon": "profile", + "link": "/identity", + "color": "#a2a2a2", + "name": "Profile" + } + ] + }, + "topBar": { + "items": [{ + "link": "https://bloodpac.org/", + "name": "About" + }, + { + "icon": "upload", + "link": "/submission", + "name": "Submit Data" + }, + { + "link": "https://www.synapse.org/#!Synapse:syn8011461/wiki/411591", + "name": "Documentation" + }, + { + "link": "bpa-support@gen3.org", + "name": "Email Support" + } + ], + "useProfileDropdown": false + }, + "login": { + "title": "BloodPAC Data Commons", + "subTitle": "Explore, Analyze, and Share Data", + "text": "This website combines liquid biopsy data from academic, government, and industry partners and aims to accelerate discovery and development of therapies, diagnostic tests, and other technologies for the treatment and prevention of cancer.", + "contact": "If you have any questions about access or the registration process, please contact ", + "email": "bpa-support@gen3.org" + }, + "certs": { + "security_quiz": { + "hasCorrectAnswers": true, + "title": "BloodPAC User agreement", + "description": "", + "questions": [{ + "name": "", + "question": "NOTICE: To protect the system from unauthorized use and to ensure that the system is functioning, activities on this system are monitored and recorded and subject to audit. Use of this system is express consent to such monitoring and recording. Any unauthorized access or use of this system is prohibited. You may only use this system if you have been designated as an Authenticated Individual by your company or institution. You may not use the system in any way that is inconsistent with the agreements between CCSR and your company or institution. Please pay particular attention to the restrictions on controlled access data and transfer it in an approved secure manner.", + "options": ["I agree"], + "answer": 0, + "hint": "Click on I agree" + }] + } + } + }, + "featureFlags": { + "explorer": true, + "analysis": true, + "explorerPublic": true, + "discovery": true, + "discoveryUseAggMDS": true, + "studyRegistration": true + }, + "analysisTools": [ + { + "title": "OHDSI Atlas", + "description": "Use this App for cohort creation. These will be automatically populated in the Gen3 GWAS App", + "applicationUrl": "https://atlas.qa-mickey.planx-pla.net/atlas/", + "image": "/custom/sponsors/gitops-sponsors/atlas-logo.png" + }, + { + "appId": "GWASUIApp", + "title": "Gen3 GWAS", + "description": "Use this App to perform high throughput GWAS on Million Veteran Program (MVP) data, using the University of Washington Genesis pipeline", + "image": "/src/img/analysis-icons/gwas.svg", + "needsTeamProject": true + }, + { + "appId": "GWASResults", + "title": "GWAS Results", + "description": "Use this App to view status & results of submitted workflows", + "image": "/src/img/analysis-icons/gwasResults.svg", + "needsTeamProject": true + } + ], + "discoveryConfig": { + "features": { + "advSearchFilters": { + "enabled": false, + "field": "advSearchFilters", + "filters": [ + { "key": "Research Focus Area" }, + { "key": "Administering IC(s)" }, + { "key": "Simulated vs. Real Data" }, + { "key": "Year Grant Awarded" }, + { "key": "Study Description" } + ] + }, + "exportToWorkspace": { + "enabled": true, + "manifestFieldName": "__manifest", + "downloadManifestButtonText": "Download", + "enableDownloadManifest": false, + "downloadZipButtonText": "Download Zip", + "enableDownloadZip": true + }, + "pageTitle": { + "enabled": false + }, + "search": { + "searchBar": { + "enabled": true, + "searchableTextFields": [ + "summary", + "location", + "study_name", + "institutions", + "investigators", + "project_title", + "project_number", + "dataset_1_title", + "dataset_2_title", + "dataset_3_title", + "dataset_4_title", + "administering_ic", + "research_program", + "research_question", + "dataset_description", + "dataset_1_description", + "dataset_2_description", + "dataset_3_description", + "dataset_4_description", + "data_dictionary" + ] + } + }, + "authorization": { + "enabled": true, + "supportedValues": { + "accessible": { + "enabled": true, + "menuText": "Accessible" + }, + "unaccessible": { + "enabled": true, + "menuText": "Not Accessible" + }, + "pending": { + "enabled": false + }, + "notAvailable": { + "enabled": false + } + } + } + }, + "aggregations": [ + { + "name": "Studies", + "field": "project_number", + "type": "count" + } + ], + "tagSelector": { + "title": "Tags by category", + "showTagCategoryNames": true + }, + "studies": [], + "studyColumns": [ + { + "name": "Project Title", + "field": "project_title", + "errorIfNotAvailable": false, + "valueIfNotAvailable": "n/a" + }, + { + "name": "Research Focus Area", + "field": "research_focus_area", + "errorIfNotAvailable": false, + "valueIfNotAvailable": "n/a" + }, + { + "name": "Investigator(s)", + "field": "investigators", + "errorIfNotAvailable": false, + "valueIfNotAvailable": "n/a" + } + ], + "studyPreviewField": { + "name": "Summary", + "field": "summary", + "contentType": "string", + "includeName": false, + "includeIfNotAvailable": true, + "valueIfNotAvailable": "No summary has been provided for this study." + }, + "studyPageFields": { + "header": { + "field": "project_title" + }, + "fieldsToShow": [ + { + "groupName": "HEAL Research Initiatives", + "groupWidth": "full", + "includeName": false, + "fields": [ + { + "name": "Project Number", + "field": "project_number" + }, + { + "name": "Research Focus Area", + "field": "research_focus_area" + }, + { + "name": "Research Program", + "field": "research_program" + }, + { + "name": "Research Question", + "field": "research_question" + } + ] + }, + { + "groupName": "Host Institution", + "groupWidth": "half", + "includeName": false, + "fields": [ + { + "name": "Institution(s)", + "field": "institutions" + }, + { + "name": "Location", + "field": "location" + }, + { + "name": "Investigator(s)", + "field": "investigators" + }, + { + "name": "Administering IC(s)", + "field": "administering_ic" + }, + { + "name": "Year Grant Awarded", + "field": "year_awarded" + } + ] + }, + { + "groupName": "Clinical Trial", + "groupWidth": "half", + "includeName": false, + "fields": [ + { + "name": "Study Type", + "field": "study_type" + }, + { + "name": "Study Name", + "field": "study_name" + }, + { + "name": "Study Description", + "field": "study_description" + }, + { + "name": "Number of Subjects", + "field": "subjects" + }, + { + "name": "Number of Sites", + "field": "sites" + }, + { + "name": "Clinical Trial Link", + "field": "clinical_trial_link" + }, + { + "name": "Protocol Name", + "field": "protocol_name" + }, + { + "name": "Dataset Description", + "field": "dataset_description" + }, + { + "name": "Dataset Category", + "field": "dataset_category" + } + ] + }, + { + "groupName": "Tags", + "includeName": false, + "groupWidth": "full", + "fields": [ + { + "name": "Tags", + "field": "tags", + "contentType": "tags", + "includeIfNotAvailable": false + } + ] + }, + { + "groupWidth": "full", + "fields": [ + { + "name": "Summary", + "field": "summary", + "contentType": "paragraphs", + "includeName": true, + "includeIfNotAvailable": true, + "valueIfNotAvailable": "No summary has been provided for this study." + } + ] + }, + { + "groupName": "Datasets", + "groupWidth": "full", + "includeName": false, + "fields": [ + { + "name": "Dataset 1 - Title", + "field": "dataset_1_title" + }, + { + "name": "Dataset 1 - Description", + "field": "dataset_1_description" + }, + { + "name": "Dataset 1 - Type", + "field": "dataset_1_type" + }, + { + "name": "Dataset 2 - Title", + "field": "dataset_2_title" + }, + { + "name": "Dataset 2 - Description", + "field": "dataset_2_description" + }, + { + "name": "Dataset 2 - Type", + "field": "dataset_2_type" + }, + { + "name": "Dataset 3 - Title", + "field": "dataset_3_title" + }, + { + "name": "Dataset 3 - Description", + "field": "dataset_3_description" + }, + { + "name": "Dataset 3 - Type", + "field": "dataset_3_type" + }, + { + "name": "Dataset 4 - Title", + "field": "dataset_4_title" + }, + { + "name": "Dataset 4 - Description", + "field": "dataset_4_description" + }, + { + "name": "Dataset 4 - Type", + "field": "dataset_4_type" + }, + { + "name": "Dataset 5 - Title", + "field": "dataset_5_title" + }, + { + "name": "Dataset 5 - Description", + "field": "dataset_5_description" + }, + { + "name": "Dataset 5 - Type", + "field": "dataset_5_type" + } + ] + } + ] + }, + "minimalFieldMapping": { + "authzField": "authz", + "tagsListFieldName": "tags", + "dataAvailabilityField": "data_availability", + "uid": "project_number" + }, + "tagCategories": [ + { + "name": "Demographic Info", + "color": "rgba(64, 34, 100, 1)", + "display": true + }, + { + "name": "Study Design", + "color": "rgba(153, 40, 107, 1)", + "display": true + }, + { + "name": "HEAL Specific", + "color": "rgba(161, 36, 46, 1)", + "display": true + }, + { + "name": "Other", + "color": "rgba(207, 54, 38, 1)", + "display": true + }, + { + "name": "Commons", + "displayName": "Data Resources", + "color": "rgba(21, 6, 76, 1)" + } + ] + }, + "studyViewerConfig": [ + { + "dataType": "subject", + "title": "Studies", + "titleField": "project_id", + "rowAccessor": "submitter_id", + "listItemConfig": { + "blockFields": ["project_id"], + "tableFields": ["data_type"] + }, + "fieldMapping": [ + { "field": "category", "name": "Study Type"}, + { "field": "project_id", "name": "Project_ID"}, + { "field": "data_type", "name": "Data_Type"}, + { "field": "condition", "name": "Condition"} + ], + "openMode": "close-all", + "buttons": [ + { + "type": "download" + }, + { + "type": "request_access", + "resourceDisplayNameField": "title", + "accessRequestedText": "DAR In Progress", + "accessRequestedTooltipText": "Your recently submitted DAR is being reviewed by NIAID" + }] + } + ], + "explorerConfig": [ + { + "tabTitle": "Data", + "charts": { + "project_id": { + "chartType": "count", + "title": "Projects" + } + }, + "filters": { + "tabs": [ + { + "title": "Project", + "fields": [ + "project_id", + "data_type", + "data_format" + ] + } + ] + }, + "table": { + "enabled": true, + "fields": [ + "project_id", + "data_format" + ] + }, + "dropdowns": { + "download": { + "title": "Download" + } + }, + "buttons": [ + { + "enabled": true, + "type": "export", + "title": "Export All to Terra", + "rightIcon": "external-link" + }, + { + "enabled": true, + "type": "export-to-seven-bridges", + "title": "Export All to Seven Bridges", + "rightIcon": "external-link" + }, + { + "enabled": true, + "type": "export-to-pfb", + "title": "Export to PFB", + "leftIcon": "datafile", + "rightIcon": "download" + }, + { + "enabled": true, + "type": "export-to-workspace", + "title": "Export to Workspace", + "leftIcon": "datafile", + "rightIcon": "download" + } + ], + "loginForDownload": true, + "guppyConfig": { + "dataType": "subject", + "nodeCountTitle": "Subjects", + "fieldMapping": [ + { "field": "disease_id", "name": "Disease ID" }, + { "field": "age_of_onset", "name": "Age of Onset" }, + { "field": "project_dbgap_accession_number", "name": "Project dbGaP Accession Number" }, + { "field": "project_dbgap_consent_text", "name":"Project dbGaP Consent Text"}, + { "field": "project_dbgap_phs", "name":"Project dbGaP Phs"} + ], + "manifestMapping": { + "resourceIndexType": "file", + "resourceIdField": "object_id", + "referenceIdFieldInResourceIndex": "_subject_id", + "referenceIdFieldInDataIndex": "_subject_id" + }, + "accessibleFieldCheckList": [ + "project_id" + ], + "accessibleValidationField": "project_id" + }, + "getAccessButtonLink": "https://dbgap.ncbi.nlm.nih.gov/", + "terraExportURL": "https://terra.biodatacatalyst.nhlbi.nih.gov/#import-data", + "terraTemplate": [ + "bdc", + "gen3" + ], + "sevenBridgesExportURL": "https://platform.sb.biodatacatalyst.nhlbi.nih.gov/import/windmill" + }, + { + "tabTitle": "File", + "charts": { + "data_type": { + "chartType": "stackedBar", + "title": "File Type" + }, + "data_format": { + "chartType": "stackedBar", + "title": "File Format" + } + }, + "filters": { + "tabs": [ + { + "title": "File", + "fields": [ + "project_id", + "data_category", + "data_type", + "data_format" + ], + "searchFields": [ + "file_name" + ] + } + ] + }, + "table": { + "enabled": true, + "fields": [ + "project_id", + "file_name", + "file_size", + "object_id" ] + }, + "loginForDownload": true, + "guppyConfig": { + "dataType": "file", + "fieldMapping": [ + { + "field": "object_id", + "name": "GUID" + }, + { + "field": "programs_name", + "name": "Program" + } + ], + "nodeCountTitle": "Files", + "manifestMapping": { + "resourceIndexType": "subject", + "resourceIdField": "_subject_id", + "referenceIdFieldInResourceIndex": "object_id", + "referenceIdFieldInDataIndex": "object_id" + }, + "accessibleFieldCheckList": [ + "project_id" + ], + "accessibleValidationField": "project_id", + "downloadAccessor": "object_id" + }, + "buttons": [ + { + "enabled": true, + "type": "file-manifest", + "title": "Download Manifest", + "leftIcon": "datafile", + "rightIcon": "download", + "fileName": "file-manifest.json", + "dropdownId": "download" + }, + { + "enabled": true, + "type": "export-files-to-workspace", + "title": "Export to Workspace", + "leftIcon": "datafile", + "rightIcon": "download" + }, + { + "enabled": true, + "type": "export-files-to-pfb", + "title": "Export All to PFB", + "rightIcon": "external-link" + }, + { + "enabled": true, + "type": "export-files", + "title": "Export All to Terra", + "rightIcon": "external-link" + }, + { + "enabled": true, + "type": "export-files-to-seven-bridges", + "title": "Export All to Seven Bridges", + "rightIcon": "external-link" + } + ], + "terraExportURL": "https://terra.biodatacatalyst.nhlbi.nih.gov/#import-data", + "terraTemplate": [ + "bdc", + "gen3" + ], + "sevenBridgesExportURL": "https://platform.sb.biodatacatalyst.nhlbi.nih.gov/import/windmill", + "enableLimitedFilePFBExport": { + "sourceNodeField": "source_node" + }, + "dropdowns": {} + }, + { + "tabTitle": "Imaging Studies", + "charts": {}, + "filters": { + "tabs": [ + { + "title": "Study Properties", + "searchFields": [ + "study_uid" + ], + "fields": [ + "study_modality", + "study_description", + "age_at_imaging", + "study_year", + "study_year_shifted", + "body_part_examined", + "days_from_study_to_pos_covid_test", + "days_from_study_to_neg_covid_test", + "project_id" + ] + } + ] + }, + "table": { + "enabled": true, + "fields": [ + "submitter_id", + "study_uid", + "case_ids", + "object_id", + "project_id" + ], + "dicomViewerId": "submitter_id", + "dicomViewerUrl": "ohif-viewer", + "linkFields": [ + "submitter_id" + ] + }, + "dropdowns": { + "download-table": { + "title": "Download Table" + } + }, + "buttons": [ + { + "enabled": true, + "type": "data-json", + "title": "JSON", + "dropdownId": "download-table", + "leftIcon": "datafile", + "rightIcon": "download", + "fileName": "Imaging_Studies_table.json" + }, + { + "enabled": true, + "type": "data-tsv", + "title": "TSV", + "dropdownId": "download-table", + "leftIcon": "datafile", + "rightIcon": "download", + "fileName": "Imaging_Studies_table.tsv" + }, + { + "enabled": true, + "type": "data-csv", + "title": "CSV", + "dropdownId": "download-table", + "leftIcon": "datafile", + "rightIcon": "download", + "fileName": "Imaging_Studies_table.csv" + }, + { + "enabled": true, + "type": "manifest", + "title": "Download File Manifest for Imaging Studies", + "leftIcon": "datafile", + "rightIcon": "download", + "fileName": "imaging_study_manifest.json" + }, + { + "enabled": false, + "type": "export-to-workspace", + "title": "Export to Workspace", + "leftIcon": "datafile", + "rightIcon": "download" + } + ], + "loginForDownload": true, + "guppyConfig": { + "dataType": "imaging_study", + "nodeCountTitle": "Imaging Studies", + "fieldMapping": [ + { + "field": "project_id", + "name": "Project ID" + }, + { + "field": "study_uid", + "name": "Study UID" + }, + { + "field": "submitter_id", + "name": "Browse in DICOM viewer" + }, + { + "field": "case_ids", + "name": "Patient ID" + } + ], + "manifestMapping": { + "resourceIndexType": "data_file", + "resourceIdField": "object_id", + "referenceIdFieldInResourceIndex": "_imaging_study_id", + "referenceIdFieldInDataIndex": "_imaging_study_id", + "useFilterForCounts": false + }, + "accessibleFieldCheckList": [ + "project_id" + ], + "accessibleValidationField": "project_id" + } + } + ] + } diff --git a/ci/perf/values/values.yaml b/ci/perf/values/values.yaml new file mode 100644 index 000000000..198ac58ad --- /dev/null +++ b/ci/perf/values/values.yaml @@ -0,0 +1,523 @@ +global: + # This will be dynamically replaced by env_setup.sh script. + hostname: + dictionaryUrl: https://s3.amazonaws.com/dictionary-artifacts/anvil/master/schema.json + pdb: true + aws: + externalSecrets: + # -- (bool) Whether to use External Secrets for aws config. + enabled: true + # -- (String) Name of Secrets Manager secret. + externalSecretAwsCreds: ci-aws-config + secretStoreServiceAccount: + # -- (bool) Set true if deploying to AWS and want to use service account and IAM role instead of aws keys. Must provide role-arn. + enabled: true + # -- (string) Name of the service account to create + name: secret-store-sa + # -- (string) AWS Role ARN for Secret Store to use + roleArn: arn:aws:iam::707767160287:role/devplanetv2-external-secrets-sa + sower_role: sower-jobs-role + externalSecrets: + deploy: true + dbCreate: true + manifestGlobalExtraValues: + # This will be dynamically replaced by env_setup.sh script. + fence_url: https://hostname/user + google_enabled: true + tierAccessLevel: libre + +mutatingWebhook: + # -- (bool) Whether to deploy the mutating webhook service. + enabled: false + +postgresql: + primary: + persistence: + enabled: true + size: 1Gi + image: + registry: quay.io + repository: cdis/postgresql + tag: 14.5.0-debian-11-r35 + +hatchery: + replicaCount: 2 + autoscaling: + enabled: true + image: + tag: master + hatchery: + containers: + - target-port: 8888 + cpu-limit: '0.5' + memory-limit: 1Gi + name: "(Tutorial) Bacpac Synthetic Data Analysis Notebook" + image: "quay.io/cdis/heal-notebooks:bacpac__10670ad79b4488cfa9f1b3681ce52bc3f1139b5a" + env: + # This will be dynamically replaced by env_setup.sh script. + FRAME_ANCESTORS: https:// + args: + - "--NotebookApp.base_url=/lw-workspace/proxy/" + - "--NotebookApp.password=''" + - "--NotebookApp.token=''" + - "--NotebookApp.quit_button=False" + command: + - start-notebook.sh + path-rewrite: "/lw-workspace/proxy/" + use-tls: 'false' + ready-probe: "/lw-workspace/proxy/" + lifecycle-post-start: + - "/bin/sh" + - "-c" + - export IAM=`whoami`; rm -rf /home/$IAM/pd/dockerHome; rm -rf /home/$IAM/pd/lost+found; ln -s /data /home/$IAM/pd/; true + user-uid: 1000 + fs-gid: 100 + +indexd: + replicaCount: 2 + autoscaling: + enabled: true + resources: + requests: + memory: "105Mi" + cpu: "15m" + # This will be dynamically replaced by env_setup.sh script. + defaultPrefix: "" + enabled: true + image: + repository: "quay.io/cdis/indexd" + tag: "master" + externalSecrets: + createK8sServiceCredsSecret: true + secrets: + userdb: + # This will be dynamically replaced by env_setup.sh script. + ssj: + # This will be dynamically replaced by env_setup.sh script. + gateway: + + +metadata: + replicaCount: 2 + autoscaling: + enabled: true + image: + repository: quay.io/cdis/metadata-service + tag: 'master' + resources: + # -- (map) The amount of resources that the container requests + requests: + memory: 25Mi + enabled: true + externalSecrets: + createK8sMetadataSecret: true + useAggMds: "True" + aggMdsNamespace: default + aggMdsConfig: | + { + "gen3_commons": { + "HEAL": { + "mds_url": "http://revproxy-service", + "commons_url": "http://revproxy-service" + } + }, + "adapter_commons": {} + } + +maxUnavailable: "" +elasticsearch: + maxUnavailable: "" + volumeClaimTemplate: + resources: + requests: + storage: 1Gi + +sower: + replicaCount: 2 + autoscaling: + enabled: true + image: + repository: quay.io/cdis/sower + tag: 'master' + enabled: true + serviceAccount: + # -- (bool) Specifies whether a service account should be created. + create: true + # -- (string) The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "sower-service-account" + externalSecrets: + createK8sPelicanServiceSecret: false + pelicanserviceG3auto: ci-pelicanservice-g3auto + createK8sSowerJobsSecret: true + sowerConfig: + - name: pelican-export + action: export + container: + name: job-task + image: quay.io/cdis/pelican-export:master + pull_policy: Always + env: + - name: DICTIONARY_URL + valueFrom: + configMapKeyRef: + name: manifest-global + key: dictionary_url + - name: GEN3_HOSTNAME + valueFrom: + configMapKeyRef: + name: manifest-global + key: hostname + - name: ROOT_NODE + value: subject + - name: DB_HOST + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: host + - name: DB_DATABASE + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: database + - name: DB_USER + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: username + - name: DB_PASS + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: password + - name: SHEEPDOG + valueFrom: + secretKeyRef: + name: indexd-service-creds + key: sheepdog + volumeMounts: + - name: pelican-creds-volume + readOnly: true + mountPath: "/pelican-creds.json" + subPath: config.json + cpu-limit: "0.5" + memory-limit: 1Gi + volumes: + - name: pelican-creds-volume + secret: + secretName: pelicanservice-g3auto #pragma: allowlist secret + restart_policy: Never + - name: pelican-export-files + action: export-files + container: + name: job-task + image: quay.io/cdis/pelican-export:master + pull_policy: Always + env: + - name: DICTIONARY_URL + valueFrom: + configMapKeyRef: + name: manifest-global + key: dictionary_url + - name: GEN3_HOSTNAME + valueFrom: + configMapKeyRef: + name: manifest-global + key: hostname + - name: ROOT_NODE + value: file + - name: EXTRA_NODES + value: "" + - name: DB_HOST + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: host + - name: DB_DATABASE + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: database + - name: DB_USER + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: username + - name: DB_PASS + valueFrom: + secretKeyRef: + name: sheepdog-dbcreds + key: password + - name: SHEEPDOG + valueFrom: + secretKeyRef: + name: indexd-service-creds + key: sheepdog + volumeMounts: + - name: pelican-creds-volume + readOnly: true + mountPath: "/pelican-creds.json" + subPath: config.json + cpu-limit: "0.5" + memory-limit: 1Gi + volumes: + - name: pelican-creds-volume + secret: + secretName: pelicanservice-g3auto #pragma: allowlist secret + restart_policy: Never + - name: batch-export + action: batch-export + activeDealineSeconds: 600 + serviceAccountName: sower-service-account + container: + name: job-task + image: quay.io/cdis/batch-export:master + pull_policy: Always + env: + - name: BUCKET + value: sower-jobs-bucket + - name: GEN3_HOSTNAME + valueFrom: + configMapKeyRef: + name: manifest-global + key: hostname + cpu-limit: "1" + memory-limit: 1Gi + restart_policy: Never + +ssjdispatcher: + replicaCount: 2 + autoscaling: + enabled: true + image: + repository: quay.io/cdis/ssjdispatcher + tag: 'master' + enabled: true + indexing: quay.io/cdis/indexs3client:master + ssjcreds: + # This will be dynamically replaced by env_setup.sh script. + sqsUrl: https://sqs.us-east-1.amazonaws.com/707767160287/gen3-helm-data-upload-bucket + # This will be dynamically replaced by env_setup.sh script. + jobPattern: "s3://gen3-helm-data-upload-bucket//*" + # This will be dynamically replaced by env_setup.sh script. + jobPassword: "replace_with_password" + # This will be dynamically replaced by env_setup.sh script. + metadataservicePassword: "replace_with_password" + serviceAccount: + create: true + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::707767160287:role/ssjdispatcher-role + name: "ssjdispatcher-service-account" + jobServiceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::707767160287:role/ssjdispatcher-role-job + externalSecrets: + createK8sSsjdispatcherSecret: true + # This will be dynamically replaced by env_setup.sh script. + gen3Namespace: default + +sheepdog: + replicaCount: 2 + autoscaling: + enabled: true + resources: + requests: + memory: "105Mi" + cpu: "15m" + enabled: true + authNamespace: '' + image: + repository: "quay.io/cdis/sheepdog" + tag: "master" + # This will be dynamically replaced by env_setup.sh script. + fenceUrl: https://hostname/user + +peregrine: + replicaCount: 2 + autoscaling: + enabled: true + resources: + requests: + memory: "105Mi" + cpu: "15m" + enabled: true + image: + repository: "quay.io/cdis/peregrine" + tag: "master" + pullPolicy: Always + +requestor: + replicaCount: 2 + autoscaling: + enabled: true + enabled: true + image: + repository: "quay.io/cdis/requestor" + tag: "master" + +audit: + replicaCount: 2 + autoscaling: + enabled: true + env: + - name: ARBORIST_URL + valueFrom: + configMapKeyRef: + name: manifest-global + key: arborist_url + optional: true + externalSecrets: + createK8sAuditSecret: true + image: + repository: quay.io/cdis/audit-service + tag: 'master' + server: + pull_from_queue: true + debug: true + sqs: + region: us-east-1 + # This will be dynamically replaced by env_setup.sh script. + url: https://sqs.us-east-1.amazonaws.com/707767160287/audit-service-sqs + enabled: true + +cedar: + enabled: true + autoscaling: + enabled: true + externalSecrets: + cedarG3auto: ci-cedar-g3auto + cedarAPIKey: ci-cedar-directory-id + cedarDirectoryId: ci-cedar-api-key + image: + repository: "quay.io/cdis/cedar-wrapper-service" + tag: "master" + +ambassador: + replicaCount: 2 + autoscaling: + enabled: true + enabled: true + image: + repository: "quay.io/datawire/ambassador" + tag: "1.4.2" + +arborist: + replicaCount: 2 + autoscaling: + enabled: true + resources: + requests: + memory: "105Mi" + cpu: "15m" + enabled: true + image: + repository: "quay.io/cdis/arborist" + tag: "master" + +revproxy: + replicaCount: 2 + autoscaling: + enabled: true + ingress: + enabled: true + annotations: + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/tags: Environment=devplanetv2 + alb.ingress.kubernetes.io/certificate-arn: 'arn:aws:acm:us-east-1:707767160287:certificate/f427a646-94fd-4b91-8bcd-80f5920c1e46' + alb.ingress.kubernetes.io/group.name: devplanetv2 + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' + alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' + alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-FIPS-2023-04 + className: alb + hosts: + # This will be dynamically replaced by env_setup.sh script. + - host: HOSTNAME + paths: + - path: / + pathType: Prefix + resources: + requests: + memory: "105Mi" + cpu: "15m" + + +wts: + replicaCount: 2 + autoscaling: + enabled: true + enabled: true + image: + repository: "quay.io/cdis/workspace-token-service" + tag: "master" + externalSecrets: + createWtsOidcClientSecret: true + createK8sWtsSecret: true + +manifestservice: + replicaCount: 2 + autoscaling: + enabled: true + image: + repository: quay.io/cdis/manifestservice + tag: "krishnaa05-patch-1" + env: + - name: REQUESTS_CA_BUNDLE + value: /etc/ssl/certs/ca-certificates.crt + - name: MANIFEST_SERVICE_CONFIG_PATH + value: /var/gen3/config/config.json + - name: GEN3_DEBUG + value: "False" + manifestserviceG3auto: + # This will be dynamically replaced by env_setup.sh script. + hostname: + bucketName: gen3-helm-manifest-bucket + prefix: 'test' + externalSecrets: + # -- (string) Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. + createK8sManifestServiceSecret: true + + +ohif-viewer: + replicaCount: 2 + autoscaling: + enabled: true + enabled: true + image: + repository: "quay.io/cdis/ohif-viewer" + tag: "gen3-v3.8.0" + +dicom-server: + replicaCount: 2 + autoscaling: + enabled: true + enabled: false + image: + repository: "quay.io/cdis/gen3-orthanc" + tag: "master" + +gen3-user-data-library: + replicaCount: 2 + autoscaling: + enabled: true + enabled: true + image: + repository: "quay.io/cdis/gen3-user-data-library" + tag: "main" + externalSecrets: + createK8sGen3UserDataLibrarySecret: true + +access-backend: + enabled: false + +orthanc: + replicaCount: 2 + autoscaling: + enabled: true + enabled: true + image: + repository: "quay.io/cdis/gen3-orthanc" + tag: "orthancteam-master-gen3-24.3.5" + # -- (map) External Secrets settings. + externalSecrets: + # -- (string) Will override the name of the aws secrets manager secret. Default is "orthanc-s3-g3auto" + orthancS3G3Auto: ci-orthanc-s3-g3auto