Fix zensical build warnings

.github/workflows/auto_updates.yml

@@ -0,0 +1,89 @@
+name: Daily API Update Check
+on:
+  # allow manual runs
+  workflow_dispatch:
+
+  # also schedule daily
+  schedule:
+    # 9:47AM central time M-F
+    # why so specific? probably a lot of people have things scheduled 
+    # on the half-hour so this avoids load times at 9:30 / 10
+    - cron: '47 9 * * 1-5'
+      timezone: "America/Chicago"
+permissions:
+  contents: write
+  pull-requests: write
+jobs:
+  api-update-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          # this uses the default repo-scoped, job-used GITHUB_TOKEN
+          # and saves it as git creds for this run only (it's
+          # removed in a post cleanup). this ensures
+          # future git commands are authed
+          persist-credentials: true
+
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email github-actions[bot]@users.noreply.github.com
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y curl
+
+      - name: Download OpenAPI spec
+        run: |
+          curl -L https://raw.githubusercontent.com/uc-cdis/gen3-ai/refs/heads/main/docs/api.html -o ./docs/gen3-resources/user-guide/api/gen3_ai_api.html
+
+      - name: Check for changes
+        id: check_changes
+        run: |
+          git add ./docs/gen3-resources/user-guide/api/gen3_ai_api.html
+
+          # if existing secrets just moved line numbers
+          # note: if detect-secrets actually fails on a NEW false
+          # positive secret from the docs generation,
+          # this workflow will still fail and require human intervention
+          git add .secrets.baseline
+
+          if git diff --cached --quiet; then
+            echo "changes=false" >> $GITHUB_OUTPUT
+          else
+            echo "changes=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Create Pull Request If Changes Detected
+        if: steps.check_changes.outputs.changes == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          BRANCH_NAME="chore/update-api-docs-${{ github.run_id }}"
+          git checkout -b $BRANCH_NAME
+
+          git add ./docs/gen3-resources/user-guide/api/gen3_ai_api.html
+
+          # if existing secrets just moved line numbers
+          # note: if detect-secrets actually fails on a NEW false
+          # positive secret from the docs generation,
+          # this workflow will still fail and require human intervention
+          git add .secrets.baseline
+
+          git commit -m "docs(api): update gen3_ai_api.html from upstream"
+          git push origin $BRANCH_NAME
+
+          # gh CLI is pre-installed in ubuntu-latest
+          # NOTE: --draft is required b/c the default GITHUB_TOKEN (which is used here)
+          #       will not trigger future `on: push` actions in the resulting PR. This is
+          #       a GH-required protection to prevent accidental infinite loops. To work around
+          #       this, we make the PR a draft, require a real person to manually "mark it ready"
+          #       and then the other workflow actions should run b/c we explicitly
+          #       tell it to run on a PR "ready" trigger.
+          gh pr create \
+            --title "docs(api): update API documentation" \
+            --body "Automated update of APIs docs from upstream repositories. Mark as 'Ready for Review' to start automated checks!" \
+            --head $BRANCH_NAME \
+            --draft

.github/workflows/publish.yml

@@ -0,0 +1,37 @@
+name: Publish Documentation
+on:
+  # any updates to main/master will actually publish to Pages
+  push:
+    branches:
+      - master
+      - main
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+jobs:
+  publish:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/configure-pages@v6
+      - uses: actions/checkout@v6
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email github-actions[bot]@users.noreply.github.com
+      - name: Add CNAME file
+        run: echo 'docs.gen3.org' > ./docs/CNAME
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
+      - uses: actions/setup-python@v6
+        with:
+          python-version: 3.x
+      - run: pip install zensical
+      - run: zensical build --clean
+      - uses: actions/upload-pages-artifact@v5
+        with:
+          path: site
+      - uses: actions/deploy-pages@v5
+        id: deployment

.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: Test Documentation
+on:
+  # push will run on every pushed commit to any branch
+  # EXCEPT to master and main, which the `publish.yml` handles
+  push:
+    branches-ignore:
+      - master
+      - main
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/configure-pages@v6
+      - uses: actions/checkout@v6
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email github-actions[bot]@users.noreply.github.com
+      - name: Add CNAME file
+        run: echo 'docs.gen3.org' > ./docs/CNAME
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
+      - uses: actions/setup-python@v6
+        with:
+          python-version: 3.x
+      - run: pip install zensical
+      - run: zensical build --clean

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ repos:
       - id: detect-secrets
         args: ["--baseline", ".secrets.baseline"]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.5.0
+    rev: v6.0.0
     hooks:
       - id: no-commit-to-branch
         args: [--branch, develop, --branch, master, --branch, main, --pattern, release/.*]

docs/gen3-resources/about.md

@@ -4,6 +4,8 @@ Gen3 is a data platform for building data commons and data meshes. The Gen3 plat
 
 You can find more information about Gen3 on [gen3.org](https://gen3.org/).
 
+You can find the source code for Gen3 on [GitHub][gen3 github].
+
 ## Powered by Gen3
 
 Gen3 is used by dozen of projects around the world to manage and analyze biomedical data.  You can find an incomplete list of current Gen3 projects [here](https://gen3.org/powered-by-gen3/).
@@ -30,11 +32,4 @@ To reach a member of the CTDS User Services Team you may contact us via email at
 
 
 
-[code of conduct]: https://forums.gen3.org/t/faq-guidelines/5
-[contact page]: ../contact.md
-[gen3 youtube]: https://www.youtube.com/channel/UCMCwQy4EDd1BaskzZgIOsNQ/featured
 [gen3 github]: https://github.com/uc-cdis
-[gen3 publications]: https://gen3.org/gen3refs/
-[community page]: https://gen3.org/community/events/
-[release notes]: https://github.com/uc-cdis/cdis-manifest/tree/master/releases
-[contributor guidelines]: developer-guide/contribute.md

docs/gen3-resources/glossary.md

@@ -11,7 +11,7 @@ A Common Services Operations Center is an operations center operated by a common
 ## Crosswalk
 Typically, used for linking patients from across data commons where some patient data exists in commons A and additional data exists in commons B. This linkage enables metadata associations across commons and the promise of richer datasets.  Crosswalks can be made for several types of metadata and are recorded in the metadata service. An example of how to set this up is found [here][crosswalk setup].
 ## Data Commons
-A data commons co-locates data with cloud computing infrastructure and commonly used software services, tools, and applications for managing, integrating, analyzing and sharing data that are exposed through web portals and APIs to create an interoperable resource for a research community. A data commons provides services so that the data is findable, accessible, interoperable and reusable (FAIR)
+A data commons co-locates data with cloud computing infrastructure and commonly used software services, tools, and applications for managing, integrating, analyzing and sharing data that are exposed through web portals and APIs to create an interoperable resource for a research community. A data commons provides services so that the data is findable, accessible, interoperable and reusable (FAIR).
 ## Data Dictionary
 Every Gen3 data commons employs a data model, which serves to describe, organize, and harmonize data sets submitted by different users. Data harmonization facilitates cross-project analyses and is thus one of the pillars of the data commons paradigm. The data model organizes experimental metadata variables, “properties”, into linked categories, “nodes”, through the use of a data dictionary. The data dictionary lists and describes all nodes in the data model, as well as defines and describes the properties in each node. A Gen3 Data Dictionary is specified by a YAML file per node. Additional details on Gen3 data dictionaries can be found [here][data dictionary overview].
 ## Data Hub
@@ -42,7 +42,7 @@ Edge nodes may be created at clinics, labs, hospitals, or academic institutions
 ## ETL
 Structured data submitted to commons are stored in PostgreSQL. Querying data from PostgreSQL with multiple join is painful and inefficient. So, we use ElasticSearch as a place to store materialized dataset. Extract-transform-load (ETL) is a process that creates the materialized data from PostgreSQL and store them in ElasticSearch.  This is accomplished via the Tube microservice. More details of running an ETL can be found [here](operator-guide/customize-search.md#etl).
 ## FAIR Data
-FAIR data are data which are findable, accessible, interoperable, and reusable[12]. There is now an extensive literature on FAIR data.
+FAIR data are data which are findable, accessible, interoperable, and reusable. There is now an extensive literature on FAIR data.
 ## Framework Services
 Framework Services or Data Commons Framework (DCF) Services is the term used by Gen3 to refer to data mesh services in the narrow middle architecture, for data meshes, such as the NCI Cancer Research Data Commons. These are set of standards-based services with open APIs for authentication, authorization, creating and accessing FAIR data objects, and for working with bulk structured data in machine-readable, self-contained format.
 

docs/gen3-resources/operator-guide/customize-frontend.md

@@ -1,4 +1,4 @@
-# Customize appearance of the front end
+f# Customize appearance of the front end
 
 Below we show a few examples of how to customize the Gen3 Data Portal.
 
@@ -62,8 +62,8 @@ The "tooltip" shows text upon hovering over the icon.
 
 ![navigationbar][navigationbar image]
 
-* [Review the code to edit icon, link, color, tooltip, and name of the navigation items][gitops.json navbar].
-* Adding a new icon requires saving the icon in [this repository][icons] and [in this file][icons inex].
+* [Review the code to edit icon, link, color, tooltip, and name of the navigation items](https://github.com/uc-cdis/cdis-manifest/blob/551f0963e60f6000ae8b9987592495406a031c81/gen3.datacommons.io/portal/gitops.json#L84-L134).
+* Adding a new icon requires saving the icon in [this repository][icons] and [in this file][icons index].
 
 ## Data Commons or Mesh Title
 

docs/gen3-resources/operator-guide/customize-search.md

@@ -101,10 +101,18 @@ To view the MDS for the Gen3 Data Hub you can go [here][gen3 data hub mds]. You
         },
 ```
 
-Instructions for working with the API are found [here][mds api].
+<!-- [] -->
 
 
 
+
+
+
+
+
+Instructions for working with the API are found [in the swagger docs][mdsapi].
+
+
 ### Aggregated Metadata Service
 An [Aggregated metadata service (aggMDS)][aggmds github] caches the metadata from two or more metadata sources to provide a unified view of the commons on the discovery page.
 
@@ -191,7 +199,7 @@ Setting up a functioning mesh where you can access files from individual commons
 [aggmds github]: https://github.com/uc-cdis/metadata-service/blob/master/docs/config_agg_mds.md
 [sdk for discovery page]: https://github.com/uc-cdis/gen3sdk-python/blob/master/gen3/cli/discovery.py
 [gen3 data hub mds]: https://gen3.datacommons.io/mds/metadata?data=True&_guid_type=discovery_metadata
-[mds api]: https://petstore.swagger.io/?url=https://raw.githubusercontent.com/uc-cdis/metadata-service/master/docs/openapi.yaml
+[mdsapi]: https://petstore.swagger.io/?url=https://raw.githubusercontent.com/uc-cdis/metadata-service/master/docs/openapi.yaml
 [aggmds sync]: https://github.com/uc-cdis/cloud-automation/blob/master/kube/services/jobs/metadata-aggregate-sync-job.yaml
 [BRH]: https://brh.data-commons.org/
 [BRH aggregated_config.json]: https://github.com/uc-cdis/cdis-manifest/blob/master/brh.data-commons.org/metadata/aggregate_config.json

docs/gen3-resources/operator-guide/helm/helm-deploy-production-example.md

@@ -265,7 +265,6 @@ spec:
 
 
 <!-- Reference Links -->
-[argo wrapper]: https://github.com/uc-cdis/argo-wrapper
 [aws cli user guide]: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html
 [terraform user guide]: https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli
 [kubectl]: https://kubernetes.io/docs/tasks/tools/

docs/gen3-resources/operator-guide/index.md

@@ -48,16 +48,12 @@ Compose-services was used to deploy Gen3 at a small scale, for experimental comm
 [Although we do not recommend using this, you can explore Gen3 compose-services here][compose-services]
 
 <!-- Links -->
-[Gen3 User Guide]: ../user-guide/index.md
-[Gen3 Developer Guide]: ../developer-guide/index.md
 [cloud-automation]: https://github.com/uc-cdis/cloud-automation/blob/master/doc/csoc-free-commons-steps.md
 [compose-services]: https://github.com/uc-cdis/compose-services/tree/master
 [helm guide]: helm/index.md
 [deploy overview]: helm/helm-deploy-overview.md
 [prerequisites]: prerequisites.md
 [infrastructure as code]: iac-overview.md
-[ssl]: ssl-cert.md
-[secrets]: secrets-mgr.md
 [authentication methods]: gen3-authn-methods.md
 [data dictionary]: create-data-dictionary.md
 [submit data]: submit-structured-data.md

docs/gen3-resources/operator-guide/submit-structured-data.md

@@ -194,5 +194,4 @@ If the submission throws errors or claims the submission to be invalid, it will
 [Fence programs and projects]: https://github.com/uc-cdis/fence/blob/master/docs/additional_documentation/user.yaml_guide.md#programs-and-projects-crud-access
 [Gen3 SDK]: https://uc-cdis.github.io/gen3sdk-python/_build/html/_modules/gen3/submission.html
 [toolbar submission]: img/Gen3_Toolbar_data_submission.png
-[submit data]: img/Gen3_Data_Submission_submit_data.png
 [upload file]: img/Gen3_Data_Submission_Use_Form.png

docs/gen3-resources/operator-guide/submit-unstructured-data.md

@@ -317,10 +317,10 @@ The below is an example of a Indexing Manifest File:
 
 | guid | File_name | File_size | md5sum | bucket_urls | auth |
 | --- | --- | --- | --- | --- | --- |
-| dg.4503/02... ...7103bbe | examplefile.txt | 34141 | c79... ...dbd | s3://nih-phs001416-c1/exfile.txt gs://nih-phs001416-c1/exfile.txt | [phs001416.c1] |
-| dg.4503/00... ...0211dfg | otherexamplefile.txt | 562256 | 65a... ...bca | s3://nih-nhlbi-topmed-released-phs001416-c1/otherexamplefile.txt gs://nih-nhlbi-topmed-released-phs001416-c1/otherexamplefile.txt | [phs001416.c1] |
-| dg.4503/00... ...7103bbe | examplefile.txt | 36564 | dca... ...774 | s3://nih-nhlbi-topmed-released-phs001416-c2/examplefile.txt gs://nih-nhlbi-topmed-released-phs001416-c2/examplefile.txt | [phs001416.c2] |
-| dg.4503/01... ...0410nnd | otherexamplefile.txt | 2675 | 742... ...f1b | s3://nih-nhlbi-topmed-released-phs001416-c2/otherexamplefile.txt gs://nih-nhlbi-topmed-released-phs001416-c2/otherexamplefile.txt | [phs001416.c2] |
+| dg.4503/02... ...7103bbe | examplefile.txt | 34141 | c79... ...dbd | s3://nih-phs001416-c1/exfile.txt gs://nih-phs001416-c1/exfile.txt | \[phs001416.c1] |
+| dg.4503/00... ...0211dfg | otherexamplefile.txt | 562256 | 65a... ...bca | s3://nih-nhlbi-topmed-released-phs001416-c1/otherexamplefile.txt gs://nih-nhlbi-topmed-released-phs001416-c1/otherexamplefile.txt | \[phs001416.c1] |
+| dg.4503/00... ...7103bbe | examplefile.txt | 36564 | dca... ...774 | s3://nih-nhlbi-topmed-released-phs001416-c2/examplefile.txt gs://nih-nhlbi-topmed-released-phs001416-c2/examplefile.txt | \[phs001416.c2] |
+| dg.4503/01... ...0410nnd | otherexamplefile.txt | 2675 | 742... ...f1b | s3://nih-nhlbi-topmed-released-phs001416-c2/otherexamplefile.txt gs://nih-nhlbi-topmed-released-phs001416-c2/otherexamplefile.txt | \[phs001416.c2] |
 
 
 ### 6. Submit file Indexing Manifest to Indexd

docs/gen3-resources/tools/data-client.md

@@ -482,7 +482,6 @@ If this happens (and you are still authorized to access the data), you can downl
 
 <!--Gen3 client download -->
 [Gen3 Client]: https://github.com/uc-cdis/cdis-data-client/releases/latest
-[sdk_github]: https://github.com/uc-cdis/gen3sdk-python
 [PATH]: data-client.md#working-from-the-command-line
 [img create API key]: img/Gen3_Keys.png
 

docs/gen3-resources/user-guide/api/README.txt

@@ -0,0 +1,6 @@
+The API HTML files here are auto-generated. 
+
+DO NOT MODIFY ANYTHING IN THIS FOLDER BY HAND.
+
+If there is a mistake or you have suggestions,
+please create an issue or reach out and we will fix it upstream.