From 5f02683f46b4e84f20f3b848b8476791cfc99dd2 Mon Sep 17 00:00:00 2001 From: avantol Date: Mon, 27 Apr 2026 14:43:55 -0500 Subject: [PATCH 1/5] feat(api): publish AI API docs, workflow to automate updates, switch to zensical Co-authored-by: Copilot --- .github/workflows/api-docs.yaml | 58 ++ .github/workflows/docs-ci.yaml | 31 +- .pre-commit-config.yaml | 2 +- docs/gen3-resources/user-guide/api/README.txt | 6 + .../user-guide/api/gen3_ai_api.html | 937 ++++++++++++++++++ docs/gen3-resources/user-guide/using-api.md | 4 +- docs/markdown.md | 98 ++ mkdocs.yml | 159 --- to-update-docs-gen3.md | 69 +- zensical.toml | 453 +++++++++ 10 files changed, 1586 insertions(+), 231 deletions(-) create mode 100644 .github/workflows/api-docs.yaml create mode 100644 docs/gen3-resources/user-guide/api/README.txt create mode 100644 docs/gen3-resources/user-guide/api/gen3_ai_api.html create mode 100644 docs/markdown.md delete mode 100644 mkdocs.yml create mode 100644 zensical.toml diff --git a/.github/workflows/api-docs.yaml b/.github/workflows/api-docs.yaml new file mode 100644 index 0000000..59e3400 --- /dev/null +++ b/.github/workflows/api-docs.yaml @@ -0,0 +1,58 @@ +name: Daily API Update Check +on: + workflow_dispatch: + schedule: + # 9:47AM central time M-F + # why so specific? probably a lot of people have things scheduled + # on the half-hour so this avoids load times at 9:30 / 10 + - cron: '47 9 * * 1-5' + timezone: "America/Chicago" +permissions: + contents: write +jobs: + api-update-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y curl + + - name: Download OpenAPI spec + run: | + # TODO: use main branch once this is merged + curl -L https://raw.githubusercontent.com/uc-cdis/gen3-ai/refs/heads/feat/docs/docs/api.html -o ./docs/gen3-resources/user-guide/api/gen3_ai_api.html + + - name: Check for changes + id: check_changes + run: | + git add ./docs/gen3-resources/user-guide/api/gen3_ai_api.html + if git diff --cached --quiet; then + echo "changes=false" >> $GITHUB_OUTPUT + else + echo "changes=true" >> $GITHUB_OUTPUT + fi + + - name: Create Pull Request If Changes Detected + if: steps.check_changes.outputs.changes == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + BRANCH_NAME="chore/update-api-docs-${{ github.run_id }}" + + git checkout -b $BRANCH_NAME + git add ./docs/gen3-resources/user-guide/api/gen3_ai_api.html + git commit -m "docs(api): update gen3_ai_api.html from upstream" + git push origin $BRANCH_NAME + + # gh CLI is pre-installed in ubuntu-latest + gh pr create \ + --title "docs(api): update API documentation" \ + --body "Automated update of APIs docs from upstream repositories." \ + --head $BRANCH_NAME diff --git a/.github/workflows/docs-ci.yaml b/.github/workflows/docs-ci.yaml index 933be3a..9bf51ad 100644 --- a/.github/workflows/docs-ci.yaml +++ b/.github/workflows/docs-ci.yaml @@ -1,31 +1,36 @@ -name: docs-ci +name: Documentation on: push: branches: - master - main permissions: - contents: write + contents: read + pages: write + id-token: write jobs: deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/configure-pages@v6 + - uses: actions/checkout@v6 - name: Configure Git Credentials run: | git config user.name github-actions[bot] git config user.email 41898282+github-actions[bot]@users.noreply.github.com - - uses: actions/setup-python@v5 - with: - python-version: 3.x - name: Add CNAME file run: echo 'docs.gen3.org' > ./docs/CNAME - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV - - uses: actions/cache@v4 + - uses: actions/setup-python@v6 + with: + python-version: 3.x + - run: pip install zensical + - run: zensical build --clean + - uses: actions/upload-pages-artifact@v5 with: - key: mkdocs-material-${{ env.cache_id }} - path: .cache - restore-keys: | - mkdocs-material- - - run: pip install mkdocs-material mkdocs-encryptcontent-plugin mkdocs-video - - run: mkdocs gh-deploy --force + path: site + - uses: actions/deploy-pages@v5 + id: deployment diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8e775c2..2ac962c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: - id: detect-secrets args: ["--baseline", ".secrets.baseline"] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 + rev: v6.0.0 hooks: - id: no-commit-to-branch args: [--branch, develop, --branch, master, --branch, main, --pattern, release/.*] diff --git a/docs/gen3-resources/user-guide/api/README.txt b/docs/gen3-resources/user-guide/api/README.txt new file mode 100644 index 0000000..199ecd9 --- /dev/null +++ b/docs/gen3-resources/user-guide/api/README.txt @@ -0,0 +1,6 @@ +The API HTML files here are auto-generated. + +DO NOT MODIFY ANYTHING IN THIS FOLDER BY HAND. + +If there is a mistake or you have suggestions, +please create an issue or reach out and we will fix it upstream. \ No newline at end of file diff --git a/docs/gen3-resources/user-guide/api/gen3_ai_api.html b/docs/gen3-resources/user-guide/api/gen3_ai_api.html new file mode 100644 index 0000000..6ee1269 --- /dev/null +++ b/docs/gen3-resources/user-guide/api/gen3_ai_api.html @@ -0,0 +1,937 @@ + + + + + + Gen3 AI + + + + + + + + + +
API docs by Redocly

Gen3 AI (1.0.0)

Download OpenAPI specification:

Gen3 AI API.

+

Embeddings

Read embedding from collection

Read a single embedding from a specific collection.

+

Args: + collection_name: Name of the collection. + embedding_uuid: UUID of the embedding. + dal: Data access layer dependency.

+

Returns: + SingleEmbeddingResult

+

Raises: + HTTPException: 404 if the collection or embedding is not found.

+
Authorizations:
bearerAuth
path Parameters
collection_name
required
string (Collection Name)
embedding_uuid
required
string <uuid> (Embedding Uuid)

Responses

Response samples

Content type
application/json
{
  • "vector": [
    ],
  • "input_index": 0,
  • "embedding_id": "18e0b745-2b45-46cb-a826-bc9049d1152c",
  • "info": {
    }
}

Update embedding in collection

Update the embedding vector for a given collection and embedding ID.

+

Args: + request: The request object + collection_name: Name of the collections. + embedding_uuid: UUID of the embedding. + body: Request body containing the new embedding vector. + dal: Data access layer dependency.

+

Returns: + SingleEmbeddingResult containing the updated embedding.

+

Raises: + HTTPException: 404 if the collection is not found; 400 if update fails.

+
Authorizations:
bearerAuth
path Parameters
collection_name
required
string (Collection Name)
embedding_uuid
required
string <uuid> (Embedding Uuid)
Request Body schema: application/json
required
Array of Embedding (numbers) or Embedding (null) (Embedding)
Metadata (object) or Metadata (null) (Metadata)

Responses

Request samples

Content type
application/json
{
  • "embedding": [
    ],
  • "metadata": { }
}

Response samples

Content type
application/json
{
  • "vector": [
    ],
  • "input_index": 0,
  • "embedding_id": "18e0b745-2b45-46cb-a826-bc9049d1152c",
  • "info": {
    }
}

Delete embedding from collection

Delete an embedding from a specific collection.

+

Args: + request: The request object + collection_name: Name of the collections. + embedding_uuid: UUID of the embedding to delete. + dal: Data access layer dependency.

+

Returns: + None on success.

+

Raises: + HTTPException: 404 if the collection or embedding is not found.

+
Authorizations:
bearerAuth
path Parameters
collection_name
required
string (Collection Name)
embedding_uuid
required
string <uuid> (Embedding Uuid)

Responses

Response samples

Content type
application/json
{
  • "detail": [
    ]
}

Read all embeddings from collection

List all embeddings within a specific collection.

+

Args: + request: The request object + collection_name: Name of the collections. + no_embeddings_info: If True, omit the 'info' block in each embedding result. + dal: Data access layer dependency.

+

Returns: + EmbeddingResponseNoCollection containing all embeddings in the collection.

+

Raises: + HTTPException: 404 if the collection is not found.

+
Authorizations:
bearerAuth
path Parameters
collection_name
required
string (Collection Name)
query Parameters
no_embeddings_info
boolean (No Embeddings Info)
Default: false
page
integer (Page) >= 1
Default: 1
page_size
integer (Page Size) [ 100 .. 1000 ]
Default: 100

Responses

Response samples

Content type
application/json
{
  • "embeddings": [
    ],
  • "page": 0,
  • "page_size": 0,
  • "next_page": 0,
  • "prev_page": 0
}

Create embeddings in collection

TODO: implementaion for StringArrayInput and ai_model +TODO: auth related +TODO: work for authz_version

+

Create one or more embeddings in a specific collection.

+

This minimal implementation only accepts raw numeric vectors.

+

Args: + request: The request object + collection_name: Name of the collection. + body: Request body containing a list of embedding vectors. + ai_model: Optional model name; not used in this minimal version. + no_embeddings_info: If True, omit the 'info' block in each embedding result. + dal: Data access layer dependency.

+

Returns: + EmbeddingResponseNoCollection containing the created embeddings.

+

Raises: + HTTPException: 404 if collection is not found; 400 if dimensions mismatch.

+
Authorizations:
bearerAuth
path Parameters
collection_name
required
string (Collection Name)
query Parameters
Ai Model (string) or Ai Model (null) (Ai Model)
no_embeddings_info
boolean (No Embeddings Info)
Default: false
Request Body schema: application/json
required
required
Array of objects (Embeddings)

Responses

Request samples

Content type
application/json
{
  • "embeddings": [
    ]
}

Response samples

Content type
application/json
{
  • "embeddings": [
    ]
}

Embeddings (Bulk Read)

Read select embeddings from unknown collections

TODO: collections list is needed as return? +TODO: update dal.get_collection_by_id_bulk. remove duplicates?

+

Read a selection of embeddings by UUID across any collection.

+

Args: + request: The request object + embedding_uuids: List of embedding UUIDs to fetch. + no_embeddings_info: If True, omit the 'info' block for each embedding. + dal: Data access layer dependency.

+

Returns: + EmbeddingResponse including collection metadata for each embedding.

+
Authorizations:
bearerAuth
query Parameters
no_embeddings_info
boolean (No Embeddings Info)
Default: false
Request Body schema: application/json
required
Array
string <uuid>

Responses

Request samples

Content type
application/json
"embedding_uuid_0"

Response samples

Content type
application/json
{
  • "embeddings": [
    ],
  • "collections": [
    ]
}

Read select embeddings from collection

Read a selection of embeddings by UUID from a specific collection.

+

Args: + request: The request object + collection_name: Name of the collections. + embedding_uuids: List of embedding UUIDs to fetch. + no_embeddings_info: If True, omit the 'info' block for each embedding. + dal: Data access layer dependency.

+

Returns: + EmbeddingResponse containing the embeddings found in the specified collection.

+

Raises: + HTTPException: 404 if the collection is not found.

+
Authorizations:
bearerAuth
path Parameters
collection_name
required
string (Collection Name)
query Parameters
no_embeddings_info
boolean (No Embeddings Info)
Default: false
Request Body schema: application/json
required
Array
string <uuid>

Responses

Request samples

Content type
application/json
"embedding_uuid_0"

Response samples

Content type
application/json
{
  • "embeddings": [
    ]
}

Vectorstore Collections

Read all collections

List all existing collections.

+

Args: + request: The request object + dal: Data access layer dependency.

+

Returns: + PaginatedCollectionsResponse containing all collections.

+
Authorizations:
bearerAuth
query Parameters
page
integer (Page) >= 1
Default: 1
page_size
integer (Page Size) [ 1 .. 1000 ]
Default: 100
collection_name
required
string (Collection Name)

Responses

Response samples

Content type
application/json
{
  • "collections": [
    ],
  • "page": 0,
  • "page_size": 0,
  • "next_page": 0,
  • "prev_page": 0
}

Create collection

Create a new collection.

+

Args: + request: The request object + body: Request body containing collection_name, description, and dimensions. + dal: Data access layer dependency.

+

Returns: + CollectionModel for the created collection.

+
Authorizations:
bearerAuth
Request Body schema: application/json
required
collection_name
required
string (Collection Name)
Description (string) or Description (null) (Description)
dimensions
required
integer (Dimensions)

Responses

Request samples

Content type
application/json
{
  • "collection_name": "string",
  • "description": "string",
  • "dimensions": 0
}

Response samples

Content type
application/json
{
  • "id": 0,
  • "collection_name": "string",
  • "description": "string",
  • "dimensions": 0,
  • "created_at": "2019-08-24T14:15:22Z",
  • "updated_at": "2019-08-24T14:15:22Z",
  • "self": "string"
}

Read collection info

Read information about a specific collection.

+

Args: + collection_name: Name of the collection. + dal: Data access layer dependency.

+

Returns: + CollectionModel representing the collection.

+

Raises: + HTTPException: 404 if collection is not found.

+
Authorizations:
bearerAuth
path Parameters
collection_name
required
string (Collection Name)

Responses

Response samples

Content type
application/json
{
  • "id": 0,
  • "collection_name": "string",
  • "description": "string",
  • "dimensions": 0,
  • "created_at": "2019-08-24T14:15:22Z",
  • "updated_at": "2019-08-24T14:15:22Z",
  • "self": "string"
}

Update collection info

Update mutable metadata fields for a collection.

+

Args: + collection_name: Name of the collection to update. + body: Request body containing fields to update (e.g., description). + dal: Data access layer dependency.

+

Returns: + A simple success status dict.

+

Raises: + HTTPException: 404 if collection is not found.

+
Authorizations:
bearerAuth
path Parameters
collection_name
required
string (Collection Name)
Request Body schema: application/json
required
Description (string) or Description (null) (Description)

Responses

Request samples

Content type
application/json
{
  • "description": "string"
}

Response samples

Content type
application/json
null

Delete collection

Delete a collection by name.

+

Args: + collection_name: Name of the collection to delete. + dal: Data access layer dependency.

+

Returns: + None on success.

+

Raises: + HTTPException: 404 if collection is not found.

+
Authorizations:
bearerAuth
path Parameters
collection_name
required
string (Collection Name)

Responses

Response samples

Content type
application/json
{
  • "detail": [
    ]
}

Search embeddings in collection

TODO: support for ai_model +TODO: raw text search

+

Perform a vector search within a specific collection.

+

Args: + request: The request object + body: SearchRequestBody containing the query vector and parameters. + collection_name: Name of the collection to search. + ai_model: Optional model name; not used in this minimal implementation. + no_embeddings_info: If True, omit the 'info' block in each embedding result. + dal: Data access layer dependency.

+

Returns: + SearchResponseNocollection containing search hits for this collection.

+

Raises: + HTTPException: 404 if collection is not found; 400 if input is invalid.

+
Authorizations:
bearerAuth
path Parameters
collection_name
required
string (Collection Name)
query Parameters
Ai Model (string) or Ai Model (null) (Ai Model)
no_embeddings_info
boolean (No Embeddings Info)
Default: false
Request Body schema: application/json
required
required
Input (string) or Array of Input (numbers) (Input)
top_k
integer (Top K)
Default: 10
Range (number) or Range (null) (Range)
Filters (object) or Filters (null) (Filters)

Responses

Request samples

Content type
application/json
{
  • "input": "string",
  • "top_k": 10,
  • "range": 0,
  • "filters": {
    }
}

Response samples

Content type
application/json
{
  • "embeddings": [
    ],
  • "collections": [
    ]
}

Search embeddings across unknown collections

TODO: support for ai_model +TODO: how to handle diffs in dimensions? current logic is not sufficient.

+

Perform a vector search across multiple collections.

+

Args: + request: The request object + body: SearchRequestBody containing the query vector and parameters. + collections: Optional comma-separated list of collection names to restrict the search. + ai_model: Optional model name; not used in this minimal implementation. + no_embeddings_info: If True, omit the 'info' block in each embedding result. + dal: Data access layer dependency.

+

Returns: + SearchResponse containing search hits across collections.

+

Raises: + HTTPException: 400 if invalid collections are specified or input is invalid.

+
Authorizations:
bearerAuth
query Parameters
Collections (string) or Collections (null) (Collections)
Ai Model (string) or Ai Model (null) (Ai Model)
no_embeddings_info
boolean (No Embeddings Info)
Default: false
Request Body schema: application/json
required
required
Input (string) or Array of Input (numbers) (Input)
top_k
integer (Top K)
Default: 10
Range (number) or Range (null) (Range)
Filters (object) or Filters (null) (Filters)

Responses

Request samples

Content type
application/json
{
  • "input": "string",
  • "top_k": 10,
  • "range": 0,
  • "filters": {
    }
}

Response samples

Content type
application/json
{
  • "embeddings": [
    ],
  • "collections": [
    ]
}

Inference

Standard Open Responses API

See official spec for details (https://openresponses.org). This OpenAPI spec here is auto-generated.

+
Authorizations:
bearerAuth
Request Body schema: application/json
required
Model (string) or Model (null) (Model)
Input (string) or (Array of Input (ItemReferenceParam (object) or ReasoningItemParam (object) or UserMessageItemParam (object) or SystemMessageItemParam (object) or DeveloperMessageItemParam (object) or AssistantMessageItemParam (object) or FunctionCallItemParam (object) or FunctionCallOutputItemParam (object))) or Input (null) (Input)
Previous Response Id (string) or Previous Response Id (null) (Previous Response Id)
Array of Include (strings) or Include (null) (Include)
Array of Tools (objects) or Tools (null) (Tools)
ToolChoice2 (object) or ToolChoice3 (string) or ToolChoice4 (object) or Tool Choice (null) (Tool Choice)
MetadataParam (object) or null
TextParam (object) or null
Temperature (number) or Temperature (null) (Temperature)
Top P (number) or Top P (null) (Top P)
Presence Penalty (number) or Presence Penalty (null) (Presence Penalty)
Frequency Penalty (number) or Frequency Penalty (null) (Frequency Penalty)
Parallel Tool Calls (boolean) or Parallel Tool Calls (null) (Parallel Tool Calls)
Stream (boolean) or Stream (null) (Stream)

Whether to stream response events as server-sent events.

+
StreamOptionsParam (object) or null
Background (boolean) or Background (null) (Background)

Whether to run the request in the background and return immediately.

+
MaxOutputTokens (integer) or null
MaxToolCalls (integer) or null
ReasoningParam (object) or null
SafetyIdentifier (string) or null
PromptCacheKey (string) or null
TruncationEnum (string) or null
Instructions (string) or Instructions (null) (Instructions)
Store (boolean) or Store (null) (Store)

Whether to store the response so it can be retrieved later.

+
ServiceTierEnum (string) or null
TopLogprobs (integer) or null

Responses

Request samples

Content type
application/json
{
  • "model": "string",
  • "input": "string",
  • "previous_response_id": "string",
  • "include": [
    ],
  • "tools": [
    ],
  • "tool_choice": { },
  • "metadata": {
    },
  • "text": {
    },
  • "temperature": 0,
  • "top_p": 0,
  • "presence_penalty": 0,
  • "frequency_penalty": 0,
  • "parallel_tool_calls": true,
  • "stream": true,
  • "stream_options": {
    },
  • "background": true,
  • "max_output_tokens": 16,
  • "max_tool_calls": 1,
  • "reasoning": {
    },
  • "safety_identifier": "string",
  • "prompt_cache_key": "string",
  • "truncation": "auto",
  • "instructions": "string",
  • "store": true,
  • "service_tier": "auto",
  • "top_logprobs": 20
}

Response samples

Content type
Example
{
  • "id": "string",
  • "object": "response",
  • "created_at": 0,
  • "completed_at": 0,
  • "status": "string",
  • "incomplete_details": {
    },
  • "model": "string",
  • "previous_response_id": "string",
  • "instructions": "string",
  • "output": [
    ],
  • "error": {
    },
  • "tools": [
    ],
  • "tool_choice": {
    },
  • "truncation": "auto",
  • "parallel_tool_calls": true,
  • "text": {
    },
  • "top_p": 0,
  • "presence_penalty": 0,
  • "frequency_penalty": 0,
  • "top_logprobs": 0,
  • "temperature": 0,
  • "reasoning": {
    },
  • "usage": {
    },
  • "max_output_tokens": 0,
  • "max_tool_calls": 0,
  • "store": true,
  • "background": true,
  • "service_tier": "string",
  • "metadata": null,
  • "safety_identifier": "string",
  • "prompt_cache_key": "string"
}
+ + + + diff --git a/docs/gen3-resources/user-guide/using-api.md b/docs/gen3-resources/user-guide/using-api.md index a0b630c..aac6020 100644 --- a/docs/gen3-resources/user-guide/using-api.md +++ b/docs/gen3-resources/user-guide/using-api.md @@ -1,5 +1,7 @@ # Using the API +> We're working on getting official API documentation auto-generated from our backend and published here. Our first batch is a pre-alpha, proposed API for some of the [Gen3 AI support](./api/gen3_ai_api.html) we're building. + ## What does the API do? The application programming interface (API) can be a set of code, rules, functions, and URLs that allow apps, software, servers or more generally speaking, systems, to communicate with each other. The communication between APIs consists of requests and (data) responses, usually in .JSON format. @@ -9,8 +11,6 @@ The beauty of a Gen3 data commons is that all the functionality of the data comm > Note: The Gen3 commons uses GraphQL as the language for querying metadata across Gen3 Data Commons. To learn the basics of writing queries in GraphQL, please visit: [http://graphql.org/learn][learn GraphQL]. You can also try out creating and executing GraphQL queries in the [Data Portal Query Page][Query_page_instructions]. - - Gen3 features a variety of API endpoints such as `/submission`, `/index`, or `/graphql`, which differ in how they access the resource and contain each a subset of REST (Representational State Transfer) APIs for networked applications. REST APIs are restricted in their interactions via HTTP request methods such as GET, POST, PATCH, PUT, or DELETE. The GET request retrieves data in read-only mode, POST typically sends data and creates a new resource, PATCH typically updates/modifies a resource, PUT typically updates/replaces a resource, and DELETE deletes a resource. At Gen3, the GET endpoint ``` /api/v0/submission///_dictionary diff --git a/docs/markdown.md b/docs/markdown.md new file mode 100644 index 0000000..7ea6d1e --- /dev/null +++ b/docs/markdown.md @@ -0,0 +1,98 @@ +--- +icon: simple/markdown +--- + +# Markdown in 5min + +## Headers +``` +# H1 Header +## H2 Header +### H3 Header +#### H4 Header +##### H5 Header +###### H6 Header +``` + +## Text formatting +``` +**bold text** +*italic text* +***bold and italic*** +~~strikethrough~~ +`inline code` +``` + +## Links and images +``` +[Link text](https://example.com) +[Link with title](https://example.com "Hover title") +![Alt text](image.jpg) +![Image with title](image.jpg "Image title") +``` + +## Lists +``` +Unordered: +- Item 1 +- Item 2 + - Nested item + +Ordered: +1. First item +2. Second item +3. Third item +``` + +## Blockquotes +``` +> This is a blockquote +> Multiple lines +>> Nested quote +``` + +## Code blocks +```` +```javascript +function hello() { + console.log("Hello, world!"); +} +``` +```` + +## Tables +``` +| Header 1 | Header 2 | Header 3 | +|----------|----------|----------| +| Row 1 | Data | Data | +| Row 2 | Data | Data | +``` + +## Horizontal rule +``` +--- +or +*** +or +___ +``` + +## Task lists +``` +- [x] Completed task +- [ ] Incomplete task +- [ ] Another task +``` + +## Escaping characters +``` +Use backslash to escape: \* \_ \# \` +``` + +## Line breaks +``` +End a line with two spaces +to create a line break. + +Or use a blank line for a new paragraph. +``` \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml deleted file mode 100644 index a49bc74..0000000 --- a/mkdocs.yml +++ /dev/null @@ -1,159 +0,0 @@ -#https://www.mkdocs.org/user-guide/configuration/# -site_name: Gen3 Documentation -site_url: https://docs.gen3.org -repo_url: https://github.com/uc-cdis/docs-gen3 -nav: - - Home: index.md - - About Gen3: gen3-resources/about.md - - Gen3 Community: gen3-resources/community.md - - User Guide: - - gen3-resources/user-guide/index.md # user guide home page, describing the contents of the user guide and linking to sections - - Search for Data: gen3-resources/user-guide/search.md - - Access Data Files: gen3-resources/user-guide/access-data.md - - Analyze Data: gen3-resources/user-guide/analyze-data.md - - Data Portal: gen3-resources/user-guide/portal.md - - Use the API: gen3-resources/user-guide/using-api.md - - Operator Guide - Deploy Gen3: - - gen3-resources/operator-guide/index.md - # homepage for operator guide, describes contents and links to different sections - - Prerequisites for Deploying Gen3: gen3-resources/operator-guide/prerequisites.md - - Pre-Deployment: - - Infrastructure as Code: - - Infrastructure as Code - Overview: gen3-resources/operator-guide/iac-overview.md - - Terraform: gen3-resources/operator-guide/iac-terraform.md - - Terragrunt: gen3-resources/operator-guide/iac-terragrunt.md - - Atlantis: gen3-resources/operator-guide/iac-atlantis.md - - Prepare SSL Certificate: gen3-resources/operator-guide/ssl-cert.md - - Authentication Methods: gen3-resources/operator-guide/gen3-authn-methods.md - - Setup Gen3 (Configure and Deploy) with Helm: - - gen3-resources/operator-guide/helm/index.md - - Deployment Overview: gen3-resources/operator-guide/helm/helm-deploy-overview.md - - Example - Minimal Local Deployment: gen3-resources/operator-guide/helm/helm-deploy-example.md - - Example - Production Deployment: gen3-resources/operator-guide/helm/helm-deploy-production-example.md - - Databases in Gen3 Helm charts: gen3-resources/operator-guide/helm/helm-deploy-databases.md - - Configure services with Helm: - - gen3-resources/operator-guide/helm/helm-config/index.md - - AuthN/AuthZ: gen3-resources/operator-guide/helm/helm-config/helm-config-auth.md - - Data-related Services: gen3-resources/operator-guide/helm/helm-config/helm-config-data-svcs.md - - Front End: gen3-resources/operator-guide/helm/helm-config/helm-config-frontend.md - - Workspaces: gen3-resources/operator-guide/helm/helm-config/helm-config-workspaces.md - - Revproxy: gen3-resources/operator-guide/helm/helm-config/helm-config-revproxy.md - - Data Submission and Search Configuration: - - Create Data Dictionary: gen3-resources/operator-guide/create-data-dictionary.md - # contains info about creating (and maintaining?) a data dictionary - - Submit Data: - - Structured Data: gen3-resources/operator-guide/submit-structured-data.md - - Unstructured Data: gen3-resources/operator-guide/submit-unstructured-data.md - - Semi-structured data: gen3-resources/operator-guide/submit-semi-structured-data.md - - Controlling data authorization: gen3-resources/operator-guide/authorization.md - - Customize Gen3 Search: gen3-resources/operator-guide/customize-search.md - # info from the current operator guide about configuring ETL and data explorer - - Customize the Front End: gen3-resources/operator-guide/customize-frontend.md - - Secrets and Security: - - Secrets Manager: gen3-resources/operator-guide/tutorial_secrets-mgr.md - - Create an AWS IAM Global User: gen3-resources/operator-guide/tutorial_global_IAM_helm_user.md - - Observability and Monitoring: - - Observability: gen3-resources/operator-guide/tutorial_observability.md - - Grafana Alloy Helm Chart: gen3-resources/operator-guide/tutorial_alloy.md - - Grafana Alloy and Faro: gen3-resources/operator-guide/tutorial_faro.md - - Microservice Jobs and Tutorials: - - About Jobs in Gen3: gen3-resources/operator-guide/tutorial_jobs_overview.md - - Fence Usersync Cronjob: gen3-resources/operator-guide/tutorial_fence_usersync_job.md - - Developer Guide - Extend Gen3: - - gen3-resources/developer-guide/index.md - - Gen3 Architecture: gen3-resources/developer-guide/architecture.md - - Key Repositories: gen3-resources/developer-guide/key_repos.md - - Contribute Code to Gen3: gen3-resources/developer-guide/contribute.md - - Tools: - - gen3-resources/tools/index.md - - Gen3 Data Client : gen3-resources/tools/data-client.md - - Gen3 SDK: gen3-resources/tools/gen3-sdk.md - - Blog: - - blog/index.md - - Frequently Asked Questions: gen3-resources/faq.md - - gen3-resources/glossary.md - - Demo @ gen3.datacommons.io: https://gen3.datacommons.io/ - - -theme: - favicon: img/Gen3-3.ico - logo: img/gen3_new_logo_white.svg - name: material - features: - - navigation.footer - - navigation.indexes - - navigation.tracking - - navigation.path - custom_dir: overrides - - palette: - - primary: black - - - # Palette toggle for light mode - - media: "(prefers-color-scheme: light)" - scheme: brh_platform - toggle: - icon: material/eye - name: Switch to dark mode - - # Palette toggle for dark mode - - media: "(prefers-color-scheme: dark)" - scheme: brh_platform_dark - toggle: - icon: material/eye-outline - name: Switch to light mode -extra_css: - - stylesheets/extra.css -extra_javascript: - - js/dd.js -use_directory_urls: true -markdown_extensions: - - admonition - - attr_list - - md_in_html - - def_list - - footnotes - - toc: - permalink: True - toc_depth: 3 - - pymdownx.superfences - - pymdownx.details - - pymdownx.caret - - pymdownx.mark - - pymdownx.tilde - - pymdownx.tasklist: - custom_checkbox: true - - pymdownx.smartsymbols - - pymdownx.emoji: - emoji_generator: !!python/name:pymdownx.emoji.to_svg - - pymdownx.inlinehilite - - pymdownx.magiclink - - pymdownx.betterem: - smart_enable: all - - pymdownx.highlight: - auto_title: true -plugins: - - search - - tags - - blog: - enabled: true - authors: true - authors_file: blog/authors.yml - blog_dir: blog - blog_toc: true - - mkdocs-video: - is_video: True - -extra: - social: - - icon: fontawesome/brands/linkedin - link: https://www.linkedin.com/company/center-for-translational-data-science/ - - icon: fontawesome/brands/slack - link: https://docs.google.com/forms/d/e/1FAIpQLSczyhhOXeCK9FdVtpQpelOHYnRj1EAq1rwwnm9q6cPAe5a7ug/viewform - - icon: fontawesome/brands/youtube - link: https://www.youtube.com/channel/UCMCwQy4EDd1BaskzZgIOsNQ/featured - - icon: fontawesome/brands/github - link: https://github.com/uc-cdis - - icon: fontawesome/brands/bluesky - link: https://bsky.app/profile/ctds-uchicago.bsky.social diff --git a/to-update-docs-gen3.md b/to-update-docs-gen3.md index dd7c256..1b0c8eb 100644 --- a/to-update-docs-gen3.md +++ b/to-update-docs-gen3.md @@ -1,14 +1,12 @@ # Updating Gen3 documentation -If you would like to submit an update to the Gen3 documentation, we encourage you to create a PR with your change. However, **we request that you first test your changes before submitting them in a PR**. You can test your changes by forking the documentation repo and serving the documentation locally with Material for MkDocs, as described below. If testing is too technically challenging, we ask that you instead [create an issue](https://github.com/uc-cdis/docs-gen3/issues/new/choose) to suggest your update. +If you would like to submit an update to the Gen3 documentation, we encourage you to create a PR with your change. However, **we request that you first test your changes before submitting them in a PR**. You can test your changes by forking the documentation repo and serving the documentation locally with Zensical, as described below. If testing is too technically challenging, we ask that you instead [create an issue](https://github.com/uc-cdis/docs-gen3/issues/new/choose) to suggest your update. ## Steps to submit your own updates to Gen3 Documentation -### Install Material for MkDocs, and some support packages +### Install Zensical -Install Material for MkDocs - [installation instructions here](https://squidfunk.github.io/mkdocs-material/getting-started/). It is recommended that you install in a [virtual environment](https://realpython.com/what-is-pip/#using-pip-in-a-python-virtual-environment) that also has Python v3.11.5 or higher installed. - -The most recent version of Material for MkDocs should also install most of the extensions and plugins our documentation uses; however, you do need to additionally install [pymdown-extensions](https://pypi.org/project/pymdown-extensions/) and the [mkdocs-video plugin](https://pypi.org/project/mkdocs-video/). (A later section will show you [how to look if there are any additional dependencies](#check-if-there-are-additional-dependencies).) +Install Zensical - [installation instructions here](https://zensical.org/docs/get-started/). It is recommended that you install in a [virtual environment](https://realpython.com/what-is-pip/#using-pip-in-a-python-virtual-environment) that also has Python v3.13 or higher. ### Fork the `docs-gen3` Github repo, then clone the fork locally @@ -18,15 +16,11 @@ Since you will not have write privileges to the repo, you will need to fork the #### Check if there are additional dependencies -In your terminal, navigate to the directory with the local clone of your `docs-gen3` fork (for clarity, we will call this `your-forked-docs`). If you installed MkDocs in a virtual environment, activate it. - -Run `mkdocs get-deps` to summarize what dependencies are defined in the documentation's mkdocs.yaml. If it includes only `mkdocs`, `mkdocs-material`, `mkdocs-video`, and `pymdown-extensions`, you do not need to install any additional packages beyond what was [described above](#install-material-for-mkdocs-and-some-support-packages). If there are any additional dependencies, install them. +In your terminal, navigate to the directory with the local clone of your `docs-gen3` fork (for clarity, we will call this `your-forked-docs`). If you installed zensical in a virtual environment, activate it. -### Locally serve the documentation using MkDocs +### Locally serve the documentation -From the `your-forked-docs` directory in the terminal, run [`mkdocs serve`](https://www.mkdocs.org/user-guide/cli/#mkdocs-serve). By default, it will serve the Gen3 documentation to the localhost ([http://127.0.0.1:8000/](http://127.0.0.1:8000/)). If you are missing any dependencies, it will give you a warning in the terminal output. - -> *Note for users familiar with MkDocs: you will not need to use the `mkdocs build` commands, as the documentation is already built in the repo you cloned.* +From the `your-forked-docs` directory in the terminal, run [`zensical serve`](https://zensical.org/docs/create-your-site/#preview-as-you-write). By default, it will serve the Gen3 documentation to the localhost ([http://localhost:8000](http://localhost:8000)). This will immediately reflect any saved changes to your local docs-gen3 fork, and so this is how you can examine whether your changes will have the expected result on the documentation. @@ -44,12 +38,12 @@ In the terminal, from the local `your-forked-docs` repo, [create a new branch](h Now, make your desired changes to the files in the `your-forked-docs` repo. As you save the changes, they will immediately show up in the served documentation on localhost, so you can navigate to the pages with your changes and see how they look. -Whenever possible, please avoid using HTML tags to achieve a formatting result. Although Markdown can be limiting in formatting, [we have implemented a number of Markdown extensions and MkDocs plugins](#markdown-extensions-and-mkdocs-plug-ins-available-in-our-documentation) that improve the ability to achieve the desired look. +Whenever possible, please avoid using HTML tags to achieve a formatting result. Although Markdown can be limiting in formatting, we have a number of Markdown extensions and Zensical plugins that improve the ability to achieve the desired look. -Here are some common mistakes with formatting in MkDocs: +Here are some common mistakes with formatting in Zensical: -* Beware that MkDocs uses a different flavor of Markdown than many other tools. (See more here: https://www.markdownguide.org/tools/mkdocs/). It’s especially finicky about indentations - improper indentation will break a list. For numbered lists, always check that multi-paragraph list items, or list items with code blocks (for example), is indented properly so that the list numbering does not restart after the additional block. -* MkDocs-flavored markdown requires a line before and after elements like headings, code blocks, lists, and image references; otherwise, it will not render elements properly +* Beware that Zensical uses a different flavor of Markdown than many other tools. (See more here: https://zensical.org/docs/setup/extensions/python-markdown/). It’s especially finicky about indentations - improper indentation will break a list. For numbered lists, always check that multi-paragraph list items, or list items with code blocks (for example), is indented properly so that the list numbering does not restart after the additional block. +* Zensical-flavored markdown requires a line before and after elements like headings, code blocks, lists, and image references; otherwise, it will not render elements properly * A single carriage return does not render as a line break. You need either two carriage returns or two spaces at the end of the line. For a list, there must be two spaces at the end of the line for the list to render as separate bulleted or numbered items. When you are done with your local testing - use CTRL+C to quit local serving, or just close the terminal window. @@ -66,43 +60,6 @@ Merge the changes into your forked repo. Once you submit your PR, a member of the CTDS staff may request changes or ask you questions regarding your update. -## Markdown extensions and MkDocs plug-ins available in our documentation - -Extensions enable you to use simple Markdown syntax to achieve some complex rendering. Here is a list of the available extensions and plug-ins: - -markdown_extensions: - - admonition - - attr_list - - md_in_html - - def_list - - footnotes - - toc: - permalink: True - toc_depth: 3 - - pymdownx.superfences - - pymdownx.details - - pymdownx.caret - - pymdownx.mark - - pymdownx.tilde - - pymdownx.tasklist: - custom_checkbox: true - - pymdownx.smartsymbols - - pymdownx.emoji: - emoji_generator: !!python/name:pymdownx.emoji.to_svg - - pymdownx.inlinehilite - - pymdownx.magiclink - - pymdownx.betterem: - smart_enable: all - - pymdownx.highlight: - auto_title: true -plugins: - - search - - tags - - blog: - enabled: true - authors: true - authors_file: blog/authors.yml - blog_dir: blog - blog_toc: true - - mkdocs-video: - is_video: True \ No newline at end of file +## Markdown extensions and Zensical plug-ins available in our documentation + +Extensions enable you to use simple Markdown syntax to achieve some complex rendering. To see the list of the available extensions and plug-ins, see the bottom of the config file [zensical.toml](zensical.toml) in the `project.markdown_extensions` sections and see [Zensical documentation](https://zensical.org/docs/setup/extensions/). diff --git a/zensical.toml b/zensical.toml new file mode 100644 index 0000000..4e00089 --- /dev/null +++ b/zensical.toml @@ -0,0 +1,453 @@ +# ============================================================================ +# +# The configuration produced by default is meant to highlight the features +# that Zensical provides and to serve as a starting point for your own +# projects. +# +# ============================================================================ + +[project] + +# The site_name is shown in the page header and the browser window title +# +# Read more: https://zensical.org/docs/setup/basics/#site_name +site_name = "Gen3 Documentation" + +# The site_description is included in the HTML head and should contain a +# meaningful description of the site content for use by search engines. +# +# Read more: https://zensical.org/docs/setup/basics/#site_description +site_description = "Documentation for the open-source software platform: Gen3" + +# The site_author attribute. This is used in the HTML head element. +# +# Read more: https://zensical.org/docs/setup/basics/#site_author +site_author = "Center for Translational Data Science" + +# The site_url is the canonical URL for your site. When building online +# documentation you should set this. +# Read more: https://zensical.org/docs/setup/basics/#site_url +site_url = "https://docs.gen3.org/" + +# The copyright notice appears in the page footer and can contain an HTML +# fragment. +# +# Read more: https://zensical.org/docs/setup/basics/#copyright +copyright = """ +Copyright © 2026 Center for Translational Data Science +""" + +# Zensical supports both implicit navigation and explicitly defined navigation. +# If you decide not to define a navigation here then Zensical will simply +# derive the navigation structure from the directory structure of your +# "docs_dir". The definition below demonstrates how a navigation structure +# can be defined using TOML syntax. +# +# Read more: https://zensical.org/docs/setup/navigation/ +# nav = [ +# { "Get started" = "index.md" }, +# { "Markdown in 5min" = "markdown.md" }, +# ] +nav = [ + { "Home" = "index.md" }, + { "About Gen3" = "gen3-resources/about.md" }, + { "Gen3 Community" = "gen3-resources/community.md" }, + { "User Guide" = [ + "gen3-resources/user-guide/index.md", + { "Search for Data" = "gen3-resources/user-guide/search.md" }, + { "Access Data Files" = "gen3-resources/user-guide/access-data.md" }, + { "Analyze Data" = "gen3-resources/user-guide/analyze-data.md" }, + { "Data Portal" = "gen3-resources/user-guide/portal.md" }, + { "Use the API" = "gen3-resources/user-guide/using-api.md" } + ]}, + { "Operator Guide - Deploy Gen3" = [ + "gen3-resources/operator-guide/index.md", + { "Prerequisites for Deploying Gen3" = "gen3-resources/operator-guide/prerequisites.md" }, + { "Pre-Deployment" = [ + { "Infrastructure as Code" = [ + { "Infrastructure as Code - Overview" = "gen3-resources/operator-guide/iac-overview.md" }, + { "Terraform" = "gen3-resources/operator-guide/iac-terraform.md" }, + { "Terragrunt" = "gen3-resources/operator-guide/iac-terragrunt.md" }, + { "Atlantis" = "gen3-resources/operator-guide/iac-atlantis.md" } + ]}, + { "Prepare SSL Certificate" = "gen3-resources/operator-guide/ssl-cert.md" }, + { "Authentication Methods" = "gen3-resources/operator-guide/gen3-authn-methods.md" } + ]}, + { "Setup Gen3 (Configure and Deploy) with Helm" = [ + "gen3-resources/operator-guide/helm/index.md", + { "Deployment Overview" = "gen3-resources/operator-guide/helm/helm-deploy-overview.md" }, + { "Example - Minimal Local Deployment" = "gen3-resources/operator-guide/helm/helm-deploy-example.md" }, + { "Example - Production Deployment" = "gen3-resources/operator-guide/helm/helm-deploy-production-example.md" }, + { "Databases in Gen3 Helm charts" = "gen3-resources/operator-guide/helm/helm-deploy-databases.md" }, + { "Configure services with Helm" = [ + "gen3-resources/operator-guide/helm/helm-config/index.md", + { "AuthN/AuthZ" = "gen3-resources/operator-guide/helm/helm-config/helm-config-auth.md" }, + { "Data-related Services" = "gen3-resources/operator-guide/helm/helm-config/helm-config-data-svcs.md" }, + { "Front End" = "gen3-resources/operator-guide/helm/helm-config/helm-config-frontend.md" }, + { "Workspaces" = "gen3-resources/operator-guide/helm/helm-config/helm-config-workspaces.md" }, + { "Revproxy" = "gen3-resources/operator-guide/helm/helm-config/helm-config-revproxy.md" } + ]} + ]}, + { "Data Submission and Search Configuration" = [ + { "Create Data Dictionary" = "gen3-resources/operator-guide/create-data-dictionary.md" }, + { "Submit Data" = [ + { "Structured Data" = "gen3-resources/operator-guide/submit-structured-data.md" }, + { "Unstructured Data" = "gen3-resources/operator-guide/submit-unstructured-data.md" }, + { "Semi-structured data" = "gen3-resources/operator-guide/submit-semi-structured-data.md" }, + { "Controlling data authorization" = "gen3-resources/operator-guide/authorization.md" } + ]}, + { "Customize Gen3 Search" = "gen3-resources/operator-guide/customize-search.md" }, + { "Customize the Front End" = "gen3-resources/operator-guide/customize-frontend.md" } + ]}, + { "Secrets and Security" = [ + { "Secrets Manager" = "gen3-resources/operator-guide/tutorial_secrets-mgr.md" }, + { "Create an AWS IAM Global User" = "gen3-resources/operator-guide/tutorial_global_IAM_helm_user.md" } + ]}, + { "Observability and Monitoring" = [ + { "Observability" = "gen3-resources/operator-guide/tutorial_observability.md" }, + { "Grafana Alloy Helm Chart" = "gen3-resources/operator-guide/tutorial_alloy.md" }, + { "Grafana Alloy and Faro" = "gen3-resources/operator-guide/tutorial_faro.md" } + ]}, + { "Microservice Jobs and Tutorials" = [ + { "About Jobs in Gen3" = "gen3-resources/operator-guide/tutorial_jobs_overview.md" }, + { "Fence Usersync Cronjob" = "gen3-resources/operator-guide/tutorial_fence_usersync_job.md" } + ]} + ]}, + { "Developer Guide - Extend Gen3" = [ + "gen3-resources/developer-guide/index.md", + { "Gen3 Architecture" = "gen3-resources/developer-guide/architecture.md" }, + { "Key Repositories" = "gen3-resources/developer-guide/key_repos.md" }, + { "Contribute Code to Gen3" = "gen3-resources/developer-guide/contribute.md" } + ]}, + { "Tools" = [ + "gen3-resources/tools/index.md", + { "Gen3 Data Client" = "gen3-resources/tools/data-client.md" }, + { "Gen3 SDK" = "gen3-resources/tools/gen3-sdk.md" } + ]}, + { "Blog" = [ "blog/index.md" ] }, + { "Frequently Asked Questions" = "gen3-resources/faq.md" }, + { "Glossary" = "gen3-resources/glossary.md" }, + { "Demo @ gen3.datacommons.io" = "https://gen3.datacommons.io/" } +] + + +# With the "extra_css" option you can add your own CSS styling to customize +# your Zensical project according to your needs. You can add any number of +# CSS files. +# +# The path provided should be relative to the "docs_dir". +# +# Read more: https://zensical.org/docs/customization/#additional-css +# +extra_css = ["stylesheets/extra.css"] + +# With the `extra_javascript` option you can add your own JavaScript to your +# project to customize the behavior according to your needs. +# +# The path provided should be relative to the "docs_dir". +# +# Read more: https://zensical.org/docs/customization/#additional-javascript +extra_javascript = ["js/dd.js"] + +# ---------------------------------------------------------------------------- +# Section for configuring theme options +# ---------------------------------------------------------------------------- +[project.theme] + +# change this to "classic" to use the traditional Material for MkDocs look. +variant = "classic" + +# Zensical allows you to override specific blocks, partials, or whole +# templates as well as to define your own templates. To do this, uncomment +# the custom_dir setting below and set it to a directory in which you +# keep your template overrides. +# +# Read more: +# - https://zensical.org/docs/customization/#extending-the-theme +# +custom_dir = "overrides" + +# With the "favicon" option you can set your own image to use as the icon +# browsers will use in the browser title bar or tab bar. The path provided +# must be relative to the "docs_dir". +# +# Read more: +# - https://zensical.org/docs/setup/logo-and-icons/#favicon +# - https://developer.mozilla.org/en-US/docs/Glossary/Favicon +# +favicon = "img/Gen3-3.ico" + +# Zensical supports more than 60 different languages. This means that the +# labels and tooltips that Zensical's templates produce are translated. +# The "language" option allows you to set the language used. This language +# is also indicated in the HTML head element to help with accessibility +# and guide search engines and translation tools. +# +# The default language is "en" (English). It is possible to create +# sites with multiple languages and configure a language selector. See +# the documentation for details. +# +# Read more: +# - https://zensical.org/docs/setup/language/ +# +language = "en" + +# Zensical provides a number of feature toggles that change the behavior +# of the documentation site. +features = [ + # Zensical includes an announcement bar. This feature allows users to + # dismiss it when they have read the announcement. + # https://zensical.org/docs/setup/header/#announcement-bar + "announce.dismiss", + + # If you have a repository configured and turn on this feature, Zensical + # will generate an edit button for the page. This works for common + # repository hosting services. + # https://zensical.org/docs/setup/repository/#content-actions + "content.action.edit", + + # If you have a repository configured and turn on this feature, Zensical + # will generate a button that allows the user to view the Markdown + # code for the current page. + # https://zensical.org/docs/setup/repository/#content-actions + "content.action.view", + + # Code annotations allow you to add an icon with a tooltip to your + # code blocks to provide explanations at crucial points. + # https://zensical.org/docs/authoring/code-blocks/#code-annotations + "content.code.annotate", + + # This feature turns on a button in code blocks that allow users to + # copy the content to their clipboard without first selecting it. + # https://zensical.org/docs/authoring/code-blocks/#code-copy-button + "content.code.copy", + + # Code blocks can include a button to allow for the selection of line + # ranges by the user. + # https://zensical.org/docs/authoring/code-blocks/#code-selection-button + "content.code.select", + + # Zensical can render footnotes as inline tooltips, so the user can read + # the footnote without leaving the context of the document. + # https://zensical.org/docs/authoring/footnotes/#footnote-tooltips + "content.footnote.tooltips", + + # If you have many content tabs that have the same titles (e.g., "Python", + # "JavaScript", "Cobol"), this feature causes all of them to switch to + # at the same time when the user chooses their language in one. + # https://zensical.org/docs/authoring/content-tabs/#linked-content-tabs + "content.tabs.link", + + # With this feature enabled users can add tooltips to links that will be + # displayed when the mouse pointer hovers the link. + # https://zensical.org/docs/authoring/tooltips/#improved-tooltips + "content.tooltips", + + # With this feature enabled, Zensical will automatically hide parts + # of the header when the user scrolls past a certain point. + # https://zensical.org/docs/setup/header/#automatic-hiding + # "header.autohide", + + # Turn on this feature to expand all collapsible sections in the + # navigation sidebar by default. + # https://zensical.org/docs/setup/navigation/#navigation-expansion + # "navigation.expand", + + # This feature turns on navigation elements in the footer that allow the + # user to navigate to a next or previous page. + # https://zensical.org/docs/setup/footer/#navigation + "navigation.footer", + + # When section index pages are enabled, documents can be directly attached + # to sections, which is particularly useful for providing overview pages. + # https://zensical.org/docs/setup/navigation/#section-index-pages + "navigation.indexes", + + # When instant navigation is enabled, clicks on all internal links will be + # intercepted and dispatched via XHR without fully reloading the page. + # https://zensical.org/docs/setup/navigation/#instant-navigation + "navigation.instant", + + # With instant prefetching, your site will start to fetch a page once the + # user hovers over a link. This will reduce the perceived loading time + # for the user. + # https://zensical.org/docs/setup/navigation/#instant-prefetching + "navigation.instant.prefetch", + + # In order to provide a better user experience on slow connections when + # using instant navigation, a progress indicator can be enabled. + # https://zensical.org/docs/setup/navigation/#progress-indicator + "navigation.instant.progress", + + # When navigation paths are activated, a breadcrumb navigation is rendered + # above the title of each page + # https://zensical.org/docs/setup/navigation/#navigation-path + "navigation.path", + + # When pruning is enabled, only the visible navigation items are included + # in the rendered HTML, reducing the size of the built site by 33% or more. + # https://zensical.org/docs/setup/navigation/#navigation-pruning + #"navigation.prune", + + # When sections are enabled, top-level sections are rendered as groups in + # the sidebar for viewports above 1220px, but remain as-is on mobile. + # https://zensical.org/docs/setup/navigation/#navigation-sections + "navigation.sections", + + # When tabs are enabled, top-level sections are rendered in a menu layer + # below the header for viewports above 1220px, but remain as-is on mobile. + # https://zensical.org/docs/setup/navigation/#navigation-tabs + #"navigation.tabs", + + # When sticky tabs are enabled, navigation tabs will lock below the header + # and always remain visible when scrolling down. + # https://zensical.org/docs/setup/navigation/#sticky-navigation-tabs + #"navigation.tabs.sticky", + + # A back-to-top button can be shown when the user, after scrolling down, + # starts to scroll up again. + # https://zensical.org/docs/setup/navigation/#back-to-top-button + "navigation.top", + + # When anchor tracking is enabled, the URL in the address bar is + # automatically updated with the active anchor as highlighted in the table + # of contents. + # https://zensical.org/docs/setup/navigation/#anchor-tracking + "navigation.tracking", + + # When search highlighting is enabled and a user clicks on a search result, + # Zensical will highlight all occurrences after following the link. + # https://zensical.org/docs/setup/search/#search-highlighting + "search.highlight", + + # When anchor following for the table of contents is enabled, the sidebar + # is automatically scrolled so that the active anchor is always visible. + # https://zensical.org/docs/setup/navigation/#anchor-following + "toc.follow", + + # When navigation integration for the table of contents is enabled, it is + # always rendered as part of the navigation sidebar on the left. + # https://zensical.org/docs/setup/navigation/#navigation-integration + #"toc.integrate", +] + +# ---------------------------------------------------------------------------- +# You can configure your own logo to be shown in the header using the "logo" +# option in the "theme" subsection. The logo must be a relative path to a file +# in your "docs_dir", e.g., to use `docs/assets/logo.png` you would set: +# ---------------------------------------------------------------------------- +logo = "img/gen3_new_logo_white.svg" + +# ---------------------------------------------------------------------------- +# If you don't have a dedicated project logo, you can use a built-in icon from +# the icon sets shipped in Zensical. Please note that the setting lives in a +# different subsection, and that the above take precedence over the icon. +# +# Read more: +# - https://zensical.org/docs/setup/logo-and-icons +# - https://github.com/zensical/ui/tree/master/dist/.icons +# ---------------------------------------------------------------------------- +#[project.theme.icon] +#logo = "lucide/smile" + +# ---------------------------------------------------------------------------- +# In the "font" subsection you can configure the fonts used. By default, fonts +# are loaded from Google Fonts, giving you a wide range of choices from a set +# of suitably licensed fonts. There are options for a normal text font and for +# a monospaced font used in code blocks. +# ---------------------------------------------------------------------------- +#[project.theme.font] +#text = "Inter" +#code = "Jetbrains Mono" + +# ---------------------------------------------------------------------------- +# In the "palette" subsection you can configure options for the color scheme. +# You can configure different color schemes, e.g., to turn on dark mode, +# that the user can switch between. Each color scheme can be further +# customized. +# +# Read more: +# - https://zensical.org/docs/setup/colors/ +# ---------------------------------------------------------------------------- +# [[project.theme.palette]] +# primary = "black" + +[[project.theme.palette]] +primary = "black" +scheme = "default" +toggle.icon = "lucide/sun" +toggle.name = "Switch to dark mode" + +[[project.theme.palette]] +primary = "black" +scheme = "slate" +toggle.icon = "lucide/moon" +toggle.name = "Switch to light mode" + +# ---------------------------------------------------------------------------- +# The "extra" section contains miscellaneous settings. +# ---------------------------------------------------------------------------- +[[project.extra.social]] +icon = "fontawesome/brands/linkedin" +link = "https://www.linkedin.com/company/center-for-translational-data-science/" + +[[project.extra.social]] +icon = "fontawesome/brands/slack" +link = "https://docs.google.com/forms/d/e/1FAIpQLSczyhhOXeCK9FdVtpQpelOHYnRj1EAq1rwwnm9q6cPAe5a7ug/viewform" + +[[project.extra.social]] +icon = "fontawesome/brands/youtube" +link = "https://www.youtube.com/channel/UCMCwQy4EDd1BaskzZgIOsNQ/featured" + +[[project.extra.social]] +icon = "fontawesome/brands/github" +link = "https://github.com/uc-cdis" + +[[project.extra.social]] +icon = "fontawesome/brands/bluesky" +link = "https://bsky.app/profile/ctds-uchicago.bsky.social" + + +# ---------------------------------------------------------------------------- +# In this section you can configure the Markdown extensions that are used when +# rendering your documentation. We enable the most useful extensions by default, +# but you can customize this list to your needs. +# +# Read more: +# - https://zensical.org/docs/setup/extensions/ +# ---------------------------------------------------------------------------- +[project.markdown_extensions.abbr] +[project.markdown_extensions.admonition] +[project.markdown_extensions.attr_list] +[project.markdown_extensions.def_list] +[project.markdown_extensions.footnotes] +[project.markdown_extensions.md_in_html] +[project.markdown_extensions.toc] +permalink = true +[project.markdown_extensions.pymdownx.arithmatex] +generic = true +[project.markdown_extensions.pymdownx.betterem] +[project.markdown_extensions.pymdownx.caret] +[project.markdown_extensions.pymdownx.details] +[project.markdown_extensions.pymdownx.emoji] +emoji_generator = "zensical.extensions.emoji.to_svg" +emoji_index = "zensical.extensions.emoji.twemoji" +[project.markdown_extensions.pymdownx.highlight] +anchor_linenums = true +line_spans = "__span" +pygments_lang_class = true +[project.markdown_extensions.pymdownx.inlinehilite] +[project.markdown_extensions.pymdownx.keys] +[project.markdown_extensions.pymdownx.magiclink] +[project.markdown_extensions.pymdownx.mark] +[project.markdown_extensions.pymdownx.smartsymbols] +[project.markdown_extensions.pymdownx.superfences] +custom_fences = [ + { name = "mermaid", class = "mermaid", format = "pymdownx.superfences.fence_code_format" } +] +[project.markdown_extensions.pymdownx.tabbed] +alternate_style = true +combine_header_slug = true +[project.markdown_extensions.pymdownx.tasklist] +custom_checkbox = true +[project.markdown_extensions.pymdownx.tilde] From 196c03102a02803e1223af1df40c459f866e023d Mon Sep 17 00:00:00 2001 From: avantol Date: Wed, 29 Apr 2026 14:47:31 -0500 Subject: [PATCH 2/5] feat(workflows): cleanup actions to support auto-creation of PRs appropriately and ensure testing of build occurs before main --- .github/workflows/api-docs.yaml | 58 ------------ .github/workflows/auto_updates.yml | 89 +++++++++++++++++++ .../workflows/{docs-ci.yaml => publish.yml} | 7 +- .github/workflows/test.yml | 26 ++++++ 4 files changed, 119 insertions(+), 61 deletions(-) delete mode 100644 .github/workflows/api-docs.yaml create mode 100644 .github/workflows/auto_updates.yml rename .github/workflows/{docs-ci.yaml => publish.yml} (83%) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/api-docs.yaml b/.github/workflows/api-docs.yaml deleted file mode 100644 index 59e3400..0000000 --- a/.github/workflows/api-docs.yaml +++ /dev/null @@ -1,58 +0,0 @@ -name: Daily API Update Check -on: - workflow_dispatch: - schedule: - # 9:47AM central time M-F - # why so specific? probably a lot of people have things scheduled - # on the half-hour so this avoids load times at 9:30 / 10 - - cron: '47 9 * * 1-5' - timezone: "America/Chicago" -permissions: - contents: write -jobs: - api-update-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Configure Git Credentials - run: | - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y curl - - - name: Download OpenAPI spec - run: | - # TODO: use main branch once this is merged - curl -L https://raw.githubusercontent.com/uc-cdis/gen3-ai/refs/heads/feat/docs/docs/api.html -o ./docs/gen3-resources/user-guide/api/gen3_ai_api.html - - - name: Check for changes - id: check_changes - run: | - git add ./docs/gen3-resources/user-guide/api/gen3_ai_api.html - if git diff --cached --quiet; then - echo "changes=false" >> $GITHUB_OUTPUT - else - echo "changes=true" >> $GITHUB_OUTPUT - fi - - - name: Create Pull Request If Changes Detected - if: steps.check_changes.outputs.changes == 'true' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - BRANCH_NAME="chore/update-api-docs-${{ github.run_id }}" - - git checkout -b $BRANCH_NAME - git add ./docs/gen3-resources/user-guide/api/gen3_ai_api.html - git commit -m "docs(api): update gen3_ai_api.html from upstream" - git push origin $BRANCH_NAME - - # gh CLI is pre-installed in ubuntu-latest - gh pr create \ - --title "docs(api): update API documentation" \ - --body "Automated update of APIs docs from upstream repositories." \ - --head $BRANCH_NAME diff --git a/.github/workflows/auto_updates.yml b/.github/workflows/auto_updates.yml new file mode 100644 index 0000000..573d165 --- /dev/null +++ b/.github/workflows/auto_updates.yml @@ -0,0 +1,89 @@ +name: Daily API Update Check +on: + # allow manual runs + workflow_dispatch: + + # also schedule daily + schedule: + # 9:47AM central time M-F + # why so specific? probably a lot of people have things scheduled + # on the half-hour so this avoids load times at 9:30 / 10 + - cron: '47 9 * * 1-5' + timezone: "America/Chicago" +permissions: + contents: write + pull-requests: write +jobs: + api-update-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + # this uses the default repo-scoped, job-used GITHUB_TOKEN + # and saves it as git creds for this run only (it's + # removed in a post cleanup). this ensures + # future git commands are authed + persist-credentials: true + + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email github-actions[bot]@users.noreply.github.com + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y curl + + - name: Download OpenAPI spec + run: | + curl -L https://raw.githubusercontent.com/uc-cdis/gen3-ai/refs/heads/main/docs/api.html -o ./docs/gen3-resources/user-guide/api/gen3_ai_api.html + + - name: Check for changes + id: check_changes + run: | + git add ./docs/gen3-resources/user-guide/api/gen3_ai_api.html + + # if existing secrets just moved line numbers + # note: if detect-secrets actually fails on a NEW false + # positive secret from the docs generation, + # this workflow will still fail and require human intervention + git add .secrets.baseline + + if git diff --cached --quiet; then + echo "changes=false" >> $GITHUB_OUTPUT + else + echo "changes=true" >> $GITHUB_OUTPUT + fi + + - name: Create Pull Request If Changes Detected + if: steps.check_changes.outputs.changes == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + BRANCH_NAME="chore/update-api-docs-${{ github.run_id }}" + git checkout -b $BRANCH_NAME + + git add ./docs/gen3-resources/user-guide/api/gen3_ai_api.html + + # if existing secrets just moved line numbers + # note: if detect-secrets actually fails on a NEW false + # positive secret from the docs generation, + # this workflow will still fail and require human intervention + git add .secrets.baseline + + git commit -m "docs(api): update gen3_ai_api.html from upstream" + git push origin $BRANCH_NAME + + # gh CLI is pre-installed in ubuntu-latest + # NOTE: --draft is required b/c the default GITHUB_TOKEN (which is used here) + # will not trigger future `on: push` actions in the resulting PR. This is + # a GH-required protection to prevent accidental infinite loops. To work around + # this, we make the PR a draft, require a real person to manually "mark it ready" + # and then the other workflow actions should run b/c we explicitly + # tell it to run on a PR "ready" trigger. + gh pr create \ + --title "docs(api): update API documentation" \ + --body "Automated update of APIs docs from upstream repositories. Mark as 'Ready for Review' to start automated checks!" \ + --head $BRANCH_NAME \ + --draft diff --git a/.github/workflows/docs-ci.yaml b/.github/workflows/publish.yml similarity index 83% rename from .github/workflows/docs-ci.yaml rename to .github/workflows/publish.yml index 9bf51ad..b851f12 100644 --- a/.github/workflows/docs-ci.yaml +++ b/.github/workflows/publish.yml @@ -1,5 +1,6 @@ -name: Documentation +name: Publish Documentation on: + # any updates to main/master will actually publish to Pages push: branches: - master @@ -9,7 +10,7 @@ permissions: pages: write id-token: write jobs: - deploy: + publish: environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} @@ -20,7 +21,7 @@ jobs: - name: Configure Git Credentials run: | git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git config user.email github-actions[bot]@users.noreply.github.com - name: Add CNAME file run: echo 'docs.gen3.org' > ./docs/CNAME - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..40ace64 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,26 @@ +name: Test Documentation +on: + # push will run on every pushed commit to any branch + # EXCEPT to master and main, which the `publish.yml` handles + push: + branches-ignore: + - master + - main +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/configure-pages@v6 + - uses: actions/checkout@v6 + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email github-actions[bot]@users.noreply.github.com + - name: Add CNAME file + run: echo 'docs.gen3.org' > ./docs/CNAME + - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + - uses: actions/setup-python@v6 + with: + python-version: 3.x + - run: pip install zensical + - run: zensical build --clean From 8ef2a1fae8434bd2f50541c1dc0b83716a453349 Mon Sep 17 00:00:00 2001 From: michaelfitzo Date: Thu, 30 Apr 2026 11:41:45 -0500 Subject: [PATCH 3/5] hide subheadings by default --- zensical.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zensical.toml b/zensical.toml index 4e00089..366764f 100644 --- a/zensical.toml +++ b/zensical.toml @@ -34,7 +34,7 @@ site_url = "https://docs.gen3.org/" # # Read more: https://zensical.org/docs/setup/basics/#copyright copyright = """ -Copyright © 2026 Center for Translational Data Science +Copyright © 2026 Center for Translational Data Science """ # Zensical supports both implicit navigation and explicitly defined navigation. @@ -292,7 +292,7 @@ features = [ # When sections are enabled, top-level sections are rendered as groups in # the sidebar for viewports above 1220px, but remain as-is on mobile. # https://zensical.org/docs/setup/navigation/#navigation-sections - "navigation.sections", + # "navigation.sections", # When tabs are enabled, top-level sections are rendered in a menu layer # below the header for viewports above 1220px, but remain as-is on mobile. @@ -372,7 +372,7 @@ logo = "img/gen3_new_logo_white.svg" # [[project.theme.palette]] # primary = "black" -[[project.theme.palette]] +[[project.theme.palette]] primary = "black" scheme = "default" toggle.icon = "lucide/sun" From e8c1588d86b020ef628b441d3d5e17a1697083cf Mon Sep 17 00:00:00 2001 From: michaelfitzo Date: Thu, 30 Apr 2026 11:53:46 -0500 Subject: [PATCH 4/5] resolving warnings --- docs/gen3-resources/about.md | 9 ++------- docs/gen3-resources/glossary.md | 4 ++-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/docs/gen3-resources/about.md b/docs/gen3-resources/about.md index 3fafda6..fbab7cd 100644 --- a/docs/gen3-resources/about.md +++ b/docs/gen3-resources/about.md @@ -4,6 +4,8 @@ Gen3 is a data platform for building data commons and data meshes. The Gen3 plat You can find more information about Gen3 on [gen3.org](https://gen3.org/). +You can find the source code for Gen3 on [GitHub][gen3 github]. + ## Powered by Gen3 Gen3 is used by dozen of projects around the world to manage and analyze biomedical data. You can find an incomplete list of current Gen3 projects [here](https://gen3.org/powered-by-gen3/). @@ -30,11 +32,4 @@ To reach a member of the CTDS User Services Team you may contact us via email at -[code of conduct]: https://forums.gen3.org/t/faq-guidelines/5 -[contact page]: ../contact.md -[gen3 youtube]: https://www.youtube.com/channel/UCMCwQy4EDd1BaskzZgIOsNQ/featured [gen3 github]: https://github.com/uc-cdis -[gen3 publications]: https://gen3.org/gen3refs/ -[community page]: https://gen3.org/community/events/ -[release notes]: https://github.com/uc-cdis/cdis-manifest/tree/master/releases -[contributor guidelines]: developer-guide/contribute.md diff --git a/docs/gen3-resources/glossary.md b/docs/gen3-resources/glossary.md index c590a2d..b5d6184 100644 --- a/docs/gen3-resources/glossary.md +++ b/docs/gen3-resources/glossary.md @@ -11,7 +11,7 @@ A Common Services Operations Center is an operations center operated by a common ## Crosswalk Typically, used for linking patients from across data commons where some patient data exists in commons A and additional data exists in commons B. This linkage enables metadata associations across commons and the promise of richer datasets. Crosswalks can be made for several types of metadata and are recorded in the metadata service. An example of how to set this up is found [here][crosswalk setup]. ## Data Commons -A data commons co-locates data with cloud computing infrastructure and commonly used software services, tools, and applications for managing, integrating, analyzing and sharing data that are exposed through web portals and APIs to create an interoperable resource for a research community. A data commons provides services so that the data is findable, accessible, interoperable and reusable (FAIR) +A data commons co-locates data with cloud computing infrastructure and commonly used software services, tools, and applications for managing, integrating, analyzing and sharing data that are exposed through web portals and APIs to create an interoperable resource for a research community. A data commons provides services so that the data is findable, accessible, interoperable and reusable (FAIR). ## Data Dictionary Every Gen3 data commons employs a data model, which serves to describe, organize, and harmonize data sets submitted by different users. Data harmonization facilitates cross-project analyses and is thus one of the pillars of the data commons paradigm. The data model organizes experimental metadata variables, “properties”, into linked categories, “nodes”, through the use of a data dictionary. The data dictionary lists and describes all nodes in the data model, as well as defines and describes the properties in each node. A Gen3 Data Dictionary is specified by a YAML file per node. Additional details on Gen3 data dictionaries can be found [here][data dictionary overview]. ## Data Hub @@ -42,7 +42,7 @@ Edge nodes may be created at clinics, labs, hospitals, or academic institutions ## ETL Structured data submitted to commons are stored in PostgreSQL. Querying data from PostgreSQL with multiple join is painful and inefficient. So, we use ElasticSearch as a place to store materialized dataset. Extract-transform-load (ETL) is a process that creates the materialized data from PostgreSQL and store them in ElasticSearch. This is accomplished via the Tube microservice. More details of running an ETL can be found [here](operator-guide/customize-search.md#etl). ## FAIR Data -FAIR data are data which are findable, accessible, interoperable, and reusable[12]. There is now an extensive literature on FAIR data. +FAIR data are data which are findable, accessible, interoperable, and reusable. There is now an extensive literature on FAIR data. ## Framework Services Framework Services or Data Commons Framework (DCF) Services is the term used by Gen3 to refer to data mesh services in the narrow middle architecture, for data meshes, such as the NCI Cancer Research Data Commons. These are set of standards-based services with open APIs for authentication, authorization, creating and accessing FAIR data objects, and for working with bulk structured data in machine-readable, self-contained format. From ca7b4bd1cbf724a888aa2644db1024be5753f0ba Mon Sep 17 00:00:00 2001 From: michaelfitzo Date: Thu, 30 Apr 2026 16:43:46 -0500 Subject: [PATCH 5/5] Resolve warnings --- .../operator-guide/customize-frontend.md | 6 +++--- .../operator-guide/customize-search.md | 12 ++++++++++-- .../helm/helm-deploy-production-example.md | 1 - docs/gen3-resources/operator-guide/index.md | 4 ---- .../operator-guide/submit-structured-data.md | 1 - .../operator-guide/submit-unstructured-data.md | 8 ++++---- docs/gen3-resources/tools/data-client.md | 1 - docs/gen3-resources/user-guide/portal.md | 4 +--- 8 files changed, 18 insertions(+), 19 deletions(-) diff --git a/docs/gen3-resources/operator-guide/customize-frontend.md b/docs/gen3-resources/operator-guide/customize-frontend.md index aaaa479..d506647 100644 --- a/docs/gen3-resources/operator-guide/customize-frontend.md +++ b/docs/gen3-resources/operator-guide/customize-frontend.md @@ -1,4 +1,4 @@ -# Customize appearance of the front end +f# Customize appearance of the front end Below we show a few examples of how to customize the Gen3 Data Portal. @@ -62,8 +62,8 @@ The "tooltip" shows text upon hovering over the icon. ![navigationbar][navigationbar image] -* [Review the code to edit icon, link, color, tooltip, and name of the navigation items][gitops.json navbar]. -* Adding a new icon requires saving the icon in [this repository][icons] and [in this file][icons inex]. +* [Review the code to edit icon, link, color, tooltip, and name of the navigation items](https://github.com/uc-cdis/cdis-manifest/blob/551f0963e60f6000ae8b9987592495406a031c81/gen3.datacommons.io/portal/gitops.json#L84-L134). +* Adding a new icon requires saving the icon in [this repository][icons] and [in this file][icons index]. ## Data Commons or Mesh Title diff --git a/docs/gen3-resources/operator-guide/customize-search.md b/docs/gen3-resources/operator-guide/customize-search.md index 97e47b5..c0b4261 100644 --- a/docs/gen3-resources/operator-guide/customize-search.md +++ b/docs/gen3-resources/operator-guide/customize-search.md @@ -101,10 +101,18 @@ To view the MDS for the Gen3 Data Hub you can go [here][gen3 data hub mds]. You }, ``` -Instructions for working with the API are found [here][mds api]. + + + + + + +Instructions for working with the API are found [in the swagger docs][mdsapi]. + + ### Aggregated Metadata Service An [Aggregated metadata service (aggMDS)][aggmds github] caches the metadata from two or more metadata sources to provide a unified view of the commons on the discovery page. @@ -191,7 +199,7 @@ Setting up a functioning mesh where you can access files from individual commons [aggmds github]: https://github.com/uc-cdis/metadata-service/blob/master/docs/config_agg_mds.md [sdk for discovery page]: https://github.com/uc-cdis/gen3sdk-python/blob/master/gen3/cli/discovery.py [gen3 data hub mds]: https://gen3.datacommons.io/mds/metadata?data=True&_guid_type=discovery_metadata -[mds api]: https://petstore.swagger.io/?url=https://raw.githubusercontent.com/uc-cdis/metadata-service/master/docs/openapi.yaml +[mdsapi]: https://petstore.swagger.io/?url=https://raw.githubusercontent.com/uc-cdis/metadata-service/master/docs/openapi.yaml [aggmds sync]: https://github.com/uc-cdis/cloud-automation/blob/master/kube/services/jobs/metadata-aggregate-sync-job.yaml [BRH]: https://brh.data-commons.org/ [BRH aggregated_config.json]: https://github.com/uc-cdis/cdis-manifest/blob/master/brh.data-commons.org/metadata/aggregate_config.json diff --git a/docs/gen3-resources/operator-guide/helm/helm-deploy-production-example.md b/docs/gen3-resources/operator-guide/helm/helm-deploy-production-example.md index 2126b9a..1b14f0a 100644 --- a/docs/gen3-resources/operator-guide/helm/helm-deploy-production-example.md +++ b/docs/gen3-resources/operator-guide/helm/helm-deploy-production-example.md @@ -265,7 +265,6 @@ spec: -[argo wrapper]: https://github.com/uc-cdis/argo-wrapper [aws cli user guide]: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html [terraform user guide]: https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli [kubectl]: https://kubernetes.io/docs/tasks/tools/ diff --git a/docs/gen3-resources/operator-guide/index.md b/docs/gen3-resources/operator-guide/index.md index 455393a..b1b02bb 100644 --- a/docs/gen3-resources/operator-guide/index.md +++ b/docs/gen3-resources/operator-guide/index.md @@ -48,16 +48,12 @@ Compose-services was used to deploy Gen3 at a small scale, for experimental comm [Although we do not recommend using this, you can explore Gen3 compose-services here][compose-services] -[Gen3 User Guide]: ../user-guide/index.md -[Gen3 Developer Guide]: ../developer-guide/index.md [cloud-automation]: https://github.com/uc-cdis/cloud-automation/blob/master/doc/csoc-free-commons-steps.md [compose-services]: https://github.com/uc-cdis/compose-services/tree/master [helm guide]: helm/index.md [deploy overview]: helm/helm-deploy-overview.md [prerequisites]: prerequisites.md [infrastructure as code]: iac-overview.md -[ssl]: ssl-cert.md -[secrets]: secrets-mgr.md [authentication methods]: gen3-authn-methods.md [data dictionary]: create-data-dictionary.md [submit data]: submit-structured-data.md diff --git a/docs/gen3-resources/operator-guide/submit-structured-data.md b/docs/gen3-resources/operator-guide/submit-structured-data.md index fb020d4..6ce002f 100644 --- a/docs/gen3-resources/operator-guide/submit-structured-data.md +++ b/docs/gen3-resources/operator-guide/submit-structured-data.md @@ -194,5 +194,4 @@ If the submission throws errors or claims the submission to be invalid, it will [Fence programs and projects]: https://github.com/uc-cdis/fence/blob/master/docs/additional_documentation/user.yaml_guide.md#programs-and-projects-crud-access [Gen3 SDK]: https://uc-cdis.github.io/gen3sdk-python/_build/html/_modules/gen3/submission.html [toolbar submission]: img/Gen3_Toolbar_data_submission.png -[submit data]: img/Gen3_Data_Submission_submit_data.png [upload file]: img/Gen3_Data_Submission_Use_Form.png diff --git a/docs/gen3-resources/operator-guide/submit-unstructured-data.md b/docs/gen3-resources/operator-guide/submit-unstructured-data.md index f66d21b..123cd6b 100644 --- a/docs/gen3-resources/operator-guide/submit-unstructured-data.md +++ b/docs/gen3-resources/operator-guide/submit-unstructured-data.md @@ -317,10 +317,10 @@ The below is an example of a Indexing Manifest File: | guid | File_name | File_size | md5sum | bucket_urls | auth | | --- | --- | --- | --- | --- | --- | -| dg.4503/02... ...7103bbe | examplefile.txt | 34141 | c79... ...dbd | s3://nih-phs001416-c1/exfile.txt gs://nih-phs001416-c1/exfile.txt | [phs001416.c1] | -| dg.4503/00... ...0211dfg | otherexamplefile.txt | 562256 | 65a... ...bca | s3://nih-nhlbi-topmed-released-phs001416-c1/otherexamplefile.txt gs://nih-nhlbi-topmed-released-phs001416-c1/otherexamplefile.txt | [phs001416.c1] | -| dg.4503/00... ...7103bbe | examplefile.txt | 36564 | dca... ...774 | s3://nih-nhlbi-topmed-released-phs001416-c2/examplefile.txt gs://nih-nhlbi-topmed-released-phs001416-c2/examplefile.txt | [phs001416.c2] | -| dg.4503/01... ...0410nnd | otherexamplefile.txt | 2675 | 742... ...f1b | s3://nih-nhlbi-topmed-released-phs001416-c2/otherexamplefile.txt gs://nih-nhlbi-topmed-released-phs001416-c2/otherexamplefile.txt | [phs001416.c2] | +| dg.4503/02... ...7103bbe | examplefile.txt | 34141 | c79... ...dbd | s3://nih-phs001416-c1/exfile.txt gs://nih-phs001416-c1/exfile.txt | \[phs001416.c1] | +| dg.4503/00... ...0211dfg | otherexamplefile.txt | 562256 | 65a... ...bca | s3://nih-nhlbi-topmed-released-phs001416-c1/otherexamplefile.txt gs://nih-nhlbi-topmed-released-phs001416-c1/otherexamplefile.txt | \[phs001416.c1] | +| dg.4503/00... ...7103bbe | examplefile.txt | 36564 | dca... ...774 | s3://nih-nhlbi-topmed-released-phs001416-c2/examplefile.txt gs://nih-nhlbi-topmed-released-phs001416-c2/examplefile.txt | \[phs001416.c2] | +| dg.4503/01... ...0410nnd | otherexamplefile.txt | 2675 | 742... ...f1b | s3://nih-nhlbi-topmed-released-phs001416-c2/otherexamplefile.txt gs://nih-nhlbi-topmed-released-phs001416-c2/otherexamplefile.txt | \[phs001416.c2] | ### 6. Submit file Indexing Manifest to Indexd diff --git a/docs/gen3-resources/tools/data-client.md b/docs/gen3-resources/tools/data-client.md index 937f7cd..6d74eb3 100644 --- a/docs/gen3-resources/tools/data-client.md +++ b/docs/gen3-resources/tools/data-client.md @@ -482,7 +482,6 @@ If this happens (and you are still authorized to access the data), you can downl [Gen3 Client]: https://github.com/uc-cdis/cdis-data-client/releases/latest -[sdk_github]: https://github.com/uc-cdis/gen3sdk-python [PATH]: data-client.md#working-from-the-command-line [img create API key]: img/Gen3_Keys.png diff --git a/docs/gen3-resources/user-guide/portal.md b/docs/gen3-resources/user-guide/portal.md index 7c95886..7bd5290 100644 --- a/docs/gen3-resources/user-guide/portal.md +++ b/docs/gen3-resources/user-guide/portal.md @@ -31,7 +31,7 @@ Each of these main tabs will have filters on the left-hand side, which can be us > Note: The main tabs in the Exploration Page, the available filters, and the properties listed in the data table are entirely customizable and will be different for various Gen3 data commons. -If the table is a list of files, there should be a button for downloading a JSON file that serves as a manifest to use with the [gen3-client][Gen3 client] for [downloading multiple files][Gen3 bulk download]. Otherwise, to download single a file listed in the table, simply click on the GUID (globally unique identifier, or object_id), which should open a page with a download button. +If the table is a list of files, there should be a button for downloading a JSON file that serves as a manifest to use with the [gen3-client][Gen3 Client] for [downloading multiple files][Gen3 bulk download]. Otherwise, to download single a file listed in the table, simply click on the GUID (globally unique identifier, or object_id), which should open a page with a download button. ![GIF showing how to use the Gen3 Explorer][img Explorer GIF] @@ -171,7 +171,6 @@ All the functionality of the data commons data portal is available by sending re [Gen3 Data Hub]: https://gen3.datacommons.io/ -[Download Data Files into a Workspace with the Python SDK]: img/profile_page.png [img profile]: img/profile_page.png [Gen3 Client]: access-data.md#download-files-using-the-gen3-client @@ -180,7 +179,6 @@ All the functionality of the data commons data portal is available by sending re [Gen3 Data Hub Exploration]: https://gen3.datacommons.io/explorer [img Gen3 Toolbar Exploration]: img/Gen3_Toolbar_exploration.png -[Gen3 client]: access-data.md#download-files-using-the-gen3-client [Gen3 bulk download]: ../tools/data-client.md#multiple-file-download-with-manifest [img Explorer GIF]: img/explorer_gif_2020.gif [Gen3 Workspace]: analyze-data.md#launch-workspace