From 86405309a00edc522b6cde5f54437c60faabdb28 Mon Sep 17 00:00:00 2001 From: TJ VanToll Date: Mon, 17 Nov 2025 20:48:34 +0000 Subject: [PATCH 1/2] feat: adding script for generating Mardown docs (and macOS Docker support) --- .devcontainer/ci/Dockerfile | 10 ++ .devcontainer/ci/devcontainer.json | 10 +- README.md | 2 + docs/BUILDING_MARKDOWN.md | 114 ++++++++++++++++ docs/CMakeLists.txt | 19 +++ docs/conf.py | 49 ++++++- docs/fix_markdown_anchors.py | 205 +++++++++++++++++++++++++++++ docs/requirements.txt | 6 + scripts/build_docs_markdown.sh | 35 +++++ 9 files changed, 446 insertions(+), 4 deletions(-) create mode 100644 docs/BUILDING_MARKDOWN.md create mode 100755 docs/fix_markdown_anchors.py create mode 100644 docs/requirements.txt create mode 100755 scripts/build_docs_markdown.sh diff --git a/.devcontainer/ci/Dockerfile b/.devcontainer/ci/Dockerfile index 334df54f..8ffa0445 100644 --- a/.devcontainer/ci/Dockerfile +++ b/.devcontainer/ci/Dockerfile @@ -24,6 +24,15 @@ ARG USER # Local Environment Variable(s) ENV LC_ALL="C.UTF-8" +# Install adduser package for user creation utilities +RUN ["dash", "-c", "\ + apt-get update --quiet \ + && apt-get install --assume-yes --no-install-recommends --quiet \ + adduser \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ +"] + # Create Non-Root User RUN ["dash", "-c", "\ addgroup \ @@ -70,6 +79,7 @@ RUN ["dash", "-c", "\ && pip install --break-system-packages \ breathe \ sphinx-rtd-theme \ + sphinx-markdown-builder \ && apt-get clean \ && apt-get purge \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ diff --git a/.devcontainer/ci/devcontainer.json b/.devcontainer/ci/devcontainer.json index 139cc19c..969517e7 100644 --- a/.devcontainer/ci/devcontainer.json +++ b/.devcontainer/ci/devcontainer.json @@ -6,9 +6,13 @@ // Sets the run context to one level up instead of the .devcontainer folder. "context": "${localWorkspaceFolder}", - // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. - "image": "ghcr.io/blues/note_c_ci:latest", - // "dockerFile": "Dockerfile", + // Use pre-built image (faster, but only works on amd64 or with Docker Desktop's platform emulation): + // "image": "ghcr.io/blues/note_c_ci:latest", + // Use local Dockerfile (works on all platforms, required for Apple Silicon): + "dockerFile": "Dockerfile", + + // Force amd64 platform for Apple Silicon compatibility + "runArgs": ["--platform=linux/amd64"], // Set *default* container specific settings.json values on container create. "settings": {}, diff --git a/README.md b/README.md index 3a290894..7a8f7895 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ it as a git subtree. The API documentation for this library can be found [here][note-c API docs]. +The documentation can also be built locally in either HTML or Markdown format. See [Building Markdown Documentation](docs/BUILDING_MARKDOWN.md) for details on generating Markdown documentation. + ## Logging Control `note-c` provides a comprehensive and flexible logging functionality. diff --git a/docs/BUILDING_MARKDOWN.md b/docs/BUILDING_MARKDOWN.md new file mode 100644 index 00000000..5313c349 --- /dev/null +++ b/docs/BUILDING_MARKDOWN.md @@ -0,0 +1,114 @@ +# Building Markdown Documentation + +The note-c documentation can now be generated in Markdown format in addition to HTML. This is useful for reusing the documentation in other contexts. + +## Prerequisites + +### Local Build + +To build the documentation locally, you need the following installed: + +1. **Doxygen** (version 1.9.8 or later) +2. **Sphinx** and required Python packages + +Install Python dependencies: +```bash +pip install -r docs/requirements.txt +``` + +Or install individually: +```bash +pip install sphinx breathe sphinx-rtd-theme sphinxcontrib-jquery sphinx-markdown-builder +``` + +### Docker Build + +Alternatively, use the provided Docker container which includes all dependencies: + +```bash +docker build .devcontainer/ci/ --tag note_c_docs +``` + +## Building Documentation + +### HTML Documentation (Original) + +To build the HTML documentation: + +```bash +# Using the build script +./scripts/build_docs.sh + +# Or using CMake directly +cmake -B build/ -DNOTE_C_BUILD_DOCS:BOOL=ON +cmake --build build/ -- docs +``` + +The HTML output will be in `build/docs/index.html` + +### Markdown Documentation (New) + +To build the Markdown documentation: + +```bash +# Using the build script +./scripts/build_docs_markdown.sh + +# Or using CMake directly +cmake -B build/ -DNOTE_C_BUILD_DOCS:BOOL=ON +cmake --build build/ -- docs-markdown +``` + +The Markdown output will be in `build/docs/markdown/` + +### Using Docker + +To build documentation using Docker: + +```bash +# Build the Docker image (if not already built) +docker build .devcontainer/ci/ --tag note_c_docs + +# Run the build inside Docker +docker run --rm --volume $(pwd):/note-c/ --workdir /note-c/ note_c_docs bash -c "./scripts/build_docs_markdown.sh" +``` + +## Output Structure + +The Markdown documentation maintains the same structure as the HTML documentation: + +- `index.md` - Main documentation index +- `api_reference.md` - API reference documentation +- `getting_started.md` - Getting started guide +- `calling_the_notecard_api.md` - API usage guide +- `library_initialization.md` - Library initialization guide +- `ports.md` - Porting guide + +The API documentation generated from Doxygen comments will be integrated into the Markdown files via the Breathe extension. + +## Configuration + +The Markdown builder configuration is in `docs/conf.py`: + +- `markdown_http_base` - Base URL for external links (set to the hosted docs URL) +- `markdown_uri_doc_suffix` - Suffix for document URIs (set to `.html` for compatibility) + +You can modify these settings if you need to adjust how links are generated in the Markdown output. + +## Troubleshooting + +### "Could NOT find Doxygen" +Install Doxygen version 1.9.8 or later, or use the Docker build method. + +### "sphinx-build: command not found" +Install Sphinx and related Python packages using `pip install -r docs/requirements.txt`. + +### "No module named 'sphinx_markdown_builder'" +Install the sphinx-markdown-builder package: `pip install sphinx-markdown-builder` + +## Notes + +- The Markdown output is generated from the same source as the HTML documentation +- Doxygen generates XML, which Breathe converts to Sphinx directives, which are then rendered to Markdown +- Some complex HTML formatting may not translate perfectly to Markdown +- The Markdown files are suitable for inclusion in other documentation systems or for viewing on platforms that render Markdown diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 6c73fdd9..3dd49584 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -11,6 +11,10 @@ set(DOCS_SRC_DIR ${PROJECT_SOURCE_DIR}/docs) set(SPHINX_OPTS "-j auto -W --keep-going -T -v") separate_arguments(SPHINX_OPTS) +# Options for markdown build (without -W to allow extension warnings) +set(SPHINX_OPTS_MARKDOWN "-j auto --keep-going -T -v") +separate_arguments(SPHINX_OPTS_MARKDOWN) + add_custom_target( docs COMMAND ${CMAKE_COMMAND} -E env @@ -24,3 +28,18 @@ add_custom_target( USES_TERMINAL COMMENT "Running Sphinx HTML build..." ) + +add_custom_target( + docs-markdown + COMMAND ${CMAKE_COMMAND} -E env + ${SPHINX_BUILD} + -b markdown + -c ${DOCS_CFG_DIR} + -w ${DOCS_BUILD_DIR}/markdown_build.log + ${SPHINX_OPTS_MARKDOWN} + ${DOCS_SRC_DIR} + ${DOCS_BUILD_DIR}/markdown + COMMAND python3 ${DOCS_CFG_DIR}/fix_markdown_anchors.py ${DOCS_BUILD_DIR}/markdown + USES_TERMINAL + COMMENT "Running Sphinx Markdown build and fixing anchors..." +) diff --git a/docs/conf.py b/docs/conf.py index 2417b022..b40fae15 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,8 +31,13 @@ 'breathe', 'sphinx.ext.autosectionlabel', 'sphinxcontrib.jquery', + 'sphinx_markdown_builder', +] +suppress_warnings = [ + 'autosectionlabel.*', + 'config.cache', # Suppress locale directory warnings + 'misc.highlighting_failure', # Suppress highlighting warnings ] -suppress_warnings = ['autosectionlabel.*'] templates_path = ['_templates'] exclude_patterns = ['build/'] master_doc = 'index' @@ -68,3 +73,45 @@ 'logo_only': True, 'prev_next_buttons_location': None } + +# Options for Markdown output + +markdown_http_base = 'https://blues.github.io/note-c' +markdown_uri_doc_suffix = '.html' + +# Custom handling for sphinx-markdown-builder compatibility +def setup(app): + """Setup custom handlers for node types not supported by sphinx-markdown-builder.""" + from docutils import nodes + from sphinx import addnodes + import logging + + # Suppress the parallel reading warning for sphinx_markdown_builder + logger = logging.getLogger('sphinx') + + # Check if we're using the markdown builder + if app.config._raw_config.get('_running_markdown_builder', False): + return + + # Add a flag to detect when markdown builder is running + def builder_inited(app): + if app.builder.name == 'markdown': + app.config._running_markdown_builder = True + # Register handlers for unsupported node types + try: + from sphinx_markdown_builder.translator import MarkdownTranslator + + # Handle desc_signature_line by treating it as a regular paragraph + def visit_desc_signature_line(self, node): + pass + + def depart_desc_signature_line(self, node): + pass + + MarkdownTranslator.visit_desc_signature_line = visit_desc_signature_line + MarkdownTranslator.depart_desc_signature_line = depart_desc_signature_line + + except ImportError: + pass + + app.connect('builder-inited', builder_inited) diff --git a/docs/fix_markdown_anchors.py b/docs/fix_markdown_anchors.py new file mode 100755 index 00000000..e0ebe0fa --- /dev/null +++ b/docs/fix_markdown_anchors.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +""" +Post-process Sphinx-generated markdown to add HTML anchors for cross-references. + +This script fixes broken cross-reference links in markdown generated by +sphinx-markdown-builder. The builder generates links with domain prefixes +(e.g., #c.JINTEGER) but doesn't create matching anchors. +""" + +import re +import sys +from pathlib import Path + + +def extract_anchor_id(link): + """Extract the anchor ID from a markdown link.""" + # Match [text](#anchor) format + match = re.search(r'\[([^\]]+)\]\(#([^)]+)\)', link) + if match: + return match.group(2) + return None + + +def find_definition_anchors(content): + """ + Find all cross-reference links and extract their anchor IDs. + Returns a dict mapping anchor IDs to the link text (identifier name). + """ + anchor_map = {} + + # Find all internal anchor links (e.g., #c.JINTEGER, #note_8h_1a234...) + # This pattern catches any internal link + pattern = r'\[([^\]]+)\]\(#([^)]+)\)' + matches = re.findall(pattern, content) + + for link_text, anchor_id in matches: + # Extract identifier from link text (strip markdown formatting) + # Remove any inline code backticks or other markdown + identifier = re.sub(r'`', '', link_text).strip() + # Store the anchor ID and what identifier it refers to + anchor_map[anchor_id] = identifier + + return anchor_map + + +def add_anchors_to_definitions(content, anchor_map): + """ + Add HTML anchors before function and type definitions. + + We look for headings that match the identifiers we found cross-references to, + and add HTML anchor tags before them. + + Args: + content: The markdown content + anchor_map: Dict mapping anchor IDs to identifier names + """ + lines = content.split('\n') + output_lines = [] + + # Build reverse map: identifier -> list of anchor IDs + identifier_to_anchors = {} + for anchor_id, identifier in anchor_map.items(): + if identifier not in identifier_to_anchors: + identifier_to_anchors[identifier] = [] + identifier_to_anchors[identifier].append(anchor_id) + + for i, line in enumerate(lines): + # Check if this is a heading line + if line.startswith('### '): + heading_text = line[4:].strip() + + # Try to extract the identifier from various heading formats + identifier = None + + # Match type pointers FIRST: "typedef ... (*NAME)(...)" + # Need to handle escaped asterisks in markdown: \* + normalized = heading_text.replace('\\*', '*') + typedef_ptr_match = re.search(r'\(\*(\w+)\)', normalized) + if typedef_ptr_match: + identifier = typedef_ptr_match.group(1) + + # Match typedef: "typedef ... NAME" + if not identifier: + typedef_match = re.match(r'^typedef\s+.*\s+(\w+)\s*$', heading_text) + if typedef_match: + identifier = typedef_match.group(1) + + # Match struct: "struct NAME" + if not identifier: + struct_match = re.match(r'^struct\s+(\w+)', heading_text) + if struct_match: + identifier = struct_match.group(1) + + # Match function definitions: "type funcName(params)" or "[type](#link) funcName(params)" + # Need to handle markdown links in the return type + if not identifier: + func_match = re.search(r'(?:\[[^\]]+\]\([^)]+\)\s+)?\*?(\w+)\s*\(', heading_text) + if func_match: + identifier = func_match.group(1) + + # Match plain identifiers (macros, simple definitions) + if not identifier: + plain_match = re.match(r'^([A-Z_][A-Z0-9_]*)\s*$', heading_text) + if plain_match: + identifier = plain_match.group(1) + + # If we found an identifier and there are matching anchor IDs, add them + if identifier and identifier in identifier_to_anchors: + anchors = identifier_to_anchors[identifier] + # Debug output for function pointers + if 'Fn' in identifier: + print(f" Adding {len(anchors)} anchor(s) for {identifier}") + for anchor_id in anchors: + output_lines.append(f'\n') + elif identifier and 'Fn' in identifier: + # Debug: identifier found but no anchors + print(f" WARNING: Found {identifier} but no anchors for it") + + output_lines.append(line) + + return '\n'.join(output_lines) + + +def fix_backticked_links(content): + """ + Remove backticks around markdown links. + + sphinx-markdown-builder sometimes wraps links in backticks, which + prevents them from being rendered as links. This function removes + those backticks. + + Examples: + `[text](#anchor)` -> [text](#anchor) + `[text](url)` -> [text](url) + """ + # Match backtick-wrapped links + pattern = r'`(\[[^\]]+\]\([^)]+\))`' + fixed = re.sub(pattern, r'\1', content) + return fixed + + +def process_markdown_file(filepath): + """Process a single markdown file to add anchors.""" + print(f"Processing {filepath}...") + + with open(filepath, 'r', encoding='utf-8') as f: + content = f.read() + + # Find all anchor IDs that are referenced + anchor_map = find_definition_anchors(content) + + if not anchor_map: + print(f" No cross-references found, skipping.") + return + + print(f" Found {len(anchor_map)} cross-references to fix") + + # Debug: show a few examples + fn_anchors = {k: v for k, v in anchor_map.items() if 'Fn' in v} + if fn_anchors: + print(f" Example function pointers in anchor map:") + for anchor_id, identifier in list(fn_anchors.items())[:5]: + print(f" {identifier} -> {anchor_id}") + + # Add anchors to definitions + updated_content = add_anchors_to_definitions(content, anchor_map) + + # Fix backticked links + updated_content = fix_backticked_links(updated_content) + + # Write back + with open(filepath, 'w', encoding='utf-8') as f: + f.write(updated_content) + + print(f" Done!") + + +def main(): + if len(sys.argv) < 2: + print("Usage: fix_markdown_anchors.py ") + sys.exit(1) + + markdown_dir = Path(sys.argv[1]) + + if not markdown_dir.exists(): + print(f"Error: Directory {markdown_dir} does not exist") + sys.exit(1) + + # Process all .md files in the directory + md_files = list(markdown_dir.glob('*.md')) + + if not md_files: + print(f"No markdown files found in {markdown_dir}") + sys.exit(1) + + print(f"Found {len(md_files)} markdown file(s) to process\n") + + for md_file in md_files: + process_markdown_file(md_file) + + print("\nAll files processed!") + + +if __name__ == '__main__': + main() diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..868b05ca --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,6 @@ +# Python dependencies for building note-c documentation +sphinx>=5.0 +breathe +sphinx-rtd-theme +sphinxcontrib-jquery +sphinx-markdown-builder diff --git a/scripts/build_docs_markdown.sh b/scripts/build_docs_markdown.sh new file mode 100755 index 00000000..e305fdf5 --- /dev/null +++ b/scripts/build_docs_markdown.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Determine the directory of the script. +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT_SRC_DIR="$SCRIPT_DIR/.." + +# Check for a CMakeLists.txt file, to ensure CMake operations +# are executed in the correct directory. +if [[ ! -f "$ROOT_SRC_DIR/CMakeLists.txt" ]]; then + echo "ERROR: Unable to locate ${ROOT_SRC_DIR}/CMakeList.txt. Expecting to use '..' to access root directory. Did the location of build_docs_markdown.sh change?" + exit 1 +fi + +# Move to the root source directory. +pushd $ROOT_SRC_DIR $@ > /dev/null + +# Configure the build with the flag to build docs. +cmake -B build/ -DNOTE_C_BUILD_DOCS:BOOL=ON +if [[ $? -ne 0 ]]; then + echo "ERROR: CMake failed to configure build." + popd > /dev/null + exit 1 +fi + +# Build the `docs-markdown` target. +cmake --build build/ -- docs-markdown -j +if [[ $? -ne 0 ]]; then + echo "ERROR: CMake failed to build markdown docs." + popd > /dev/null + exit 1 +fi + +echo "SUCCESS: Built markdown docs." +echo "Markdown files are located in: build/docs/markdown/" +popd > /dev/null From 208da65e625f7d5d237f8a08ec14514f24931dad Mon Sep 17 00:00:00 2001 From: TJ VanToll Date: Tue, 18 Nov 2025 19:19:23 +0000 Subject: [PATCH 2/2] only include the markdown dependency when building markdown --- docs/conf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index b40fae15..15d43ada 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,8 +31,11 @@ 'breathe', 'sphinx.ext.autosectionlabel', 'sphinxcontrib.jquery', - 'sphinx_markdown_builder', ] + +# Only load sphinx_markdown_builder when building markdown +if args.builder == 'markdown': + extensions.append('sphinx_markdown_builder') suppress_warnings = [ 'autosectionlabel.*', 'config.cache', # Suppress locale directory warnings