diff --git a/docs/.custom_wordlist.txt b/docs/.custom_wordlist.txt new file mode 100644 index 00000000..6a24c7b6 --- /dev/null +++ b/docs/.custom_wordlist.txt @@ -0,0 +1,21 @@ +AGPL +Armored +Levenshtein +OpenPGP +Starlark +Subcommands +ZSTD +armor +armored +basename +esm +fips +http +https +jsonwall +optimize +subcommand +subcommands +symlink +ubuntu +yaml diff --git a/docs/.readthedocs.yaml b/docs/.readthedocs.yaml new file mode 100644 index 00000000..f170b006 --- /dev/null +++ b/docs/.readthedocs.yaml @@ -0,0 +1,30 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + jobs: + pre_install: + - git fetch --unshallow || true + +# Build documentation in the docs/ directory with Sphinx +sphinx: + builder: dirhtml + configuration: docs/conf.py + fail_on_warning: true + +# If using Sphinx, optionally build your docs in additional formats such as PDF +formats: + - pdf + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: docs/.sphinx/requirements.txt diff --git a/docs/.sphinx/.markdownlint.json b/docs/.sphinx/.markdownlint.json new file mode 100644 index 00000000..8aafaf6d --- /dev/null +++ b/docs/.sphinx/.markdownlint.json @@ -0,0 +1,16 @@ +{ + "default": false, + "MD003": { "style": "atx" }, + "MD013": { "code_blocks": false, "tables": false, "stern": true, "line_length": 150}, + "MD014": true, + "MD018": true, + "MD022": true, + "MD023": true, + "MD026": { "punctuation": ".,;。,;"}, + "MD031": { "list_items": false}, + "MD032": true, + "MD035": true, + "MD042": true, + "MD045": true, + "MD052": true +} \ No newline at end of file diff --git a/docs/.sphinx/.wordlist.txt b/docs/.sphinx/.wordlist.txt new file mode 100644 index 00000000..be5021a1 --- /dev/null +++ b/docs/.sphinx/.wordlist.txt @@ -0,0 +1,64 @@ +ACME +ACME's +addons +AGPLv +API +APIs +balancer +Charmhub +CLI +DCO +Diátaxis +Dqlite +dropdown +EBS +EKS +enablement +favicon +Furo +Git +GitHub +Grafana +IAM +installable +JSON +Juju +Kubeflow +Kubernetes +Launchpad +linter +LTS +LXD +Makefile +Makefiles +Matrix +Mattermost +MicroCeph +MicroCloud +MicroOVN +MyST +namespace +namespaces +NodePort +Numbat +observability +OEM +OLM +Permalink +pre +Quickstart +ReadMe +reST +reStructuredText +roadmap +RTD +subdirectories +subfolders +subtree +TODO +Ubuntu +UI +UUID +VM +webhook +YAML diff --git a/docs/.sphinx/_static/css/pdf.css b/docs/.sphinx/_static/css/pdf.css new file mode 100644 index 00000000..f9e28dd2 --- /dev/null +++ b/docs/.sphinx/_static/css/pdf.css @@ -0,0 +1,15 @@ +/* Only relevant for specific PDF generation issues */ + +.rubric>.hclass2 { + display: block; + font-size: 2em; + border-radius: .5rem; + font-weight: 300; + line-height: 1.25; + margin-top: 1.75rem; + margin-right: -0.5rem; + margin-bottom: 0.5rem; + margin-left: -0.5rem; + padding-left: .5rem; + padding-right: .5rem; +} \ No newline at end of file diff --git a/docs/.sphinx/_static/favicon.png b/docs/.sphinx/_static/favicon.png new file mode 100644 index 00000000..7f175e46 Binary files /dev/null and b/docs/.sphinx/_static/favicon.png differ diff --git a/docs/.sphinx/_static/tag.png b/docs/.sphinx/_static/tag.png new file mode 100644 index 00000000..f6f6e5aa Binary files /dev/null and b/docs/.sphinx/_static/tag.png differ diff --git a/docs/.sphinx/_templates/header.html b/docs/.sphinx/_templates/header.html new file mode 100644 index 00000000..30c52c5c --- /dev/null +++ b/docs/.sphinx/_templates/header.html @@ -0,0 +1,52 @@ + diff --git a/docs/.sphinx/fonts/LICENCE.txt b/docs/.sphinx/fonts/LICENCE.txt new file mode 100644 index 00000000..ae78a8f9 --- /dev/null +++ b/docs/.sphinx/fonts/LICENCE.txt @@ -0,0 +1,96 @@ +------------------------------- +UBUNTU FONT LICENCE Version 1.0 +------------------------------- + +PREAMBLE +This licence allows the licensed fonts to be used, studied, modified and +redistributed freely. The fonts, including any derivative works, can be +bundled, embedded, and redistributed provided the terms of this licence +are met. The fonts and derivatives, however, cannot be released under +any other licence. The requirement for fonts to remain under this +licence does not require any document created using the fonts or their +derivatives to be published under this licence, as long as the primary +purpose of the document is not to be a vehicle for the distribution of +the fonts. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this licence and clearly marked as such. This may +include source files, build scripts and documentation. + +"Original Version" refers to the collection of Font Software components +as received under this licence. + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to +a new environment. + +"Copyright Holder(s)" refers to all individuals and companies who have a +copyright ownership of the Font Software. + +"Substantially Changed" refers to Modified Versions which can be easily +identified as dissimilar to the Font Software by users of the Font +Software comparing the Original Version with the Modified Version. + +To "Propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification and with or without charging +a redistribution fee), making available to the public, and in some +countries other activities as well. + +PERMISSION & CONDITIONS +This licence does not grant any rights under trademark law and all such +rights are reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of the Font Software, to propagate the Font Software, subject to +the below conditions: + +1) Each copy of the Font Software must contain the above copyright +notice and this licence. These can be included either as stand-alone +text files, human-readable headers or in the appropriate machine- +readable metadata fields within text or binary files as long as those +fields can be easily viewed by the user. + +2) The font name complies with the following: +(a) The Original Version must retain its name, unmodified. +(b) Modified Versions which are Substantially Changed must be renamed to +avoid use of the name of the Original Version or similar names entirely. +(c) Modified Versions which are not Substantially Changed must be +renamed to both (i) retain the name of the Original Version and (ii) add +additional naming elements to distinguish the Modified Version from the +Original Version. The name of such Modified Versions must be the name of +the Original Version, with "derivative X" where X represents the name of +the new work, appended to that name. + +3) The name(s) of the Copyright Holder(s) and any contributor to the +Font Software shall not be used to promote, endorse or advertise any +Modified Version, except (i) as required by this licence, (ii) to +acknowledge the contribution(s) of the Copyright Holder(s) or (iii) with +their explicit written permission. + +4) The Font Software, modified or unmodified, in part or in whole, must +be distributed entirely under this licence, and must not be distributed +under any other licence. The requirement for fonts to remain under this +licence does not affect any document created using the Font Software, +except any version of the Font Software extracted from a document +created using the Font Software may only be distributed under this +licence. + +TERMINATION +This licence becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM OTHER +DEALINGS IN THE FONT SOFTWARE. diff --git a/docs/.sphinx/fonts/Ubuntu-B.ttf b/docs/.sphinx/fonts/Ubuntu-B.ttf new file mode 100644 index 00000000..b173da27 Binary files /dev/null and b/docs/.sphinx/fonts/Ubuntu-B.ttf differ diff --git a/docs/.sphinx/fonts/Ubuntu-R.ttf b/docs/.sphinx/fonts/Ubuntu-R.ttf new file mode 100644 index 00000000..d748728a Binary files /dev/null and b/docs/.sphinx/fonts/Ubuntu-R.ttf differ diff --git a/docs/.sphinx/fonts/Ubuntu-RI.ttf b/docs/.sphinx/fonts/Ubuntu-RI.ttf new file mode 100644 index 00000000..4f2d2bc7 Binary files /dev/null and b/docs/.sphinx/fonts/Ubuntu-RI.ttf differ diff --git a/docs/.sphinx/fonts/UbuntuMono-B.ttf b/docs/.sphinx/fonts/UbuntuMono-B.ttf new file mode 100644 index 00000000..7bd66657 Binary files /dev/null and b/docs/.sphinx/fonts/UbuntuMono-B.ttf differ diff --git a/docs/.sphinx/fonts/UbuntuMono-R.ttf b/docs/.sphinx/fonts/UbuntuMono-R.ttf new file mode 100644 index 00000000..fdd309d7 Binary files /dev/null and b/docs/.sphinx/fonts/UbuntuMono-R.ttf differ diff --git a/docs/.sphinx/fonts/UbuntuMono-RI.ttf b/docs/.sphinx/fonts/UbuntuMono-RI.ttf new file mode 100644 index 00000000..18f81a29 Binary files /dev/null and b/docs/.sphinx/fonts/UbuntuMono-RI.ttf differ diff --git a/docs/.sphinx/get_vale_conf.py b/docs/.sphinx/get_vale_conf.py new file mode 100644 index 00000000..9ee2d0b5 --- /dev/null +++ b/docs/.sphinx/get_vale_conf.py @@ -0,0 +1,53 @@ +#! /usr/bin/env python + +import requests +import os + +DIR = os.getcwd() + + +def main(): + if os.path.exists(f"{DIR}/.sphinx/styles/Canonical"): + print("Vale directory exists") + else: + os.makedirs(f"{DIR}/.sphinx/styles/Canonical") + + url = ( + "https://api.github.com/repos/canonical/praecepta/" + + "contents/styles/Canonical" + ) + r = requests.get(url) + for item in r.json(): + download = requests.get(item["download_url"]) + file = open(".sphinx/styles/Canonical/" + item["name"], "w") + file.write(download.text) + file.close() + + if os.path.exists(f"{DIR}/.sphinx/styles/config/vocabularies/Canonical"): + print("Vocab directory exists") + else: + os.makedirs(f"{DIR}/.sphinx/styles/config/vocabularies/Canonical") + + url = ( + "https://api.github.com/repos/canonical/praecepta/" + + "contents/styles/config/vocabularies/Canonical" + ) + r = requests.get(url) + for item in r.json(): + download = requests.get(item["download_url"]) + file = open( + ".sphinx/styles/config/vocabularies/Canonical/" + item["name"], + "w" + ) + file.write(download.text) + file.close() + config = requests.get( + "https://raw.githubusercontent.com/canonical/praecepta/main/vale.ini" + ) + file = open(".sphinx/vale.ini", "w") + file.write(config.text) + file.close() + + +if __name__ == "__main__": + main() diff --git a/docs/.sphinx/images/Canonical-logo-4x.png b/docs/.sphinx/images/Canonical-logo-4x.png new file mode 100644 index 00000000..fd75696e Binary files /dev/null and b/docs/.sphinx/images/Canonical-logo-4x.png differ diff --git a/docs/.sphinx/images/front-page-light.pdf b/docs/.sphinx/images/front-page-light.pdf new file mode 100644 index 00000000..bb68cdf8 Binary files /dev/null and b/docs/.sphinx/images/front-page-light.pdf differ diff --git a/docs/.sphinx/images/normal-page-footer.pdf b/docs/.sphinx/images/normal-page-footer.pdf new file mode 100644 index 00000000..dfd73cbc Binary files /dev/null and b/docs/.sphinx/images/normal-page-footer.pdf differ diff --git a/docs/.sphinx/latex_elements_template.txt b/docs/.sphinx/latex_elements_template.txt new file mode 100644 index 00000000..322412c7 --- /dev/null +++ b/docs/.sphinx/latex_elements_template.txt @@ -0,0 +1,119 @@ +{ + 'papersize': 'a4paper', + 'pointsize': '11pt', + 'fncychap': '', + 'preamble': r''' +%\usepackage{charter} +%\usepackage[defaultsans]{lato} +%\usepackage{inconsolata} +\setmainfont[UprightFont = *-R, BoldFont = *-B, ItalicFont=*-RI, Extension = .ttf]{Ubuntu} +\setmonofont[UprightFont = *-R, BoldFont = *-B, ItalicFont=*-RI, Extension = .ttf]{UbuntuMono} +\usepackage[most]{tcolorbox} +\tcbuselibrary{breakable} +\usepackage{lastpage} +\usepackage{tabto} +\usepackage{ifthen} +\usepackage{etoolbox} +\usepackage{fancyhdr} +\usepackage{graphicx} +\usepackage{titlesec} +\usepackage{fontspec} +\usepackage{tikz} +\usepackage{changepage} +\usepackage{array} +\usepackage{tabularx} +\definecolor{yellowgreen}{RGB}{154, 205, 50} +\definecolor{title}{RGB}{76, 17, 48} +\definecolor{subtitle}{RGB}{116, 27, 71} +\definecolor{label}{RGB}{119, 41, 100} +\definecolor{copyright}{RGB}{174, 167, 159} +\makeatletter +\def\tcb@finalize@environment{% + \color{.}% hack for xelatex + \tcb@layer@dec% +} +\makeatother +\newenvironment{sphinxclassprompt}{\color{yellowgreen}\setmonofont[Color = 9ACD32, UprightFont = *-R, Extension = .ttf]{UbuntuMono}}{} +\tcbset{enhanced jigsaw, colback=black, fontupper=\color{white}} +\newtcolorbox{termbox}{use color stack, breakable, colupper=white, halign=flush left} +\newenvironment{sphinxclassterminal}{\setmonofont[Color = white, UprightFont = *-R, Extension = .ttf]{UbuntuMono}\sphinxsetup{VerbatimColor={black}}\begin{termbox}}{\end{termbox}} +\newcommand{\dimtorightedge}{% + \dimexpr\paperwidth-1in-\hoffset-\oddsidemargin\relax} +\newcommand{\dimtotop}{% + \dimexpr\height-1in-\voffset-\topmargin-\headheight-\headsep\relax} +\newtoggle{tpage} +\AtBeginEnvironment{titlepage}{\global\toggletrue{tpage}} +\fancypagestyle{plain}{ + \fancyhf{} + \fancyfoot[R]{\thepage\ of \pageref*{LastPage}} + \renewcommand{\headrulewidth}{0pt} + \renewcommand{\footrulewidth}{0pt} +} +\fancypagestyle{normal}{ + \fancyhf{} + \fancyfoot[R]{\thepage\ of \pageref*{LastPage}} + \renewcommand{\headrulewidth}{0pt} + \renewcommand{\footrulewidth}{0pt} +} +\fancypagestyle{titlepage}{% + \fancyhf{} + \fancyfoot[L]{\footnotesize \textcolor{copyright}{© 2024 Canonical Ltd. All rights reserved.}} +} +\newcommand\sphinxbackoftitlepage{\thispagestyle{titlepage}} +\titleformat{\chapter}[block]{\Huge \color{title} \bfseries\filright}{\thechapter .}{1.5ex}{} +\titlespacing{\chapter}{0pt}{0pt}{0pt} +\titleformat{\section}[block]{\huge \bfseries\filright}{\thesection .}{1.5ex}{} +\titlespacing{\section}{0pt}{0pt}{0pt} +\titleformat{\subsection}[block]{\Large \bfseries\filright}{\thesubsection .}{1.5ex}{} +\titlespacing{\subsection}{0pt}{0pt}{0pt} +\setcounter{tocdepth}{1} +\renewcommand\pagenumbering[1]{} +''', + 'sphinxsetup': 'verbatimwithframe=false, pre_border-radius=0pt, verbatimvisiblespace=\\phantom{}, verbatimcontinued=\\phantom{}', + 'extraclassoptions': 'openany,oneside', + 'maketitle': r''' +\begin{titlepage} +\begin{flushleft} + \begin{tikzpicture}[remember picture,overlay] + \node[anchor=south east, inner sep=0] at (current page.south east) { + \includegraphics[width=\paperwidth, height=\paperheight]{front-page-light} + }; + \end{tikzpicture} +\end{flushleft} + +\vspace*{3cm} + +\begin{adjustwidth}{8cm}{0pt} +\begin{flushleft} + \huge \textcolor{black}{\textbf{}{\raggedright{$PROJECT}}} +\end{flushleft} +\end{adjustwidth} + +\vfill + +\begin{adjustwidth}{8cm}{0pt} +\begin{tabularx}{0.5\textwidth}{ l l } + \textcolor{lightgray}{© 2024 Canonical Ltd.} & \hspace{3cm} \\ + \textcolor{lightgray}{All rights reserved.} & \hspace{3cm} \\ + & \hspace{3cm} \\ + & \hspace{3cm} \\ + +\end{tabularx} +\end{adjustwidth} + +\end{titlepage} +\RemoveFromHook{shipout/background} +\AddToHook{shipout/background}{ + \begin{tikzpicture}[remember picture,overlay] + \node[anchor=south west, align=left, inner sep=0] at (current page.south west) { + \includegraphics[width=\paperwidth]{normal-page-footer} + }; + \end{tikzpicture} + \begin{tikzpicture}[remember picture,overlay] + \node[anchor=north east, opacity=0.5, inner sep=35] at (current page.north east) { + \includegraphics[width=4cm]{Canonical-logo-4x} + }; + \end{tikzpicture} + } +''', +} \ No newline at end of file diff --git a/docs/.sphinx/metrics/build_metrics.sh b/docs/.sphinx/metrics/build_metrics.sh new file mode 100755 index 00000000..bd1ff1cb --- /dev/null +++ b/docs/.sphinx/metrics/build_metrics.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# shellcheck disable=all + +links=0 +images=0 + +# count number of links +links=$(find . -type d -path './.sphinx' -prune -o -name '*.html' -exec cat {} + | grep -o " \n" \ + "------------------------------------------------------------- \n" + +%: + $(MAKE) -f Makefile.sp sp-$@ diff --git a/docs/Makefile.sp b/docs/Makefile.sp new file mode 100644 index 00000000..beaf79df --- /dev/null +++ b/docs/Makefile.sp @@ -0,0 +1,170 @@ +# Minimal makefile for Sphinx documentation +# +# `Makefile.sp` is from the Sphinx starter pack and should not be +# modified. +# Add your customisation to `Makefile` instead. + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXDIR = .sphinx +SPHINXOPTS ?= -c . -d $(SPHINXDIR)/.doctrees -j auto +SPHINXBUILD ?= $(VENVDIR)/bin/sphinx-build +SOURCEDIR = . +BUILDDIR = _build +VENVDIR = $(SPHINXDIR)/venv +PA11Y = $(SPHINXDIR)/node_modules/pa11y/bin/pa11y.js --config $(SPHINXDIR)/pa11y.json +VENV = $(VENVDIR)/bin/activate +TARGET = * +ALLFILES = *.rst **/*.rst +METRICSDIR = $(SOURCEDIR)/.sphinx/metrics +REQPDFPACKS = latexmk fonts-freefont-otf texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-font-utils texlive-lang-cjk texlive-xetex plantuml xindy tex-gyre dvipng +CONFIRM_SUDO ?= N + +.PHONY: sp-full-help sp-woke-install sp-spellcheck-install sp-pa11y-install sp-install sp-run sp-html \ + sp-epub sp-serve sp-clean sp-clean-doc sp-spelling sp-spellcheck sp-linkcheck sp-woke \ + sp-allmetrics sp-pa11y sp-pdf-prep-force sp-pdf-prep sp-pdf Makefile.sp sp-vale sp-bash + +sp-full-help: $(VENVDIR) + @. $(VENV); $(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + @echo "\n\033[1;31mNOTE: This help texts shows unsupported targets!\033[0m" + @echo "Run 'make help' to see supported targets." + +# If requirements are updated, venv should be rebuilt and timestamped. +$(VENVDIR): + python3 -c "import venv" || \ + (echo "You must install python3-venv before you can build the documentation."; exit 1) + @echo "... setting up virtualenv" + python3 -m venv $(VENVDIR) + . $(VENV); pip install $(PIPOPTS) --require-virtualenv \ + --upgrade -r $(SPHINXDIR)/requirements.txt \ + --log $(VENVDIR)/pip_install.log + @test ! -f $(VENVDIR)/pip_list.txt || \ + mv $(VENVDIR)/pip_list.txt $(VENVDIR)/pip_list.txt.bak + @. $(VENV); pip list --local --format=freeze > $(VENVDIR)/pip_list.txt + @touch $(VENVDIR) + +sp-woke-install: + @type woke >/dev/null 2>&1 || \ + { \ + echo "Installing system-wide \"woke\" snap..."; \ + confirm_sudo=$(CONFIRM_SUDO); \ + if [ "$$confirm_sudo" != "y" ] && [ "$$confirm_sudo" != "Y" ]; then \ + read -p "This requires sudo privileges. Proceed? [y/N]: " confirm_sudo; \ + fi; \ + if [ "$$confirm_sudo" = "y" ] || [ "$$confirm_sudo" = "Y" ]; then \ + sudo snap install woke; \ + else \ + echo "Installation cancelled."; \ + fi \ + } + +sp-spellcheck-install: + @type aspell >/dev/null 2>&1 || \ + { \ + echo "Installing system-wide \"aspell\" packages..."; \ + confirm_sudo=$(CONFIRM_SUDO); \ + if [ "$$confirm_sudo" != "y" ] && [ "$$confirm_sudo" != "Y" ]; then \ + read -p "This requires sudo privileges. Proceed? [y/N]: " confirm_sudo; \ + fi; \ + if [ "$$confirm_sudo" = "y" ] || [ "$$confirm_sudo" = "Y" ]; then \ + sudo apt-get install aspell aspell-en; \ + else \ + echo "Installation cancelled."; \ + fi \ + } + +sp-pa11y-install: + @type $(PA11Y) >/dev/null 2>&1 || { \ + echo "Installing \"pa11y\" from npm... \n"; \ + mkdir -p $(SPHINXDIR)/node_modules/ ; \ + npm install --prefix $(SPHINXDIR) pa11y; \ + } + +sp-install: $(VENVDIR) + +sp-run: sp-install + . $(VENV); $(VENVDIR)/bin/sphinx-autobuild -b dirhtml "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) + +# Doesn't depend on $(BUILDDIR) to rebuild properly at every run. +sp-html: sp-install + . $(VENV); $(SPHINXBUILD) -W --keep-going -b dirhtml "$(SOURCEDIR)" "$(BUILDDIR)" -w $(SPHINXDIR)/warnings.txt $(SPHINXOPTS) + +sp-epub: sp-install + . $(VENV); $(SPHINXBUILD) -b epub "$(SOURCEDIR)" "$(BUILDDIR)" -w $(SPHINXDIR)/warnings.txt $(SPHINXOPTS) + +sp-serve: sp-html + cd "$(BUILDDIR)"; python3 -m http.server --bind 127.0.0.1 8000 + +sp-clean: sp-clean-doc + @test ! -e "$(VENVDIR)" -o -d "$(VENVDIR)" -a "$(abspath $(VENVDIR))" != "$(VENVDIR)" + rm -rf $(VENVDIR) + rm -rf $(SPHINXDIR)/node_modules/ + rm -rf $(SPHINXDIR)/styles + rm -rf $(SPHINXDIR)/vale.ini + +sp-clean-doc: + git clean -fx "$(BUILDDIR)" + rm -rf $(SPHINXDIR)/.doctrees + +sp-spellcheck: sp-spellcheck-install + . $(VENV) ; python3 -m pyspelling -c $(SPHINXDIR)/spellingcheck.yaml -j $(shell nproc) + +sp-spelling: sp-html sp-spellcheck + +sp-linkcheck: sp-install + . $(VENV) ; $(SPHINXBUILD) -b linkcheck "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) || { grep --color -F "[broken]" "$(BUILDDIR)/output.txt"; exit 1; } + exit 0 + +sp-woke: sp-woke-install + woke $(ALLFILES) --exit-1-on-failure \ + -c https://raw.githubusercontent.com/canonical/Inclusive-naming/main/config.yml + +sp-pa11y: sp-pa11y-install sp-html + find $(BUILDDIR) -name *.html -print0 | xargs -n 1 -0 $(PA11Y) + +sp-vale: sp-install + @. $(VENV); test -d $(SPHINXDIR)/venv/lib/python*/site-packages/vale || pip install vale + @. $(VENV); test -f $(SPHINXDIR)/vale.ini || python3 $(SPHINXDIR)/get_vale_conf.py + @. $(VENV); find $(SPHINXDIR)/venv/lib/python*/site-packages/vale/vale_bin -size 195c -exec vale --config "$(SPHINXDIR)/vale.ini" $(TARGET) > /dev/null \; + @cat $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt > $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt + @cat $(SPHINXDIR)/.wordlist.txt $(SOURCEDIR)/.custom_wordlist.txt >> $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt + @echo "" + @echo "Running Vale against $(TARGET). To change target set TARGET= with make command" + @echo "" + @. $(VENV); vale --config "$(SPHINXDIR)/vale.ini" --glob='*.{md,txt,rst}' $(TARGET) || true + @cat $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt > $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt && rm $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt + +sp-pdf-prep: sp-install + @for packageName in $(REQPDFPACKS); do (dpkg-query -W -f='$${Status}' $$packageName 2>/dev/null | \ + grep -c "ok installed" >/dev/null && echo "Package $$packageName is installed") && continue || \ + (echo "\nPDF generation requires the installation of the following packages: $(REQPDFPACKS)" && \ + echo "" && echo "Run 'sudo make pdf-prep-force' to install these packages" && echo "" && echo \ + "Please be aware these packages will be installed to your system") && exit 1 ; done + +sp-pdf-prep-force: + apt-get update + apt-get upgrade -y + apt-get install --no-install-recommends -y $(REQPDFPACKS) \ + +sp-pdf: sp-pdf-prep + @. $(VENV); sphinx-build -M latexpdf "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) + @rm ./$(BUILDDIR)/latex/front-page-light.pdf || true + @rm ./$(BUILDDIR)/latex/normal-page-footer.pdf || true + @find ./$(BUILDDIR)/latex -name "*.pdf" -exec mv -t ./$(BUILDDIR) {} + + @rm -r $(BUILDDIR)/latex + @echo "\nOutput can be found in ./$(BUILDDIR)\n" + +sp-allmetrics: sp-html + @echo "Recording documentation metrics..." + @echo "Checking for existence of vale..." + . $(VENV) + @. $(VENV); test -d $(SPHINXDIR)/venv/lib/python*/site-packages/vale || pip install vale + @. $(VENV); test -f $(SPHINXDIR)/vale.ini || python3 $(SPHINXDIR)/get_vale_conf.py + @. $(VENV); find $(SPHINXDIR)/venv/lib/python*/site-packages/vale/vale_bin -size 195c -exec vale --config "$(SPHINXDIR)/vale.ini" $(TARGET) > /dev/null \; + @eval '$(METRICSDIR)/source_metrics.sh $(PWD)' + @eval '$(METRICSDIR)/build_metrics.sh $(PWD) $(METRICSDIR)' + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile.sp + . $(VENV); $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/MO-1.svg b/docs/_static/MO-1.svg new file mode 100644 index 00000000..c363f5bb --- /dev/null +++ b/docs/_static/MO-1.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/_static/MO-2.svg b/docs/_static/MO-2.svg new file mode 100644 index 00000000..e24a9040 --- /dev/null +++ b/docs/_static/MO-2.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/_static/MO-3.svg b/docs/_static/MO-3.svg new file mode 100644 index 00000000..de464949 --- /dev/null +++ b/docs/_static/MO-3.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/_static/MO-4.svg b/docs/_static/MO-4.svg new file mode 100644 index 00000000..43991b00 --- /dev/null +++ b/docs/_static/MO-4.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..1ea6652a --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,324 @@ +import ast +import datetime +import os +import yaml + +# Configuration for the Sphinx documentation builder. +# All configuration specific to your project should be done in this file. +# +# If you're new to Sphinx and don't want any advanced or custom features, +# just go through the items marked 'TODO'. +# +# A complete list of built-in Sphinx configuration values: +# https://www.sphinx-doc.org/en/master/usage/configuration.html +# +# Our starter pack uses the custom Canonical Sphinx extension +# to keep all documentation based on it consistent and on brand: +# https://github.com/canonical/canonical-sphinx + + +####################### +# Project information # +####################### + +# Project name +# +# TODO: Update with the official name of your project or product + +project = "Chisel" +author = "Canonical Ltd." + + +# Sidebar documentation title; best kept reasonably short +# +# TODO: To include a version number, add it here (hardcoded or automated). +# +# TODO: To disable the title, set to an empty string. + +html_title = project + " documentation" + + +# Copyright string; shown at the bottom of the page +# +# Now, the starter pack uses CC-BY-SA as the license +# and the current year as the copyright year. +# +# TODO: If your docs need another license, specify it instead of 'CC-BY-SA'. +# +# TODO: If your documentation is a part of the code repository of your project, +# it inherits the code license instead; specify it instead of 'CC-BY-SA'. +# +# NOTE: For static works, it is common to provide the first publication year. +# Another option is to provide both the first year of publication +# and the current year, especially for docs that frequently change, +# e.g. 2022–2023 (note the en-dash). +# +# A way to check a repo's creation date is to get a classic GitHub token +# with 'repo' permissions; see https://github.com/settings/tokens +# Next, use 'curl' and 'jq' to extract the date from the API's output: +# +# curl -H 'Authorization: token ' \ +# -H 'Accept: application/vnd.github.v3.raw' \ +# https://api.github.com/repos/canonical/ | jq '.created_at' + +copyright = "%s AGPL-3.0, %s" % (datetime.date.today().year, author) + + +# Documentation website URL +# +# TODO: Update with the official URL of your docs or leave empty if unsure. +# +# NOTE: The Open Graph Protocol (OGP) enhances page display in a social graph +# and is used by social media platforms; see https://ogp.me/ + +ogp_site_url = "https://canonical-chisel.readthedocs-hosted.com/" + + +# Preview name of the documentation website +# +# TODO: To use a different name for the project in previews, update as needed. + +ogp_site_name = project + + +# Preview image URL +# +# TODO: To customise the preview image, update as needed. + +ogp_image = "https://assets.ubuntu.com/v1/253da317-image-document-ubuntudocs.svg" + + +# Product favicon; shown in bookmarks, browser tabs, etc. + +# TODO: To customise the favicon, uncomment and update as needed. + +# html_favicon = '.sphinx/_static/favicon.png' + + +# Dictionary of values to pass into the Sphinx context for all pages: +# https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-html_context + +html_context = { + # Product page URL; can be different from product docs URL + # + # TODO: Change to your product website URL, + # dropping the 'https://' prefix, e.g. 'ubuntu.com/lxd'. + # + # TODO: If there's no such website, + # remove the {{ product_page }} link from the page header template + # (usually .sphinx/_templates/header.html; also, see README.rst). + "product_page": "github.com/canonical/chisel", + # Product tag image; the orange part of your logo, shown in the page header + # + # TODO: To add a tag image, uncomment and update as needed. + # 'product_tag': '_static/tag.png', + # Your Discourse instance URL + # + # TODO: Change to your Discourse instance URL or leave empty. + # + # NOTE: If set, adding ':discourse: 123' to an .rst file + # will add a link to Discourse topic 123 at the bottom of the page. + "discourse": "https://discourse.ubuntu.com/tags/c/rocks/117/chisel", + # Your Mattermost channel URL + # + # TODO: Change to your Mattermost channel URL or leave empty. + "mattermost": "", + # Your Matrix channel URL + # + # TODO: Change to your Matrix channel URL or leave empty. + "matrix": "https://matrix.to/#/#chisel:ubuntu.com", + # Your documentation GitHub repository URL + # + # TODO: Change to your documentation GitHub repository URL or leave empty. + # + # NOTE: If set, links for viewing the documentation source files + # and creating GitHub issues are added at the bottom of each page. + "github_url": "https://github.com/canonical/chisel", + # Docs branch in the repo; used in links for viewing the source files + # + # TODO: To customise the branch, uncomment and update as needed. + # 'github_version': 'main', + # Docs location in the repo; used in links for viewing the source files + # + # TODO: To customise the directory, uncomment and update as needed. + "github_folder": "/docs/", + # TODO: To enable or disable the Previous / Next buttons at the bottom of pages + # Valid options: none, prev, next, both + "sequential_nav": "both", +} + +# Project slug; see https://meta.discourse.org/t/what-is-category-slug/87897 +# +# TODO: If your documentation is hosted on https://docs.ubuntu.com/, +# uncomment and update as needed. + +# slug = '' + + +# Template and asset locations + +html_static_path = [".sphinx/_static"] +templates_path = [".sphinx/_templates"] + + +############# +# Redirects # +############# + +# To set up redirects: https://documatt.gitlab.io/sphinx-reredirects/usage.html +# For example: 'explanation/old-name.html': '../how-to/prettify.html', + +# To set up redirects in the Read the Docs project dashboard: +# https://docs.readthedocs.io/en/stable/guides/redirects.html + +# NOTE: If undefined, set to None, or empty, +# the sphinx_reredirects extension will be disabled. + +redirects = {} + + +########################### +# Link checker exceptions # +########################### + +# A regex list of URLs that are ignored by 'make linkcheck' +# +# TODO: Remove or adjust the ACME entry after you update the contributing guide + +linkcheck_ignore = ["http://127.0.0.1:8000"] + + +# A regex list of URLs where anchors are ignored by 'make linkcheck' + +linkcheck_anchors_ignore_for_url = [r"https://github\.com/.*"] + + +######################## +# Configuration extras # +######################## + +# Custom MyST syntax extensions; see +# https://myst-parser.readthedocs.io/en/latest/syntax/optional.html +# +# NOTE: By default, the following MyST extensions are enabled: +# substitution, deflist, linkify + +# myst_enable_extensions = set() + + +# Custom Sphinx extensions; see +# https://www.sphinx-doc.org/en/master/usage/extensions/index.html + +# NOTE: The canonical_sphinx extension is required for the starter pack. +# It automatically enables the following extensions: +# - custom-rst-roles +# - myst_parser +# - notfound.extension +# - related-links +# - sphinx_copybutton +# - sphinx_design +# - sphinx_reredirects +# - sphinx_tabs.tabs +# - sphinxcontrib.jquery +# - sphinxext.opengraph +# - terminal-output +# - youtube-links + +extensions = [ + "canonical_sphinx", + "sphinxcontrib.cairosvgconverter", + "sphinx_last_updated_by_git", +] + + +# Excludes files or directories from processing + +exclude_patterns = [ + "doc-cheat-sheet*", +] + +# Adds custom CSS files, located under 'html_static_path' + +html_css_files = [ + "css/pdf.css", +] + + +# Adds custom JavaScript files, located under 'html_static_path' + +# html_js_files = [] + + +# Specifies a reST snippet to be appended to each .rst file + +if os.path.exists('./reuse/links.txt'): + rst_epilog = """ + .. include:: /reuse/links.txt + """ + +# To reuse sentences or paragraphs that have little markup and special +# formatting, use substitutions. +# https://canonical-documentation-with-sphinx-and-readthedocscom.readthedocs-hosted.com/style-guide-myst/#substitution + +if os.path.exists('./reuse/substitutions.yaml'): + with open('./reuse/substitutions.yaml', 'r') as fd: + myst_substitutions = yaml.safe_load(fd.read()) + +# Feedback button at the top; enabled by default +# +# TODO: To disable the button, uncomment this. + +# disable_feedback_button = True + + +# Your manpage URL +# +# TODO: To enable manpage links, uncomment and update as needed. +# +# NOTE: If set, adding ':manpage:' to an .rst file +# adds a link to the corresponding man section at the bottom of the page. + +# manpages_url = f'https://manpages.ubuntu.com/manpages/{codename}/en/' + \ +# f'man{section}/{page}.{section}.html' + + +# Specifies a reST snippet to be prepended to each .rst file +# This defines a :center: role that centers table cell content. +# This defines a :h2: role that styles content for use with PDF generation. + +rst_prolog = """ +.. role:: center + :class: align-center +.. role:: h2 + :class: hclass2 +""" + +# Workaround for https://github.com/canonical/canonical-sphinx/issues/34 + +if "discourse_prefix" not in html_context and "discourse" in html_context: + html_context["discourse_prefix"] = html_context["discourse"] + "/t/" + +##################### +# PDF configuration # +##################### + +latex_additional_files = [ + "./.sphinx/fonts/Ubuntu-B.ttf", + "./.sphinx/fonts/Ubuntu-R.ttf", + "./.sphinx/fonts/Ubuntu-RI.ttf", + "./.sphinx/fonts/UbuntuMono-R.ttf", + "./.sphinx/fonts/UbuntuMono-RI.ttf", + "./.sphinx/fonts/UbuntuMono-B.ttf", + "./.sphinx/images/Canonical-logo-4x.png", + "./.sphinx/images/front-page-light.pdf", + "./.sphinx/images/normal-page-footer.pdf", +] + +latex_engine = "xelatex" +latex_show_pagerefs = True +latex_show_urls = "footnote" + +with open(".sphinx/latex_elements_template.txt", "rt") as file: + latex_config = file.read() + +latex_elements = ast.literal_eval(latex_config.replace("$PROJECT", project)) diff --git a/docs/explanation/index.md b/docs/explanation/index.md new file mode 100644 index 00000000..ad132e03 --- /dev/null +++ b/docs/explanation/index.md @@ -0,0 +1,10 @@ +# Explanation + +This section of the documentation covers the core concepts of Chisel. + +```{toctree} +:maxdepth: 1 + +mode-of-operation +slices +``` diff --git a/docs/explanation/mode-of-operation.md b/docs/explanation/mode-of-operation.md new file mode 100644 index 00000000..01f573bb --- /dev/null +++ b/docs/explanation/mode-of-operation.md @@ -0,0 +1,95 @@ +(chisel_mo_explanation)= +# How it works + +The main purpose of Chisel is to create a slice of Ubuntu - a minimal root file system +comprising only the application and its runtime dependencies. + +Such task can be accomplished with the {{cut_cmd}} command, whose workflow is depicted below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +```{image} /_static/MO-1.svg + :align: center + :alt: Read and parse chisel-releases +``` + + + +Chisel fetches, reads and validates the {ref}`chisel-release`. +This includes parsing the {ref}`chisel_yaml_ref` and {ref}`slice +definitions` while validating the release and checking for conflicting paths across packages. + +
+ +```{image} /_static/MO-2.svg + :align: center + :alt: Talk to archives and fetch packages +``` + + + +Chisel talks to the {ref}`chisel_yaml_format_spec_archives` directly. +It fetches, validates and parses their `InRelease` files. +It then resolves which archive holds the **requested** packages and fetches +the corresponding package tarballs. + +
+ +```{image} /_static/MO-3.svg + :align: center + :alt: Install package slices +``` + + + +Chisel groups and merges all slice definitions per package. Then, +for every package, it extracts the **specified slices' paths** into +the provided root file system. + +
+ +```{image} /_static/MO-4.svg + :align: center + :alt: Run mutation scripts +``` + + + +Chisel then runs the {{mutation_scripts}}. Only the +{ref}`mutable` files may be +modified at this stage. Finally, the files specified with +{ref}`until:mutate` are +removed from the provided root file system. + + +
diff --git a/docs/explanation/slices.md b/docs/explanation/slices.md new file mode 100644 index 00000000..98b1cad9 --- /dev/null +++ b/docs/explanation/slices.md @@ -0,0 +1,50 @@ +(slices_explanation)= +# Slices + +## What are package slices? + +Since Debian packages are simply archives that can be inspected, navigated and +deconstructed, it is possible to define slices of packages that contain +minimal, complementary, loosely-coupled sets of files based on package metadata +and content. Such **package slices** are subsets of Debian packages, with their +own content and set of dependencies to other internal and external slices. + +The use of package slices provides the ability to build minimal root file +system from the wider set of Ubuntu packages. + +```{image} /_static/package-slices.svg + :align: center + :width: 75% + :alt: Debian package slices with dependencies +``` + +This image illustrates the simple case where, at a package level, package _B_ +depends on package _A_. However, there might be files in _A_ that _B_ doesn't +actually need, but which are provided for convenience or completeness. By +identifying the files in _A_ that are actually needed by _B_, we can divide _A_ +into slices that serve this purpose. In this example, the files in the package +slice, _A_slice3_, are not needed for _B_ to function. To make package _B_ +usable in the same way, it can also be divided into slices. + +With these slice definitions in place, Chisel is able to extract a +highly-customised and specialised slice of the Ubuntu distribution, which one +could see as a block of stone from which we can carve and extract only the +small and relevant parts that we need to run our applications, thus keeping the +file system small and less exposed to vulnerabilities. + +## Defining slices + +A package's slices can be defined via a YAML slice definitions file. Check +{ref}`slice_definitions_ref` for more information about this file's format. + +## Naming convention + +In Chisel, slices are recognized by the following pattern: +`_`. + +For example, the slice `libc6_libs` refers to the slice definition `libs` of the +package `libc6`. + + +The use of an underscore in this pattern is what distinguishes package names from +slice names, as this character is not allowed in Debian package names. diff --git a/docs/how-to/index.md b/docs/how-to/index.md new file mode 100644 index 00000000..3d17812a --- /dev/null +++ b/docs/how-to/index.md @@ -0,0 +1,17 @@ +(how_to_guides)= +# How-to guides + +These guides will walk you through steps of using Chisel through its operations +life cycle. + +## Installation + +Installation follows a broadly similar pattern on all architectures, and you +can choose to install the pre-built binary, Snap or build it from the source by +yourself. + +```{toctree} +:maxdepth: 1 + +install-chisel +``` diff --git a/docs/how-to/install-chisel.md b/docs/how-to/install-chisel.md new file mode 100644 index 00000000..1d4e8254 --- /dev/null +++ b/docs/how-to/install-chisel.md @@ -0,0 +1,110 @@ +# How to install Chisel + +To install the latest version of Chisel, you can choose any of the following +methods: + +- {ref}`install_chisel_binary` +- {ref}`install_chisel_source` +- {ref}`install_chisel_snap` + + +(install_chisel_binary)= +## Install the binary + +We publish pre-built binaries for every version of Chisel on GitHub. To install +the latest Chisel binary: + + + +1. Visit the {{latest_release_page}} to determine the latest version, for example, + `v1.0.0` (update the following steps according to your desired version and + architecture). + +2. Run the following command to download the file. + ```sh + wget https://github.com/canonical/chisel/releases/download/v1.0.0/chisel_v1.0.0_linux_amd64.tar.gz + ``` + +3. We publish checksum files for the release tarballs. Download the appropriate + checksum file with the following command. + ```sh + wget https://github.com/canonical/chisel/releases/download/v1.0.0/chisel_v1.0.0_linux_amd64.tar.gz.sha384 + ``` + +4. Verify the checksum with the following command: + ```sh + sha384sum -c chisel_v1.0.0_linux_amd64.tar.gz.sha384 + ``` + +5. Extract the contents of the downloaded tarball by running: + ```sh + tar zxvf chisel_v1.0.0_linux_amd64.tar.gz + ``` + +6. Install the Chisel binary. Make sure the installation directory is included + in your system’s `PATH` environment variable. For example: + ```sh + sudo mv chisel /usr/local/bin/ + ``` + + +(install_chisel_source)= +## Install from source + +Alternatively, you can install the latest version of Chisel from source: + +1. Follow the [official Go documentation](https://go.dev/doc/install) to + download and install Go. + +2. After installing, you will want to add the `$GOBIN` directory to your + `$PATH` so you can use the installed tools. For more information, refer to + the [official documentation](https://go.dev/doc/install/source#environment). + +3. Run the following command to build and install the latest version of Chisel: + ```sh + go install github.com/canonical/chisel/cmd/chisel@latest + ``` + + +(install_chisel_snap)= +## Install Snap + +You can also install the latest version of Chisel from the Snap store. Run the +following command to install from the `latest/stable` track: + +```sh +sudo snap install chisel +``` + +```{note} +This snap can only install the slices in a location inside the user's `$HOME` +directory, i.e. the `--root` option of the {ref}`cut_command_reference` must point to a location +inside `$HOME`. +``` + + +## Verify installation + + + +Once the installation is complete, verify that Chisel has been installed +correctly by running: + +```sh +chisel +``` + +This should produce output similar to the following: + +```{terminal} +:input: chisel + +Chisel can slice a Linux distribution using a release database +and construct a new filesystem using the finely defined parts. + +Usage: chisel [...] + +... +``` + + diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..3454534f --- /dev/null +++ b/docs/index.md @@ -0,0 +1,74 @@ +# Chisel + +**Chisel** is a tool for carving and cutting Ubuntu packages. + +It is built on the idea of {{package_slices}} - minimal, complimentary, and +loosely coupled sets of files, based on the package's metadata and content. +Slices are basically subsets of the Ubuntu packages, with their own content and +set of dependencies to other internal and external slices. + +Chisel is able to extract a highly customised and specialised _Slice_ of the +Ubuntu distribution, which one could see as a block of stone from which we can +carve and extract the small and relevant parts we need to run our applications. + +It operates similar to a package manager, but for package slices, thus being +particularly useful for supporting developers in the creation of smaller but +equally functional container images. + +--------- + +## In this documentation + +````{grid} 1 1 2 2 + +```{grid-item-card} [Tutorial](tutorial/getting-started) + +**Get started** - become familiar with Chisel by slicing Ubuntu packages to create +a minimal root file system. +``` + +```{grid-item-card} [How-to guides](how-to/index) + +**Step-by-step guides** - learn key operations and common tasks. +``` + +```` + +````{grid} 1 1 2 2 +:reverse: + +```{grid-item-card} [Reference](reference/index) + +**Technical information** - understand the CLI commands, slice definitions files +and Chisel manifests. +``` + +```{grid-item-card} [Explanations](explanation/index) + +**Discussion and clarification** - explore Chisel's mode of operation and learn +about fundamental topics such as package slices. +``` + +```` + +--------- + +## Project and community + +Chisel is free software and released under {{AGPL3}}. + +The Chisel project is sponsored by {{Canonical}}. + +- [Code of conduct](https://ubuntu.com/community/ethos/code-of-conduct) +- [Contribute](https://github.com/canonical/chisel) + + +```{toctree} +:hidden: +:maxdepth: 2 + +Tutorial +how-to/index +explanation/index +reference/index +``` diff --git a/docs/reference/chisel-releases/chisel.yaml.md b/docs/reference/chisel-releases/chisel.yaml.md new file mode 100644 index 00000000..8b53f1b8 --- /dev/null +++ b/docs/reference/chisel-releases/chisel.yaml.md @@ -0,0 +1,306 @@ +(chisel_yaml_ref)= +# chisel.yaml + +The `chisel.yaml` file defines various configuration values for Chisel, +for a given _chisel-release_. + + +(chisel_yaml_location)= +## Location + +The file must be placed in the root level of a {ref}`chisel-releases_ref` +directory. + + +(chisel_yaml_format_spec)= +## Format specification + +(chisel_yaml_format_spec_format)= +### `format` + +| Field | Type | Required | Supported values | +| -------- | ----- | -------- | ---------------- | +| `format` | `str` | Required | `v1` | + +Used to define the supported schemas for the {ref}`chisel-releases_ref`. +For example: + +```yaml +format: v1 +``` + +```{note} +New formats are typically introduced with new _chisel-releases_ and may +introduce disruptive changes to the previous formats. +``` + + +(chisel_yaml_format_spec_archives)= +### `archives` + +| Field | Type | Required | +| ---------- | ----------------- | -------- | +| `archives` | `dict(str, dict)` | Required | + +Tells Chisel which Ubuntu archives to fetch packages from. + +```{note} +Chisel only supports fetching packages from the official Ubuntu archives, +including ESM. +``` + +For example: + +```yaml +archives: + ubuntu-esm-apps: + pro: esm-apps + priority: 16 + version: 24.04 + components: [main] + suites: [noble-apps-security, noble-apps-updates] + public-keys: [ubuntu-apps-key] +``` + +If {ref}`chisel_yaml_format_spec_archives_pro` is not specified, the archives +point to: + +- http://archive.ubuntu.com/ for `amd64` and `i386` package architecture, and +- http://ports.ubuntu.com/ubuntu-ports/ for other architectures. + +otherwise, the archive point to the Ubuntu Pro archives listed +{ref}`below`. + + +(chisel_yaml_format_spec_archives_default)= +### `archives..default` + +| Field | Type | Required | Supported values | +| --------- | ------ | --------------------------------------------------------------------------------------------------- | ---------------- | +| `default` | `bool` | Required with multiple archives, if no {ref}`priorities` | `true`, `false` | + +If `default` is `true`, Chisel fetches packages from this archive, unless +otherwise specified by the field {ref}`"archive"` +in the slice definitions file. + +In case there are multiple archives, one, **and only one**, must be the default, +**otherwise**, use {ref}`priorities`. + +```{tip} +{ref}`chisel_yaml_format_spec_archives_default` will soon be deprecated, +so use {ref}`chisel_yaml_format_spec_archives_priority` instead. +``` + + +(chisel_yaml_format_spec_archives_version)= +### `archives..version` + +| Field | Type | Required | Supported values | +| --------- | ----- | -------- | ------------------------------------------------------- | +| `version` | `str` | Required | Ubuntu release in `xx.yy` format e.g. 22.04, 24.04 etc. | + +Indicates the Ubuntu release this archive should fetch the +packages for. This value is currently only used for logging, and does not change +the archive behaviour. + + +(chisel_yaml_format_spec_archives_suites)= +### `archives..suites` + +| Field | Type | Required | Supported values | +| -------- | ----------- | -------- | ------------------------------------------------------------- | +| `suites` | `list(str)` | Required | Ubuntu archive suite names e.g. `jammy`, `noble-updates` etc. | + +Lists the archive suites to fetch packages from. Read more +about suites in the [Ubuntu packaging +guide](https://canonical-ubuntu-packaging-guide.readthedocs-hosted.com/en/latest/explanation/archive/#suite). + + +(chisel_yaml_format_spec_archives_components)= +### `archives..components` + +| Field | Type | Required | Supported values | +| ------------ | ----------- | -------- | -------------------------------------------------- | +| `components` | `list(str)` | Required | Suite component names e.g. `main`, `universe` etc. | + +Lists the components of the archive suites to fetch +packages from. Read more about components in the [Ubuntu packaging +guide](https://canonical-ubuntu-packaging-guide.readthedocs-hosted.com/en/latest/explanation/archive/#components). + +Chisel reads the `InRelease` files from each `(suite, component)` combination to +locate packages. + + +(chisel_yaml_format_spec_archives_public_keys)= +### `archives..public-keys` + +| Field | Type | Required | Supported values | +| ------------- | ----------- | -------- | --------------------------------------------------------------------------- | +| `public-keys` | `list(str)` | Required | List of key names, as defined in {ref}`chisel_yaml_format_spec_public_keys` | + +Lists the names of the OpenPGP public keys needed to verify the archive's `InRelease` +file signatures. These key names must be defined in +{ref}`chisel_yaml_format_spec_public_keys`. + + +(chisel_yaml_format_spec_archives_priority)= +### `archives..priority` + +| Field | Type | Required | Supported values | +| ---------- | ----- | ------------------------------------------------ | ---------------------------------- | +| `priority` | `int` | Required with multiple archives, if no `default` | Any integer between -1000 and 1000 | + +Describes the priority of an archive compared to other +archives. It is used to support multiple archives in Chisel. If a package is +available in two archives, it is fetched from the archive with higher priority, +unless the package's slice definitions file specifies {ref}`"archive"`. + +Note that: +- an unspecified `priority` field **does not** yield a 0 value, and +- two archives cannot have the same `priority` value. + + +(chisel_yaml_format_spec_archives_pro)= +### `archives..pro` + +| Field | Type | Required | Supported values | +| ----- | ----- | -------- | ------------------------------------------------ | +| `pro` | `str` | Optional | `fips`, `fips-updates`, `esm-apps`, `esm-infra`. | + +Specifies the [Ubuntu Pro](https://ubuntu.com/pro) archive to +fetch and install packages from. + +```{important} +To chisel a Pro package you need to have a Pro-enabled host. +``` + +Chisel reads the Pro archives' credentials from the directory defined by the +environment variable `CHISEL_AUTH_DIR` (which defaults to `/etc/apt/auth.conf.d`). + +The following `pro` values are supported, and if specified, the +archive points to their corresponding base URLs. + +| `pro` value | Base repository URL | +| -------------- | ------------------------------------------ | +| `fips` | https://esm.ubuntu.com/fips/ubuntu | +| `fips-updates` | https://esm.ubuntu.com/fips-updates/ubuntu | +| `esm-apps` | https://esm.ubuntu.com/apps/ubuntu | +| `esm-infra` | https://esm.ubuntu.com/infra/ubuntu | + +```{tip} +Although not enforced, the following `priority` values are suggested when +`pro` is used: + +| `pro` value | Suggested `priority` | +| ---------------------------------------------------------------------------------- | -------------------- | +| `fips` | 20 | +| `fips-updates` | 21 | +| `esm-apps` | 16 | +| `esm-infra` | 15 | +| `""` (empty, indicates a {ref}`non-Pro archive`) | 10 | +``` + +(chisel_yaml_format_spec_public_keys)= +### `public-keys` + +| Field | Type | Required | +| ------------- | ----------------- | -------- | +| `public-keys` | `dict(str, dict)` | Required | + +The top-level `public-keys` field is used to define OpenPGP public keys that are +needed to verify the `InRelease` file signatures of the +{ref}`chisel_yaml_format_spec_archives`. + +For example: + +```yaml +public-keys: + ubuntu-archive-key-2018: + id: 871920D1991BC93C + armor: | # Armored ASCII data + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFufwdoBEADv/Gxytx/LcSXYuM0MwKojbBye81s0G1nEx+lz6VAUpIUZnbkq + ... + -----END PGP PUBLIC KEY BLOCK----- +``` + +The key names are then referenced in +{ref}`chisel_yaml_format_spec_archives_public_keys` as needed. + + +(chisel_yaml_format_spec_public_keys_id)= +### `public-keys..id` + +| Field | Type | Required | +| ----- | ----- | -------- | +| `id` | `str` | Required | + +The `id` field specifies the OpenPGP public key fingerprint in capital hex e.g. +`871920D1991BC93C`. It must be 16 chars long and must match the decoded +fingerprint in {ref}`chisel_yaml_format_spec_public_keys_armor`. + + +(chisel_yaml_format_spec_public_keys_armor)= +### `public-keys..armor` + +| Field | Type | Required | +| ------- | ----- | -------- | +| `armor` | `str` | Required | + +The `armor` field contains the multi-line armored ASCII data of OpenPGP public +key. + + +(chisel_yaml_example)= +## Example + +The following `chisel.yaml` is used in Ubuntu 24.04 (Noble) release: + +```yaml +format: v1 + +archives: + ubuntu: + default: true + version: 24.04 + components: [main, universe] + suites: [noble, noble-security, noble-updates] + public-keys: [ubuntu-archive-key-2018] + +public-keys: + # Ubuntu Archive Automatic Signing Key (2018) + # rsa4096/f6ecb3762474eda9d21b7022871920d1991bc93c 2018-09-17T15:01:46Z + ubuntu-archive-key-2018: + id: "871920D1991BC93C" + armor: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFufwdoBEADv/Gxytx/LcSXYuM0MwKojbBye81s0G1nEx+lz6VAUpIUZnbkq + dXBHC+dwrGS/CeeLuAjPRLU8AoxE/jjvZVp8xFGEWHYdklqXGZ/gJfP5d3fIUBtZ + HZEJl8B8m9pMHf/AQQdsC+YzizSG5t5Mhnotw044LXtdEEkx2t6Jz0OGrh+5Ioxq + X7pZiq6Cv19BohaUioKMdp7ES6RYfN7ol6HSLFlrMXtVfh/ijpN9j3ZhVGVeRC8k + KHQsJ5PkIbmvxBiUh7SJmfZUx0IQhNMaDHXfdZAGNtnhzzNReb1FqNLSVkrS/Pns + AQzMhG1BDm2VOSF64jebKXffFqM5LXRQTeqTLsjUbbrqR6s/GCO8UF7jfUj6I7ta + LygmsHO/JD4jpKRC0gbpUBfaiJyLvuepx3kWoqL3sN0LhlMI80+fA7GTvoOx4tpq + VlzlE6TajYu+jfW3QpOFS5ewEMdL26hzxsZg/geZvTbArcP+OsJKRmhv4kNo6Ayd + yHQ/3ZV/f3X9mT3/SPLbJaumkgp3Yzd6t5PeBu+ZQk/mN5WNNuaihNEV7llb1Zhv + Y0Fxu9BVd/BNl0rzuxp3rIinB2TX2SCg7wE5xXkwXuQ/2eTDE0v0HlGntkuZjGow + DZkxHZQSxZVOzdZCRVaX/WEFLpKa2AQpw5RJrQ4oZ/OfifXyJzP27o03wQARAQAB + tEJVYnVudHUgQXJjaGl2ZSBBdXRvbWF0aWMgU2lnbmluZyBLZXkgKDIwMTgpIDxm + dHBtYXN0ZXJAdWJ1bnR1LmNvbT6JAjgEEwEKACIFAlufwdoCGwMGCwkIBwMCBhUI + AgkKCwQWAgMBAh4BAheAAAoJEIcZINGZG8k8LHMQAKS2cnxz/5WaoCOWArf5g6UH + beOCgc5DBm0hCuFDZWWv427aGei3CPuLw0DGLCXZdyc5dqE8mvjMlOmmAKKlj1uG + g3TYCbQWjWPeMnBPZbkFgkZoXJ7/6CB7bWRht1sHzpt1LTZ+SYDwOwJ68QRp7DRa + Zl9Y6QiUbeuhq2DUcTofVbBxbhrckN4ZteLvm+/nG9m/ciopc66LwRdkxqfJ32Cy + q+1TS5VaIJDG7DWziG+Kbu6qCDM4QNlg3LH7p14CrRxAbc4lvohRgsV4eQqsIcdF + kuVY5HPPj2K8TqpY6STe8Gh0aprG1RV8ZKay3KSMpnyV1fAKn4fM9byiLzQAovC0 + LZ9MMMsrAS/45AvC3IEKSShjLFn1X1dRCiO6/7jmZEoZtAp53hkf8SMBsi78hVNr + BumZwfIdBA1v22+LY4xQK8q4XCoRcA9G+pvzU9YVW7cRnDZZGl0uwOw7z9PkQBF5 + KFKjWDz4fCk+K6+YtGpovGKekGBb8I7EA6UpvPgqA/QdI0t1IBP0N06RQcs1fUaA + QEtz6DGy5zkRhR4pGSZn+dFET7PdAjEK84y7BdY4t+U1jcSIvBj0F2B7LwRL7xGp + SpIKi/ekAXLs117bvFHaCvmUYN7JVp1GMmVFxhIdx6CFm3fxG8QjNb5tere/YqK+ + uOgcXny1UlwtCUzlrSaP + =9AdM + -----END PGP PUBLIC KEY BLOCK----- +``` diff --git a/docs/reference/chisel-releases/index.md b/docs/reference/chisel-releases/index.md new file mode 100644 index 00000000..0b433ab2 --- /dev/null +++ b/docs/reference/chisel-releases/index.md @@ -0,0 +1,31 @@ +(chisel-releases_ref)= +# chisel-releases + +Chisel uses **slice definitions files** (aka SDFs) to define the slices of packages. +SDFs are YAML files, and there is one per package and per Ubuntu release, named +after the package name. + +For a given Ubuntu release, the collection of SDFs plus a configuration file +named `chisel.yaml` form what is called a _chisel-release_. + +The {{chisel_releases_repo}} contains a number of branches for various +_chisel-releases_, matching the corresponding Ubuntu releases. + +A _chisel-release_ is simply a directory with the following structure: + +``` +├── chisel.yaml +└── slices + ├── pkgA.yaml + ├── pkgB.yaml + └── ... +``` + +The following pages provide more details on: + +```{toctree} +:maxdepth: 1 + +chisel.yaml +slice-definitions +``` diff --git a/docs/reference/chisel-releases/slice-definitions.md b/docs/reference/chisel-releases/slice-definitions.md new file mode 100644 index 00000000..dfb4d52e --- /dev/null +++ b/docs/reference/chisel-releases/slice-definitions.md @@ -0,0 +1,436 @@ +(slice_definitions_ref)= +# Slice definitions + +{ref}`Slices ` are described in slice definitions files (aka SDFs). +These are YAML files, named after the package name. + + +(slice_definitions_location)= +## Location + +The slice definitions files are located in the `slices/` directory of +{ref}`chisel-releases_ref`. And the slices for a given package `hello` are defined in +`slices/hello.yaml`. + +```{tip} +Although the `hello.yaml` file can be placed in a sub-directory of `slices/` e.g. +`slices/dir/hello.yaml`, it is generally recommended to keep them at +`slices/hello.yaml`. The {{chisel_releases_repo}} follows the latter. +``` + + +(slice_definitions_format)= +## Format specification + +(slice_definitions_format_package)= +### `package` + +| Field | Type | Required | +| --------- | ----- | -------- | +| `package` | `str` | Required | + +Indicates the package name. It must follow the +[Debian policy for package name](https://www.debian.org/doc/debian-policy/ch-binary.html#the-package-name). + +As indicated above, the value must also match the YAML file basename. +For example: + +```yaml +package: hello +``` + + +(slice_definitions_format_archive)= +### `archive` + +| Field | Type | Required | Supported values | +| --------- | ----- | -------- | --------------------------------------------------------------------- | +| `archive` | `str` | Optional | Archive name, from {ref}`archives`. | + +Specifies a particular +{ref}`archive` from where this package should be +fetched from. If specified, Chisel fetches this package from that archive despite the +{ref}`chisel_yaml_format_spec_archives_default` and +{ref}`chisel_yaml_format_spec_archives_priority` settings in +{ref}`chisel_yaml_ref`. + +The archive name must be defined in {ref}`chisel_yaml_format_spec_archives`. +For example: + +```yaml +archive: ubuntu +``` + + +(slice_definitions_format_essential)= +### `essential` + +| Field | Type | Required | Supported values | +| ----------- | ----------- | -------- | ------------------ | +| `essential` | `list(str)` | Optional | An existing slice. | + +Lists the slices that are needed for **every slice** of the +current package. Slices in this list must be written in their full name, e.g. +`hello_copyright`. This field is similar to +{ref}`slice_definitions_format_slices_essential`, but applicable for every slice +within the package. + +In the following example, the `hello_copyright` slice is an _essential_ for +every slice including the `hello_bins` slice. + +```yaml +package: hello +essential: + - hello_copyright +slices: + bins: + contents: + ... + copyright: + ... +``` + + +(slice_definitions_format_slices)= +### `slices` + +| Field | Type | Required | +| -------- | ----------------- | -------- | +| `slices` | `dict(str, dict)` | Required | + +Defines the slices of a package. + +The slice names must consist only of lower case letters(`a-z`), digits(`0-9`) +and minus (`-`) signs. They must be at least three characters long and must start +with an alphanumeric character. + +For example, a slice definition called `data`, for the `ca-certificates` package, +could look like the following: + +```yaml +slices: + data: + essential: + - openssl_data + contents: + /etc/ssl/certs/ca-certificates.crt: {text: FIXME, mutable: true} + /usr/share/ca-certificates/mozilla/: {until: mutate} + /usr/share/ca-certificates/mozilla/**: {until: mutate} + mutate: | + certs_dir = "/usr/share/ca-certificates/mozilla/" + certs = [ + content.read(certs_dir + path) for path in content.list(certs_dir) + ] + content.write("/etc/ssl/certs/ca-certificates.crt", "".join(certs)) +``` + + +(slice_definitions_format_slices_essential)= +### `slices..essential` + +| Field | Type | Required | Supported values | +| ----------- | ----------- | -------- | ------------------ | +| `essential` | `list(str)` | Optional | An existing slice. | + +Lists the slices that are needed and that must be installed before the current slice. +Slices in this list must be written in their full name +e.g. `hello_copyright`. This field is similar to +{ref}`slice_definitions_format_essential`, but only applicable for the current +slice. + +In the following example, `libc6_libs` is a requirement for the `bins` +slice and must be installed when installing the `bins` slice. + +```yaml +slices: + bins: + essential: + - libc6_libs +``` + + +(slice_definitions_format_slices_contents)= +### `slices..contents` + +| Field | Type | Required | +| ---------- | ----------------- | -------- | +| `contents` | `dict(str, dict)` | Optional | + +Describes the paths that come from this slice. + +```{note} +Paths must be absolute and must start with `/`. + + +Also, paths can have wildcard characters (`?`, `*` and `**`), where + * `?` matches any one character, except for `/`, + * `*` matches zero or more characters, except for `/`, and + * `**` matches zero or more characters, including `/`. +``` + +(slice_definitions_format_slices_contents_copy)= +### `slices..contents..copy` + +| Field | Type | Required | +| ------ | ----- | -------- | +| `copy` | `str` | Optional | + +The `copy` field refers to the path Chisel should copy the target path from. + +In the following example, Chisel copies the `/bin/original` file from the +package onto `/bin/moved`. + +```yaml + contents: + /bin/moved: {copy: /bin/original} +``` + +```{note} +This field is only applicable to paths with no wildcards, and its value +must also be an absolute path with no wildcards. +``` + +(slice_definitions_format_slices_contents_make)= +### `slices..contents..make` + +| Field | Type | Required | Supported values | +| ------ | ------ | -------- | ---------------- | +| `make` | `bool` | Optional | `true`, `false` | + +If `make` is true, Chisel creates the specified directory path. Note that, the +path must be an absolute directory path with a trailing `/`. If +{ref}`mode` is not specified, Chisel +creates the directory with `0755`. + +```yaml + contents: + /path/to/dir/: {make: true} +``` + +```{note} +This field is only applicable for paths with no wildcards. +``` + +(slice_definitions_format_slices_contents_text)= +### `slices..contents..text` + +| Field | Type | Required | +| ------ | ----- | -------- | +| `text` | `str` | Optional | + +The `text` field instructs Chisel to create a text file with the specified +value as the file content. If empty, Chisel creates an empty file of 0 bytes. + +In the following example, `/file` is created with the content `Hello world!`. +If {ref}`mode` is not specified, +Chisel creates the file with `0644`. + +```yaml + contents: + /file: {text: "Hello world!"} +``` + +```{note} +This field is only applicable for paths with no wildcards. +``` + +(slice_definitions_format_slices_contents_symlink)= +### `slices..contents..symlink` + +| Field | Type | Required | +| --------- | ----- | -------- | +| `symlink` | `str` | Optional | + +The `symlink` field is used to create symbolic links. If specified, Chisel +creates a symlink to the target path specified by the `symlink` value. The value +must be an absolute path with no wildcards. + +In the following example, Chisel creates the symlink `/link` which points to +`/file`. + +```yaml + contents: + /link: {symlink: /file} +``` + +```{note} +This field is only applicable for paths with no wildcards. +``` + +(slice_definitions_format_slices_contents_mode)= +### `slices..contents..mode` + +| Field | Type | Required | +| ------ | ----- | -------- | +| `mode` | `int` | Optional | + +The `mode` field is used to specify the permission bits for any path Chisel +creates. It takes in a 32 bit unsigned integer, preferably in an octal value +format e.g. `0755` or `0o755`. For example: + +```yaml + contents: + /file: {text: "Hello world!", mode: 0755} +``` + +```{note} +It can only be used with +{ref}`copy`, +{ref}`make` and +{ref}`text`. + +This field is only applicable for paths with no wildcards. +``` + +(slice_definitions_format_slices_contents_arch)= +### `slices..contents..arch` + +| Field | Type | Required | Supported values | +| ------ | -------------------- | -------- | ---------------------------------------------------------------- | +| `arch` | `str` or `list(str)` | Optional | `amd64`, `arm64`, `armhf`, `i386`, `ppc64el`, `riscv64`, `s390x` | + +Used to specify the package architectures a path should be +installed for. This field can take a single architecture string or a list, as its +value. + +In the following example, `/foo` will be installed for `i386` installations and +`/bar` will be installed for `amd64` or `arm64` installations. + +```yaml + contents: + /foo: {arch: i386} + /bar: {arch: [amd64, arm64]} +``` + + +(slice_definitions_format_slices_contents_mutable)= +### `slices..contents..mutable` + +| Field | Type | Required | Supported values | +| --------- | ------ | -------- | ---------------- | +| `mutable` | `bool` | Optional | `true`, `false` | + +If `mutable` f set to `true`, indicates that this path can be later +_mutated_ (modified) by the {{mutation_scripts}}. + + +(slice_definitions_format_slices_contents_until)= +### `slices..contents..until` + +| Field | Type | Required | Supported values | +| ------- | ----- | -------- | ---------------- | +| `until` | `str` | Optional | `mutate` | + +The `until` field indicates that the path will be available until a certain +event takes place. The file is eventually not installed in the final root file +system. + +It currently accepts only one value - `mutate`. If specified, the path will only +be available until the {{mutation_scripts}} execute, and is removed afterwards. + +In the following example, `/file` will not be installed in the final root file +system but will exist throughout the execution of the {{mutation_scripts}}. + +```yaml + contents: + /file: {until: mutate} +``` + + +(slice_definitions_format_slices_contents_generate)= +### `slices..contents..generate` + +| Field | Type | Required | Supported values | +| ---------- | ----- | -------- | ---------------- | +| `generate` | `str` | Optional | `manifest` | + +Used to specify the location where Chisel should produce metadata at. The path +this field applies to must not have any other fields applied to it. + +The specified path must be a directory and must end with double-asterisks (`**`). +Additionally, the path must not contain any other wildcard characters except the trailing double-asterisks (`**`). + +Currently, `generate` only accepts one value - `manifest`. If specified, Chisel +creates the {ref}`chisel_manifest_ref` file in that directory. + +In the following example, Chisel creates the `/var/lib/chisel` directory with +`0755` mode and produces a {ref}`"manifest.wall"` file within the directory. + +```yaml + contents: + /var/lib/chisel/**: {generate: manifest} +``` + + +(slice_definitions_format_slices_mutate)= +### `slices..mutate` + +| Field | Type | Required | Supported values | +| -------- | ----- | -------- | ------------------- | +| `mutate` | `str` | Optional | {{Starlark}} script | + +Describes a slice's mutation scripts. The mutation scripts are conceptually similar +to [Debian's maintainer +script](https://www.debian.org/doc/debian-policy/ch-maintainerscripts.html). + +The mutation scripts are written in Google's {{Starlark}} language and are executed +after the files of every slice have been installed in the root file system. The +mutation scripts are run once per each installed slice, in the same order of +slices. + +In addition to {{Starlark}}'s native syntax, Chisel introduces the following +functions: + +| Function | Return type | Description | +| --------------------- | ----------- | ---------------------------------------------------------------- | +| `content.list(d)` | `list(str)` | Lists and returns directory `d`'s contents (similar to GNU `ls`) | +| `content.read(f)` | `str` | Reads a text file `f` and returns its contents | +| `content.write(f, s)` | - | Writes the text content `s` to a file `f` | + +Reusing the above {ref}`"ca-certificates_data"` example, +Chisel initially creates the `/etc/ssl/certs/ca-certificates.crt` text file with `FIXME` as its content. When the mutation scripts execute, Chisel concatenates the contents of +every file in the `/usr/share/ca-certificates/mozilla/` directory and writes the +concatenated data to the previously created `/etc/ssl/certs/ca-certificates.crt` +file. + +```yaml + contents: + /etc/ssl/certs/ca-certificates.crt: {text: FIXME, mutable: true} + /usr/share/ca-certificates/mozilla/: {until: mutate} + /usr/share/ca-certificates/mozilla/**: {until: mutate} + mutate: | + certs_dir = "/usr/share/ca-certificates/mozilla/" + certs = [ + content.read(certs_dir + path) for path in content.list(certs_dir) + ] + content.write("/etc/ssl/certs/ca-certificates.crt", "".join(certs)) +``` + +Due to the usage of `until`, the `/usr/share/ca-certificates/mozilla/` directory +and the files inside are not present in the final root file system. + + +(slice_definitions_example)= +## Example + +The slice definitions files can be found in the {{chisel_releases_repo}}, or +inspected via the {{info_cmd}}. Here is a short example of the `hello` package +slice definitions: + +```yaml +package: hello + +essential: + - hello_copyright + +slices: + bins: + essential: + - libc6_libs + contents: + /usr/bin/hello: + + copyright: + contents: + /usr/share/doc/hello/copyright: +``` diff --git a/docs/reference/cmd/cut.md b/docs/reference/cmd/cut.md new file mode 100644 index 00000000..11419d01 --- /dev/null +++ b/docs/reference/cmd/cut.md @@ -0,0 +1,54 @@ +(cut_command_reference)= +# cut command + +The **cut** command uses the provided selection of package slices to create a +new file system tree in the root location. + +By default it fetches the slices for the same Ubuntu version as the +current host, unless the `--release` option is used. + +## Options + + + +- `--release` is a {{chisel_releases_repo}} branch or local directory (e.g. ubuntu-22.04). +- `--root` is the path for the resulting root file system. +- `--arch` is used to specify the desired package architecture. + + + +## Example + +To install the `hello_bins` slice from Ubuntu 24.04 for `amd64` architecture, +we can run the following: + + + +```{terminal} +:input: chisel cut --release ubuntu-24.04 --root rootfs/ hello_bins + +2024/11/26 12:21:35 Consulting release repository... +2024/11/26 12:21:37 Cached ubuntu-24.04 release is still up-to-date. +2024/11/26 12:21:37 Processing ubuntu-24.04 release... +2024/11/26 12:22:12 Selecting slices... +2024/11/26 12:22:12 Fetching ubuntu 24.04 noble suite details... +2024/11/26 12:22:15 Release date: Thu, 25 Apr 2024 15:10:33 UTC +2024/11/26 12:22:15 Fetching index for ubuntu 24.04 noble main component... +2024/11/26 12:22:15 Fetching index for ubuntu 24.04 noble universe component... +2024/11/26 12:22:16 Fetching ubuntu 24.04 noble-security suite details... +2024/11/26 12:22:16 Release date: Tue, 26 Nov 2024 3:33:31 UTC +2024/11/26 12:22:16 Fetching index for ubuntu 24.04 noble-security main component... +2024/11/26 12:22:16 Fetching index for ubuntu 24.04 noble-security universe component... +2024/11/26 12:22:16 Fetching ubuntu 24.04 noble-updates suite details... +2024/11/26 12:22:17 Release date: Tue, 26 Nov 2024 5:53:50 UTC +2024/11/26 12:22:17 Fetching index for ubuntu 24.04 noble-updates main component... +2024/11/26 12:22:18 Fetching index for ubuntu 24.04 noble-updates universe component... +2024/11/26 12:22:19 Fetching pool/main/b/base-files/base-files_13ubuntu10.1_amd64.deb... +2024/11/26 12:22:19 Fetching pool/main/h/hello/hello_2.10-3build1_amd64.deb... +2024/11/26 12:22:19 Fetching pool/main/g/glibc/libc6_2.39-0ubuntu8.3_amd64.deb... +2024/11/26 12:22:19 Extracting files from package "base-files"... +2024/11/26 12:22:19 Extracting files from package "hello"... +2024/11/26 12:22:19 Extracting files from package "libc6"... +``` + + diff --git a/docs/reference/cmd/find.md b/docs/reference/cmd/find.md new file mode 100644 index 00000000..43870b99 --- /dev/null +++ b/docs/reference/cmd/find.md @@ -0,0 +1,45 @@ +# find command + +The **find** command queries {{chisel_releases_repo}} for matching slices. + +Globs (`*` and `?`) are allowed in the query. + +By default it fetches the slices for the same Ubuntu version as the +current host, unless the `--release` option is used. + +## Options + +- `--release` is a {{chisel_releases_repo}} branch or local directory (e.g. ubuntu-22.04). + +## Example + +Run the following command to search python3.10 slices in `ubuntu-22.04` release: + +```{terminal} +:input: chisel find --release ubuntu-22.04 python3.10 + +2024/11/26 13:11:08 Consulting release repository... +2024/11/26 13:11:10 Fetching current ubuntu-22.04 release... +2024/11/26 13:11:10 Processing ubuntu-22.04 release... +Slice Summary +python3.10_copyright - +python3.10_core - +python3.10_standard - +python3.10_utils - +python3.11_copyright - +python3.11_core - +python3.11_standard - +python3.11_utils - +``` + +```{note} +Notice that there are some python3.11 slices in the output as well. This is +because the find command finds partially-matched slices with +{{Levenshtein_distance}} of up to 1. +``` + +The first three lines are logs, which you can ignore with: + +```sh +chisel find --release ubuntu-22.04 python3.10 2>logs +``` diff --git a/docs/reference/cmd/help.md b/docs/reference/cmd/help.md new file mode 100644 index 00000000..be8812b9 --- /dev/null +++ b/docs/reference/cmd/help.md @@ -0,0 +1,29 @@ +# help command + +The **help** command displays information about commands. + + +## Example + +```{terminal} +:scroll: +:input: chisel help --all + +Chisel can slice a Linux distribution using a release database +and construct a new filesystem using the finely defined parts. + +Usage: chisel [...] + +Commands can be classified as follows: + + Basic (general operations): + find Find existing slices + info Show information about package slices + help Show help about a command + version Show version details + + Action (make things happen): + cut Cut a tree with selected slices + +For more information about a command, run 'chisel help '. +``` diff --git a/docs/reference/cmd/index.md b/docs/reference/cmd/index.md new file mode 100644 index 00000000..b92dbe85 --- /dev/null +++ b/docs/reference/cmd/index.md @@ -0,0 +1,18 @@ +# CLI Commands + +Chisel uses subcommands, and is invoked using `chisel `. + +Subcommands are keywords that invoke a new set of options and features, and each +Chisel subcommand has its own set of flags. + +Here’s the list of all Chisel subcommands: + +```{toctree} +:maxdepth: 1 + +cut +find +info +version +help +``` diff --git a/docs/reference/cmd/info.md b/docs/reference/cmd/info.md new file mode 100644 index 00000000..5d21e264 --- /dev/null +++ b/docs/reference/cmd/info.md @@ -0,0 +1,51 @@ +# info command + +The **info** command shows detailed information about package slices. + +It complements the {{find_cmd}}. Whereas the `find` command searches for existing +slices, the `info` command takes in slice or package name(s) and displays the +slice definition(s). + +It accepts a whitespace-separated list of strings. The list can be +composed of package names, slice names, or a combination of both. The +default output format is YAML. When multiple arguments are provided, +the output is a list of YAML documents separated by a `---` line. + +Slice definitions are shown according to their definition in +the selected release. For example, globs are not expanded. + + +## Options + +- `--release` is a {{chisel_releases_repo}} branch or local directory (e.g. ubuntu-22.04). + +## Example + +The following illustrates using the `info` command to inspect multiple slices: + +```{terminal} +:input: chisel info hello libgcc-s1_libs 2>logs + +package: hello +archive: ubuntu +slices: + bins: + essential: + - hello_copyright + - libc6_libs + contents: + /usr/bin/hello: {} + copyright: + contents: + /usr/share/doc/hello/copyright: {} +--- +package: libgcc-s1 +archive: ubuntu +slices: + libs: + essential: + - libgcc-s1_copyright + - libc6_libs + contents: + /usr/lib/*-linux-*/libgcc_s.so.*: {} +``` diff --git a/docs/reference/cmd/version.md b/docs/reference/cmd/version.md new file mode 100644 index 00000000..eefa0ccf --- /dev/null +++ b/docs/reference/cmd/version.md @@ -0,0 +1,12 @@ +# version command + +The **version** command displays the Chisel version and exits. + + +## Example + +```{terminal} +:input: chisel version + +v1.0.0 +``` diff --git a/docs/reference/index.md b/docs/reference/index.md new file mode 100644 index 00000000..2744852e --- /dev/null +++ b/docs/reference/index.md @@ -0,0 +1,9 @@ +# Reference + +```{toctree} +:maxdepth: 1 + +cmd/index +chisel-releases/index +Manifest +``` diff --git a/docs/reference/manifest.md b/docs/reference/manifest.md new file mode 100644 index 00000000..77ac423c --- /dev/null +++ b/docs/reference/manifest.md @@ -0,0 +1,157 @@ +(chisel_manifest_ref)= +# Chisel Manifest + +The Chisel manifest is a ZSTD-compressed file which lists the metadata about +installed packages, slices and files. The uncompressed file is in the +["jsonwall" format](chisel_manifest_format) - a collection of JSON objects, +one-per-line. + +When building fine-grained and yet functional root file systems, Chisel creates +this manifest as a way for ensuring file-level integrity and trace all the +installed package slices, in a format that 3rd party tools (such as SBOM +generators and vulnerability scanners) can work with. + + +(chisel_manifest_location)= +## Location of the manifest + +Chisel manifests may be generated anywhere in the newly created root file system. +To specify the location, Chisel must be instructed to install a slice where at least +one of its contents points to a path with the property +{ref}`generate: manifest`. + +When such a slice is installed, a `manifest.wall` file is generated at the specified +path. If there are multiple paths of this kind being installed, a manifest will be +created in each one of them. + +(chisel_manifest_pre-defined_location)= +### Pre-defined location + +There is a pre-defined slice named `base-files_chisel` that is available in all +supported Ubuntu releases in the {{chisel_releases_repo}}. + + +```yaml +package: base-files +slices: + chisel: + contents: + /var/lib/chisel/**: {generate: manifest} + ... +``` + +Installing the `base-files_chisel` slice produces a manifest at +`/var/lib/chisel/manifest.wall`. + + +(chisel_manifest_format)= +## Manifest format + +The uncompressed manifest is a "jsonwall" file. This is a custom database file +format where there is one JSON object per line. All JSON objects (except the +{ref}`header`) are sorted in a lexicographic order, +according to the "kind" of object, to optimize for searching and iterating over the manifest. + + +(chisel_manifest_jsonwall_header)= +### Header + +The "jsonwall" header is a single JSON object on the first line of the file. For example: + +```json +{"jsonwall":"1.0","schema":"1.0","count":84} +``` + +Where: + +| Field | Type | Required | Description | +| ---------- | ----- | -------- | ----------------------------------------------------------------- | +| `jsonwall` | `str` | required | version of jsonwall. | +| `schema` | `str` | required | schema version that Chisel manifest uses. | +| `count` | `int` | required | number of JSON entries in this file, including the header itself. | + + +(chisel_manifest_packages)= +### Packages + +For each package installed, a JSON object with `"kind":"package"` is present in +the manifest. For example: + +```json +{"kind":"package","name":"hello","version":"2.10-3build1","sha256":"e68cf4365b7aa9c4e2af4af6eee1710d6f967059b7b4af62786e8870d7366333","arch":"amd64"} +``` + +Where: + +| Field | Type | Required | Description | +| --------- | ----- | -------- | ------------------------------------------------------------- | +| `kind` | `str` | required | type of JSON object -- must always be `package` for packages. | +| `name` | `str` | required | name of the package. | +| `version` | `str` | required | version of the package. | +| `sha256` | `str` | required | digest of the package (in hex format). | +| `arch` | `str` | required | architecture of the package. | + + +(chisel_manifest_slices)= +### Slices + +For each slice installed in the file system, a JSON object with `"kind":"slice"` +is present in the manifest. For example: + +```json +{"kind":"slice","name":"hello_bins"} +``` + +Where: + +| Field | Type | Required | Description | +| ------ | ----- | -------- | --------------------------------------------------------- | +| `kind` | `str` | required | type of JSON object -- must always be `slice` for slices. | +| `name` | `str` | required | name of the slice, in the `pkg_slice` format. | + + +(chisel_manifest_paths)= +### Paths + +For each path (file, directory, symlink, etc.) that Chisel installs in the file +system, a JSON object with `"kind":"path"` is present in the manifest. For +example: + +```json +{"kind":"path","path":"/etc/ssl/certs/ca-certificates.crt","mode":"0644","slices":["ca-certificates_data"],"sha256":"8f2adf96b87e9da120f700d292f446ffe20062d9f57eaa2449ae67a09af970c3","final_sha256":"6d84ab71cb726c0641b0af84303c316e3fa50db941dc8507d09045eb2fa5d238","size":219342} +{"kind":"path","path":"/lib64","mode":"0777","slices":["base-files_lib"],"link":"usr/lib64"} +{"kind":"path","path":"/run/","mode":"0755","slices":["base-files_var"]} +{"kind":"path","path":"/usr/bin/hello","mode":"0755","slices":["hello_bins"],"sha256":"d288b98ce5f0a3981ea833f3b1d6484dfdde9ee36a00ee3b50bd3a9f7b01f75f","size":26856} +``` + +Where: + +| Field | Type | Required | Description | +| -------------- | ------ | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `kind` | `str` | required | type of JSON object -- must always be `path` for paths. | +| `path` | `str` | required | location of the path. | +| `mode` | `str` | required | permissions of the path, in an octal value format. | +| `slices` | `list` | required | the slices that have added or modified this path. | +| `sha256` | `str` | optional | original checksum of the file as in the Debian package (in hex format). This attribute is required for all regular files, except the `manifest.wall` file itself, which is an exception. | +| `final_sha256` | `str` | optional | checksum of the file after it has been modified during installation (in hex format). This attribute is required only for files that have been mutated. | +| `size` | `int` | optional | final size of the file, in bytes. This attribute is required for regular files, except the `manifest.wall` file itself, which is an exception. | +| `link` | `str` | optional | the target, if the file is a symbolic link. | + + +(chisel_manifest_list_of_paths_in_slice)= +### List of {ref}`Paths` under a Slice + +To state the paths that a slice has added/modified, JSON objects with +`"kind":"content"` are used. For example: + +```json +{"kind":"content","slice":"hello_bins","path":"/usr/bin/hello"} +``` + +Where: + +| Field | Type | Required | Description | +| ------- | ----- | -------- | ------------------------------------------------------------------- | +| `kind` | `str` | required | type of JSON object -- must always be `content` for slice contents. | +| `slice` | `str` | required | name of the slice. | +| `path` | `str` | required | location of the path. | diff --git a/docs/reuse/substitutions.yaml b/docs/reuse/substitutions.yaml new file mode 100644 index 00000000..705acc4b --- /dev/null +++ b/docs/reuse/substitutions.yaml @@ -0,0 +1,40 @@ +# To reuse sentences or paragraphs that have little markup and special +# formatting, put them here. Read more at [1]. +# +# This file is loaded into the myst_substitutions[2] variable in conf.py. +# +# [1] https://canonical-documentation-with-sphinx-and-readthedocscom.readthedocs-hosted.com/style-guide-myst/#reuse +# [2] https://myst-parser.readthedocs.io/en/v0.13.5/using/syntax-optional.html#substitutions-with-jinja2 + + +######### +# Links # +######### + + +AGPL3: "[AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.en.html)" +Canonical: "[Canonical Ltd](https://www.canonical.com)" +Docker: "[Docker](https://docs.docker.com/)" +Rocks: "[Rocks](https://documentation.ubuntu.com/rockcraft/)" +Levenshtein_distance: | + [Levenshtein Distance](https://en.wikipedia.org/wiki/Levenshtein_distance) +latest_release_page: | + [latest release page](https://github.com/canonical/chisel/releases/latest) +chisel_releases_repo: | + [chisel-releases repository](https://github.com/canonical/chisel-releases) +hello_pkg: "[hello](https://packages.ubuntu.com/noble/hello)" +Starlark: "[Starlark](https://github.com/google/starlark-go)" + +slices: "[slices](/explanation/slices)" +package_slices: "[package slices](/explanation/slices)" +cut_cmd: "{doc}`/reference/cmd/cut`" +find_cmd: "{doc}`/reference/cmd/find`" +info_cmd: "{doc}`/reference/cmd/info`" +mutation_scripts: | + {ref}`mutation scripts ` + + +############### +# Code blocks # +############### + diff --git a/docs/tutorial/getting-started.md b/docs/tutorial/getting-started.md new file mode 100644 index 00000000..1611961c --- /dev/null +++ b/docs/tutorial/getting-started.md @@ -0,0 +1,139 @@ +# Getting started with Chisel + +This tutorial will walk you through the creation of your first +chiseled Ubuntu root filesystem, from installation to the slicing of +Ubuntu packages. + +The goal is to give you a basic understanding of what Chisel is and +how to use it to install {{package_slices}}. By the end of it, +you will have a minimal slice of Ubuntu comprising only the necessary +files to run our application. + +## Prerequisites + +- A Linux machine. + +## Download and install Chisel + +Let's install the Chisel from its GitHub releases page. If you prefer a +different installation method, see [](/how-to/install-chisel). + +To install the latest Chisel binary: + +```{include} /how-to/install-chisel.md + :start-after: "" + :end-before: "" +``` + +### Verify the Chisel installation + +```{include} /how-to/install-chisel.md + :start-after: "" + :end-before: "" +``` + +## Slice the "{{hello_pkg}}" package + +The {{hello_pkg}} package has already been sliced and its slice definitions are +[available in the chisel-releases +repository](https://github.com/canonical/chisel-releases/blob/ubuntu-24.04/slices/hello.yaml). + +Before running Chisel, creating the empty directory where the root file system should be +located: +``` +mkdir rootfs +``` + +Finally, use the {{cut_cmd}} to install the `hello_bins` slice from the `ubuntu-24.04` +_chisel-release_: + +```{include} /reference/cmd/cut.md + :start-after: "" + :end-before: "" +``` + +### Inspect the chiseled root file system + +A quick look at the `rootfs/` directory will show that the {{hello_pkg}} binary +has been installed at `rootfs/usr/bin/hello`. + +```{terminal} +:input: find rootfs + +rootfs +rootfs/lib64 +rootfs/usr +rootfs/usr/lib64 +rootfs/usr/lib64/ld-linux-x86-64.so.2 +rootfs/usr/share +rootfs/usr/share/doc +rootfs/usr/share/doc/base-files +rootfs/usr/share/doc/base-files/copyright +rootfs/usr/share/doc/hello +rootfs/usr/share/doc/hello/copyright +rootfs/usr/share/doc/libc6 +rootfs/usr/share/doc/libc6/copyright +rootfs/usr/bin +rootfs/usr/bin/hello +rootfs/usr/lib +rootfs/usr/lib/x86_64-linux-gnu +rootfs/usr/lib/x86_64-linux-gnu/libpthread.so.0 +rootfs/usr/lib/x86_64-linux-gnu/libthread_db.so.1 +rootfs/usr/lib/x86_64-linux-gnu/libm.so.6 +rootfs/usr/lib/x86_64-linux-gnu/libc.so.6 +rootfs/usr/lib/x86_64-linux-gnu/libmemusage.so +rootfs/usr/lib/x86_64-linux-gnu/libdl.so.2 +rootfs/usr/lib/x86_64-linux-gnu/librt.so.1 +rootfs/usr/lib/x86_64-linux-gnu/libnss_files.so.2 +rootfs/usr/lib/x86_64-linux-gnu/libanl.so.1 +rootfs/usr/lib/x86_64-linux-gnu/libnss_hesiod.so.2 +rootfs/usr/lib/x86_64-linux-gnu/libc_malloc_debug.so.0 +rootfs/usr/lib/x86_64-linux-gnu/libmvec.so.1 +rootfs/usr/lib/x86_64-linux-gnu/libutil.so.1 +rootfs/usr/lib/x86_64-linux-gnu/libresolv.so.2 +rootfs/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 +rootfs/usr/lib/x86_64-linux-gnu/libBrokenLocale.so.1 +rootfs/usr/lib/x86_64-linux-gnu/libpcprofile.so +rootfs/usr/lib/x86_64-linux-gnu/libnss_dns.so.2 +rootfs/usr/lib/x86_64-linux-gnu/libnsl.so.1 +rootfs/usr/lib/x86_64-linux-gnu/libnss_compat.so.2 +rootfs/lib +``` + +````{note} +Notice, however, that there are a few other files besides the `hello` binary. +This is because the `hello_bins` slice depends on other slices, such as `libc6_libs`, +which provides necessary runtime libraries: + +```{terminal} +:input: chisel info --release ubuntu-24.04 hello_bins 2>/dev/null +package: hello +archive: ubuntu +slices: + bins: + essential: + - hello_copyright + - libc6_libs + contents: + /usr/bin/hello: {} +``` + +When installing a slice, Chisel installs its dependencies as well. +```` + +## Test the application + +To run `hello` from the chiseled root file system, do the following: + +```{terminal} +:input: sudo chroot rootfs/ hello + +Hello, world! +``` + +## Next steps + + +See what other [](/reference/cmd/index) are there, and have a look at +the {ref}`how_to_guides` for learning about other typical +Chisel operations.