Merge branch 'main' into dependabot/github_actions/actions-rs/toolchain-1.0.7

Author: molpopgen

.github/dependabot.yml

@@ -1,15 +1,16 @@
 version: 2
 updates:
-- package-ecosystem: cargo
-  directory: "/"
-  schedule:
-    interval: monthly
-    time: "13:00"
-  open-pull-requests-limit: 10
-  commit-message:
-    prefix: "chore: "
-  allow:
-   - dependency-type: "direct"
+# Generates far too much PR noise
+# - package-ecosystem: cargo
+#   directory: "/"
+#   schedule:
+#     interval: monthly
+#     time: "13:00"
+#   open-pull-requests-limit: 10
+#   commit-message:
+#     prefix: "chore: "
+#   allow:
+#    - dependency-type: "direct"
 - package-ecosystem: github-actions
   directory: "/"
   schedule:

.github/workflows/python.yml

@@ -0,0 +1,51 @@
+on:
+  push:
+    branches: [main, dev]
+  pull_request:
+
+name: Python data round trips
+
+jobs:
+  test-metadata:
+    name: Test Python metadata round trips
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-24.04]
+        rust:
+          - stable
+        python: [ "3.13" ]
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          submodules: recursive
+      - uses: dtolnay/rust-toolchain@v1
+        with:
+          toolchain: ${{ matrix.rust }}
+      - uses: Swatinem/[email protected]
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          activate-environment: true
+          version: "latest"
+          python-version: ${{ matrix.python }}
+      - name: run JSON metadata example
+        run: |
+          cargo run --example json_metadata --features derive
+      - name: run bincode metadata example
+        run: |
+          cargo run --example bincode_metadata --features derive
+      - name: setup Python and run tests
+        run: |
+          uv venv -p ${{ matrix.python }}
+          source .venv/bin/activate
+          uv pip install -r python/requirements_locked_3_13.txt
+          uv pip install python/tskit_glue
+          python -m pytest python
+

.github/workflows/semver.yml

@@ -20,8 +20,12 @@ jobs:
           toolchain: ${{ matrix.rust }}
           components: clippy
       - name: Install cargo-semver-checks
-        uses: baptiste0928/cargo-install@v3.1.1
+        uses: baptiste0928/cargo-install@v3.3.2
         with:
           crate: cargo-semver-checks
+          version: '0.43.0'
       - name: run check-release
-        run: cargo semver-checks check-release
+      # We set the cargo version here to dodge ecosystem
+      # issues that can arise b/c semver-checks doesnt'
+      # respect our Cargo.lock
+        run: cargo +1.90.0 semver-checks check-release

.github/workflows/test32bit.yml

@@ -38,6 +38,9 @@ jobs:
       - name: Install rust tooling for 32 bit builds
         run: |
           rustup target install i686-unknown-linux-gnu
+      - name: Install rust toolchain
+        run: |
+          rustup toolchain install
       - name: cargo check
         run: |
             cargo hack check --all-targets --target=i686-unknown-linux-gnu --feature-powerset

.github/workflows/tests.yml

@@ -33,6 +33,8 @@ jobs:
       - uses: taiki-e/install-action@cargo-hack
       - run: sudo apt-get update -y
         if: matrix.os == 'ubuntu-24.04'
+      - name: update toolchain
+        run: rustup toolchain install
       - name: cargo check (powerset)
         run: cargo hack check --feature-powerset --no-dev-deps
       - name: cargo check examples (powerset)
@@ -74,6 +76,8 @@ jobs:
           components: clippy
       - uses: Swatinem/[email protected]
       - uses: taiki-e/install-action@cargo-hack
+      - name: update toolchain
+        run: rustup toolchain install
       - name: clippy (all targets, feature powerset)
         run: cargo hack clippy --all-targets --feature-powerset -- -D warnings
 
@@ -83,7 +87,7 @@ jobs:
     strategy:
       matrix:
         rust:
-          - 1.71.0
+          - 1.75.0
     steps:
       - uses: actions/[email protected]
       - uses: dtolnay/rust-toolchain@v1

Cargo.lock

@@ -8,7 +8,7 @@ description = "rust interface to tskit"
 license = "MIT"
 homepage = "https://github.com/tskit-dev/tskit-rust"
 repository = "https://github.com/tskit-dev/tskit-rust"
-rust-version = "1.71.0"
+rust-version = "1.75.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [lints.rust]
@@ -20,7 +20,6 @@ lto = "fat"
 codegen-units=1
 
 [dependencies]
-thiserror = "1.0"
 libc = "0.2.155"
 streaming-iterator = "0.1.5"
 serde = {version = "1.0.203", features = ["derive"], optional = true}
@@ -56,3 +55,14 @@ rustdoc-args = ["--cfg", "doc_cfg"]
 # Not run during tests
 [[example]]
 name = "tree_traversals"
+
+[[example]]
+name = "json_metadata"
+required-features = ["derive"]
+
+[[example]]
+name = "bincode_metadata"
+required-features = ["derive"]
+
+[[example]]
+name = "manual_metadata_encoding"

Cargo.toml

@@ -19,7 +19,7 @@
 * [Metadata](./metadata.md)
   - [Defining metadata types in rust](./metadata_derive.md)
   - [Metadata and tables](./metadata_tables.md)
-  - [Metadata schema](./metadata_schema.md)
+  - [Metadata processing with Python](./metadata_python.md)
   - [Advanced topics](./metadata_advanced.md)
 
 * [Error handling](./error_handling.md)

book/src/SUMMARY.md

@@ -0,0 +1,31 @@
+# Metadata processing with Python
+
+## `JSON` metadata
+
+If your metadata are generated in `JSON` format via `serde` (see [here](metadata_derive.md)), then the metadata are simple to access from Python.
+The code repository for `tskit-rust` contains examples in the `python/` subdirectory.
+
+You may work with `JSON` metadata with or without a metadata schema (see [here](https://tskit.dev/tskit/docs/stable/metadata.html)).
+A schema is useful for data validation but there is an unfortunate inefficiency if your input to Python is a tree sequence rather than a table collection.
+You will have to copy the tables, add the metadata schema, and regenerate a tree sequence.
+See the examples mentioned above.
+
+## Other formats
+
+The `tskit-python` API only supports `JSON` and Python's `struct` data formats.
+It is useful to use a format other than `JSON` in order to minimize storage requirements.
+However, doing so will require that you provide a method to covert the data into a valid Python object.
+
+An easy way to provide conversion methods is to use [pyo3](https://pyo3.rs) to create a small Python module to deserialize your metadata into Python objects.
+The `tskit-rust` code repository contains an example of this in the `python/` subdirectory.
+The module is shown in its entirety below:
+
+```rust, noplaygound, ignore
+{{#include ../../python/tskit_glue/src/lib.rs}}
+```
+
+Using it in Python is just a matter of importing the module:
+
+```python
+{{#include ../../python/test_bincode_metadata.py}}
+```