Merge branch 'tskit-dev:main' into main

Author: Unic-X

.github/workflows/python.yml

@@ -38,10 +38,14 @@ jobs:
       - name: run JSON metadata example
         run: |
           cargo run --example json_metadata --features derive
+      - name: run bincode metadata example
+        run: |
+          cargo run --example bincode_metadata --features derive
       - name: setup Python and run tests
         run: |
           uv venv -p ${{ matrix.python }}
           source .venv/bin/activate
           uv pip install -r python/requirements_locked_3_13.txt
+          uv pip install python/tskit_glue
           python -m pytest python
 

.github/workflows/test32bit.yml

@@ -38,6 +38,9 @@ jobs:
       - name: Install rust tooling for 32 bit builds
         run: |
           rustup target install i686-unknown-linux-gnu
+      - name: Install rust toolchain
+        run: |
+          rustup toolchain install
       - name: cargo check
         run: |
             cargo hack check --all-targets --target=i686-unknown-linux-gnu --feature-powerset

.github/workflows/tests.yml

@@ -33,6 +33,8 @@ jobs:
       - uses: taiki-e/install-action@cargo-hack
       - run: sudo apt-get update -y
         if: matrix.os == 'ubuntu-24.04'
+      - name: update toolchain
+        run: rustup toolchain install
       - name: cargo check (powerset)
         run: cargo hack check --feature-powerset --no-dev-deps
       - name: cargo check examples (powerset)
@@ -76,6 +78,8 @@ jobs:
           components: clippy
       - uses: Swatinem/[email protected]
       - uses: taiki-e/install-action@cargo-hack
+      - name: update toolchain
+        run: rustup toolchain install
       - name: clippy (all targets, feature powerset)
         run: cargo hack clippy --all-targets --feature-powerset -- -D warnings
 
@@ -85,7 +89,7 @@ jobs:
     strategy:
       matrix:
         rust:
-          - 1.71.0
+          - 1.75.0
     steps:
       - uses: actions/[email protected]
         with: 

Cargo.toml

@@ -8,7 +8,7 @@ description = "rust interface to tskit"
 license = "MIT"
 homepage = "https://github.com/tskit-dev/tskit-rust"
 repository = "https://github.com/tskit-dev/tskit-rust"
-rust-version = "1.71.0"
+rust-version = "1.75.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [lints.rust]
@@ -60,3 +60,10 @@ name = "tree_traversals"
 [[example]]
 name = "json_metadata"
 required-features = ["derive"]
+
+[[example]]
+name = "bincode_metadata"
+required-features = ["derive"]
+
+[[example]]
+name = "manual_metadata_encoding"

book/src/SUMMARY.md

@@ -19,7 +19,7 @@
 * [Metadata](./metadata.md)
   - [Defining metadata types in rust](./metadata_derive.md)
   - [Metadata and tables](./metadata_tables.md)
-  - [Metadata schema](./metadata_schema.md)
+  - [Metadata processing with Python](./metadata_python.md)
   - [Advanced topics](./metadata_advanced.md)
 
 * [Error handling](./error_handling.md)

book/src/metadata_python.md

@@ -0,0 +1,31 @@
+# Metadata processing with Python
+
+## `JSON` metadata
+
+If your metadata are generated in `JSON` format via `serde` (see [here](metadata_derive.md)), then the metadata are simple to access from Python.
+The code repository for `tskit-rust` contains examples in the `python/` subdirectory.
+
+You may work with `JSON` metadata with or without a metadata schema (see [here](https://tskit.dev/tskit/docs/stable/metadata.html)).
+A schema is useful for data validation but there is an unfortunate inefficiency if your input to Python is a tree sequence rather than a table collection.
+You will have to copy the tables, add the metadata schema, and regenerate a tree sequence.
+See the examples mentioned above.
+
+## Other formats
+
+The `tskit-python` API only supports `JSON` and Python's `struct` data formats.
+It is useful to use a format other than `JSON` in order to minimize storage requirements.
+However, doing so will require that you provide a method to covert the data into a valid Python object.
+
+An easy way to provide conversion methods is to use [pyo3](https://pyo3.rs) to create a small Python module to deserialize your metadata into Python objects.
+The `tskit-rust` code repository contains an example of this in the `python/` subdirectory.
+The module is shown in its entirety below:
+
+```rust, noplaygound, ignore
+{{#include ../../python/tskit_glue/src/lib.rs}}
+```
+
+Using it in Python is just a matter of importing the module:
+
+```python
+{{#include ../../python/test_bincode_metadata.py}}
+```

book/src/metadata_schema.md

@@ -0,0 +1,51 @@
+#[derive(serde::Serialize, serde::Deserialize, tskit::metadata::MutationMetadata)]
+#[serializer("bincode")]
+struct MutationMetadata {
+    effect_size: f64,
+    dominance: f64,
+}
+
+#[derive(serde::Serialize, serde::Deserialize, tskit::metadata::IndividualMetadata)]
+#[serializer("bincode")]
+struct IndividualMetadata {
+    name: String,
+    phenotypes: Vec<i32>,
+}
+
+fn main() {
+    let ts = make_treeseq().unwrap();
+    ts.dump("with_bincode_metadata.trees", 0).unwrap();
+}
+
+fn make_tables() -> anyhow::Result<tskit::TableCollection> {
+    let mut tables = tskit::TableCollection::new(100.0)?;
+    let pop0 = tables.add_population()?;
+    let ind0 = tables.add_individual_with_metadata(
+        0,
+        None,
+        None,
+        &IndividualMetadata {
+            name: "Jerome".to_string(),
+            phenotypes: vec![0, 1, 2, 0],
+        },
+    )?;
+    let node0 = tables.add_node(tskit::NodeFlags::new_sample(), 0.0, pop0, ind0)?;
+    let site0 = tables.add_site(50.0, Some("A".as_bytes()))?;
+    let _ = tables.add_mutation_with_metadata(
+        site0,
+        node0,
+        tskit::MutationId::NULL,
+        1.0,
+        Some("G".as_bytes()),
+        &MutationMetadata {
+            effect_size: -1e-3,
+            dominance: 0.1,
+        },
+    )?;
+    tables.build_index()?;
+    Ok(tables)
+}
+
+fn make_treeseq() -> anyhow::Result<tskit::TreeSequence> {
+    Ok(make_tables()?.tree_sequence(0)?)
+}

examples/bincode_metadata.rs

@@ -156,7 +156,7 @@ fn stress_test_total_branch_length() {
         // stress test the branch length fn b/c it is not a trivial
         // wrapper around the C API.
         {
-            use streaming_iterator::StreamingIterator;
+            use tskit::StreamingIterator;
             let mut x = f64::NAN;
             if let Ok(mut tree_iter) = ts.tree_iterator(0) {
                 // We will only do the first tree to save time.

examples/haploid_wright_fisher.rs

@@ -0,0 +1,137 @@
+use core::str;
+
+use tskit::metadata::MetadataRoundtrip;
+
+struct MutationMetadata {
+    effect_size: f64,
+    dominance: f64,
+}
+
+impl MetadataRoundtrip for MutationMetadata {
+    fn encode(&self) -> Result<Vec<u8>, tskit::metadata::MetadataError> {
+        let mut rv = vec![];
+        rv.extend_from_slice(&self.effect_size.to_le_bytes());
+        rv.extend_from_slice(&self.dominance.to_le_bytes());
+        Ok(rv)
+    }
+
+    fn decode(md: &[u8]) -> Result<Self, tskit::metadata::MetadataError>
+    where
+        Self: Sized,
+    {
+        let slice: [u8; 8] = md[0..8].try_into().unwrap();
+        let effect_size = f64::from_le_bytes(slice);
+        let slice: [u8; 8] = md[8..].try_into().unwrap();
+        let dominance = f64::from_le_bytes(slice);
+        Ok(Self {
+            effect_size,
+            dominance,
+        })
+    }
+}
+
+impl tskit::metadata::MutationMetadata for MutationMetadata {}
+
+struct IndividualMetadata {
+    name: String,
+    phenotypes: Vec<i32>,
+}
+
+impl MetadataRoundtrip for IndividualMetadata {
+    fn encode(&self) -> Result<Vec<u8>, tskit::metadata::MetadataError> {
+        let mut rv = vec![];
+        rv.extend_from_slice(self.name.len().to_le_bytes().as_slice());
+        rv.extend_from_slice(self.name.as_bytes());
+        rv.extend_from_slice(self.phenotypes.len().to_le_bytes().as_slice());
+        for i in self.phenotypes.iter() {
+            rv.extend_from_slice(i.to_le_bytes().as_slice());
+        }
+        Ok(rv)
+    }
+    fn decode(md: &[u8]) -> Result<Self, tskit::metadata::MetadataError>
+    where
+        Self: Sized,
+    {
+        let size: [u8; std::mem::size_of::<usize>()] =
+            md[0..std::mem::size_of::<usize>()].try_into().unwrap();
+        let size = usize::from_le_bytes(size);
+        let md = &md[std::mem::size_of::<usize>()..];
+        let name = str::from_utf8(&md[0..size]).unwrap().to_string();
+        let md = &md[size..];
+        let md = &md[std::mem::size_of::<usize>()..];
+        let mut phenotypes = vec![];
+        // NOTE: production code would want to validate that
+        // the remaining number of bytes are correct
+        let chunks = md.chunks_exact(std::mem::size_of::<i32>());
+        for c in chunks {
+            // Unwrap b/c the conversion cannot fail b/c the chunk size is correct!
+            let a: [u8; std::mem::size_of::<i32>()] = c.try_into().unwrap();
+            phenotypes.push(i32::from_le_bytes(a));
+        }
+        Ok(Self { name, phenotypes })
+    }
+}
+
+impl tskit::metadata::IndividualMetadata for IndividualMetadata {}
+
+fn main() {
+    let ts = make_treeseq().unwrap();
+    ts.dump("with_manual_metadata.trees", 0).unwrap();
+}
+
+fn make_tables() -> anyhow::Result<tskit::TableCollection> {
+    let mut tables = tskit::TableCollection::new(100.0)?;
+    let pop0 = tables.add_population()?;
+    let ind0 = tables.add_individual_with_metadata(
+        0,
+        None,
+        None,
+        &IndividualMetadata {
+            name: "Jerome".to_string(),
+            phenotypes: vec![0, 1, 2, 0],
+        },
+    )?;
+    let node0 = tables.add_node(tskit::NodeFlags::new_sample(), 0.0, pop0, ind0)?;
+    let site0 = tables.add_site(50.0, Some("A".as_bytes()))?;
+    let _ = tables.add_mutation_with_metadata(
+        site0,
+        node0,
+        tskit::MutationId::NULL,
+        1.0,
+        Some("G".as_bytes()),
+        &MutationMetadata {
+            effect_size: -1e-3,
+            dominance: 0.1,
+        },
+    )?;
+    tables.build_index()?;
+    Ok(tables)
+}
+
+fn make_treeseq() -> anyhow::Result<tskit::TreeSequence> {
+    Ok(make_tables()?.tree_sequence(0)?)
+}
+
+#[test]
+fn test_mutation_metadata_roundtrip() {
+    let md = MutationMetadata {
+        effect_size: 0.1,
+        dominance: 0.25,
+    };
+    let encoded = md.encode().unwrap();
+    let decoded = MutationMetadata::decode(&encoded).unwrap();
+    assert_eq!(md.effect_size, decoded.effect_size);
+    assert_eq!(md.dominance, decoded.dominance);
+}
+
+#[test]
+fn test_individual_metadata_roundtrip() {
+    let md = IndividualMetadata {
+        name: "Jerome".to_string(),
+        phenotypes: vec![10, 9],
+    };
+    let encoded = md.encode().unwrap();
+    let decoded = IndividualMetadata::decode(&encoded).unwrap();
+    assert_eq!(md.name, decoded.name);
+    assert_eq!(md.phenotypes, decoded.phenotypes);
+}