- use tokio instead of os threads

- buffer writes - also write in parallel
2026-05-08 18:55:07 +02:00
parent 8883ae7e5e
commit 2fc0bf8c9f
9 changed files with 456 additions and 449 deletions
@@ -1,178 +0,0 @@
-name: Publish
-
-on: [push, pull_request, workflow_call]
-
-permissions:
-  contents: read
-
-jobs:
-  publish_pytest:
-    uses: ./.github/workflows/pytest.yml
-  linux:
-    needs: [ publish_pytest ]
-    runs-on: ${{ matrix.platform.runner }}
-    strategy:
-      matrix:
-        platform:
-          - runner: ubuntu-latest
-            target: x86_64
-          - runner: ubuntu-latest
-            target: x86
-          - runner: ubuntu-latest
-            target: aarch64
-          - runner: ubuntu-latest
-            target: armv7
-          - runner: ubuntu-latest
-            target: s390x
-          - runner: ubuntu-latest
-            target: ppc64le
-    steps:
-      - uses: actions/checkout@v6
-      - uses: actions/setup-python@v6
-        with:
-          python-version: '3.13'
-      - name: Build wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.platform.target }}
-          args: --release --out dist
-          sccache: 'true'
-          manylinux: auto
-      - name: Upload wheels
-        uses: actions/upload-artifact@v6
-        with:
-          name: wheels-linux-${{ matrix.platform.target }}
-          path: dist
-
-  windows:
-    needs: [ publish_pytest ]
-    runs-on: ${{ matrix.platform.runner }}
-    strategy:
-      matrix:
-        platform:
-          - runner: windows-latest
-            target: x64
-#          - runner: windows-11-arm
-#            target: aarch64
-    steps:
-      - uses: actions/checkout@v6
-      - uses: actions/setup-python@v6
-        with:
-          python-version: '3.13'
-          architecture: ${{ matrix.platform.target }}
-      - name: Build wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.platform.target }}
-          args: --release --out dist
-          sccache: 'true'
-      - name: Upload wheels
-        uses: actions/upload-artifact@v6
-        with:
-          name: wheels-windows-${{ matrix.platform.target }}
-          path: dist
-
-  macos:
-    needs: [ publish_pytest ]
-    runs-on: ${{ matrix.platform.runner }}
-    strategy:
-      matrix:
-        platform:
-          - runner: macos-latest
-            target: x86_64
-          - runner: macos-14
-            target: aarch64
-    steps:
-      - uses: actions/checkout@v6
-      - uses: actions/setup-python@v6
-        with:
-          python-version: '3.13'
-      - name: Build wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.platform.target }}
-          args: --release --out dist
-          sccache: 'true'
-      - name: Upload wheels
-        uses: actions/upload-artifact@v6
-        with:
-          name: wheels-macos-${{ matrix.platform.target }}
-          path: dist
-
-  sdist:
-    needs: [ publish_pytest ]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v6
-      - name: Build sdist
-        uses: PyO3/maturin-action@v1
-        with:
-          command: sdist
-          args: --out dist
-      - name: Upload sdist
-        uses: actions/upload-artifact@v6
-        with:
-          name: wheels-sdist
-          path: dist
-
-  release:
-    name: Release
-    runs-on: ubuntu-latest
-    needs: [linux, windows, macos, sdist]
-    steps:
-      - uses: actions/download-artifact@v7
-      - name: Publish to PyPI
-        uses: PyO3/maturin-action@v1
-        env:
-          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
-        with:
-          command: upload
-          args: --non-interactive --skip-existing wheels-*/*
-
-  crates_io_publish:
-    name: Publish (crates.io)
-    needs: [ publish_pytest ]
-    runs-on: ubuntu-latest
-    timeout-minutes: 25
-    steps:
-      - uses: actions/checkout@v6
-      - uses: dtolnay/rust-toolchain@stable
-
-      - name: cargo-release Cache
-        id: cargo_release_cache
-        uses: actions/cache@v5
-        with:
-          path: ~/.cargo/bin/cargo-release
-          key: ${{ runner.os }}-cargo-release
-
-      - run: cargo install cargo-release
-        if: steps.cargo_release_cache.outputs.cache-hit != 'true'
-
-      - name: cargo login
-        run: cargo login ${{ secrets.CRATES_IO_API_TOKEN }}
-
-      # allow-branch HEAD is because GitHub actions switches
-      # to the tag while building, which is a detached head
-
-      # Publishing is currently messy, because:
-      #
-      # * `peace_rt_model_core` exports `NativeError` or `WebError` depending on the target.
-      # * `peace_rt_model_web` fails to build when publishing the workspace for a native target.
-      # * `peace_rt_model_web` still needs its dependencies to be published before it can be
-      #    published.
-      # * `peace_rt_model_hack` needs `peace_rt_model_web` to be published before it can be
-      #    published.
-      #
-      # We *could* pass through `--no-verify` so `cargo` doesn't build the crate before publishing,
-      # which is reasonable, since this job only runs after the Linux, Windows, and WASM builds
-      # have passed.
-      - name: "cargo release publish"
-        run: |-
-          cargo release \
-            publish \
-            --workspace \
-            --all-features \
-            --allow-branch "main" \
-            --no-confirm \
-            --no-verify \
-            --execute
@@ -0,0 +1,69 @@
+name: Publish
+
+on: [workflow_dispatch]
+
+permissions:
+  contents: read
+
+jobs:
+  publish_pytest:
+    uses: ./.github/workflows/pytest.yml
+  crates_io_publish:
+    needs: [ publish_pytest ]
+    name: Publish (crates.io)
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@v6
+      - name: Restore cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.cargo
+          key: cache-ubuntu-cargo-publish
+
+      - name: Install Rust
+        run: |-
+          export PATH="$HOME/.cargo/bin:$PATH"
+          if ! command -v rustc >/dev/null 2>&1; then
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+          else
+            rustup update
+          fi
+          cargo install cargo-release
+        shell: bash
+
+      # allow-branch HEAD is because GitHub actions switches
+      # to the tag while building, which is a detached head
+
+      # Publishing is currently messy, because:
+      #
+      # * `peace_rt_model_core` exports `NativeError` or `WebError` depending on the target.
+      # * `peace_rt_model_web` fails to build when publishing the workspace for a native target.
+      # * `peace_rt_model_web` still needs its dependencies to be published before it can be
+      #    published.
+      # * `peace_rt_model_hack` needs `peace_rt_model_web` to be published before it can be
+      #    published.
+      #
+      # We *could* pass through `--no-verify` so `cargo` doesn't build the crate before publishing,
+      # which is reasonable, since this job only runs after the Linux, Windows, and WASM builds
+      # have passed.
+      - name: "cargo release publish"
+        run: |-
+          export PATH="$HOME/.osxcross/bin:$PATH"
+          cargo login ${{ secrets.CRATES_IO_API_TOKEN }}
+          cargo release \
+            publish \
+            --workspace \
+            --all-features \
+            --allow-branch "main" \
+            --no-confirm \
+            --no-verify \
+            --execute
+
+      - name: Store cache
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            ~/.cargo
+          key: cache-ubuntu-cargo-publish
@@ -0,0 +1,117 @@
+name: CI
+
+on: [workflow_dispatch]
+
+permissions:
+  contents: read
+
+jobs:
+  publish_pytest:
+    uses: ./.github/workflows/pytest.yml
+  pypi_publish:
+    needs: [ publish_pytest ]
+    name: Publish (pypi.org)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+
+      - name: Restore cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.cache/pip-wheel
+            ~/.cache/sccache
+            ~/.cache/cargo-xwin
+            ~/.cargo
+            ~/.osxcross
+          key: cache-ubuntu-maturin-cross-compile
+
+      - name: Install llvm
+        run: |
+          if ! command -v llvm-dlltool >/dev/null 2>&1; then
+            sudo apt update
+            sudo apt install -y llvm
+          fi
+        shell: bash
+
+      - name: Install Rust
+        run: |
+          export PATH="$HOME/.cargo/bin:$PATH"
+          if ! command -v rustc >/dev/null 2>&1; then
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+          else
+            rustup update
+          fi
+        shell: bash
+
+      - name: Install sccache and maturin
+        run: |
+          export PATH="$HOME/.cargo/bin:$PATH"
+          python -m pip install --upgrade pip
+          pip install maturin ziglang
+          if ! command -v sccache >/dev/null 2>&1; then
+            cargo install sccache || pip install sccache
+          fi
+        shell: bash
+
+      - name: Install xwin
+        run: |
+          export PATH="$HOME/.cargo/bin:$PATH"
+          if ! command -v cargo-xwin >/dev/null 2>&1; then
+            cargo install cargo-xwin || pip install cargo-xwin
+            cargo xwin cache xwin
+          fi
+        shell: bash
+
+      - name: Install osxcross
+        run: |
+          export PATH="$HOME/.osxcross/bin:$PATH"
+          if ! command -v osxcross >/dev/null 2>&1; then
+            wget ${{ secrets.OSXCROSS_LINK }} -O osxcross.tar.gz
+            tar -xzf osxcross.tar.gz -C ~/
+            mv ~/osxcross ~/.osxcross
+          fi
+
+      - name: Build wheels
+        run: |
+          export PATH="$HOME/.cargo/bin:$HOME/.osxcross/bin:$PATH"
+          maturin sdist --out dist
+          rustup default nightly
+          
+          rustup target add x86_64-unknown-linux-gnu --toolchain nightly
+          maturin build --release --out dist --target x86_64-unknown-linux-gnu
+          rustup target add aarch64-unknown-linux-gnu --toolchain nightly
+          maturin build --release --out dist --target aarch64-unknown-linux-gnu --zig
+          
+          rustup target add x86_64-pc-windows-msvc --toolchain nightly
+          maturin build --release --out dist --target x86_64-pc-windows-msvc
+          rustup target add aarch64-pc-windows-msvc --toolchain nightly
+          maturin build --release --out dist --target aarch64-pc-windows-msvc
+          
+          rustup target add x86_64-apple-darwin --toolchain nightly
+          maturin build --release --out dist --target x86_64-apple-darwin --zig
+          rustup target add aarch64-apple-darwin --toolchain nightly
+          maturin build --release --out dist --target aarch64-apple-darwin --zig
+
+      - name: Store cache
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.cache/pip-wheel
+            ~/.cache/sccache
+            ~/.cache/cargo-xwin
+            ~/.cargo
+            ~/.osxcross
+          key: cache-ubuntu-maturin-cross-compile
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+        env:
+          GITHUB_WORKFLOW_REF: 1.10.1
@@ -4,19 +4,63 @@ on: [push, pull_request, workflow_call]

 jobs:
  pytest:
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
    strategy:
      matrix:
-        python-version: ["3.10", "3.12", "3.13"]
-        os: [ubuntu-latest, windows-latest, macOS-latest]
+        python-version: ["3.10", "3.12", "3.14"]

    steps:
      - uses: actions/checkout@v6
      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
+
+      - name: Restore cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.cache/pip-wheel
+            ~/.cache/sccache
+            ~/.cache/cargo-xwin
+            ~/.cargo
+            ~/.osxcross
+          key: cache-ubuntu-maturin-cross-compile
+
+      - name: Install Rust
+        run: |
+          export PATH="$HOME/.cargo/bin:$PATH"
+          if ! command -v rustc >/dev/null 2>&1; then
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+          else
+            rustup update
+          fi
+        shell: bash
+
+      - name: Install sccache and maturin
+        run: |
+          export PATH="$HOME/.cargo/bin:$PATH"
+          python -m pip install --upgrade pip
+          pip install maturin ziglang
+          if ! command -v sccache >/dev/null 2>&1; then
+            cargo install sccache || pip install sccache
+          fi
+        shell: bash
+
      - name: Install
        run: pip install .[test]
      - name: Test with pytest
        run: pytest
+
+      - name: Store cache
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.cache/pip-wheel
+            ~/.cache/sccache
+            ~/.cache/cargo-xwin
+            ~/.cargo
+            ~/.osxcross
+          key: cache-ubuntu-maturin-cross-compile
@@ -1,13 +1,13 @@
 [package]
 name = "tiffwrite"
-version = "2026.1.1"
+version = "2026.5.0"
 edition = "2024"
-rust-version = "1.85.1"
+rust-version = "1.88.0"
 authors = ["Wim Pomp <w.pomp@nki.nl>"]
 license = "MIT OR Apache-2.0"
 description = "Write BioFormats/ImageJ compatible tiffs with zstd compression in parallel."
-homepage = "https://github.com/wimpomp/tiffwrite"
-repository = "https://github.com/wimpomp/tiffwrite"
+homepage = "https://git.wimpomp.nl/wim/tiffwrite"
+repository = "https://git.wimpomp.nl/wim/tiffwrite"
 documentation = "https://docs.rs/tiffwrite"
 readme = "README.md"
 keywords = ["bioformats", "tiff", "ndarray", "zstd", "fiji"]
@@ -24,17 +24,15 @@ color-eyre = { version = "0.6", optional = true }
 chrono = "0.4"
 css-color = "0.2"
 flate2 = "1"
+lazy_static = "1"
 ndarray = "0.17"
 num = "0.4"
-numpy = { version = "0.27", optional = true }
+numpy = { version = "0.28", optional = true }
+pyo3 = { version = "0.28", features = ["abi3-py310", "eyre", "generate-import-lib", "multiple-pymethods"], optional = true }
 rayon = "1"
 thiserror = "2"
+tokio = { version = "1", features = ["fs", "rt", "rt-multi-thread", "time"] }
 zstd = "0.13"

-[dependencies.pyo3]
-version = "0.27"
-features = ["extension-module", "abi3-py310", "eyre", "generate-import-lib", "multiple-pymethods"]
-optional = true
-
 [features]
 python = ["dep:pyo3", "dep:numpy", "dep:color-eyre"]
@@ -1,29 +1,27 @@
 [build-system]
-requires = ["maturin>=1.5,<2.0"]
+requires = ["maturin>=1.9.4,<2.0"]
 build-backend = "maturin"

 [project]
 name = "tiffwrite"
-dynamic = ["version"]
-authors = [
-    { name = "Wim Pomp", email = "w.pomp@nki.nl" }
+dynamic = [
+    "version",
+    "description",
+    "readme",
+    "license",
+    "license-files",
+    "authors",
+    "maintainers",
+    "keywords",
+    "urls",
 ]
-license = "MIT"
-readme = "README.md"
-keywords = ["bioformats", "tiff", "ndarray", "zstd", "fiji"]
-description = "Write BioFormats/ImageJ compatible tiffs with zstd compression in parallel."
 requires-python = ">=3.10"
 classifiers = [
+    "License :: OSI Approved :: Apache Software License",
    "License :: OSI Approved :: MIT License",
    "Programming Language :: Rust",
    "Programming Language :: Python",
    "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.12",
-    "Programming Language :: Python :: 3.13",
-    "Programming Language :: Python :: 3.14",
 ]

 dependencies = ["numpy", "tqdm"]
@@ -6,8 +6,14 @@ pub enum Error {
    IO(#[from] std::io::Error),
    #[error(transparent)]
    ColorCet(#[from] colorcet::ColorcetError),
+    #[error(transparent)]
+    Tokio(#[from] tokio::task::JoinError),
    #[error("could not parse color: {0}")]
    ColorParse(String),
    #[error("could not covert ColorMap into LinearGradient")]
    Conversion,
+    #[error("mutex was poisoned, this is a bug, please report it!")]
+    MutexPoisoned,
+    #[error("cannot express {0} as Rational32")]
+    Rational(f64),
 }
@@ -8,20 +8,19 @@ use colorcet::ColorMap;
 use colorgrad::{Gradient, LinearGradient};
 use css_color::Srgb;
 use flate2::write::ZlibEncoder;
+use lazy_static::lazy_static;
 use ndarray::{ArcArray2, AsArray, Ix2, s};
 use num::{Complex, FromPrimitive, Rational32, traits::ToBytes};
 use rayon::prelude::*;
 use std::collections::HashSet;
 use std::fs::{File, OpenOptions};
 use std::hash::{DefaultHasher, Hash, Hasher};
-use std::io::{Read, Seek, SeekFrom, Write};
+use std::io::{BufWriter, Read, Seek, SeekFrom, Write};
 use std::path::Path;
-use std::time::Duration;
+use std::sync::{Arc, Mutex};
+use std::thread::available_parallelism;
 use std::{cmp::Ordering, collections::HashMap};
-use std::{
-    thread,
-    thread::{JoinHandle, available_parallelism, sleep},
-};
+use tokio::{runtime::Runtime, task::JoinHandle};
 use zstd::zstd_safe::CompressionLevel;
 use zstd::{DEFAULT_COMPRESSION_LEVEL, stream::Encoder};

@@ -29,8 +28,12 @@ const TAG_SIZE: usize = 20;
 const OFFSET_SIZE: usize = 8;
 const OFFSET: u64 = 16;

+lazy_static! {
+    static ref RT: Runtime = Runtime::new().unwrap();
+}
+
 /// Compression: deflate or zstd
-#[derive(Clone, Debug)]
+#[derive(Copy, Clone, Debug)]
 pub enum Compression {
    Deflate,
    Zstd(CompressionLevel),
@@ -62,32 +65,29 @@ impl IFD {

    fn write(
        &mut self,
-        ijtifffile: &mut IJTiffFile,
+        hashes: &mut HashMap<u64, u64>,
+        file: &mut BufWriter<File>,
        where_to_write_offset: u64,
    ) -> Result<u64, Error> {
        let mut tags = self.tags.drain().collect::<Vec<_>>();
        tags.sort();
-        ijtifffile.file.seek(SeekFrom::End(0))?;
-        if ijtifffile.file.stream_position()? % 2 == 1 {
-            ijtifffile.file.write_all(&[0])?;
+        file.seek(SeekFrom::End(0))?;
+        if file.stream_position()? % 2 == 1 {
+            file.write_all(&[0])?;
        }
-        let offset = ijtifffile.file.stream_position()?;
-        ijtifffile
-            .file
-            .write_all(&(tags.len() as u64).to_le_bytes())?;
+        let offset = file.stream_position()?;
+        file.write_all(&(tags.len() as u64).to_le_bytes())?;

        for tag in tags.iter_mut() {
-            tag.write_tag(ijtifffile)?;
+            tag.write_tag(file)?;
        }
-        let where_to_write_next_ifd_offset = ijtifffile.file.stream_position()?;
-        ijtifffile.file.write_all(&[0; OFFSET_SIZE])?;
+        let where_to_write_next_ifd_offset = file.stream_position()?;
+        file.write_all(&[0; OFFSET_SIZE])?;
        for tag in tags.iter() {
-            tag.write_data(ijtifffile)?;
+            tag.write_data(hashes, file)?;
        }
-        ijtifffile
-            .file
-            .seek(SeekFrom::Start(where_to_write_offset))?;
-        ijtifffile.file.write_all(&offset.to_le_bytes())?;
+        file.seek(SeekFrom::Start(where_to_write_offset))?;
+        file.write_all(&offset.to_le_bytes())?;
        Ok(where_to_write_next_ifd_offset)
    }
 }
@@ -291,13 +291,16 @@ impl Tag {
    }

    pub fn short_long_or_long8(code: u16, value: &[u64]) -> Self {
-        let m = *value.iter().max().unwrap();
-        if m < 65536 {
-            Tag::short(code, &value.iter().map(|x| *x as u16).collect::<Vec<_>>())
-        } else if m < 4294967296 {
-            Tag::long(code, &value.iter().map(|x| *x as u32).collect::<Vec<_>>())
+        if let Some(&m) = value.iter().max() {
+            if m < 65536 {
+                Tag::short(code, &value.iter().map(|x| *x as u16).collect::<Vec<_>>())
+            } else if m < 4294967296 {
+                Tag::long(code, &value.iter().map(|x| *x as u32).collect::<Vec<_>>())
+            } else {
+                Tag::long8(code, value)
+            }
        } else {
-            Tag::long8(code, value)
+            Tag::short(code, &[])
        }
    }

@@ -327,32 +330,34 @@ impl Tag {
        c as u64
    }

-    fn write_tag(&mut self, ijtifffile: &mut IJTiffFile) -> Result<(), Error> {
-        self.offset = ijtifffile.file.stream_position()?;
-        ijtifffile.file.write_all(&self.code.to_le_bytes())?;
-        ijtifffile.file.write_all(&self.ttype.to_le_bytes())?;
-        ijtifffile.file.write_all(&self.count().to_le_bytes())?;
+    fn write_tag(&mut self, file: &mut BufWriter<File>) -> Result<(), Error> {
+        self.offset = file.stream_position()?;
+        file.write_all(&self.code.to_le_bytes())?;
+        file.write_all(&self.ttype.to_le_bytes())?;
+        file.write_all(&self.count().to_le_bytes())?;
        if self.bytes.len() <= OFFSET_SIZE {
-            ijtifffile.file.write_all(&self.bytes)?;
-            ijtifffile
-                .file
-                .write_all(&vec![0; OFFSET_SIZE - self.bytes.len()])?;
+            file.write_all(&self.bytes)?;
+            file.write_all(&vec![0; OFFSET_SIZE - self.bytes.len()])?;
        } else {
-            ijtifffile.file.write_all(&[0; OFFSET_SIZE])?;
+            file.write_all(&[0; OFFSET_SIZE])?;
        }
        Ok(())
    }

-    fn write_data(&self, ijtifffile: &mut IJTiffFile) -> Result<(), Error> {
+    fn write_data(
+        &self,
+        hashes: &mut HashMap<u64, u64>,
+        file: &mut BufWriter<File>,
+    ) -> Result<(), Error> {
        if self.bytes.len() > OFFSET_SIZE {
-            ijtifffile.file.seek(SeekFrom::End(0))?;
-            let offset = ijtifffile.write(&self.bytes)?;
-            ijtifffile.file.seek(SeekFrom::Start(
+            file.seek(SeekFrom::End(0))?;
+            let offset = IJTiffFile::write(hashes, file, &self.bytes)?;
+            file.seek(SeekFrom::Start(
                self.offset + (TAG_SIZE - OFFSET_SIZE) as u64,
            ))?;
-            ijtifffile.file.write_all(&offset.to_le_bytes())?;
-            if ijtifffile.file.stream_position()? % 2 == 1 {
-                ijtifffile.file.write_all(&[0])?;
+            file.write_all(&offset.to_le_bytes())?;
+            if file.stream_position()? % 2 == 1 {
+                file.write_all(&[0])?;
            }
        }
        Ok(())
@@ -360,18 +365,24 @@ impl Tag {
 }

 #[derive(Debug)]
-struct CompressedFrame {
-    bytes: Vec<Vec<u8>>,
+struct Frame {
+    offsets: Vec<u64>,
+    bytecounts: Vec<u64>,
    image_width: u32,
    image_length: u32,
-    tile_width: usize,
-    tile_length: usize,
+    tile_width: u16,
+    tile_length: u16,
    bits_per_sample: u16,
    sample_format: u16,
 }

-impl CompressedFrame {
-    fn new<T>(frame: ArcArray2<T>, compression: Compression) -> CompressedFrame
+impl Frame {
+    fn new<T>(
+        hashes: Arc<Mutex<HashMap<u64, u64>>>,
+        file: Arc<Mutex<BufWriter<File>>>,
+        frame: ArcArray2<T>,
+        compression: Compression,
+    ) -> Result<Frame, Error>
    where
        T: Bytes + Send + Sync,
    {
@@ -394,7 +405,7 @@ impl CompressedFrame {
                    (j + 1) * tile_length,
                ));
            }
-            if shape[1] % tile_length != 0 {
+            if !shape[1].is_multiple_of(tile_length) {
                slices.push((
                    i * tile_width,
                    (i + 1) * tile_width,
@@ -403,7 +414,7 @@ impl CompressedFrame {
                ));
            }
        }
-        if shape[0] % tile_width != 0 {
+        if !shape[0].is_multiple_of(tile_width) {
            for j in 0..m {
                slices.push((
                    n * tile_width,
@@ -412,7 +423,7 @@ impl CompressedFrame {
                    (j + 1) * tile_length,
                ));
            }
-            if shape[1] % tile_length != 0 {
+            if !shape[1].is_multiple_of(tile_length) {
                slices.push((n * tile_width, shape[0], m * tile_length, shape[1]));
            }
        }
@@ -423,28 +434,16 @@ impl CompressedFrame {
                    slices
                        .into_par_iter()
                        .map(|slice| {
-                            CompressedFrame::compress_tile_deflate(
-                                frame.clone(),
-                                slice,
-                                tile_size,
-                                tile_size,
-                            )
-                            .unwrap()
+                            Frame::compress_tile_deflate(frame.clone(), slice, tile_size, tile_size)
                        })
-                        .collect()
+                        .collect::<Result<Vec<_>, Error>>()?
                } else {
                    slices
                        .into_iter()
                        .map(|slice| {
-                            CompressedFrame::compress_tile_deflate(
-                                frame.clone(),
-                                slice,
-                                tile_size,
-                                tile_size,
-                            )
-                            .unwrap()
+                            Frame::compress_tile_deflate(frame.clone(), slice, tile_size, tile_size)
                        })
-                        .collect()
+                        .collect::<Result<Vec<_>, Error>>()?
                }
            }

@@ -453,43 +452,51 @@ impl CompressedFrame {
                    slices
                        .into_par_iter()
                        .map(|slice| {
-                            CompressedFrame::compress_tile_zstd(
+                            Frame::compress_tile_zstd(
                                frame.clone(),
                                slice,
                                tile_size,
                                tile_size,
                                level,
                            )
-                            .unwrap()
                        })
-                        .collect()
+                        .collect::<Result<Vec<_>, Error>>()?
                } else {
                    slices
                        .into_iter()
                        .map(|slice| {
-                            CompressedFrame::compress_tile_zstd(
+                            Frame::compress_tile_zstd(
                                frame.clone(),
                                slice,
                                tile_size,
                                tile_size,
                                level,
                            )
-                            .unwrap()
                        })
-                        .collect()
+                        .collect::<Result<Vec<_>, Error>>()?
                }
            }
        };

-        CompressedFrame {
-            bytes,
+        let mut offsets = Vec::new();
+        let mut bytecounts = Vec::new();
+        let mut file = file.lock().map_err(|_| Error::MutexPoisoned)?;
+        let mut hashes = hashes.lock().map_err(|_| Error::MutexPoisoned)?;
+        for tile in bytes {
+            bytecounts.push(tile.len() as u64);
+            offsets.push(IJTiffFile::write(&mut hashes, &mut file, &tile)?);
+        }
+
+        Ok(Frame {
+            offsets,
+            bytecounts,
            image_width: shape[1] as u32,
            image_length: shape[0] as u32,
-            tile_width,
-            tile_length,
+            tile_width: tile_width as u16,
+            tile_length: tile_length as u16,
            bits_per_sample: T::BITS_PER_SAMPLE,
            sample_format: T::SAMPLE_FORMAT,
-        }
+        })
    }

    fn encode<W, T>(
@@ -534,7 +541,7 @@ impl CompressedFrame {
        T: Bytes,
    {
        let mut encoder = ZlibEncoder::new(Vec::new(), flate2::Compression::default());
-        encoder = CompressedFrame::encode(encoder, frame, slice, tile_width, tile_length)?;
+        encoder = Frame::encode(encoder, frame, slice, tile_width, tile_length)?;
        Ok(encoder.finish()?)
    }

@@ -553,50 +560,13 @@ impl CompressedFrame {
        let bytes_per_sample = (T::BITS_PER_SAMPLE / 8) as usize;
        encoder.include_contentsize(true)?;
        encoder.set_pledged_src_size(Some((bytes_per_sample * tile_width * tile_length) as u64))?;
-        encoder.include_checksum(true)?;
-        encoder = CompressedFrame::encode(encoder, frame, slice, tile_width, tile_length)?;
+        encoder.include_checksum(false)?;
+        encoder = Frame::encode(encoder, frame, slice, tile_width, tile_length)?;
        encoder.finish()?;
        Ok(dest)
    }
 }

-#[derive(Clone, Debug)]
-struct Frame {
-    offsets: Vec<u64>,
-    bytecounts: Vec<u64>,
-    image_width: u32,
-    image_length: u32,
-    bits_per_sample: u16,
-    sample_format: u16,
-    tile_width: u16,
-    tile_length: u16,
-}
-
-impl Frame {
-    #[allow(clippy::too_many_arguments)]
-    fn new(
-        offsets: Vec<u64>,
-        bytecounts: Vec<u64>,
-        image_width: u32,
-        image_length: u32,
-        bits_per_sample: u16,
-        sample_format: u16,
-        tile_width: u16,
-        tile_length: u16,
-    ) -> Self {
-        Frame {
-            offsets,
-            bytecounts,
-            image_width,
-            image_length,
-            bits_per_sample,
-            sample_format,
-            tile_width,
-            tile_length,
-        }
-    }
-}
-
 /// trait to convert numbers to bytes
 pub trait Bytes {
    const BITS_PER_SAMPLE: u16;
@@ -654,10 +624,10 @@ pub enum Colors {
 /// save 2d arrays in a tif file compatible with Fiji/ImageJ
 #[derive(Debug)]
 pub struct IJTiffFile {
-    file: File,
+    file: Arc<Mutex<BufWriter<File>>>,
    frames: HashMap<(usize, usize, usize), Frame>,
-    hashes: HashMap<u64, u64>,
-    threads: HashMap<(usize, usize, usize), JoinHandle<CompressedFrame>>,
+    hashes: Arc<Mutex<HashMap<u64, u64>>>,
+    threads: HashMap<(usize, usize, usize), JoinHandle<Result<Frame, Error>>>,
    /// zstd: -7 ..= 22
    pub compression: Compression,
    pub colors: Colors,
@@ -681,7 +651,7 @@ impl Drop for IJTiffFile {
 }

 impl IJTiffFile {
-    /// create new tifffile from path, use it's save() method to save frames
+    /// create new tifffile from path, use its save() method to save frames
    /// the file is finalized when it goes out of scope
    pub fn new<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
        let mut file = OpenOptions::new()
@@ -696,9 +666,9 @@ impl IJTiffFile {
        file.write_all(&0u16.to_le_bytes())?;
        file.write_all(&OFFSET.to_le_bytes())?;
        Ok(IJTiffFile {
-            file,
+            file: Arc::new(Mutex::new(BufWriter::new(file))),
            frames: HashMap::new(),
-            hashes: HashMap::new(),
+            hashes: Arc::new(Mutex::new(HashMap::new())),
            threads: HashMap::new(),
            compression: Compression::Zstd(DEFAULT_COMPRESSION_LEVEL),
            colors: Colors::None,
@@ -842,29 +812,31 @@ impl IJTiffFile {
        hasher.finish()
    }

-    fn hash_check(&mut self, bytes: &Vec<u8>, offset: u64) -> Result<bool, Error> {
-        let current_offset = self.file.stream_position()?;
-        self.file.seek(SeekFrom::Start(offset))?;
+    fn hash_check(f: &mut BufWriter<File>, bytes: &Vec<u8>, offset: u64) -> Result<bool, Error> {
+        let current_offset = f.stream_position()?;
+        f.seek(SeekFrom::Start(offset))?;
        let mut buffer = vec![0; bytes.len()];
-        self.file.read_exact(&mut buffer)?;
+        f.get_ref().read_exact(&mut buffer)?;
        let same = bytes == &buffer;
-        self.file.seek(SeekFrom::Start(current_offset))?;
+        f.seek(SeekFrom::Start(current_offset))?;
        Ok(same)
    }

-    fn write(&mut self, bytes: &Vec<u8>) -> Result<u64, Error> {
+    fn write(
+        hashes: &mut HashMap<u64, u64>,
+        file: &mut BufWriter<File>,
+        bytes: &Vec<u8>,
+    ) -> Result<u64, Error> {
        let hash = IJTiffFile::hash(&bytes);
-        if self.hashes.contains_key(&hash)
-            && self.hash_check(bytes, *self.hashes.get(&hash).unwrap())?
-        {
-            Ok(*self.hashes.get(&hash).unwrap())
+        if hashes.contains_key(&hash) && Self::hash_check(file, bytes, hashes[&hash])? {
+            Ok(hashes[&hash])
        } else {
-            if self.file.stream_position()? % 2 == 1 {
-                self.file.write_all(&[0])?;
+            if file.stream_position()? % 2 == 1 {
+                file.write_all(&[0])?;
            }
-            let offset = self.file.stream_position()?;
-            self.hashes.insert(hash, offset);
-            self.file.write_all(bytes)?;
+            let offset = file.stream_position()?;
+            hashes.insert(hash, offset);
+            file.write_all(bytes)?;
            Ok(offset)
        }
    }
@@ -875,58 +847,31 @@ impl IJTiffFile {
        A: AsArray<'a, T, Ix2>,
        T: Bytes + Clone + Send + Sync + 'static,
    {
-        let n_threads = usize::from(available_parallelism()?);
-        loop {
-            self.collect_threads(false)?;
-            if self.threads.len() < n_threads {
-                break;
-            }
-            sleep(Duration::from_millis(100));
-        }
-        let compression = self.compression.clone();
+        self.collect_threads(false, usize::from(available_parallelism()?))?;
+        let hashes = self.hashes.clone();
+        let file = self.file.clone();
        let frame = frame.into().to_shared();
+        let compression = self.compression;
        self.threads.insert(
            (c, z, t),
-            thread::spawn(move || CompressedFrame::new(frame, compression)),
+            RT.spawn_blocking(move || Frame::new(hashes, file, frame, compression)),
        );
        Ok(())
    }

-    fn collect_threads(&mut self, block: bool) -> Result<(), Error> {
-        for (c, z, t) in self.threads.keys().cloned().collect::<Vec<_>>() {
-            if block || self.threads[&(c, z, t)].is_finished() {
-                if let Some(thread) = self.threads.remove(&(c, z, t)) {
-                    self.write_frame(thread.join().unwrap(), c, z, t)?;
+    fn collect_threads(&mut self, block: bool, max_threads: usize) -> Result<(), Error> {
+        RT.block_on(async {
+            while self.threads.len() > max_threads {
+                for (c, z, t) in self.threads.keys().cloned().collect::<Vec<_>>() {
+                    if (block || self.threads[&(c, z, t)].is_finished())
+                        && let Some(thread) = self.threads.remove(&(c, z, t))
+                    {
+                        self.frames.insert((c, z, t), thread.await??);
+                    }
                }
            }
-        }
-        Ok(())
-    }
-
-    fn write_frame(
-        &mut self,
-        frame: CompressedFrame,
-        c: usize,
-        z: usize,
-        t: usize,
-    ) -> Result<(), Error> {
-        let mut offsets = Vec::new();
-        let mut bytecounts = Vec::new();
-        for tile in frame.bytes {
-            bytecounts.push(tile.len() as u64);
-            offsets.push(self.write(&tile)?);
-        }
-        let frame = Frame::new(
-            offsets,
-            bytecounts,
-            frame.image_width,
-            frame.image_length,
-            frame.bits_per_sample,
-            frame.sample_format,
-            frame.tile_width as u16,
-            frame.tile_length as u16,
-        );
-        self.frames.insert((c, z, t), frame);
+            Ok::<(), Error>(())
+        })?;
        Ok(())
    }

@@ -957,7 +902,7 @@ impl IJTiffFile {
    }

    fn close(&mut self) -> Result<(), Error> {
-        self.collect_threads(true)?;
+        self.collect_threads(true, 0)?;
        let mut c_size = 1;
        let mut z_size = 1;
        let mut t_size = 1;
@@ -970,6 +915,8 @@ impl IJTiffFile {
        let mut where_to_write_next_ifd_offset = OFFSET - OFFSET_SIZE as u64;
        let mut warn = Vec::new();
        let (samples_per_pixel, n_frames) = self.spp_and_n_frames(c_size, t_size, z_size);
+        let mut file = self.file.lock().map_err(|_| Error::MutexPoisoned)?;
+        let mut hashes = self.hashes.lock().map_err(|_| Error::MutexPoisoned)?;
        for frame_number in 0..n_frames {
            if let Some(frame) = self
                .frames
@@ -1009,7 +956,7 @@ impl IJTiffFile {
                    ifd.tags.insert(Tag::short(339, &[frame.sample_format]));
                }
                if let Some(px_size) = self.px_size {
-                    let r = [Rational32::from_f64(px_size).unwrap()];
+                    let r = [Rational32::from_f64(px_size).ok_or_else(|| Error::Rational(px_size))?];
                    ifd.tags.insert(Tag::rational(282, &r));
                    ifd.tags.insert(Tag::rational(283, &r));
                    ifd.tags.insert(Tag::short(296, &[1]));
@@ -1019,27 +966,27 @@ impl IJTiffFile {
                } else if let Colors::None = self.colors {
                    ifd.tags.insert(Tag::short(262, &[1]));
                }
-                if frame_number == 0 {
-                    if let Colors::Colormap(colormap) = &self.colors {
-                        ifd.tags.insert(Tag::short(
-                            320,
-                            &self.get_colormap(colormap, frame.bits_per_sample),
-                        ));
-                    }
+                if frame_number == 0
+                    && let Colors::Colormap(colormap) = &self.colors
+                {
+                    ifd.tags.insert(Tag::short(
+                        320,
+                        &self.get_colormap(colormap, frame.bits_per_sample),
+                    ));
                }
-                if frame_number < c_size {
-                    if let Colors::Colors(colors) = &self.colors {
-                        ifd.tags.insert(Tag::short(
-                            320,
-                            &self.get_color(&colors[frame_number], frame.bits_per_sample),
-                        ));
-                        ifd.tags.insert(Tag::short(262, &[3]));
-                    }
+                if frame_number < c_size
+                    && let Colors::Colors(colors) = &self.colors
+                {
+                    ifd.tags.insert(Tag::short(
+                        320,
+                        &self.get_color(&colors[frame_number], frame.bits_per_sample),
+                    ));
+                    ifd.tags.insert(Tag::short(262, &[3]));
                }
-                if let Colors::None = &self.colors {
-                    if c_size > 1 {
-                        ifd.tags.insert(Tag::short(284, &[2]));
-                    }
+                if let Colors::None = &self.colors
+                    && c_size > 1
+                {
+                    ifd.tags.insert(Tag::short(284, &[2]));
                }
                for channel in 0..samples_per_pixel {
                    let czt = self.get_czt(frame_number, channel, c_size, z_size);
@@ -1060,7 +1007,8 @@ impl IJTiffFile {
                        &format!("{}", Utc::now().format("%Y:%m:%d %H:%M:%S")),
                    ));
                }
-                where_to_write_next_ifd_offset = ifd.write(self, where_to_write_next_ifd_offset)?;
+                where_to_write_next_ifd_offset =
+                    ifd.write(&mut hashes, &mut file, where_to_write_next_ifd_offset)?;
            } else {
                warn.push((frame_number, 0));
            }
@@ -1076,9 +1024,8 @@ impl IJTiffFile {
                )
            }
        }
-        self.file
-            .seek(SeekFrom::Start(where_to_write_next_ifd_offset))?;
-        self.file.write_all(&0u64.to_le_bytes())?;
+        file.seek(SeekFrom::Start(where_to_write_next_ifd_offset))?;
+        file.write_all(&0u64.to_le_bytes())?;
        Ok(())
    }
 }
@@ -10,8 +10,7 @@ impl From<crate::error::Error> for PyErr {
    }
 }

-#[pyclass(subclass)]
-#[pyo3(name = "Tag")]
+#[pyclass(name = "Tag", module = "tiffwrite_rs", subclass, from_py_object)]
 #[derive(Clone, Debug)]
 struct PyTag {
    tag: Tag,
@@ -163,8 +162,7 @@ impl PyTag {
    }
 }

-#[pyclass(subclass)]
-#[pyo3(name = "IJTiffFile")]
+#[pyclass(name = "IJTiffFile", module = "tiffwrite_rs", subclass)]
 #[derive(Debug)]
 struct PyIJTiffFile {
    ijtifffile: Option<IJTiffFile>,
@@ -199,10 +197,10 @@ impl PyIJTiffFile {

    #[getter]
    fn get_colors(&self) -> PyResult<Option<Vec<Vec<u8>>>> {
-        if let Some(ijtifffile) = &self.ijtifffile {
-            if let Colors::Colors(colors) = &ijtifffile.colors {
-                return Ok(Some(colors.to_owned()));
-            }
+        if let Some(ijtifffile) = &self.ijtifffile
+            && let Colors::Colors(colors) = &ijtifffile.colors
+        {
+            return Ok(Some(colors.to_owned()));
        }
        Ok(None)
    }
@@ -217,10 +215,10 @@ impl PyIJTiffFile {

    #[getter]
    fn get_colormap(&mut self) -> PyResult<Option<Vec<Vec<u8>>>> {
-        if let Some(ijtifffile) = &self.ijtifffile {
-            if let Colors::Colormap(colormap) = &ijtifffile.colors {
-                return Ok(Some(colormap.to_owned()));
-            }
+        if let Some(ijtifffile) = &self.ijtifffile
+            && let Colors::Colormap(colormap) = &ijtifffile.colors
+        {
+            return Ok(Some(colormap.to_owned()));
        }
        Ok(None)
    }
@@ -303,25 +301,25 @@ impl PyIJTiffFile {

    #[pyo3(signature = (tag, czt=None))]
    fn append_extra_tag(&mut self, tag: PyTag, czt: Option<(usize, usize, usize)>) {
-        if let Some(ijtifffile) = self.ijtifffile.as_mut() {
-            if let Some(extra_tags) = ijtifffile.extra_tags.get_mut(&czt) {
-                extra_tags.push(tag.tag)
-            }
+        if let Some(ijtifffile) = self.ijtifffile.as_mut()
+            && let Some(extra_tags) = ijtifffile.extra_tags.get_mut(&czt)
+        {
+            extra_tags.push(tag.tag)
        }
    }

    #[pyo3(signature = (czt=None))]
    fn get_tags(&self, czt: Option<(usize, usize, usize)>) -> PyResult<Vec<PyTag>> {
-        if let Some(ijtifffile) = &self.ijtifffile {
-            if let Some(extra_tags) = ijtifffile.extra_tags.get(&czt) {
-                let v = extra_tags
-                    .iter()
-                    .map(|tag| PyTag {
-                        tag: tag.to_owned(),
-                    })
-                    .collect();
-                return Ok(v);
-            }
+        if let Some(ijtifffile) = &self.ijtifffile
+            && let Some(extra_tags) = ijtifffile.extra_tags.get(&czt)
+        {
+            let v = extra_tags
+                .iter()
+                .map(|tag| PyTag {
+                    tag: tag.to_owned(),
+                })
+                .collect();
+            return Ok(v);
        }
        Ok(Vec::new())
    }
@@ -369,9 +367,17 @@ impl_save! {

 #[pymodule]
 #[pyo3(name = "tiffwrite_rs")]
-fn tiffwrite_rs(m: &Bound<'_, PyModule>) -> PyResult<()> {
-    color_eyre::install()?;
-    m.add_class::<PyTag>()?;
-    m.add_class::<PyIJTiffFile>()?;
-    Ok(())
+mod tiffwrite_rs {
+    use pyo3::prelude::*;
+
+    #[pymodule_export]
+    use super::PyTag;
+
+    #[pymodule_export]
+    use super::PyIJTiffFile;
+
+    #[pymodule_init]
+    fn init(_: &Bound<'_, PyModule>) -> PyResult<()> {
+        Ok(color_eyre::install()?)
+    }
 }