From 82931f7715529ddc0b18c0d569f48ec20afa6565 Mon Sep 17 00:00:00 2001 From: Wim Pomp Date: Sun, 6 Oct 2024 20:30:57 +0200 Subject: [PATCH 01/13] rewrite in rust --- .gitignore | 1 + Cargo.lock | 596 +++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 21 ++ pyproject.toml | 52 ++-- src/lib.rs | 515 +++++++++++++++++++++++++++++++++++ src/main.rs | 28 ++ src/py.rs | 158 +++++++++++ tiffwrite/__init__.py | 604 +----------------------------------------- 8 files changed, 1339 insertions(+), 636 deletions(-) create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 src/lib.rs create mode 100644 src/main.rs create mode 100644 src/py.rs mode change 100755 => 100644 tiffwrite/__init__.py diff --git a/.gitignore b/.gitignore index 5a5a156..f5af85d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ /tiffwrite.egg-info/ /.pytest_cache/ /venv/ +/target/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..efdd80a --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,596 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "cc" +version = "1.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "fraction" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7" +dependencies = [ + "lazy_static", + "num", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + +[[package]] +name = "js-sys" +version = "0.3.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.159" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "matrixmultiply" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" + +[[package]] +name = "portable-atomic-util" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcdd8420072e66d54a407b3316991fe946ce3ab1083a7f575b2463866624704d" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15ee168e30649f7f234c3d49ef5a7a6cbf5134289bc46c29ff3155fa3221c225" +dependencies = [ + "anyhow", + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e61cef80755fe9e46bb8a0b8f20752ca7676dcc07a5277d8b7768c6172e529b3" +dependencies = [ + "once_cell", + "python3-dll-a", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ce096073ec5405f5ee2b8b31f03a68e02aa10d5d4f565eca04acc41931fa1c" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2440c6d12bc8f3ae39f1e775266fa5122fd0c8891ce7520fa6048e683ad3de28" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1be962f0e06da8f8465729ea2cb71a416d2257dff56cbe40a70d3e62a93ae5d1" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "python3-dll-a" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0b78171a90d808b319acfad166c4790d9e9759bbc14ac8273fe133673dd41b" +dependencies = [ + "cc", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + +[[package]] +name = "tiffwrite" +version = "2024.10.0" +dependencies = [ + "anyhow", + "chrono", + "fraction", + "ndarray", + "num", + "pyo3", + "rayon", +] + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" + +[[package]] +name = "unindent" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + +[[package]] +name = "wasm-bindgen" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..8680bd6 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "tiffwrite" +version = "2024.10.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "tiffwrite" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = { version = "0.22.3", features = ["extension-module", "abi3-py310", "generate-import-lib", "anyhow"] } +anyhow = "1.0.89" +rayon = "1.10.0" +fraction = "0.15.3" +num = "0.4.3" +ndarray = "0.16.1" +chrono = "0.4.38" + +[features] +nopython = [] diff --git a/pyproject.toml b/pyproject.toml index 2db2f95..484a62e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,39 +1,21 @@ -[tool.poetry] +[build-system] +requires = ["maturin>=1.5,<2.0"] +build-backend = "maturin" + +[project] name = "tiffwrite" -version = "2024.10.1" -description = "Parallel tiff writer compatible with ImageJ." -authors = ["Wim Pomp, Lenstra lab NKI "] -license = "GPL-3.0-or-later" -readme = "README.md" -packages = [{include = "tiffwrite"}] -repository = "https://github.com/wimpomp/tiffwrite" +dynamic = ["version"] +authors = [{ name = "Wim Pomp", email = "w.pomp@nki.nl" }] +requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] -[tool.poetry.dependencies] -python = "^3.10" -tifffile = "*" -imagecodecs = "*" -numpy = "*" -tqdm = "*" -colorcet = "*" -matplotlib = "*" -parfor = ">=2024.9.2" -pytest = { version = "*", optional = true } -mypy = { version = "*", optional = true } - -[tool.poetry.extras] -test = ["pytest", "mypy"] - -[tool.pytest.ini_options] -filterwarnings = ["ignore:::(?!tiffwrite)"] +[tool.maturin] +features = ["pyo3/extension-module"] +module-name = "tiffwrite" [tool.isort] -line_length = 119 - -[tool.mypy] -disable_error_code = ["import-untyped", "return"] -implicit_optional = true -exclude = ["build"] - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" +line_length = 119 \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..13f946f --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,515 @@ +use std::cmp::Ordering; +use std::collections::HashMap; +use std::fs::{File, OpenOptions}; +use std::io::{Read, Seek, SeekFrom, Write}; +use anyhow::Result; +use fraction::Fraction; +use num::{Complex, Zero}; +use num::complex::ComplexFloat; +use ndarray::{s, Array2}; +use num::traits::ToBytes; +use std::hash::{DefaultHasher, Hash, Hasher}; +use chrono::Utc; + + +const TAG_SIZE: usize = 20; +const OFFSET_SIZE: usize = 8; +const OFFSET: u64 = 16; + + +#[derive(Clone, Debug)] +struct IFD { + tags: Vec +} + +impl IFD { + pub fn new() -> Self { + IFD { tags: Vec::new() } + } + + fn push_tag(&mut self, tag: Tag) { + if !self.tags.contains(&tag) { + self.tags.push(tag); + } + } + + fn extend_tags(&mut self, tags: Vec) { + for tag in tags { + self.push_tag(tag); + } + } + + fn write(&mut self, ijtifffile: &mut IJTiffFile, where_to_write_offset: u64) -> Result { + self.tags.sort(); + ijtifffile.file.seek(SeekFrom::End(0))?; + if ijtifffile.file.stream_position()? % 2 == 1 { + ijtifffile.file.write(&[0])?; + } + let offset = ijtifffile.file.stream_position()?; + ijtifffile.file.write(&(self.tags.len() as u64).to_le_bytes())?; + + for tag in self.tags.iter_mut() { + tag.write_tag(ijtifffile)?; + } + let where_to_write_next_ifd_offset = ijtifffile.file.stream_position()?; + ijtifffile.file.write(&vec![0u8; OFFSET_SIZE])?; + for tag in self.tags.iter() { + tag.write_data(ijtifffile)?; + } + ijtifffile.file.seek(SeekFrom::Start(where_to_write_offset))?; + ijtifffile.file.write(&offset.to_le_bytes())?; + Ok(where_to_write_next_ifd_offset) + } +} + + +#[derive(Clone, Debug, Eq)] +pub struct Tag { + code: u16, + bytes: Vec, + ttype: u16, + offset: u64 +} + +impl PartialOrd for Tag { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Tag { + fn cmp(&self, other: &Self) -> Ordering { + self.code.cmp(&other.code) + } +} + +impl PartialEq for Tag { + fn eq(&self, other: &Self) -> bool { + self.code == other.code + } +} + +impl Tag { + pub fn new(code: u16, bytes: Vec, ttype: u16) -> Self { + Tag { code, bytes, ttype, offset: 0 } + } + + pub fn byte(code: u16, byte: Vec) -> Self { + Tag::new(code, byte, 1) + } + + pub fn ascii(code: u16, ascii: &str) -> Self { + let mut bytes = ascii.as_bytes().to_vec(); + bytes.push(0); + Tag::new(code, bytes, 2) + } + + pub fn short(code: u16, short: Vec) -> Self { + Tag::new(code, short.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 3) + } + + pub fn long(code: u16, long: Vec) -> Self { + Tag::new(code, long.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 4) + } + + pub fn rational(code: u16, rational: Vec) -> Self { + Tag::new(code, rational.into_iter().map(|x| + u32::try_from(*x.denom().unwrap()).unwrap().to_le_bytes().into_iter().chain( + u32::try_from(*x.numer().unwrap()).unwrap().to_le_bytes()).collect::>() + ).flatten().collect(), 5) + } + + pub fn sbyte(code: u16, sbyte: Vec) -> Self { + Tag::new(code, sbyte.iter().map(|x| x.to_le_bytes()).flatten().collect(), 6) + } + + pub fn sshort(code: u16, sshort: Vec) -> Self { + Tag::new(code, sshort.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 8) + } + + pub fn slong(code: u16, slong: Vec) -> Self { + Tag::new(code, slong.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 9) + } + + pub fn srational(code: u16, srational: Vec) -> Self { + Tag::new(code, srational.into_iter().map(|x| + i32::try_from(*x.denom().unwrap()).unwrap().to_le_bytes().into_iter().chain( + i32::try_from(*x.numer().unwrap()).unwrap().to_le_bytes()).collect::>() + ).flatten().collect(), 10) + } + + pub fn float(code: u16, float: Vec) -> Self { + Tag::new(code, float.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 11) + } + + pub fn double(code: u16, double: Vec) -> Self { + Tag::new(code, double.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 12) + } + + pub fn ifd(code: u16, ifd: Vec) -> Self { + Tag::new(code, ifd.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 13) + } + + pub fn unicode(code: u16, unicode: &str) -> Self { + let mut bytes: Vec = unicode.encode_utf16().map(|x| x.to_le_bytes()).flatten().collect(); + bytes.push(0); + Tag::new(code, bytes, 14) + } + + pub fn complex(code: u16, complex: Vec>) -> Self { + Tag::new(code, complex.into_iter().map(|x| + x.re().to_le_bytes().into_iter().chain(x.im().to_le_bytes()).collect::>() + ).flatten().collect(), 15) + } + + pub fn long8(code: u16, long8: Vec) -> Self { + Tag::new(code, long8.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 16) + } + + pub fn slong8(code: u16, slong8: Vec) -> Self { + Tag::new(code, slong8.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 17) + } + + pub fn ifd8(code: u16, ifd8: Vec) -> Self { + Tag::new(code, ifd8.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 18) + } + + pub fn count(&self) -> u64 { + let c = match self.ttype { + 1 => self.bytes.len(), // BYTE + 2 => self.bytes.len(), // ASCII + 3 => self.bytes.len() / 2, // SHORT + 4 => self.bytes.len() / 4, // LONG + 5 => self.bytes.len() / 8, // RATIONAL + 6 => self.bytes.len(), // SBYTE + 7 => self.bytes.len(), // UNDEFINED + 8 => self.bytes.len() / 2, // SSHORT + 9 => self.bytes.len() / 4, // SLONG + 10 => self.bytes.len() / 8, // SRATIONAL + 11 => self.bytes.len() / 4, // FLOAT + 12 => self.bytes.len() / 8, // DOUBLE + 13 => self.bytes.len() / 4, // IFD + 14 => self.bytes.len() / 2, // UNICODE + 15 => self.bytes.len() / 8, // COMPLEX + 16 => self.bytes.len() / 8, // LONG8 + 17 => self.bytes.len() / 8, // SLONG8 + 18 => self.bytes.len() / 8, // IFD8 + _ => self.bytes.len(), + }; + c as u64 + } + + fn write_tag(&mut self, ijtifffile: &mut IJTiffFile) -> Result<()> { + self.offset = ijtifffile.file.stream_position()?; + ijtifffile.file.write(&self.code.to_le_bytes())?; + ijtifffile.file.write(&self.ttype.to_le_bytes())?; + ijtifffile.file.write(&self.count().to_le_bytes())?; + if self.bytes.len() <= OFFSET_SIZE { + ijtifffile.file.write(&self.bytes)?; + for _ in self.bytes.len()..OFFSET_SIZE { + ijtifffile.file.write(&[0])?; + } + } else { + ijtifffile.file.write(&vec![0u8; OFFSET_SIZE])?; + } + Ok(()) + } + + fn write_data(&self, ijtifffile: &mut IJTiffFile) -> Result<()> { + if self.bytes.len() > OFFSET_SIZE { + ijtifffile.file.seek(SeekFrom::End(0))?; + let offset = ijtifffile.write(&self.bytes)?; + ijtifffile.file.seek(SeekFrom::Start( + self.offset + (TAG_SIZE - OFFSET_SIZE) as u64))?; + ijtifffile.file.write(&offset.to_le_bytes())?; + if ijtifffile.file.stream_position()? % 2 == 1 { + ijtifffile.file.write(&[0u8])?; + } + } + Ok(()) + } +} + + +#[derive(Clone, Debug)] +struct Frame { + tilebyteoffsets: Vec, + tilebytecounts: Vec, + image_width: u32, + image_length: u32, + bits_per_sample: u16, + compression: u16, + tile_width: u16, + tile_length: u16, + extra_tags: Vec +} + +impl Frame { + fn new( + tilebyteoffsets: Vec, tilebytecounts: Vec, image_width: u32, + image_length: u32, bits_per_sample: u16, compression: u16, tile_width: u16, tile_length: u16 + ) -> Self { + Frame { + tilebyteoffsets, tilebytecounts, image_width, image_length, bits_per_sample, + compression, tile_width, tile_length, extra_tags: Vec::new() + } + } +} + + +#[derive(Debug)] +pub struct IJTiffFile { + file: File, + frames: HashMap<(usize, u8), Frame>, + hashes: HashMap, + pub shape: (usize, usize, usize), + pub n_frames: usize, + pub samples_per_pixel: u8, + pub colormap: Option>, + pub colors: Option>, + pub comment: Option, + pub delta_z: Option, + pub timeinterval: Option, + pub extra_tags: Option> +} + +impl Drop for IJTiffFile { + fn drop(&mut self) { + if let Err(e) = self.close() { + println!("Error closing IJTiffFile: {:?}", e); + } + } +} + +impl IJTiffFile { + pub fn new(path: &str, shape: (usize, usize, usize)) -> Result { + let mut file = OpenOptions::new().create(true).truncate(true) + .write(true).read(true).open(path)?; + file.write(b"II")?; + file.write(&43u16.to_le_bytes())?; + file.write(&8u16.to_le_bytes())?; + file.write(&0u16.to_le_bytes())?; + file.write(&OFFSET.to_le_bytes())?; + let colormap: Option> = None; + let (spp, n_frames) = if let None = &colormap { + (shape.0 as u8, shape.1 * shape.2) + } else { + (1, shape.0 * shape.1 * shape.2) + }; + Ok(IJTiffFile { file, frames: HashMap::new(), hashes: HashMap::new(), shape, n_frames, + samples_per_pixel: spp, colormap: None, colors: None, comment: None, delta_z: None, + timeinterval: None, extra_tags: None } ) + } + + pub fn description(&self) -> String { + let mut desc: String = String::from("ImageJ=1.11a"); + if let (None, None) = (self.colormap.as_ref(), self.colors.as_ref()) { + desc += &format!("\nimages={}", self.shape.0); + desc += &format!("\nslices={}", self.shape.1); + desc += &format!("\nframes={}", self.shape.2); + } else { + desc += &format!("\nimages={}", self.shape.0 * self.shape.1 * self.shape.2); + desc += &format!("\nchannels={}", self.shape.0); + desc += &format!("\nslices={}", self.shape.1); + desc += &format!("\nframes={}", self.shape.2); + }; + if self.shape.0 == 1 { + desc += "\nmode=grayscale"; + } else { + desc += "\nmode=composite"; + } + desc += "\nhyperstack=true\nloop=false\nunit=micron"; + if let Some(delta_z) = self.delta_z { + desc += &format!("\nspacing={}", delta_z); + } + if let Some(timeinterval) = self.timeinterval { + desc += &format!("\ninterval={}", timeinterval); + } + if let Some(comment) = &self.comment { + desc += &format!("\ncomment={}", comment); + } + desc + } + + pub fn save(&mut self, frame: Array2, c: usize, z: usize, t: usize, + extra_tags: Option>) -> Result<()> { + let mut compressed_frame = self.compress_frame(frame)?; + if let Some(tags) = extra_tags { + for tag in tags { + compressed_frame.extra_tags.push(tag); + } + } + + self.frames.insert(self.get_frame_number(c, z, t), compressed_frame); + Ok(()) + } + + fn get_frame_number(&self, c: usize, z: usize, t: usize) -> (usize, u8) { + if let (None, None) = (self.colormap.as_ref(), self.colors.as_ref()) { + (z + t * self.shape.1, c as u8) + } else { + (c + z * self.shape.0 + t * self.shape.0 * self.shape.1, 0) + } + } + + pub fn hash(value: &T) -> u64 { + let mut hasher = DefaultHasher::new(); + value.hash(&mut hasher); + hasher.finish() + } + + fn hash_check(&mut self, bytes: &Vec, offset: u64) -> Result { + let current_offset = self.file.stream_position()?; + self.file.seek(SeekFrom::Start(offset))?; + let mut buffer = vec![0u8; bytes.len()]; + self.file.read_exact(&mut buffer)?; + let same = bytes == &buffer; + self.file.seek(SeekFrom::Start(current_offset))?; + Ok(same) + } + + fn write(&mut self, bytes: &Vec) -> Result { + let hash = IJTiffFile::hash(&bytes); + if self.hashes.contains_key(&hash) && self.hash_check(&bytes, *self.hashes.get(&hash).unwrap())? { + Ok(*self.hashes.get(&hash).unwrap()) + } else { + if self.file.stream_position()? % 2 == 1 { + self.file.write(&[0])?; + } + let offset = self.file.stream_position()?; + self.hashes.insert(hash, offset); + self.file.write(&bytes)?; + Ok(offset) + } + } + + fn compress_frame(&mut self, frame: Array2) -> Result { + let image_width = frame.shape()[0] as u32; + let image_length = frame.shape()[1] as u32; + let mut tilebyteoffsets = Vec::new(); + let mut tilebytecounts = Vec::new(); + let tiles = IJTiffFile::tile(frame.reversed_axes(), 64); + for tile in tiles { + let bytes: Vec = tile.into_flat().into_iter().map( + |x| x.to_le_bytes()).into_iter().flatten().collect(); + tilebytecounts.push(bytes.len() as u64); + tilebyteoffsets.push(self.write(&bytes)?); + } + + Ok(Frame::new(tilebyteoffsets, tilebytecounts, image_width, image_length, + 16, 1, 64, 64)) + } + + fn tile(frame: Array2, size: usize) -> Vec> { + let shape = frame.shape(); + let mut tiles = Vec::new(); + let (n, m) = (shape[0] / size, shape[1] / size); + for i in 0..n { + for j in 0..m { + tiles.push(frame.slice( + s![i * size..(i + 1) * size, j * size..(j + 1) * size]).to_owned()); + } + if shape[1] % size != 0 { + let mut tile = Array2::::zeros((size, size)); + tile.slice_mut( + s![.., ..shape[1] - m * size] + ).assign(&frame.slice(s![i * size..(i + 1) * size, m * size..])); + tiles.push(tile); + } + } + if shape[0] % size != 0 { + for j in 0..m { + let mut tile = Array2::::zeros((size, size)); + tile.slice_mut( + s![..shape[0] - n * size, ..] + ).assign(&frame.slice(s![n * size.., j * size..(j + 1) * size])); + tiles.push(tile); + } + if shape[1] % size != 0 { + let mut tile = Array2::::zeros((size, size)); + tile.slice_mut( + s![..shape[0] - n * size, ..shape[1] - m * size] + ).assign(&frame.slice(s![n * size.., m * size..])); + tiles.push(tile); + } + } + tiles + } + + fn get_colormap(&self, colormap: &Vec) -> Result> { + todo!(); + Ok(Vec::new()) + } + + fn get_color(&self, colors: (u8, u8, u8)) -> Result> { + todo!(); + Ok(Vec::new()) + } + + pub fn close(&mut self) -> Result<()> { + let mut where_to_write_next_ifd_offset = OFFSET - OFFSET_SIZE as u64; + let mut warn = false; + for frame_number in 0..self.n_frames { + if let Some(frame) = self.frames.get(&(frame_number, 0)) { + let mut tilebyteoffsets = Vec::new(); + let mut tilebytecounts = Vec::new(); + let mut frame_count = 0; + for channel in 0..self.samples_per_pixel { + if let Some(frame_n) = self.frames.get(&(frame_number, channel)) { + tilebyteoffsets.extend(frame_n.tilebyteoffsets.iter()); + tilebytecounts.extend(frame_n.tilebytecounts.iter()); + frame_count += 1; + } else { + warn = true; + } + } + let mut ifd = IFD::new(); + ifd.push_tag(Tag::long(256, vec![frame.image_width])); + ifd.push_tag(Tag::long(257, vec![frame.image_length])); + ifd.push_tag(Tag::short(258, vec![frame.bits_per_sample; frame_count])); + ifd.push_tag(Tag::short(259, vec![1])); + ifd.push_tag(Tag::ascii(270, &self.description())); + ifd.push_tag(Tag::short(277, vec![frame_count as u16])); + ifd.push_tag(Tag::ascii(305, "tiffwrite_rs")); + ifd.push_tag(Tag::short(322, vec![frame.tile_width])); + ifd.push_tag(Tag::short(323, vec![frame.tile_length])); + ifd.push_tag(Tag::long8(324, tilebyteoffsets)); + ifd.push_tag(Tag::long8(325, tilebytecounts)); + if frame_number == 0 { + if let Some(colormap) = &self.colormap { + ifd.push_tag(Tag::short(320, self.get_colormap(colormap)?)); + ifd.push_tag(Tag::short(262, vec![3])); // PhotometricInterpretation PHOTOMETRIC_PALETTE + } else if let None = self.colors { + ifd.push_tag(Tag::short(262, vec![1])); // PhotometricInterpretation PHOTOMETRIC_PALETTE + } + } + if frame_number < self.samples_per_pixel as usize { + if let Some(color) = &self.colors { + ifd.push_tag(Tag::short(320, self.get_color(color[frame_number])?)); + ifd.push_tag(Tag::short(262, vec![3])); // PhotometricInterpretation PHOTOMETRIC_PALETTE + } + } + if let (None, None) = (&self.colormap, &self.colors) { + if self.shape.0 > 1 { + ifd.push_tag(Tag::short(284, vec![2])) + } + } + ifd.extend_tags(frame.extra_tags.to_owned()); + if let Some(extra_tags) = &self.extra_tags { + ifd.extend_tags(extra_tags.to_owned()); + } + ifd.push_tag(Tag::ascii(306, &format!("{}", Utc::now().format("%Y:%m:%d %H:%M:%S")))); + where_to_write_next_ifd_offset = ifd.write(self, where_to_write_next_ifd_offset)?; + } else { + warn = true; + } + if warn { + println!("Some frames were not added to the tif file, either you forgot them, \ + or an error occurred and the tif file was closed prematurely.") + } + } + self.file.seek(SeekFrom::Start(where_to_write_next_ifd_offset))?; + self.file.write(&0u64.to_le_bytes())?; + Ok(()) + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..f3fd0a5 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,28 @@ +#[cfg(not(feature = "nopython"))] +mod py; +mod lib; + +use anyhow::Result; +use ndarray::{s, Array2}; +use rayon::prelude::*; +use crate::lib::IJTiffFile; + + +fn main() -> Result<()> { + println!("Hello World!"); + let mut f = IJTiffFile::new("foo.tif", (1, 2, 1))?; + let mut arr = Array2::::zeros((100, 100)); + // for i in 0..arr.shape()[0] { + // for j in 0..arr.shape()[1] { + // arr[[i, j]] = i as u16; + // } + // } + f.save(arr.to_owned(), 0, 0, 0, None)?; + + // let mut arr = Array2::::zeros((100, 100)); + // arr.slice_mut(s![64.., ..64]).fill(1); + // arr.slice_mut(s![..64, 64..]).fill(2); + // arr.slice_mut(s![64.., 64..]).fill(3); + f.save(arr.to_owned(), 0, 1,0, None)?; + Ok(()) +} \ No newline at end of file diff --git a/src/py.rs b/src/py.rs new file mode 100644 index 0000000..4ba2d78 --- /dev/null +++ b/src/py.rs @@ -0,0 +1,158 @@ +use pyo3::prelude::*; +use crate::lib::{IJTiffFile, Tag}; +use std::time::Duration; +use pyo3::types::{PyInt, PyString}; +use fraction::Fraction; +use num::Complex; + + +#[pyclass(subclass)] +#[pyo3(name = "Tag")] +#[derive(Clone, Debug)] +struct PyTag { + tag: Tag +} + +#[pymethods] +impl PyTag { + #[staticmethod] + fn byte(code: u16, byte: Vec) -> Self { + PyTag { tag: Tag::byte(code, byte) } + } + + #[staticmethod] + fn ascii(code: u16, ascii: &str) -> Self { + PyTag { tag: Tag::ascii(code, ascii) } + } + + #[staticmethod] + fn short(code: u16, short: Vec) -> Self { + PyTag { tag: Tag::short(code, short) } + } + + #[staticmethod] + fn long(code: u16, long: Vec) -> Self { + PyTag { tag: Tag::long(code, long) } + } + + #[staticmethod] + fn rational(code: u16, rational: Vec) -> Self { + PyTag { tag: Tag::rational(code, rational.into_iter().map(|x| Fraction::from(x)).collect()) } + } + + #[staticmethod] + fn sbyte(code: u16, sbyte: Vec) -> Self { + PyTag { tag: Tag::sbyte(code, sbyte) } + } + + #[staticmethod] + fn sshort(code: u16, sshort: Vec) -> Self { + PyTag { tag: Tag::sshort(code, sshort) } + } + + #[staticmethod] + fn slong(code: u16, slong: Vec) -> Self { + PyTag { tag: Tag::slong(code, slong) } + } + + #[staticmethod] + fn srational(code: u16, srational: Vec) -> Self { + PyTag { tag: Tag::srational(code, srational.into_iter().map(|x| Fraction::from(x)).collect()) } + } + + #[staticmethod] + fn float(code: u16, float: Vec) -> Self { + PyTag { tag: Tag::float(code, float) } + } + + #[staticmethod] + fn double(code: u16, double: Vec) -> Self { + PyTag { tag: Tag::double(code, double) } + } + + #[staticmethod] + fn ifd(code: u16, ifd: Vec) -> Self { + PyTag { tag: Tag::ifd(code, ifd) } + } + + #[staticmethod] + fn unicode(code: u16, unicode: &str) -> Self { + PyTag { tag: Tag::unicode(code, unicode) } + } + + #[staticmethod] + fn complex(code: u16, complex: Vec<(f32, f32)>) -> Self { + PyTag { tag: Tag::complex(code, complex.into_iter().map(|(x, y)| Complex { re: x, im: y }).collect()) } + } + + #[staticmethod] + fn long8(code: u16, long8: Vec) -> Self { + PyTag { tag: Tag::long8(code, long8) } + } + + #[staticmethod] + fn slong8(code: u16, slong8: Vec) -> Self { + PyTag { tag: Tag::slong8(code, slong8) } + } + + #[staticmethod] + fn ifd8(code: u16, ifd8: Vec) -> Self { + PyTag { tag: Tag::ifd8(code, ifd8) } + } +} + + +#[pyclass(subclass)] +#[pyo3(name = "IJTiffFile")] +#[derive(Debug)] +struct PyIJTiffFile { + ijtifffile: IJTiffFile +} + +#[pymethods] +impl PyIJTiffFile { + #[new] + fn new(path: &str, shape: (usize, usize, usize), dtype: &str) -> PyResult { + Ok(PyIJTiffFile { ijtifffile: IJTiffFile::new(path, shape)? } ) + } + + fn with_colors(&mut self, colors: (u8, u8, u8)) -> Self { + todo!() + } + + fn with_colormap(&mut self, colormap: Vec<(u8, u8, u8)>) -> Self { + todo!() + } + + fn with_px_size(&mut self, pxsize: f64) -> Self { + todo!() + } + + fn with_delta_z(&mut self, delta_z: f64) -> Self { + todo!() + } + + fn with_time_interval(&mut self, time_interval: f64) -> Self { + todo!() + } + + fn with_comments(&mut self, comments: String) -> Self { + todo!() + } + + fn append_extra_tag(&mut self, extra_tag: PyTag) { + if let Some(extra_tags) = self.ijtifffile.extra_tags.as_mut() { + extra_tags.push(extra_tag.tag); + } else { + self.ijtifffile.extra_tags = Some(vec![extra_tag.tag]); + } + } + + +} + +#[pymodule] +fn tiffwrite(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + Ok(()) +} \ No newline at end of file diff --git a/tiffwrite/__init__.py b/tiffwrite/__init__.py old mode 100755 new mode 100644 index e4c0d1e..7f1ac1f --- a/tiffwrite/__init__.py +++ b/tiffwrite/__init__.py @@ -1,603 +1,5 @@ -from __future__ import annotations +from . import tiffwrite as rs -import struct -import warnings -from collections.abc import Iterable -from contextlib import contextmanager -from datetime import datetime -from fractions import Fraction -from functools import cached_property -from hashlib import sha1 -from importlib.metadata import version -from io import BytesIO -from itertools import product -from pathlib import Path -from typing import Any, BinaryIO, Callable, Generator, Literal, Optional, Sequence -import colorcet -import numpy as np -import tifffile -from matplotlib import colors as mpl_colors -from numpy.typing import DTypeLike -from parfor import ParPool, PoolSingleton -from tqdm.auto import tqdm - -__all__ = ["IJTiffFile", "Tag", "tiffwrite"] - - -try: - __version__ = version("tiffwrite") -except Exception: # noqa - __version__ = "unknown" - - -Strip = tuple[list[int], list[int]] -CZT = tuple[int, int, int] - - -def tiffwrite(file: str | Path, data: np.ndarray, axes: str = 'TZCXY', dtype: DTypeLike = None, bar: bool = False, - *args: Any, **kwargs: Any) -> None: - """ file: string; filename of the new tiff file - data: 2 to 5D numpy array - axes: string; order of dimensions in data, default: TZCXY for 5D, ZCXY for 4D, CXY for 3D, XY for 2D data - dtype: string; datatype to use when saving to tiff - bar: bool; whether to show a progress bar - other args: see IJTiffFile - """ - - axes = axes[-np.ndim(data):].upper() - if not axes == 'CZTXY': - axes_shuffle = [axes.find(i) for i in 'CZTXY'] - axes_add = [i for i, j in enumerate(axes_shuffle) if j < 0] - axes_shuffle = [i for i in axes_shuffle if i >= 0] - data = np.transpose(data, axes_shuffle) - for axis in axes_add: - data = np.expand_dims(data, axis) - - shape = data.shape[:3] - with IJTiffFile(file, shape, data.dtype if dtype is None else dtype, *args, **kwargs) as f: # type: ignore - at_least_one = False - for n in tqdm(product(*[range(i) for i in shape]), total=np.prod(shape), desc='Saving tiff', disable=not bar): - if np.any(data[n]) or not at_least_one: - f.save(data[n], *n) - at_least_one = True - - -class Header: - def __init__(self, filehandle_or_byteorder: BinaryIO | Literal['>', '<'] | None = None, - bigtiff: bool = True) -> None: - if filehandle_or_byteorder is None or isinstance(filehandle_or_byteorder, str): - self.byteorder = filehandle_or_byteorder or '<' - self.bigtiff = bigtiff - if self.bigtiff: - self.tagsize = 20 - self.tagnoformat = 'Q' - self.offsetsize = 8 - self.offsetformat = 'Q' - self.offset = 16 - else: - self.tagsize = 12 - self.tagnoformat = 'H' - self.offsetsize = 4 - self.offsetformat = 'I' - self.offset = 8 - else: - fh = filehandle_or_byteorder - fh.seek(0) - self.byteorder = '>' if fh.read(2) == b'MM' else '<' - self.bigtiff = {42: False, 43: True}[struct.unpack(self.byteorder + 'H', fh.read(2))[0]] - if self.bigtiff: - self.tagsize = 20 - self.tagnoformat = 'Q' - self.offsetsize = struct.unpack(self.byteorder + 'H', fh.read(2))[0] - self.offsetformat = {8: 'Q', 16: '2Q'}[self.offsetsize] - assert struct.unpack(self.byteorder + 'H', fh.read(2))[0] == 0, 'Not a TIFF-file' - self.offset = struct.unpack(self.byteorder + self.offsetformat, fh.read(self.offsetsize))[0] - else: - self.tagsize = 12 - self.tagnoformat = 'H' - self.offsetformat = 'I' - self.offsetsize = 4 - self.offset = struct.unpack(self.byteorder + self.offsetformat, fh.read(self.offsetsize))[0] - - def write(self, fh: BinaryIO) -> None: - fh.write({'<': b'II', '>': b'MM'}[self.byteorder]) - if self.bigtiff: - fh.write(struct.pack(self.byteorder + 'H', 43)) - fh.write(struct.pack(self.byteorder + 'H', 8)) - fh.write(struct.pack(self.byteorder + 'H', 0)) - fh.write(struct.pack(self.byteorder + 'Q', self.offset)) - else: - fh.write(struct.pack(self.byteorder + 'H', 42)) - fh.write(struct.pack(self.byteorder + 'I', self.offset)) - - -class Tag: - Value = bytes | str | float | Fraction | Sequence[bytes | str | float | Fraction] - tiff_tag_registry = tifffile.TiffTagRegistry({key: value.lower() for key, value in tifffile.TIFF.TAGS.items()}) - - @staticmethod - def from_dict(tags: dict[str | int, Value | Tag]) -> dict[int, Tag]: - return {(key if isinstance(key, int) - else (int(key[3:]) if key.lower().startswith('tag') - else Tag.tiff_tag_registry[key.lower()])): tag if isinstance(tag, Tag) else Tag(tag) - for key, tag in tags.items()} - - @staticmethod - def fraction(numerator: float = 0, denominator: int = None) -> Fraction: - return Fraction(numerator, denominator).limit_denominator( # type: ignore - 2 ** (31 if numerator < 0 or (denominator is not None and denominator < 0) else 32) - 1) - - def __init__(self, ttype_or_value: str | Value, value: Value = None, - offset: int = None) -> None: - self._value: bytes | str | Sequence[bytes | str | float | Fraction] - self.fh: Optional[BinaryIO] = None - self.header: Optional[Header] = None - self.bytes_data: Optional[bytes] = None - if value is None: - self.value = ttype_or_value # type: ignore - if isinstance(self.value, (str, bytes)) or all([isinstance(value, (str, bytes)) for value in self.value]): - ttype = 'ascii' - elif all([isinstance(value, int) for value in self.value]): - min_value: int = np.min(self.value) # type: ignore - max_value: int = np.max(self.value) # type: ignore - type_map = {'uint8': 'byte', 'int8': 'sbyte', 'uint16': 'short', 'int16': 'sshort', - 'uint32': 'long', 'int32': 'slong', 'uint64': 'long8', 'int64': 'slong8'} - for dtype, ttype in type_map.items(): - if np.iinfo(dtype).min <= min_value and max_value <= np.iinfo(dtype).max: - break - else: - ttype = 'undefined' - elif all([isinstance(value, Fraction) for value in self.value]): - if all([value.numerator < 0 or value.denominator < 0 for value in self.value]): # type: ignore - ttype = 'srational' - else: - ttype = 'rational' - elif all([isinstance(value, (float, int)) for value in self.value]): - min_value = np.min(np.asarray(self.value)[np.isfinite(self.value)]) # type: ignore - max_value = np.max(np.asarray(self.value)[np.isfinite(self.value)]) # type: ignore - type_map = {'float32': 'float', 'float64': 'double'} - for dtype, ttype in type_map.items(): - if np.finfo(dtype).min <= min_value and max_value <= np.finfo(dtype).max: - break - else: - ttype = 'undefined' - elif all([isinstance(value, complex) for value in self.value]): - ttype = 'complex' - else: - ttype = 'undefined' - self.ttype = tifffile.TIFF.DATATYPES[ttype.upper()] # noqa - else: - self.value = value # type: ignore - self.ttype = tifffile.TIFF.DATATYPES[ttype_or_value.upper()] if isinstance(ttype_or_value, str) \ - else ttype_or_value # type: ignore - self.dtype = tifffile.TIFF.DATA_FORMATS[self.ttype] - self.offset = offset - self.type_check() - - @property - def value(self) -> bytes | str | Sequence[bytes | str | float | Fraction]: - return self._value - - @value.setter - def value(self, value: Value) -> None: - self._value = value if isinstance(value, Iterable) else (value,) - - def __repr__(self) -> str: - if self.offset is None: - return f'{tifffile.TIFF.DATATYPES(self.ttype).name}: {self.value!r}' - else: - return f'{tifffile.TIFF.DATATYPES(self.ttype).name} @ {self.offset}: {self.value!r}' - - def type_check(self) -> None: - try: - self.bytes_and_count(Header()) - except Exception: - raise ValueError(f"tif tag type '{tifffile.TIFF.DATATYPES(self.ttype).name}' and " - f"data type '{type(self.value[0]).__name__}' do not correspond") - - def bytes_and_count(self, header: Header) -> tuple[bytes, int]: - if isinstance(self.value, bytes): - return self.value, len(self.value) // struct.calcsize(self.dtype) - elif self.ttype in (2, 14): - if isinstance(self.value, str): - bytes_value = self.value.encode('ascii') + b'\x00' - else: - bytes_value = b'\x00'.join([value.encode('ascii') for value in self.value]) + b'\x00' # type: ignore - return bytes_value, len(bytes_value) - elif self.ttype in (5, 10): - return b''.join([struct.pack(header.byteorder + self.dtype, # type: ignore - *((value.denominator, value.numerator) if isinstance(value, Fraction) - else value)) for value in self.value]), len(self.value) - else: - return b''.join([struct.pack(header.byteorder + self.dtype, value) for value in self.value]), \ - len(self.value) - - def write_tag(self, fh: BinaryIO, key: int, header: Header, offset: int = None) -> None: - self.fh = fh - self.header = header - if offset is None: - self.offset = fh.tell() - else: - fh.seek(offset) - self.offset = offset - fh.write(struct.pack(header.byteorder + 'HH', key, self.ttype)) - bytes_tag, count = self.bytes_and_count(header) - fh.write(struct.pack(header.byteorder + header.offsetformat, count)) - len_bytes = len(bytes_tag) - if len_bytes <= header.offsetsize: - fh.write(bytes_tag) - self.bytes_data = None - empty_bytes = header.offsetsize - len_bytes - else: - self.bytes_data = bytes_tag - empty_bytes = header.offsetsize - if empty_bytes: - fh.write(empty_bytes * b'\x00') - - def write_data(self, write: Callable[[BinaryIO, bytes], None] = None) -> None: - if self.bytes_data and self.fh is not None and self.header is not None and self.offset is not None: - self.fh.seek(0, 2) - if write is None: - offset = self.write(self.bytes_data) - else: - offset = write(self.fh, self.bytes_data) - self.fh.seek(self.offset + self.header.tagsize - self.header.offsetsize) - self.fh.write(struct.pack(self.header.byteorder + self.header.offsetformat, offset)) - - def write(self, bytes_value: bytes) -> Optional[int]: - if self.fh is not None: - if self.fh.tell() % 2: - self.fh.write(b'\x00') - offset = self.fh.tell() - self.fh.write(bytes_value) - return offset - - def copy(self) -> Tag: - return self.__class__(self.ttype, self.value[:], self.offset) - - -class IFD(dict): - def __init__(self, fh: BinaryIO = None) -> None: - super().__init__() - self.fh = fh - self.header: Optional[Header] = None - self.offset: Optional[int] = None - self.where_to_write_next_ifd_offset: Optional[int] = None - if fh is not None: - header = Header(fh) - fh.seek(header.offset) - n_tags = struct.unpack(header.byteorder + header.tagnoformat, - fh.read(struct.calcsize(header.tagnoformat)))[0] - assert n_tags < 4096, 'Too many tags' - addr = [] - addroffset = [] - - length = 8 if header.bigtiff else 2 - length += n_tags * header.tagsize + header.offsetsize - - for i in range(n_tags): - pos = header.offset + struct.calcsize(header.tagnoformat) + header.tagsize * i - fh.seek(pos) - - code, ttype = struct.unpack(header.byteorder + 'HH', fh.read(4)) - count = struct.unpack(header.byteorder + header.offsetformat, fh.read(header.offsetsize))[0] - - dtype = tifffile.TIFF.DATA_FORMATS[ttype] - dtypelen = struct.calcsize(dtype) - - toolong = struct.calcsize(dtype) * count > header.offsetsize - if toolong: - addr.append(fh.tell() - header.offset) - caddr = struct.unpack(header.byteorder + header.offsetformat, fh.read(header.offsetsize))[0] - addroffset.append(caddr - header.offset) - cp = fh.tell() - fh.seek(caddr) - - if ttype == 1: - value: Tag.Value = fh.read(count) - elif ttype == 2: - value = fh.read(count).decode('ascii').rstrip('\x00') - elif ttype in (5, 10): - value = [struct.unpack(header.byteorder + dtype, fh.read(dtypelen)) # type: ignore - for _ in range(count)] - else: - value = [struct.unpack(header.byteorder + dtype, fh.read(dtypelen))[0] for _ in range(count)] - - if toolong: - fh.seek(cp) # noqa - - self[code] = Tag(ttype, value, pos) - fh.seek(header.offset) - - def __setitem__(self, key: str | int, tag: str | float | Fraction | Tag) -> None: - super().__setitem__(Tag.tiff_tag_registry[key.lower()] if isinstance(key, str) else key, - tag if isinstance(tag, Tag) else Tag(tag)) - - def items(self) -> Generator[tuple[int, Tag], None, None]: # type: ignore[override] - return ((key, self[key]) for key in sorted(self)) - - def keys(self) -> Generator[int, None, None]: # type: ignore[override] - return (key for key in sorted(self)) - - def values(self) -> Generator[Tag, None, None]: # type: ignore[override] - return (self[key] for key in sorted(self)) - - def write(self, fh: BinaryIO, header: Header, write: Callable[[BinaryIO, bytes], None] = None) -> BinaryIO: - self.fh = fh - self.header = header - if fh.seek(0, 2) % 2: - fh.write(b'\x00') - self.offset = fh.tell() - fh.write(struct.pack(header.byteorder + header.tagnoformat, len(self))) - for key, tag in self.items(): - tag.write_tag(fh, key, header) - self.where_to_write_next_ifd_offset = fh.tell() - fh.write(b'\x00' * header.offsetsize) - for tag in self.values(): - tag.write_data(write) - return fh - - def write_offset(self, where_to_write_offset: int) -> None: - if self.fh is not None and self.header is not None: - self.fh.seek(where_to_write_offset) - self.fh.write(struct.pack(self.header.byteorder + self.header.offsetformat, self.offset)) - - def copy(self) -> IFD: - new = self.__class__() - new.update({key: tag.copy() for key, tag in self.items()}) - return new - - -FrameInfo = tuple[IFD, Strip, CZT] - - -class IJTiffFile: - """ Writes a tiff file in a format that the BioFormats reader in Fiji understands. - file: filename of the new tiff file - shape: shape (CZT) of the data to be written - dtype: datatype to use when saving to tiff - colors: a tuple with a color per channel, chosen from matplotlib.colors, html colors are also possible - colormap: name of a colormap from colorcet - pxsize: pixel size in um - deltaz: z slice interval in um - timeinterval: time between frames in seconds - extratags: other tags to be saved, example: Artist='John Doe', Tag4567=[400, 500] - or Copyright=Tag('ascii', 'Made by me'). See tiff_tag_registry.items(). - wp@tl20200214 - """ - def __init__(self, path: str | Path, shape: tuple[int, int, int], dtype: DTypeLike = 'uint16', - colors: Sequence[str] = None, colormap: str = None, pxsize: float = None, - deltaz: float = None, timeinterval: float = None, - compression: tuple[int, int] = (50000, 22), comment: str = None, - **extratags: Tag.Value | Tag) -> None: - assert len(shape) >= 3, 'please specify all c, z, t for the shape' - assert len(shape) <= 3, 'please specify only c, z, t for the shape' - assert np.dtype(dtype).char in 'BbHhf', 'datatype not supported' - assert colors is None or colormap is None, 'cannot have colors and colormap simultaneously' - - self.path = Path(path) - self.shape = shape - self.dtype = np.dtype(dtype) - self.colors = colors - self.colormap = colormap - self.pxsize = pxsize - self.deltaz = deltaz - self.timeinterval = timeinterval - self.compression = compression - self.comment = comment - self.extratags = {} if extratags is None else Tag.from_dict(extratags) # type: ignore - if pxsize is not None: - pxsize_fraction = Tag.fraction(pxsize) - self.extratags.update({282: Tag(pxsize_fraction), 283: Tag(pxsize_fraction)}) - - self.header = Header() - self.spp = self.shape[0] if self.colormap is None and self.colors is None else 1 # samples/pixel - self.nframes = np.prod(self.shape[1:]) if self.colormap is None and self.colors is None else np.prod(self.shape) - self.frame_extra_tags: dict[tuple[int, int, int], dict[int, Tag]] = {} - self.fh = FileHandle(self.path) - self.pool = ParPool(self.compress_frame) # type: ignore - self.hashes = PoolSingleton().manager.dict() - self.main_process = True - - with self.fh.lock() as fh: # noqa - self.header.write(fh) - - def __setstate__(self, state: dict[str, Any]) -> None: - self.__dict__.update(state) - self.main_process = False - - def __hash__(self) -> int: - return hash(self.path) - - def get_frame_number(self, n: tuple[int, int, int]) -> tuple[int, int]: - if self.colormap is None and self.colors is None: - return n[1] + n[2] * self.shape[1], n[0] - else: - return n[0] + n[1] * self.shape[0] + n[2] * self.shape[0] * self.shape[1], 0 - - def ij_tiff_frame(self, frame: np.ndarray) -> bytes: - with BytesIO() as frame_bytes: - with tifffile.TiffWriter(frame_bytes, bigtiff=self.header.bigtiff, - byteorder=self.header.byteorder) as t: # type: ignore - # predictor=True might save a few bytes, but requires the package imagecodes to save floats - t.write(frame, compression=self.compression, contiguous=True, predictor=False) # type: ignore - return frame_bytes.getvalue() - - def save(self, frame: np.ndarray | Any, c: int, z: int, t: int, - **extratags: Tag.Value | Tag) -> None: - """ save a 2d numpy array to the tiff at channel=c, slice=z, time=t, with optional extra tif tags - """ - assert (c, z, t) not in self.pool.tasks, f'frame {c} {z} {t} is added already' - assert all([0 <= i < s for i, s in zip((c, z, t), self.shape)]), \ - 'frame {} {} {} is outside shape {} {} {}'.format(c, z, t, *self.shape) - self.pool(frame.astype(self.dtype) if hasattr(frame, 'astype') else frame, handle=(c, z, t)) - if extratags: - self.frame_extra_tags[(c, z, t)] = Tag.from_dict(extratags) # type: ignore - - @property - def description(self) -> bytes: - desc = ['ImageJ=1.11a'] - if self.colormap is None and self.colors is None: - desc.extend((f'images={np.prod(self.shape[:1])}', f'slices={self.shape[1]}', f'frames={self.shape[2]}')) - else: - desc.extend((f'images={np.prod(self.shape)}', f'channels={self.shape[0]}', f'slices={self.shape[1]}', - f'frames={self.shape[2]}')) - if self.shape[0] == 1: - desc.append('mode=grayscale') - else: - desc.append('mode=composite') - desc.extend(('hyperstack=true', 'loop=false', 'unit=micron')) - if self.deltaz is not None: - desc.append(f'spacing={self.deltaz}') - if self.timeinterval is not None: - desc.append(f'interval={self.timeinterval}') - desc_bytes = [bytes(d, 'ascii') for d in desc] - if self.comment is not None: - desc_bytes.append(b'') - if isinstance(self.comment, bytes): - desc_bytes.append(self.comment) - else: - desc_bytes.append(bytes(self.comment, 'ascii')) - return b'\n'.join(desc_bytes) + b'\0' - - @cached_property - def colormap_bytes(self) -> Optional[bytes]: - if self.colormap: - colormap = getattr(colorcet, self.colormap) - colormap[0] = '#ffffff' - colormap[-1] = '#000000' - colormap = 65535 * np.array( - [[int(''.join(i), 16) for i in zip(*[iter(s[1:])] * 2)] for s in colormap]) // 255 - if np.dtype(self.dtype).itemsize == 2: - colormap = np.tile(colormap, 256).reshape((-1, 3)) - return b''.join([struct.pack(self.header.byteorder + 'H', c) for c in colormap.T.flatten()]) - - @cached_property - def colors_bytes(self) -> list[bytes]: - return [b''.join([struct.pack(self.header.byteorder + 'H', c) - for c in np.linspace(0, 65535 * np.array(mpl_colors.to_rgb(color)), - 65536 if np.dtype(self.dtype).itemsize == 2 else 256, - dtype=int).T.flatten()]) for color in self.colors] if self.colors else [] - - def close(self) -> None: - if self.main_process: - ifds, strips = {}, {} - for n in list(self.pool.tasks): - for ifd, strip, delta in self.pool[n]: - framenr, channel = self.get_frame_number(tuple(i + j for i, j in zip(n, delta))) # type: ignore - ifds[framenr], strips[(framenr, channel)] = ifd, strip - - self.pool.close() - with self.fh.lock() as fh: # noqa - for n, tags in self.frame_extra_tags.items(): - framenr, _ = self.get_frame_number(n) - ifds[framenr].update(tags) - if 0 in ifds and self.colormap is not None: - ifds[0][320] = Tag('SHORT', self.colormap_bytes) - ifds[0][262] = Tag('SHORT', 3) - if self.colors is not None: - for c, color in enumerate(self.colors_bytes): - if c in ifds: - ifds[c][320] = Tag('SHORT', color) - ifds[c][262] = Tag('SHORT', 3) - if 0 in ifds and 306 not in ifds[0]: - ifds[0][306] = Tag('ASCII', datetime.now().strftime('%Y:%m:%d %H:%M:%S')) - wrn = False - for framenr in range(self.nframes): - if framenr in ifds and all([(framenr, channel) in strips for channel in range(self.spp)]): - stripbyteoffsets, stripbytecounts = zip(*[strips[(framenr, channel)] - for channel in range(self.spp)]) - ifds[framenr][258].value = self.spp * ifds[framenr][258].value - ifds[framenr][270] = Tag('ASCII', self.description) - ifds[framenr][273] = Tag('LONG8', sum(stripbyteoffsets, [])) - ifds[framenr][277] = Tag('SHORT', self.spp) - ifds[framenr][279] = Tag('LONG8', sum(stripbytecounts, [])) - ifds[framenr][305] = Tag('ASCII', 'tiffwrite_tllab_NKI') - if self.extratags is not None: - ifds[framenr].update(self.extratags) - if self.colormap is None and self.colors is None and self.shape[0] > 1: - ifds[framenr][284] = Tag('SHORT', 2) - ifds[framenr].write(fh, self.header, self.write) - if framenr: - ifds[framenr].write_offset(ifds[framenr - 1].where_to_write_next_ifd_offset) - else: - ifds[framenr].write_offset(self.header.offset - self.header.offsetsize) - else: - wrn = True - if wrn: - warnings.warn('Some frames were not added to the tif file, either you forgot them, ' - 'or an error occurred and the tif file was closed prematurely.') - - def __enter__(self) -> IJTiffFile: - return self - - def __exit__(self, *args: Any, **kwargs: Any) -> None: - self.close() - - @staticmethod - def hash_check(fh: BinaryIO, bvalue: bytes, offset: int) -> bool: - addr = fh.tell() - fh.seek(offset) - same = bvalue == fh.read(len(bvalue)) - fh.seek(addr) - return same - - def write(self, fh: BinaryIO, bvalue: bytes) -> int: - hash_value = sha1(bvalue).hexdigest() # hash uses a random seed making hashes different in different processes - if hash_value in self.hashes and self.hash_check(fh, bvalue, self.hashes[hash_value]): - return self.hashes[hash_value] # reuse previously saved data - else: - if fh.tell() % 2: - fh.write(b'\x00') - offset = fh.tell() - self.hashes[hash_value] = offset - fh.write(bvalue) - return offset - - def compress_frame(self, frame: np.ndarray) -> Sequence[FrameInfo]: - """ This is run in a different process. Turns an image into bytes, writes them and returns the ifd, strip info - and czt delta. When subclassing IJTiffWrite this can be overridden to write one or more (using czt delta) - frames. - """ - stripbytecounts, ifd, chunks = self.get_chunks(self.ij_tiff_frame(frame)) - stripbyteoffsets = [] - with self.fh.lock() as fh: # noqa - for chunk in chunks: - stripbyteoffsets.append(self.write(fh, chunk)) - return (ifd, (stripbyteoffsets, stripbytecounts), (0, 0, 0)), - - @staticmethod - def get_chunks(frame: bytes) -> tuple[list[int], IFD, list[bytes]]: - with BytesIO(frame) as fh: - ifd = IFD(fh) - stripoffsets = ifd[273].value - stripbytecounts = ifd[279].value - chunks = [] - for stripoffset, stripbytecount in zip(stripoffsets, stripbytecounts): - fh.seek(stripoffset) - chunks.append(fh.read(stripbytecount)) - return stripbytecounts, ifd, chunks - - -class FileHandle: - """ Process safe file handle """ - def __init__(self, path: Path) -> None: - manager = PoolSingleton().manager - if path.exists(): - path.unlink() - with open(path, 'xb'): - pass - self.path = path - self._lock = manager.RLock() - self._pos = manager.Value('i', 0) - - @contextmanager - def lock(self) -> Generator[BinaryIO, None, None]: - with self._lock: - with open(self.path, 'rb+') as f: - try: - f.seek(self._pos.value) - yield f - finally: - self._pos.value = f.tell() +class IJTiffFile(rs.IJTiffFile): + pass From 52785037b9403889ad3cb1b219d30053d31c2493 Mon Sep 17 00:00:00 2001 From: Wim Pomp Date: Wed, 9 Oct 2024 12:05:11 +0200 Subject: [PATCH 02/13] - can now save the common types --- Cargo.lock | 596 ------------------------------------------ Cargo.toml | 10 +- src/lib.rs | 105 ++++++-- src/main.rs | 29 +- src/py.rs | 71 ++++- tiffwrite/__init__.py | 89 ++++++- 6 files changed, 241 insertions(+), 659 deletions(-) delete mode 100644 Cargo.lock diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index efdd80a..0000000 --- a/Cargo.lock +++ /dev/null @@ -1,596 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anyhow" -version = "1.0.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" - -[[package]] -name = "autocfg" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" - -[[package]] -name = "bumpalo" -version = "3.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" - -[[package]] -name = "cc" -version = "1.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" -dependencies = [ - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "chrono" -version = "0.4.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", - "num-traits", - "wasm-bindgen", - "windows-targets", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "crossbeam-deque" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" - -[[package]] -name = "either" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" - -[[package]] -name = "fraction" -version = "0.15.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7" -dependencies = [ - "lazy_static", - "num", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "iana-time-zone" -version = "0.1.61" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "indoc" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" - -[[package]] -name = "js-sys" -version = "0.3.70" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "libc" -version = "0.2.159" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" - -[[package]] -name = "log" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" - -[[package]] -name = "matrixmultiply" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" -dependencies = [ - "autocfg", - "rawpointer", -] - -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - -[[package]] -name = "ndarray" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" -dependencies = [ - "matrixmultiply", - "num-complex", - "num-integer", - "num-traits", - "portable-atomic", - "portable-atomic-util", - "rawpointer", -] - -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - -[[package]] -name = "num-bigint" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "once_cell" -version = "1.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" - -[[package]] -name = "portable-atomic" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" - -[[package]] -name = "portable-atomic-util" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdd8420072e66d54a407b3316991fe946ce3ab1083a7f575b2463866624704d" -dependencies = [ - "portable-atomic", -] - -[[package]] -name = "proc-macro2" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "pyo3" -version = "0.22.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ee168e30649f7f234c3d49ef5a7a6cbf5134289bc46c29ff3155fa3221c225" -dependencies = [ - "anyhow", - "cfg-if", - "indoc", - "libc", - "memoffset", - "once_cell", - "portable-atomic", - "pyo3-build-config", - "pyo3-ffi", - "pyo3-macros", - "unindent", -] - -[[package]] -name = "pyo3-build-config" -version = "0.22.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e61cef80755fe9e46bb8a0b8f20752ca7676dcc07a5277d8b7768c6172e529b3" -dependencies = [ - "once_cell", - "python3-dll-a", - "target-lexicon", -] - -[[package]] -name = "pyo3-ffi" -version = "0.22.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ce096073ec5405f5ee2b8b31f03a68e02aa10d5d4f565eca04acc41931fa1c" -dependencies = [ - "libc", - "pyo3-build-config", -] - -[[package]] -name = "pyo3-macros" -version = "0.22.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2440c6d12bc8f3ae39f1e775266fa5122fd0c8891ce7520fa6048e683ad3de28" -dependencies = [ - "proc-macro2", - "pyo3-macros-backend", - "quote", - "syn", -] - -[[package]] -name = "pyo3-macros-backend" -version = "0.22.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1be962f0e06da8f8465729ea2cb71a416d2257dff56cbe40a70d3e62a93ae5d1" -dependencies = [ - "heck", - "proc-macro2", - "pyo3-build-config", - "quote", - "syn", -] - -[[package]] -name = "python3-dll-a" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd0b78171a90d808b319acfad166c4790d9e9759bbc14ac8273fe133673dd41b" -dependencies = [ - "cc", -] - -[[package]] -name = "quote" -version = "1.0.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rawpointer" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" - -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "syn" -version = "2.0.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "target-lexicon" -version = "0.12.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" - -[[package]] -name = "tiffwrite" -version = "2024.10.0" -dependencies = [ - "anyhow", - "chrono", - "fraction", - "ndarray", - "num", - "pyo3", - "rayon", -] - -[[package]] -name = "unicode-ident" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" - -[[package]] -name = "unindent" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" - -[[package]] -name = "wasm-bindgen" -version = "0.2.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" -dependencies = [ - "cfg-if", - "once_cell", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" - -[[package]] -name = "windows-core" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" diff --git a/Cargo.toml b/Cargo.toml index 8680bd6..71603b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,21 +1,23 @@ [package] name = "tiffwrite" -version = "2024.10.0" +version = "2024.10.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] name = "tiffwrite" -crate-type = ["cdylib"] +crate-type = ["cdylib", "rlib"] [dependencies] -pyo3 = { version = "0.22.3", features = ["extension-module", "abi3-py310", "generate-import-lib", "anyhow"] } +pyo3 = { version = "0.21.2", features = ["extension-module", "abi3-py310", "generate-import-lib", "anyhow", "multiple-pymethods"] } anyhow = "1.0.89" rayon = "1.10.0" fraction = "0.15.3" num = "0.4.3" -ndarray = "0.16.1" +ndarray = "0.15.6" chrono = "0.4.38" +numpy = "0.21.0" +futures = "0.3.31" [features] nopython = [] diff --git a/src/lib.rs b/src/lib.rs index 13f946f..0953369 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,6 @@ +#[cfg(not(feature = "nopython"))] +mod py; + use std::cmp::Ordering; use std::collections::HashMap; use std::fs::{File, OpenOptions}; @@ -15,6 +18,7 @@ use chrono::Utc; const TAG_SIZE: usize = 20; const OFFSET_SIZE: usize = 8; const OFFSET: u64 = 16; +const COMPRESSION: u16 = 1; #[derive(Clone, Debug)] @@ -238,7 +242,7 @@ struct Frame { image_width: u32, image_length: u32, bits_per_sample: u16, - compression: u16, + sample_format: u16, tile_width: u16, tile_length: u16, extra_tags: Vec @@ -246,17 +250,63 @@ struct Frame { impl Frame { fn new( - tilebyteoffsets: Vec, tilebytecounts: Vec, image_width: u32, - image_length: u32, bits_per_sample: u16, compression: u16, tile_width: u16, tile_length: u16 + tilebyteoffsets: Vec, tilebytecounts: Vec, image_width: u32, image_length: u32, + bits_per_sample: u16, sample_format: u16, tile_width: u16, tile_length: u16 ) -> Self { Frame { tilebyteoffsets, tilebytecounts, image_width, image_length, bits_per_sample, - compression, tile_width, tile_length, extra_tags: Vec::new() + sample_format, tile_width, tile_length, extra_tags: Vec::new() } } } +pub trait Bytes { + const BITS_PER_SAMPLE: u16; + const SAMPLE_FORMAT: u16; + + fn bytes(&self) -> Vec; +} + + +macro_rules! bytes_impl { + ($T:ty, $bits_per_sample:expr, $sample_format:expr) => { + impl Bytes for $T { + const BITS_PER_SAMPLE: u16 = $bits_per_sample; + const SAMPLE_FORMAT: u16 = $sample_format; + + #[inline] + fn bytes(&self) -> Vec + { + self.to_le_bytes().to_vec() + } + } + }; +} + +bytes_impl!(u8, 8, 1); +bytes_impl!(u16, 16, 1); +bytes_impl!(u32, 32, 1); +bytes_impl!(u64, 64, 1); +bytes_impl!(u128, 128, 1); +#[cfg(target_pointer_width = "64")] +bytes_impl!(usize, 64, 1); +#[cfg(target_pointer_width = "32")] +bytes_impl!(usize, 32, 1); + +bytes_impl!(i8, 8, 2); +bytes_impl!(i16, 16, 2); +bytes_impl!(i32, 32, 2); +bytes_impl!(i64, 64, 2); +bytes_impl!(i128, 128, 2); +#[cfg(target_pointer_width = "64")] +bytes_impl!(isize, 64, 2); +#[cfg(target_pointer_width = "32")] +bytes_impl!(isize, 32, 2); +bytes_impl!(f32, 32, 3); +bytes_impl!(f64, 64, 3); + + #[derive(Debug)] pub struct IJTiffFile { file: File, @@ -331,19 +381,6 @@ impl IJTiffFile { desc } - pub fn save(&mut self, frame: Array2, c: usize, z: usize, t: usize, - extra_tags: Option>) -> Result<()> { - let mut compressed_frame = self.compress_frame(frame)?; - if let Some(tags) = extra_tags { - for tag in tags { - compressed_frame.extra_tags.push(tag); - } - } - - self.frames.insert(self.get_frame_number(c, z, t), compressed_frame); - Ok(()) - } - fn get_frame_number(&self, c: usize, z: usize, t: usize) -> (usize, u8) { if let (None, None) = (self.colormap.as_ref(), self.colors.as_ref()) { (z + t * self.shape.1, c as u8) @@ -352,7 +389,7 @@ impl IJTiffFile { } } - pub fn hash(value: &T) -> u64 { + fn hash(value: &T) -> u64 { let mut hasher = DefaultHasher::new(); value.hash(&mut hasher); hasher.finish() @@ -383,21 +420,32 @@ impl IJTiffFile { } } - fn compress_frame(&mut self, frame: Array2) -> Result { + pub fn save(&mut self, frame: Array2, c: usize, z: usize, t: usize, + extra_tags: Option>) -> Result<()> { + self.compress_frame(frame, c, z, t, extra_tags); + Ok(()) + } + + fn compress_frame(&mut self, frame: Array2, c: usize, z: usize, t: usize, + extra_tags: Option>) { let image_width = frame.shape()[0] as u32; let image_length = frame.shape()[1] as u32; let mut tilebyteoffsets = Vec::new(); let mut tilebytecounts = Vec::new(); let tiles = IJTiffFile::tile(frame.reversed_axes(), 64); for tile in tiles { - let bytes: Vec = tile.into_flat().into_iter().map( - |x| x.to_le_bytes()).into_iter().flatten().collect(); + let bytes: Vec = tile.map(|x| x.bytes()).into_iter().flatten().collect(); tilebytecounts.push(bytes.len() as u64); - tilebyteoffsets.push(self.write(&bytes)?); + tilebyteoffsets.push(self.write(&bytes).unwrap()); } - - Ok(Frame::new(tilebyteoffsets, tilebytecounts, image_width, image_length, - 16, 1, 64, 64)) + let mut frame = Frame::new(tilebyteoffsets, tilebytecounts, image_width, image_length, + T::BITS_PER_SAMPLE, T::SAMPLE_FORMAT, 64, 64); + if let Some(tags) = extra_tags { + for tag in tags { + frame.extra_tags.push(tag); + } + } + self.frames.insert(self.get_frame_number(c, z, t), frame); } fn tile(frame: Array2, size: usize) -> Vec> { @@ -438,15 +486,13 @@ impl IJTiffFile { fn get_colormap(&self, colormap: &Vec) -> Result> { todo!(); - Ok(Vec::new()) } fn get_color(&self, colors: (u8, u8, u8)) -> Result> { todo!(); - Ok(Vec::new()) } - pub fn close(&mut self) -> Result<()> { + fn close(&mut self) -> Result<()> { let mut where_to_write_next_ifd_offset = OFFSET - OFFSET_SIZE as u64; let mut warn = false; for frame_number in 0..self.n_frames { @@ -467,7 +513,7 @@ impl IJTiffFile { ifd.push_tag(Tag::long(256, vec![frame.image_width])); ifd.push_tag(Tag::long(257, vec![frame.image_length])); ifd.push_tag(Tag::short(258, vec![frame.bits_per_sample; frame_count])); - ifd.push_tag(Tag::short(259, vec![1])); + ifd.push_tag(Tag::short(259, vec![COMPRESSION])); ifd.push_tag(Tag::ascii(270, &self.description())); ifd.push_tag(Tag::short(277, vec![frame_count as u16])); ifd.push_tag(Tag::ascii(305, "tiffwrite_rs")); @@ -475,6 +521,7 @@ impl IJTiffFile { ifd.push_tag(Tag::short(323, vec![frame.tile_length])); ifd.push_tag(Tag::long8(324, tilebyteoffsets)); ifd.push_tag(Tag::long8(325, tilebytecounts)); + ifd.push_tag(Tag::short(339, vec![frame.sample_format])); if frame_number == 0 { if let Some(colormap) = &self.colormap { ifd.push_tag(Tag::short(320, self.get_colormap(colormap)?)); diff --git a/src/main.rs b/src/main.rs index f3fd0a5..658eac3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,28 +1,23 @@ -#[cfg(not(feature = "nopython"))] -mod py; -mod lib; - use anyhow::Result; use ndarray::{s, Array2}; -use rayon::prelude::*; -use crate::lib::IJTiffFile; +use tiffwrite::IJTiffFile; fn main() -> Result<()> { println!("Hello World!"); - let mut f = IJTiffFile::new("foo.tif", (1, 2, 1))?; + let mut f = IJTiffFile::new("foo.tif", (2, 1, 1))?; let mut arr = Array2::::zeros((100, 100)); - // for i in 0..arr.shape()[0] { - // for j in 0..arr.shape()[1] { - // arr[[i, j]] = i as u16; - // } - // } + for i in 0..arr.shape()[0] { + for j in 0..arr.shape()[1] { + arr[[i, j]] = i as u16; + } + } f.save(arr.to_owned(), 0, 0, 0, None)?; - // let mut arr = Array2::::zeros((100, 100)); - // arr.slice_mut(s![64.., ..64]).fill(1); - // arr.slice_mut(s![..64, 64..]).fill(2); - // arr.slice_mut(s![64.., 64..]).fill(3); - f.save(arr.to_owned(), 0, 1,0, None)?; + let mut arr = Array2::::zeros((100, 100)); + arr.slice_mut(s![64.., ..64]).fill(1); + arr.slice_mut(s![..64, 64..]).fill(2); + arr.slice_mut(s![64.., 64..]).fill(3); + f.save(arr.to_owned(), 1, 0,0, None)?; Ok(()) } \ No newline at end of file diff --git a/src/py.rs b/src/py.rs index 4ba2d78..e8f92ef 100644 --- a/src/py.rs +++ b/src/py.rs @@ -1,9 +1,8 @@ use pyo3::prelude::*; -use crate::lib::{IJTiffFile, Tag}; -use std::time::Duration; -use pyo3::types::{PyInt, PyString}; +use crate::{IJTiffFile, Tag}; use fraction::Fraction; use num::Complex; +use numpy::{PyReadonlyArray2, PyArrayMethods}; #[pyclass(subclass)] @@ -106,14 +105,14 @@ impl PyTag { #[pyo3(name = "IJTiffFile")] #[derive(Debug)] struct PyIJTiffFile { - ijtifffile: IJTiffFile + ijtifffile: Option } #[pymethods] impl PyIJTiffFile { #[new] - fn new(path: &str, shape: (usize, usize, usize), dtype: &str) -> PyResult { - Ok(PyIJTiffFile { ijtifffile: IJTiffFile::new(path, shape)? } ) + fn new(path: &str, shape: (usize, usize, usize)) -> PyResult { + Ok(PyIJTiffFile { ijtifffile: Some(IJTiffFile::new(path, shape)?) } ) } fn with_colors(&mut self, colors: (u8, u8, u8)) -> Self { @@ -140,19 +139,67 @@ impl PyIJTiffFile { todo!() } - fn append_extra_tag(&mut self, extra_tag: PyTag) { - if let Some(extra_tags) = self.ijtifffile.extra_tags.as_mut() { - extra_tags.push(extra_tag.tag); - } else { - self.ijtifffile.extra_tags = Some(vec![extra_tag.tag]); + fn append_extra_tag(&mut self, tag: PyTag) { + if let Some(ijtifffile) = self.ijtifffile.as_mut() { + if let Some(extra_tags) = ijtifffile.extra_tags.as_mut() { + extra_tags.push(tag.tag); + } else { + ijtifffile.extra_tags = Some(vec![tag.tag]); + } } } + fn extend_extra_tags(&mut self, tags: Vec) { + if let Some(ijtifffile) = self.ijtifffile.as_mut() { + if let Some(extra_tags) = ijtifffile.extra_tags.as_mut() { + extra_tags.extend(tags.into_iter().map(|x| x.tag)); + } else { + ijtifffile.extra_tags = Some(tags.into_iter().map(|x| x.tag).collect()); + } + } + } + fn close(&mut self) -> PyResult<()> { + self.ijtifffile.take(); + Ok(()) + } } + +macro_rules! impl_save { + ($T:ty, $t:ident) => { + #[pymethods] + impl PyIJTiffFile { + fn $t(&mut self, frame: PyReadonlyArray2<$T>, c: usize, t: usize, z: usize, + extra_tags: Option>) -> PyResult<()> { + let extra_tags = if let Some(extra_tags) = extra_tags { + Some(extra_tags.into_iter().map(|x| x.tag).collect()) + } else { + None + }; + if let Some(ijtifffile) = self.ijtifffile.as_mut() { + ijtifffile.save(frame.to_owned_array(), c, t, z, extra_tags)?; + } + Ok(()) + } + } + }; +} + +impl_save!(u8, save_u8); +impl_save!(u16, save_u16); +impl_save!(u32, save_u32); +impl_save!(u64, save_u64); +impl_save!(i8, save_i8); +impl_save!(i16, save_i16); +impl_save!(i32, save_i32); +impl_save!(i64, save_i64); +impl_save!(f32, save_f32); +impl_save!(f64, save_f64); + #[pymodule] fn tiffwrite(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; m.add_class::()?; Ok(()) -} \ No newline at end of file +} diff --git a/tiffwrite/__init__.py b/tiffwrite/__init__.py index 7f1ac1f..ebf2282 100644 --- a/tiffwrite/__init__.py +++ b/tiffwrite/__init__.py @@ -1,5 +1,92 @@ +from __future__ import annotations + +import numpy as np +from typing import Any, Self, Sequence +from pathlib import Path +from numpy.typing import ArrayLike, DTypeLike + from . import tiffwrite as rs -class IJTiffFile(rs.IJTiffFile): +__all__ = ['Tag', 'IJTiffFile', 'tiffwrite'] + + +class IFD(dict): pass + + +class Tag(rs.Tag): + pass + + +class IJTiffFile(rs.IJTiffFile): + def __new__(cls, path: str | Path, shape: tuple[int, int, int], dtype: DTypeLike = 'uint16', + colors: Sequence[str] = None, colormap: str = None, pxsize: float = None, + deltaz: float = None, timeinterval: float = None, comment: str = None, + **extratags: Tag.Value | Tag) -> None: + new = super().__new__(cls, str(path), shape) + if colors is not None: + new = new.with_colors(colors) + if colormap is not None: + new = new.with_colormap(colormap) + if pxsize is not None: + new = new.with_pxsize(pxsize) + if deltaz is not None: + new = new.with_deltaz(deltaz) + if timeinterval is not None: + new = new.with_timeinterval(timeinterval) + if comment is not None: + new = new.with_comment(comment) + if extratags: + new = new.extend_extratags(extratags) + return new + + def __init__(self, path: str | Path, shape: tuple[int, int, int], dtype: DTypeLike = 'uint16', + colors: Sequence[str] = None, colormap: str = None, pxsize: float = None, + deltaz: float = None, timeinterval: float = None, comment: str = None, + **extratags: Tag.Value | Tag) -> None: + self.path = Path(path) + self.dtype = np.dtype(dtype) + + def __enter__(self) -> Self: + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def save(self, frame: ArrayLike, c: int, z: int, t: int) -> None: + frame = np.asarray(frame).astype(self.dtype) + match self.dtype: + case np.uint8: + self.save_u8(frame, c, z, t) + case np.uint16: + self.save_u16(frame, c, z, t) + case np.uint32: + self.save_u32(frame, c, z, t) + case np.uint64: + self.save_u64(frame, c, z, t) + case np.int8: + self.save_i8(frame, c, z, t) + case np.int16: + self.save_i16(frame, c, z, t) + case np.int32: + self.save_i32(frame, c, z, t) + case np.int64: + self.save_i64(frame, c, z, t) + case np.float32: + self.save_f32(frame, c, z, t) + case np.float64: + self.save_f64(frame, c, z, t) + case _: + raise TypeError(f'Cannot save type {self.dtype}') + + +def tiffwrite(file: str | Path, data: ArrayLike, axes: str = 'TZCXY', dtype: DTypeLike = None, bar: bool = False, + *args: Any, **kwargs: Any) -> None: + """ file: string; filename of the new tiff file + data: 2 to 5D numpy array + axes: string; order of dimensions in data, default: TZCXY for 5D, ZCXY for 4D, CXY for 3D, XY for 2D data + dtype: string; datatype to use when saving to tiff + bar: bool; whether to show a progress bar + other args: see IJTiffFile + """ From f62b711692d75523b527fd85542dbf379c9cefa4 Mon Sep 17 00:00:00 2001 From: Wim Pomp Date: Wed, 9 Oct 2024 15:07:38 +0200 Subject: [PATCH 03/13] - add tiffwrite function for python - parallel zstd compression --- .github/workflows/mypy.yml | 21 -------- .gitignore | 2 + Cargo.toml | 3 +- {tiffwrite => py/tiffwrite}/__init__.py | 51 ++++++++++++++---- pyproject.toml | 6 ++- src/lib.rs | 72 +++++++++++++------------ src/py.rs | 15 ++++-- 7 files changed, 97 insertions(+), 73 deletions(-) delete mode 100644 .github/workflows/mypy.yml rename {tiffwrite => py/tiffwrite}/__init__.py (64%) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml deleted file mode 100644 index 9146014..0000000 --- a/.github/workflows/mypy.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: MyPy - -on: [push, pull_request] - -jobs: - mypy: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10", "3.12"] - - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: pip install .[test] - - name: Test with mypy - run: mypy . \ No newline at end of file diff --git a/.gitignore b/.gitignore index f5af85d..889ae51 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ /.pytest_cache/ /venv/ /target/ +/Cargo.lock +/foo.tif diff --git a/Cargo.toml b/Cargo.toml index 71603b6..0bc729b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,12 +12,11 @@ crate-type = ["cdylib", "rlib"] pyo3 = { version = "0.21.2", features = ["extension-module", "abi3-py310", "generate-import-lib", "anyhow", "multiple-pymethods"] } anyhow = "1.0.89" rayon = "1.10.0" -fraction = "0.15.3" num = "0.4.3" ndarray = "0.15.6" chrono = "0.4.38" numpy = "0.21.0" -futures = "0.3.31" +zstd = "0.13.2" [features] nopython = [] diff --git a/tiffwrite/__init__.py b/py/tiffwrite/__init__.py similarity index 64% rename from tiffwrite/__init__.py rename to py/tiffwrite/__init__.py index ebf2282..bef208d 100644 --- a/tiffwrite/__init__.py +++ b/py/tiffwrite/__init__.py @@ -1,16 +1,22 @@ from __future__ import annotations -import numpy as np -from typing import Any, Self, Sequence +from itertools import product from pathlib import Path +from typing import Any, Sequence + +import numpy as np from numpy.typing import ArrayLike, DTypeLike +from tqdm.auto import tqdm -from . import tiffwrite as rs +from . import tiffwrite_rs as rs # noqa -__all__ = ['Tag', 'IJTiffFile', 'tiffwrite'] +__all__ = ['Header', 'IJTiffFile', 'IFD', 'FrameInfo', 'Tag', 'Strip', 'tiffwrite'] +class Header: + pass + class IFD(dict): pass @@ -19,11 +25,16 @@ class Tag(rs.Tag): pass +Strip = tuple[list[int], list[int]] +CZT = tuple[int, int, int] +FrameInfo = tuple[IFD, Strip, CZT] + + class IJTiffFile(rs.IJTiffFile): def __new__(cls, path: str | Path, shape: tuple[int, int, int], dtype: DTypeLike = 'uint16', colors: Sequence[str] = None, colormap: str = None, pxsize: float = None, deltaz: float = None, timeinterval: float = None, comment: str = None, - **extratags: Tag.Value | Tag) -> None: + **extratags: Tag.Value | Tag) -> IJTiffFile: new = super().__new__(cls, str(path), shape) if colors is not None: new = new.with_colors(colors) @@ -41,14 +52,14 @@ class IJTiffFile(rs.IJTiffFile): new = new.extend_extratags(extratags) return new - def __init__(self, path: str | Path, shape: tuple[int, int, int], dtype: DTypeLike = 'uint16', - colors: Sequence[str] = None, colormap: str = None, pxsize: float = None, - deltaz: float = None, timeinterval: float = None, comment: str = None, - **extratags: Tag.Value | Tag) -> None: + def __init__(self, path: str | Path, shape: tuple[int, int, int], dtype: DTypeLike = 'uint16', # noqa + colors: Sequence[str] = None, colormap: str = None, pxsize: float = None, # noqa + deltaz: float = None, timeinterval: float = None, comment: str = None, # noqa + **extratags: Tag.Value | Tag) -> None: # noqa self.path = Path(path) self.dtype = np.dtype(dtype) - def __enter__(self) -> Self: + def __enter__(self) -> IJTiffFile: return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -81,7 +92,7 @@ class IJTiffFile(rs.IJTiffFile): raise TypeError(f'Cannot save type {self.dtype}') -def tiffwrite(file: str | Path, data: ArrayLike, axes: str = 'TZCXY', dtype: DTypeLike = None, bar: bool = False, +def tiffwrite(file: str | Path, data: np.ndarray, axes: str = 'TZCXY', dtype: DTypeLike = None, bar: bool = False, *args: Any, **kwargs: Any) -> None: """ file: string; filename of the new tiff file data: 2 to 5D numpy array @@ -90,3 +101,21 @@ def tiffwrite(file: str | Path, data: ArrayLike, axes: str = 'TZCXY', dtype: DTy bar: bool; whether to show a progress bar other args: see IJTiffFile """ + + axes = axes[-np.ndim(data):].upper() + if not axes == 'CZTXY': + axes_shuffle = [axes.find(i) for i in 'CZTXY'] + axes_add = [i for i, j in enumerate(axes_shuffle) if j < 0] + axes_shuffle = [i for i in axes_shuffle if i >= 0] + data = np.transpose(data, axes_shuffle) + for axis in axes_add: + data = np.expand_dims(data, axis) + + shape = data.shape[:3] + with IJTiffFile(file, shape, data.dtype if dtype is None else dtype, *args, **kwargs) as f: # type: ignore + at_least_one = False + for n in tqdm(product(*[range(i) for i in shape]), total=np.prod(shape), # type: ignore + desc='Saving tiff', disable=not bar): + if np.any(data[n]) or not at_least_one: + f.save(data[n], *n) + at_least_one = True diff --git a/pyproject.toml b/pyproject.toml index 484a62e..7e13236 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,9 +13,13 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] +[project.optional-dependencies] +test = ["pytest"] + [tool.maturin] +python-source = "py" features = ["pyo3/extension-module"] -module-name = "tiffwrite" +module-name = "tiffwrite.tiffwrite_rs" [tool.isort] line_length = 119 \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 0953369..e900cf5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,19 +6,19 @@ use std::collections::HashMap; use std::fs::{File, OpenOptions}; use std::io::{Read, Seek, SeekFrom, Write}; use anyhow::Result; -use fraction::Fraction; -use num::{Complex, Zero}; -use num::complex::ComplexFloat; +use num::{Complex, Rational32, Zero}; use ndarray::{s, Array2}; use num::traits::ToBytes; use std::hash::{DefaultHasher, Hash, Hasher}; use chrono::Utc; +use zstd::stream::encode_all; +use rayon::prelude::*; const TAG_SIZE: usize = 20; const OFFSET_SIZE: usize = 8; const OFFSET: u64 = 16; -const COMPRESSION: u16 = 1; +const COMPRESSION: u16 = 50000; #[derive(Clone, Debug)] @@ -116,10 +116,10 @@ impl Tag { Tag::new(code, long.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 4) } - pub fn rational(code: u16, rational: Vec) -> Self { + pub fn rational(code: u16, rational: Vec) -> Self { Tag::new(code, rational.into_iter().map(|x| - u32::try_from(*x.denom().unwrap()).unwrap().to_le_bytes().into_iter().chain( - u32::try_from(*x.numer().unwrap()).unwrap().to_le_bytes()).collect::>() + u32::try_from(*x.denom()).unwrap().to_le_bytes().into_iter().chain( + u32::try_from(*x.numer()).unwrap().to_le_bytes()).collect::>() ).flatten().collect(), 5) } @@ -135,10 +135,10 @@ impl Tag { Tag::new(code, slong.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 9) } - pub fn srational(code: u16, srational: Vec) -> Self { + pub fn srational(code: u16, srational: Vec) -> Self { Tag::new(code, srational.into_iter().map(|x| - i32::try_from(*x.denom().unwrap()).unwrap().to_le_bytes().into_iter().chain( - i32::try_from(*x.numer().unwrap()).unwrap().to_le_bytes()).collect::>() + i32::try_from(*x.denom()).unwrap().to_le_bytes().into_iter().chain( + i32::try_from(*x.numer()).unwrap().to_le_bytes()).collect::>() ).flatten().collect(), 10) } @@ -162,7 +162,7 @@ impl Tag { pub fn complex(code: u16, complex: Vec>) -> Self { Tag::new(code, complex.into_iter().map(|x| - x.re().to_le_bytes().into_iter().chain(x.im().to_le_bytes()).collect::>() + x.re.to_le_bytes().into_iter().chain(x.im.to_le_bytes()).collect::>() ).flatten().collect(), 15) } @@ -237,7 +237,7 @@ impl Tag { #[derive(Clone, Debug)] struct Frame { - tilebyteoffsets: Vec, + tileoffsets: Vec, tilebytecounts: Vec, image_width: u32, image_length: u32, @@ -250,11 +250,11 @@ struct Frame { impl Frame { fn new( - tilebyteoffsets: Vec, tilebytecounts: Vec, image_width: u32, image_length: u32, + tileoffsets: Vec, tilebytecounts: Vec, image_width: u32, image_length: u32, bits_per_sample: u16, sample_format: u16, tile_width: u16, tile_length: u16 ) -> Self { Frame { - tilebyteoffsets, tilebytecounts, image_width, image_length, bits_per_sample, + tileoffsets, tilebytecounts, image_width, image_length, bits_per_sample, sample_format, tile_width, tile_length, extra_tags: Vec::new() } } @@ -276,8 +276,7 @@ macro_rules! bytes_impl { const SAMPLE_FORMAT: u16 = $sample_format; #[inline] - fn bytes(&self) -> Vec - { + fn bytes(&self) -> Vec { self.to_le_bytes().to_vec() } } @@ -293,7 +292,6 @@ bytes_impl!(u128, 128, 1); bytes_impl!(usize, 64, 1); #[cfg(target_pointer_width = "32")] bytes_impl!(usize, 32, 1); - bytes_impl!(i8, 8, 2); bytes_impl!(i16, 16, 2); bytes_impl!(i32, 32, 2); @@ -422,30 +420,38 @@ impl IJTiffFile { pub fn save(&mut self, frame: Array2, c: usize, z: usize, t: usize, extra_tags: Option>) -> Result<()> { - self.compress_frame(frame, c, z, t, extra_tags); + self.compress_frame(frame.reversed_axes(), c, z, t, extra_tags)?; Ok(()) } - fn compress_frame(&mut self, frame: Array2, c: usize, z: usize, t: usize, - extra_tags: Option>) { + fn compress_frame(&mut self, frame: Array2, + c: usize, z: usize, t: usize, + extra_tags: Option>) -> Result<()> { let image_width = frame.shape()[0] as u32; let image_length = frame.shape()[1] as u32; - let mut tilebyteoffsets = Vec::new(); + let tile_size = 2usize.pow(((image_width as f64 * image_length as f64 / 64f64).log2() / 2f64).round() as u32).max(16).min(1024); + let mut tileoffsets = Vec::new(); let mut tilebytecounts = Vec::new(); - let tiles = IJTiffFile::tile(frame.reversed_axes(), 64); - for tile in tiles { - let bytes: Vec = tile.map(|x| x.bytes()).into_iter().flatten().collect(); - tilebytecounts.push(bytes.len() as u64); - tilebyteoffsets.push(self.write(&bytes).unwrap()); + let tiles = IJTiffFile::tile(frame.reversed_axes(), tile_size); + let byte_tiles: Vec> = tiles.into_iter().map( + |tile| tile.map(|x| x.bytes()).into_iter().flatten().collect() + ).collect(); + for tile in byte_tiles.into_par_iter().map(|x| encode_all(&*x, 3)).collect::>() { + if let Ok(bytes) = tile { + tilebytecounts.push(bytes.len() as u64); + tileoffsets.push(self.write(&bytes)?); + } } - let mut frame = Frame::new(tilebyteoffsets, tilebytecounts, image_width, image_length, - T::BITS_PER_SAMPLE, T::SAMPLE_FORMAT, 64, 64); + + let mut frame = Frame::new(tileoffsets, tilebytecounts, image_width, image_length, + T::BITS_PER_SAMPLE, T::SAMPLE_FORMAT, tile_size as u16, tile_size as u16); if let Some(tags) = extra_tags { for tag in tags { frame.extra_tags.push(tag); } } self.frames.insert(self.get_frame_number(c, z, t), frame); + Ok(()) } fn tile(frame: Array2, size: usize) -> Vec> { @@ -484,11 +490,11 @@ impl IJTiffFile { tiles } - fn get_colormap(&self, colormap: &Vec) -> Result> { + fn get_colormap(&self, _colormap: &Vec) -> Result> { todo!(); } - fn get_color(&self, colors: (u8, u8, u8)) -> Result> { + fn get_color(&self, _colors: (u8, u8, u8)) -> Result> { todo!(); } @@ -497,12 +503,12 @@ impl IJTiffFile { let mut warn = false; for frame_number in 0..self.n_frames { if let Some(frame) = self.frames.get(&(frame_number, 0)) { - let mut tilebyteoffsets = Vec::new(); + let mut tileoffsets = Vec::new(); let mut tilebytecounts = Vec::new(); let mut frame_count = 0; for channel in 0..self.samples_per_pixel { if let Some(frame_n) = self.frames.get(&(frame_number, channel)) { - tilebyteoffsets.extend(frame_n.tilebyteoffsets.iter()); + tileoffsets.extend(frame_n.tileoffsets.iter()); tilebytecounts.extend(frame_n.tilebytecounts.iter()); frame_count += 1; } else { @@ -519,7 +525,7 @@ impl IJTiffFile { ifd.push_tag(Tag::ascii(305, "tiffwrite_rs")); ifd.push_tag(Tag::short(322, vec![frame.tile_width])); ifd.push_tag(Tag::short(323, vec![frame.tile_length])); - ifd.push_tag(Tag::long8(324, tilebyteoffsets)); + ifd.push_tag(Tag::long8(324, tileoffsets)); ifd.push_tag(Tag::long8(325, tilebytecounts)); ifd.push_tag(Tag::short(339, vec![frame.sample_format])); if frame_number == 0 { diff --git a/src/py.rs b/src/py.rs index e8f92ef..5da17e5 100644 --- a/src/py.rs +++ b/src/py.rs @@ -1,7 +1,6 @@ use pyo3::prelude::*; use crate::{IJTiffFile, Tag}; -use fraction::Fraction; -use num::Complex; +use num::{Complex, Rational32, FromPrimitive}; use numpy::{PyReadonlyArray2, PyArrayMethods}; @@ -36,7 +35,7 @@ impl PyTag { #[staticmethod] fn rational(code: u16, rational: Vec) -> Self { - PyTag { tag: Tag::rational(code, rational.into_iter().map(|x| Fraction::from(x)).collect()) } + PyTag { tag: Tag::rational(code, rational.into_iter().map(|x| Rational32::from_f64(x).unwrap()).collect()) } } #[staticmethod] @@ -56,7 +55,7 @@ impl PyTag { #[staticmethod] fn srational(code: u16, srational: Vec) -> Self { - PyTag { tag: Tag::srational(code, srational.into_iter().map(|x| Fraction::from(x)).collect()) } + PyTag { tag: Tag::srational(code, srational.into_iter().map(|x| Rational32::from_f64(x).unwrap()).collect()) } } #[staticmethod] @@ -98,6 +97,10 @@ impl PyTag { fn ifd8(code: u16, ifd8: Vec) -> Self { PyTag { tag: Tag::ifd8(code, ifd8) } } + + fn count(&self) -> u64 { + self.tag.count() + } } @@ -197,8 +200,10 @@ impl_save!(i64, save_i64); impl_save!(f32, save_f32); impl_save!(f64, save_f64); + #[pymodule] -fn tiffwrite(m: &Bound<'_, PyModule>) -> PyResult<()> { +#[pyo3(name = "tiffwrite_rs")] +fn tiffwrite_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; Ok(()) From 984df9441af0985c3676cc08cc0bfc89a4264c22 Mon Sep 17 00:00:00 2001 From: Wim Pomp Date: Wed, 9 Oct 2024 15:11:23 +0200 Subject: [PATCH 04/13] - add python dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 7e13236..a658afd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] +dependencies = ["numpy", "tqdm"] [project.optional-dependencies] test = ["pytest"] From 7678585bba86635e25635141ce0a8384dd7dc2e3 Mon Sep 17 00:00:00 2001 From: Wim Pomp Date: Wed, 9 Oct 2024 20:30:45 +0200 Subject: [PATCH 05/13] - some more threading --- .gitignore | 1 + src/lib.rs | 112 +++++++++++++++++++++++++++++++++++++---------------- src/py.rs | 12 +----- 3 files changed, 81 insertions(+), 44 deletions(-) diff --git a/.gitignore b/.gitignore index 889ae51..cb58d10 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ /target/ /Cargo.lock /foo.tif +*.so diff --git a/src/lib.rs b/src/lib.rs index e900cf5..758632b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -#[cfg(not(feature = "nopython"))] +// #[cfg(not(feature = "nopython"))] mod py; use std::cmp::Ordering; @@ -10,6 +10,8 @@ use num::{Complex, Rational32, Zero}; use ndarray::{s, Array2}; use num::traits::ToBytes; use std::hash::{DefaultHasher, Hash, Hasher}; +use std::thread; +use std::thread::JoinHandle; use chrono::Utc; use zstd::stream::encode_all; use rayon::prelude::*; @@ -235,6 +237,16 @@ impl Tag { } +struct CompressedFrame { + bytes: Vec>, + image_width: u32, + image_length: u32, + tile_size: usize, + bits_per_sample: u16, + sample_format: u16 +} + + #[derive(Clone, Debug)] struct Frame { tileoffsets: Vec, @@ -245,7 +257,6 @@ struct Frame { sample_format: u16, tile_width: u16, tile_length: u16, - extra_tags: Vec } impl Frame { @@ -255,7 +266,7 @@ impl Frame { ) -> Self { Frame { tileoffsets, tilebytecounts, image_width, image_length, bits_per_sample, - sample_format, tile_width, tile_length, extra_tags: Vec::new() + sample_format, tile_width, tile_length } } } @@ -310,6 +321,7 @@ pub struct IJTiffFile { file: File, frames: HashMap<(usize, u8), Frame>, hashes: HashMap, + threads: HashMap<(usize, u8), JoinHandle>, pub shape: (usize, usize, usize), pub n_frames: usize, pub samples_per_pixel: u8, @@ -318,7 +330,8 @@ pub struct IJTiffFile { pub comment: Option, pub delta_z: Option, pub timeinterval: Option, - pub extra_tags: Option> + pub extra_tags: Vec, + pub extra_tags_frame: HashMap> } impl Drop for IJTiffFile { @@ -344,9 +357,10 @@ impl IJTiffFile { } else { (1, shape.0 * shape.1 * shape.2) }; - Ok(IJTiffFile { file, frames: HashMap::new(), hashes: HashMap::new(), shape, n_frames, + Ok(IJTiffFile { file, frames: HashMap::new(), hashes: HashMap::new(), + threads: HashMap::new(), shape, n_frames, samples_per_pixel: spp, colormap: None, colors: None, comment: None, delta_z: None, - timeinterval: None, extra_tags: None } ) + timeinterval: None, extra_tags: Vec::new(), extra_tags_frame: HashMap::new() } ) } pub fn description(&self) -> String { @@ -418,39 +432,64 @@ impl IJTiffFile { } } - pub fn save(&mut self, frame: Array2, c: usize, z: usize, t: usize, - extra_tags: Option>) -> Result<()> { - self.compress_frame(frame.reversed_axes(), c, z, t, extra_tags)?; + pub fn save(&mut self, frame: Array2, c: usize, z: usize, t: usize, + extra_tags: Option>) -> Result<()> + where T: Bytes + Clone + Send + Sync + Zero + 'static { + let key = self.get_frame_number(c, z, t); + if let Some(extra_tags) = extra_tags { + if let Some(extra_tags_frame) = self.extra_tags_frame.get_mut(&key.0) { + extra_tags_frame.extend(extra_tags); + } else { + self.extra_tags_frame.insert(key.0, extra_tags); + } + } + self.compress_frame(frame.reversed_axes(), key)?; Ok(()) } - fn compress_frame(&mut self, frame: Array2, - c: usize, z: usize, t: usize, - extra_tags: Option>) -> Result<()> { - let image_width = frame.shape()[0] as u32; - let image_length = frame.shape()[1] as u32; - let tile_size = 2usize.pow(((image_width as f64 * image_length as f64 / 64f64).log2() / 2f64).round() as u32).max(16).min(1024); - let mut tileoffsets = Vec::new(); - let mut tilebytecounts = Vec::new(); - let tiles = IJTiffFile::tile(frame.reversed_axes(), tile_size); - let byte_tiles: Vec> = tiles.into_iter().map( - |tile| tile.map(|x| x.bytes()).into_iter().flatten().collect() - ).collect(); - for tile in byte_tiles.into_par_iter().map(|x| encode_all(&*x, 3)).collect::>() { - if let Ok(bytes) = tile { - tilebytecounts.push(bytes.len() as u64); - tileoffsets.push(self.write(&bytes)?); + fn compress_frame(&mut self, frame: Array2, key: (usize, u8)) -> Result<()> + where T: Bytes + Clone + Zero + Send + 'static { + fn compress(frame: Array2) -> CompressedFrame + where T: Bytes + Clone + Zero { + let image_width = frame.shape()[0] as u32; + let image_length = frame.shape()[1] as u32; + let tile_size = 2usize.pow(((image_width as f64 * image_length as f64 / 64f64 + ).log2() / 2f64).round() as u32).max(16).min(1024); + let tiles = IJTiffFile::tile(frame.reversed_axes(), tile_size); + let byte_tiles: Vec> = tiles.into_iter().map( + |tile| tile.map(|x| x.bytes()).into_iter().flatten().collect() + ).collect(); + let bytes = byte_tiles.into_par_iter().map(|x| encode_all(&*x, 3).unwrap()).collect::>(); + CompressedFrame { bytes, image_width, image_length, tile_size, + bits_per_sample: T::BITS_PER_SAMPLE, sample_format: T::SAMPLE_FORMAT } + } + self.threads.insert(key, thread::spawn(move || compress(frame))); + for key in self.threads.keys().cloned().collect::>() { + if self.threads[&key].is_finished() { + } } - let mut frame = Frame::new(tileoffsets, tilebytecounts, image_width, image_length, - T::BITS_PER_SAMPLE, T::SAMPLE_FORMAT, tile_size as u16, tile_size as u16); - if let Some(tags) = extra_tags { - for tag in tags { - frame.extra_tags.push(tag); + for key in self.threads.keys().cloned().collect::>() { + if self.threads[&key].is_finished() { + if let Some(thread) = self.threads.remove(&key) { + self.write_frame(thread.join().unwrap(), key)?; + } } } - self.frames.insert(self.get_frame_number(c, z, t), frame); + Ok(()) + } + + fn write_frame(&mut self, frame: CompressedFrame, key: (usize, u8)) -> Result<()> { + let mut tileoffsets = Vec::new(); + let mut tilebytecounts = Vec::new(); + for tile in frame.bytes { + tilebytecounts.push(tile.len() as u64); + tileoffsets.push(self.write(&tile)?); + } + let frame = Frame::new(tileoffsets, tilebytecounts, frame.image_width, frame.image_length, + frame.bits_per_sample, frame.sample_format, frame.tile_size as u16, frame.tile_size as u16); + self.frames.insert(key, frame); Ok(()) } @@ -499,6 +538,11 @@ impl IJTiffFile { } fn close(&mut self) -> Result<()> { + for key in self.threads.keys().cloned().collect::>() { + if let Some(thread) = self.threads.remove(&key) { + self.write_frame(thread.join().unwrap(), key)?; + } + } let mut where_to_write_next_ifd_offset = OFFSET - OFFSET_SIZE as u64; let mut warn = false; for frame_number in 0..self.n_frames { @@ -547,10 +591,10 @@ impl IJTiffFile { ifd.push_tag(Tag::short(284, vec![2])) } } - ifd.extend_tags(frame.extra_tags.to_owned()); - if let Some(extra_tags) = &self.extra_tags { - ifd.extend_tags(extra_tags.to_owned()); + if self.extra_tags_frame.contains_key(&frame_number) { + ifd.extend_tags(self.extra_tags_frame[&frame_number].to_owned()); } + ifd.extend_tags(self.extra_tags.to_owned()); ifd.push_tag(Tag::ascii(306, &format!("{}", Utc::now().format("%Y:%m:%d %H:%M:%S")))); where_to_write_next_ifd_offset = ifd.write(self, where_to_write_next_ifd_offset)?; } else { diff --git a/src/py.rs b/src/py.rs index 5da17e5..3c91f4b 100644 --- a/src/py.rs +++ b/src/py.rs @@ -144,21 +144,13 @@ impl PyIJTiffFile { fn append_extra_tag(&mut self, tag: PyTag) { if let Some(ijtifffile) = self.ijtifffile.as_mut() { - if let Some(extra_tags) = ijtifffile.extra_tags.as_mut() { - extra_tags.push(tag.tag); - } else { - ijtifffile.extra_tags = Some(vec![tag.tag]); - } + ijtifffile.extra_tags.push(tag.tag); } } fn extend_extra_tags(&mut self, tags: Vec) { if let Some(ijtifffile) = self.ijtifffile.as_mut() { - if let Some(extra_tags) = ijtifffile.extra_tags.as_mut() { - extra_tags.extend(tags.into_iter().map(|x| x.tag)); - } else { - ijtifffile.extra_tags = Some(tags.into_iter().map(|x| x.tag).collect()); - } + ijtifffile.extra_tags.extend(tags.into_iter().map(|x| x.tag)); } } From 1197806a6fd0d070d66eaf73d7e385c34a34d76e Mon Sep 17 00:00:00 2001 From: Wim Pomp Date: Thu, 10 Oct 2024 15:28:14 +0200 Subject: [PATCH 06/13] - start some color(map) support in python - compress_frame function in python for backwards compatibility - save all extra tags in a single hashmap - construct tags from references - store frames by c, z, t - save px_size in tiff - some getters and setters in py.rs --- py/tiffwrite/__init__.py | 83 +++--- pyproject.toml | 2 +- src/lib.rs | 616 ++++++++++++++++++++++++++------------- src/main.rs | 7 +- src/py.rs | 266 +++++++++++++---- 5 files changed, 681 insertions(+), 293 deletions(-) diff --git a/py/tiffwrite/__init__.py b/py/tiffwrite/__init__.py index bef208d..85edd53 100644 --- a/py/tiffwrite/__init__.py +++ b/py/tiffwrite/__init__.py @@ -4,7 +4,9 @@ from itertools import product from pathlib import Path from typing import Any, Sequence +import colorcet import numpy as np +from matplotlib import colors as mpl_colors from numpy.typing import ArrayLike, DTypeLike from tqdm.auto import tqdm @@ -27,29 +29,29 @@ class Tag(rs.Tag): Strip = tuple[list[int], list[int]] CZT = tuple[int, int, int] -FrameInfo = tuple[IFD, Strip, CZT] +FrameInfo = tuple[np.ndarray, None, CZT] class IJTiffFile(rs.IJTiffFile): def __new__(cls, path: str | Path, shape: tuple[int, int, int], dtype: DTypeLike = 'uint16', colors: Sequence[str] = None, colormap: str = None, pxsize: float = None, deltaz: float = None, timeinterval: float = None, comment: str = None, - **extratags: Tag.Value | Tag) -> IJTiffFile: + **extratags: Tag) -> IJTiffFile: new = super().__new__(cls, str(path), shape) if colors is not None: - new = new.with_colors(colors) + new.colors = np.array([get_color(color) for color in colors]) if colormap is not None: - new = new.with_colormap(colormap) + new.colormap = get_colormap(colormap) if pxsize is not None: - new = new.with_pxsize(pxsize) + new.px_size = float(pxsize) if deltaz is not None: - new = new.with_deltaz(deltaz) + new.delta_z = float(deltaz) if timeinterval is not None: - new = new.with_timeinterval(timeinterval) + new.time_interval = float(timeinterval) if comment is not None: - new = new.with_comment(comment) - if extratags: - new = new.extend_extratags(extratags) + new.comment = comment + for extra_tag in extratags: + new.append_extra_tag(extra_tag, None) return new def __init__(self, path: str | Path, shape: tuple[int, int, int], dtype: DTypeLike = 'uint16', # noqa @@ -66,30 +68,43 @@ class IJTiffFile(rs.IJTiffFile): self.close() def save(self, frame: ArrayLike, c: int, z: int, t: int) -> None: - frame = np.asarray(frame).astype(self.dtype) - match self.dtype: - case np.uint8: - self.save_u8(frame, c, z, t) - case np.uint16: - self.save_u16(frame, c, z, t) - case np.uint32: - self.save_u32(frame, c, z, t) - case np.uint64: - self.save_u64(frame, c, z, t) - case np.int8: - self.save_i8(frame, c, z, t) - case np.int16: - self.save_i16(frame, c, z, t) - case np.int32: - self.save_i32(frame, c, z, t) - case np.int64: - self.save_i64(frame, c, z, t) - case np.float32: - self.save_f32(frame, c, z, t) - case np.float64: - self.save_f64(frame, c, z, t) - case _: - raise TypeError(f'Cannot save type {self.dtype}') + for frame, _, (cn, zn, tn) in self.compress_frame(frame): + frame = np.asarray(frame).astype(self.dtype) + match self.dtype: + case np.uint8: + self.save_u8(frame, c + cn, z + zn, t + tn) + case np.uint16: + self.save_u16(frame, c + cn, z + zn, t + tn) + case np.uint32: + self.save_u32(frame, c + cn, z + zn, t + tn) + case np.uint64: + self.save_u64(frame, c + cn, z + zn, t + tn) + case np.int8: + self.save_i8(frame, c + cn, z + zn, t + tn) + case np.int16: + self.save_i16(frame, c + cn, z + zn, t + tn) + case np.int32: + self.save_i32(frame, c + cn, z + zn, t + tn) + case np.int64: + self.save_i64(frame, c + cn, z + zn, t + tn) + case np.float32: + self.save_f32(frame, c + cn, z + zn, t + tn) + case np.float64: + self.save_f64(frame, c + cn, z + zn, t + tn) + case _: + raise TypeError(f'Cannot save type {self.dtype}') + + def compress_frame(self, frame: ArrayLike) -> tuple[FrameInfo]: # noqa + return (frame, None, (0, 0, 0)), + +def get_colormap(colormap: str) -> np.ndarray: + colormap = getattr(colorcet, colormap) + colormap[0] = '#ffffff' + colormap[-1] = '#000000' + return np.array([[int(''.join(i), 16) for i in zip(*[iter(s[1:])] * 2)] for s in colormap]).astype('uint8') + +def get_color(color: str) -> np.ndarray: + return np.array([int(''.join(i), 16) for i in zip(*[iter(mpl_colors.to_hex(color)[1:])] * 2)]).astype('uint8') def tiffwrite(file: str | Path, data: np.ndarray, axes: str = 'TZCXY', dtype: DTypeLike = None, bar: bool = False, diff --git a/pyproject.toml b/pyproject.toml index a658afd..0078504 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] -dependencies = ["numpy", "tqdm"] +dependencies = ["colorcet", "matplotlib", "numpy", "tqdm"] [project.optional-dependencies] test = ["pytest"] diff --git a/src/lib.rs b/src/lib.rs index 758632b..580fbcf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,31 +1,29 @@ // #[cfg(not(feature = "nopython"))] mod py; +use anyhow::Result; +use chrono::Utc; +use ndarray::{s, Array2}; +use num::traits::ToBytes; +use num::{Complex, FromPrimitive, Rational32, Zero}; +use rayon::prelude::*; use std::cmp::Ordering; use std::collections::HashMap; use std::fs::{File, OpenOptions}; -use std::io::{Read, Seek, SeekFrom, Write}; -use anyhow::Result; -use num::{Complex, Rational32, Zero}; -use ndarray::{s, Array2}; -use num::traits::ToBytes; use std::hash::{DefaultHasher, Hash, Hasher}; +use std::io::{Read, Seek, SeekFrom, Write}; use std::thread; use std::thread::JoinHandle; -use chrono::Utc; use zstd::stream::encode_all; -use rayon::prelude::*; - const TAG_SIZE: usize = 20; const OFFSET_SIZE: usize = 8; const OFFSET: u64 = 16; const COMPRESSION: u16 = 50000; - #[derive(Clone, Debug)] struct IFD { - tags: Vec + tags: Vec, } impl IFD { @@ -39,12 +37,6 @@ impl IFD { } } - fn extend_tags(&mut self, tags: Vec) { - for tag in tags { - self.push_tag(tag); - } - } - fn write(&mut self, ijtifffile: &mut IJTiffFile, where_to_write_offset: u64) -> Result { self.tags.sort(); ijtifffile.file.seek(SeekFrom::End(0))?; @@ -52,7 +44,9 @@ impl IFD { ijtifffile.file.write(&[0])?; } let offset = ijtifffile.file.stream_position()?; - ijtifffile.file.write(&(self.tags.len() as u64).to_le_bytes())?; + ijtifffile + .file + .write(&(self.tags.len() as u64).to_le_bytes())?; for tag in self.tags.iter_mut() { tag.write_tag(ijtifffile)?; @@ -62,19 +56,20 @@ impl IFD { for tag in self.tags.iter() { tag.write_data(ijtifffile)?; } - ijtifffile.file.seek(SeekFrom::Start(where_to_write_offset))?; + ijtifffile + .file + .seek(SeekFrom::Start(where_to_write_offset))?; ijtifffile.file.write(&offset.to_le_bytes())?; Ok(where_to_write_next_ifd_offset) } } - #[derive(Clone, Debug, Eq)] pub struct Tag { code: u16, bytes: Vec, ttype: u16, - offset: u64 + offset: u64, } impl PartialOrd for Tag { @@ -97,11 +92,16 @@ impl PartialEq for Tag { impl Tag { pub fn new(code: u16, bytes: Vec, ttype: u16) -> Self { - Tag { code, bytes, ttype, offset: 0 } + Tag { + code, + bytes, + ttype, + offset: 0, + } } - pub fn byte(code: u16, byte: Vec) -> Self { - Tag::new(code, byte, 1) + pub fn byte(code: u16, byte: &Vec) -> Self { + Tag::new(code, byte.to_owned(), 1) } pub fn ascii(code: u16, ascii: &str) -> Self { @@ -110,96 +110,213 @@ impl Tag { Tag::new(code, bytes, 2) } - pub fn short(code: u16, short: Vec) -> Self { - Tag::new(code, short.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 3) + pub fn short(code: u16, short: &Vec) -> Self { + Tag::new( + code, + short + .into_iter() + .map(|x| x.to_le_bytes()) + .flatten() + .collect(), + 3, + ) } - pub fn long(code: u16, long: Vec) -> Self { - Tag::new(code, long.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 4) + pub fn long(code: u16, long: &Vec) -> Self { + Tag::new( + code, + long.into_iter() + .map(|x| x.to_le_bytes()) + .flatten() + .collect(), + 4, + ) } - pub fn rational(code: u16, rational: Vec) -> Self { - Tag::new(code, rational.into_iter().map(|x| - u32::try_from(*x.denom()).unwrap().to_le_bytes().into_iter().chain( - u32::try_from(*x.numer()).unwrap().to_le_bytes()).collect::>() - ).flatten().collect(), 5) + pub fn rational(code: u16, rational: &Vec) -> Self { + Tag::new( + code, + rational + .into_iter() + .map(|x| { + u32::try_from(*x.denom()) + .unwrap() + .to_le_bytes() + .into_iter() + .chain(u32::try_from(*x.numer()).unwrap().to_le_bytes()) + .collect::>() + }) + .flatten() + .collect(), + 5, + ) } - pub fn sbyte(code: u16, sbyte: Vec) -> Self { - Tag::new(code, sbyte.iter().map(|x| x.to_le_bytes()).flatten().collect(), 6) + pub fn sbyte(code: u16, sbyte: &Vec) -> Self { + Tag::new( + code, + sbyte.iter().map(|x| x.to_le_bytes()).flatten().collect(), + 6, + ) } - pub fn sshort(code: u16, sshort: Vec) -> Self { - Tag::new(code, sshort.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 8) + pub fn sshort(code: u16, sshort: &Vec) -> Self { + Tag::new( + code, + sshort + .into_iter() + .map(|x| x.to_le_bytes()) + .flatten() + .collect(), + 8, + ) } - pub fn slong(code: u16, slong: Vec) -> Self { - Tag::new(code, slong.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 9) + pub fn slong(code: u16, slong: &Vec) -> Self { + Tag::new( + code, + slong + .into_iter() + .map(|x| x.to_le_bytes()) + .flatten() + .collect(), + 9, + ) } - pub fn srational(code: u16, srational: Vec) -> Self { - Tag::new(code, srational.into_iter().map(|x| - i32::try_from(*x.denom()).unwrap().to_le_bytes().into_iter().chain( - i32::try_from(*x.numer()).unwrap().to_le_bytes()).collect::>() - ).flatten().collect(), 10) + pub fn srational(code: u16, srational: &Vec) -> Self { + Tag::new( + code, + srational + .into_iter() + .map(|x| { + i32::try_from(*x.denom()) + .unwrap() + .to_le_bytes() + .into_iter() + .chain(i32::try_from(*x.numer()).unwrap().to_le_bytes()) + .collect::>() + }) + .flatten() + .collect(), + 10, + ) } - pub fn float(code: u16, float: Vec) -> Self { - Tag::new(code, float.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 11) + pub fn float(code: u16, float: &Vec) -> Self { + Tag::new( + code, + float + .into_iter() + .map(|x| x.to_le_bytes()) + .flatten() + .collect(), + 11, + ) } - pub fn double(code: u16, double: Vec) -> Self { - Tag::new(code, double.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 12) + pub fn double(code: u16, double: &Vec) -> Self { + Tag::new( + code, + double + .into_iter() + .map(|x| x.to_le_bytes()) + .flatten() + .collect(), + 12, + ) } - pub fn ifd(code: u16, ifd: Vec) -> Self { - Tag::new(code, ifd.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 13) + pub fn ifd(code: u16, ifd: &Vec) -> Self { + Tag::new( + code, + ifd.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), + 13, + ) } pub fn unicode(code: u16, unicode: &str) -> Self { - let mut bytes: Vec = unicode.encode_utf16().map(|x| x.to_le_bytes()).flatten().collect(); + let mut bytes: Vec = unicode + .encode_utf16() + .map(|x| x.to_le_bytes()) + .flatten() + .collect(); bytes.push(0); Tag::new(code, bytes, 14) } - pub fn complex(code: u16, complex: Vec>) -> Self { - Tag::new(code, complex.into_iter().map(|x| - x.re.to_le_bytes().into_iter().chain(x.im.to_le_bytes()).collect::>() - ).flatten().collect(), 15) + pub fn complex(code: u16, complex: &Vec>) -> Self { + Tag::new( + code, + complex + .into_iter() + .map(|x| { + x.re.to_le_bytes() + .into_iter() + .chain(x.im.to_le_bytes()) + .collect::>() + }) + .flatten() + .collect(), + 15, + ) } - pub fn long8(code: u16, long8: Vec) -> Self { - Tag::new(code, long8.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 16) + pub fn long8(code: u16, long8: &Vec) -> Self { + Tag::new( + code, + long8 + .into_iter() + .map(|x| x.to_le_bytes()) + .flatten() + .collect(), + 16, + ) } - pub fn slong8(code: u16, slong8: Vec) -> Self { - Tag::new(code, slong8.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 17) + pub fn slong8(code: u16, slong8: &Vec) -> Self { + Tag::new( + code, + slong8 + .into_iter() + .map(|x| x.to_le_bytes()) + .flatten() + .collect(), + 17, + ) } - pub fn ifd8(code: u16, ifd8: Vec) -> Self { - Tag::new(code, ifd8.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 18) + pub fn ifd8(code: u16, ifd8: &Vec) -> Self { + Tag::new( + code, + ifd8.into_iter() + .map(|x| x.to_le_bytes()) + .flatten() + .collect(), + 18, + ) } pub fn count(&self) -> u64 { let c = match self.ttype { - 1 => self.bytes.len(), // BYTE - 2 => self.bytes.len(), // ASCII + 1 => self.bytes.len(), // BYTE + 2 => self.bytes.len(), // ASCII 3 => self.bytes.len() / 2, // SHORT 4 => self.bytes.len() / 4, // LONG 5 => self.bytes.len() / 8, // RATIONAL - 6 => self.bytes.len(), // SBYTE - 7 => self.bytes.len(), // UNDEFINED + 6 => self.bytes.len(), // SBYTE + 7 => self.bytes.len(), // UNDEFINED 8 => self.bytes.len() / 2, // SSHORT 9 => self.bytes.len() / 4, // SLONG - 10 => self.bytes.len() / 8, // SRATIONAL - 11 => self.bytes.len() / 4, // FLOAT - 12 => self.bytes.len() / 8, // DOUBLE - 13 => self.bytes.len() / 4, // IFD - 14 => self.bytes.len() / 2, // UNICODE - 15 => self.bytes.len() / 8, // COMPLEX - 16 => self.bytes.len() / 8, // LONG8 - 17 => self.bytes.len() / 8, // SLONG8 - 18 => self.bytes.len() / 8, // IFD8 + 10 => self.bytes.len() / 8, // SRATIONAL + 11 => self.bytes.len() / 4, // FLOAT + 12 => self.bytes.len() / 8, // DOUBLE + 13 => self.bytes.len() / 4, // IFD + 14 => self.bytes.len() / 2, // UNICODE + 15 => self.bytes.len() / 8, // COMPLEX + 16 => self.bytes.len() / 8, // LONG8 + 17 => self.bytes.len() / 8, // SLONG8 + 18 => self.bytes.len() / 8, // IFD8 _ => self.bytes.len(), }; c as u64 @@ -226,7 +343,8 @@ impl Tag { ijtifffile.file.seek(SeekFrom::End(0))?; let offset = ijtifffile.write(&self.bytes)?; ijtifffile.file.seek(SeekFrom::Start( - self.offset + (TAG_SIZE - OFFSET_SIZE) as u64))?; + self.offset + (TAG_SIZE - OFFSET_SIZE) as u64, + ))?; ijtifffile.file.write(&offset.to_le_bytes())?; if ijtifffile.file.stream_position()? % 2 == 1 { ijtifffile.file.write(&[0u8])?; @@ -236,17 +354,16 @@ impl Tag { } } - +#[derive(Debug)] struct CompressedFrame { bytes: Vec>, image_width: u32, image_length: u32, tile_size: usize, bits_per_sample: u16, - sample_format: u16 + sample_format: u16, } - #[derive(Clone, Debug)] struct Frame { tileoffsets: Vec, @@ -261,17 +378,28 @@ struct Frame { impl Frame { fn new( - tileoffsets: Vec, tilebytecounts: Vec, image_width: u32, image_length: u32, - bits_per_sample: u16, sample_format: u16, tile_width: u16, tile_length: u16 + tileoffsets: Vec, + tilebytecounts: Vec, + image_width: u32, + image_length: u32, + bits_per_sample: u16, + sample_format: u16, + tile_width: u16, + tile_length: u16, ) -> Self { Frame { - tileoffsets, tilebytecounts, image_width, image_length, bits_per_sample, - sample_format, tile_width, tile_length + tileoffsets, + tilebytecounts, + image_width, + image_length, + bits_per_sample, + sample_format, + tile_width, + tile_length, } } } - pub trait Bytes { const BITS_PER_SAMPLE: u16; const SAMPLE_FORMAT: u16; @@ -279,7 +407,6 @@ pub trait Bytes { fn bytes(&self) -> Vec; } - macro_rules! bytes_impl { ($T:ty, $bits_per_sample:expr, $sample_format:expr) => { impl Bytes for $T { @@ -315,23 +442,26 @@ bytes_impl!(isize, 32, 2); bytes_impl!(f32, 32, 3); bytes_impl!(f64, 64, 3); +#[derive(Clone, Debug)] +pub enum Colors { + None, + Colors(Vec>), + Colormap(Vec>), +} #[derive(Debug)] pub struct IJTiffFile { file: File, - frames: HashMap<(usize, u8), Frame>, + frames: HashMap<(usize, usize, usize), Frame>, hashes: HashMap, - threads: HashMap<(usize, u8), JoinHandle>, + threads: HashMap<(usize, usize, usize), JoinHandle>, pub shape: (usize, usize, usize), - pub n_frames: usize, - pub samples_per_pixel: u8, - pub colormap: Option>, - pub colors: Option>, + pub colors: Colors, pub comment: Option, + pub px_size: Option, pub delta_z: Option, - pub timeinterval: Option, - pub extra_tags: Vec, - pub extra_tags_frame: HashMap> + pub time_interval: Option, + pub extra_tags: HashMap, Vec>, } impl Drop for IJTiffFile { @@ -344,28 +474,35 @@ impl Drop for IJTiffFile { impl IJTiffFile { pub fn new(path: &str, shape: (usize, usize, usize)) -> Result { - let mut file = OpenOptions::new().create(true).truncate(true) - .write(true).read(true).open(path)?; + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .read(true) + .open(path)?; file.write(b"II")?; file.write(&43u16.to_le_bytes())?; file.write(&8u16.to_le_bytes())?; file.write(&0u16.to_le_bytes())?; file.write(&OFFSET.to_le_bytes())?; - let colormap: Option> = None; - let (spp, n_frames) = if let None = &colormap { - (shape.0 as u8, shape.1 * shape.2) - } else { - (1, shape.0 * shape.1 * shape.2) - }; - Ok(IJTiffFile { file, frames: HashMap::new(), hashes: HashMap::new(), - threads: HashMap::new(), shape, n_frames, - samples_per_pixel: spp, colormap: None, colors: None, comment: None, delta_z: None, - timeinterval: None, extra_tags: Vec::new(), extra_tags_frame: HashMap::new() } ) + Ok(IJTiffFile { + file, + frames: HashMap::new(), + hashes: HashMap::new(), + threads: HashMap::new(), + shape, + colors: Colors::None, + comment: None, + px_size: None, + delta_z: None, + time_interval: None, + extra_tags: HashMap::new(), + }) } pub fn description(&self) -> String { let mut desc: String = String::from("ImageJ=1.11a"); - if let (None, None) = (self.colormap.as_ref(), self.colors.as_ref()) { + if let Colors::None = self.colors { desc += &format!("\nimages={}", self.shape.0); desc += &format!("\nslices={}", self.shape.1); desc += &format!("\nframes={}", self.shape.2); @@ -384,7 +521,7 @@ impl IJTiffFile { if let Some(delta_z) = self.delta_z { desc += &format!("\nspacing={}", delta_z); } - if let Some(timeinterval) = self.timeinterval { + if let Some(timeinterval) = self.time_interval { desc += &format!("\ninterval={}", timeinterval); } if let Some(comment) = &self.comment { @@ -393,11 +530,27 @@ impl IJTiffFile { desc } - fn get_frame_number(&self, c: usize, z: usize, t: usize) -> (usize, u8) { - if let (None, None) = (self.colormap.as_ref(), self.colors.as_ref()) { - (z + t * self.shape.1, c as u8) + fn get_czt(&self, frame_number: usize, channel: u8) -> (usize, usize, usize) { + if let Colors::None = self.colors { + ( + channel as usize, + frame_number % self.shape.1, + frame_number / self.shape.1, + ) } else { - (c + z * self.shape.0 + t * self.shape.0 * self.shape.1, 0) + ( + frame_number % self.shape.0, + frame_number / self.shape.0 % self.shape.1, + frame_number / self.shape.0 / self.shape.1, + ) + } + } + + fn spp_and_n_frames(&self) -> (u8, usize) { + if let Colors::None = &self.colors { + (self.shape.0 as u8, self.shape.1 * self.shape.2) + } else { + (1, self.shape.0 * self.shape.1 * self.shape.2) } } @@ -419,7 +572,9 @@ impl IJTiffFile { fn write(&mut self, bytes: &Vec) -> Result { let hash = IJTiffFile::hash(&bytes); - if self.hashes.contains_key(&hash) && self.hash_check(&bytes, *self.hashes.get(&hash).unwrap())? { + if self.hashes.contains_key(&hash) + && self.hash_check(&bytes, *self.hashes.get(&hash).unwrap())? + { Ok(*self.hashes.get(&hash).unwrap()) } else { if self.file.stream_position()? % 2 == 1 { @@ -432,64 +587,88 @@ impl IJTiffFile { } } - pub fn save(&mut self, frame: Array2, c: usize, z: usize, t: usize, - extra_tags: Option>) -> Result<()> - where T: Bytes + Clone + Send + Sync + Zero + 'static { - let key = self.get_frame_number(c, z, t); - if let Some(extra_tags) = extra_tags { - if let Some(extra_tags_frame) = self.extra_tags_frame.get_mut(&key.0) { - extra_tags_frame.extend(extra_tags); - } else { - self.extra_tags_frame.insert(key.0, extra_tags); - } - } - self.compress_frame(frame.reversed_axes(), key)?; + pub fn save(&mut self, frame: Array2, c: usize, z: usize, t: usize) -> Result<()> + where + T: Bytes + Clone + Send + Sync + Zero + 'static, + { + self.compress_frame(frame.reversed_axes(), c, z, t)?; Ok(()) } - fn compress_frame(&mut self, frame: Array2, key: (usize, u8)) -> Result<()> - where T: Bytes + Clone + Zero + Send + 'static { + fn compress_frame(&mut self, frame: Array2, c: usize, z: usize, t: usize) -> Result<()> + where + T: Bytes + Clone + Zero + Send + 'static, + { fn compress(frame: Array2) -> CompressedFrame - where T: Bytes + Clone + Zero { + where + T: Bytes + Clone + Zero, + { let image_width = frame.shape()[0] as u32; let image_length = frame.shape()[1] as u32; - let tile_size = 2usize.pow(((image_width as f64 * image_length as f64 / 64f64 - ).log2() / 2f64).round() as u32).max(16).min(1024); + let tile_size = 2usize + .pow( + ((image_width as f64 * image_length as f64 / 64f64).log2() / 2f64).round() + as u32, + ) + .max(16) + .min(1024); let tiles = IJTiffFile::tile(frame.reversed_axes(), tile_size); - let byte_tiles: Vec> = tiles.into_iter().map( - |tile| tile.map(|x| x.bytes()).into_iter().flatten().collect() - ).collect(); - let bytes = byte_tiles.into_par_iter().map(|x| encode_all(&*x, 3).unwrap()).collect::>(); - CompressedFrame { bytes, image_width, image_length, tile_size, - bits_per_sample: T::BITS_PER_SAMPLE, sample_format: T::SAMPLE_FORMAT } - } - self.threads.insert(key, thread::spawn(move || compress(frame))); - for key in self.threads.keys().cloned().collect::>() { - if self.threads[&key].is_finished() { - + let byte_tiles: Vec> = tiles + .into_iter() + .map(|tile| tile.map(|x| x.bytes()).into_iter().flatten().collect()) + .collect(); + let bytes = byte_tiles + .into_par_iter() + .map(|x| encode_all(&*x, 3).unwrap()) + .collect::>(); + CompressedFrame { + bytes, + image_width, + image_length, + tile_size, + bits_per_sample: T::BITS_PER_SAMPLE, + sample_format: T::SAMPLE_FORMAT, } } + self.threads + .insert((c, z, t), thread::spawn(move || compress(frame))); + for key in self + .threads + .keys() + .cloned() + .collect::>() + { + if self.threads[&key].is_finished() {} + } - for key in self.threads.keys().cloned().collect::>() { - if self.threads[&key].is_finished() { - if let Some(thread) = self.threads.remove(&key) { - self.write_frame(thread.join().unwrap(), key)?; + for (c, z, t) in self.threads.keys().cloned().collect::>() { + if self.threads[&(c, z, t)].is_finished() { + if let Some(thread) = self.threads.remove(&(c, z, t)) { + self.write_frame(thread.join().unwrap(), c, z, t)?; } } } - Ok(()) + Ok(()) } - fn write_frame(&mut self, frame: CompressedFrame, key: (usize, u8)) -> Result<()> { + fn write_frame(&mut self, frame: CompressedFrame, c: usize, z: usize, t: usize) -> Result<()> { let mut tileoffsets = Vec::new(); let mut tilebytecounts = Vec::new(); for tile in frame.bytes { tilebytecounts.push(tile.len() as u64); tileoffsets.push(self.write(&tile)?); } - let frame = Frame::new(tileoffsets, tilebytecounts, frame.image_width, frame.image_length, - frame.bits_per_sample, frame.sample_format, frame.tile_size as u16, frame.tile_size as u16); - self.frames.insert(key, frame); + let frame = Frame::new( + tileoffsets, + tilebytecounts, + frame.image_width, + frame.image_length, + frame.bits_per_sample, + frame.sample_format, + frame.tile_size as u16, + frame.tile_size as u16, + ); + self.frames.insert((c, z, t), frame); Ok(()) } @@ -499,59 +678,74 @@ impl IJTiffFile { let (n, m) = (shape[0] / size, shape[1] / size); for i in 0..n { for j in 0..m { - tiles.push(frame.slice( - s![i * size..(i + 1) * size, j * size..(j + 1) * size]).to_owned()); + tiles.push( + frame + .slice(s![i * size..(i + 1) * size, j * size..(j + 1) * size]) + .to_owned(), + ); } if shape[1] % size != 0 { let mut tile = Array2::::zeros((size, size)); - tile.slice_mut( - s![.., ..shape[1] - m * size] - ).assign(&frame.slice(s![i * size..(i + 1) * size, m * size..])); + tile.slice_mut(s![.., ..shape[1] - m * size]) + .assign(&frame.slice(s![i * size..(i + 1) * size, m * size..])); tiles.push(tile); } } if shape[0] % size != 0 { for j in 0..m { let mut tile = Array2::::zeros((size, size)); - tile.slice_mut( - s![..shape[0] - n * size, ..] - ).assign(&frame.slice(s![n * size.., j * size..(j + 1) * size])); + tile.slice_mut(s![..shape[0] - n * size, ..]) + .assign(&frame.slice(s![n * size.., j * size..(j + 1) * size])); tiles.push(tile); } if shape[1] % size != 0 { let mut tile = Array2::::zeros((size, size)); - tile.slice_mut( - s![..shape[0] - n * size, ..shape[1] - m * size] - ).assign(&frame.slice(s![n * size.., m * size..])); + tile.slice_mut(s![..shape[0] - n * size, ..shape[1] - m * size]) + .assign(&frame.slice(s![n * size.., m * size..])); tiles.push(tile); } } tiles } - fn get_colormap(&self, _colormap: &Vec) -> Result> { - todo!(); + fn get_colormap(&self, colormap: &Vec>, bits_per_sample: u16) -> Vec { + if bits_per_sample == 8 { + colormap + .iter() + .flatten() + .map(|x| (*x as u16) * 256) + .collect() + } else { + colormap + .iter() + .map(|x| vec![x; 256]) + .flatten() + .flatten() + .map(|x| (*x as u16) * 256) + .collect() + } } - fn get_color(&self, _colors: (u8, u8, u8)) -> Result> { + fn get_color(&self, _colors: &Vec, _bits_per_sample: u16) -> Result> { todo!(); } fn close(&mut self) -> Result<()> { - for key in self.threads.keys().cloned().collect::>() { - if let Some(thread) = self.threads.remove(&key) { - self.write_frame(thread.join().unwrap(), key)?; + for (c, z, t) in self.threads.keys().cloned().collect::>() { + if let Some(thread) = self.threads.remove(&(c, z, t)) { + self.write_frame(thread.join().unwrap(), c, z, t)?; } } let mut where_to_write_next_ifd_offset = OFFSET - OFFSET_SIZE as u64; let mut warn = false; - for frame_number in 0..self.n_frames { - if let Some(frame) = self.frames.get(&(frame_number, 0)) { + let (samples_per_pixel, n_frames) = self.spp_and_n_frames(); + for frame_number in 0..n_frames { + if let Some(frame) = self.frames.get(&self.get_czt(frame_number, 0)) { let mut tileoffsets = Vec::new(); let mut tilebytecounts = Vec::new(); let mut frame_count = 0; - for channel in 0..self.samples_per_pixel { - if let Some(frame_n) = self.frames.get(&(frame_number, channel)) { + for channel in 0..samples_per_pixel { + if let Some(frame_n) = self.frames.get(&self.get_czt(frame_number, channel)) { tileoffsets.extend(frame_n.tileoffsets.iter()); tilebytecounts.extend(frame_n.tilebytecounts.iter()); frame_count += 1; @@ -560,52 +754,80 @@ impl IJTiffFile { } } let mut ifd = IFD::new(); - ifd.push_tag(Tag::long(256, vec![frame.image_width])); - ifd.push_tag(Tag::long(257, vec![frame.image_length])); - ifd.push_tag(Tag::short(258, vec![frame.bits_per_sample; frame_count])); - ifd.push_tag(Tag::short(259, vec![COMPRESSION])); + ifd.push_tag(Tag::long(256, &vec![frame.image_width])); + ifd.push_tag(Tag::long(257, &vec![frame.image_length])); + ifd.push_tag(Tag::short(258, &vec![frame.bits_per_sample; frame_count])); + ifd.push_tag(Tag::short(259, &vec![COMPRESSION])); ifd.push_tag(Tag::ascii(270, &self.description())); - ifd.push_tag(Tag::short(277, vec![frame_count as u16])); + ifd.push_tag(Tag::short(277, &vec![frame_count as u16])); ifd.push_tag(Tag::ascii(305, "tiffwrite_rs")); - ifd.push_tag(Tag::short(322, vec![frame.tile_width])); - ifd.push_tag(Tag::short(323, vec![frame.tile_length])); - ifd.push_tag(Tag::long8(324, tileoffsets)); - ifd.push_tag(Tag::long8(325, tilebytecounts)); - ifd.push_tag(Tag::short(339, vec![frame.sample_format])); + ifd.push_tag(Tag::short(322, &vec![frame.tile_width])); + ifd.push_tag(Tag::short(323, &vec![frame.tile_length])); + ifd.push_tag(Tag::long8(324, &tileoffsets)); + ifd.push_tag(Tag::long8(325, &tilebytecounts)); + ifd.push_tag(Tag::short(339, &vec![frame.sample_format])); + if let Some(px_size) = self.px_size { + let r = vec![Rational32::from_f64(px_size).unwrap()]; + ifd.push_tag(Tag::rational(282, &r)); + ifd.push_tag(Tag::rational(283, &r)); + ifd.push_tag(Tag::short(296, &vec![1])); + } + if frame_number == 0 { - if let Some(colormap) = &self.colormap { - ifd.push_tag(Tag::short(320, self.get_colormap(colormap)?)); - ifd.push_tag(Tag::short(262, vec![3])); // PhotometricInterpretation PHOTOMETRIC_PALETTE - } else if let None = self.colors { - ifd.push_tag(Tag::short(262, vec![1])); // PhotometricInterpretation PHOTOMETRIC_PALETTE + if let Colors::Colormap(colormap) = &self.colors { + ifd.push_tag(Tag::short( + 320, + &self.get_colormap(colormap, frame.bits_per_sample), + )); + ifd.push_tag(Tag::short(262, &vec![3])); + } else if let Colors::None = self.colors { + ifd.push_tag(Tag::short(262, &vec![1])); } } - if frame_number < self.samples_per_pixel as usize { - if let Some(color) = &self.colors { - ifd.push_tag(Tag::short(320, self.get_color(color[frame_number])?)); - ifd.push_tag(Tag::short(262, vec![3])); // PhotometricInterpretation PHOTOMETRIC_PALETTE + if frame_number < samples_per_pixel as usize { + if let Colors::Colors(colors) = &self.colors { + ifd.push_tag(Tag::short( + 320, + &self.get_color(&colors[frame_number], frame.bits_per_sample)?, + )); + ifd.push_tag(Tag::short(262, &vec![3])); } } - if let (None, None) = (&self.colormap, &self.colors) { + if let Colors::None = &self.colors { if self.shape.0 > 1 { - ifd.push_tag(Tag::short(284, vec![2])) + ifd.push_tag(Tag::short(284, &vec![2])) } } - if self.extra_tags_frame.contains_key(&frame_number) { - ifd.extend_tags(self.extra_tags_frame[&frame_number].to_owned()); + for channel in 0..samples_per_pixel { + let czt = self.get_czt(frame_number, channel); + if let Some(extra_tags) = self.extra_tags.get(&Some(czt)) { + for tag in extra_tags { + ifd.push_tag(tag.to_owned()) + } + } } - ifd.extend_tags(self.extra_tags.to_owned()); - ifd.push_tag(Tag::ascii(306, &format!("{}", Utc::now().format("%Y:%m:%d %H:%M:%S")))); + if let Some(extra_tags) = self.extra_tags.get(&None) { + for tag in extra_tags { + ifd.push_tag(tag.to_owned()) + } + } + ifd.push_tag(Tag::ascii( + 306, + &format!("{}", Utc::now().format("%Y:%m:%d %H:%M:%S")), + )); where_to_write_next_ifd_offset = ifd.write(self, where_to_write_next_ifd_offset)?; } else { warn = true; } if warn { - println!("Some frames were not added to the tif file, either you forgot them, \ - or an error occurred and the tif file was closed prematurely.") + println!( + "Some frames were not added to the tif file, either you forgot them, \ + or an error occurred and the tif file was closed prematurely." + ) } } - self.file.seek(SeekFrom::Start(where_to_write_next_ifd_offset))?; + self.file + .seek(SeekFrom::Start(where_to_write_next_ifd_offset))?; self.file.write(&0u64.to_le_bytes())?; Ok(()) } diff --git a/src/main.rs b/src/main.rs index 658eac3..75a03da 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,7 +2,6 @@ use anyhow::Result; use ndarray::{s, Array2}; use tiffwrite::IJTiffFile; - fn main() -> Result<()> { println!("Hello World!"); let mut f = IJTiffFile::new("foo.tif", (2, 1, 1))?; @@ -12,12 +11,12 @@ fn main() -> Result<()> { arr[[i, j]] = i as u16; } } - f.save(arr.to_owned(), 0, 0, 0, None)?; + f.save(arr.to_owned(), 0, 0, 0)?; let mut arr = Array2::::zeros((100, 100)); arr.slice_mut(s![64.., ..64]).fill(1); arr.slice_mut(s![..64, 64..]).fill(2); arr.slice_mut(s![64.., 64..]).fill(3); - f.save(arr.to_owned(), 1, 0,0, None)?; + f.save(arr.to_owned(), 1, 0, 0)?; Ok(()) -} \ No newline at end of file +} diff --git a/src/py.rs b/src/py.rs index 3c91f4b..693b338 100644 --- a/src/py.rs +++ b/src/py.rs @@ -1,101 +1,153 @@ +use crate::{Colors, IJTiffFile, Tag}; +use ndarray::s; +use num::{Complex, FromPrimitive, Rational32}; +use numpy::{PyArrayMethods, PyReadonlyArray2}; use pyo3::prelude::*; -use crate::{IJTiffFile, Tag}; -use num::{Complex, Rational32, FromPrimitive}; -use numpy::{PyReadonlyArray2, PyArrayMethods}; - #[pyclass(subclass)] #[pyo3(name = "Tag")] #[derive(Clone, Debug)] struct PyTag { - tag: Tag + tag: Tag, } #[pymethods] impl PyTag { #[staticmethod] fn byte(code: u16, byte: Vec) -> Self { - PyTag { tag: Tag::byte(code, byte) } + PyTag { + tag: Tag::byte(code, &byte), + } } #[staticmethod] fn ascii(code: u16, ascii: &str) -> Self { - PyTag { tag: Tag::ascii(code, ascii) } + PyTag { + tag: Tag::ascii(code, ascii), + } } #[staticmethod] fn short(code: u16, short: Vec) -> Self { - PyTag { tag: Tag::short(code, short) } + PyTag { + tag: Tag::short(code, &short), + } } #[staticmethod] fn long(code: u16, long: Vec) -> Self { - PyTag { tag: Tag::long(code, long) } + PyTag { + tag: Tag::long(code, &long), + } } #[staticmethod] fn rational(code: u16, rational: Vec) -> Self { - PyTag { tag: Tag::rational(code, rational.into_iter().map(|x| Rational32::from_f64(x).unwrap()).collect()) } + PyTag { + tag: Tag::rational( + code, + &rational + .into_iter() + .map(|x| Rational32::from_f64(x).unwrap()) + .collect(), + ), + } } #[staticmethod] fn sbyte(code: u16, sbyte: Vec) -> Self { - PyTag { tag: Tag::sbyte(code, sbyte) } + PyTag { + tag: Tag::sbyte(code, &sbyte), + } } #[staticmethod] fn sshort(code: u16, sshort: Vec) -> Self { - PyTag { tag: Tag::sshort(code, sshort) } + PyTag { + tag: Tag::sshort(code, &sshort), + } } #[staticmethod] fn slong(code: u16, slong: Vec) -> Self { - PyTag { tag: Tag::slong(code, slong) } + PyTag { + tag: Tag::slong(code, &slong), + } } #[staticmethod] fn srational(code: u16, srational: Vec) -> Self { - PyTag { tag: Tag::srational(code, srational.into_iter().map(|x| Rational32::from_f64(x).unwrap()).collect()) } + PyTag { + tag: Tag::srational( + code, + &srational + .into_iter() + .map(|x| Rational32::from_f64(x).unwrap()) + .collect(), + ), + } } #[staticmethod] fn float(code: u16, float: Vec) -> Self { - PyTag { tag: Tag::float(code, float) } + PyTag { + tag: Tag::float(code, &float), + } } #[staticmethod] fn double(code: u16, double: Vec) -> Self { - PyTag { tag: Tag::double(code, double) } + PyTag { + tag: Tag::double(code, &double), + } } #[staticmethod] fn ifd(code: u16, ifd: Vec) -> Self { - PyTag { tag: Tag::ifd(code, ifd) } + PyTag { + tag: Tag::ifd(code, &ifd), + } } #[staticmethod] fn unicode(code: u16, unicode: &str) -> Self { - PyTag { tag: Tag::unicode(code, unicode) } + PyTag { + tag: Tag::unicode(code, unicode), + } } #[staticmethod] fn complex(code: u16, complex: Vec<(f32, f32)>) -> Self { - PyTag { tag: Tag::complex(code, complex.into_iter().map(|(x, y)| Complex { re: x, im: y }).collect()) } + PyTag { + tag: Tag::complex( + code, + &complex + .into_iter() + .map(|(x, y)| Complex { re: x, im: y }) + .collect(), + ), + } } #[staticmethod] fn long8(code: u16, long8: Vec) -> Self { - PyTag { tag: Tag::long8(code, long8) } + PyTag { + tag: Tag::long8(code, &long8), + } } #[staticmethod] fn slong8(code: u16, slong8: Vec) -> Self { - PyTag { tag: Tag::slong8(code, slong8) } + PyTag { + tag: Tag::slong8(code, &slong8), + } } #[staticmethod] fn ifd8(code: u16, ifd8: Vec) -> Self { - PyTag { tag: Tag::ifd8(code, ifd8) } + PyTag { + tag: Tag::ifd8(code, &ifd8), + } } fn count(&self) -> u64 { @@ -103,55 +155,157 @@ impl PyTag { } } - #[pyclass(subclass)] #[pyo3(name = "IJTiffFile")] #[derive(Debug)] struct PyIJTiffFile { - ijtifffile: Option + ijtifffile: Option, } #[pymethods] impl PyIJTiffFile { #[new] fn new(path: &str, shape: (usize, usize, usize)) -> PyResult { - Ok(PyIJTiffFile { ijtifffile: Some(IJTiffFile::new(path, shape)?) } ) + Ok(PyIJTiffFile { + ijtifffile: Some(IJTiffFile::new(path, shape)?), + }) } - fn with_colors(&mut self, colors: (u8, u8, u8)) -> Self { - todo!() + #[getter] + fn get_colors(&self) -> PyResult>>> { + if let Some(ijtifffile) = &self.ijtifffile { + if let Colors::Colors(colors) = &ijtifffile.colors { + return Ok(Some(colors.to_owned())); + } + } + Ok(None) } - fn with_colormap(&mut self, colormap: Vec<(u8, u8, u8)>) -> Self { - todo!() + #[setter] + fn set_colors(&mut self, colors: PyReadonlyArray2) -> PyResult<()> { + if let Some(ijtifffile) = &mut self.ijtifffile { + let a = colors.to_owned_array(); + ijtifffile.colors = Colors::Colors( + (0..a.shape()[0]) + .map(|i| Vec::from(a.slice(s![i, ..]).as_slice().unwrap())) + .collect(), + ); + } + Ok(()) } - fn with_px_size(&mut self, pxsize: f64) -> Self { - todo!() + #[getter] + fn get_colormap(&mut self) -> PyResult>>> { + if let Some(ijtifffile) = &self.ijtifffile { + if let Colors::Colormap(colormap) = &ijtifffile.colors { + return Ok(Some(colormap.to_owned())); + } + } + Ok(None) } - fn with_delta_z(&mut self, delta_z: f64) -> Self { - todo!() + #[setter] + fn set_colormap(&mut self, colormap: PyReadonlyArray2) -> PyResult<()> { + if let Some(ijtifffile) = &mut self.ijtifffile { + let a = colormap.to_owned_array(); + ijtifffile.colors = Colors::Colormap( + (0..a.shape()[0]) + .map(|i| Vec::from(a.slice(s![i, ..]).as_slice().unwrap())) + .collect(), + ); + } + Ok(()) } - fn with_time_interval(&mut self, time_interval: f64) -> Self { - todo!() - } - - fn with_comments(&mut self, comments: String) -> Self { - todo!() - } - - fn append_extra_tag(&mut self, tag: PyTag) { - if let Some(ijtifffile) = self.ijtifffile.as_mut() { - ijtifffile.extra_tags.push(tag.tag); + #[getter] + fn get_px_size(&self) -> PyResult> { + if let Some(ijtifffile) = &self.ijtifffile { + Ok(ijtifffile.px_size) + } else { + Ok(None) } } - fn extend_extra_tags(&mut self, tags: Vec) { - if let Some(ijtifffile) = self.ijtifffile.as_mut() { - ijtifffile.extra_tags.extend(tags.into_iter().map(|x| x.tag)); + #[setter] + fn set_px_size(&mut self, px_size: f64) -> PyResult<()> { + if let Some(ijtifffile) = &mut self.ijtifffile { + ijtifffile.px_size = Some(px_size); } + Ok(()) + } + + #[getter] + fn get_delta_z(&self) -> PyResult> { + if let Some(ijtifffile) = &self.ijtifffile { + Ok(ijtifffile.delta_z) + } else { + Ok(None) + } + } + + #[setter] + fn set_delta_z(&mut self, delta_z: f64) -> PyResult<()> { + if let Some(ijtifffile) = &mut self.ijtifffile { + ijtifffile.delta_z = Some(delta_z); + } + Ok(()) + } + + #[getter] + fn get_time_interval(&self) -> PyResult> { + if let Some(ijtifffile) = &self.ijtifffile { + Ok(ijtifffile.time_interval) + } else { + Ok(None) + } + } + + #[setter] + fn set_time_interval(&mut self, time_interval: f64) -> PyResult<()> { + if let Some(ijtifffile) = &mut self.ijtifffile { + ijtifffile.time_interval = Some(time_interval); + } + Ok(()) + } + + #[getter] + fn get_comment(&self) -> PyResult> { + if let Some(ijtifffile) = &self.ijtifffile { + Ok(ijtifffile.comment.clone()) + } else { + Ok(None) + } + } + + #[setter] + fn set_comment(&mut self, comment: &str) -> PyResult<()> { + if let Some(ijtifffile) = &mut self.ijtifffile { + ijtifffile.comment = Some(String::from(comment)); + } + Ok(()) + } + + fn append_extra_tag(&mut self, tag: PyTag, czt: Option<(usize, usize, usize)>) { + if let Some(ijtifffile) = self.ijtifffile.as_mut() { + if let Some(extra_tags) = ijtifffile.extra_tags.get_mut(&czt) { + extra_tags.push(tag.tag) + } + } + } + + fn get_tags(&self, czt: Option<(usize, usize, usize)>) -> PyResult> { + if let Some(ijtifffile) = &self.ijtifffile { + if let Some(extra_tags) = ijtifffile.extra_tags.get(&czt) { + let v = extra_tags + .iter() + .map(|tag| PyTag { + tag: tag.to_owned(), + }) + .collect(); + return Ok(v); + } + } + Ok(Vec::new()) } fn close(&mut self) -> PyResult<()> { @@ -160,20 +314,19 @@ impl PyIJTiffFile { } } - macro_rules! impl_save { ($T:ty, $t:ident) => { #[pymethods] impl PyIJTiffFile { - fn $t(&mut self, frame: PyReadonlyArray2<$T>, c: usize, t: usize, z: usize, - extra_tags: Option>) -> PyResult<()> { - let extra_tags = if let Some(extra_tags) = extra_tags { - Some(extra_tags.into_iter().map(|x| x.tag).collect()) - } else { - None - }; + fn $t( + &mut self, + frame: PyReadonlyArray2<$T>, + c: usize, + t: usize, + z: usize, + ) -> PyResult<()> { if let Some(ijtifffile) = self.ijtifffile.as_mut() { - ijtifffile.save(frame.to_owned_array(), c, t, z, extra_tags)?; + ijtifffile.save(frame.to_owned_array(), c, t, z)?; } Ok(()) } @@ -192,7 +345,6 @@ impl_save!(i64, save_i64); impl_save!(f32, save_f32); impl_save!(f64, save_f64); - #[pymodule] #[pyo3(name = "tiffwrite_rs")] fn tiffwrite_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { From 4d31933a38a5cd8865d3282d27742b016339c710 Mon Sep 17 00:00:00 2001 From: Wim Pomp Date: Fri, 11 Oct 2024 18:52:49 +0200 Subject: [PATCH 07/13] - make zstd block include content size so fiji can actually read it - add compression level argument - remove shape argument - some more pytest tests --- Cargo.toml | 2 +- py/tiffwrite/__init__.py | 12 ++- pyproject.toml | 2 +- src/lib.rs | 201 +++++++++++++++++++++++++-------------- src/main.rs | 3 +- src/py.rs | 10 +- tests/test_multiple.py | 2 +- tests/test_single.py | 28 ++++-- 8 files changed, 171 insertions(+), 89 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0bc729b..180fbb9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tiffwrite" -version = "2024.10.1" +version = "2024.10.2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/py/tiffwrite/__init__.py b/py/tiffwrite/__init__.py index 85edd53..25cb020 100644 --- a/py/tiffwrite/__init__.py +++ b/py/tiffwrite/__init__.py @@ -33,11 +33,15 @@ FrameInfo = tuple[np.ndarray, None, CZT] class IJTiffFile(rs.IJTiffFile): - def __new__(cls, path: str | Path, shape: tuple[int, int, int], dtype: DTypeLike = 'uint16', + def __new__(cls, path: str | Path, shape: tuple[int, int, int] = None, dtype: DTypeLike = 'uint16', colors: Sequence[str] = None, colormap: str = None, pxsize: float = None, - deltaz: float = None, timeinterval: float = None, comment: str = None, + deltaz: float = None, timeinterval: float = None, compression: int = None, comment: str = None, **extratags: Tag) -> IJTiffFile: - new = super().__new__(cls, str(path), shape) + new = super().__new__(cls, str(path)) + if compression is not None: + if isinstance(compression, Sequence): + compression = compression[-1] + new.set_compression_level(compression) if colors is not None: new.colors = np.array([get_color(color) for color in colors]) if colormap is not None: @@ -54,7 +58,7 @@ class IJTiffFile(rs.IJTiffFile): new.append_extra_tag(extra_tag, None) return new - def __init__(self, path: str | Path, shape: tuple[int, int, int], dtype: DTypeLike = 'uint16', # noqa + def __init__(self, path: str | Path, shape: tuple[int, int, int] = None, dtype: DTypeLike = 'uint16', # noqa colors: Sequence[str] = None, colormap: str = None, pxsize: float = None, # noqa deltaz: float = None, timeinterval: float = None, comment: str = None, # noqa **extratags: Tag.Value | Tag) -> None: # noqa diff --git a/pyproject.toml b/pyproject.toml index 0078504..7ccaf6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ classifiers = [ dependencies = ["colorcet", "matplotlib", "numpy", "tqdm"] [project.optional-dependencies] -test = ["pytest"] +test = ["pytest", "tifffile"] [tool.maturin] python-source = "py" diff --git a/src/lib.rs b/src/lib.rs index 580fbcf..bf45a63 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,26 +1,43 @@ -// #[cfg(not(feature = "nopython"))] +#[cfg(not(feature = "nopython"))] mod py; use anyhow::Result; use chrono::Utc; use ndarray::{s, Array2}; -use num::traits::ToBytes; -use num::{Complex, FromPrimitive, Rational32, Zero}; +use num::{traits::ToBytes, Complex, FromPrimitive, Rational32, Zero}; use rayon::prelude::*; -use std::cmp::Ordering; -use std::collections::HashMap; +use std::{cmp::Ordering, collections::HashMap}; use std::fs::{File, OpenOptions}; use std::hash::{DefaultHasher, Hash, Hasher}; -use std::io::{Read, Seek, SeekFrom, Write}; -use std::thread; -use std::thread::JoinHandle; -use zstd::stream::encode_all; +use std::io::{copy, Read, Seek, SeekFrom, Write}; +use std::{thread, thread::JoinHandle}; +use zstd::{DEFAULT_COMPRESSION_LEVEL, stream::Encoder}; const TAG_SIZE: usize = 20; const OFFSET_SIZE: usize = 8; const OFFSET: u64 = 16; const COMPRESSION: u16 = 50000; +pub fn encode_all(source: Vec, level: i32) -> Result> { + let mut result = Vec::::new(); + copy_encode(&*source, &mut result, level, source.len() as u64)?; + Ok(result) +} + +/// copy_encode from zstd crate, but let it include the content size in the zstd block header +pub fn copy_encode(mut source: R, destination: W, level: i32, length: u64) -> Result<()> +where + R: Read, + W: Write, +{ + let mut encoder = Encoder::new(destination, level)?; + encoder.include_contentsize(true)?; + encoder.set_pledged_src_size(Some(length))?; + copy(&mut source, &mut encoder)?; + encoder.finish()?; + Ok(()) +} + #[derive(Clone, Debug)] struct IFD { tags: Vec, @@ -366,8 +383,8 @@ struct CompressedFrame { #[derive(Clone, Debug)] struct Frame { - tileoffsets: Vec, - tilebytecounts: Vec, + offsets: Vec, + bytecounts: Vec, image_width: u32, image_length: u32, bits_per_sample: u16, @@ -378,8 +395,8 @@ struct Frame { impl Frame { fn new( - tileoffsets: Vec, - tilebytecounts: Vec, + offsets: Vec, + bytecounts: Vec, image_width: u32, image_length: u32, bits_per_sample: u16, @@ -388,8 +405,8 @@ impl Frame { tile_length: u16, ) -> Self { Frame { - tileoffsets, - tilebytecounts, + offsets, + bytecounts, image_width, image_length, bits_per_sample, @@ -455,7 +472,7 @@ pub struct IJTiffFile { frames: HashMap<(usize, usize, usize), Frame>, hashes: HashMap, threads: HashMap<(usize, usize, usize), JoinHandle>, - pub shape: (usize, usize, usize), + pub compression_level: i32, pub colors: Colors, pub comment: Option, pub px_size: Option, @@ -473,7 +490,7 @@ impl Drop for IJTiffFile { } impl IJTiffFile { - pub fn new(path: &str, shape: (usize, usize, usize)) -> Result { + pub fn new(path: &str) -> Result { let mut file = OpenOptions::new() .create(true) .truncate(true) @@ -490,7 +507,7 @@ impl IJTiffFile { frames: HashMap::new(), hashes: HashMap::new(), threads: HashMap::new(), - shape, + compression_level: DEFAULT_COMPRESSION_LEVEL, colors: Colors::None, comment: None, px_size: None, @@ -500,19 +517,23 @@ impl IJTiffFile { }) } - pub fn description(&self) -> String { + pub fn set_compression_level(&mut self, compression_level: i32) { + self.compression_level = compression_level; + } + + pub fn description(&self, c_size: usize, z_size: usize, t_size: usize) -> String { let mut desc: String = String::from("ImageJ=1.11a"); if let Colors::None = self.colors { - desc += &format!("\nimages={}", self.shape.0); - desc += &format!("\nslices={}", self.shape.1); - desc += &format!("\nframes={}", self.shape.2); + desc += &format!("\nimages={}", c_size); + desc += &format!("\nslices={}", z_size); + desc += &format!("\nframes={}", t_size); } else { - desc += &format!("\nimages={}", self.shape.0 * self.shape.1 * self.shape.2); - desc += &format!("\nchannels={}", self.shape.0); - desc += &format!("\nslices={}", self.shape.1); - desc += &format!("\nframes={}", self.shape.2); + desc += &format!("\nimages={}", c_size * z_size * t_size); + desc += &format!("\nchannels={}", c_size); + desc += &format!("\nslices={}", z_size); + desc += &format!("\nframes={}", t_size); }; - if self.shape.0 == 1 { + if c_size == 1 { desc += "\nmode=grayscale"; } else { desc += "\nmode=composite"; @@ -530,27 +551,33 @@ impl IJTiffFile { desc } - fn get_czt(&self, frame_number: usize, channel: u8) -> (usize, usize, usize) { + fn get_czt( + &self, + frame_number: usize, + channel: u8, + c_size: usize, + z_size: usize, + ) -> (usize, usize, usize) { if let Colors::None = self.colors { ( channel as usize, - frame_number % self.shape.1, - frame_number / self.shape.1, + frame_number % z_size, + frame_number / z_size, ) } else { ( - frame_number % self.shape.0, - frame_number / self.shape.0 % self.shape.1, - frame_number / self.shape.0 / self.shape.1, + frame_number % c_size, + frame_number / c_size % z_size, + frame_number / c_size / z_size, ) } } - fn spp_and_n_frames(&self) -> (u8, usize) { + fn spp_and_n_frames(&self, c_size: usize, z_size: usize, t_size: usize) -> (u8, usize) { if let Colors::None = &self.colors { - (self.shape.0 as u8, self.shape.1 * self.shape.2) + (c_size as u8, z_size * t_size) } else { - (1, self.shape.0 * self.shape.1 * self.shape.2) + (1, c_size * z_size * t_size) } } @@ -599,7 +626,7 @@ impl IJTiffFile { where T: Bytes + Clone + Zero + Send + 'static, { - fn compress(frame: Array2) -> CompressedFrame + fn compress(frame: Array2, compression_level: i32) -> CompressedFrame where T: Bytes + Clone + Zero, { @@ -607,7 +634,7 @@ impl IJTiffFile { let image_length = frame.shape()[1] as u32; let tile_size = 2usize .pow( - ((image_width as f64 * image_length as f64 / 64f64).log2() / 2f64).round() + ((image_width as f64 * image_length as f64 / 2f64).log2() / 2f64).round() as u32, ) .max(16) @@ -619,7 +646,7 @@ impl IJTiffFile { .collect(); let bytes = byte_tiles .into_par_iter() - .map(|x| encode_all(&*x, 3).unwrap()) + .map(|x| encode_all(x, compression_level).unwrap()) .collect::>(); CompressedFrame { bytes, @@ -630,8 +657,11 @@ impl IJTiffFile { sample_format: T::SAMPLE_FORMAT, } } - self.threads - .insert((c, z, t), thread::spawn(move || compress(frame))); + let compression_level = self.compression_level; + self.threads.insert( + (c, z, t), + thread::spawn(move || compress(frame, compression_level)), + ); for key in self .threads .keys() @@ -652,15 +682,15 @@ impl IJTiffFile { } fn write_frame(&mut self, frame: CompressedFrame, c: usize, z: usize, t: usize) -> Result<()> { - let mut tileoffsets = Vec::new(); - let mut tilebytecounts = Vec::new(); + let mut offsets = Vec::new(); + let mut bytecounts = Vec::new(); for tile in frame.bytes { - tilebytecounts.push(tile.len() as u64); - tileoffsets.push(self.write(&tile)?); + bytecounts.push(tile.len() as u64); + offsets.push(self.write(&tile)?); } let frame = Frame::new( - tileoffsets, - tilebytecounts, + offsets, + bytecounts, frame.image_width, frame.image_length, frame.bits_per_sample, @@ -674,8 +704,8 @@ impl IJTiffFile { fn tile(frame: Array2, size: usize) -> Vec> { let shape = frame.shape(); - let mut tiles = Vec::new(); let (n, m) = (shape[0] / size, shape[1] / size); + let mut tiles = Vec::new(); for i in 0..n { for j in 0..m { tiles.push( @@ -726,8 +756,15 @@ impl IJTiffFile { } } - fn get_color(&self, _colors: &Vec, _bits_per_sample: u16) -> Result> { - todo!(); + fn get_color(&self, colors: &Vec, bits_per_sample: u16) -> Result> { + let mut c = Vec::new(); + let lvl = if bits_per_sample == 8 { 255 } else { 65535 }; + for i in 0..=lvl { + c.push(i * (colors[0] as u16) / 255); + c.push(i * (colors[1] as u16) / 255); + c.push(i * (colors[2] as u16) / 255); + } + Ok(c) } fn close(&mut self) -> Result<()> { @@ -736,18 +773,33 @@ impl IJTiffFile { self.write_frame(thread.join().unwrap(), c, z, t)?; } } + let mut c_size = 1; + let mut z_size = 1; + let mut t_size = 1; + for (c, z, t) in self.frames.keys() { + c_size = c_size.max(c + 1); + z_size = z_size.max(z + 1); + t_size = t_size.max(t + 1); + } + let mut where_to_write_next_ifd_offset = OFFSET - OFFSET_SIZE as u64; let mut warn = false; - let (samples_per_pixel, n_frames) = self.spp_and_n_frames(); + let (samples_per_pixel, n_frames) = self.spp_and_n_frames(c_size, t_size, z_size); for frame_number in 0..n_frames { - if let Some(frame) = self.frames.get(&self.get_czt(frame_number, 0)) { - let mut tileoffsets = Vec::new(); - let mut tilebytecounts = Vec::new(); + if let Some(frame) = self + .frames + .get(&self.get_czt(frame_number, 0, c_size, z_size)) + { + let mut offsets = Vec::new(); + let mut bytecounts = Vec::new(); let mut frame_count = 0; for channel in 0..samples_per_pixel { - if let Some(frame_n) = self.frames.get(&self.get_czt(frame_number, channel)) { - tileoffsets.extend(frame_n.tileoffsets.iter()); - tilebytecounts.extend(frame_n.tilebytecounts.iter()); + if let Some(frame_n) = + self.frames + .get(&self.get_czt(frame_number, channel, c_size, z_size)) + { + offsets.extend(frame_n.offsets.iter()); + bytecounts.extend(frame_n.bytecounts.iter()); frame_count += 1; } else { warn = true; @@ -758,30 +810,33 @@ impl IJTiffFile { ifd.push_tag(Tag::long(257, &vec![frame.image_length])); ifd.push_tag(Tag::short(258, &vec![frame.bits_per_sample; frame_count])); ifd.push_tag(Tag::short(259, &vec![COMPRESSION])); - ifd.push_tag(Tag::ascii(270, &self.description())); + ifd.push_tag(Tag::ascii(270, &self.description(c_size, z_size, t_size))); ifd.push_tag(Tag::short(277, &vec![frame_count as u16])); - ifd.push_tag(Tag::ascii(305, "tiffwrite_rs")); + ifd.push_tag(Tag::ascii(305, "tiffwrite_tllab_NKI")); ifd.push_tag(Tag::short(322, &vec![frame.tile_width])); ifd.push_tag(Tag::short(323, &vec![frame.tile_length])); - ifd.push_tag(Tag::long8(324, &tileoffsets)); - ifd.push_tag(Tag::long8(325, &tilebytecounts)); - ifd.push_tag(Tag::short(339, &vec![frame.sample_format])); + ifd.push_tag(Tag::long8(324, &offsets)); + ifd.push_tag(Tag::long8(325, &bytecounts)); + if frame.sample_format > 1 { + ifd.push_tag(Tag::short(339, &vec![frame.sample_format])); + } if let Some(px_size) = self.px_size { let r = vec![Rational32::from_f64(px_size).unwrap()]; ifd.push_tag(Tag::rational(282, &r)); ifd.push_tag(Tag::rational(283, &r)); ifd.push_tag(Tag::short(296, &vec![1])); } - + if let Colors::Colormap(_) = &self.colors { + ifd.push_tag(Tag::short(262, &vec![3])); + } else if let Colors::None = self.colors { + ifd.push_tag(Tag::short(262, &vec![1])); + } if frame_number == 0 { if let Colors::Colormap(colormap) = &self.colors { ifd.push_tag(Tag::short( 320, &self.get_colormap(colormap, frame.bits_per_sample), )); - ifd.push_tag(Tag::short(262, &vec![3])); - } else if let Colors::None = self.colors { - ifd.push_tag(Tag::short(262, &vec![1])); } } if frame_number < samples_per_pixel as usize { @@ -794,12 +849,12 @@ impl IJTiffFile { } } if let Colors::None = &self.colors { - if self.shape.0 > 1 { + if c_size > 1 { ifd.push_tag(Tag::short(284, &vec![2])) } } for channel in 0..samples_per_pixel { - let czt = self.get_czt(frame_number, channel); + let czt = self.get_czt(frame_number, channel, c_size, z_size); if let Some(extra_tags) = self.extra_tags.get(&Some(czt)) { for tag in extra_tags { ifd.push_tag(tag.to_owned()) @@ -811,10 +866,12 @@ impl IJTiffFile { ifd.push_tag(tag.to_owned()) } } - ifd.push_tag(Tag::ascii( - 306, - &format!("{}", Utc::now().format("%Y:%m:%d %H:%M:%S")), - )); + if frame_number == 0 { + ifd.push_tag(Tag::ascii( + 306, + &format!("{}", Utc::now().format("%Y:%m:%d %H:%M:%S")), + )); + } where_to_write_next_ifd_offset = ifd.write(self, where_to_write_next_ifd_offset)?; } else { warn = true; diff --git a/src/main.rs b/src/main.rs index 75a03da..449463d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,8 @@ use tiffwrite::IJTiffFile; fn main() -> Result<()> { println!("Hello World!"); - let mut f = IJTiffFile::new("foo.tif", (2, 1, 1))?; + let mut f = IJTiffFile::new("foo.tif")?; + f.set_compression_level(10); let mut arr = Array2::::zeros((100, 100)); for i in 0..arr.shape()[0] { for j in 0..arr.shape()[1] { diff --git a/src/py.rs b/src/py.rs index 693b338..ce567f2 100644 --- a/src/py.rs +++ b/src/py.rs @@ -165,12 +165,18 @@ struct PyIJTiffFile { #[pymethods] impl PyIJTiffFile { #[new] - fn new(path: &str, shape: (usize, usize, usize)) -> PyResult { + fn new(path: &str) -> PyResult { Ok(PyIJTiffFile { - ijtifffile: Some(IJTiffFile::new(path, shape)?), + ijtifffile: Some(IJTiffFile::new(path)?), }) } + fn set_compression_level(&mut self, compression_level: i32) { + if let Some(ref mut ijtifffile) = self.ijtifffile { + ijtifffile.set_compression_level(compression_level); + } + } + #[getter] fn get_colors(&self) -> PyResult>>> { if let Some(ijtifffile) = &self.ijtifffile { diff --git a/tests/test_multiple.py b/tests/test_multiple.py index 37e7abb..936c4bc 100644 --- a/tests/test_multiple.py +++ b/tests/test_multiple.py @@ -12,7 +12,7 @@ def test_mult(tmp_path: Path) -> None: shape = (2, 3, 5) paths = [tmp_path / f'test{i}.tif' for i in range(6)] with ExitStack() as stack: - tifs = [stack.enter_context(IJTiffFile(path, shape)) for path in paths] # noqa + tifs = [stack.enter_context(IJTiffFile(path)) for path in paths] # noqa for c, z, t in tqdm(product(range(shape[0]), range(shape[1]), range(shape[2])), total=np.prod(shape)): # noqa for tif in tifs: tif.save(np.random.randint(0, 255, (64, 64)), c, z, t) diff --git a/tests/test_single.py b/tests/test_single.py index 2d47470..8583b9b 100644 --- a/tests/test_single.py +++ b/tests/test_single.py @@ -1,14 +1,28 @@ -from itertools import product from pathlib import Path import numpy as np +import pytest +from tifffile import imread from tiffwrite import IJTiffFile -def test_single(tmp_path: Path) -> None: - path = tmp_path / 'test.tif' - with IJTiffFile(path, (3, 4, 5)) as tif: - for c, z, t in product(range(3), range(4), range(5)): - tif.save(np.random.randint(0, 255, (64, 64)), c, z, t) - assert path.exists() +@pytest.mark.parametrize('dtype', ('uint8', 'uint16', 'uint32', 'uint64', + 'int8', 'int16', 'int32', 'int64', 'float32', 'float64')) +def test_single(tmp_path: Path, dtype) -> None: + with IJTiffFile(tmp_path / 'test.tif', dtype=dtype) as tif: + a0, b0 = np.meshgrid(range(100), range(100)) + a0[::2, :] = 0 + b0[:, ::2] = 1 + tif.save(a0, 0, 0, 0) + tif.save(b0, 1, 0, 0) + + a1, b1 = np.meshgrid(range(100), range(100)) + a1[:, ::2] = 0 + b1[::2, :] = 1 + tif.save(a1, 0, 0, 1) + tif.save(b1, 1, 0, 1) + + t = imread(tmp_path / 'test.tif') + assert t.dtype == np.dtype(dtype), "data type does not match" + assert np.all(np.stack(((a0, b0), (a1, b1))) == t), "data does not match" From 625b222a0de6ac4fb18681308860622f8ca31d4b Mon Sep 17 00:00:00 2001 From: Wim Pomp Date: Sat, 12 Oct 2024 13:19:22 +0200 Subject: [PATCH 08/13] - some warnings - better color(map) support - make python module optional in rust - max 48 threads --- .github/workflows/publish.yml | 24 ++++++++ .github/workflows/pytest.yml | 2 +- .github/workflows/wheels.yml | 42 +++++++++++++ Cargo.toml | 17 +++--- README.md | 24 ++++---- py/tiffwrite/__init__.py | 109 ++++++++++++++++++++++++---------- pyproject.toml | 4 +- src/lib.rs | 105 ++++++++++++++++---------------- tests/test_single.py | 2 +- 9 files changed, 224 insertions(+), 105 deletions(-) create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/wheels.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..2006944 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,24 @@ +name: Publish + +on: workflow_dispatch + +jobs: + publish_wheels: + uses: ./.github/workflows/wheels.yml + publish: + name: publish + needs: publish_wheels + runs-on: ubuntu-latest + environment: pypi + steps: + - uses: actions/download-artifact@v4 + with: + # unpacks all tiffwrite artifacts into dist/ + pattern: tiffwrite-* + path: dist + merge-multiple: true + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} + repository-url: https://upload.pypi.org/legacy/ \ No newline at end of file diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index c61decd..dafe279 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,6 +1,6 @@ name: PyTest -on: [push, pull_request] +on: [workflow_call, push, pull_request] jobs: pytest: diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..a93bec7 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,42 @@ +name: Wheels + +on: workflow_call + +jobs: + wheels_pytest: + uses: ./.github/workflows/pytest.yml + build_wheels: + name: Build wheels on ${{ matrix.os }} + needs: [ wheels_pytest ] + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-latest, windows-latest, macos-13, macos-latest ] + + steps: + - uses: actions/checkout@v4 + + - name: Build wheels + uses: pypa/cibuildwheel@v2.21.2 + + - uses: actions/upload-artifact@v4 + with: + name: tiffwrite-wheels-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl + + build_sdist: + name: Build source distribution + needs: [ wheels_pytest ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build sdist + run: | + pip install build + python -m build --sdist + + - uses: actions/upload-artifact@v4 + with: + name: tiffwrite-sdist + path: dist/*.tar.gz \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 180fbb9..c00fafc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,20 +3,23 @@ name = "tiffwrite" version = "2024.10.2" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] name = "tiffwrite" crate-type = ["cdylib", "rlib"] [dependencies] -pyo3 = { version = "0.21.2", features = ["extension-module", "abi3-py310", "generate-import-lib", "anyhow", "multiple-pymethods"] } anyhow = "1.0.89" -rayon = "1.10.0" -num = "0.4.3" -ndarray = "0.15.6" chrono = "0.4.38" -numpy = "0.21.0" +ndarray = "0.15.6" +num = "0.4.3" +rayon = "1.10.0" zstd = "0.13.2" +numpy = { version = "0.22.0", optional = true } + +[dependencies.pyo3] +version = "0.22.2" +features = ["extension-module", "abi3-py310", "generate-import-lib", "anyhow", "multiple-pymethods"] +optional = true [features] -nopython = [] +python = ["dep:pyo3", "dep:numpy"] diff --git a/README.md b/README.md index bb9af0a..308162a 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,26 @@ -[![mypy](https://github.com/wimpomp/tiffwrite/actions/workflows/mypy.yml/badge.svg)](https://github.com/wimpomp/tiffwrite/actions/workflows/mypy.yml) [![pytest](https://github.com/wimpomp/tiffwrite/actions/workflows/pytest.yml/badge.svg)](https://github.com/wimpomp/tiffwrite/actions/workflows/pytest.yml) # Tiffwrite -Exploiting [tifffile](https://pypi.org/project/tifffile/) in parallel to write BioFormats/ImageJ compatible tiffs with -good compression. +Write BioFormats/ImageJ compatible tiffs with zstd compression in parallel using Rust. ## Features - Writes bigtiff files that open in ImageJ as hyperstack with correct dimensions. - Parallel compression. - Write individual frames in random order. - Compresses even more by referencing tag or image data which otherwise would have been saved several times. -For example empty frames, or a long string tag on every frame. +For example empty frames, or a long string tag on every frame. Editing tiffs becomes mostly impossible, but compression +makes that very hard anyway. - Enables memory efficient scripts by saving frames whenever they're ready to be saved, not waiting for the whole stack. -- Colormaps, extra tags, globally or frame dependent. +- Colormaps +- Extra tags, globally or frame dependent. ## Installation pip install tiffwrite or +- install [rust](https://rustup.rs/) + + pip install tiffwrite@git+https://github.com/wimpomp/tiffwrite # Usage @@ -67,11 +70,10 @@ or from tiffwrite import IJTiffFile import numpy as np - shape = (3, 5, 10) # channels, z, time - with IJTiffFile('file.tif', shape, pxsize=0.09707) as tif: - for c in range(shape[0]): - for z in range(shape[1]): - for t in range(shape[2]): + with IJTiffFile('file.tif', pxsize=0.09707) as tif: + for c in range(3): + for z in range(5): + for t in range(10): tif.save(np.random.randint(0, 10, (32, 32)), c, z, t) ## Saving multiple tiffs simultaneously @@ -79,7 +81,7 @@ or import numpy as np shape = (3, 5, 10) # channels, z, time - with IJTiffFile('fileA.tif', shape) as tif_a, IJTiffFile('fileB.tif', shape) as tif_b: + with IJTiffFile('fileA.tif') as tif_a, IJTiffFile('fileB.tif') as tif_b: for c in range(shape[0]): for z in range(shape[1]): for t in range(shape[2]): diff --git a/py/tiffwrite/__init__.py b/py/tiffwrite/__init__.py index 25cb020..edcd6a2 100644 --- a/py/tiffwrite/__init__.py +++ b/py/tiffwrite/__init__.py @@ -3,8 +3,10 @@ from __future__ import annotations from itertools import product from pathlib import Path from typing import Any, Sequence +from warnings import warn import colorcet +import matplotlib import numpy as np from matplotlib import colors as mpl_colors from numpy.typing import ArrayLike, DTypeLike @@ -12,13 +14,13 @@ from tqdm.auto import tqdm from . import tiffwrite_rs as rs # noqa - __all__ = ['Header', 'IJTiffFile', 'IFD', 'FrameInfo', 'Tag', 'Strip', 'tiffwrite'] class Header: pass + class IFD(dict): pass @@ -32,38 +34,60 @@ CZT = tuple[int, int, int] FrameInfo = tuple[np.ndarray, None, CZT] +class TiffWriteWarning(UserWarning): + pass + + class IJTiffFile(rs.IJTiffFile): - def __new__(cls, path: str | Path, shape: tuple[int, int, int] = None, dtype: DTypeLike = 'uint16', - colors: Sequence[str] = None, colormap: str = None, pxsize: float = None, - deltaz: float = None, timeinterval: float = None, compression: int = None, comment: str = None, - **extratags: Tag) -> IJTiffFile: - new = super().__new__(cls, str(path)) + """ Writes a tiff file in a format that the BioFormats reader in Fiji understands. + file: filename of the new tiff file + shape: not used anymore + dtype: datatype to use when saving to tiff + colors: a tuple with a color per channel, chosen from matplotlib.colors, html colors are also possible + colormap: name of a colormap from colorcet + pxsize: pixel size in um + deltaz: z slice interval in um + timeinterval: time between frames in seconds + extratags: other tags to be saved, example: (Tag.ascii(315, 'John Doe'), Tag.bytes(4567, [400, 500]) + or (Tag.ascii(33432, 'Made by me'),). + """ + def __new__(cls, path: str | Path, *args, **kwargs) -> IJTiffFile: + return super().__new__(cls, str(path)) + + def __init__(self, path: str | Path, shape: tuple[int, int, int] = None, dtype: DTypeLike = 'uint16', + colors: Sequence[str] = None, colormap: str = None, pxsize: float = None, + deltaz: float = None, timeinterval: float = None, compression: int = None, comment: str = None, + extratags: Sequence[Tag] = None) -> None: + self.path = Path(path) + self.shape = shape + self.dtype = np.dtype(dtype) if compression is not None: if isinstance(compression, Sequence): compression = compression[-1] - new.set_compression_level(compression) + self.set_compression_level(compression) if colors is not None: - new.colors = np.array([get_color(color) for color in colors]) + self.colors = np.array([get_color(color) for color in colors]) if colormap is not None: - new.colormap = get_colormap(colormap) + self.colormap = get_colormap(colormap) if pxsize is not None: - new.px_size = float(pxsize) + self.px_size = float(pxsize) if deltaz is not None: - new.delta_z = float(deltaz) + self.delta_z = float(deltaz) if timeinterval is not None: - new.time_interval = float(timeinterval) + self.time_interval = float(timeinterval) if comment is not None: - new.comment = comment - for extra_tag in extratags: - new.append_extra_tag(extra_tag, None) - return new - - def __init__(self, path: str | Path, shape: tuple[int, int, int] = None, dtype: DTypeLike = 'uint16', # noqa - colors: Sequence[str] = None, colormap: str = None, pxsize: float = None, # noqa - deltaz: float = None, timeinterval: float = None, comment: str = None, # noqa - **extratags: Tag.Value | Tag) -> None: # noqa - self.path = Path(path) - self.dtype = np.dtype(dtype) + self.comment = comment + if extratags is not None: + for extra_tag in extratags: + self.append_extra_tag(extra_tag, None) + if self.dtype.itemsize == 1 and colors is not None: + warn('Fiji will not interpret colors saved in an (u)int8 tif, save as (u)int16 instead.', + TiffWriteWarning, stacklevel=2) + if shape is not None: + warn('Providing shape is not needed anymore, the argument will be removed in the future.', + DeprecationWarning, stacklevel=2) + if colors is not None and colormap is not None: + warn('Cannot have colors and colormap simultaneously.', TiffWriteWarning, stacklevel=2) def __enter__(self) -> IJTiffFile: return self @@ -71,7 +95,8 @@ class IJTiffFile(rs.IJTiffFile): def __exit__(self, exc_type, exc_val, exc_tb): self.close() - def save(self, frame: ArrayLike, c: int, z: int, t: int) -> None: + def save(self, frame: ArrayLike, c: int, z: int, t: int, extratags: Sequence[Tag] = None) -> None: + """ save a 2d numpy array to the tiff at channel=c, slice=z, time=t, with optional extra tif tags """ for frame, _, (cn, zn, tn) in self.compress_frame(frame): frame = np.asarray(frame).astype(self.dtype) match self.dtype: @@ -97,15 +122,36 @@ class IJTiffFile(rs.IJTiffFile): self.save_f64(frame, c + cn, z + zn, t + tn) case _: raise TypeError(f'Cannot save type {self.dtype}') + if extratags is not None: + for extra_tag in extratags: + self.append_extra_tag(extra_tag, (c, z, t)) def compress_frame(self, frame: ArrayLike) -> tuple[FrameInfo]: # noqa + """ backwards compatibility """ return (frame, None, (0, 0, 0)), + def get_colormap(colormap: str) -> np.ndarray: - colormap = getattr(colorcet, colormap) - colormap[0] = '#ffffff' - colormap[-1] = '#000000' - return np.array([[int(''.join(i), 16) for i in zip(*[iter(s[1:])] * 2)] for s in colormap]).astype('uint8') + if hasattr(colorcet, colormap.rstrip('_r')): + cm = np.array([[int(''.join(i), 16) for i in zip(*[iter(s[1:])] * 2)] + for s in getattr(colorcet, colormap.rstrip('_r'))]).astype('uint8') + if colormap.endswith('_r'): + cm = cm[::-1] + if colormap.startswith('glasbey') or colormap.endswith('glasbey'): + cm[0] = 0, 0, 0 + cm[-1] = 255, 255, 255 + else: + cmap = matplotlib.colormaps.get_cmap(colormap) + if cmap.N < 256: + cm = (255 * np.vstack(((0, 0, 0), + matplotlib.cm.ScalarMappable(matplotlib.colors.Normalize(1, 254), + cmap).to_rgba(np.arange(1, 254))[:, :3], + (1, 1, 1)))).astype('uint8') + else: + cm = (255 * matplotlib.cm.ScalarMappable(matplotlib.colors.Normalize(0, 255), cmap) + .to_rgba(np.arange(256))[:, :3]).astype('uint8') + return cm + def get_color(color: str) -> np.ndarray: return np.array([int(''.join(i), 16) for i in zip(*[iter(mpl_colors.to_hex(color)[1:])] * 2)]).astype('uint8') @@ -131,10 +177,7 @@ def tiffwrite(file: str | Path, data: np.ndarray, axes: str = 'TZCXY', dtype: DT data = np.expand_dims(data, axis) shape = data.shape[:3] - with IJTiffFile(file, shape, data.dtype if dtype is None else dtype, *args, **kwargs) as f: # type: ignore - at_least_one = False + with IJTiffFile(file, dtype=data.dtype if dtype is None else dtype, *args, **kwargs) as f: for n in tqdm(product(*[range(i) for i in shape]), total=np.prod(shape), # type: ignore desc='Saving tiff', disable=not bar): - if np.any(data[n]) or not at_least_one: - f.save(data[n], *n) - at_least_one = True + f.save(data[n], *n) diff --git a/pyproject.toml b/pyproject.toml index 7ccaf6b..7bddf15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,11 +15,11 @@ classifiers = [ dependencies = ["colorcet", "matplotlib", "numpy", "tqdm"] [project.optional-dependencies] -test = ["pytest", "tifffile"] +test = ["pytest", "tifffile", "imagecodecs"] [tool.maturin] python-source = "py" -features = ["pyo3/extension-module"] +features = ["pyo3/extension-module", "python"] module-name = "tiffwrite.tiffwrite_rs" [tool.isort] diff --git a/src/lib.rs b/src/lib.rs index bf45a63..b293b83 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -#[cfg(not(feature = "nopython"))] +#[cfg(feature = "python")] mod py; use anyhow::Result; @@ -6,26 +6,30 @@ use chrono::Utc; use ndarray::{s, Array2}; use num::{traits::ToBytes, Complex, FromPrimitive, Rational32, Zero}; use rayon::prelude::*; -use std::{cmp::Ordering, collections::HashMap}; use std::fs::{File, OpenOptions}; use std::hash::{DefaultHasher, Hash, Hasher}; use std::io::{copy, Read, Seek, SeekFrom, Write}; -use std::{thread, thread::JoinHandle}; -use zstd::{DEFAULT_COMPRESSION_LEVEL, stream::Encoder}; +use std::time::Duration; +use std::{cmp::Ordering, collections::HashMap}; +use std::{ + thread, + thread::{sleep, JoinHandle}, +}; +use zstd::{stream::Encoder, DEFAULT_COMPRESSION_LEVEL}; const TAG_SIZE: usize = 20; const OFFSET_SIZE: usize = 8; const OFFSET: u64 = 16; const COMPRESSION: u16 = 50000; -pub fn encode_all(source: Vec, level: i32) -> Result> { +fn encode_all(source: Vec, level: i32) -> Result> { let mut result = Vec::::new(); copy_encode(&*source, &mut result, level, source.len() as u64)?; Ok(result) } /// copy_encode from zstd crate, but let it include the content size in the zstd block header -pub fn copy_encode(mut source: R, destination: W, level: i32, length: u64) -> Result<()> +fn copy_encode(mut source: R, destination: W, level: i32, length: u64) -> Result<()> where R: Read, W: Write, @@ -518,7 +522,7 @@ impl IJTiffFile { } pub fn set_compression_level(&mut self, compression_level: i32) { - self.compression_level = compression_level; + self.compression_level = compression_level.max(-7).min(22); } pub fn description(&self, c_size: usize, z_size: usize, t_size: usize) -> String { @@ -644,10 +648,17 @@ impl IJTiffFile { .into_iter() .map(|tile| tile.map(|x| x.bytes()).into_iter().flatten().collect()) .collect(); - let bytes = byte_tiles - .into_par_iter() - .map(|x| encode_all(x, compression_level).unwrap()) - .collect::>(); + let bytes = if byte_tiles.len() > 4 { + byte_tiles + .into_par_iter() + .map(|x| encode_all(x, compression_level).unwrap()) + .collect::>() + } else { + byte_tiles + .into_iter() + .map(|x| encode_all(x, compression_level).unwrap()) + .collect::>() + }; CompressedFrame { bytes, image_width, @@ -657,22 +668,24 @@ impl IJTiffFile { sample_format: T::SAMPLE_FORMAT, } } + loop { + self.collect_threads(false)?; + if self.threads.len() < 48 { + break; + } + sleep(Duration::from_millis(100)); + } let compression_level = self.compression_level; self.threads.insert( (c, z, t), thread::spawn(move || compress(frame, compression_level)), ); - for key in self - .threads - .keys() - .cloned() - .collect::>() - { - if self.threads[&key].is_finished() {} - } + Ok(()) + } + fn collect_threads(&mut self, block: bool) -> Result<()> { for (c, z, t) in self.threads.keys().cloned().collect::>() { - if self.threads[&(c, z, t)].is_finished() { + if block | self.threads[&(c, z, t)].is_finished() { if let Some(thread) = self.threads.remove(&(c, z, t)) { self.write_frame(thread.join().unwrap(), c, z, t)?; } @@ -739,40 +752,33 @@ impl IJTiffFile { } fn get_colormap(&self, colormap: &Vec>, bits_per_sample: u16) -> Vec { - if bits_per_sample == 8 { - colormap - .iter() - .flatten() - .map(|x| (*x as u16) * 256) - .collect() - } else { - colormap - .iter() - .map(|x| vec![x; 256]) - .flatten() - .flatten() - .map(|x| (*x as u16) * 256) - .collect() + let mut r = Vec::new(); + let mut g = Vec::new(); + let mut b = Vec::new(); + let n = 2usize.pow(bits_per_sample as u32 - 8); + for color in colormap { + r.extend(vec![(color[0] as u16) * 257; n]); + g.extend(vec![(color[1] as u16) * 257; n]); + b.extend(vec![(color[2] as u16) * 257; n]); } + r.extend(g); + r.extend(b); + r } - fn get_color(&self, colors: &Vec, bits_per_sample: u16) -> Result> { + fn get_color(&self, colors: &Vec, bits_per_sample: u16) -> Vec { let mut c = Vec::new(); - let lvl = if bits_per_sample == 8 { 255 } else { 65535 }; - for i in 0..=lvl { - c.push(i * (colors[0] as u16) / 255); - c.push(i * (colors[1] as u16) / 255); - c.push(i * (colors[2] as u16) / 255); + let n = 2usize.pow(bits_per_sample as u32 - 8); + for color in colors { + for i in 0..256 { + c.extend(vec![i * (*color as u16) / 255 * 257; n]) + } } - Ok(c) + c } fn close(&mut self) -> Result<()> { - for (c, z, t) in self.threads.keys().cloned().collect::>() { - if let Some(thread) = self.threads.remove(&(c, z, t)) { - self.write_frame(thread.join().unwrap(), c, z, t)?; - } - } + self.collect_threads(true)?; let mut c_size = 1; let mut z_size = 1; let mut t_size = 1; @@ -812,7 +818,7 @@ impl IJTiffFile { ifd.push_tag(Tag::short(259, &vec![COMPRESSION])); ifd.push_tag(Tag::ascii(270, &self.description(c_size, z_size, t_size))); ifd.push_tag(Tag::short(277, &vec![frame_count as u16])); - ifd.push_tag(Tag::ascii(305, "tiffwrite_tllab_NKI")); + ifd.push_tag(Tag::ascii(305, "tiffwrite_rs")); ifd.push_tag(Tag::short(322, &vec![frame.tile_width])); ifd.push_tag(Tag::short(323, &vec![frame.tile_length])); ifd.push_tag(Tag::long8(324, &offsets)); @@ -824,7 +830,6 @@ impl IJTiffFile { let r = vec![Rational32::from_f64(px_size).unwrap()]; ifd.push_tag(Tag::rational(282, &r)); ifd.push_tag(Tag::rational(283, &r)); - ifd.push_tag(Tag::short(296, &vec![1])); } if let Colors::Colormap(_) = &self.colors { ifd.push_tag(Tag::short(262, &vec![3])); @@ -839,11 +844,11 @@ impl IJTiffFile { )); } } - if frame_number < samples_per_pixel as usize { + if frame_number < c_size { if let Colors::Colors(colors) = &self.colors { ifd.push_tag(Tag::short( 320, - &self.get_color(&colors[frame_number], frame.bits_per_sample)?, + &self.get_color(&colors[frame_number], frame.bits_per_sample), )); ifd.push_tag(Tag::short(262, &vec![3])); } diff --git a/tests/test_single.py b/tests/test_single.py index 8583b9b..0d88e69 100644 --- a/tests/test_single.py +++ b/tests/test_single.py @@ -10,7 +10,7 @@ from tiffwrite import IJTiffFile @pytest.mark.parametrize('dtype', ('uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16', 'int32', 'int64', 'float32', 'float64')) def test_single(tmp_path: Path, dtype) -> None: - with IJTiffFile(tmp_path / 'test.tif', dtype=dtype) as tif: + with IJTiffFile(tmp_path / 'test.tif', dtype=dtype, pxsize=0.1, deltaz=0.5, timeinterval=6.5) as tif: a0, b0 = np.meshgrid(range(100), range(100)) a0[::2, :] = 0 b0[:, ::2] = 1 From 95a49bb28e174e9eab4dc3c1210336708757c354 Mon Sep 17 00:00:00 2001 From: Wim Pomp Date: Sat, 12 Oct 2024 13:30:11 +0200 Subject: [PATCH 09/13] - bump ndarray --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index c00fafc..c58af4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] anyhow = "1.0.89" chrono = "0.4.38" -ndarray = "0.15.6" +ndarray = "0.16.1" num = "0.4.3" rayon = "1.10.0" zstd = "0.13.2" From 140e7eaf388fd23547d35e174adf825e0e8cd48a Mon Sep 17 00:00:00 2001 From: Wim Pomp Date: Sun, 13 Oct 2024 15:52:52 +0200 Subject: [PATCH 10/13] - use HashSet for storing tags in ifd - make offset and count tags short or long if possible --- src/lib.rs | 94 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 55 insertions(+), 39 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b293b83..37e33c0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ use chrono::Utc; use ndarray::{s, Array2}; use num::{traits::ToBytes, Complex, FromPrimitive, Rational32, Zero}; use rayon::prelude::*; +use std::collections::HashSet; use std::fs::{File, OpenOptions}; use std::hash::{DefaultHasher, Hash, Hasher}; use std::io::{copy, Read, Seek, SeekFrom, Write}; @@ -44,37 +45,32 @@ where #[derive(Clone, Debug)] struct IFD { - tags: Vec, + tags: HashSet, } impl IFD { pub fn new() -> Self { - IFD { tags: Vec::new() } - } - - fn push_tag(&mut self, tag: Tag) { - if !self.tags.contains(&tag) { - self.tags.push(tag); + IFD { + tags: HashSet::new(), } } fn write(&mut self, ijtifffile: &mut IJTiffFile, where_to_write_offset: u64) -> Result { - self.tags.sort(); + let mut tags = self.tags.drain().collect::>(); + tags.sort(); ijtifffile.file.seek(SeekFrom::End(0))?; if ijtifffile.file.stream_position()? % 2 == 1 { ijtifffile.file.write(&[0])?; } let offset = ijtifffile.file.stream_position()?; - ijtifffile - .file - .write(&(self.tags.len() as u64).to_le_bytes())?; + ijtifffile.file.write(&(tags.len() as u64).to_le_bytes())?; - for tag in self.tags.iter_mut() { + for tag in tags.iter_mut() { tag.write_tag(ijtifffile)?; } let where_to_write_next_ifd_offset = ijtifffile.file.stream_position()?; ijtifffile.file.write(&vec![0u8; OFFSET_SIZE])?; - for tag in self.tags.iter() { + for tag in tags.iter() { tag.write_data(ijtifffile)?; } ijtifffile @@ -111,6 +107,12 @@ impl PartialEq for Tag { } } +impl Hash for Tag { + fn hash(&self, state: &mut H) { + self.code.hash(state); + } +} + impl Tag { pub fn new(code: u16, bytes: Vec, ttype: u16) -> Self { Tag { @@ -307,10 +309,11 @@ impl Tag { ) } - pub fn ifd8(code: u16, ifd8: &Vec) -> Self { + pub fn ifd8(code: u16, value: &Vec) -> Self { Tag::new( code, - ifd8.into_iter() + value + .into_iter() .map(|x| x.to_le_bytes()) .flatten() .collect(), @@ -318,6 +321,17 @@ impl Tag { ) } + pub fn short_long_or_long8(code: u16, value: &Vec) -> Self { + let m = *value.iter().max().unwrap(); + if m < 65536 { + Tag::short(code, &value.into_iter().map(|x| *x as u16).collect()) + } else if m < 4294967296 { + Tag::long(code, &value.into_iter().map(|x| *x as u32).collect()) + } else { + Tag::long8(code, value) + } + } + pub fn count(&self) -> u64 { let c = match self.ttype { 1 => self.bytes.len(), // BYTE @@ -685,7 +699,7 @@ impl IJTiffFile { fn collect_threads(&mut self, block: bool) -> Result<()> { for (c, z, t) in self.threads.keys().cloned().collect::>() { - if block | self.threads[&(c, z, t)].is_finished() { + if block || self.threads[&(c, z, t)].is_finished() { if let Some(thread) = self.threads.remove(&(c, z, t)) { self.write_frame(thread.join().unwrap(), c, z, t)?; } @@ -812,33 +826,35 @@ impl IJTiffFile { } } let mut ifd = IFD::new(); - ifd.push_tag(Tag::long(256, &vec![frame.image_width])); - ifd.push_tag(Tag::long(257, &vec![frame.image_length])); - ifd.push_tag(Tag::short(258, &vec![frame.bits_per_sample; frame_count])); - ifd.push_tag(Tag::short(259, &vec![COMPRESSION])); - ifd.push_tag(Tag::ascii(270, &self.description(c_size, z_size, t_size))); - ifd.push_tag(Tag::short(277, &vec![frame_count as u16])); - ifd.push_tag(Tag::ascii(305, "tiffwrite_rs")); - ifd.push_tag(Tag::short(322, &vec![frame.tile_width])); - ifd.push_tag(Tag::short(323, &vec![frame.tile_length])); - ifd.push_tag(Tag::long8(324, &offsets)); - ifd.push_tag(Tag::long8(325, &bytecounts)); + ifd.tags.insert(Tag::long(256, &vec![frame.image_width])); + ifd.tags.insert(Tag::long(257, &vec![frame.image_length])); + ifd.tags + .insert(Tag::short(258, &vec![frame.bits_per_sample; frame_count])); + ifd.tags.insert(Tag::short(259, &vec![COMPRESSION])); + ifd.tags + .insert(Tag::ascii(270, &self.description(c_size, z_size, t_size))); + ifd.tags.insert(Tag::short(277, &vec![frame_count as u16])); + ifd.tags.insert(Tag::ascii(305, "tiffwrite_rs")); + ifd.tags.insert(Tag::short(322, &vec![frame.tile_width])); + ifd.tags.insert(Tag::short(323, &vec![frame.tile_length])); + ifd.tags.insert(Tag::short_long_or_long8(324, &offsets)); + ifd.tags.insert(Tag::short_long_or_long8(325, &bytecounts)); if frame.sample_format > 1 { - ifd.push_tag(Tag::short(339, &vec![frame.sample_format])); + ifd.tags.insert(Tag::short(339, &vec![frame.sample_format])); } if let Some(px_size) = self.px_size { let r = vec![Rational32::from_f64(px_size).unwrap()]; - ifd.push_tag(Tag::rational(282, &r)); - ifd.push_tag(Tag::rational(283, &r)); + ifd.tags.insert(Tag::rational(282, &r)); + ifd.tags.insert(Tag::rational(283, &r)); } if let Colors::Colormap(_) = &self.colors { - ifd.push_tag(Tag::short(262, &vec![3])); + ifd.tags.insert(Tag::short(262, &vec![3])); } else if let Colors::None = self.colors { - ifd.push_tag(Tag::short(262, &vec![1])); + ifd.tags.insert(Tag::short(262, &vec![1])); } if frame_number == 0 { if let Colors::Colormap(colormap) = &self.colors { - ifd.push_tag(Tag::short( + ifd.tags.insert(Tag::short( 320, &self.get_colormap(colormap, frame.bits_per_sample), )); @@ -846,33 +862,33 @@ impl IJTiffFile { } if frame_number < c_size { if let Colors::Colors(colors) = &self.colors { - ifd.push_tag(Tag::short( + ifd.tags.insert(Tag::short( 320, &self.get_color(&colors[frame_number], frame.bits_per_sample), )); - ifd.push_tag(Tag::short(262, &vec![3])); + ifd.tags.insert(Tag::short(262, &vec![3])); } } if let Colors::None = &self.colors { if c_size > 1 { - ifd.push_tag(Tag::short(284, &vec![2])) + ifd.tags.insert(Tag::short(284, &vec![2])); } } for channel in 0..samples_per_pixel { let czt = self.get_czt(frame_number, channel, c_size, z_size); if let Some(extra_tags) = self.extra_tags.get(&Some(czt)) { for tag in extra_tags { - ifd.push_tag(tag.to_owned()) + ifd.tags.insert(tag.to_owned()); } } } if let Some(extra_tags) = self.extra_tags.get(&None) { for tag in extra_tags { - ifd.push_tag(tag.to_owned()) + ifd.tags.insert(tag.to_owned()); } } if frame_number == 0 { - ifd.push_tag(Tag::ascii( + ifd.tags.insert(Tag::ascii( 306, &format!("{}", Utc::now().format("%Y:%m:%d %H:%M:%S")), )); From 654755ab83c5f1aa376f8ee8bf5464f87a03177c Mon Sep 17 00:00:00 2001 From: Wim Pomp Date: Mon, 14 Oct 2024 19:03:24 +0200 Subject: [PATCH 11/13] - switch black and white at ends of quantitative colormaps --- Cargo.toml | 2 +- py/tiffwrite/__init__.py | 8 ++--- src/lib.rs | 64 ++++++++++++++++++++-------------------- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c58af4c..d4a7d5f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tiffwrite" -version = "2024.10.2" +version = "2024.10.3" edition = "2021" [lib] diff --git a/py/tiffwrite/__init__.py b/py/tiffwrite/__init__.py index edcd6a2..c3b9328 100644 --- a/py/tiffwrite/__init__.py +++ b/py/tiffwrite/__init__.py @@ -138,15 +138,15 @@ def get_colormap(colormap: str) -> np.ndarray: if colormap.endswith('_r'): cm = cm[::-1] if colormap.startswith('glasbey') or colormap.endswith('glasbey'): - cm[0] = 0, 0, 0 - cm[-1] = 255, 255, 255 + cm[0] = 255, 255, 255 + cm[-1] = 0, 0, 0 else: cmap = matplotlib.colormaps.get_cmap(colormap) if cmap.N < 256: - cm = (255 * np.vstack(((0, 0, 0), + cm = (255 * np.vstack(((1, 1, 1), matplotlib.cm.ScalarMappable(matplotlib.colors.Normalize(1, 254), cmap).to_rgba(np.arange(1, 254))[:, :3], - (1, 1, 1)))).astype('uint8') + (0, 0, 0)))).astype('uint8') else: cm = (255 * matplotlib.cm.ScalarMappable(matplotlib.colors.Normalize(0, 255), cmap) .to_rgba(np.arange(256))[:, :3]).astype('uint8') diff --git a/src/lib.rs b/src/lib.rs index 37e33c0..eef6968 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -123,20 +123,20 @@ impl Tag { } } - pub fn byte(code: u16, byte: &Vec) -> Self { - Tag::new(code, byte.to_owned(), 1) + pub fn byte(code: u16, value: &Vec) -> Self { + Tag::new(code, value.to_owned(), 1) } - pub fn ascii(code: u16, ascii: &str) -> Self { - let mut bytes = ascii.as_bytes().to_vec(); + pub fn ascii(code: u16, value: &str) -> Self { + let mut bytes = value.as_bytes().to_vec(); bytes.push(0); Tag::new(code, bytes, 2) } - pub fn short(code: u16, short: &Vec) -> Self { + pub fn short(code: u16, value: &Vec) -> Self { Tag::new( code, - short + value .into_iter() .map(|x| x.to_le_bytes()) .flatten() @@ -145,10 +145,10 @@ impl Tag { ) } - pub fn long(code: u16, long: &Vec) -> Self { + pub fn long(code: u16, value: &Vec) -> Self { Tag::new( code, - long.into_iter() + value.into_iter() .map(|x| x.to_le_bytes()) .flatten() .collect(), @@ -156,10 +156,10 @@ impl Tag { ) } - pub fn rational(code: u16, rational: &Vec) -> Self { + pub fn rational(code: u16, value: &Vec) -> Self { Tag::new( code, - rational + value .into_iter() .map(|x| { u32::try_from(*x.denom()) @@ -175,18 +175,18 @@ impl Tag { ) } - pub fn sbyte(code: u16, sbyte: &Vec) -> Self { + pub fn sbyte(code: u16, value: &Vec) -> Self { Tag::new( code, - sbyte.iter().map(|x| x.to_le_bytes()).flatten().collect(), + value.iter().map(|x| x.to_le_bytes()).flatten().collect(), 6, ) } - pub fn sshort(code: u16, sshort: &Vec) -> Self { + pub fn sshort(code: u16, value: &Vec) -> Self { Tag::new( code, - sshort + value .into_iter() .map(|x| x.to_le_bytes()) .flatten() @@ -195,10 +195,10 @@ impl Tag { ) } - pub fn slong(code: u16, slong: &Vec) -> Self { + pub fn slong(code: u16, value: &Vec) -> Self { Tag::new( code, - slong + value .into_iter() .map(|x| x.to_le_bytes()) .flatten() @@ -207,10 +207,10 @@ impl Tag { ) } - pub fn srational(code: u16, srational: &Vec) -> Self { + pub fn srational(code: u16, value: &Vec) -> Self { Tag::new( code, - srational + value .into_iter() .map(|x| { i32::try_from(*x.denom()) @@ -226,10 +226,10 @@ impl Tag { ) } - pub fn float(code: u16, float: &Vec) -> Self { + pub fn float(code: u16, value: &Vec) -> Self { Tag::new( code, - float + value .into_iter() .map(|x| x.to_le_bytes()) .flatten() @@ -238,10 +238,10 @@ impl Tag { ) } - pub fn double(code: u16, double: &Vec) -> Self { + pub fn double(code: u16, value: &Vec) -> Self { Tag::new( code, - double + value .into_iter() .map(|x| x.to_le_bytes()) .flatten() @@ -250,16 +250,16 @@ impl Tag { ) } - pub fn ifd(code: u16, ifd: &Vec) -> Self { + pub fn ifd(code: u16, value: &Vec) -> Self { Tag::new( code, - ifd.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), + value.into_iter().map(|x| x.to_le_bytes()).flatten().collect(), 13, ) } - pub fn unicode(code: u16, unicode: &str) -> Self { - let mut bytes: Vec = unicode + pub fn unicode(code: u16, value: &str) -> Self { + let mut bytes: Vec = value .encode_utf16() .map(|x| x.to_le_bytes()) .flatten() @@ -268,10 +268,10 @@ impl Tag { Tag::new(code, bytes, 14) } - pub fn complex(code: u16, complex: &Vec>) -> Self { + pub fn complex(code: u16, value: &Vec>) -> Self { Tag::new( code, - complex + value .into_iter() .map(|x| { x.re.to_le_bytes() @@ -285,10 +285,10 @@ impl Tag { ) } - pub fn long8(code: u16, long8: &Vec) -> Self { + pub fn long8(code: u16, value: &Vec) -> Self { Tag::new( code, - long8 + value .into_iter() .map(|x| x.to_le_bytes()) .flatten() @@ -297,10 +297,10 @@ impl Tag { ) } - pub fn slong8(code: u16, slong8: &Vec) -> Self { + pub fn slong8(code: u16, value: &Vec) -> Self { Tag::new( code, - slong8 + value .into_iter() .map(|x| x.to_le_bytes()) .flatten() From 83c0e221fbbdb02904bf929691b354a45837a28f Mon Sep 17 00:00:00 2001 From: Wim Pomp Date: Wed, 16 Oct 2024 14:26:30 +0200 Subject: [PATCH 12/13] - special IJTiffParallel class to help generate frames in parallel - warning now shows which frames are missing --- Cargo.toml | 4 +- py/tiffwrite/__init__.py | 121 +++++++++++++++++++++++++-------------- src/lib.rs | 19 +++--- 3 files changed, 90 insertions(+), 54 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d4a7d5f..a14a93e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tiffwrite" -version = "2024.10.3" +version = "2024.10.4" edition = "2021" [lib] @@ -17,7 +17,7 @@ zstd = "0.13.2" numpy = { version = "0.22.0", optional = true } [dependencies.pyo3] -version = "0.22.2" +version = "0.22.5" features = ["extension-module", "abi3-py310", "generate-import-lib", "anyhow", "multiple-pymethods"] optional = true diff --git a/py/tiffwrite/__init__.py b/py/tiffwrite/__init__.py index c3b9328..322236e 100644 --- a/py/tiffwrite/__init__.py +++ b/py/tiffwrite/__init__.py @@ -14,24 +14,19 @@ from tqdm.auto import tqdm from . import tiffwrite_rs as rs # noqa -__all__ = ['Header', 'IJTiffFile', 'IFD', 'FrameInfo', 'Tag', 'Strip', 'tiffwrite'] +__all__ = ['IJTiffFile', 'IJTiffParallel', 'FrameInfo', 'Tag', 'tiffwrite'] + + +Tag = rs.Tag +FrameInfo = tuple[np.ndarray, int, int, int] class Header: - pass + """ deprecated """ class IFD(dict): - pass - - -class Tag(rs.Tag): - pass - - -Strip = tuple[list[int], list[int]] -CZT = tuple[int, int, int] -FrameInfo = tuple[np.ndarray, None, CZT] + """ deprecated """ class TiffWriteWarning(UserWarning): @@ -40,6 +35,7 @@ class TiffWriteWarning(UserWarning): class IJTiffFile(rs.IJTiffFile): """ Writes a tiff file in a format that the BioFormats reader in Fiji understands. + Zstd compression is done in parallel using Rust. file: filename of the new tiff file shape: not used anymore dtype: datatype to use when saving to tiff @@ -97,38 +93,33 @@ class IJTiffFile(rs.IJTiffFile): def save(self, frame: ArrayLike, c: int, z: int, t: int, extratags: Sequence[Tag] = None) -> None: """ save a 2d numpy array to the tiff at channel=c, slice=z, time=t, with optional extra tif tags """ - for frame, _, (cn, zn, tn) in self.compress_frame(frame): - frame = np.asarray(frame).astype(self.dtype) - match self.dtype: - case np.uint8: - self.save_u8(frame, c + cn, z + zn, t + tn) - case np.uint16: - self.save_u16(frame, c + cn, z + zn, t + tn) - case np.uint32: - self.save_u32(frame, c + cn, z + zn, t + tn) - case np.uint64: - self.save_u64(frame, c + cn, z + zn, t + tn) - case np.int8: - self.save_i8(frame, c + cn, z + zn, t + tn) - case np.int16: - self.save_i16(frame, c + cn, z + zn, t + tn) - case np.int32: - self.save_i32(frame, c + cn, z + zn, t + tn) - case np.int64: - self.save_i64(frame, c + cn, z + zn, t + tn) - case np.float32: - self.save_f32(frame, c + cn, z + zn, t + tn) - case np.float64: - self.save_f64(frame, c + cn, z + zn, t + tn) - case _: - raise TypeError(f'Cannot save type {self.dtype}') - if extratags is not None: - for extra_tag in extratags: - self.append_extra_tag(extra_tag, (c, z, t)) - - def compress_frame(self, frame: ArrayLike) -> tuple[FrameInfo]: # noqa - """ backwards compatibility """ - return (frame, None, (0, 0, 0)), + frame = np.asarray(frame).astype(self.dtype) + match self.dtype: + case np.uint8: + self.save_u8(frame, c, z, t) + case np.uint16: + self.save_u16(frame, c, z, t) + case np.uint32: + self.save_u32(frame, c, z, t) + case np.uint64: + self.save_u64(frame, c, z, t) + case np.int8: + self.save_i8(frame, c, z, t) + case np.int16: + self.save_i16(frame, c, z, t) + case np.int32: + self.save_i32(frame, c, z, t) + case np.int64: + self.save_i64(frame, c, z, t) + case np.float32: + self.save_f32(frame, c, z, t) + case np.float64: + self.save_f64(frame, c, z, t) + case _: + raise TypeError(f'Cannot save type {self.dtype}') + if extratags is not None: + for extra_tag in extratags: + self.append_extra_tag(extra_tag, (c, z, t)) def get_colormap(colormap: str) -> np.ndarray: @@ -181,3 +172,45 @@ def tiffwrite(file: str | Path, data: np.ndarray, axes: str = 'TZCXY', dtype: DT for n in tqdm(product(*[range(i) for i in shape]), total=np.prod(shape), # type: ignore desc='Saving tiff', disable=not bar): f.save(data[n], *n) + + +try: + from parfor import ParPool, Task + from abc import abstractmethod, ABCMeta + from functools import wraps + + class IJTiffParallel(ParPool, metaclass=ABCMeta): + """ wraps IJTiffFile.save in a parallel pool, the method 'parallel' needs to be overloaded """ + + @abstractmethod + def parallel(self, frame: Any) -> Sequence[tuple[ArrayLike, int, int, int]]: + """ does something with frame in a parallel process, + and returns a sequence of frames and offsets to c, z and t to save in the tif """ + + @wraps(IJTiffFile.__init__) + def __init__(self, *args: Any, **kwargs: Any) -> None: + self.ijtifffile = IJTiffFile(*args, **kwargs) + super().__init__(self.parallel) # noqa + + def done(self, task: Task) -> None: + c, z, t = task.handle + super().done(task) + for frame, cn, zn, tn in self[c, z, t]: + self.ijtifffile.save(frame, c + cn, z + zn, t + tn) + + @wraps(IJTiffFile.close) + def close(self) -> None: + while len(self.tasks): + self.get_newest() + super().close() + self.ijtifffile.close() + + @wraps(IJTiffFile.save) + def save(self, frame: Any, c: int, z: int, t: int, extratags: Sequence[Tag] = None) -> None: + self[c, z, t] = frame + if extratags is not None: + for extra_tag in extratags: + self.ijtifffile.append_extra_tag(extra_tag, (c, z, t)) + +except ImportError: + IJTiffPool = None diff --git a/src/lib.rs b/src/lib.rs index eef6968..2df7557 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -803,7 +803,7 @@ impl IJTiffFile { } let mut where_to_write_next_ifd_offset = OFFSET - OFFSET_SIZE as u64; - let mut warn = false; + let mut warn = Vec::new(); let (samples_per_pixel, n_frames) = self.spp_and_n_frames(c_size, t_size, z_size); for frame_number in 0..n_frames { if let Some(frame) = self @@ -822,7 +822,7 @@ impl IJTiffFile { bytecounts.extend(frame_n.bytecounts.iter()); frame_count += 1; } else { - warn = true; + warn.push((frame_number, channel)); } } let mut ifd = IFD::new(); @@ -895,13 +895,16 @@ impl IJTiffFile { } where_to_write_next_ifd_offset = ifd.write(self, where_to_write_next_ifd_offset)?; } else { - warn = true; + warn.push((frame_number, 0)); } - if warn { - println!( - "Some frames were not added to the tif file, either you forgot them, \ - or an error occurred and the tif file was closed prematurely." - ) + if warn.len() > 0 { + println!("The following frames were not added to the tif file"); + for (frame_number, channel) in &warn { + let (c, z, t) = self.get_czt(*frame_number, *channel, c_size, z_size); + println!("{c}, {z}, {t}") + } + println!("Either you forgot them, \ + or an error occurred and the tif file was closed prematurely.") } } self.file From 4451ad7c61d303c4799f08842f5754f7587a18cb Mon Sep 17 00:00:00 2001 From: Wim Pomp Date: Wed, 16 Oct 2024 14:36:21 +0200 Subject: [PATCH 13/13] - change publish.yml --- .github/workflows/publish.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2006944..901cc38 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,16 +9,16 @@ jobs: name: publish needs: publish_wheels runs-on: ubuntu-latest - environment: pypi + environment: + name: pypi + url: https://pypi.org/p/tiffwrite + permissions: + id-token: write steps: - - uses: actions/download-artifact@v4 + - name: Download all the dists + uses: actions/download-artifact@v4 with: - # unpacks all tiffwrite artifacts into dist/ - pattern: tiffwrite-* - path: dist - merge-multiple: true - - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} - repository-url: https://upload.pypi.org/legacy/ \ No newline at end of file + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file