- parallel reading when writing tiffs

- add version argument to distinguish between versions of the .iss-pt file
- update readme

README.md

@@ -1,2 +1,29 @@
 # issfile
-LIbrary for opening ISS (iss.com) files and conversion to tiff.
+Library for opening [ISS](https://iss.com) files and their conversion to tiff.
+
+## Installation
+
+    pip install git+https://github.com/wimpomp/issfile.git
+
+## Converting .iss-pt files to .tiff files
+
+    iss2tiff --help
+
+    iss2tiff file.iss-pt
+
+this will create file.tiff and file.carpet.tiff containing images and carpets respectively.
+Metadata is also saved in the tiffs in the description tag.
+
+## Use as a library
+
+    from matplotlib import pyplot as plt
+    from issfile import IssFile
+
+    with IssFile(file) as iss:
+        image = iss.get_image(c=1, t=5)
+        carpet = iss.get_carpet(c=1, t=5)
+    
+        plt.figure()
+        plt.imshow(image)
+        plt.figure()
+        plt.plot(carpet.sum(1))

issfile/__init__.py

@@ -1,18 +1,21 @@
-import sys
 import zipfile
+import re
+import pickle
 import xml.etree.ElementTree as ET
 import numpy as np
-import re
 from struct import unpack
 from tiffwrite import IJTiffFile
 from tqdm.auto import tqdm
 from itertools import product
 from yaml import safe_load
+from argparse import ArgumentParser
 
 
-class IssTrack:
+class IssFile:
-    def __init__(self, file):
+    def __init__(self, file, version=388):
-        self.zip = zipfile.ZipFile(file)
+        self.file = file
+        self.version = version
+        self.zip = zipfile.ZipFile(self.file)
         self.data = self.zip.open('data/PrimaryDecayData.bin')
         self.metadata = ET.fromstring(self.zip.read('dataProps/Core.xml'))
         dimensions = self.metadata.find('Dimensions')
@@ -24,19 +27,16 @@ class IssTrack:
         self.shape = size_x, size_y, size_c, size_t // 2 + size_t % 2, size_t // 2
         self.exposure_time = float(self.metadata.find('FrameIntervalTime').text)
         self.pxsize = float(self.metadata.find('Boundary').find('FrameWidth').text) / self.shape[0]
-
+        self.alba_metadata = safe_load('\n'.join([IssFile.parse_line(line)
-        self.alba_metadata = safe_load('\n'.join([IssTrack.parse_line(line)
             for line in self.metadata.find('AlbaSystemSettings').find('withComments').text.splitlines()]))
         particle_tracking = self.alba_metadata['ParticleTracking']
         self.points_per_orbit = particle_tracking['ScanCirclePointCount']
-        self.n_orbits = particle_tracking['OrbitCountPerTrack']
+        self.orbits_per_cycle = particle_tracking['OrbitCountPerTrack']
-        self.orbit_time = particle_tracking['PacketTrackTime_ms']
+        self.radius = particle_tracking['ScanRadius_um']
-        self.start_times = [float(re.match(r'[.\d]+', value).group(0))
+        self.orbit_pxsize = self.radius * 2 * np.pi / self.points_per_orbit
-                            for key, value in self.alba_metadata.items()
+        self.data_bytes_len = self.zip.getinfo('data/PrimaryDecayData.bin').file_size
-                            if re.match(r'Time Series (\d+) started at', key)]
+        self.delta = self.data_bytes_len // (self.shape[0] * self.shape[1] * self.shape[2] *
-        self.time_interval = np.mean(np.diff(self.start_times)[1::2])
+                                             (self.shape[3] + self.shape[4]))
-        self.delta = self.shape[2] * 2 ** int(np.ceil(np.log2(self.time_interval * 1000 / self.orbit_time)))
-        self.delta = 768  # TODO: figure out if this number is the same for all files
 
     def __enter__(self):
         return self
@@ -44,6 +44,14 @@ class IssTrack:
     def __exit__(self, *args, **kwargs):
         self.close()
 
+    def __getstate__(self):
+        return {key: value for key, value in self.__dict__.items() if key not in ('zip', 'data')}
+
+    def __setstate__(self, state):
+        self.__dict__.update(state)
+        self.zip = zipfile.ZipFile(self.file)
+        self.data = self.zip.open('data/PrimaryDecayData.bin')
+
     def close(self):
         try:
             self.data.close()
@@ -61,32 +69,58 @@ class IssTrack:
             data.append(unpack('<I', self.data.read(4)))
         return np.reshape(data, self.shape[:2])
 
-    def get_carpet(self, c, t):
+    def get_carpet(self, c, t, min_n_lines=1):
         assert c < self.shape[2] and t < self.shape[4],\
             f'carpet {c = }, {t = } not in shape {self.shape[2]}, {self.shape[4]}'
         frame = c + (2 * t + 1) * self.shape[2]
         frame_bytes = self.shape[0] * self.shape[1] * self.delta
         data = []
         self.data.seek(frame * frame_bytes)
-        for i in range(frame_bytes // (2 * self.points_per_orbit * self.n_orbits)):
+        for i in range(frame_bytes // (2 * self.points_per_orbit * self.orbits_per_cycle)):
-            line = [unpack('<H', self.data.read(2))[0] for _ in range(self.points_per_orbit * self.n_orbits)]
+            line = [unpack('<H', self.data.read(2))[0] for _ in range(self.points_per_orbit * self.orbits_per_cycle)]
-            if np.any(line):
+            if self.version == 388:
+                _position_and_time = self.data.read(16)
+            if np.any(line) or i < min_n_lines:
                 data.append(line)
             else:
                 break
-        return np.vstack(data)
+        return np.vstack(data).reshape((-1, self.points_per_orbit))
+
+    @property
+    def tiff_writer(self):
+        class TiffFile(IJTiffFile):
+            """ Modify the tiff writer so that it can read from the .iss-pt file by itself in parallel processes. """
+            def __init__(self, *args, iss, bar, **kwargs):
+                if 'processes' not in kwargs:
+                    kwargs['processes'] = 'all'
+                super().__init__(*args, **kwargs)
+                self.iss = pickle.dumps(iss)
+                self.bar = bar
+
+            def update(self):
+                self.bar.update()
+
+            def compress_frame(self, frame):
+                if isinstance(self.iss, bytes):
+                    self.iss = pickle.loads(self.iss)
+                frame = self.iss.get_carpet(*frame[1:]) if frame[0] else self.iss.get_image(*frame[1:])
+                return super().compress_frame(frame.astype(self.dtype))
+
+        return TiffFile
 
     def save_images_as_tiff(self, file):
-        with IJTiffFile(file, (self.shape[2], 1, self.shape[3]),
+        with tqdm(total=self.shape[2] * self.shape[3], desc='Writing  images') as bar:
+            with self.tiff_writer(file, (self.shape[2], 1, self.shape[3]), iss=self, bar=bar,
                                   pxsize=self.pxsize, comment=ET.tostring(self.metadata)) as tif:
-            for c, t in tqdm(product(range(self.shape[2]), range(self.shape[3])), total=self.shape[2] * self.shape[3]):
+                for c, t in product(range(self.shape[2]), range(self.shape[3])):
-                tif.save(self.get_image(c, t), c, 0, t)
+                    tif.save(np.array((0, c, t)), c, 0, t)
 
     def save_carpets_as_tiff(self, file):
-        with IJTiffFile(file, (self.shape[2], 1, self.shape[4]),
+        with tqdm(total=self.shape[2] * self.shape[4], desc='Writing carpets') as bar:
-                        pxsize=self.pxsize, comment=ET.tostring(self.metadata)) as tif:
+            with self.tiff_writer(file, (self.shape[2], 1, self.shape[4]), iss=self, bar=bar,
-            for c, t in tqdm(product(range(self.shape[2]), range(self.shape[4])), total=self.shape[2] * self.shape[4]):
+                                  pxsize=self.orbit_pxsize, comment=ET.tostring(self.metadata)) as tif:
-                tif.save(self.get_carpet(c, t), c, 0, t)
+                for c, t in product(range(self.shape[2]), range(self.shape[4])):
+                    tif.save(np.array((1, c, t)), c, 0, t)
 
     @staticmethod
     def parse_line(line):
@@ -99,8 +133,14 @@ class IssTrack:
 
 
 def main():
-    for file in sys.argv[1:]:
+    parser = ArgumentParser(description='Convert .iss-pt files into .tiff files.')
-        with IssTrack(file) as iss_file:
+    parser.add_argument('files', help='files to be converted', nargs='*')
+    parser.add_argument('-v', '--version', type=int, default=388,
+                        help='version of VistaVision with which the .iss-pt was written, default: 388')
+    args = parser.parse_args()
+
+    for file in args.files:
+        with IssFile(file, args.version) as iss_file:
             iss_file.save_images_as_tiff(file.replace('.iss-pt', '.tif'))
             iss_file.save_carpets_as_tiff(file.replace('.iss-pt', '.carpet.tif'))
 

setup.py

@@ -5,7 +5,7 @@ with open('README.md', 'r') as fh:
 
 setuptools.setup(
     name='issfile',
-    version='2022.10.12',
+    version='2022.10.1',
     author='Wim Pomp @ Lenstra lab NKI',
     author_email='w.pomp@nki.nl',
     description='Open ISS files.',
@@ -19,6 +19,6 @@ setuptools.setup(
         'Operating System :: OS Independent',
     ],
     python_requires='>=3.8',
-    install_requires=['numpy', 'tiffwrite>=2022.10.0'],
+    install_requires=['numpy', 'tqdm', 'pyyaml', 'tiffwrite>=2022.10.1'],
     entry_points={'console_scripts': ['iss2tiff=issfile:main']}
 )