wim/issfile
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- parallel reading when writing tiffs

Browse Source 
- add version argument to distinguish between versions of the .iss-pt file
- update readme
This commit is contained in:
Wim Pomp
2022-10-12 17:15:20 +02:00
parent 40b314d627
commit 13cbcfee92
4 changed files with 100 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
README.md
View File

@@ -1,2 +1,29 @@
# issfile
LIbrary for opening ISS (iss.com) files and conversion to tiff.
Library for opening [ISS](https://iss.com) files and their conversion to tiff.
## Installation
pip install git+https://github.com/wimpomp/issfile.git
## Converting .iss-pt files to .tiff files
iss2tiff --help
iss2tiff file.iss-pt
this will create file.tiff and file.carpet.tiff containing images and carpets respectively.
Metadata is also saved in the tiffs in the description tag.
## Use as a library
from matplotlib import pyplot as plt
from issfile import IssFile
with IssFile(file) as iss:
image = iss.get_image(c=1, t=5)
carpet = iss.get_carpet(c=1, t=5)
plt.figure()
plt.imshow(image)
plt.figure()
plt.plot(carpet.sum(1))

BIN
example_v388.iss-pt Executable file
View File

Binary file not shown.

100
issfile/__init__.py
View File

@@ -1,18 +1,21 @@
import sys
import zipfile
import re
import pickle
import xml.etree.ElementTree as ET
import numpy as np
import re
from struct import unpack
from tiffwrite import IJTiffFile
from tqdm.auto import tqdm
from itertools import product
from yaml import safe_load
from argparse import ArgumentParser
class IssTrack:
def __init__(self, file):
self.zip = zipfile.ZipFile(file)
class IssFile:
def __init__(self, file, version=388):
self.file = file
self.version = version
self.zip = zipfile.ZipFile(self.file)
self.data = self.zip.open('data/PrimaryDecayData.bin')
self.metadata = ET.fromstring(self.zip.read('dataProps/Core.xml'))
dimensions = self.metadata.find('Dimensions')
@@ -24,19 +27,16 @@ class IssTrack:
self.shape = size_x, size_y, size_c, size_t // 2 + size_t % 2, size_t // 2
self.exposure_time = float(self.metadata.find('FrameIntervalTime').text)
self.pxsize = float(self.metadata.find('Boundary').find('FrameWidth').text) / self.shape[0]
self.alba_metadata = safe_load('\n'.join([IssTrack.parse_line(line)
self.alba_metadata = safe_load('\n'.join([IssFile.parse_line(line)
for line in self.metadata.find('AlbaSystemSettings').find('withComments').text.splitlines()]))
particle_tracking = self.alba_metadata['ParticleTracking']
self.points_per_orbit = particle_tracking['ScanCirclePointCount']
self.n_orbits = particle_tracking['OrbitCountPerTrack']
self.orbit_time = particle_tracking['PacketTrackTime_ms']
self.start_times = [float(re.match(r'[.\d]+', value).group(0))
for key, value in self.alba_metadata.items()
if re.match(r'Time Series (\d+) started at', key)]
self.time_interval = np.mean(np.diff(self.start_times)[1::2])
self.delta = self.shape[2] * 2 ** int(np.ceil(np.log2(self.time_interval * 1000 / self.orbit_time)))
self.delta = 768 # TODO: figure out if this number is the same for all files
self.orbits_per_cycle = particle_tracking['OrbitCountPerTrack']
self.radius = particle_tracking['ScanRadius_um']
self.orbit_pxsize = self.radius * 2 * np.pi / self.points_per_orbit
self.data_bytes_len = self.zip.getinfo('data/PrimaryDecayData.bin').file_size
self.delta = self.data_bytes_len // (self.shape[0] * self.shape[1] * self.shape[2] *
(self.shape[3] + self.shape[4]))
def __enter__(self):
return self
@@ -44,6 +44,14 @@ class IssTrack:
def __exit__(self, *args, **kwargs):
self.close()
def __getstate__(self):
return {key: value for key, value in self.__dict__.items() if key not in ('zip', 'data')}
def __setstate__(self, state):
self.__dict__.update(state)
self.zip = zipfile.ZipFile(self.file)
self.data = self.zip.open('data/PrimaryDecayData.bin')
def close(self):
try:
self.data.close()
@@ -61,32 +69,58 @@ class IssTrack:
data.append(unpack('<I', self.data.read(4)))
return np.reshape(data, self.shape[:2])
def get_carpet(self, c, t):
def get_carpet(self, c, t, min_n_lines=1):
assert c < self.shape[2] and t < self.shape[4],\
f'carpet {c = }, {t = } not in shape {self.shape[2]}, {self.shape[4]}'
frame = c + (2 * t + 1) * self.shape[2]
frame_bytes = self.shape[0] * self.shape[1] * self.delta
data = []
self.data.seek(frame * frame_bytes)
for i in range(frame_bytes // (2 * self.points_per_orbit * self.n_orbits)):
line = [unpack('<H', self.data.read(2))[0] for _ in range(self.points_per_orbit * self.n_orbits)]
if np.any(line):
for i in range(frame_bytes // (2 * self.points_per_orbit * self.orbits_per_cycle)):
line = [unpack('<H', self.data.read(2))[0] for _ in range(self.points_per_orbit * self.orbits_per_cycle)]
if self.version == 388:
_position_and_time = self.data.read(16)
if np.any(line) or i < min_n_lines:
data.append(line)
else:
break
return np.vstack(data)
return np.vstack(data).reshape((-1, self.points_per_orbit))
@property
def tiff_writer(self):
class TiffFile(IJTiffFile):
""" Modify the tiff writer so that it can read from the .iss-pt file by itself in parallel processes. """
def __init__(self, *args, iss, bar, **kwargs):
if 'processes' not in kwargs:
kwargs['processes'] = 'all'
super().__init__(*args, **kwargs)
self.iss = pickle.dumps(iss)
self.bar = bar
def update(self):
self.bar.update()
def compress_frame(self, frame):
if isinstance(self.iss, bytes):
self.iss = pickle.loads(self.iss)
frame = self.iss.get_carpet(*frame[1:]) if frame[0] else self.iss.get_image(*frame[1:])
return super().compress_frame(frame.astype(self.dtype))
return TiffFile
def save_images_as_tiff(self, file):
with IJTiffFile(file, (self.shape[2], 1, self.shape[3]),
pxsize=self.pxsize, comment=ET.tostring(self.metadata)) as tif:
for c, t in tqdm(product(range(self.shape[2]), range(self.shape[3])), total=self.shape[2] * self.shape[3]):
tif.save(self.get_image(c, t), c, 0, t)
with tqdm(total=self.shape[2] * self.shape[3], desc='Writing images') as bar:
with self.tiff_writer(file, (self.shape[2], 1, self.shape[3]), iss=self, bar=bar,
pxsize=self.pxsize, comment=ET.tostring(self.metadata)) as tif:
for c, t in product(range(self.shape[2]), range(self.shape[3])):
tif.save(np.array((0, c, t)), c, 0, t)
def save_carpets_as_tiff(self, file):
with IJTiffFile(file, (self.shape[2], 1, self.shape[4]),
pxsize=self.pxsize, comment=ET.tostring(self.metadata)) as tif:
for c, t in tqdm(product(range(self.shape[2]), range(self.shape[4])), total=self.shape[2] * self.shape[4]):
tif.save(self.get_carpet(c, t), c, 0, t)
with tqdm(total=self.shape[2] * self.shape[4], desc='Writing carpets') as bar:
with self.tiff_writer(file, (self.shape[2], 1, self.shape[4]), iss=self, bar=bar,
pxsize=self.orbit_pxsize, comment=ET.tostring(self.metadata)) as tif:
for c, t in product(range(self.shape[2]), range(self.shape[4])):
tif.save(np.array((1, c, t)), c, 0, t)
@staticmethod
def parse_line(line):
@@ -99,8 +133,14 @@ class IssTrack:
def main():
for file in sys.argv[1:]:
with IssTrack(file) as iss_file:
parser = ArgumentParser(description='Convert .iss-pt files into .tiff files.')
parser.add_argument('files', help='files to be converted', nargs='*')
parser.add_argument('-v', '--version', type=int, default=388,
help='version of VistaVision with which the .iss-pt was written, default: 388')
args = parser.parse_args()
for file in args.files:
with IssFile(file, args.version) as iss_file:
iss_file.save_images_as_tiff(file.replace('.iss-pt', '.tif'))
iss_file.save_carpets_as_tiff(file.replace('.iss-pt', '.carpet.tif'))

4
setup.py
View File

@@ -5,7 +5,7 @@ with open('README.md', 'r') as fh:
setuptools.setup(
name='issfile',
version='2022.10.12',
version='2022.10.1',
author='Wim Pomp @ Lenstra lab NKI',
author_email='w.pomp@nki.nl',
description='Open ISS files.',
@@ -19,6 +19,6 @@ setuptools.setup(
'Operating System :: OS Independent',
],
python_requires='>=3.8',
install_requires=['numpy', 'tiffwrite>=2022.10.0'],
install_requires=['numpy', 'tqdm', 'pyyaml', 'tiffwrite>=2022.10.1'],
entry_points={'console_scripts': ['iss2tiff=issfile:main']}
)