From 0721e2125cd0fedaf7b93427370c7176d1799aa1 Mon Sep 17 00:00:00 2001 From: Wim Pomp Date: Mon, 4 Jan 2021 12:25:19 +0100 Subject: [PATCH] - differentiate use of nP into nP and rP - default to using one process per cpu core - update requirements with versions --- parfor/__init__.py | 32 ++++++++++++++++++-------------- setup.py | 4 ++-- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/parfor/__init__.py b/parfor/__init__.py index ccc9bb7..df329ff 100644 --- a/parfor/__init__.py +++ b/parfor/__init__.py @@ -151,7 +151,7 @@ class tqdmm(tqdm): super(tqdmm, self).__exit__(exc_type, exc_value, traceback) def parfor(*args, **kwargs): - """ @parfor(iterator=None, args=(), kwargs={}, length=None, desc=None, bar=True, qbar=True, np1/3 serial=4, debug=False): + """ @parfor(iterator=None, args=(), kwargs={}, length=None, desc=None, bar=True, qbar=True, rP=1/3, serial=4, debug=False): decorator to parallize for-loops required arguments: @@ -165,7 +165,8 @@ def parfor(*args, **kwargs): desc: string with description of the progress bar bar: bool enable progress bar pbar: bool enable buffer indicator bar - nP: number of workers, default: number of cpu's/3 + rP: ratio workers to cpu cores, default: 1 + nP: number of workers, default: None, overrides rP if not None serial: switch to serial if number of tasks less than serial, default: 4 debug: if an error occurs in an iteration, return the erorr instead of retrying in the main process @@ -238,14 +239,16 @@ class parpool(object): """ Parallel processing with addition of iterations at any time and request of that result any time after that. The target function and its argument can be changed at any time. """ - def __init__(self, fun=None, args=None, kwargs=None, nP=0.33, bar=None, qbar=None, terminator=None, debug=False): - """ fun, args, kwargs: target function and its arguments and keyword arguments. - nP: number of workers, or fraction of cpu cores to use, default: 0.33. - bar, qbar: instances of tqdm and tqdmm to use for monitoring buffer and progress. """ - if nP is None: - self.nP = int(multiprocessing.cpu_count()/3) - elif nP<1: - self.nP = int(nP*multiprocessing.cpu_count()) + def __init__(self, fun=None, args=None, kwargs=None, rP=None, nP=None, bar=None, qbar=None, terminator=None, + debug=False): + """ fun, args, kwargs: target function and its arguments and keyword arguments + rP: ratio workers to cpu cores, default: 1 + nP: number of workers, default, None, overrides rP if not None + bar, qbar: instances of tqdm and tqdmm to use for monitoring buffer and progress """ + if rP is None and nP is None: + self.nP = int(multiprocessing.cpu_count()) + elif nP is None: + self.nP = int(round(rP * multiprocessing.cpu_count())) else: self.nP = int(nP) self.fun = fun or (lambda x: x) @@ -267,7 +270,7 @@ class parpool(object): self.bar = bar self.qbar = qbar if not self.qbar is None: - self.qbar.total = 3*nP + self.qbar.total = 3*self.nP @property def fun(self): @@ -445,7 +448,7 @@ class parpool(object): return obj def pmap(fun, iterable=None, args=None, kwargs=None, length=None, desc=None, bar=True, qbar=False, terminator=None, - nP=0.33, serial=4, debug=False): + rP=1, nP=None, serial=4, debug=False): """ map a function fun to each iteration in iterable best use: iterable is a generator and length is given to this function @@ -458,7 +461,8 @@ def pmap(fun, iterable=None, args=None, kwargs=None, length=None, desc=None, bar bar: bool enable progress bar pbar: bool enable buffer indicator bar terminator: function which is executed in each worker after all the work is done - nP: number of workers, default: number of cpu's/3 + rP: ratio workers to cpu cores, default: 1 + nP: number of workers, default, None, overrides rP if not None serial: switch to serial if number of tasks less than serial, default: 4 debug: if an error occurs in an iteration, return the erorr instead of retrying in the main process """ @@ -473,7 +477,7 @@ def pmap(fun, iterable=None, args=None, kwargs=None, length=None, desc=None, bar else: #parallel case with tqdmm(total=0, desc='Task buffer', disable=not qbar, leave=False) as qbar,\ tqdm(total=length, desc=desc, disable=not bar) as bar: - with parpool(fun, args, kwargs, nP, bar, qbar, terminator, debug) as p: + with parpool(fun, args, kwargs, rP, nP, bar, qbar, terminator, debug) as p: length = 0 for i, j in enumerate(iterable): #add work to the queue p[i] = j diff --git a/setup.py b/setup.py index fac7e58..3b08ccb 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="parfor", - version="2020.12.28", + version="2021.1.4", author="Wim Pomp", author_email="wimpomp@gmail.com", description="A package to mimic the use of parfor as done in Matlab.", @@ -20,5 +20,5 @@ setuptools.setup( "Operating System :: OS Independent", ], python_requires='>=2.7', - install_requires=['tqdm', 'dill'], + install_requires=['tqdm>=4.50.0', 'dill>=0.3.0'], ) \ No newline at end of file