- add PoolSingleton to __init__
- update readme
This commit is contained in:
21
README.md
21
README.md
@@ -12,8 +12,13 @@ Tested on linux, Windows and OSX with python 3.10 and 3.12.
|
|||||||
- Easy to use
|
- Easy to use
|
||||||
- Using dill instead of pickle: a lot more objects can be used when parallelizing
|
- Using dill instead of pickle: a lot more objects can be used when parallelizing
|
||||||
- Progress bars are built-in
|
- Progress bars are built-in
|
||||||
|
- Automatically use multithreading instead of multiprocessing when the GIL is disabled
|
||||||
|
|
||||||
## How it works
|
## How it works
|
||||||
|
This depends on whether the GIL is currently disabled or not. Disabling the GIL in Python is currently an experimental
|
||||||
|
feature in Python3.13, and not the standard.
|
||||||
|
|
||||||
|
### Python with GIL enabled
|
||||||
The work you want parfor to do is divided over a number of processes. These processes are started by parfor and put
|
The work you want parfor to do is divided over a number of processes. These processes are started by parfor and put
|
||||||
together in a pool. This pool is reused when you want parfor to do more work, or shut down when no new work arrives
|
together in a pool. This pool is reused when you want parfor to do more work, or shut down when no new work arrives
|
||||||
within 10 minutes.
|
within 10 minutes.
|
||||||
@@ -24,6 +29,12 @@ worker is requesting it. The manager deletes objects automatically when they're
|
|||||||
|
|
||||||
When the work is done the result is sent back for collection in the main process.
|
When the work is done the result is sent back for collection in the main process.
|
||||||
|
|
||||||
|
### Python with GIL disabled
|
||||||
|
The work you want parfor to do is given to a new thread. These threads are started by parfor and put together in a pool.
|
||||||
|
The threads and pool are not reused and closed automatically when done.
|
||||||
|
|
||||||
|
When the work is done a message is sent to the main thread to update the status of the pool.
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
`pip install parfor`
|
`pip install parfor`
|
||||||
|
|
||||||
@@ -35,12 +46,14 @@ an iterator.
|
|||||||
tqdm, dill
|
tqdm, dill
|
||||||
|
|
||||||
## Limitations
|
## Limitations
|
||||||
Objects passed to the pool need to be dillable (dill needs to serialize them). Generators and SwigPyObjects are examples
|
If you're using Python with the GIL enabaled, then objects passed to the pool need to be dillable (dill needs to
|
||||||
of objects that cannot be used. They can be used however, for the iterator argument when using parfor, but its
|
serialize them). Generators and SwigPyObjects are examples of objects that cannot be used. They can be used however, for
|
||||||
iterations need to be dillable. You might be able to make objects dillable anyhow using `dill.register` or with
|
the iterator argument when using parfor, but its iterations need to be dillable. You might be able to make objects
|
||||||
`__reduce__`, `__getstate__`, etc.
|
dillable anyhow using `dill.register` or with `__reduce__`, `__getstate__`, etc.
|
||||||
|
|
||||||
## Arguments
|
## Arguments
|
||||||
|
To functions `parfor.parfor`, `parfor.pmap` and `parfor.gmap`.
|
||||||
|
|
||||||
### Required:
|
### Required:
|
||||||
fun: function taking arguments: iteration from iterable, other arguments defined in args & kwargs
|
fun: function taking arguments: iteration from iterable, other arguments defined in args & kwargs
|
||||||
iterable: iterable or iterator from which an item is given to fun as a first argument
|
iterable: iterable or iterator from which an item is given to fun as a first argument
|
||||||
|
|||||||
@@ -19,23 +19,18 @@ Result = TypeVar('Result')
|
|||||||
Iteration = TypeVar('Iteration')
|
Iteration = TypeVar('Iteration')
|
||||||
|
|
||||||
|
|
||||||
|
def select():
|
||||||
|
return nogil if hasattr(sys, '_is_gil_enabled') and not sys._is_gil_enabled() else gil # noqa
|
||||||
|
|
||||||
|
|
||||||
class ParPool:
|
class ParPool:
|
||||||
def __new__(cls, *args, **kwargs):
|
def __new__(cls, *args, **kwargs):
|
||||||
try:
|
return select().ParPool(*args, **kwargs)
|
||||||
if not sys._is_gil_enabled(): # noqa
|
|
||||||
return nogil.ParPool(*args, **kwargs)
|
|
||||||
except AttributeError:
|
|
||||||
pass
|
|
||||||
return gil.ParPool(*args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def nested():
|
class PoolSingleton:
|
||||||
try:
|
def __new__(cls, *args, **kwargs):
|
||||||
if not sys._is_gil_enabled(): # noqa
|
return select().PoolSingleton(*args, **kwargs)
|
||||||
return nogil.Worker.nested
|
|
||||||
except AttributeError:
|
|
||||||
pass
|
|
||||||
return gil.Worker.nested
|
|
||||||
|
|
||||||
|
|
||||||
class Chunks(Iterable):
|
class Chunks(Iterable):
|
||||||
@@ -222,7 +217,7 @@ def gmap(fun: Callable[[Iteration, Any, ...], Result], iterable: Iterable[Iterat
|
|||||||
bar_kwargs['desc'] = desc
|
bar_kwargs['desc'] = desc
|
||||||
if 'disable' not in bar_kwargs:
|
if 'disable' not in bar_kwargs:
|
||||||
bar_kwargs['disable'] = not bar
|
bar_kwargs['disable'] = not bar
|
||||||
if serial is True or (serial is None and len(iterable) < min(cpu_count, 4)) or nested(): # serial case
|
if serial is True or (serial is None and len(iterable) < min(cpu_count, 4)) or select().Worker.nested: # serial case
|
||||||
|
|
||||||
def tqdm_chunks(chunks: Chunks, *args, **kwargs) -> Iterable[Iteration]: # noqa
|
def tqdm_chunks(chunks: Chunks, *args, **kwargs) -> Iterable[Iteration]: # noqa
|
||||||
with tqdm(*args, **kwargs) as b:
|
with tqdm(*args, **kwargs) as b:
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[tool.poetry]
|
[tool.poetry]
|
||||||
name = "parfor"
|
name = "parfor"
|
||||||
version = "2024.10.0"
|
version = "2024.10.1"
|
||||||
description = "A package to mimic the use of parfor as done in Matlab."
|
description = "A package to mimic the use of parfor as done in Matlab."
|
||||||
authors = ["Wim Pomp <wimpomp@gmail.com>"]
|
authors = ["Wim Pomp <wimpomp@gmail.com>"]
|
||||||
license = "GPLv3"
|
license = "GPLv3"
|
||||||
|
|||||||
Reference in New Issue
Block a user