Skip to content

Commit

Permalink
Fix
Browse files Browse the repository at this point in the history
  • Loading branch information
jonathanmetzman committed Dec 27, 2024
1 parent 8a38750 commit 7a6e60e
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 20 deletions.
21 changes: 2 additions & 19 deletions src/clusterfuzz/_internal/base/concurrency.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,33 +21,16 @@
POOL_SIZE = multiprocessing.cpu_count()


class SingleThreadPool:
"""Single thread pool for when it's not worth using Python's thread
implementation."""

def __init__(self, size):
del size

def map(self, f, l):
return list(map(f, l))

def imap_unordered(self, f, l):
return list(map(f, l))


@contextlib.contextmanager
def make_pool(pool_size=POOL_SIZE, cpu_bound=False, max_pool_size=None):
def make_pool(pool_size=POOL_SIZE, max_pool_size=None):
"""Returns a pool that can (usually) execute tasks concurrently."""
if max_pool_size is not None:
pool_size = min(pool_size, max_pool_size)

# Don't use processes on Windows and unittests to avoid hangs.
if (environment.get_value('PY_UNITTESTS') or
environment.platform() == 'WINDOWS'):
if cpu_bound:
yield SingleThreadPool(pool_size)
else:
yield futures.ThreadPoolExecutor(pool_size)
yield futures.ThreadPoolExecutor(pool_size)
else:
yield futures.ProcessPoolExecutor(pool_size)

Expand Down
2 changes: 1 addition & 1 deletion src/clusterfuzz/_internal/google_cloud_utils/storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -1383,7 +1383,7 @@ def parallel_map(func, argument_list):
will OOM."""
max_size = 2
timeout = 120
with concurrency.make_pool(cpu_bound=True, max_pool_size=max_size) as pool:
with concurrency.make_pool(max_pool_size=max_size) as pool:
calls = {pool.submit(func, argument) for argument in argument_list}
while calls:
finished_calls, _ = futures.wait(
Expand Down

0 comments on commit 7a6e60e

Please sign in to comment.