kwargs are copied for every worker, which is different to partial

[Karol-G]

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from multiprocessing import Pool\n",
    "from functools import partial\n",
    "\n",
    "from mzarr import Mzarr\n",
    "import numpy as np\n",
    "from patchly.sampler import GridSampler\n",
    "from tqdmp import tqdmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img = Mzarr(\"/home/s522r/network_drives/E132-Projekte/Projects/Helmholtz_Imaging_ACVL/HMGU_2022_DIADEM/for_karol/2024_02_07_S001.mzarr\", mode=\"r\").numpy()\n",
    "\n",
    "sampler = GridSampler(None, patch_size=(256, 256), step_size=(128, 128), spatial_size=img.shape[1:], spatial_first=False)\n",
    "mask = np.load(\"/home/s522r/network_drives/E132-Projekte/Projects/Helmholtz_Imaging_ACVL/HMGU_2022_DIADEM/for_karol/2024_02_07_S001.npy\")#, mmap_mode=\"r\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_valid_idx(arg, mask):\n",
    "    i, sampler_iter = arg\n",
    "    patch_bbox = sampler_iter\n",
    "   \n",
    "    patch_mask = mask[patch_bbox[0,0]:patch_bbox[0,1], patch_bbox[1,0]:patch_bbox[1,1]]\n",
    "    if not np.any((patch_mask==1) | (patch_mask==2)):\n",
    "        return i\n",
    "\n",
    "pool  = Pool(4)\n",
    "methd = partial(find_valid_idx, mask = mask)\n",
    "res = pool.map(methd, zip(range(len(sampler)), sampler))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_workers = 4\n",
    "\n",
    "def find_valid_idx(i, sampler_iter):\n",
    "    patch_bbox = sampler_iter\n",
    "\n",
    "    patch_mask = mask[patch_bbox[0,0]:patch_bbox[0,1], patch_bbox[1,0]:patch_bbox[1,1]]\n",
    "    if not np.any((patch_mask==1) | (patch_mask==2)):\n",
    "        return i\n",
    "    \n",
    "    \n",
    "valid_idx = tqdmp(find_valid_idx, (range(len(sampler)), sampler), num_workers, mult_iter=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_workers = 0\n",
    "\n",
    "def find_valid_idx(i, sampler_iter, mask):\n",
    "    patch_bbox = sampler_iter\n",
    "    \n",
    "    patch_mask = mask[patch_bbox[0,0]:patch_bbox[0,1], patch_bbox[1,0]:patch_bbox[1,1]]\n",
    "    if not np.any((patch_mask==1) | (patch_mask==2)):\n",
    "        return i\n",
    "    \n",
    "    \n",
    "valid_idx = tqdmp(find_valid_idx, (range(len(sampler)), sampler), num_workers, mult_iter=True, mask=mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "clam2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}