Issues with deepcopy

[floli]

Hello,
I have some issues with deepcopy, if objects are not pickable, e.g.

Traceback (most recent call last):
  File "mappings.py", line 45, in merge_mapping
    return dict(merge(*(p.dict() for p in outputs_sequence), strategy=Strategy.ADDITIVE))
  File "/home/user/.virtualenvs/dt_launcher/lib/python3.8/site-packages/mergedeep/mergedeep.py", line 100, in merge
    return reduce(partial(_deepmerge, strategy=strategy), sources, destination)
  File "/home/user/.virtualenvs/dt_launcher/lib/python3.8/site-packages/mergedeep/mergedeep.py", line 87, in _deepmerge
    dst[key] = deepcopy(src[key])
  File "/usr/lib/python3.8/copy.py", line 146, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python3.8/copy.py", line 230, in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
  File "/usr/lib/python3.8/copy.py", line 172, in deepcopy
    y = _reconstruct(x, memo, *rv)
  File "/usr/lib/python3.8/copy.py", line 270, in _reconstruct
    state = deepcopy(state, memo)
  File "/usr/lib/python3.8/copy.py", line 146, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python3.8/copy.py", line 230, in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
  File "/usr/lib/python3.8/copy.py", line 172, in deepcopy
    y = _reconstruct(x, memo, *rv)
  File "/usr/lib/python3.8/copy.py", line 270, in _reconstruct
    state = deepcopy(state, memo)
  File "/usr/lib/python3.8/copy.py", line 146, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python3.8/copy.py", line 230, in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
  File "/usr/lib/python3.8/copy.py", line 161, in deepcopy
    rv = reductor(4)
TypeError: cannot pickle '_thread.lock' object

I see two ways to approach that:

• Use "normal" copy, i.e. target = source, which afaik uses references as default
• Maybe add a parameter copy_fun with can default do deepcopy but can be overwritten by a function which uses shallow copy.

[jonathanabila]

    return merge({}, self._custom_request_kwargs, kwargs)
  File "/opt/python/mergedeep/mergedeep.py", line 106, in merge
    return reduce(partial(_deepmerge, strategy=strategy), sources, destination)
  File "/opt/python/mergedeep/mergedeep.py", line 94, in _deepmerge
    dst[key] = deepcopy(src[key])
  File "/var/lang/lib/python3.8/copy.py", line 172, in deepcopy
    y = _reconstruct(x, memo, *rv)
  File "/var/lang/lib/python3.8/copy.py", line 270, in _reconstruct
    state = deepcopy(state, memo)
  File "/var/lang/lib/python3.8/copy.py", line 146, in deepcopy
    y = copier(x, memo)
  File "/var/lang/lib/python3.8/copy.py", line 230, in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
  File "/var/lang/lib/python3.8/copy.py", line 146, in deepcopy
    y = copier(x, memo)
  File "/var/lang/lib/python3.8/copy.py", line 237, in _deepcopy_method
    return type(x)(x.__func__, deepcopy(x.__self__, memo))
  File "/var/lang/lib/python3.8/copy.py", line 172, in deepcopy
    y = _reconstruct(x, memo, *rv)
  File "/var/lang/lib/python3.8/copy.py", line 270, in _reconstruct
    state = deepcopy(state, memo)
  File "/var/lang/lib/python3.8/copy.py", line 146, in deepcopy
    y = copier(x, memo)
  File "/var/lang/lib/python3.8/copy.py", line 230, in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
  File "/var/lang/lib/python3.8/copy.py", line 161, in deepcopy
    rv = reductor(4)
TypeError: cannot pickle 'module' object

Probably not very useful, but it's happening inside a multithread context.

I tried to replicate the same scenario in a test, but it didn't work:

    def test__merge_kwargs(self, requests_mock):
        """
        Normally, private methods shouldn't have tests, however this one had problems in production,
        so now we have tests for it.
        """
        def auth_method(client, method, uri, headers, body):
            body = add_params_to_qs(
                body or "",
                [
                    ("client_id", client.client_id),
                    ("client_secret", client.client_secret or ""),
                ],
            )
            if "Content-Length" in headers:
                headers["Content-Length"] = str(len(body))

            headers["Content-Type"] = "application/x-www-form-urlencoded"
            return uri, headers, body

        client = OAuthClient("client_id", "client_secret", token_endpoint_auth_method="auth_method")
        client.register_client_auth_method(("auth_method", auth_method))

        requests_mock.post(self.url, json={"refresh_token": "123123"})

        client.refresh_token(self.url, refresh_token="123123")

Maybe be a shallow copy would work here too, I couldn't pinpoint the problem, but it seems to be the same issue with threads.