Use our normal config setting style for the pickled metadata warning

Author: poodlewars

docs/mkdocs/docs/tutorials/metadata.md

@@ -54,7 +54,7 @@ Pickling can have [serious downsides](https://nedbatchelder.com/blog/202006/pick
 unpickle data written with one set of library versions from a client with a different set of library versions.
 
 Because of this, we log a warning when metadata gets pickled. You can disable the warning by setting an environment
-variable `PICKLED_METADATA_LOGLEVEL_DEBUG` to any value. The log message looks like:
+variable `ARCTICDB_PickledMetadata_loglevel_str` to `DEBUG`. The log message looks like:
 
 ```
 Pickling metadata - may not be readable by other clients

python/arcticdb/version_store/_normalization.py

@@ -19,6 +19,7 @@
 import pickle
 from abc import ABCMeta, abstractmethod
 
+from arcticdb_ext import get_config_string
 from pandas.api.types import is_integer_dtype
 from arcticc.pb2.descriptors_pb2 import UserDefinedMetadata, NormalizationMetadata, MsgPackSerialization
 from arcticc.pb2.storage_pb2 import VersionStoreConfig
@@ -79,7 +80,26 @@ def check_is_utc_if_newer_pandas(*args, **kwargs):
 NormalizedInput = NamedTuple("NormalizedInput", [("item", NPDDataFrame), ("metadata", NormalizationMetadata)])
 
 
-PICKLED_METADATA_LOGLEVEL = LogLevel.DEBUG if os.getenv("PICKLED_METADATA_LOGLEVEL_DEBUG") else LogLevel.WARN
+_PICKLED_METADATA_LOGLEVEL = None # set lazily with function below
+
+
+def get_pickled_metadata_loglevel():
+    global _PICKLED_METADATA_LOGLEVEL
+    if _PICKLED_METADATA_LOGLEVEL:
+        return _PICKLED_METADATA_LOGLEVEL
+
+    log_level = get_config_string("PickledMetadata.LogLevel")
+    expected_settings = ("DEBUG", "INFO", "WARN", "ERROR")
+    if log_level:
+        if log_level.upper() not in expected_settings:
+            log.warn(f"Expected PickledMetadata.LogLevel setting to be in {expected_settings} or absent but was {log_level}")
+            _PICKLED_METADATA_LOGLEVEL = LogLevel.WARN
+        else:
+            _PICKLED_METADATA_LOGLEVEL = getattr(LogLevel, log_level.upper())
+    else:
+        _PICKLED_METADATA_LOGLEVEL = LogLevel.WARN
+
+    return _PICKLED_METADATA_LOGLEVEL
 
 
 # To simplify unit testing of serialization logic. This maps the cpp _FrameData exposed object
@@ -1068,7 +1088,7 @@ def read(data, pickled_in_python2=False):
 
     @staticmethod
     def write(obj):
-        log.log(PICKLED_METADATA_LOGLEVEL, f"Pickling metadata - may not be readable by other clients")
+        log.log(get_pickled_metadata_loglevel(), f"Pickling metadata - may not be readable by other clients")
         return pickle.dumps(obj, protocol=PICKLE_PROTOCOL)
 
 

python/tests/integration/arcticdb/version_store/test_metadata_support.py

@@ -7,6 +7,7 @@
 """
 import numpy as np
 import pandas as pd
+from arcticdb_ext import set_config_string, unset_config_string
 from pandas import DataFrame, Timestamp
 import pytest
 
@@ -32,6 +33,36 @@ def test_rt_df_with_small_meta(object_and_mem_and_lmdb_version_store):
     assert meta == vit.metadata
 
 
+@pytest.mark.parametrize("log_level", ("error", "warn", "debug", "info", "ERROR", "eRror", "", None))
+def test_pickled_metadata_warning(lmdb_version_store_v1, log_level):
+    import arcticdb.version_store._normalization as norm
+    norm._PICKLED_METADATA_LOGLEVEL = None
+    if log_level is not None:
+        set_config_string("PickledMetadata.LogLevel", log_level)
+    lib = lmdb_version_store_v1
+    df = DataFrame(data=["A", "B", "C"])
+    meta = df
+    lib.write("pandas", df, metadata=meta)
+    vit = lib.read("pandas")
+    assert_frame_equal(df, vit.data)
+    assert_frame_equal(df, vit.metadata)
+    unset_config_string("PickledMetadata.LogLevel")
+
+
+def test_pickled_metadata_warning_bad_config(lmdb_version_store_v1):
+    """Don't block writes just because they set this wrong."""
+    import arcticdb.version_store._normalization as norm
+    norm._PICKLED_METADATA_LOGLEVEL = None
+    set_config_string("PickledMetadata.LogLevel", "cat")
+    lib = lmdb_version_store_v1
+    df = DataFrame(data=["A", "B", "C"])
+    meta = df
+    lib.write("pandas", df, metadata=meta)
+    vit = lib.read("pandas")
+    assert_frame_equal(df, vit.data)
+    assert_frame_equal(df, vit.metadata)
+
+
 def test_rt_df_with_humonguous_meta(object_and_mem_and_lmdb_version_store):
     with pytest.raises(ArcticDbNotYetImplemented):
         from arcticdb.version_store._normalization import _MAX_USER_DEFINED_META as MAX