Merge pull request #261 from skalenetwork/optimize-config

Migrate configuration to Pydantic settings

Author: dmytrotkk

bounty_agent.py

@@ -25,11 +25,12 @@
 import logging
 import socket
 import time
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 import tenacity
 from apscheduler.events import EVENT_JOB_ERROR, EVENT_JOB_EXECUTED
 from apscheduler.schedulers.background import BackgroundScheduler
+from skale.core.settings import SkaleSettings, get_settings
 from skale.transactions.exceptions import TransactionError
 from web3.logs import DISCARD
 
@@ -56,10 +57,16 @@
 
 
 class BountyAgent:
-    def __init__(self, skale, node_id=None):
+    def __init__(self, skale, settings: SkaleSettings, node_id=None):
         self.agent_name = get_agent_name(self.__class__.__name__)
         self.logger = logging.getLogger(self.agent_name)
-        add_file_handler(self.logger, self.agent_name, node_id)
+        add_file_handler(
+            self.logger,
+            self.agent_name,
+            node_id,
+            str(settings.sgx_url),
+            str(settings.endpoint),
+        )
         self.logger.info(f'Initialization of {self.agent_name} ...')
         if node_id is None:
             self.id = get_id_from_config(NODE_CONFIG_FILEPATH)
@@ -90,7 +97,7 @@ def get_reward_date(self):
         except Exception as err:
             self.notifier.send(f'Cannot get reward date from SKALE Manager: {err}', MsgIcon.ERROR)
             raise
-        return datetime.utcfromtimestamp(reward_date)
+        return datetime.fromtimestamp(reward_date, timezone.utc)
 
     def get_bounty(self):
         try:
@@ -134,7 +141,7 @@ def job(self) -> None:
         reward_date = self.get_reward_date()
         last_block_number = self.skale.web3.eth.block_number
         block_data = call_retry(self.skale.web3.eth.get_block, last_block_number)
-        block_timestamp = datetime.utcfromtimestamp(block_data['timestamp'])
+        block_timestamp = datetime.fromtimestamp(block_data['timestamp'], timezone.utc)
         self.logger.info(f'Reward date: {reward_date}')
         self.logger.info(f'Block timestamp:  {block_timestamp}')
         if reward_date > block_timestamp:
@@ -145,7 +152,7 @@ def job(self) -> None:
     def job_listener(self, event):
         if event.exception:
             self.logger.info('"Get Bounty" job failed')
-            utc_now = datetime.utcnow()
+            utc_now = datetime.now(timezone.utc)
             self.scheduler.add_job(
                 self.job, 'date', run_date=utc_now + timedelta(seconds=DELAY_AFTER_ERR)
             )
@@ -156,7 +163,7 @@ def job_listener(self, event):
                 reward_date = self.get_reward_date()
                 self.notifier.send(f'Next reward date: {reward_date}', MsgIcon.BOUNTY)
             except Exception:
-                reward_date = datetime.utcnow() + timedelta(seconds=DELAY_AFTER_ERR)
+                reward_date = datetime.now(timezone.utc) + timedelta(seconds=DELAY_AFTER_ERR)
                 self.logger.info(f'Next try to get reward date: {reward_date}')
             self.scheduler.add_job(self.job, 'date', run_date=reward_date)
             self.scheduler.print_jobs()
@@ -165,7 +172,7 @@ def run(self) -> None:
         """Starts agent."""
         reward_date = self.get_reward_date()
         self.logger.info(f"Next reward date on agent's start: {reward_date}")
-        utc_now = datetime.utcnow()
+        utc_now = datetime.now(timezone.utc)
         if utc_now > reward_date:
             reward_date = utc_now
         self.scheduler.add_job(self.job, 'date', run_date=reward_date)
@@ -179,11 +186,12 @@ def stop(self):
 
 
 if __name__ == '__main__':
-    init_logger()
+    st = get_settings(SkaleSettings)
+    init_logger(str(st.sgx_url), str(st.endpoint))
     while True:
         try:
-            skale = init_skale()
-            bounty_agent = BountyAgent(skale)
+            skale = init_skale(st)
+            bounty_agent = BountyAgent(skale, settings=st)
             bounty_agent.run()
             while not bounty_agent.is_stopped:
                 time.sleep(1)

configs/__init__.py

@@ -1,19 +1,20 @@
 import os
+from pathlib import Path
+
+from skale.core.settings import SkaleSettings
 
 ENV = os.environ.get('ENV')
 
 LONG_LINE = '-' * 100
 
 NOTIFIER_URL = 'http://localhost:3007/send-tg-notification'
-SKALE_VOLUME_PATH = os.getenv('SKALE_VOLUME_PATH', '/skale_vol')
-NODE_DATA_PATH = os.getenv('NODE_DATA_PATH', '/skale_node_data')
+NODE_DATA_FOLDER_NAME: str = 'node_data'
 
-NODE_CONFIG_FILENAME = 'node_config.json'
-NODE_CONFIG_FILEPATH = os.path.join(NODE_DATA_PATH, NODE_CONFIG_FILENAME)
+SKALE_VOLUME_PATH: Path = Path(os.getenv('SKALE_VOLUME_PATH', '/skale_vol'))
+NODE_DATA_PATH: Path = SKALE_VOLUME_PATH / NODE_DATA_FOLDER_NAME
 
-STATE_FILENAME = os.getenv('STATE_FILENAME')
-STATE_BASE_PATH = os.path.join(NODE_DATA_PATH, 'eth-state')
-STATE_FILEPATH = None if not STATE_FILENAME else os.path.join(STATE_BASE_PATH, STATE_FILENAME)
+NODE_CONFIG_FILENAME = 'node_config.json'
+NODE_CONFIG_FILEPATH: Path = NODE_DATA_PATH / NODE_CONFIG_FILENAME
 
 MIN_ETH_AMOUNT_IN_SKL = 0.01
 MIN_ETH_AMOUNT = int(MIN_ETH_AMOUNT_IN_SKL * (10**18))
@@ -22,14 +23,11 @@
 MISFIRE_GRACE_TIME = 365 * 24 * 60 * 60  # in seconds
 DELAY_AFTER_ERR = 60  # in seconds
 
-SGX_SERVER_URL = os.getenv('SGX_SERVER_URL')
-SGX_CERTIFICATES_FOLDER_NAME = os.getenv('SGX_CERTIFICATES_DIR_NAME')
-
-SGX_CERTIFICATES_FOLDER = None
-if SGX_CERTIFICATES_FOLDER_NAME:
-    SGX_CERTIFICATES_FOLDER = os.path.join(NODE_DATA_PATH, SGX_CERTIFICATES_FOLDER_NAME)
-else:
-    SGX_CERTIFICATES_FOLDER = os.getenv('SGX_CERTIFICATES_FOLDER')
+SGX_CERTIFICATES_FOLDER = NODE_DATA_PATH / 'sgx_certs'
 
 DEFAULT_POOL = 'transactions'
 REDIS_URI = os.getenv('REDIS_URI', 'redis://@127.0.0.1:6379')
+
+SETTINGS_FOLDER_PATH: Path = NODE_DATA_PATH / 'settings'
+NODE_SETTINGS_PATH: Path = SETTINGS_FOLDER_PATH / 'node.toml'
+SkaleSettings.model_config['toml_file'] = NODE_SETTINGS_PATH

configs/web3.py

@@ -3,16 +3,16 @@ name = "bounty-agent"
 version = "3.1.0"
 requires-python = ">=3.13"
 dependencies = [
-    "apscheduler==3.11.0",
+    "apscheduler==3.11.2",
     "peewee==3.18.2",
     "schedule==1.2.2",
-    "skale.py==7.6dev3",
-    "tenacity==9.1.2",
+    "skale.py==7.13dev1",
+    "tenacity==9.1.4",
 ]
 [project.optional-dependencies]
-dev = ["ruff==0.14.1", "mypy==1.18.2"]
+dev = ["ruff==0.15.1", "mypy==1.19.1"]
 test = [
-    "pytest==8.4.2",
+    "pytest==9.0.2",
     "pytest-cov==7.0.0",
     "freezegun==1.5.5",
     "codecov==2.1.13",
@@ -25,6 +25,9 @@ log_cli_format = "[%(asctime)s] [%(levelname)8s] [%(threadName)s] %(message)s (%
 log_cli_date_format = "%Y-%m-%d %H:%M:%S"
 filterwarnings = ["ignore::DeprecationWarning"]
 
+[tool.uv]
+prerelease = "allow"
+
 [tool.ruff]
 line-length = 100
 

helper-scripts

@@ -7,3 +7,4 @@ export SKALE_VOLUME_PATH=$DIR/../skale_vol
 export NODE_DATA_PATH=$DIR/../skale_node_data
 export PYTHONPATH=$PYTHONPATH:.
 export ALLOWED_TS_DIFF=-1
+export DEFAULT_GAS_PRICE_WEI=1000000000

pyproject.toml

@@ -4,12 +4,18 @@
 
 import pytest
 from skale import SkaleManager
+from skale.core.settings import SkaleSettings
 from skale.utils.helper import get_skale_manager_address
 from skale.utils.web3_utils import init_web3
 from skale.wallets import Web3Wallet
 
 from tests.constants import ENDPOINT, ETH_PRIVATE_KEY, N_TEST_NODES, TEST_ABI_FILEPATH
-from tests.prepare_validator import create_dirs, create_set_of_nodes, get_active_ids
+from tests.prepare_validator import (
+    create_dirs,
+    create_set_of_nodes,
+    get_active_ids,
+    setup_validator,
+)
 
 
 @pytest.fixture(scope='session', autouse=True)
@@ -18,16 +24,38 @@ def setup_test_paths():
     os.environ['SKALE_VOLUME_PATH'] = './skale_vol'
     os.environ['NODE_DATA_PATH'] = './skale_node_data'
 
+    # Force low gas price for tests to avoid Insufficient Funds in local anvil
+    import skale.config as skale_config
+    skale_config.DEFAULT_GAS_PRICE = 1000000000  # 1 Gwei
+
+
+@pytest.fixture(scope='session')
+def settings():
+    manager_address = get_skale_manager_address(TEST_ABI_FILEPATH)
+    return SkaleSettings(
+        env_type='devnet',
+        endpoint=ENDPOINT,
+        node_version='0.0.0',
+        block_device='/dev/sda',
+        manager_contracts=manager_address,
+        ima_contracts='0x' + '0' * 40,
+        docker_lvmpy_version='0.0.0',
+        sgx_url='http://localhost:1026',
+        default_gas_price_wei=1000000000,
+    )
+
 
 @pytest.fixture(scope='session')
-def skale():
+def skale(settings):
     """Returns a SKALE instance with provider from config"""
-    web3 = init_web3(ENDPOINT)
+    web3 = init_web3(str(settings.endpoint))
     wallet = Web3Wallet(ETH_PRIVATE_KEY, web3)
-    manager_address = get_skale_manager_address(TEST_ABI_FILEPATH)
-    skale = SkaleManager(ENDPOINT, manager_address, wallet)
+    skale = SkaleManager(str(settings.endpoint), settings.manager_contracts, wallet)
 
     create_dirs()
+
+    setup_validator(skale)
+
     ids = get_active_ids(skale)
     print(f'Existing Node IDs = {ids}')
     cur_node_id = max(ids) + 1 if len(ids) else 0

scripts/export_env.sh

@@ -18,7 +18,7 @@
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import time
-from datetime import datetime
+from datetime import datetime, timezone
 
 import pytest
 from freezegun import freeze_time
@@ -40,8 +40,8 @@ def node_id(skale):
 
 
 @pytest.fixture(scope='module')
-def bounty_collector(skale, node_id):
-    return bounty_agent.BountyAgent(skale, node_id)
+def bounty_collector(skale, settings, node_id):
+    return bounty_agent.BountyAgent(skale, settings, node_id)
 
 
 def test_check_if_node_is_registered(skale, node_id):
@@ -54,7 +54,7 @@ def test_check_if_node_is_registered(skale, node_id):
 def test_get_bounty_neg(skale, bounty_collector):
     last_block_number = skale.web3.eth.block_number
     block_data = skale.web3.eth.get_block(last_block_number)
-    block_timestamp = datetime.utcfromtimestamp(block_data['timestamp'])
+    block_timestamp = datetime.fromtimestamp(block_data['timestamp'], timezone.utc)
     reward_date = bounty_collector.get_reward_date()
     print(f'Reward date: {reward_date}')
     print(f'Timestamp: {block_timestamp}')
@@ -74,7 +74,7 @@ def get_bounty_events(skale, node_id):
         args = log['args']
         tx_block_number = log['blockNumber']
         block_data = skale.web3.eth.get_block(tx_block_number)
-        block_timestamp = datetime.utcfromtimestamp(block_data['timestamp'])
+        block_timestamp = datetime.fromtimestamp(block_data['timestamp'], timezone.utc)
         bounty_events.append(
             (
                 args['nodeIndex'],
@@ -101,14 +101,14 @@ def test_bounty_job_saves_data(skale, bounty_collector):
     assert len(bounties) == 1
 
 
-def test_run_agent(skale, node_id):
-    bounty_collector = bounty_agent.BountyAgent(skale, node_id)
+def test_run_agent(skale, settings, node_id):
+    bounty_collector = bounty_agent.BountyAgent(skale, settings, node_id)
     reward_date = skale.nodes.contract.functions.getNodeNextRewardDate(bounty_collector.id).call()
     print(f'Reward date: {reward_date}')
     go_to_date(skale.web3, reward_date + REWARD_DATE_OFFSET)
     print('Latest block timestamp', skale.web3.eth.get_block('latest')['timestamp'])
 
-    with freeze_time(datetime.utcfromtimestamp(reward_date + REWARD_DATE_OFFSET)):
+    with freeze_time(datetime.fromtimestamp(reward_date + REWARD_DATE_OFFSET, timezone.utc)):
         bounty_collector.run()
         bounty_collector.stop()
 

tests/conftest.py

@@ -26,34 +26,34 @@
 from tools.exceptions import NodeNotFoundException
 
 
-def test_init_agent_pos(skale):
+def test_init_agent_pos(skale, settings):
     print("Test agent init with a given node id")
-    agent0 = BountyAgent(skale, 0)
+    agent0 = BountyAgent(skale, settings, 0)
     assert agent0.id == 0
 
     print("Test agent init without given node id - read id from file")
     with open(NODE_CONFIG_FILEPATH, 'w') as json_file:
         json.dump({'node_id': 1}, json_file)
 
-    agent1 = BountyAgent(skale)
+    agent1 = BountyAgent(skale, settings)
     assert agent1.id == 1
 
 
-def test_init_agent_neg(skale):
+def test_init_agent_neg(skale, settings):
     print("Test agent init with a non-existing node id")
     with pytest.raises(NodeNotFoundException):
-        BountyAgent(skale, 100)
+        BountyAgent(skale, settings, 100)
 
     print("Test agent init with a negative node id")
     with open(NODE_CONFIG_FILEPATH, 'w') as json_file:
         json.dump({'node_id': -1}, json_file)
 
     with pytest.raises(Exception):
-        BountyAgent(skale)
+        BountyAgent(skale, settings)
 
     print("Test agent init with a non-integer node id")
     with open(NODE_CONFIG_FILEPATH, 'w') as json_file:
         json.dump({'node_id': 'one'}, json_file)
 
     with pytest.raises(Exception):
-        BountyAgent(skale)
+        BountyAgent(skale, settings)