BU-ISCIII · Shettland · Mar 2, 2026 · Dec 15, 2025 · Dec 15, 2025 · Dec 15, 2025
diff --git a/.github/workflows/test_modules.yml b/.github/workflows/test_modules.yml
@@ -47,9 +47,15 @@ jobs:
       run: |
         pip install -r requirements.txt
         pip install .
+
+    - name: Load profile config (relecov)
+      run: |
+        relecov-tools add-extra-config \
+          --config_file relecov_tools/conf/initial_config-relecov.yaml --force
+
     - name: Run each module tests
       run: |
-        relecov-tools map -j tests/data/map_validate/processed_metadata_lab_test.json -p relecov_tools/schema/relecov_schema.json ${{ matrix.map_args }} -o .
+        relecov-tools --debug map -j tests/data/map_validate/processed_metadata_lab_test.json -p relecov_tools/schema/relecov_schema.json ${{ matrix.map_args }} -o .
       env:
         OUTPUT_LOCATION: ${{ github.workspace }}/tests/
     - name: Upload output file
@@ -87,6 +93,11 @@ jobs:
       run: |
         pip install -r requirements.txt
         pip install .
+
+    - name: Load profile config (relecov)
+      run: |
+        relecov-tools add-extra-config \
+          --config_file relecov_tools/conf/initial_config-relecov.yaml --force
 
     - name: Run read-lab-metadata module
       if: matrix.modules == 'read-lab-metadata'
@@ -110,18 +121,18 @@ jobs:
       run: |
         relecov-tools read-bioinfo-metadata \
           --json_file tests/data/read_bioinfo_metadata/validated_samples.json \
+          --json_schema_file relecov_tools/schema/relecov_schema.json \
           --input_folder tests/data/read_bioinfo_metadata/analysis_folder/ \
           --software_name viralrecon \
           --soft_validation \
           -o $OUTPUT_LOCATION
 
-    - name: Run validate module with upload files (default)
+    - name: Run validate module without --upload-files param (no sftp)
       if: matrix.modules == 'validate'
       run: |
-        python3 tests/test_validate.py \
+        relecov-tools validate \
           -j tests/data/map_validate/processed_metadata_lab_test.json \
           -m tests/data/map_validate/metadata_lab_test.xlsx \
-          --upload_files \
           -o tests/data/map_validate/ \
           -s relecov_tools/schema/relecov_schema.json \
           -l tests/data/map_validate/previous_processes_log_summary.json

diff --git a/.github/workflows/test_sftp_modules.yml b/.github/workflows/test_sftp_modules.yml
@@ -1,6 +1,8 @@
 name: test_sftp_modules
 
 on:
+  push:
+    branches: "**"
   pull_request_target:
     types: [opened, reopened, synchronize]
     branches: "**"
@@ -53,6 +55,11 @@ jobs:
       run: |
         pip install -r requirements.txt
         pip install .
+
+    - name: Load profile config (relecov)
+      run: |
+        relecov-tools add-extra-config \
+          --config_file relecov_tools/conf/initial_config-relecov.yaml --force
 
     - name: Run sftp_handle tests
       run: |
@@ -88,12 +95,60 @@ jobs:
       run: |
         pip install -r requirements.txt
         pip install .
-
+
+    - name: Load profile config (relecov)
+      run: |
+        relecov-tools add-extra-config \
+          --config_file relecov_tools/conf/initial_config-relecov.yaml --force
+
     - name: Run Wrapper tests
       run: |
         python3 tests/test_wrapper_handle.py --download_option ${{ matrix.download_options }}
       env:
         TEST_USER: ${{ secrets.TEST_USER }}
         TEST_PASSWORD: ${{ secrets.TEST_PASSWORD }}
         TEST_PORT: ${{ secrets.TEST_PORT }}
-        OUTPUT_LOCATION: ${{ github.workspace }}/tests/
+        OUTPUT_LOCATION: ${{ github.workspace }}/tests/
+
+
+  test_validate_upload:
+    runs-on: ubuntu-latest
+    needs: test_wrapper
+    if: github.repository_owner == 'BU-ISCIII'
+
+    steps:
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.12'
+
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        fetch-depth: 0
+
+    - name: Install package and dependencies
+      run: |
+        pip install -r requirements.txt
+        pip install .
+
+    - name: Load profile config (relecov)
+      run: |
+        relecov-tools add-extra-config \
+          --config_file relecov_tools/conf/initial_config-relecov.yaml --force
+    - name: Run validate module with upload files (expected default)
+      if: github.repository_owner == 'BU-ISCIII'
+      run: |
+        python3 tests/test_validate.py \
+          -j tests/data/map_validate/processed_metadata_lab_test.json \
+          -m tests/data/map_validate/metadata_lab_test.xlsx \
+          --upload_files \
+          -o tests/data/map_validate/ \
+          -s relecov_tools/schema/relecov_schema.json \
+          -l tests/data/map_validate/previous_processes_log_summary.json
+      env:
+        TEST_USER: ${{ secrets.TEST_USER }}
+        TEST_PASSWORD: ${{ secrets.TEST_PASSWORD }}
+        TEST_PORT: ${{ secrets.TEST_PORT }}
+        OUTPUT_LOCATION: ${{ github.workspace }}/tests/
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,50 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.8.0] - 2025-27-02 : <https://github.com/BU-ISCIII/relecov-tools/releases/tag/v1.8.0>
+
+### Credits
+
+- [Enrique Sapena](https://github.com/ESapenaVentura)
+- [Pablo Mata](https://github.com/shettland)
+- [Alejandro Bernabeu](https://github.com/aberdur)
+
+#### Added enhancements
+
+- Added handling of $refs for enums and typo fixing/style formatting for build_schema module [#844](https://github.com/BU-ISCIII/relecov-tools/pull/844)
+- Refactor configuration.json migration to initial-config profiles [#860]https://github.com/BU-ISCIII/relecov-tools/pull/860
+- Included configuration overwriting and validation via add-extra-config module [#861]https://github.com/BU-ISCIII/relecov-tools/pull/861
+- Included a new arg `topic_config` in add-extra-config to narrow config removal [#861]https://github.com/BU-ISCIII/relecov-tools/pull/861
+- Updated add-extra-config and included Initial configuration description in README.md [#861]https://github.com/BU-ISCIII/relecov-tools/pull/861
+- Updated test workflows to load initial_config-relecov.yaml first [#861]https://github.com/BU-ISCIII/relecov-tools/pull/861
+- Included three initial_config.yaml files for three projects: relecov, mepram & EQA2026 [#861]https://github.com/BU-ISCIII/relecov-tools/pull/861
+- Normalize compact YYYYMMDD dates in read-lab-metadata [#867]https://github.com/BU-ISCIII/relecov-tools/pull/867
+
+#### Fixes
+
+- Fix small bug on type/examples consistency for float/integer fields in JSON schema generation (build_schema module) [#844](https://github.com/BU-ISCIII/relecov-tools/pull/844)
+- Fix header generation for `METADATA_LAB` and `DATA_VALIDATION`: now "CAMPO" is the first row, "REQUIRED" last [#849](https://github.com/BU-ISCIII/relecov-tools/pull/849)
+- Now Metadatalab validation sheets (DATA_VALIDATION and DROPDOWNS) only generate the included fields, not all of them [#849](https://github.com/BU-ISCIII/relecov-tools/pull/849)
+- Fixed datetime recognition for schema example generation and validation [#854](https://github.com/BU-ISCIII/relecov-tools/pull/854)
+- Restore formatting and validation behavior after header reordering + add Excel warnings for MEPRAM [#856] (https://github.com/BU-ISCIII/relecov-tools/pull/856)
+- Adapted github actions workflows to load extra_config first [#863](https://github.com/BU-ISCIII/relecov-tools/pull/863)
+- Fix missing env vars in test_upload_validate step for test_sftp_modules [#866](https://github.com/BU-ISCIII/relecov-tools/pull/866)
+- read-lab-metadata: keep submitting_institution_id from metadata in EQA initial-config [#868](https://github.com/BU-ISCIII/relecov-tools/pull/868)
+
+#### Changed
+
+- Changed `build_schema.build_new_schema` function: Now recursively iterates on complex properties to generate subschemas [#849](https://github.com/BU-ISCIII/relecov-tools/pull/849)
+- Changed `assets/schema_utils/metadatalab_template.py`: Now supports iterative recursion to flatten nested schemas [#849](https://github.com/BU-ISCIII/relecov-tools/pull/849)
+- Updated how project name is extracted from config in read-lab-metadata [#861]https://github.com/BU-ISCIII/relecov-tools/pull/861
+- Include extra_config=True in all modules that did not have it [#863](https://github.com/BU-ISCIII/relecov-tools/pull/863)
+- Update relecov_schema.json title and add space removal check [#867]https://github.com/BU-ISCIII/relecov-tools/pull/867
+
+#### Removed
+
+- Removed all project-specific config from configuration.json and moved it to initial_config.yaml files [#861]https://github.com/BU-ISCIII/relecov-tools/pull/861
+
+### Requirements
+
 ## [1.7.4] - 2025-12-15 : <https://github.com/BU-ISCIII/relecov-tools/releases/tag/v1.7.4>
 
 ### Credits

diff --git a/README.md b/README.md
@@ -108,7 +108,7 @@ $ relecov-tools --help
 \    \  /   |__ / |__  |    |___ |    |   |  \    /
 /    /  \   |  \  |    |    |    |    |   |   \  /
 /    |--|   |   \ |___ |___ |___ |___ |___|    \/
-RELECOV-tools version 1.7.4
+RELECOV-tools version 1.8.0
 Usage: relecov-tools [OPTIONS] COMMAND [ARGS]...
 
 Options:
@@ -148,6 +148,11 @@ Further explanation for each argument:
 - `--debug`: Activate DEBUG logs. When not provided, logs will only show the most relevant information.
 - `--hex-code`: By default all files generated will include a date and an unique hexadecimal code which is randomly generated upon execution. Using this argument you can pre-define the resulting hexadecimal code. NOTE: Keep in mind that this could overwrite existing files.
 
+### Initial configuration.
+
+Prior to using any module you will need to setup specific configuration for your project using `add-extra-config` module, usage documentation [here](#add-extra-config). There are examples of `initial_config.yaml` files in `relecov_tools/conf/` that can be used as template.
+You can also override the config in `relecov_tools/conf/configuration.json` with this method except for the `required_conf` keys which we recommend to preserve as they are always validated at runtime. You can add new `required_conf` elements into your `extra_config.json` but the resulting list will be the sum of both, not overriden.
+
 ## Modules
 
 #### download
@@ -480,7 +485,7 @@ Options:
 
 #### add-extra-config
 
-This command is used to create an additional config file that will override the configuration in `conf/configuration.json`. You may pass this configuration in a YAML or JSON file. If you want the keys in your additional configuration to be grouped under a certain keyname, use param `-n, --config_name`. Otherwise, the file content will be parsed with no additional processing.
+This command is used to create an additional config file that will override the configuration in `conf/configuration.json` except for `required_conf` keys which are merged from the two configs. Passing the whole configuration in a YAML or JSON file is recommended. If you want the keys in your additional configuration to be grouped under a certain keyname, use param `-n, --config_name`. Otherwise, the file content will be parsed with no additional processing. The resulting configuration can be found in `/.relecov_tools/extra_config.json`
 
 ```
 Usage: relecov-tools add-extra-config [OPTIONS]
@@ -492,8 +497,10 @@ Options:
   -f, --config_file TEXT  Path to the input file: Json or Yaml format
   --force                 Force replacement of existing configuration if
                           needed
-  --clear_config          Remove given config_name from extra config: Use with
+  --remove_config         Remove given config_name from extra config: Use with
                           empty --config_name to remove all
+  --topic_config          Name of the primary config key. Only used with 
+                          --remove_config to narrow the search for --config_name"
   --help                  Show this message and exit.
 ```
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "relecov-tools"
-version = "1.7.4"
+version = "1.8.0"
 description = "Tools for managing and proccessing relecov network data."
 readme = "README.md"
 requires-python = ">=3.10"

diff --git a/relecov_tools/__main__.py b/relecov_tools/__main__.py
@@ -39,7 +39,7 @@
     stderr=True, force_terminal=relecov_tools.utils.rich_force_colors()
 )
 
-__version__ = "1.7.4"
+__version__ = "1.8.0"
 
 # IMPORTANT: When defining a Click command function in this script,
 # you MUST include both 'ctx' (for @click.pass_context) and ALL the parameters
@@ -102,7 +102,6 @@ def merge_with_extra_config(ctx, add_extra_config=False):
     sig = inspect.signature(func)
     valid_keys = sig.parameters.keys()
     filtered = {k: v for k, v in merged.items() if k in valid_keys}
-
     return filtered
 
 
@@ -1381,22 +1380,31 @@ def upload_results(
     help="Force replacement of existing configuration if needed",
 )
 @click.option(
-    "--clear_config",
+    "-r",
+    "--remove_config",
     is_flag=True,
     default=False,
     help="Remove given config_name from extra config: Use with empty --config_name to remove all",
 )
+@click.option(
+    "-t",
+    "--topic_config",
+    default=None,
+    help="Name of the primary config key. Only used with --remove_config to narrow the search for --config_name",
+)
 @click.pass_context
-def add_extra_config(ctx, config_name, config_file, force, clear_config):
+def add_extra_config(ctx, config_name, config_file, force, remove_config, topic_config):
     """Save given file content as additional configuration"""
     debug = ctx.obj.get("debug", False)
     try:
         if os.path.isfile(relecov_tools.config_json.ConfigJson._extra_config_path):
             config_json = relecov_tools.config_json.ConfigJson(extra_config=True)
         else:
             config_json = relecov_tools.config_json.ConfigJson()
-        if clear_config:
-            config_json.remove_extra_config(config_name)
+        if remove_config:
+            config_json.remove_extra_config(
+                topic=topic_config, deep=config_name, force=force
+            )
         else:
             config_json.include_extra_config(
                 config_file, config_name=config_name, force=force

diff --git a/relecov_tools/assets/Relecov_metadata_template_v3.2.3.xlsx b/relecov_tools/assets/Relecov_metadata_template_v3.2.3.xlsx
diff --git a/relecov_tools/assets/Relecov_metadata_template_v3.2.4.xlsx b/relecov_tools/assets/Relecov_metadata_template_v3.2.4.xlsx
diff --git a/relecov_tools/assets/pipeline_utils/utils.py b/relecov_tools/assets/pipeline_utils/utils.py
@@ -2,6 +2,7 @@
 """
 Common utility function used for relecov_tools package.
 """
+
 import json
 import logging
 import os

diff --git a/relecov_tools/assets/schema_utils/custom_validators.py b/relecov_tools/assets/schema_utils/custom_validators.py
@@ -32,7 +32,7 @@ def validate_with_exceptions(schema, data, errors):
         if (
             error.validator == "type"
             and error.instance
-            == relecov_tools.config_json.ConfigJson().get_topic_data(
+            == relecov_tools.config_json.ConfigJson(extra_config=True).get_topic_data(
                 "generic", "not_provided_field"
             )
             and prop_schema.get("type") in ["integer", "number"]
@@ -43,7 +43,7 @@ def validate_with_exceptions(schema, data, errors):
         if (
             error.validator == "format"
             and error.instance
-            == relecov_tools.config_json.ConfigJson().get_topic_data(
+            == relecov_tools.config_json.ConfigJson(extra_config=True).get_topic_data(
                 "generic", "not_provided_field"
             )
             and prop_schema.get("type") == "string"