Merge pull request #39 from SciCatProject/background-ingestor

Background ingestor

Author: YooSunYoung

src/background-ingestor.py

@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2024 ScicatProject contributors (https://github.com/ScicatProject)
+import json
+import logging
+from collections.abc import Generator
+from contextlib import contextmanager
+
+from scicat_configuration import (
+    build_background_ingestor_arg_parser,
+    build_scicat_config,
+)
+from scicat_logging import build_logger
+
+# import scippnexus as snx
+
+
+def quit(logger: logging.Logger, unexpected: bool = True) -> None:
+    """Log the message and exit the program."""
+    import sys
+
+    logger.info("Exiting ingestor")
+    sys.exit(1 if unexpected else 0)
+
+
+@contextmanager
+def exit_at_exceptions(logger: logging.Logger) -> Generator[None, None, None]:
+    """Exit the program if an exception is raised."""
+    try:
+        yield
+    except KeyboardInterrupt:
+        logger.info("Received keyboard interrupt.")
+        quit(logger, unexpected=False)
+    except Exception as e:
+        logger.error("An exception occurred: %s", e)
+        quit(logger, unexpected=True)
+    else:
+        logger.error("Loop finished unexpectedly.")
+        quit(logger, unexpected=True)
+
+
+def main() -> None:
+    """Main entry point of the app."""
+    arg_parser = build_background_ingestor_arg_parser()
+    arg_namespace = arg_parser.parse_args()
+    config = build_scicat_config(arg_namespace)
+    logger = build_logger(config)
+
+    # Log the configuration as dictionary so that it is easier to read from the logs
+    logger.info(
+        'Starting the Scicat background Ingestor with the following configuration:'
+    )
+    logger.info(config.to_dict())
+
+    with exit_at_exceptions(logger):
+        nexus_file = arg_namespace.nexus_file
+        logger.info("Nexus file to be ingested : ")
+        logger.info(nexus_file)
+
+        done_writing_message_file = (
+            arg_namespace.arg_namespace.done_writing_message_file
+        )
+        logger.info("Done writing message file linked to nexus file : ")
+        logger.info(done_writing_message_file)
+
+        # open and read done writing message input file
+        with open(done_writing_message_file, 'r') as f:
+            done_writing_message = json.load(f)
+
+        print(done_writing_message)
+        # open nexus file
+        # nxs = snx.File(nexus_file)
+
+        # extract instrument
+
+        # load instrument metadata configuration
+
+        # retrieve information regarding the proposal
+
+        # extract and prepare metadata
+
+        # create b2blake hash of all the files
+
+        # create and populate scicat dataset entry
+
+        # create and populate scicat origdatablock entry
+        # with files and hashes previously computed
+
+        pass

src/scicat_configuration.py

@@ -94,6 +94,39 @@ def build_main_arg_parser() -> argparse.ArgumentParser:
     return parser
 
 
+def build_background_ingestor_arg_parser() -> argparse.ArgumentParser:
+    parser = build_main_arg_parser()
+
+    group = parser.add_argument_group('Scicat Background Ingestor Options')
+
+    group.add_argument(
+        '-f',
+        '--nf',
+        '--file',
+        '--nexus-file',
+        default='',
+        dest='nexus_file',
+        help='Full path of the input nexus file to be ingested',
+        type=str,
+    )
+
+    group.add_argument(
+        '-m',
+        '--dm',
+        '--wrdm',
+        '--done-writing-message-file',
+        default='',
+        dest='done_writing_message_file',
+        help="""
+          Full path of the input done writing message
+          file that match the nexus file to be ingested
+        """,
+        type=str,
+    )
+
+    return parser
+
+
 @dataclass
 class GraylogOptions:
     host: str = ""

src/scicat_ingestor.py

@@ -41,7 +41,7 @@ def main() -> None:
     logger = build_logger(config)
 
     # Log the configuration as dictionary so that it is easier to read from the logs
-    logger.info('Starting the Scicat Ingestor with the following configuration:')
+    logger.info('Starting the Scicat online Ingestor with the following configuration:')
     logger.info(config.to_dict())
 
     with exit_at_exceptions(logger):