runBacklogWorker.py

# This file is part of rubintv_production.
#
# Developed for the LSST Data Management System.
# This product includes software developed by the LSST Project
# (https://www.lsst.org).
# See the COPYRIGHT file at the top-level directory of this distribution
# for details of code ownership.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

import logging
import os
from time import sleep

from lsst.daf.butler import Butler
from lsst.rubintv.production.pipelineRunning import SingleCorePipelineRunner
from lsst.rubintv.production.podDefinition import PodDetails, PodFlavor
from lsst.rubintv.production.utils import (
    getAutomaticLocationConfig,
    getDoRaise,
    getPodWorkerNumber,
    setupSentry,
)
from lsst.summit.utils.utils import setupLogging

setupSentry()
setupLogging()
log = logging.getLogger(__name__)
instrument = "LSSTCam"

workerNum = getPodWorkerNumber()
detectorDepth = workerNum  # flat set like step1b workers, so depth is workerNum

sleepDuration = float(os.getenv("BUTLER_ROLLOUT_PAUSE", 0)) * workerNum
log.info(f"Sleeping worker {workerNum} {detectorDepth}) for {sleepDuration}s ease postgres load")
sleep(sleepDuration)
log.info(f"Worker {workerNum} {detectorDepth}) starting up...")

locationConfig = getAutomaticLocationConfig()
podDetails = PodDetails(
    instrument=instrument, podFlavor=PodFlavor.BACKLOG_WORKER, detectorNumber=None, depth=detectorDepth
)
log.info(
    f"Running {podDetails.instrument} {podDetails.podFlavor.name} at {locationConfig.location},"
    f"consuming from {podDetails.queueName}..."
)

locationConfig = getAutomaticLocationConfig()
butler = Butler.from_config(
    locationConfig.lsstCamButlerPath,
    instrument=instrument,
    collections=[
        f"{instrument}/defaults",
        locationConfig.getOutputChain(instrument),
        "u/gmegias/intrinsic_aberrations_collection_temp",
    ],
    writeable=True,
)

runner = SingleCorePipelineRunner(
    butler=butler,
    locationConfig=locationConfig,
    instrument=instrument,
    step="step1a",
    awaitsDataProduct="raw",
    podDetails=podDetails,
    doRaise=getDoRaise(),
)
runner.run()