Dockerfile

FROM ghcr.io/whiletrue-industries/datapackage-pipelines:latest-slim


RUN apt-get update --fix-missing && apt-get install -y sudo curl ssh
RUN adduser --disabled-password --home /home/dpp dpp && adduser dpp dpp && \
    adduser dpp root && adduser dpp redis && adduser redis dpp && \
    mkdir -p /var/datapackages && chown dpp.dpp /var/datapackages -R && chmod -R a+r /var/datapackages && \
    mkdir -p /home/dpp/.ssh && chown dpp.dpp /home/dpp/.ssh -R && \
    chown dpp.dpp /var/redis -R && \
    chown dpp.dpp /var/log/redis -R && \
    chown dpp.dpp /var/run/dpp -R && \
    chmod 700 /home/dpp/.ssh && \
    echo '%root ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/root
RUN mkdir -p /usr/share/man/man1 && apt-get install -y antiword poppler-utils libmagic1 default-jre-headless

RUN pip install "numpy<2" && pip install textract==1.5.0 pyquery "rfc3986<1.0" filemagic tabula-py paramiko boto3

ADD ./ /

ADD .dpp-runners.tzabar /datapackage_pipelines_budgetkey/pipelines/
RUN mv /datapackage_pipelines_budgetkey/pipelines/.dpp-runners.tzabar /datapackage_pipelines_budgetkey/pipelines/dpp-runners.yaml

RUN chown dpp.dpp /datapackage_pipelines_budgetkey -R
RUN pip install -e /
# RUN pip install -U -r /requirements-dev.txt

# Fallback only: google_chrome.py resolves a chromedriver matching whatever
# Chrome the google-chrome-in-a-box container reports, and only falls back to
# this one if that lookup fails. Keep it roughly in step with that image.
ARG CHROMEDRIVER_VERSION=150.0.7871.124
RUN apt-get install -y sudo unzip wget libglib2.0-0 && \
    wget -q "https://storage.googleapis.com/chrome-for-testing-public/${CHROMEDRIVER_VERSION}/linux64/chromedriver-linux64.zip" && \
    unzip -j chromedriver-linux64.zip 'chromedriver-linux64/chromedriver' && \
    rm chromedriver-linux64.zip && \
    chmod +x chromedriver && \
    mv chromedriver /usr/local/bin/ && \
    cp /certs/* /usr/local/share/ca-certificates/ && \
    rm -rf /certs/ && \
    cd /usr/local/share/ca-certificates/ && for i in `ls `; do ln -s $i /etc/ssl/certs/$i; done && \
    update-ca-certificates --fresh

USER dpp

ENV PYTHONPATH=/
ENV DPP_PROCESSOR_PATH=/datapackage_pipelines_budgetkey/processors
ENV DPP_REDIS_HOST=localhost
ENV REDIS_USER=dpp
ENV REDIS_GROUP=dpp
ENV DPP_NUM_WORKERS=4
ENV DPP_MAX_TASKS_PER_CHILD=100
ENV DPP_MAX_MEMORY_PER_CHILD=750000
ENV DPP_MANAGEMENT_MAX_TASKS_PER_CHILD=200
ENV DPP_MANAGEMENT_MAX_MEMORY_PER_CHILD=500000

WORKDIR /datapackage_pipelines_budgetkey/pipelines/

EXPOSE 5000

ENTRYPOINT ["/startup.sh"]