diff --git a/client/.dockerignore b/client/.dockerignore index fc3abfcdd..df8f09d58 100644 --- a/client/.dockerignore +++ b/client/.dockerignore @@ -8,6 +8,5 @@ __pycache__ .pytest_cache .idea .mypy_cache -examples node_modules .coverage diff --git a/client/README.md b/client/README.md index c7f8a8c57..56a470bfb 100644 --- a/client/README.md +++ b/client/README.md @@ -47,6 +47,21 @@ the distribution-specific commands to install these are: * Red Hat-based distros: `sudo yum install python38 python38-pip` (for e.g. Python 3.8) +### GCP + +You need the Google Cloud CLI client `gcloud` installed. [Read installation +steps for the `gcloud` CLI package.](https://cloud.google.com/sdk/docs/install) + +You need to have credentials configured locally for a GCP user with suitable +permissions to perform batch translation. [Read how `gcloud` searches for +application default credentials.](https://cloud.google.com/docs/authentication/application-default-credentials) + +You need a GCP project and a Google Cloud Storage bucket to use for uploading +your input SQL files and downloading the translated output. [Learn how to +create a GCS bucket manually][creating buckets], or see the [instructions for +using `provision.sh`](#running-using-runsh) to automatically provision a +bucket for translation. + ### Support for Encodings other than UTF-8 If all of the files you wish to translate are UTF-8 encoded @@ -54,7 +69,7 @@ If all of the files you wish to translate are UTF-8 encoded Otherwise, you will need to install additional system dependencies: * Debian-based distros: `sudo apt install pkg-config libicu-dev` -* RedHat-based distros: `sudo yum install gcc gcc-c++ libicu-devel +* Red Hat-based distros: `sudo yum install gcc gcc-c++ libicu-devel python38-devel` **You must also remember**, upon reaching the step to `pip install` further down @@ -64,14 +79,6 @@ in the Quickstart section below, to use this command instead: pip install ../dwh-migration-tools/client[icu] ``` -### GCP - -You need a GCP project and a Google Cloud Storage bucket to use for uploading -your input SQL files and downloading the translated output. [Learn how to -create a GCS bucket manually][creating buckets], or see the [instructions for -using `provision.sh`](#running-using-runsh) to automatically provision a -bucket for translation. - ## Quickstart 1. Download the repo from [google/dwh-migration-tools] in your choice of diff --git a/client/tests/packaging/Dockerfile-RedHat b/client/tests/packaging/Dockerfile-RedHat new file mode 100644 index 000000000..094a03d0a --- /dev/null +++ b/client/tests/packaging/Dockerfile-RedHat @@ -0,0 +1,31 @@ +FROM redhat/ubi9 + +# See https://cloud.google.com/sdk/docs/install +COPY tests/packaging/redhat-google-cloud-sdk.repo /etc/yum.repos.d/google-cloud-sdk.repo +RUN dnf install -y \ + google-cloud-cli \ + python3 \ + python3-pip + +WORKDIR /workspace +COPY . /workspace/dwh-migration-tools/client +COPY examples/teradata/sql /workspace/project + +ARG ICU_ENABLED=false +RUN if [ "$ICU_ENABLED" = "true" ]; then \ + dnf install -y \ + gcc \ + gcc-c++ \ + libicu-devel \ + python3-devel \ + && python3 -m pip install dwh-migration-tools/client[icu] ; \ + else \ + python3 -m pip install dwh-migration-tools/client ; \ + fi + +RUN dnf clean all -y && rm -rf /var/cache +WORKDIR /workspace/project +ENV BQMS_VERBOSE="True" +ENV BQMS_MULTITHREADED="True" + +ENTRYPOINT [ "/workspace/project/run.sh" ] diff --git a/client/tests/packaging/Dockerfile-Ubuntu b/client/tests/packaging/Dockerfile-Ubuntu new file mode 100644 index 000000000..fc8594c86 --- /dev/null +++ b/client/tests/packaging/Dockerfile-Ubuntu @@ -0,0 +1,32 @@ +FROM ubuntu:22.04 + +RUN apt-get update -y && apt-get install -y \ + curl \ + python3-pip +# See https://cloud.google.com/sdk/docs/install +RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" \ + | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \ + | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - \ + && apt-get update -y \ + && apt-get install -y google-cloud-cli + +WORKDIR /workspace +COPY . /workspace/dwh-migration-tools/client +COPY examples/teradata/sql /workspace/project + +ARG ICU_ENABLED=false +RUN if [ "$ICU_ENABLED" = "true" ]; then \ + apt-get install -y \ + libicu-dev \ + pkg-config \ + && python3 -m pip install dwh-migration-tools/client[icu] ; \ + else \ + python3 -m pip install dwh-migration-tools/client ; \ + fi + +RUN rm -rf /var/lib/apt/lists/* +WORKDIR /workspace/project +ENV BQMS_VERBOSE="True" +ENV BQMS_MULTITHREADED="True" + +ENTRYPOINT [ "/workspace/project/run.sh" ] diff --git a/client/tests/packaging/redhat-google-cloud-sdk.repo b/client/tests/packaging/redhat-google-cloud-sdk.repo new file mode 100644 index 000000000..e0bc693d1 --- /dev/null +++ b/client/tests/packaging/redhat-google-cloud-sdk.repo @@ -0,0 +1,7 @@ +[google-cloud-cli] +name=Google Cloud CLI +baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el9-x86_64 +enabled=1 +gpgcheck=1 +repo_gpgcheck=0 +gpgkey=https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg diff --git a/client/tests/packaging/run-packaging-tests.sh b/client/tests/packaging/run-packaging-tests.sh new file mode 100755 index 000000000..aa3115944 --- /dev/null +++ b/client/tests/packaging/run-packaging-tests.sh @@ -0,0 +1,16 @@ +#!/bin/bash -e + +for BASE_OS in RedHat Ubuntu; do + for ICU_ENABLED in false true; do + docker build \ + -t dwh-migration-tools-test \ + -f Dockerfile-$BASE_OS \ + --build-arg ICU_ENABLED=$ICU_ENABLED \ + ../.. + docker run -it --rm \ + -v ~/.config/gcloud:/root/.config/gcloud \ + -e BQMS_PROJECT=$BQMS_PROJECT \ + -e BQMS_GCS_BUCKET=$BQMS_GCS_BUCKET \ + dwh-migration-tools-test + done +done