diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 846d4a2f..8132f7a5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,10 @@ repos: # Note for all linters: do not forget to update pyproject.toml when updating version. - repo: https://github.com/python-poetry/poetry - rev: 1.8.0 + rev: 2.3.2 hooks: - - id: poetry-lock - args: ["--check"] + - id: poetry-check + args: ["--lock"] - repo: https://github.com/psf/black-pre-commit-mirror rev: 24.10.0 diff --git a/Dockerfile b/Dockerfile index edfcdb57..56bcf957 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ ENV PYTHONUNBUFFERED=1 \ PYSETUP_PATH="/opt/pysetup" \ VENV_PATH="/opt/pysetup/.venv" \ POETRY_HOME="/opt/poetry" \ - POETRY_VERSION=1.8.3 \ + POETRY_VERSION=2.3.2 \ POETRY_VIRTUALENVS_IN_PROJECT=true \ POETRY_NO_INTERACTION=1 ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH" diff --git a/app/utils/download.py b/app/utils/download.py index ae15fbd4..79e912e5 100644 --- a/app/utils/download.py +++ b/app/utils/download.py @@ -93,14 +93,17 @@ def download_file(url: str, output_path: Path): etag = r.headers.get("ETag", "").strip("'\"") tmp_output_path = output_path.with_name(output_path.name + ".part") - with tmp_output_path.open("wb") as f, tqdm.tqdm( - unit="B", - unit_scale=True, - unit_divisor=1024, - miniters=1, - desc=str(output_path), - total=int(r.headers.get("content-length", 0)), - ) as pbar: + with ( + tmp_output_path.open("wb") as f, + tqdm.tqdm( + unit="B", + unit_scale=True, + unit_divisor=1024, + miniters=1, + desc=str(output_path), + total=int(r.headers.get("content-length", 0)), + ) as pbar, + ): for chunk in r.iter_content(chunk_size=4096): f.write(chunk) pbar.update(len(chunk)) diff --git a/poetry.lock b/poetry.lock index 5ed5982e..351a6547 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1386,5 +1386,5 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [metadata] lock-version = "2.1" -python-versions = "^3.11" -content-hash = "41e2590e1056ed2e3d1c8ed6a8d541dede5ae073aeecdd4f8baf37bd42ba0e00" +python-versions = ">=3.11,<3.12" +content-hash = "e26893be9d85f958166c66396e9a8cc2f9c17dbb68023a88e0631538e9def140" diff --git a/pyproject.toml b/pyproject.toml index 76d4b77d..d6c95d47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,42 +1,28 @@ -[tool.poetry] +[project] name = "search-a-licious" version = "0.1.0" description = "" -authors = ["Open Food Facts team"] -license = "AGPL-3.0 licence" readme = "README.md" -package-mode = false - -[tool.poetry.dependencies] -python = "^3.11" -elasticsearch-dsl = "~8.9.0" -fastapi = "~0.115.0" -requests = "~2.32.4" -redis = "~5.0.0" -uvicorn = "~0.23.2" -tqdm = "~4.66.1" -cachetools = "~5.3.2" -typer = "~0.9.0" -luqum = "~0.13.0" -pydantic-settings = "~2.0.3" -sentry-sdk = {extras = ["fastapi"], version = "~1.34.0"} -jinja2 = "~3.1.6" -pyyaml = "~6.0.1" -orjson = "~3.9.15" -py-healthcheck = "^1.10.1" - - -[tool.poetry.group.dev.dependencies] -pytest = "^7.4.3" -coverage = {extras = ["toml"], version = "^7.3.2"} -pytest-cov = "^4.1.0" -mypy = "^1.10.0" -types-cachetools = "^5.3.0.7" -types-requests = "^2.31.0.10" -types-pyyaml = "^6.0.12.12" -pre-commit = "^3.5.0" -factory-boy = "^3.3.1" -httpx = "^0.27.1" +requires-python = ">=3.11,<3.12" +license = "AGPL-3.0-only" +authors = [{ name = "Open Food Facts team" }] +dependencies = [ + "cachetools~=5.3.2", + "elasticsearch-dsl~=8.9.0", + "fastapi~=0.115.0", + "jinja2~=3.1.6", + "luqum~=0.13.0", + "orjson~=3.9.15", + "py-healthcheck>=1.10.1,<2.0.0", + "pydantic-settings~=2.0.3", + "pyyaml~=6.0.1", + "redis~=5.0.0", + "requests~=2.32.4", + "sentry-sdk[fastapi]~=1.34.0", + "tqdm~=4.66.1", + "typer~=0.9.0", + "uvicorn~=0.23.2", +] [build-system] requires = ["poetry-core"] @@ -52,15 +38,6 @@ exclude = ''' )/ ''' -[tool.mypy] -ignore_missing_imports = true -exclude = '''(?x)( - .*\.cache/pre-commit/.* # avoid pre-commit cache -)''' - -[tool.isort] -profile = "black" - [tool.coverage.run] data_file = ".cov/.coverage" source = ["app"] @@ -70,3 +47,27 @@ output = ".cov/coverage.xml" [tool.coverage.html] directory = ".cov/htmlcov" + +[tool.isort] +profile = "black" + +[tool.mypy] +ignore_missing_imports = true +exclude = '''(?x)( + .*\.cache/pre-commit/.* # avoid pre-commit cache +)''' + +[tool.poetry] +package-mode = false + +[tool.poetry.group.dev.dependencies] +coverage = { extras = ["toml"], version = "^7.3.2" } +factory-boy = "^3.3.1" +httpx = "^0.27.1" +mypy = "^1.10.0" +pre-commit = "^3.5.0" +pytest = "^7.4.3" +pytest-cov = "^4.1.0" +types-cachetools = "^5.3.0.7" +types-pyyaml = "^6.0.12.12" +types-requests = "^2.31.0.10" diff --git a/scripts/Dockerfile.sphinx b/scripts/Dockerfile.sphinx index b6383ed4..b7e3cf32 100644 --- a/scripts/Dockerfile.sphinx +++ b/scripts/Dockerfile.sphinx @@ -1,24 +1,56 @@ -# in your Dockerfile -FROM sphinxdoc/sphinx:7.4.7 - ARG USER_UID=1000 ARG USER_GID=1000 +ARG PYTHON_VERSION=3.12 + +# This first stage install Poetry and exports a requirements.txt, +# which we will then use in the second stage to install dependencies. +FROM python:${PYTHON_VERSION}-slim AS poetry-builder + +ARG POETRY_VERSION=2.3.2 + +ENV POETRY_HOME="/opt/poetry" +ENV PATH="$POETRY_HOME/bin:$PATH" + +WORKDIR /build + +RUN python3 -m venv /opt/poetry && \ + /opt/poetry/bin/pip install "poetry==${POETRY_VERSION}" && \ + poetry config virtualenvs.create false && \ + poetry self add poetry-plugin-export + +COPY poetry.lock pyproject.toml ./ +RUN poetry export --output requirements.txt + + +FROM sphinxdoc/sphinx:7.4.7 AS docs-builder + +ARG USER_UID +ARG USER_GID + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y curl cargo && \ + apt-get autoremove --purge && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + WORKDIR /docs + # add user with right id -RUN addgroup --gid $USER_GID user && adduser --uid $USER_UID --ingroup user --no-create-home --disabled-password --quiet user +RUN addgroup --gid ${USER_GID} user && \ + adduser --uid ${USER_UID} --ingroup user --disabled-password --quiet user + # ensure directories and permisisons -RUN mkdir -p /docs/build && mkdir -p /docs/source && chown user:user /docs -# install poetry, and export dependencies as a requirement.txt -COPY poetry.lock pyproject.toml ./ -RUN apt update && apt install -y curl cargo -RUN ( curl -sSL https://install.python-poetry.org | python3 - ) && \ - /root/.local/bin/poetry self add poetry-plugin-export && \ - /root/.local/bin/poetry export --output requirements.txt -# install those dependencies -RUN pip install -U pip && pip install -r requirements.txt -# install some useful plugin for sphinx -RUN pip install autodoc_pydantic sphinxcontrib-typer +RUN mkdir -p /docs/build && \ + mkdir -p /docs/source && \ + chown -R user:user /docs + USER user +COPY --from=poetry-builder /build/requirements.txt . + +# install those dependencies +RUN pip install -U pip && pip install -r requirements.txt diff --git a/scripts/build_mkdocs.sh b/scripts/build_mkdocs.sh index 0b14417f..9d626a84 100755 --- a/scripts/build_mkdocs.sh +++ b/scripts/build_mkdocs.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -echo '::group::{build_mkdocs}' set -e # Renders markdown doc in docs to html in gh_pages @@ -15,5 +14,3 @@ docker run --rm \ -e USER_ID=$UID -e GROUP_ID=$GID \ -v $(pwd):/app -w /app \ mkdocs-builder build --strict - -echo "::endgroup::" \ No newline at end of file diff --git a/scripts/build_sphinx.sh b/scripts/build_sphinx.sh index 676760c5..f87cae26 100755 --- a/scripts/build_sphinx.sh +++ b/scripts/build_sphinx.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash # Build sphinx documentation -echo '::group::{build_sphinx}' - set -e # get group id to use it in the docker @@ -28,5 +26,3 @@ docker run --rm --user user \ rm -rf gh_pages/devs/ref-python || true mv gh_pages/sphinx/html gh_pages/devs/ref-python rm -rf gh_pages/sphinx/ - -echo "::endgroup::" \ No newline at end of file diff --git a/scripts/generate_doc.sh b/scripts/generate_doc.sh index 9197f3e8..c3ce570b 100755 --- a/scripts/generate_doc.sh +++ b/scripts/generate_doc.sh @@ -1,5 +1,37 @@ #!/usr/bin/env bash +COLOR_RESET="\033[0m" +COLOR_BOLD="\033[1m" +COLOR_BLUE="\033[34m" + +is_tty() { + [[ -t 1 && "$TERM" != "dumb" ]] +} + +# Start a log group +start_group() { + local title="${1}" + if [ -n "$GITHUB_ACTIONS" ]; then + printf "::group::%s\n" "$title" + else + local prefix="==> ${title}" + if is_tty; then + printf "${COLOR_BOLD}${COLOR_BLUE}%s${COLOR_RESET}\n" "$prefix" + else + printf "%s\n" "$prefix" + fi + fi +} + +# End a log group +end_group() { + if [ -n "$GITHUB_ACTIONS" ]; then + echo "::endgroup::" + else + echo "" + fi +} + # generating documentation # fail on errors @@ -8,40 +40,43 @@ set -e mkdir -p gh_pages # script to generate documentation -echo "Build documentation with MkDocs" +start_group "Generate documentation with mkdocs" scripts/build_mkdocs.sh +end_group -echo "Generate documentation for configuration file and settings" +start_group "Generate documentation for configuration file and settings" scripts/build_schema.sh config scripts/build_schema.sh settings +end_group -echo "Generate OpenAPI documentation" -echo '::group::{generate_openapi}' +start_group "Generate OpenAPI documentation" make generate-openapi -echo "::endgroup::" +end_group -echo "Generate openapi html with redocly" -echo '::group::{generate_openapi_html}' +start_group "Generate openapi html with redocly" docker run --rm \ - -v $(pwd)/data:/data -v $(pwd)/gh_pages/:/output \ - ghcr.io/redocly/redoc/cli:latest \ - build -o /output/users/ref-openapi/index.html searchalicious-openapi.yml -sudo chown $UID -R gh_pages/users/ref-openapi -echo "::endgroup::" - -echo "Generate webcomponents documentation" -echo '::group::{generate_custom_elements}' + -v $(pwd)/data:/data \ + -v $(pwd)/gh_pages/:/output \ + --user "$(id -u):$(id -g)" \ + ghcr.io/redocly/redoc/cli:latest \ + build -o /output/users/ref-openapi/index.html searchalicious-openapi.yml +end_group + +start_group "Generate webcomponents documentation" make generate-custom-elements mkdir -p gh_pages/users/ref-web-components/dist cp frontend/public/dist/custom-elements.json gh_pages/users/ref-web-components/dist/custom-elements.json sudo chown $UID -R gh_pages/users/ref-web-components -echo "::endgroup::" +end_group -echo "Generate python code documentation using sphinx" +start_group "Generate python code documentation using sphinx" scripts/build_sphinx.sh +end_group # tell GitHub we don't want to use jekyll ! (otherwise it will remove _static) # see https://github.blog/news-insights/the-library/bypassing-jekyll-on-github-pages/ -touch gh_pages/.nojekyll +if [ -n "$GITHUB_ACTIONS" ]; then + touch gh_pages/.nojekyll +fi -echo "To see your doc locally, run: python3 -m http.server -d gh_pages 8001" \ No newline at end of file +echo "To see your doc locally, run: python3 -m http.server -d gh_pages 8001" diff --git a/tests/unit/test__import.py b/tests/unit/test__import.py index 93d4bab9..8be0e125 100644 --- a/tests/unit/test__import.py +++ b/tests/unit/test__import.py @@ -421,9 +421,11 @@ def test_run_update_daemon(default_global_config: Config): load_document_fetcher_mock = MagicMock(return_value=document_fetcher_mock) # Patch the necessary functions and objects - with patch("app._import.connection", connection_mock), patch( - "app._import.load_document_fetcher", load_document_fetcher_mock - ), patch("app._import.get_new_updates", MagicMock(return_value=updates)): + with ( + patch("app._import.connection", connection_mock), + patch("app._import.load_document_fetcher", load_document_fetcher_mock), + patch("app._import.get_new_updates", MagicMock(return_value=updates)), + ): # Call the function run_update_daemon(default_global_config)