diff --git a/readthedocs/doc_builder/director.py b/readthedocs/doc_builder/director.py index 0fa9b865e35..4b7b9d406ec 100644 --- a/readthedocs/doc_builder/director.py +++ b/readthedocs/doc_builder/director.py @@ -29,6 +29,7 @@ from readthedocs.projects.constants import BUILD_COMMANDS_OUTPUT_PATH_HTML from readthedocs.projects.constants import GENERIC from readthedocs.projects.exceptions import RepositoryError +from readthedocs.projects.models import Feature from readthedocs.projects.notifications import MESSAGE_PROJECT_SSH_KEY_WITH_WRITE_ACCESS from readthedocs.projects.signals import after_build from readthedocs.projects.signals import before_build @@ -724,6 +725,21 @@ def build_docs_class(self, builder_class): success = builder.build() return success + def _add_parallel_env_var(self, env): + cpus = os.cpu_count() + # These variables are useful for projects that compile libraries during the build. + env.update( + { + # https://cmake.org/cmake/help/latest/envvar/CMAKE_BUILD_PARALLEL_LEVEL.html + "CMAKE_BUILD_PARALLEL_LEVEL": cpus, + # https://www.gnu.org/software/make/manual/html_node/Options_002fRecursion.html + "MAKEFLAGS": f"-j {cpus}", + # Pillow and other libraries use this variable + # https://pillow.readthedocs.io/en/stable/installation/building-from-source.html#build-options + "MAX_CONCURRENCY": cpus, + } + ) + def _add_git_ssh_command_env_var(self, env): if settings.ALLOW_PRIVATE_REPOS: # Set GIT_SSH_COMMAND to use ssh with options that disable host key checking @@ -812,6 +828,9 @@ def get_build_env_vars(self): self._add_git_ssh_command_env_var(env) + if self.data.project.has_feature(Feature.BUILD_IN_PARALLEL): + self._add_parallel_env_var(env) + # Update environment from Project's specific environment variables, # avoiding to expose private environment variables # if the version is external (i.e. a PR build). diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index ffd54a61b92..8b1bb881b37 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -39,10 +39,30 @@ def _get_clone_url(self): return f"{parsed_url.scheme}://$READTHEDOCS_GIT_CLONE_TOKEN@{parsed_url.netloc}{parsed_url.path}" return self.repo_url + def _set_parallel_configs(self): + """ + Set git configs to enable parallelization. + """ + # https://git-scm.com/docs/git-config#Documentation/git-config.txt-fetchparallel + self.run("git", "config", "--global", "fetch.parallel", "0") + # https://git-scm.com/docs/git-config#Documentation/git-config.txt-checkoutworkers + self.run("git", "config", "--global", "checkout.workers", "0") + # https://git-scm.com/docs/git-config#Documentation/git-config.txt-checkoutthresholdForParallelism + self.run("git", "config", "--global", "checkout.thresholdForParallelism", "100") + # https://git-scm.com/docs/git-config#Documentation/git-config.txt-submodulefetchJobs + self.run("git", "config", "--global", "submodule.fetchJobs", "0") + # https://git-scm.com/docs/git-config#Documentation/git-config.txt-packthreads + self.run("git", "config", "--global", "pack.threads", "0") + def update(self): """Clone and/or fetch remote repository.""" super().update() + from readthedocs.projects.models import Feature + + if self.project.has_feature(Feature.BUILD_IN_PARALLEL): + self._set_parallel_configs() + if self.project.git_checkout_command: # Run custom checkout step if defined if isinstance(self.project.git_checkout_command, list):