From 5163521a5894c9c7c2a5a3fad58487c71462ea82 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 9 Apr 2020 16:46:58 +0200 Subject: [PATCH] Refactor `--config` setup Define top-level configurations for the user to specify, e.g. `--config linux-nixpkgs`. These will then define the required flags for that platform/toolchain combination. Configure CI by writing `--config` flags to `.bazelrc.local` instead of passing them on the command-line. This makes sure that the same configuration is used even if Bazel is called indirectly through a script. This is useful for tools like `ghcide` and `hie-bios`. --- .bazelrc | 70 ++++++++++++++++++++++++++------ .buildkite/bindists-pipeline | 17 +++----- .buildkite/pipeline.yml | 10 +++-- .circleci/config.yml | 25 ++++++------ README.md | 25 ++++++++++++ azure-pipelines.yml | 79 ++++++++++++++++++++---------------- haskell/ghc.BUILD.tpl | 5 +++ tests/RunTests.hs | 6 +-- 8 files changed, 156 insertions(+), 81 deletions(-) diff --git a/.bazelrc b/.bazelrc index a2a8a8aaa..532de5546 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,30 +1,74 @@ # See https://docs.bazel.build/versions/master/user-manual.html#bazelrc. +# Global Configuration +# -------------------- + +# test environment does not propagate locales by default some tests reads files +# written in UTF8, we need to propagate the correct environment variables, such +# as LOCALE_ARCHIVE We also need to setup an utf8 locale +test --test_env=LANG=C.UTF-8 --test_env=LOCALE_ARCHIVE + +# Platform / Toolchain Selection +# ------------------------------ +build:linux-nixpkgs --host_platform=@rules_haskell//haskell/platforms:linux_x86_64_nixpkgs +build:macos-nixpkgs --host_platform=@rules_haskell//haskell/platforms:darwin_x86_64_nixpkgs # Use this configuration when targeting Windows. Eventually this will # no longer be required: # https://bazel.build/roadmaps/platforms.html#replace---cpu-and---host_cpu-flags. -build:windows --crosstool_top=@rules_haskell_ghc_windows_amd64//:cc_toolchain -s --verbose_failures --sandbox_debug +build:windows-bindist --crosstool_top=@rules_haskell_ghc_windows_amd64//:cc_toolchain + +# Build and Test Filters +# ---------------------- +# Note tag filters do not compose. If you wish to augment these filters on the +# command-line you have to repeat the full list. +build:linux-bindist --build_tag_filters -requires_lz4,-requires_proto,-requires_zlib,-requires_doctest,-requires_c2hs,-requires_shellcheck,-requires_threaded_rts,-dont_test_with_bindist +build:linux-bindist --test_tag_filters -requires_lz4,-requires_proto,-requires_zlib,-requires_doctest,-requires_c2hs,-requires_shellcheck,-requires_threaded_rts,-dont_test_with_bindist + +build:macos-nixpkgs --build_tag_filters -dont_test_on_darwin +build:macos-nixpkgs --test_tag_filters -dont_test_on_darwin + +build:macos-bindist --build_tag_filters -dont_test_on_darwin,-dont_test_on_darwin_with_bindist,-requires_lz4,-requires_proto,-requires_zlib,-requires_doctest,-requires_c2hs,-requires_shellcheck,-requires_threaded_rts,-dont_test_with_bindist +build:macos-bindist --test_tag_filters -dont_test_on_darwin,-dont_test_on_darwin_with_bindist,-requires_lz4,-requires_proto,-requires_zlib,-requires_doctest,-requires_c2hs,-requires_shellcheck,-requires_threaded_rts,-dont_test_with_bindist + +# CI Configuration +# ---------------- +common:ci --color=no -build:ci_linux --remote_http_cache=https://storage.googleapis.com/tweag-bazel-cache-buildkite --remote_upload_local_results=true --google_default_credentials build:ci --loading_phase_threads=1 build:ci --verbose_failures # Make sure we don't rely on the names of convenience symlinks because those # can be changed by user. build:ci --symlink_prefix=bazel-ci- -common:ci --color=no + test:ci --test_output=errors +build:ci-linux-bindist --remote_http_cache=https://storage.googleapis.com/tweag-bazel-cache-buildkite --remote_upload_local_results=true --google_default_credentials +build:ci-linux-nixpkgs --remote_http_cache=https://storage.googleapis.com/tweag-bazel-cache-buildkite --remote_upload_local_results=true --google_default_credentials + +build:ci-linux-bindist --repository_cache ~/repo_cache +# XXX: @com_google_protobuf sets `use_default_shell_env = True`, so we enable +# strict action env to avoid changes in `PATH` invalidating the cache. +build:ci-linux-bindist --experimental_strict_action_env + +build:ci-macos-bindist --disk_cache=~/.cache/bazel/ +build:ci-macos-nixpkgs --disk_cache=~/.cache/bazel/ + +build:ci-windows-bindist --subcommands --sandbox_debug + # Note [backward compatible options] -common:ci --incompatible_enable_cc_toolchain_resolution -common:ci --incompatible_load_cc_rules_from_bzl +build:ci-linux-bindist --incompatible_enable_cc_toolchain_resolution +build:ci-macos-bindist --incompatible_enable_cc_toolchain_resolution +# rules_nixpkgs is not yet compatible with cc toolchain resolution, see +# https://github.com/tweag/rules_nixpkgs/issues/88 +#build:ci-linux-nixpkgs --incompatible_enable_cc_toolchain_resolution +#build:ci-macos-nixpkgs --incompatible_enable_cc_toolchain_resolution +# Windows still uses --crosstool_top. +#build:ci-windows-bindist --incompatible_enable_cc_toolchain_resolution +build:ci --incompatible_load_cc_rules_from_bzl # Blocked by https://github.com/bazelbuild/buildtools/issues/757 -#common:ci --incompatible_load_proto_rules_from_bzl -common:ci --incompatible_load_python_rules_from_bzl - -# test environment does not propagate locales by default -# some tests reads files written in UTF8, we need to propagate the correct -# environment variables, such as LOCALE_ARCHIVE -# We also need to setup an utf8 locale -test --test_env=LANG=C.UTF-8 --test_env=LOCALE_ARCHIVE +#build:ci --incompatible_load_proto_rules_from_bzl +build:ci --incompatible_load_python_rules_from_bzl +# User Configuration +# ------------------ try-import %workspace%/.bazelrc.local diff --git a/.buildkite/bindists-pipeline b/.buildkite/bindists-pipeline index f8be8a9d4..9b0a9a26b 100755 --- a/.buildkite/bindists-pipeline +++ b/.buildkite/bindists-pipeline @@ -6,18 +6,13 @@ set -euo pipefail BAZEL_DIR="$(.ci/fetch-bazel-bindist)" -REPO_CACHE="$HOME/repo_cache" trap "rm -rf '$BAZEL_DIR'" EXIT export PATH="$BAZEL_DIR:$PATH" -EXCLUDED="-requires_lz4,-requires_proto,-requires_zlib,-requires_doctest,-requires_c2hs,-requires_shellcheck,-requires_threaded_rts,-dont_test_with_bindist" -echo "common:ci --build_tag_filters $EXCLUDED" > .bazelrc.local -echo "common:ci --test_tag_filters $EXCLUDED" >> .bazelrc.local -echo "build:ci --config=ci_linux" >> .bazelrc.local -echo "common:ci --repository_cache $REPO_CACHE" >> .bazelrc.local -# XXX: @com_google_protobuf sets `use_default_shell_env = True`, so we enable -# strict action env to avoid changes in `PATH` invalidating the cache. -echo "build:ci --experimental_strict_action_env" >> .bazelrc.local +cat >.bazelrc.local < .bazelrc.local - echo "build:ci --config=ci_linux" >> .bazelrc.local + cat >.bazelrc.local <> .bazelrc.local - echo "build:ci --disk_cache=~/.cache/bazel/" >> .bazelrc.local - echo "common:ci --test_tag_filters -dont_test_on_darwin" >> .bazelrc.local + cat >.bazelrc.local <> .bazelrc.local - EXCLUDED="-dont_test_on_darwin,-dont_test_on_darwin_with_bindist,-requires_lz4,-requires_proto,-requires_zlib,-requires_doctest,-requires_c2hs,-requires_shellcheck,-requires_threaded_rts,-dont_test_with_bindist" - echo "common:ci --build_tag_filters $EXCLUDED" >> .bazelrc.local - echo "common:ci --test_tag_filters $EXCLUDED" >> .bazelrc.local + cat >.bazelrc.local <>.bazelrc.local +``` + ### Saving common command-line flags to a file If you find yourself constantly passing the same flags on the diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 655194cc1..35eaae67c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,6 +22,13 @@ jobs: Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -Value 1 displayName: "Enable da long paths" + - bash: | + cat >.bazelrc.local < Process.CreateProcess --- Note: --config=ci is intercalated between the action and the list of --- arguments. It should appear after the action, but before any @--@ --- following argument. -bazel (command:args) = Process.proc "bazel" (command:"--config=ci":args) -bazel [] = Process.proc "bazel" [] +bazel args = Process.proc "bazel" args -- | Runs a bazel query and return the list of matching targets bazelQuery :: String -> SpecM a [String]