From f9ecc2aa23a63af78ca605bf525f39ce82bd3249 Mon Sep 17 00:00:00 2001 From: Eric Rahm Date: Tue, 30 Sep 2025 21:09:24 +0000 Subject: [PATCH 1/4] Add build.json and generator script This commit introduces a new script, `manage_build_json.py`, which queries Bazel to determine the source files for the Emboss compiler and C++ runtime. The script generates a `build.json` file in the project root, which will serve as the source of truth for generating build helper files for other build systems. The script also adds support for verifying that `build.json` is in sync with the bazel build targets. --- build.json | 68 ++++++++ scripts/build_helpers/manage_build_json.py | 193 +++++++++++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 build.json create mode 100755 scripts/build_helpers/manage_build_json.py diff --git a/build.json b/build.json new file mode 100644 index 0000000..4576ad1 --- /dev/null +++ b/build.json @@ -0,0 +1,68 @@ +{ + // A list of all source files required to build the Emboss compiler. + "embossc_sources": [ + "compiler/__init__.py", + "compiler/back_end/__init__.py", + "compiler/back_end/cpp/__init__.py", + "compiler/back_end/cpp/attributes.py", + "compiler/back_end/cpp/emboss_codegen_cpp.py", + "compiler/back_end/cpp/generated_code_templates", + "compiler/back_end/cpp/header_generator.py", + "compiler/back_end/util/__init__.py", + "compiler/back_end/util/code_template.py", + "compiler/front_end/__init__.py", + "compiler/front_end/attribute_checker.py", + "compiler/front_end/attributes.py", + "compiler/front_end/constraints.py", + "compiler/front_end/dependency_checker.py", + "compiler/front_end/emboss_front_end.py", + "compiler/front_end/error_examples", + "compiler/front_end/expression_bounds.py", + "compiler/front_end/generated/cached_parser.py", + "compiler/front_end/glue.py", + "compiler/front_end/lr1.py", + "compiler/front_end/make_parser.py", + "compiler/front_end/module_ir.py", + "compiler/front_end/parser.py", + "compiler/front_end/prelude.emb", + "compiler/front_end/reserved_words", + "compiler/front_end/symbol_resolver.py", + "compiler/front_end/synthetics.py", + "compiler/front_end/tokenizer.py", + "compiler/front_end/type_check.py", + "compiler/front_end/write_inference.py", + "compiler/util/__init__.py", + "compiler/util/attribute_util.py", + "compiler/util/error.py", + "compiler/util/expression_parser.py", + "compiler/util/ir_data.py", + "compiler/util/ir_data_fields.py", + "compiler/util/ir_data_utils.py", + "compiler/util/ir_util.py", + "compiler/util/name_conversion.py", + "compiler/util/parser_types.py", + "compiler/util/parser_util.py", + "compiler/util/resources.py", + "compiler/util/simple_memoizer.py", + "compiler/util/traverse_ir.py", + "embossc" + ], + // A list of all source files required for the Emboss C++ runtime. + "emboss_runtime_cpp_sources": [ + "runtime/cpp/emboss_arithmetic.h", + "runtime/cpp/emboss_arithmetic_all_known_generated.h", + "runtime/cpp/emboss_arithmetic_maximum_operation_generated.h", + "runtime/cpp/emboss_array_view.h", + "runtime/cpp/emboss_bit_util.h", + "runtime/cpp/emboss_constant_view.h", + "runtime/cpp/emboss_cpp_types.h", + "runtime/cpp/emboss_cpp_util.h", + "runtime/cpp/emboss_defines.h", + "runtime/cpp/emboss_enum_view.h", + "runtime/cpp/emboss_maybe.h", + "runtime/cpp/emboss_memory_util.h", + "runtime/cpp/emboss_prelude.h", + "runtime/cpp/emboss_text_util.h", + "runtime/cpp/emboss_view_parameters.h" + ] +} diff --git a/scripts/build_helpers/manage_build_json.py b/scripts/build_helpers/manage_build_json.py new file mode 100755 index 0000000..3d356c1 --- /dev/null +++ b/scripts/build_helpers/manage_build_json.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Creates and validates the `build.json` file.""" + +import argparse +import json +import os +import subprocess +import sys + +# The Bazel targets that define the embossc compiler sources. +EMBOSSC_TARGETS = [ + "//compiler/back_end/cpp:emboss_codegen_cpp", + "//compiler/front_end:emboss_front_end", +] + +# The Bazel target for the C++ runtime sources. +RUNTIME_CPP_TARGET = "//runtime/cpp:cpp_utils" + + +def get_bazel_query_output(query): + """Runs a Bazel query and returns the output as a list of strings.""" + result = subprocess.run( + ["bazel", "query", query], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip().split("\n") + + +def get_source_files_for_targets(targets): + """Queries Bazel for all source files in the transitive dependencies.""" + query = f"kind('source file', deps({'+'.join(targets)}))" + output = get_bazel_query_output(query) + return sorted( + [ + f.replace("//", "").replace(":", "/") + for f in output + if f.startswith("//") and not f.startswith("//external") + ] + ) + + +def read_build_json(file_path): + """Reads build.json, strips comments, and returns the parsed JSON data.""" + with open(file_path, "r") as f: + content = "".join(line for line in f if not line.strip().startswith("//")) + return json.loads(content) + + +def find_init_py_files(source_files): + """Finds all __init__.py files in the directories of the source files.""" + init_py_files = set() + checked_dirs = set() + + for source_file in source_files: + dir_path = os.path.dirname(source_file) + while dir_path and dir_path not in checked_dirs: + checked_dirs.add(dir_path) + init_py = os.path.join(dir_path, "__init__.py") + if os.path.exists(init_py): + init_py_files.add(init_py) + # Move to the parent directory + parent_dir = os.path.dirname(dir_path) + if parent_dir == dir_path: # Root directory + break + dir_path = parent_dir + + return sorted(list(init_py_files)) + + +def generate_build_json(output_path): + """Generates the build.json file from Bazel sources.""" + print("Generating fresh source lists from Bazel...") + embossc_sources = get_source_files_for_targets(EMBOSSC_TARGETS) + embossc_sources.append("embossc") + init_py_sources = find_init_py_files(embossc_sources) + all_embossc_sources = sorted(list(set(embossc_sources + init_py_sources))) + + runtime_cpp_sources = get_source_files_for_targets([RUNTIME_CPP_TARGET]) + + build_data = { + "embossc_sources": all_embossc_sources, + "emboss_runtime_cpp_sources": runtime_cpp_sources, + } + + # Use json.dumps to get a formatted string, then inject comments. + json_string = json.dumps(build_data, indent=2) + json_string = json_string.replace( + '"embossc_sources":', + '// A list of all source files required to build the Emboss compiler.\n "embossc_sources":', + ) + json_string = json_string.replace( + '"emboss_runtime_cpp_sources":', + '// A list of all source files required for the Emboss C++ runtime.\n "emboss_runtime_cpp_sources":', + ) + + with open(output_path, "w") as f: + f.write(json_string) + f.write("\n") + + print(f"Successfully generated {output_path}") + return 0 + + +def validate_build_json(file_path): + """Validates that the on-disk build.json file is up-to-date.""" + print("Generating fresh source lists from Bazel for validation...") + fresh_embossc_sources = get_source_files_for_targets(EMBOSSC_TARGETS) + fresh_embossc_sources.append("embossc") + init_py_sources = find_init_py_files(fresh_embossc_sources) + all_fresh_embossc_sources = sorted( + list(set(fresh_embossc_sources + init_py_sources)) + ) + fresh_runtime_cpp_sources = get_source_files_for_targets([RUNTIME_CPP_TARGET]) + + print(f"Reading existing sources from {file_path}...") + on_disk_data = read_build_json(file_path) + on_disk_embossc_sources = on_disk_data.get("embossc_sources", []) + on_disk_runtime_cpp_sources = on_disk_data.get("emboss_runtime_cpp_sources", []) + + embossc_diff = set(all_fresh_embossc_sources) ^ set(on_disk_embossc_sources) + runtime_diff = set(fresh_runtime_cpp_sources) ^ set(on_disk_runtime_cpp_sources) + + if not embossc_diff and not runtime_diff: + print("build.json is up-to-date.") + return 0 + + print("\nERROR: build.json is out of date!", file=sys.stderr) + + def print_diff(title, fresh_set, on_disk_set): + added = sorted(list(fresh_set - on_disk_set)) + removed = sorted(list(on_disk_set - fresh_set)) + if added or removed: + print(f" {title}:", file=sys.stderr) + for f in added: + print(f" + {f}", file=sys.stderr) + for f in removed: + print(f" - {f}", file=sys.stderr) + + print_diff( + "embossc_sources", + set(fresh_embossc_sources), + set(on_disk_embossc_sources), + ) + print_diff( + "emboss_runtime_cpp_sources", + set(fresh_runtime_cpp_sources), + set(on_disk_runtime_cpp_sources), + ) + + print( + "\nPlease run 'scripts/build_helpers/manage_build_json.py' to update.", + file=sys.stderr, + ) + return 1 + + +def main(): + parser = argparse.ArgumentParser(description="Manage the build.json file.") + parser.add_argument( + "--validate", + action="store_true", + help="Validate that build.json is up-to-date.", + ) + args = parser.parse_args() + + workspace_root = os.environ.get("BUILD_WORKSPACE_DIRECTORY", os.getcwd()) + build_json_path = os.path.join(workspace_root, "build.json") + + if args.validate: + return validate_build_json(build_json_path) + else: + return generate_build_json(build_json_path) + + +if __name__ == "__main__": + sys.exit(main()) From bbe2b6a4fa9089311504474389ac175cd7555b73 Mon Sep 17 00:00:00 2001 From: Eric Rahm Date: Wed, 1 Oct 2025 17:30:21 +0000 Subject: [PATCH 2/4] Add CI verification of build.json Adds a check to the `verify-pull-request` CI workflow to make sure that `build.json` is in sync with the bazel build targets. --- .github/workflows/verify-pull-request.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/verify-pull-request.yml b/.github/workflows/verify-pull-request.yml index 9c57be7..744b5c9 100644 --- a/.github/workflows/verify-pull-request.yml +++ b/.github/workflows/verify-pull-request.yml @@ -27,3 +27,14 @@ jobs: with: options: "--check --verbose" version: "24.8.0" + check-build-json: + name: "Check build.json" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: bazel-contrib/setup-bazel@0.8.5 + with: + bazelisk-cache: true + disk-cache: "verify-pr:check-build-json" + repository-cache: true + - run: python3 scripts/build_helpers/manage_build_json.py --validate From 11a529dee7933100bcb1feb63056022454047833 Mon Sep 17 00:00:00 2001 From: Eric Rahm Date: Wed, 1 Oct 2025 17:32:22 +0000 Subject: [PATCH 3/4] Add build files and generator script Adds a `generate_build_files.py` script that is used to generate build files for use in bazel, cmake, and GN. Additionally a clean json file is added for more generic usage. These files are intended to be included in downstream build systems to explicitly list out the files required by embossc and the C++ runtime. The generated build files are included as well. --- gen/sources.bzl | 83 ++++++++++++ gen/sources.cmake | 83 ++++++++++++ gen/sources.gni | 83 ++++++++++++ gen/sources.json | 66 +++++++++ scripts/build_helpers/generate_build_files.py | 128 ++++++++++++++++++ 5 files changed, 443 insertions(+) create mode 100644 gen/sources.bzl create mode 100644 gen/sources.cmake create mode 100644 gen/sources.gni create mode 100644 gen/sources.json create mode 100755 scripts/build_helpers/generate_build_files.py diff --git a/gen/sources.bzl b/gen/sources.bzl new file mode 100644 index 0000000..3a1f135 --- /dev/null +++ b/gen/sources.bzl @@ -0,0 +1,83 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Automatically generated by generate_build_files.py. +# Do not edit this file directly. + +EMBOSSC_SOURCES = [ + "compiler/__init__.py", + "compiler/back_end/__init__.py", + "compiler/back_end/cpp/__init__.py", + "compiler/back_end/cpp/attributes.py", + "compiler/back_end/cpp/emboss_codegen_cpp.py", + "compiler/back_end/cpp/generated_code_templates", + "compiler/back_end/cpp/header_generator.py", + "compiler/back_end/util/__init__.py", + "compiler/back_end/util/code_template.py", + "compiler/front_end/__init__.py", + "compiler/front_end/attribute_checker.py", + "compiler/front_end/attributes.py", + "compiler/front_end/constraints.py", + "compiler/front_end/dependency_checker.py", + "compiler/front_end/emboss_front_end.py", + "compiler/front_end/error_examples", + "compiler/front_end/expression_bounds.py", + "compiler/front_end/generated/cached_parser.py", + "compiler/front_end/glue.py", + "compiler/front_end/lr1.py", + "compiler/front_end/make_parser.py", + "compiler/front_end/module_ir.py", + "compiler/front_end/parser.py", + "compiler/front_end/prelude.emb", + "compiler/front_end/reserved_words", + "compiler/front_end/symbol_resolver.py", + "compiler/front_end/synthetics.py", + "compiler/front_end/tokenizer.py", + "compiler/front_end/type_check.py", + "compiler/front_end/write_inference.py", + "compiler/util/__init__.py", + "compiler/util/attribute_util.py", + "compiler/util/error.py", + "compiler/util/expression_parser.py", + "compiler/util/ir_data.py", + "compiler/util/ir_data_fields.py", + "compiler/util/ir_data_utils.py", + "compiler/util/ir_util.py", + "compiler/util/name_conversion.py", + "compiler/util/parser_types.py", + "compiler/util/parser_util.py", + "compiler/util/resources.py", + "compiler/util/simple_memoizer.py", + "compiler/util/traverse_ir.py", + "embossc", +] + +EMBOSS_RUNTIME_CPP_SOURCES = [ + "runtime/cpp/emboss_arithmetic.h", + "runtime/cpp/emboss_arithmetic_all_known_generated.h", + "runtime/cpp/emboss_arithmetic_maximum_operation_generated.h", + "runtime/cpp/emboss_array_view.h", + "runtime/cpp/emboss_bit_util.h", + "runtime/cpp/emboss_constant_view.h", + "runtime/cpp/emboss_cpp_types.h", + "runtime/cpp/emboss_cpp_util.h", + "runtime/cpp/emboss_defines.h", + "runtime/cpp/emboss_enum_view.h", + "runtime/cpp/emboss_maybe.h", + "runtime/cpp/emboss_memory_util.h", + "runtime/cpp/emboss_prelude.h", + "runtime/cpp/emboss_text_util.h", + "runtime/cpp/emboss_view_parameters.h", +] diff --git a/gen/sources.cmake b/gen/sources.cmake new file mode 100644 index 0000000..60d009b --- /dev/null +++ b/gen/sources.cmake @@ -0,0 +1,83 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Automatically generated by generate_build_files.py. +# Do not edit this file directly. + +set(EMBOSSC_SOURCES + "compiler/__init__.py" + "compiler/back_end/__init__.py" + "compiler/back_end/cpp/__init__.py" + "compiler/back_end/cpp/attributes.py" + "compiler/back_end/cpp/emboss_codegen_cpp.py" + "compiler/back_end/cpp/generated_code_templates" + "compiler/back_end/cpp/header_generator.py" + "compiler/back_end/util/__init__.py" + "compiler/back_end/util/code_template.py" + "compiler/front_end/__init__.py" + "compiler/front_end/attribute_checker.py" + "compiler/front_end/attributes.py" + "compiler/front_end/constraints.py" + "compiler/front_end/dependency_checker.py" + "compiler/front_end/emboss_front_end.py" + "compiler/front_end/error_examples" + "compiler/front_end/expression_bounds.py" + "compiler/front_end/generated/cached_parser.py" + "compiler/front_end/glue.py" + "compiler/front_end/lr1.py" + "compiler/front_end/make_parser.py" + "compiler/front_end/module_ir.py" + "compiler/front_end/parser.py" + "compiler/front_end/prelude.emb" + "compiler/front_end/reserved_words" + "compiler/front_end/symbol_resolver.py" + "compiler/front_end/synthetics.py" + "compiler/front_end/tokenizer.py" + "compiler/front_end/type_check.py" + "compiler/front_end/write_inference.py" + "compiler/util/__init__.py" + "compiler/util/attribute_util.py" + "compiler/util/error.py" + "compiler/util/expression_parser.py" + "compiler/util/ir_data.py" + "compiler/util/ir_data_fields.py" + "compiler/util/ir_data_utils.py" + "compiler/util/ir_util.py" + "compiler/util/name_conversion.py" + "compiler/util/parser_types.py" + "compiler/util/parser_util.py" + "compiler/util/resources.py" + "compiler/util/simple_memoizer.py" + "compiler/util/traverse_ir.py" + "embossc" +) + +set(EMBOSS_RUNTIME_CPP_SOURCES + "runtime/cpp/emboss_arithmetic.h" + "runtime/cpp/emboss_arithmetic_all_known_generated.h" + "runtime/cpp/emboss_arithmetic_maximum_operation_generated.h" + "runtime/cpp/emboss_array_view.h" + "runtime/cpp/emboss_bit_util.h" + "runtime/cpp/emboss_constant_view.h" + "runtime/cpp/emboss_cpp_types.h" + "runtime/cpp/emboss_cpp_util.h" + "runtime/cpp/emboss_defines.h" + "runtime/cpp/emboss_enum_view.h" + "runtime/cpp/emboss_maybe.h" + "runtime/cpp/emboss_memory_util.h" + "runtime/cpp/emboss_prelude.h" + "runtime/cpp/emboss_text_util.h" + "runtime/cpp/emboss_view_parameters.h" +) diff --git a/gen/sources.gni b/gen/sources.gni new file mode 100644 index 0000000..6207e5b --- /dev/null +++ b/gen/sources.gni @@ -0,0 +1,83 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Automatically generated by generate_build_files.py. +# Do not edit this file directly. + +embossc_sources = [ + "compiler/__init__.py", + "compiler/back_end/__init__.py", + "compiler/back_end/cpp/__init__.py", + "compiler/back_end/cpp/attributes.py", + "compiler/back_end/cpp/emboss_codegen_cpp.py", + "compiler/back_end/cpp/generated_code_templates", + "compiler/back_end/cpp/header_generator.py", + "compiler/back_end/util/__init__.py", + "compiler/back_end/util/code_template.py", + "compiler/front_end/__init__.py", + "compiler/front_end/attribute_checker.py", + "compiler/front_end/attributes.py", + "compiler/front_end/constraints.py", + "compiler/front_end/dependency_checker.py", + "compiler/front_end/emboss_front_end.py", + "compiler/front_end/error_examples", + "compiler/front_end/expression_bounds.py", + "compiler/front_end/generated/cached_parser.py", + "compiler/front_end/glue.py", + "compiler/front_end/lr1.py", + "compiler/front_end/make_parser.py", + "compiler/front_end/module_ir.py", + "compiler/front_end/parser.py", + "compiler/front_end/prelude.emb", + "compiler/front_end/reserved_words", + "compiler/front_end/symbol_resolver.py", + "compiler/front_end/synthetics.py", + "compiler/front_end/tokenizer.py", + "compiler/front_end/type_check.py", + "compiler/front_end/write_inference.py", + "compiler/util/__init__.py", + "compiler/util/attribute_util.py", + "compiler/util/error.py", + "compiler/util/expression_parser.py", + "compiler/util/ir_data.py", + "compiler/util/ir_data_fields.py", + "compiler/util/ir_data_utils.py", + "compiler/util/ir_util.py", + "compiler/util/name_conversion.py", + "compiler/util/parser_types.py", + "compiler/util/parser_util.py", + "compiler/util/resources.py", + "compiler/util/simple_memoizer.py", + "compiler/util/traverse_ir.py", + "embossc", +] + +emboss_runtime_cpp_sources = [ + "runtime/cpp/emboss_arithmetic.h", + "runtime/cpp/emboss_arithmetic_all_known_generated.h", + "runtime/cpp/emboss_arithmetic_maximum_operation_generated.h", + "runtime/cpp/emboss_array_view.h", + "runtime/cpp/emboss_bit_util.h", + "runtime/cpp/emboss_constant_view.h", + "runtime/cpp/emboss_cpp_types.h", + "runtime/cpp/emboss_cpp_util.h", + "runtime/cpp/emboss_defines.h", + "runtime/cpp/emboss_enum_view.h", + "runtime/cpp/emboss_maybe.h", + "runtime/cpp/emboss_memory_util.h", + "runtime/cpp/emboss_prelude.h", + "runtime/cpp/emboss_text_util.h", + "runtime/cpp/emboss_view_parameters.h", +] diff --git a/gen/sources.json b/gen/sources.json new file mode 100644 index 0000000..46212da --- /dev/null +++ b/gen/sources.json @@ -0,0 +1,66 @@ +{ + "embossc_sources": [ + "compiler/__init__.py", + "compiler/back_end/__init__.py", + "compiler/back_end/cpp/__init__.py", + "compiler/back_end/cpp/attributes.py", + "compiler/back_end/cpp/emboss_codegen_cpp.py", + "compiler/back_end/cpp/generated_code_templates", + "compiler/back_end/cpp/header_generator.py", + "compiler/back_end/util/__init__.py", + "compiler/back_end/util/code_template.py", + "compiler/front_end/__init__.py", + "compiler/front_end/attribute_checker.py", + "compiler/front_end/attributes.py", + "compiler/front_end/constraints.py", + "compiler/front_end/dependency_checker.py", + "compiler/front_end/emboss_front_end.py", + "compiler/front_end/error_examples", + "compiler/front_end/expression_bounds.py", + "compiler/front_end/generated/cached_parser.py", + "compiler/front_end/glue.py", + "compiler/front_end/lr1.py", + "compiler/front_end/make_parser.py", + "compiler/front_end/module_ir.py", + "compiler/front_end/parser.py", + "compiler/front_end/prelude.emb", + "compiler/front_end/reserved_words", + "compiler/front_end/symbol_resolver.py", + "compiler/front_end/synthetics.py", + "compiler/front_end/tokenizer.py", + "compiler/front_end/type_check.py", + "compiler/front_end/write_inference.py", + "compiler/util/__init__.py", + "compiler/util/attribute_util.py", + "compiler/util/error.py", + "compiler/util/expression_parser.py", + "compiler/util/ir_data.py", + "compiler/util/ir_data_fields.py", + "compiler/util/ir_data_utils.py", + "compiler/util/ir_util.py", + "compiler/util/name_conversion.py", + "compiler/util/parser_types.py", + "compiler/util/parser_util.py", + "compiler/util/resources.py", + "compiler/util/simple_memoizer.py", + "compiler/util/traverse_ir.py", + "embossc" + ], + "emboss_runtime_cpp_sources": [ + "runtime/cpp/emboss_arithmetic.h", + "runtime/cpp/emboss_arithmetic_all_known_generated.h", + "runtime/cpp/emboss_arithmetic_maximum_operation_generated.h", + "runtime/cpp/emboss_array_view.h", + "runtime/cpp/emboss_bit_util.h", + "runtime/cpp/emboss_constant_view.h", + "runtime/cpp/emboss_cpp_types.h", + "runtime/cpp/emboss_cpp_util.h", + "runtime/cpp/emboss_defines.h", + "runtime/cpp/emboss_enum_view.h", + "runtime/cpp/emboss_maybe.h", + "runtime/cpp/emboss_memory_util.h", + "runtime/cpp/emboss_prelude.h", + "runtime/cpp/emboss_text_util.h", + "runtime/cpp/emboss_view_parameters.h" + ] +} diff --git a/scripts/build_helpers/generate_build_files.py b/scripts/build_helpers/generate_build_files.py new file mode 100755 index 0000000..480b3b2 --- /dev/null +++ b/scripts/build_helpers/generate_build_files.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generates build helper files from `build.json`.""" + +import json +import os + +COPYRIGHT_HEADER = """# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + + +def read_build_json(file_path): + """Reads build.json, strips comments, and returns the parsed JSON data.""" + with open(file_path, "r") as f: + # Simple comment stripping for JSONC format + content = "".join(line for line in f if not line.strip().startswith("//")) + return json.loads(content) + + +def generate_sources_json(output_dir, data): + """Generates gen/sources.json.""" + output_path = os.path.join(output_dir, "sources.json") + with open(output_path, "w") as f: + json.dump(data, f, indent=4) + f.write("\n") + print(f"Generated {output_path}") + + +def generate_sources_bzl(output_dir, data): + """Generates gen/sources.bzl.""" + output_path = os.path.join(output_dir, "sources.bzl") + with open(output_path, "w") as f: + f.write(COPYRIGHT_HEADER) + f.write("\n\n") + f.write("# Automatically generated by generate_build_files.py.\n") + f.write("# Do not edit this file directly.\n") + f.write("\n") + f.write("EMBOSSC_SOURCES = [\n") + for source in data["embossc_sources"]: + f.write(f' "{source}",\n') + f.write("]\n\n") + f.write("EMBOSS_RUNTIME_CPP_SOURCES = [\n") + for source in data["emboss_runtime_cpp_sources"]: + f.write(f' "{source}",\n') + f.write("]\n") + print(f"Generated {output_path}") + + +def generate_sources_cmake(output_dir, data): + """Generates gen/sources.cmake.""" + output_path = os.path.join(output_dir, "sources.cmake") + with open(output_path, "w") as f: + f.write(COPYRIGHT_HEADER) + f.write("\n\n") + f.write("# Automatically generated by generate_build_files.py.\n") + f.write("# Do not edit this file directly.\n\n") + f.write("set(EMBOSSC_SOURCES\n") + for source in data["embossc_sources"]: + f.write(f' "{source}"\n') + f.write(")\n\n") + f.write("set(EMBOSS_RUNTIME_CPP_SOURCES\n") + for source in data["emboss_runtime_cpp_sources"]: + f.write(f' "{source}"\n') + f.write(")\n") + print(f"Generated {output_path}") + + +def generate_sources_gni(output_dir, data): + """Generates gen/sources.gni.""" + output_path = os.path.join(output_dir, "sources.gni") + with open(output_path, "w") as f: + f.write(COPYRIGHT_HEADER) + f.write("\n\n") + f.write("# Automatically generated by generate_build_files.py.\n") + f.write("# Do not edit this file directly.\n\n") + f.write("embossc_sources = [\n") + for source in data["embossc_sources"]: + f.write(f' "{source}",\n') + f.write("]\n\n") + f.write("emboss_runtime_cpp_sources = [\n") + for source in data["emboss_runtime_cpp_sources"]: + f.write(f' "{source}",\n') + f.write("]\n") + print(f"Generated {output_path}") + + +def main(): + workspace_root = os.environ.get("BUILD_WORKSPACE_DIRECTORY", os.getcwd()) + build_json_path = os.path.join(workspace_root, "build.json") + gen_dir = os.path.join(workspace_root, "gen") + + os.makedirs(gen_dir, exist_ok=True) + + data = read_build_json(build_json_path) + + generate_sources_json(gen_dir, data) + generate_sources_bzl(gen_dir, data) + generate_sources_cmake(gen_dir, data) + generate_sources_gni(gen_dir, data) + + +if __name__ == "__main__": + main() From e2de64f1daebbf35b65cd65c4e4555a44808a5fd Mon Sep 17 00:00:00 2001 From: Eric Rahm Date: Wed, 1 Oct 2025 17:36:00 +0000 Subject: [PATCH 4/4] Update docs to include notes on build.json Add details on how to update `build.json` and generate the build files. --- doc/contributing.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/doc/contributing.md b/doc/contributing.md index f6146a4..8074449 100644 --- a/doc/contributing.md +++ b/doc/contributing.md @@ -29,6 +29,41 @@ comes from a non-Google open source project needs to have an acceptable license and be committed to the Emboss repository in a specific location. +### Updating Build Helper Files + +The Emboss project provides a set of generated files in the `gen/` directory +that can be used by other build systems (like CMake, GN, etc.) to easily +include the Emboss compiler and C++ runtime sources. + +These files are generated from a single source of truth: `build.json`. + +- **`build.json`**: This file, located in the project root, contains the + canonical lists of source files required for the compiler and the C++ + runtime. It is the only file in this system that should be manually + edited, though it is typically updated via a script. + +- **`gen/`**: This directory contains the generated helper files. **Do not + edit these files directly**, as they will be overwritten. + +If you change the dependencies of the Emboss compiler or runtime in the Bazel +build system, you will need to update `build.json` and regenerate the helper +files. + +- To update `build.json` with the latest sources from Bazel, run: + ```bash + ./scripts/build_helpers/manage_build_json.py + ``` + +- To regenerate the files in `gen/` after modifying `build.json`, run: + ```bash + ./scripts/build_helpers/generate_build_files.py + ``` + +A CI check is in place to ensure that `build.json` is always in sync with the +Bazel build. If this check fails, you will need to run the `manage_build_json.py` +script and commit the changes. + + ### How-To Guides This document covers the process of getting a change into the main Emboss