Modularize chipyard

Author: jerryz123

.github/actions/run-tests/action.yml

@@ -20,7 +20,7 @@ runs:
       run: |
         conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)
         git submodule sync
-        ./scripts/init-submodules-no-riscv-tools.sh
+        ./scripts/init-submodules-no-riscv-tools.sh --full
       shell: bash -leo pipefail {0}
 
     # Note: You shouldn't need the other inputs since it shouldn't build RTL from scratch

.github/scripts/remote-do-rtl-build.sh

@@ -16,7 +16,7 @@ source $SCRIPT_DIR/defaults.sh
 
 cd $REMOTE_CHIPYARD_DIR
 git submodule sync
-./scripts/init-submodules-no-riscv-tools.sh
+./scripts/init-submodules-no-riscv-tools.sh --full
 
 # Constellation can run without espresso, but this improves
 # elaboration time drastically

.github/workflows/chipyard-run-tests.yml

@@ -365,7 +365,7 @@ jobs:
       - name: Build Gemmini FireMarshal
         run: |
           conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)
-          cd ${{ github.workspace }} && ./scripts/init-submodules-no-riscv-tools.sh
+          cd ${{ github.workspace }} && ./scripts/init-submodules-no-riscv-tools.sh --full
           cd ${{ github.workspace }} && source ./scripts/fix-open-files.sh
           git submodule update --init software/firemarshal && cd software/firemarshal && ./init-submodules.sh
           cd ${{ github.workspace }}/generators/gemmini/software && ${{ github.workspace }}/software/firemarshal/marshal -v -d build gemmini-smoke.json

build.sbt

@@ -153,20 +153,51 @@ lazy val testchipip = (project in file("generators/testchipip"))
   .settings(libraryDependencies ++= rocketLibDeps.value)
   .settings(commonSettings)
 
-lazy val chipyard = (project in file("generators/chipyard"))
-  .dependsOn(testchipip, rocketchip, boom, rocketchip_blocks, rocketchip_inclusive_cache,
-    dsptools, rocket_dsp_utils,
-    radiance, gemmini, icenet, tracegen, cva6, nvdla, sodor, ibex, fft_generator,
-    constellation, mempress, barf, shuttle, caliptra_aes, rerocc,
-    compressacc, saturn, ara, firrtl2_bridge, vexiiriscv, tacit)
-  .settings(libraryDependencies ++= rocketLibDeps.value)
-  .settings(
-    libraryDependencies ++= Seq(
-      "org.reflections" % "reflections" % "0.10.2"
+lazy val chipyard = {
+  // Base chipyard project with always-on dependencies
+  // Use explicit Project(...) so the project id remains 'chipyard'
+  var cy = Project(id = "chipyard", base = file("generators/chipyard"))
+    .dependsOn(
+      testchipip, rocketchip, boom, rocketchip_blocks, rocketchip_inclusive_cache,
+      dsptools, rocket_dsp_utils,
+      radiance, gemmini, icenet, tracegen, cva6, nvdla, sodor, ibex, fft_generator,
+      constellation, barf, shuttle, rerocc,
+      firrtl2_bridge, vexiiriscv, tacit
     )
+    .settings(libraryDependencies ++= rocketLibDeps.value)
+    .settings(
+      libraryDependencies ++= Seq(
+        "org.reflections" % "reflections" % "0.10.2"
+      )
+    )
+    .settings(commonSettings)
+    .settings(Compile / unmanagedSourceDirectories += file("tools/stage/src/main/scala"))
+
+  // Optional modules discovered via initialized submodules (no env or manifest)
+  val optionalModules: Seq[(String, ProjectReference)] = Seq(
+    "ara" -> ara,
+    "saturn" -> saturn,
+    "caliptra-aes-acc" -> caliptra_aes,
+    "compress-acc" -> compressacc,
+    "mempress" -> mempress
   )
-  .settings(commonSettings)
-  .settings(Compile / unmanagedSourceDirectories += file("tools/stage/src/main/scala"))
+
+  val discovered = optionalModules.filter { case (dir, _) => (file(s"generators/$dir/.git")).exists }
+
+  // Wire in project dependencies only for discovered modules
+  if (discovered.nonEmpty) {
+    // dependsOn requires ClasspathDep[ProjectReference]; wrap explicitly
+    cy = cy.dependsOn(discovered.map { case (_, pr) => sbt.ClasspathDependency(pr, None) }: _*)
+  }
+
+  // Also add their Chipyard-facing sources without symlinks
+  cy.settings(
+    Compile / unmanagedSourceDirectories ++=
+      discovered.map { case (dir, _) => baseDirectory.value / s"generators/$dir/chipyard" }.filter(_.exists)
+  )
+
+  cy
+}
 
 lazy val compressacc = (project in file("generators/compress-acc"))
   .dependsOn(rocketchip)

common.mk

@@ -76,29 +76,32 @@ HELP_COMMANDS += \
 # include additional subproject make fragments
 # see HELP_COMPILATION_VARIABLES
 #########################################################################################
-include $(base_dir)/generators/cva6/cva6.mk
-include $(base_dir)/generators/ibex/ibex.mk
-include $(base_dir)/generators/ara/ara.mk
 include $(base_dir)/generators/tracegen/tracegen.mk
-include $(base_dir)/generators/nvdla/nvdla.mk
-include $(base_dir)/generators/radiance/radiance.mk
 include $(base_dir)/tools/torture.mk
+# Optional generator make fragments should not fail build if absent
+-include $(base_dir)/generators/cva6/cva6.mk
+-include $(base_dir)/generators/ibex/ibex.mk
+-include $(base_dir)/generators/nvdla/nvdla.mk
+-include $(base_dir)/generators/radiance/radiance.mk
+# Wildcard include for standardized per-generator make fragments
+-include $(wildcard $(base_dir)/generators/*/chipyard.mk)
+
 
 #########################################################################################
 # Prerequisite lists
 #########################################################################################
 # Returns a list of files in directories $1 with single file extension $2.
 # If available, use 'fd' to find the list of files, which is faster than 'find'.
 ifeq ($(shell which fd 2> /dev/null),)
-	lookup_srcs = $(shell find -L $(1)/ -name target -prune -o \( -iname "*.$(2)" ! -iname ".*" \) -print 2> /dev/null)
+	lookup_srcs = $(shell find -L $(1)/ -name target -prune -o \( ! -xtype l -a -iname "*.$(2)" ! -iname ".*" \) -print 2> /dev/null)
 else
 	lookup_srcs = $(shell fd -L -t f -e $(2) . $(1))
 endif
 
 # Returns a list of files in directories $1 with *any* of the file extensions in $2
 lookup_srcs_by_multiple_type = $(foreach type,$(2),$(call lookup_srcs,$(1),$(type)))
 
-CHECK_SUBMODULES_COMMAND = echo "Checking all submodules in generators/ are initialized. Uninitialized submodules will be displayed" ; ! git submodule status $(base_dir)/generators | grep ^-
+CHECK_SUBMODULES_COMMAND = echo "Checking required submodules in generators/ are initialized. Uninitialized submodules will be displayed" ; ! git submodule status $(base_dir)/generators | grep '^-.*' | grep -vE "(ara|caliptra|compress|mempress|saturn)"
 
 SCALA_EXT = scala
 VLOG_EXT = sv v

docs/Generators/Ara.rst

@@ -3,7 +3,7 @@ Ara
 
 `Ara <https://github.com/pulp-platform/ara>`__ is a RISC-V vector unit developed by the PULP project.
 The Ara vector unit supports integration with either the Rocket or Shuttle in-order cores, following a similar methodology as used in the original Ara+CVA6 system.
-Example Ara configurations are listed in ``generators/chipyard/src/main/scala/config/AraConfigs.scala``.
+Example Ara configurations are provided by the Ara repository and compiled directly from ``generators/ara/chipyard/`` when the Ara submodule is initialized.
 
 .. Warning:: Ara only supports a partial subset of the full V-extension. Notably, we do not support virtual memory or precise traps with Ara.
 

docs/Generators/Saturn.rst

@@ -5,7 +5,7 @@ Saturn is a parameterized RISC-V Vector Unit generator currently supporting inte
 Saturn implements a compact short-vector-length vector microarchitecture suitable for deployment in a DSP-optimized core or area-efficient general-purpose core.
 
 More documentation on Saturn will be released in the future.
-A partial listing of supported Saturn configurations is in ``generators/chipyard/src/main/scala/config/SaturnConfigs.scala``.
+A partial listing of supported Saturn configurations is maintained in the Saturn repository and compiled directly from ``generators/saturn/chipyard/`` when the Saturn submodule is initialized.
 
 For now, the recommended Saturn configuration is ``GENV256D128ShuttleConfig``, which builds a dual-issue core with 256-bit VLEN, 128-bit wide SIMD datapath, and separate floating-point and integer vector issue units.
 
@@ -20,4 +20,3 @@ For now, the recommended Saturn configuration is ``GENV256D128ShuttleConfig``, w
  * ``Zve64d`` support for vector FP32 and FP64
  * Configurable vector length, from ``Zvl64b`` up (tested to ``Zvl4096b``)
  * Configurable datapath width, from 64b up (tested to 512b)
-