Rv64

machines/pod_1x1/Makefile.machine.include

@@ -1,6 +1,8 @@
 BSG_MACHINE_PODS_X                    = 1
 BSG_MACHINE_PODS_Y                    = 1
 
+BSG_MACHINE_XLEN                      = 32
+
 BSG_MACHINE_GLOBAL_X                  = 16
 BSG_MACHINE_GLOBAL_Y                  = 8
 

machines/pod_1x1_2X2Y/Makefile.machine.include

@@ -1,6 +1,8 @@
 BSG_MACHINE_PODS_X									  = 1
 BSG_MACHINE_PODS_Y									  = 1
 
+BSG_MACHINE_XLEN                      = 32
+
 BSG_MACHINE_GLOBAL_X                  = 2
 BSG_MACHINE_GLOBAL_Y                  = 2
 

machines/pod_1x1_4X2Y/Makefile.machine.include

@@ -1,6 +1,8 @@
 BSG_MACHINE_PODS_X									  = 1
 BSG_MACHINE_PODS_Y									  = 1
 
+BSG_MACHINE_XLEN                      = 32
+
 BSG_MACHINE_GLOBAL_X                  = 4
 BSG_MACHINE_GLOBAL_Y                  = 2
 

machines/pod_1x1_4X2Y_64b/Makefile.machine.include

@@ -0,0 +1,44 @@
+BSG_MACHINE_PODS_X									  = 1
+BSG_MACHINE_PODS_Y									  = 1
+
+BSG_MACHINE_XLEN                      = 64
+
+BSG_MACHINE_GLOBAL_X                  = 4
+BSG_MACHINE_GLOBAL_Y                  = 2
+
+BSG_MACHINE_X_CORD_WIDTH              = 7
+BSG_MACHINE_Y_CORD_WIDTH              = 7
+
+BSG_MACHINE_RUCHE_FACTOR_X            = 3
+BSG_MACHINE_BARRIER_RUCHE_FACTOR_X    = 3
+BSG_MACHINE_WH_RUCHE_FACTOR           = 2
+
+BSG_MACHINE_NUM_VCACHE_ROWS           = 1
+BSG_MACHINE_VCACHE_SET                = 64
+BSG_MACHINE_VCACHE_WAY                = 4
+BSG_MACHINE_VCACHE_BLOCK_SIZE_WORDS   = 8
+BSG_MACHINE_VCACHE_DMA_DATA_WIDTH     = 32
+BSG_MACHINE_NUM_VCACHES_PER_CHANNEL   = 2
+BSG_MACHINE_VCACHE_WORD_TRACKING      = 1
+
+
+BSG_MACHINE_DRAM_SIZE_WORDS           = 536870912
+BSG_MACHINE_DRAM_BANK_SIZE_WORDS      = 16777216
+BSG_MACHINE_DRAM_INCLUDED             = 1
+
+BSG_MACHINE_MAX_EPA_WIDTH             = 28
+BSG_MACHINE_BRANCH_TRACE_EN           = 0
+BSG_MACHINE_HETERO_TYPE_VEC           = default:0
+
+BSG_MACHINE_ORIGIN_Y_CORD             = 2
+BSG_MACHINE_ORIGIN_X_CORD             = 4
+
+BSG_MACHINE_HOST_Y_CORD               = 0
+BSG_MACHINE_HOST_X_CORD               = 4
+
+BSG_MACHINE_NETWORK_CFG               = e_network_half_ruche_x
+BSG_MACHINE_MEM_CFG                   = e_vcache_test_mem
+BSG_MACHINE_DRAMSIM3_PKG              = bsg_dramsim3_hbm2_8gb_x128_pkg
+
+BSG_MACHINE_SUBARRAY_X                = 1
+BSG_MACHINE_SUBARRAY_Y                = 1

machines/pod_1x1_hbm2/Makefile.machine.include

@@ -1,6 +1,8 @@
 BSG_MACHINE_PODS_X                    = 1
 BSG_MACHINE_PODS_Y                    = 1
 
+BSG_MACHINE_XLEN                      = 32
+
 BSG_MACHINE_GLOBAL_X                  = 16
 BSG_MACHINE_GLOBAL_Y                  = 8
 

machines/pod_4x4/Makefile.machine.include

@@ -1,6 +1,8 @@
 BSG_MACHINE_PODS_X                    = 4
 BSG_MACHINE_PODS_Y                    = 4
 
+BSG_MACHINE_XLEN                      = 32
+
 BSG_MACHINE_GLOBAL_X                  = 16
 BSG_MACHINE_GLOBAL_Y                  = 8
 

machines/pod_4x4_hbm2/Makefile.machine.include

@@ -1,6 +1,8 @@
 BSG_MACHINE_PODS_X                    = 4
 BSG_MACHINE_PODS_Y                    = 4
 
+BSG_MACHINE_XLEN                      = 32
+
 BSG_MACHINE_GLOBAL_X                  = 16
 BSG_MACHINE_GLOBAL_Y                  = 8
 

software/mk/Makefile.builddefs

@@ -1,8 +1,22 @@
 #########################################################
 # Chip Architecture
 
-ARCH_OP=rv32imaf
-ABI=ilp32f
+# The Chip Architecture is based on the machine's XLEN
+include $(BSG_MACHINE_PATH)/Makefile.machine.include
+
+BSG_XLEN = $(BSG_MACHINE_XLEN)
+
+$(info BSG_XLEN: $(BSG_XLEN))
+
+ifeq ($(BSG_XLEN), 64)
+	ARCH_OP=rv64imaf
+	ABI=lp64f
+	ARCH_NAME=riscv64
+else
+	ARCH_OP=rv32imaf
+	ABI=ilp32f
+	ARCH_NAME=riscv32
+endif
 
 
 #######################################################
@@ -56,11 +70,11 @@ ifeq ($(RISCV_BIN_DIR),)
 $(error RISCV_BIN_DIR not defined)
 endif
 
-RISCV_GCC     ?= $(RISCV_BIN_DIR)/riscv32-unknown-elf-dramfs-gcc
-RISCV_GXX     ?= $(RISCV_BIN_DIR)/riscv32-unknown-elf-dramfs-g++
+RISCV_GCC     ?= $(RISCV_BIN_DIR)/riscv64-unknown-elf-dramfs-gcc
+RISCV_GXX     ?= $(RISCV_BIN_DIR)/riscv64-unknown-elf-dramfs-g++
 RISCV_ELF2HEX ?= LD_LIBRARY_PATH=$(RISCV_BIN_DIR)/../lib $(RISCV_BIN_DIR)/elf2hex
-RISCV_OBJCOPY ?= $(RISCV_BIN_DIR)/riscv32-unknown-elf-dramfs-objcopy
-RISCV_AR      ?= $(RISCV_BIN_DIR)/riscv32-unknown-elf-dramfs-ar
+RISCV_OBJCOPY ?= $(RISCV_BIN_DIR)/riscv64-unknown-elf-dramfs-objcopy
+RISCV_AR      ?= $(RISCV_BIN_DIR)/riscv64-unknown-elf-dramfs-ar
 RISCV_SIM     ?= $(RISCV_BIN_DIR)/spike
 
 BSG_ROM_GEN     = $(BSG_IP_CORES_DIR)/bsg_mem/bsg_ascii_to_rom.py
@@ -83,7 +97,7 @@ vpath %.S $(COMMON_SRC_DIRS)
 # flags
 OPT_LEVEL ?= -O2
 RISCV_GCC_EXTRA_OPTS ?=
-RISCV_GCC_OPTS  =-march=$(ARCH_OP) -static -std=gnu99 -ffast-math -fno-common -mtune=bsg_vanilla_2020
+RISCV_GCC_OPTS  =-march=$(ARCH_OP) -mabi=$(ABI) -static -std=gnu99 -ffast-math -fno-common -mtune=bsg_vanilla_2020 -mcmodel=medany
 
 RISCV_GCC_OPTS +=$(RISCV_GCC_EXTRA_OPTS) -I$(BSG_MANYCORE_DIR)/software/spmd/common/ -I$(BSG_MANYCORE_DIR)/software/bsg_manycore_lib
 RISCV_GCC_OPTS +=-Dbsg_tiles_X=$(bsg_tiles_X) -Dbsg_tiles_Y=$(bsg_tiles_Y)
@@ -124,7 +138,7 @@ LFS_BLOCK_SIZE ?= 128
 LFS_BLOCK_COUNT ?= 64
 
 lfs.c: $(IN_FILES)
-	$(RISCV_BIN_DIR)/../riscv32-unknown-elf-dramfs/bin/dramfs_mklfs $(LFS_BLOCK_SIZE) $(LFS_BLOCK_COUNT) $(IN_FILES) > $@
+	$(RISCV_BIN_DIR)/../riscv64-unknown-elf-dramfs/bin/dramfs_mklfs $(LFS_BLOCK_SIZE) $(LFS_BLOCK_COUNT) $(IN_FILES) > $@
 endif
 
 # Manycore C library archiving rule
@@ -160,6 +174,7 @@ else ifeq ($(BSG_ELF_OFF_CHIP_MEM), 0)
 else
   $(error Invalid BSG_ELF_OFF_CHIP_MEM = $(BSG_ELF_OFF_CHIP_MEM); Only 0 and 1 are valid)
 endif
+  LINK_GEN_OPTS += --xlen=$(BSG_XLEN)
 
 LINK_SCRIPT ?= $(CURR_DIR)/bsg_link.ld
 
@@ -204,12 +219,12 @@ LLVM_CLANG        ?= $(LLVM_DIR)/bin/clang
 LLVM_CLANGPP      ?= $(LLVM_DIR)/bin/clang++
 LLVM_OPT          ?= $(LLVM_DIR)/bin/opt
 LLVM_LLC          ?= $(LLVM_DIR)/bin/llc
-CLANG_TARGET_OPTS ?= --target=riscv32 -march=$(ARCH_OP) -mabi=$(ABI)
+CLANG_TARGET_OPTS ?= --target=$(ARCH_NAME) -march=$(ARCH_OP) -mabi=$(ABI)
 
 # Ideally LLC should infer targets option provided to Clang. But LLC 11.0.0 fails
 # to infer the architecture extensions correctly. Hence the information is repeated
 # in LLC target options below.
-LLC_TARGET_OPTS   ?= -march=riscv32 -mcpu=hb-rv32 -mattr=+m,+a,+f
+LLC_TARGET_OPTS   ?= -march=$(ARCH_NAME) -mcpu=hb-32 -mattr=+m,+a,+f
 
 ifdef ENABLE_LLVM_PASSES
 PASS_DIR          ?= $(BSG_MANYCORE_DIR)/software/manycore-llvm-pass
@@ -230,15 +245,15 @@ $(LLVM_DIR):
 # Opt can run optimizations in any order, so it doesn't matter
 %.ll: %.c $(LLVM_DIR) $(RUNTIME_FNS)
 	$(LLVM_CLANG) $(CLANG_TARGET_OPTS) $(RISCV_GCC_OPTS) \
-		--sysroot=$(RISCV_BIN_DIR)/../riscv32-unknown-elf-dramfs \
+		--sysroot=$(RISCV_BIN_DIR)/../riscv64-unknown-elf-dramfs \
 		$(OPT_LEVEL) $(spmd_defs) -c -emit-llvm -S $(INCS) $< -o $@
 
 # do the same for C++ sources
 %.ll: %.cpp $(LLVM_DIR) $(RUNTIME_FNS)
 	$(LLVM_CLANGPP) $(CLANG_TARGET_OPTS) $(RISCV_GXX_OPTS) \
-		--sysroot=$(RISCV_BIN_DIR)/../riscv32-unknown-elf-dramfs \
-		-I$(RISCV_BIN_DIR)/../riscv32-unknown-elf-dramfs/include/c++/9.2.0 \
-		-I$(RISCV_BIN_DIR)/../riscv32-unknown-elf-dramfs/include/c++/9.2.0/riscv32-unknown-elf-dramfs \
+		--sysroot=$(RISCV_BIN_DIR)/../riscv64-unknown-elf-dramfs \
+		-I$(RISCV_BIN_DIR)/../riscv64-unknown-elf-dramfs/include/c++/9.2.0 \
+		-I$(RISCV_BIN_DIR)/../riscv64-unknown-elf-dramfs/include/c++/9.2.0/riscv64-unknown-elf-dramfs \
 		$(OPT_LEVEL) $(spmd_defs) -c -emit-llvm -S $(INCS) $< -o $@
 
 ifdef ENABLE_LLVM_PASSES
@@ -313,13 +328,13 @@ SKIP_ZEROS ?= 0
 		$(bsg_pods_X) $(bsg_pods_Y) \
 		$(BSG_MACHINE_NUM_VCACHE_ROWS) \
 		$(SKIP_DRAM_INSTRUCTION_LOAD) \
-		$(SKIP_ZEROS) > $*.nbf
+		$(SKIP_ZEROS) $(BSG_XLEN) > $*.nbf
 
 %.bin:  %.hex
 	python $(HEX2BIN) $< 32 > $@
 
 %.dis: %.riscv
-	$(RISCV_BIN_DIR)/riscv32-unknown-elf-dramfs-objdump -M numeric --disassemble-all -S $<
+	$(RISCV_BIN_DIR)/riscv64-unknown-elf-dramfs-objdump -M numeric --disassemble-all -S $<
 
 PROG_NAME ?= main
 

software/py/bsg_manycore_link_gen.py

@@ -44,16 +44,18 @@ class bsg_manycore_link_gen:
     "/*********************************************************\n" \
     + " BSG Manycore Linker Script \n\n"
 
-  def __init__(self, default_data_loc, dram_size, imem_size, sp):
+  def __init__(self, default_data_loc, dram_size, imem_size, sp, xlen):
     self._default_data_loc = default_data_loc
     self._dram_size = dram_size
     self._imem_size = imem_size
     self._sp = sp
+    self._xlen = xlen
     self._opening_comment += \
         " data default: {0}\n".format(default_data_loc) \
         + " dram memory size: 0x{0:08x}\n".format(dram_size) \
         + " imem allocated size: 0x{0:08x}\n".format(imem_size) \
         + " stack pointer init: 0x{0:08x}\n".format(sp) \
+        + " RISC-V XLEN: {0}".format(xlen)\
         + "\n" \
         + " Generated at " + str(datetime.now()) + "\n" \
       + "**********************************************************/\n"
@@ -234,8 +236,14 @@ def script(self):
 
       # .text section virtual address starts at 0x0 but
       # loaded at 0x80000000
+
+      # For 64b code, we currently set vaddr to 0x80000000
+      # so the toolchain can link successfully
       if sec == ".text.dram":
-        vaddr = "0x0"
+        if self._xlen == '64':
+          vaddr = "0x80000000"
+        else:
+          vaddr = "0x0"
 
       # Append .dram to output section name
       if re.search(".dram$", sec) == None and self._default_data_loc == 'dram':
@@ -315,12 +323,16 @@ def script(self):
   parser.add_argument('--out',
     help = 'Output file name',
     default = None)
+  parser.add_argument('--xlen',
+    help = 'RISC-V XLEN',
+    default = '32',
+    choices = ['32', '64'])
   args = parser.parse_args()
 
 
   # Generate linker script
   link_gen = bsg_manycore_link_gen(args.default_data_loc, args.dram_size,
-      args.imem_size, args.sp)
+      args.imem_size, args.sp, args.xlen)
 
   if args.out is None:
     print(link_gen.script())

software/py/nbf.py

@@ -38,7 +38,7 @@ def __init__(self, config):
 
     # fixed arch params
     self.icache_size = 1024
-    self.data_width = 32
+    self.data_width = config["machine_xlen"]
 
     # input binary
     self.riscv_file = config["riscv_file"]
@@ -102,13 +102,14 @@ def print_nbf(self, x, y, epa, data):
     print(line)
 
   # read objcopy dumped in 'verilog' format.
-  # return in EPA (word addr) and 32-bit value dictionary
+  # return in EPA (word addr) and 32-bit / 64-bit value dictionary
   def read_objcopy(self, section, output_file):
 
     # make sure that you have riscv tool binaries in
     # bsg_manycore/software/riscv-tools/riscv-install/bin
     dirname = os.path.abspath(os.path.dirname(__file__))
-    objcopy_path = os.path.join(dirname, "../riscv-tools/riscv-install/bin/riscv32-unknown-elf-dramfs-objcopy")
+
+    objcopy_path = os.path.join(dirname, "../riscv-tools/riscv-install/bin/riscv64-unknown-elf-dramfs-objcopy")
 
     if not os.path.isfile(objcopy_path):
       print("install riscv-tools first...")
@@ -478,7 +479,7 @@ def dump(self):
 #
 if __name__ == "__main__":
 
-  if len(sys.argv) == 23:
+  if len(sys.argv) == 24:
     # config setting
     config = {
       "riscv_file" : sys.argv[1],
@@ -503,7 +504,8 @@ def dump(self):
       "num_pods_y" : int(sys.argv[19]),
       "num_vcache_rows" : int(sys.argv[20]),
       "skip_dram_instruction_load": int(sys.argv[21]),
-      "skip_zeros": int(sys.argv[22])
+      "skip_zeros": int(sys.argv[22]),
+      "machine_xlen": int(sys.argv[23])
     }
 
     converter = NBF(config)
@@ -519,6 +521,6 @@ def dump(self):
     command += "{num_pods_x} {num_pods_y}"
     command += "{num_vcache_rows}"
     command += "{skip_dram_instruction_load}"
-    command += "{skip_zeros}"
+    command += "{skip_zeros} {machine_xlen}"
     print(command)
 

software/riscv-tools/Makefile

@@ -27,9 +27,9 @@
 #
 .DEFAULT_GOAL := help
 
-TARGET_ARCH       := rv32imaf
-TARGET_ABI        := ilp32f
-TARGET_CFLAGS     := '-fno-common'
+TARGET_MULTILIB   := rv32imaf-ilp32f-- rv64imaf-lp64f--
+TARGET_CFLAGS     := '-fno-common -mstrict-align -mcmodel=medany'
+TARGET_CXXFLAGS   := '-fno-common -mstrict-align -mcmodel=medany'
 
 TOOLCHAIN_REPO    := riscv-gnu-toolchain
 TOOLCHAIN_URL     := https://github.com/bespoke-silicon-group/$(TOOLCHAIN_REPO)
@@ -135,14 +135,16 @@ configure-riscv-gnu-tools:
 	@echo "====================================="
 	@echo "Configuring toolchain..."
 	@echo "====================================="
+	./$(TOOLCHAIN_REPO)/riscv-gcc/gcc/config/riscv/multilib-generator $(TARGET_MULTILIB) > $(TOOLCHAIN_REPO)/riscv-gcc/gcc/config/riscv/t-elf-multilib
 	cd riscv-gnu-toolchain && \
-		./configure --prefix=$(RISCV) --disable-linux --with-arch=$(TARGET_ARCH) --with-abi=$(TARGET_ABI) --disable-gdb --with-tune=bsg_vanilla_2020
+		./configure --prefix=$(RISCV) --disable-linux --enable-multilib --disable-gdb --with-tune=bsg_vanilla_2020 --with-cmodel=medany
 
 build-riscv-gnu-tools: configure-riscv-gnu-tools
 	@echo "====================================="
 	@echo "Building toolchain..."
 	@echo "====================================="
-	cd riscv-gnu-toolchain && $(MAKE) -j 16 CFLAGS_FOR_TARGET_EXTRA=$(TARGET_CFLAGS)
+	cd riscv-gnu-toolchain && \
+		$(MAKE) -j 16 CFLAGS_FOR_TARGET_EXTRA=$(TARGET_CFLAGS) CXXFLAGS_FOR_TARGET_EXTRA=$(TARGET_CXXFLAGS)
 
 build-spike:
 	@echo "====================================="