Merge pull request #1461 from clasp-developers/apple-silicon

Add support for LLVM16 and Apple Silicon (via bytecode)

Author: yitzchak

RELEASE_NOTES.md

@@ -1,9 +1,13 @@
-# Version 2.3.0 (LLVM15) Pending
+# Version 2.3.0 (LLVM15/LLVM16) Pending
 
 ## Added
 * Bytecode compiled images via build mode `:bytecode`.
 * FASL magic number sniffing for CL:LOAD. This enables FASL files to be
   loaded even if the filename extension is incorrect.
+* Preliminary Apple Silicon support. Must be compiled with bytecode
+  either by specifying `--build-mode=bytecode` when invoking koga or
+  by adding `:build-mode :bytecode` to the config.sexp file.
+* LLVM16 support. LLVM15 is still supported.
 
 ## Changed
 * Updated to Eclector v0.9.0

include/clasp/core/lispCallingConvention.h

@@ -92,7 +92,7 @@ LCC_RETURN_RAW general_entry_point_redirect_7(core::T_O* closure, core::T_O* far
 
 
 /*! This is X86_64 dependent code */
-#if defined(X86_64) && defined(_ADDRESS_MODEL_64)
+#if (defined(X86_64) || defined(ARM64)) && defined(_ADDRESS_MODEL_64)
 
 // This is VERY HACKISH
 // it's based on the System V Application Binary Interface for X86_64

include/clasp/llvmo/code.h

@@ -224,12 +224,9 @@ class CodeBlock_O : public core::CxxObject_O {
   uintptr_t     _HeadOffset;
   uintptr_t     _TailOffset;
   uintptr_t     _TotalSize;
-#ifdef USE_MMAP_CODEBLOCK
   void*         _mmapBlock;
   uintptr_t     _mmapSize;
-#else
   gctools::GCArray_moveable<uint8_t> _DataCode;
-#endif
   static constexpr size_t DefaultSize = 8*1024*1024;
 public:
   template <typename Stage>

include/clasp/llvmo/debugInfoExpose.h

@@ -834,6 +834,7 @@ namespace translate {
 
 
 
+#if __clang_major__ < 16
 namespace translate {
 template <>
 struct from_object<llvm::Optional<llvm::DIFile::ChecksumInfo<llvm::StringRef>>, std::true_type> {
@@ -873,7 +874,49 @@ struct from_object<llvm::Optional<llvm::DIFile::ChecksumInfo<llvm::StringRef>>,
   }
 };
 };
+#else
+namespace translate {
+template <>
+struct from_object<std::optional<llvm::DIFile::ChecksumInfo<llvm::StringRef>>, std::true_type> {
+  typedef std::optional<llvm::DIFile::ChecksumInfo<llvm::StringRef>> DeclareType;
+  DeclareType _v;
+  std::string _Storage;
+  from_object(core::T_sp object) {
+//    printf("%s:%d:%s object = %s\n", __FILE__, __LINE__, __FUNCTION__, core::_rep_(object).c_str());
+    core::SymbolToEnumConverter_sp converter = gc::As<core::SymbolToEnumConverter_sp>(llvmo::_sym_CSKEnum->symbolValue());
+    if (object.nilp()) {
+      DeclareType none;
+      this->_v = none;
+//      printf("%s:%d:%s ChecksumInfo RESET  this->_v -> %d\n", __FILE__, __LINE__, __FUNCTION__, this->_v.hasValue() );
+    } else if (gc::IsA<core::Symbol_sp>(object)) {
+      core::Symbol_sp sobject = gc::As<core::Symbol_sp>(object);
+      this->_Storage = sobject->symbolNameAsString();
+      llvm::DIFile::ChecksumKind kind = converter->enumForSymbol<llvm::DIFile::ChecksumKind>(sobject);
+      for ( int p=0; p<this->_Storage.size(); p++ ) {
+        if (this->_Storage[p] == '-') this->_Storage[p] = '_';
+      }
+      llvm::DIFile::ChecksumInfo<llvm::StringRef> checksum(kind,this->_Storage);
+      this->_v = checksum;
+//      printf("%s:%d:%s ChecksumInfo kind = %d  str = %s \n", __FILE__, __LINE__, __FUNCTION__, kind, this->_Storage.c_str() );
+    } else {
+      SIMPLE_ERROR("You must pass a valid Checksum like :CSK_MD5");
+    }
+  }
+  from_object(const from_object& orig) = delete;
+  from_object(from_object&& orig) : _Storage(std::move(orig._Storage)), _v(orig._v) {
+    if (this->_v.has_value()) {
+//      printf("%s:%d:%s from_object move ctor\n", __FILE__, __LINE__, __FUNCTION__ );
+      llvm::DIFile::ChecksumInfo<llvm::StringRef> checksum(this->_v->Kind,this->_Storage);
+      this->_v = checksum;
+    } else {
+//      printf("%s:%d:%s from_object move ctor NIL\n", __FILE__, __LINE__, __FUNCTION__ );
+    }
+  }
+};
+};
+#endif
 
+#if __clang_major__ < 16
 namespace translate {
 template <>
 struct from_object<llvm::Optional<llvm::StringRef>, std::true_type> {
@@ -897,6 +940,31 @@ struct from_object<llvm::Optional<llvm::StringRef>, std::true_type> {
   }
 };
 };
+#else
+namespace translate {
+template <>
+struct from_object<std::optional<llvm::StringRef>, std::true_type> {
+  typedef std::optional<llvm::StringRef> DeclareType;
+  std::string _Storage;
+  DeclareType _v;
+  from_object(core::T_sp object) {
+    if (object.nilp()) {
+      DeclareType none;
+      _v = none;
+    } else if (gc::IsA<core::String_sp>(object)) {
+      _Storage = gc::As<core::String_sp>(object)->get_std_string();
+      _v = _Storage;
+    } else {
+      SIMPLE_ERROR("You must pass a String or NIL");
+    }
+  }
+  from_object(const from_object& orig) = delete;
+  from_object(from_object&& orig) : _Storage(std::move(orig._Storage)), _v(orig._v) {
+    if (_v.has_value()) _v = _Storage;
+  }
+};
+};
+#endif
 
 // DIContext_O
 namespace llvmo {

include/clasp/llvmo/llvmoExpose.h

@@ -852,6 +852,7 @@ struct from_object<llvm::CodeGenOpt::Level, std::true_type> {
   }
 };
 
+#if __clang_major__ < 16
 template <>
   struct from_object<llvm::Optional<llvm::Reloc::Model>, std::true_type> {
   typedef llvm::Optional<llvm::Reloc::Model> DeclareType;
@@ -872,6 +873,28 @@ template <>
     }
   }
 };
+#else
+template <>
+  struct from_object<std::optional<llvm::Reloc::Model>, std::true_type> {
+  typedef std::optional<llvm::Reloc::Model> DeclareType;
+  DeclareType _v;
+  from_object(T_P object) {
+    if (object.nilp()) {
+//      SIMPLE_ERROR(("You must pass a valid RelocModel"));
+    }
+    if (core::Symbol_sp so = object.asOrNull<core::Symbol_O>()) {
+      if ( so == llvmo::_sym_RelocModel_undefined ) {
+        //printf("%s:%d Leaving llvm::Reloc::Model Undefined\n", __FILE__, __LINE__ );
+      } else {
+        core::SymbolToEnumConverter_sp converter = gc::As<core::SymbolToEnumConverter_sp>(llvmo::_sym_RelocModel->symbolValue());
+        this->_v = converter->enumForSymbol<llvm::Reloc::Model>(so);
+      }
+    } else {
+      SIMPLE_ERROR("You must pass a valid RelocModel or {}", _rep_(llvmo::_sym_RelocModel_undefined));
+    }
+  }
+};
+#endif
 
 template <>
 struct from_object<llvm::CodeModel::Model, std::true_type> {
@@ -4685,7 +4708,7 @@ ENUM_TRANSLATOR(llvm::GlobalValue::UnnamedAddr,llvmo::_sym_STARGlobalValueUnname
 
 
 
-
+#if __clang_major__ < 16
 namespace translate {
  template <typename T>
 struct from_object<llvm::Optional<T>> {
@@ -4703,6 +4726,25 @@ struct from_object<llvm::Optional<T>> {
    from_object(from_object&& orig) : _v(std::move(orig._v)) {};
  };
 }
+#else
+namespace translate {
+ template <typename T>
+struct from_object<std::optional<T>> {
+   typedef std::optional<T> DeclareType;
+   DeclareType _v;
+   from_object(core::T_sp o) {
+     if (o.unboundp()) {
+       return;
+     }
+     std::optional<T> val(from_object<T>(o)._v);
+     this->_v = val;
+     return;
+   }
+   from_object(const from_object& orig) = delete;
+   from_object(from_object&& orig) : _v(std::move(orig._v)) {};
+ };
+}
+#endif
 
 namespace llvmo {
 

src/analysis/clasp_gc.sif

@@ -1111,6 +1111,10 @@
              :offset-base-ctype "llvmo::CodeBlock_O" :layout-offset-field-names ("_TailOffset")}
 {fixed-field :offset-type-cxx-identifier "ctype_unsigned_long" :offset-ctype "unsigned long"
              :offset-base-ctype "llvmo::CodeBlock_O" :layout-offset-field-names ("_TotalSize")}
+{fixed-field :offset-type-cxx-identifier "RAW_POINTER_OFFSET" :offset-ctype "UnknownType"
+             :offset-base-ctype "llvmo::CodeBlock_O" :layout-offset-field-names ("_mmapBlock")}
+{fixed-field :offset-type-cxx-identifier "ctype_unsigned_long" :offset-ctype "unsigned long"
+             :offset-base-ctype "llvmo::CodeBlock_O" :layout-offset-field-names ("_mmapSize")}
 {variable-array0 :offset-base-ctype "llvmo::CodeBlock_O" :field-names ("_DataCode" "._Data")}
 {variable-capacity :ctype "unsigned char" :offset-base-ctype "llvmo::CodeBlock_O"
                    :length-field-names ("_DataCode" "._MaybeSignedLength")

src/analysis/clasp_gc_cando.sif

@@ -2967,6 +2967,10 @@
              :offset-base-ctype "llvmo::CodeBlock_O" :layout-offset-field-names ("_TailOffset")}
 {fixed-field :offset-type-cxx-identifier "ctype_unsigned_long" :offset-ctype "unsigned long"
              :offset-base-ctype "llvmo::CodeBlock_O" :layout-offset-field-names ("_TotalSize")}
+{fixed-field :offset-type-cxx-identifier "RAW_POINTER_OFFSET" :offset-ctype "UnknownType"
+             :offset-base-ctype "llvmo::CodeBlock_O" :layout-offset-field-names ("_mmapBlock")}
+{fixed-field :offset-type-cxx-identifier "ctype_unsigned_long" :offset-ctype "unsigned long"
+             :offset-base-ctype "llvmo::CodeBlock_O" :layout-offset-field-names ("_mmapSize")}
 {variable-array0 :offset-base-ctype "llvmo::CodeBlock_O" :field-names ("_DataCode" "._Data")}
 {variable-capacity :ctype "unsigned char" :offset-base-ctype "llvmo::CodeBlock_O"
                    :length-field-names ("_DataCode" "._MaybeSignedLength")
@@ -12867,6 +12871,12 @@
 {variable-field :offset-type-cxx-identifier "SMART_PTR_OFFSET"
                 :fixup-ctype-offset-type-key "gctools::smart_ptr<core::Symbol_O>"
                 :fixup-ctype-key "chem::AtomInfo" :layout-offset-field-names ("_Type")}
+{variable-field :offset-type-cxx-identifier "ctype_unsigned_long"
+                :fixup-ctype-offset-type-key "unsigned long" :fixup-ctype-key "chem::AtomInfo"
+                :layout-offset-field-names ("_MoleculeIndex")}
+{variable-field :offset-type-cxx-identifier "ctype_unsigned_long"
+                :fixup-ctype-offset-type-key "unsigned long" :fixup-ctype-key "chem::AtomInfo"
+                :layout-offset-field-names ("_ResidueIndex")}
 {container-kind :stamp-name "STAMPWTAG_gctools__GCVector_moveable_gctools__smart_ptr_chem__FFVdw_O__"
                 :stamp-key "gctools::GCVector_moveable<gctools::smart_ptr<chem::FFVdw_O>>"
                 :parent-class "gctools::GCContainer" :lisp-class-base nil

src/asttooling/astExpose1.cc

@@ -508,8 +508,10 @@ void initialize_astExpose() {
   scope_Type_TemplateTypeParm.def("desugar", &clang::TemplateTypeParmType::desugar);
   scope_Type_SubstTemplateTypeParm.def("desugar", &clang::SubstTemplateTypeParmType::desugar);
   scope_Type_TemplateSpecialization.def("getTemplateName", &clang::TemplateSpecializationType::getTemplateName);
+#if __clang_major__ < 16
   scope_Type_TemplateSpecialization.def("getNumArgs", &clang::TemplateSpecializationType::getNumArgs);
   scope_Type_TemplateSpecialization.def("getArg", &clang::TemplateSpecializationType::getArg);
+#endif
   scope_Type_TemplateSpecialization.def("desugar", &clang::TemplateSpecializationType::desugar);
   scope_Type_TemplateSpecialization.def("getAliasedType", &clang::TemplateSpecializationType::getAliasedType);
   scope_Type_TemplateSpecialization.def("isTypeAlias", &clang::TemplateSpecializationType::isTypeAlias);

src/asttooling/clangTooling.cc

@@ -526,10 +526,17 @@ CL_DEFUN core::T_sp ast_tooling__clangVersionString() {
 
 core::T_sp ast_tooling__getSingleMatcher(core::T_sp variantMatcher) {
   clang::ast_matchers::dynamic::VariantMatcher *vp = gc::As<core::WrappedPointer_sp>(variantMatcher)->cast<clang::ast_matchers::dynamic::VariantMatcher>();
+#if __clang_major__ < 16  
   llvm::Optional<clang::ast_matchers::internal::DynTypedMatcher> dtm = vp->getSingleMatcher();
   if (dtm.hasValue()) {
     return clbind::Wrapper<clang::ast_matchers::internal::DynTypedMatcher>::make_wrapper(*dtm, reg::registered_class<clang::ast_matchers::internal::DynTypedMatcher>::id);
   }
+#else
+  std::optional<clang::ast_matchers::internal::DynTypedMatcher> dtm = vp->getSingleMatcher();
+  if (dtm.has_value()) {
+    return clbind::Wrapper<clang::ast_matchers::internal::DynTypedMatcher>::make_wrapper(*dtm, reg::registered_class<clang::ast_matchers::internal::DynTypedMatcher>::id);
+  }
+#endif
   return nil<core::T_O>();
 };
 
@@ -998,8 +1005,13 @@ CL_DEFUN core::T_sp ast_tooling__parse_dynamic_matcher(const string& matcher)
 //  printf("%s:%d:%s got matcher %s\n", __FILE__, __LINE__, __FUNCTION__, matcher.c_str());
   llvm::StringRef matchersr(matcher);
   clang::ast_matchers::dynamic::Diagnostics error;
+#if __clang_major__ < 16  
   llvm::Optional<clang::ast_matchers::dynamic::DynTypedMatcher> Matcher =
     clang::ast_matchers::dynamic::Parser::parseMatcherExpression(matchersr, NULL, NULL, &error);
+#else
+  std::optional<clang::ast_matchers::dynamic::DynTypedMatcher> Matcher =
+    clang::ast_matchers::dynamic::Parser::parseMatcherExpression(matchersr, NULL, NULL, &error);
+#endif
   if (!Matcher) {
     SIMPLE_ERROR("Could not parse expression {}", matcher);
   }

src/core/backtrace.cc

@@ -135,14 +135,24 @@ static T_sp getSourcePosInfoForAddress(llvmo::DWARFContext_sp dcontext, llvmo::S
   }
 
   std::string source_path = info.FileName;
+#if __clang_major__ < 16
   if (info.Source.hasValue()) {
     std::smatch match;
     std::string source = info.Source.getValue().str();
     if (std::regex_search(source, match, logical_pathname_regex)) {
       source_path = match[1];
     }
   }
-
+#else
+  if (info.Source.has_value()) {
+    std::smatch match;
+    std::string source = info.Source.value().str();
+    if (std::regex_search(source, match, logical_pathname_regex)) {
+      source_path = match[1];
+    }
+  }
+#endif
+  
   return core__makeSourcePosInfo(source_path, true, 0, false, info.Line, true, info.Column, true);
 }