feat(semantic tokens): add primitive semantic token type

docs/en/features/semantic-tokens.md

@@ -104,6 +104,10 @@ Kinds derived from the token stream itself, independent of the AST.
   #endif
 
   #pragma pack(1)
+
+  #
+  #define STRINGIZE(x) #x
+  const char* stringized = STRINGIZE(abc);
   ```
 
   </details>
@@ -173,7 +177,7 @@ Kinds derived from the token stream itself, independent of the AST.
 
   </details>
 
-- [ ] Primitive token type — a distinct kind for built-in types instead of plain `keyword`
+- [x] Primitive token type — a distinct kind for built-in types instead of plain `keyword`
 
   <details>
   <summary>Example</summary>
@@ -182,6 +186,9 @@ Kinds derived from the token stream itself, independent of the AST.
   int number = 0;
   float ratio = 0.5f;
   void act();
+  unsigned long long wide_number = 0;
+  __int128 extended_int = 0;
+  _Float16 extended_float = 0;
   ```
 
   </details>

docs/zh/features/semantic-tokens.md

@@ -146,7 +146,7 @@
 
 ### 额外 Token 类型
 
-- [ ] Primitive token 类型用于内置类型（`int`、`float`、`void` 等）
+- [x] Primitive token 类型用于内置类型（`int`、`float`、`void` 等）
 - [ ] Bracket token 类型用于匹配的括号对（`[]`、`()`、`{}`、`<>`）
 
 ## Token 修饰符

editors/vscode/package.json

@@ -164,6 +164,11 @@
         "id": "angle",
         "description": "Angle brackets `<>`",
         "superType": "operator"
+      },
+      {
+        "id": "primitive",
+        "description": "C/C++ primitive type (e.g., int, float)",
+        "superType": "keyword"
       }
     ],
     "views": {

src/feature/document_symbols.cpp

@@ -42,6 +42,7 @@ auto to_protocol_symbol_kind(SymbolKind kind) -> protocol::SymbolKind {
         case SymbolKind::Parameter:
         case SymbolKind::Label:
         case SymbolKind::Keyword:
+        case SymbolKind::Primitive:
         case SymbolKind::Directive:
         case SymbolKind::MacroParameter:
         case SymbolKind::Attribute: return Variable;

src/feature/semantic_tokens.cpp

@@ -14,6 +14,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/DeclObjC.h"
+#include "clang/Basic/TokenKinds.h"
 
 namespace clice::feature {
 
@@ -322,47 +323,69 @@ class SemanticTokensCollector {
     /// small state machine for preprocessor directive context.
     Classified classify_lexical(const clang::syntax::Token& token, std::uint32_t offset) {
         Classified lexical;
+        bool is_identifier_like = clang::tok::isAnyIdentifier(token.kind());
 
         switch(token.kind()) {
             case clang::tok::numeric_constant: lexical.kind = SymbolKind::Number; break;
+
+            /// Character literals
             case clang::tok::char_constant:
             case clang::tok::wide_char_constant:
             case clang::tok::utf8_char_constant:
             case clang::tok::utf16_char_constant:
             case clang::tok::utf32_char_constant: lexical.kind = SymbolKind::Character; break;
+
+            /// String literals
             case clang::tok::string_literal:
             case clang::tok::wide_string_literal:
             case clang::tok::utf8_string_literal:
             case clang::tok::utf16_string_literal:
             case clang::tok::utf32_string_literal: lexical.kind = SymbolKind::String; break;
-            case clang::tok::hash: {
-                if(directive_context == DirectiveContext::None) {
-                    lexical.kind = SymbolKind::Directive;
-                    directive_context = DirectiveContext::AfterHash;
-                }
+
+            /// Fundamental and Clang/GNU builtin types; `__fp16` lexes as
+            /// `kw_half`.
+            case clang::tok::kw_bool:
+            case clang::tok::kw_char:
+            case clang::tok::kw_wchar_t:
+            case clang::tok::kw_char8_t:
+            case clang::tok::kw_char16_t:
+            case clang::tok::kw_char32_t:
+            case clang::tok::kw_double:
+            case clang::tok::kw_float:
+            case clang::tok::kw_int:
+            case clang::tok::kw_long:
+            case clang::tok::kw_short:
+            case clang::tok::kw_signed:
+            case clang::tok::kw_unsigned:
+            case clang::tok::kw_void:
+            case clang::tok::kw_half:
+            case clang::tok::kw__BitInt:
+            case clang::tok::kw__Bool:
+            case clang::tok::kw__Complex:
+            case clang::tok::kw__Decimal128:
+            case clang::tok::kw__Decimal32:
+            case clang::tok::kw__Decimal64:
+            case clang::tok::kw__ExtInt:
+            case clang::tok::kw__Float16:
+            case clang::tok::kw__Imaginary:
+            case clang::tok::kw___bf16:
+            case clang::tok::kw___float128:
+            case clang::tok::kw___ibm128:
+            case clang::tok::kw___int64:
+            case clang::tok::kw___int128: {
+                lexical.kind = SymbolKind::Primitive;
+                is_identifier_like = true;
                 break;
             }
+
+            /// PP directive hash
+            case clang::tok::hash: break;
+
             default: {
-                if(directive_context == DirectiveContext::AfterHash) {
-                    /// The directive name right after `#`, e.g. `include`, `if`.
-                    lexical.kind = SymbolKind::Directive;
-                    auto spelling = content.substr(offset, token.length());
-                    if(spelling == "include" || spelling == "include_next" ||
-                       spelling == "import" || spelling == "embed") {
-                        directive_context = DirectiveContext::InIncludeName;
-                    } else if(spelling == "define") {
-                        directive_context = DirectiveContext::AfterDefine;
-                    } else {
-                        directive_context = DirectiveContext::InDirective;
-                    }
-                } else if(directive_context == DirectiveContext::AfterDefine) {
-                    /// The macro name of a #define. Also covers preamble
-                    /// defines under a PCH, where no MacroDefine node exists
-                    /// (the preamble's directives live in the PCH compile).
-                    lexical.kind = SymbolKind::Macro;
-                    directive_context = DirectiveContext::InDirective;
-                } else if(clang::tok::getKeywordSpelling(token.kind())) {
+                if(clang::tok::getKeywordSpelling(token.kind())) {
                     lexical.kind = SymbolKind::Keyword;
+                    is_identifier_like = true;
+                    break;
                 } else if(auto* punctuator = clang::tok::getPunctuatorSpelling(token.kind())) {
                     /// Alternative operator spellings (and, or, not, ...) lex
                     /// as their punctuator kinds but are written as words.
@@ -376,26 +399,66 @@ class SemanticTokensCollector {
             }
         }
 
-        /// The filename of an #include: either a string literal or the
-        /// `<vector>` token sequence; adjacent merging joins the pieces.
-        /// The header context ends with the filename, so directive operands
-        /// after it (e.g. #embed parameters) keep their own classification.
-        if(directive_context == DirectiveContext::InIncludeName &&
-           lexical.kind != SymbolKind::Directive) {
-            if(token.kind() == clang::tok::less) {
-                directive_context = DirectiveContext::InAngledName;
-                lexical = {SymbolKind::Header, 0};
-            } else if(lexical.kind == SymbolKind::String) {
-                directive_context = DirectiveContext::InDirective;
+        /// Move the directive state machine to classify tokens in a PP directive.
+        switch(directive_context) {
+            case DirectiveContext::None: {
+                if(token.kind() == clang::tok::hash) {
+                    directive_context = DirectiveContext::AfterHash;
+                    lexical.kind = SymbolKind::Directive;
+                }
+                break;
+            }
+            case DirectiveContext::AfterHash: {
+                /// The directive name right after `#`, e.g. `include`, `if`.
+                if(is_identifier_like) {
+                    lexical.kind = SymbolKind::Directive;
+                }
+
+                auto spelling = content.substr(offset, token.length());
+                if(spelling == "include" || spelling == "include_next" || spelling == "import" ||
+                   spelling == "embed") {
+                    directive_context = DirectiveContext::InIncludeName;
+                } else if(spelling == "define") {
+                    directive_context = DirectiveContext::AfterDefine;
+                } else {
+                    directive_context = DirectiveContext::InDirective;
+                }
+                break;
+            }
+            case DirectiveContext::InIncludeName: {
+                /// The filename of an #include: either a string literal or the
+                /// `<vector>` token sequence; adjacent merging joins the pieces.
+                /// The header context ends with the filename, so directive operands
+                /// after it (e.g. #embed parameters) keep their own classification.
+                if(token.kind() == clang::tok::less) {
+                    directive_context = DirectiveContext::InAngledName;
+                    lexical = {SymbolKind::Header, 0};
+                } else if(lexical.kind == SymbolKind::String) {
+                    directive_context = DirectiveContext::InDirective;
+                    lexical = {SymbolKind::Header, 0};
+                } else {
+                    directive_context = DirectiveContext::InDirective;
+                }
+                break;
+            }
+            case DirectiveContext::InAngledName: {
+                if(token.kind() == clang::tok::greater) {
+                    directive_context = DirectiveContext::InDirective;
+                }
                 lexical = {SymbolKind::Header, 0};
-            } else {
-                directive_context = DirectiveContext::InDirective;
+                break;
             }
-        } else if(directive_context == DirectiveContext::InAngledName) {
-            if(token.kind() == clang::tok::greater) {
+            case DirectiveContext::AfterDefine: {
+                /// The macro name of a #define. Also covers preamble
+                /// defines under a PCH, where no MacroDefine node exists
+                /// (the preamble's directives live in the PCH compile).
+                if(is_identifier_like) {
+                    lexical.kind = SymbolKind::Macro;
+                }
                 directive_context = DirectiveContext::InDirective;
+                break;
             }
-            lexical = {SymbolKind::Header, 0};
+            case DirectiveContext::InDirective: break;
         }
 
         return lexical;
@@ -647,10 +710,10 @@ class SemanticTokensCollector {
     enum class DirectiveContext : std::uint8_t {
         None,
         AfterHash,
-        InDirective,
         InIncludeName,
         InAngledName,
         AfterDefine,
+        InDirective,
     };
 
     CompilationUnitRef unit;

src/index/serialization.h

@@ -103,7 +103,7 @@ namespace clice::index {
 /// Bump it whenever a persisted type's reflected layout changes, or when
 /// symbol-hash semantics change (stored rows keyed by old hashes would
 /// silently stop matching newly computed ones).
-constexpr inline std::uint32_t index_format_version = 7;
+constexpr inline std::uint32_t index_format_version = 8;
 
 /// Serialize a reflected index blob to `os` as a verified-readable
 /// flatbuffer. Encoding only fails on structural impossibilities (e.g. more

src/semantic/symbol.cpp

@@ -38,8 +38,4 @@ SymbolKind SymbolKind::from(const clang::Decl* decl) {
     }
 }
 
-SymbolKind SymbolKind::from(const clang::tok::TokenKind kind) {
-    return {};
-}
-
 }  // namespace clice

src/semantic/symbol.h

@@ -4,7 +4,6 @@
 #include <string>
 
 #include "clang/AST/Decl.h"
-#include "clang/Basic/TokenKinds.h"
 
 namespace clice {
 
@@ -49,6 +48,7 @@ struct SymbolKind {
         Brace,           ///> `{` and `}`.
         Angle,           ///> `<` and `>`.
         Conflict,        ///> This token have multiple kinds.
+        Primitive,       ///< C/C++ built-in primitive type keyword.
         Invalid,
     };
 
@@ -72,8 +72,6 @@ struct SymbolKind {
 
     static SymbolKind from(const clang::Decl* decl);
 
-    static SymbolKind from(const clang::tok::TokenKind kind);
-
 private:
     Kind kind_value = Invalid;
 };

tests/snap/semantic_tokens/comments.snap.yml

@@ -15,11 +15,11 @@ input_file: comments.cpp
 - { loc: "11:0", text: " * spanning several lines", kind: comment }
 - { loc: "12:0", text: " */", kind: comment }
 - { loc: "13:0", text: "/// a doc comment", kind: comment }
-- { loc: "14:0", text: "int", kind: keyword }
+- { loc: "14:0", text: "int", kind: primitive }
 - { loc: "14:4", text: "after_comments", kind: variable, modifiers: [definition] }
 - { loc: "14:21", text: "0", kind: number }
 - { loc: "16:0", text: "/* first", kind: comment }
 - { loc: "17:0", text: "second */", kind: comment }
-- { loc: "17:10", text: "int", kind: keyword }
+- { loc: "17:10", text: "int", kind: primitive }
 - { loc: "17:14", text: "after_block", kind: variable, modifiers: [definition] }
 - { loc: "17:28", text: "1", kind: number }

tests/snap/semantic_tokens/declaration_tokens.snap.yml

@@ -14,12 +14,12 @@ input_file: declaration_tokens.cpp
 - { loc: "6:24", text: "Green", kind: enumMember, modifiers: [definition, readonly] }
 - { loc: "8:0", text: "struct", kind: keyword }
 - { loc: "8:7", text: "Shape", kind: struct, modifiers: [definition] }
-- { loc: "9:4", text: "int", kind: keyword }
+- { loc: "9:4", text: "int", kind: primitive }
 - { loc: "9:8", text: "sides", kind: field, modifiers: [definition] }
-- { loc: "12:0", text: "int", kind: keyword }
+- { loc: "12:0", text: "int", kind: primitive }
 - { loc: "12:4", text: "area", kind: function, modifiers: [declaration] }
-- { loc: "12:9", text: "int", kind: keyword }
+- { loc: "12:9", text: "int", kind: primitive }
 - { loc: "12:13", text: "width", kind: parameter, modifiers: [definition] }
-- { loc: "12:20", text: "int", kind: keyword }
+- { loc: "12:20", text: "int", kind: primitive }
 - { loc: "12:24", text: "height", kind: parameter, modifiers: [definition] }
 - { loc: "14:3", text: "// namespace demo", kind: comment }

tests/snap/semantic_tokens/directives.cpp

@@ -19,3 +19,7 @@ int flagged = 2;
 #endif
 
 #pragma pack(1)
+
+#
+#define STRINGIZE(x) #x
+const char* stringized = STRINGIZE(abc);

tests/snap/semantic_tokens/directives.snap.yml

@@ -8,24 +8,31 @@ input_file: directives.cpp
 - { loc: "3:0", text: "///", kind: comment }
 - { loc: "4:0", text: "/// - status: supported", kind: comment }
 - { loc: "5:0", text: "/// - order: 4", kind: comment }
-- { loc: "7:0", text: "int", kind: keyword }
+- { loc: "7:0", text: "int", kind: primitive }
 - { loc: "7:4", text: "before_conditional", kind: variable, modifiers: [definition] }
 - { loc: "7:25", text: "0", kind: number }
 - { loc: "9:0", text: "#if", kind: directive }
 - { loc: "9:4", text: "0", kind: number }
-- { loc: "10:0", text: "int", kind: keyword }
+- { loc: "10:0", text: "int", kind: primitive }
 - { loc: "11:0", text: "#else", kind: directive }
-- { loc: "12:0", text: "int", kind: keyword }
+- { loc: "12:0", text: "int", kind: primitive }
 - { loc: "12:4", text: "enabled_branch", kind: variable, modifiers: [definition] }
 - { loc: "12:21", text: "1", kind: number }
 - { loc: "13:0", text: "#endif", kind: directive }
 - { loc: "15:0", text: "#define", kind: directive }
 - { loc: "15:8", text: "FLAG", kind: macro, modifiers: [definition] }
 - { loc: "16:0", text: "#ifdef", kind: directive }
 - { loc: "16:7", text: "FLAG", kind: macro }
-- { loc: "17:0", text: "int", kind: keyword }
+- { loc: "17:0", text: "int", kind: primitive }
 - { loc: "17:4", text: "flagged", kind: variable, modifiers: [definition] }
 - { loc: "17:14", text: "2", kind: number }
 - { loc: "18:0", text: "#endif", kind: directive }
 - { loc: "20:0", text: "#pragma", kind: directive }
 - { loc: "20:13", text: "1", kind: number }
+- { loc: "22:0", text: "#", kind: directive }
+- { loc: "23:0", text: "#define", kind: directive }
+- { loc: "23:8", text: "STRINGIZE", kind: macro, modifiers: [definition] }
+- { loc: "24:0", text: "const", kind: keyword }
+- { loc: "24:6", text: "char", kind: primitive }
+- { loc: "24:12", text: "stringized", kind: variable, modifiers: [definition, readonly] }
+- { loc: "24:25", text: "STRINGIZE", kind: macro }