diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index bc6e369..a80bb17 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -18,7 +18,7 @@ Two design choices make this graph useful in a way that "throw the repo at an LL 2. **Every fact is derived, never inferred.** This is the core invariant: - > **Nothing in the graph is guessed by a language model.** Every node and edge comes from a real parser (Go's `go/ast`, a tree-sitter grammar, a language-specific scanner) or a deterministic algorithm (Tarjan's strongly-connected-components for cycles, pattern matching for layers). Run enola twice on the same commit and you get the same graph, byte for byte. + > **Nothing in the graph is guessed by a language model.** Every node and edge comes from a real parser (Go's `go/ast`, a tree-sitter grammar, a YAML/JSON scanner) or a deterministic algorithm (Tarjan's strongly-connected-components for cycles, pattern matching for layers). Run enola twice on the same commit and you get the same graph, byte for byte. That determinism is the whole point. An AI agent reasoning over enola's graph is standing on ground truth it can trust, instead of re-deriving the structure of your code — imperfectly, and at the cost of tokens — on every single task. @@ -343,7 +343,7 @@ Each extractor is detected by characteristic project files and then parses what | Python | tree-sitter | `pyproject.toml`, `setup.py`, `requirements.txt`, `Pipfile`, `pytest.ini`, `mypy.ini`, `tox.ini`, or `setup.cfg` (root or up to 3 levels deep) | | TypeScript | tree-sitter | `tsconfig.json`, `tsconfig.base.json`, or `package.json` with TypeScript (root or one level deep) | | Swift | tree-sitter | `Package.swift`, `.xcodeproj`, or `.xcworkspace` present | -| Ruby | regex scanner | `Gemfile` present | +| Ruby | tree-sitter | `Gemfile` present | | C++ | tree-sitter | a C++ source (`.cpp`/`.cc`/`.cxx`/`.hpp`/...) present, or a build file (`CMakeLists.txt`/`Makefile`/`meson.build`/`*.vcxproj`) plus any header | | OpenAPI | YAML/JSON scanner| any file containing `openapi:` or `swagger:` | @@ -361,7 +361,7 @@ Each extractor is detected by characteristic project files and then parses what **OpenAPI** scans for spec files independently of the main walker (so it finds them even when `*.yaml`/`*.json` are globally ignored), confirming candidates by an `openapi:`/`swagger:` key. It emits one `route` per operation enriched with method, `operationId`, summary, tags, and a spec back-reference; specs under an `openapi/client/` directory are marked `role:"client"`. Gateway extensions (`x-gateway-config`, `x-gateway-capabilities`) are parsed into props. -**Ruby** is Rails-aware: ActiveRecord models (`has_many`, `belongs_to`, scopes, table inference), the route DSL in `config/routes.rb` (resources, namespaces, member/collection blocks), and Packwerk package boundaries (`package.yml` dependency enforcement). It also tracks modules, classes, methods with visibility, mixins (`include`/`extend`/`prepend`), `ActiveSupport::Concern`, constants, and attributes. +**Ruby** is parsed with tree-sitter, replacing the former line-based regex scanner — the grammar handles heredocs, endless methods (`def x = expr`), multi-line expressions, and the nested scopes that tripped up the line scanner. It is Rails-aware: ActiveRecord models (`has_many`/`has_one`/`belongs_to`/`has_and_belongs_to_many`, scopes, table inference, explicit `self.table_name`) emit `storage` facts; the route DSL in `config/routes.rb` (plus `config/routes/*.rb` and packwerk `draw`) is walked from the real block structure, so nested `namespace`/`scope`/`resources`/`member`/`collection` blocks produce one `route` per RESTful action (honoring `only:`/`except:`); and Packwerk package boundaries (`package.yml` dependency enforcement, `app/public/` privacy) are parsed. It tracks modules, classes, methods with `public`/`private`/`protected` visibility, `class << self` eigenclass and `module_function` methods — now correctly typed as class methods rather than instance methods — mixins (`include`/`extend`/`prepend` → `implements` edges), `ActiveSupport::Concern` (flagged `concern: true`), constants, and `attr_*` accessors. Like the other AST extractors, it walks method bodies for call sites, emitting `calls` edges (qualified `Const.method`/`Ns::Class.method` and receiver `var.method`, deduplicated) and `implements` edges for superclasses — so Ruby participates in `traverse`, `find_path`, and `impact_analysis`. **C++** is parsed with tree-sitter and handles the header/source split that defines the language. Classes, structs, unions, enums (incl. `enum class`), namespaces, free functions and methods, data members, and `typedef`/`using` aliases become symbol facts named `.` — enola's `.` module convention on the outside, native C++ `::` scope inside. Because an out-of-line definition `Class::method` (parsed from a `qualified_identifier`) yields the same canonical name as its in-class declaration, a dedup pass **merges a header's method prototype with its `.cpp` definition** into a single symbol (the definition wins for file/line and carries the call-graph edges). Base classes become `implements` edges; method bodies are walked for `calls`/`instantiates` edges; quoted `#include "x.h"` becomes a `dependency` resolved to the declaring module, while system `<...>` includes are skipped. Templates are unwrapped to their inner declaration and flagged `templated`, and the walker descends through `#if`/`#ifdef` preprocessor guards (so code wrapped in `#if defined(HAVE_*)` and headers behind include guards are still extracted). *Limitation:* header/source merging relies on the `.h` and `.cpp` living in the same directory (the common layout); split `include/` + `src/` trees are not merged. diff --git a/README.md b/README.md index 2120a5d..da1e903 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ Working across several repos? Generate the first, then add the rest with append Framework- and platform-specific detection for each language is described in **[ARCHITECTURE.md → Supported languages](ARCHITECTURE.md#supported-languages)**. -> Python is parsed with tree-sitter and now contributes call and dependency edges to the graph, so `traverse`, `find_path`, and `impact_analysis` reach into Python code — not just modules and routes. +> Python and Ruby are parsed with tree-sitter and contribute call and dependency edges to the graph, so `traverse`, `find_path`, and `impact_analysis` reach into them — not just modules and routes. --- diff --git a/go.mod b/go.mod index 602fa96..bcb686c 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/tree-sitter/tree-sitter-cpp v0.22.4-0.20240818224355-b1a4e2b25148 github.com/tree-sitter/tree-sitter-java v0.21.1-0.20240824015150-576d8097e495 github.com/tree-sitter/tree-sitter-python v0.23.6 + github.com/tree-sitter/tree-sitter-ruby v0.21.1-0.20240818211811-7dbc1e2d0e2d github.com/tree-sitter/tree-sitter-typescript v0.23.2 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/internal/extractors/rubyextractor/routes.go b/internal/extractors/rubyextractor/routes.go index 66e8572..2aa400c 100644 --- a/internal/extractors/rubyextractor/routes.go +++ b/internal/extractors/rubyextractor/routes.go @@ -1,7 +1,6 @@ package rubyextractor import ( - "bufio" "log" "os" "path/filepath" @@ -11,20 +10,8 @@ import ( "github.com/enola-labs/enola/internal/facts" ) -// Route DSL regex patterns. -var ( - httpVerbRe = regexp.MustCompile(`^\s*(get|post|put|patch|delete)\s+['"]([^'"]+)['"](?:\s*,\s*to:\s*['"]([^'"]+)['"])?`) - resourcesRe = regexp.MustCompile(`^\s*resources?\s+:(\w+)`) - namespaceRe = regexp.MustCompile(`^\s*namespace\s+:(\w+)`) - scopePathRe = regexp.MustCompile(`^\s*scope\s+['"]([^'"]+)['"]`) - scopeModRe = regexp.MustCompile(`^\s*scope\s+module:\s*[:'"](\w+)`) - rootRe = regexp.MustCompile(`^\s*root\s+(?:to:\s*)?['"]([^'"]+)['"]`) - drawRe = regexp.MustCompile(`^\s*draw\s*\(\s*:(\w+)\s*\)`) - memberRe = regexp.MustCompile(`^\s*(member|collection)\s+do\b`) - doBlockRe = regexp.MustCompile(`\bdo\s*(?:\|[^|]*\|)?\s*$`) - onlyRe = regexp.MustCompile(`only:\s*\[([^\]]*)\]`) - exceptRe = regexp.MustCompile(`except:\s*\[([^\]]*)\]`) -) +// symbolListRe extracts symbol names from a string like ":index, :show, :create". +var symbolListRe = regexp.MustCompile(`:(\w+)`) // extractAllRoutes finds and parses all Rails route files in the repository. func extractAllRoutes(repoPath string, files []string) []facts.Fact { @@ -43,14 +30,12 @@ func extractAllRoutes(repoPath string, files []string) []facts.Fact { for _, relFile := range routeFiles { absFile := filepath.Join(repoPath, relFile) - f, err := os.Open(absFile) + src, err := os.ReadFile(absFile) if err != nil { log.Printf("[ruby-extractor] error reading route file %s: %v", relFile, err) continue } - routeFacts := parseRouteFile(f, relFile) - f.Close() - allFacts = append(allFacts, routeFacts...) + allFacts = append(allFacts, parseRouteFileAST(src, relFile)...) } return allFacts @@ -82,206 +67,6 @@ type routeScope struct { module string } -// parseRouteFile parses a single Rails route file. -func parseRouteFile(f *os.File, relFile string) []facts.Fact { - var result []facts.Fact - - scanner := bufio.NewScanner(f) - scanner.Buffer(make([]byte, 0, 256*1024), 1024*1024) - - var ( - lineNum int - scopeStack []routeScope - depth int - currentResource string - ) - - for scanner.Scan() { - lineNum++ - line := scanner.Text() - trimmed := strings.TrimSpace(line) - - if trimmed == "" || strings.HasPrefix(trimmed, "#") { - continue - } - - // Track end keywords. - if trimmed == "end" { - depth-- - if depth < 0 { - depth = 0 - } - if depth < len(scopeStack) { - scopeStack = scopeStack[:depth] - } - currentResource = "" - continue - } - - prefix := buildPrefix(scopeStack) - - // draw(:package_name) -- delegation to packwerk package routes. - if m := drawRe.FindStringSubmatch(line); m != nil { - result = append(result, facts.Fact{ - Kind: facts.KindRoute, - Name: prefix + "/" + m[1], - File: relFile, - Line: lineNum, - Props: map[string]any{ - "method": "DRAW", - "framework": "rails", - "language": "ruby", - "delegate": m[1], - }, - }) - continue - } - - // Namespace. - if m := namespaceRe.FindStringSubmatch(line); m != nil { - scopeStack = append(scopeStack, routeScope{ - pathPrefix: "/" + m[1], - module: m[1], - }) - depth++ - continue - } - - // Scope with path. - if m := scopePathRe.FindStringSubmatch(line); m != nil { - path := m[1] - if !strings.HasPrefix(path, "/") { - path = "/" + path - } - scopeStack = append(scopeStack, routeScope{pathPrefix: path}) - if doBlockRe.MatchString(line) { - depth++ - } - continue - } - - // Scope with module. - if m := scopeModRe.FindStringSubmatch(line); m != nil { - scopeStack = append(scopeStack, routeScope{module: m[1]}) - if doBlockRe.MatchString(line) { - depth++ - } - continue - } - - // Root route. - if m := rootRe.FindStringSubmatch(line); m != nil { - result = append(result, facts.Fact{ - Kind: facts.KindRoute, - Name: prefix + "/", - File: relFile, - Line: lineNum, - Props: map[string]any{ - "method": "GET", - "framework": "rails", - "language": "ruby", - "handler": m[1], - }, - Relations: []facts.Relation{ - {Kind: facts.RelDeclares, Target: filepath.Dir(relFile)}, - }, - }) - continue - } - - // HTTP verb routes: get '/path', post '/path', etc. - if m := httpVerbRe.FindStringSubmatch(line); m != nil { - method := strings.ToUpper(m[1]) - path := m[2] - handler := m[3] - - if !strings.HasPrefix(path, "/") { - path = "/" + path - } - fullPath := prefix + path - - props := map[string]any{ - "method": method, - "framework": "rails", - "language": "ruby", - } - if handler != "" { - props["handler"] = handler - } - - result = append(result, facts.Fact{ - Kind: facts.KindRoute, - Name: fullPath, - File: relFile, - Line: lineNum, - Props: props, - Relations: []facts.Relation{ - {Kind: facts.RelDeclares, Target: filepath.Dir(relFile)}, - }, - }) - continue - } - - // resources / resource. - if m := resourcesRe.FindStringSubmatch(line); m != nil { - resourceName := m[1] - currentResource = resourceName - resourcePath := prefix + "/" + resourceName - - actions := restfulActions(line) - for _, action := range actions { - method := action.method - path := resourcePath + action.suffix - - props := map[string]any{ - "method": method, - "framework": "rails", - "language": "ruby", - "resource": resourceName, - "action": action.name, - } - - result = append(result, facts.Fact{ - Kind: facts.KindRoute, - Name: path, - File: relFile, - Line: lineNum, - Props: props, - Relations: []facts.Relation{ - {Kind: facts.RelDeclares, Target: filepath.Dir(relFile)}, - }, - }) - } - - // If there's a do block, push resource as a scope. - if doBlockRe.MatchString(line) { - scopeStack = append(scopeStack, routeScope{pathPrefix: "/" + resourceName}) - depth++ - } - continue - } - - // member do / collection do. - if m := memberRe.FindStringSubmatch(line); m != nil { - blockType := m[1] - memberPrefix := "" - if blockType == "member" && currentResource != "" { - memberPrefix = "/:id" - } - scopeStack = append(scopeStack, routeScope{pathPrefix: memberPrefix}) - depth++ - continue - } - - // Track other do blocks for depth. - if doBlockRe.MatchString(line) { - depth++ - } - } - - return result -} - // buildPrefix constructs the current URL prefix from the scope stack. func buildPrefix(stack []routeScope) string { var parts []string @@ -300,8 +85,9 @@ type restAction struct { suffix string } -// restfulActions returns the set of REST actions for a resources declaration. -func restfulActions(line string) []restAction { +// restfulActions returns the set of REST actions for a resources declaration, +// honoring only:/except: filters parsed from the declaration's arguments. +func restfulActions(only, except map[string]bool) []restAction { all := []restAction{ {name: "index", method: "GET", suffix: ""}, {name: "create", method: "POST", suffix: ""}, @@ -312,18 +98,12 @@ func restfulActions(line string) []restAction { {name: "destroy", method: "DELETE", suffix: "/:id"}, } - // Check for only: [...] filter. - if m := onlyRe.FindStringSubmatch(line); m != nil { - allowed := parseSymbolList(m[1]) - return filterActions(all, allowed, true) + if len(only) > 0 { + return filterActions(all, only, true) } - - // Check for except: [...] filter. - if m := exceptRe.FindStringSubmatch(line); m != nil { - excluded := parseSymbolList(m[1]) - return filterActions(all, excluded, false) + if len(except) > 0 { + return filterActions(all, except, false) } - return all } diff --git a/internal/extractors/rubyextractor/routes_ast.go b/internal/extractors/rubyextractor/routes_ast.go new file mode 100644 index 0000000..0ad76f7 --- /dev/null +++ b/internal/extractors/rubyextractor/routes_ast.go @@ -0,0 +1,243 @@ +package rubyextractor + +import ( + "path/filepath" + "strings" + + "github.com/enola-labs/enola/internal/facts" + sitter "github.com/tree-sitter/go-tree-sitter" + ruby "github.com/tree-sitter/tree-sitter-ruby/bindings/go" +) + +// parseRouteFileAST parses a Rails route file with tree-sitter and emits KindRoute +// facts. Block boundaries come from the grammar (do_block) rather than counting +// `do`/`end`, so nested namespaces/resources/scopes are tracked precisely. +func parseRouteFileAST(src []byte, relFile string) []facts.Fact { + parser := sitter.NewParser() + defer parser.Close() + if err := parser.SetLanguage(sitter.NewLanguage(ruby.Language())); err != nil { + return nil + } + tree := parser.Parse(src, nil) + defer tree.Close() + + rw := &routeWalker{src: src, relFile: relFile, dir: filepath.Dir(relFile)} + rw.walk(tree.RootNode(), nil) + return rw.out +} + +type routeWalker struct { + src []byte + relFile string + dir string + out []facts.Fact +} + +// walk iterates the statements of a program / body_statement, dispatching each +// route-DSL call with the current scope stack. +func (rw *routeWalker) walk(node *sitter.Node, stack []routeScope) { + if node == nil { + return + } + for i := uint(0); i < node.ChildCount(); i++ { + c := node.Child(i) + if c.Kind() == "call" { + rw.handleCall(c, stack) + } + } +} + +// blockBody returns the body_statement of a call's do/brace block, or nil. +func blockBody(call *sitter.Node) *sitter.Node { + block := call.ChildByFieldName("block") + if block == nil { + return nil + } + return block.ChildByFieldName("body") +} + +func (rw *routeWalker) handleCall(call *sitter.Node, stack []routeScope) { + method := rubyText(call.ChildByFieldName("method"), rw.src) + args := call.ChildByFieldName("arguments") + body := blockBody(call) + prefix := buildPrefix(stack) + + switch method { + case "get", "post", "put", "patch", "delete": + path := firstStringArg(args, rw.src) + if path == "" { + return + } + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + props := map[string]any{ + "method": strings.ToUpper(method), + "framework": "rails", + "language": "ruby", + } + if handler := pairString(args, "to", rw.src); handler != "" { + props["handler"] = handler + } + rw.emit(prefix+path, line(call), props) + + case "root": + handler := pairString(args, "to", rw.src) + if handler == "" { + handler = firstStringArg(args, rw.src) + } + props := map[string]any{ + "method": "GET", + "framework": "rails", + "language": "ruby", + } + if handler != "" { + props["handler"] = handler + } + rw.emit(prefix+"/", line(call), props) + + case "resources", "resource": + name := firstSymbolArg(args, rw.src) + if name == "" { + return + } + only := pairSymbols(args, "only", rw.src) + except := pairSymbols(args, "except", rw.src) + resourcePath := prefix + "/" + name + for _, a := range restfulActions(only, except) { + rw.emit(resourcePath+a.suffix, line(call), map[string]any{ + "method": a.method, + "framework": "rails", + "language": "ruby", + "resource": name, + "action": a.name, + }) + } + if body != nil { + rw.walk(body, append(stack, routeScope{pathPrefix: "/" + name})) + } + + case "namespace": + name := firstSymbolArg(args, rw.src) + if name == "" || body == nil { + return + } + rw.walk(body, append(stack, routeScope{pathPrefix: "/" + name, module: name})) + + case "scope": + ns := routeScope{} + if path := firstStringArg(args, rw.src); path != "" { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + ns.pathPrefix = path + } else if mod := pairSymbol(args, "module", rw.src); mod != "" { + ns.module = mod + } + if body != nil { + rw.walk(body, append(stack, ns)) + } + + case "member", "collection": + memberPrefix := "" + if method == "member" { + memberPrefix = "/:id" + } + if body != nil { + rw.walk(body, append(stack, routeScope{pathPrefix: memberPrefix})) + } + + case "draw": + // `draw do ... end` is the routes wrapper (Rails.application.routes.draw, + // engine routers, etc.) — recurse into the block. `draw(:pkg)` with no + // block is a packwerk delegation — emit a DRAW route. + if body != nil { + rw.walk(body, stack) + return + } + if pkg := firstSymbolArg(args, rw.src); pkg != "" { + rw.out = append(rw.out, facts.Fact{ + Kind: facts.KindRoute, + Name: prefix + "/" + pkg, + File: rw.relFile, + Line: line(call), + Props: map[string]any{ + "method": "DRAW", + "framework": "rails", + "language": "ruby", + "delegate": pkg, + }, + }) + } + + default: + // Unknown DSL call (constraints, concern, authenticate, ...) — descend into + // any block so nested routes are still discovered. + if body != nil { + rw.walk(body, stack) + } + } +} + +// emit appends a route fact with a declares relation to the file's directory. +func (rw *routeWalker) emit(name string, lineNum int, props map[string]any) { + rw.out = append(rw.out, facts.Fact{ + Kind: facts.KindRoute, + Name: name, + File: rw.relFile, + Line: lineNum, + Props: props, + Relations: []facts.Relation{{Kind: facts.RelDeclares, Target: rw.dir}}, + }) +} + +// --- keyword-argument helpers --- + +// pairString returns the string content of a `key: "value"` pair. +func pairString(args *sitter.Node, key string, src []byte) string { + if v := findPairValue(args, key, src); v != nil { + return firstStringArg(v, src) + } + return "" +} + +// pairSymbol returns the symbol name of a `key: :value` pair. +func pairSymbol(args *sitter.Node, key string, src []byte) string { + if v := findPairValue(args, key, src); v != nil && v.Kind() == "simple_symbol" { + return strings.TrimPrefix(rubyText(v, src), ":") + } + return "" +} + +// pairSymbols returns the symbol names of a `key: [:a, :b]` pair. +func pairSymbols(args *sitter.Node, key string, src []byte) map[string]bool { + out := make(map[string]bool) + v := findPairValue(args, key, src) + if v == nil { + return out + } + for i := uint(0); i < v.ChildCount(); i++ { + if v.Child(i).Kind() == "simple_symbol" { + out[strings.TrimPrefix(rubyText(v.Child(i), src), ":")] = true + } + } + return out +} + +// findPairValue returns the value node of a `key: value` pair in an argument_list. +func findPairValue(args *sitter.Node, key string, src []byte) *sitter.Node { + if args == nil { + return nil + } + for i := uint(0); i < args.ChildCount(); i++ { + c := args.Child(i) + if c.Kind() != "pair" { + continue + } + k := c.ChildByFieldName("key") + if k != nil && strings.TrimSuffix(rubyText(k, src), ":") == key { + return c.ChildByFieldName("value") + } + } + return nil +} diff --git a/internal/extractors/rubyextractor/ruby.go b/internal/extractors/rubyextractor/ruby.go index 46774b7..5614946 100644 --- a/internal/extractors/rubyextractor/ruby.go +++ b/internal/extractors/rubyextractor/ruby.go @@ -1,18 +1,17 @@ package rubyextractor import ( - "bufio" "context" "log" "os" "path/filepath" - "regexp" "strings" "github.com/enola-labs/enola/internal/facts" ) -// RubyExtractor extracts architectural facts from Ruby source code using line-based regex parsing. +// RubyExtractor extracts architectural facts from Ruby source code using the +// tree-sitter Ruby grammar (in line with the other language extractors). type RubyExtractor struct{} // New creates a new RubyExtractor. @@ -63,26 +62,16 @@ func (e *RubyExtractor) Extract(ctx context.Context, repoPath string, files []st } absFile := filepath.Join(repoPath, relFile) - f, err := os.Open(absFile) + src, err := os.ReadFile(absFile) if err != nil { log.Printf("[ruby-extractor] error reading %s: %v", relFile, err) continue } exported := isPublicAPI(relFile, pkgInfo) - fileFacts := extractFile(f, relFile, isRails, exported) - f.Close() - - // Collect storage facts from ActiveRecord patterns found during file parsing. - storageFacts := extractStorageFacts(relFile, fileFacts) - allFacts = append(allFacts, fileFacts...) - allFacts = append(allFacts, storageFacts...) - - // Re-read the file to extract association details if models were found. - if len(storageFacts) > 0 { - assocFacts := extractAssociationsFromFile(filepath.Join(repoPath, relFile), relFile) - allFacts = append(allFacts, assocFacts...) - } + // extractFileAST emits symbols, imports, mixins, constants, attrs, calls, + // and ActiveRecord storage/associations in a single AST pass. + allFacts = append(allFacts, extractFileAST(src, relFile, isRails, exported)...) dir := filepath.Dir(relFile) modules[dir] = true @@ -131,565 +120,6 @@ func detectRailsProject(repoPath string) bool { return false } -// --- Regex patterns --- - -var ( - moduleRe = regexp.MustCompile(`^\s*module\s+([\w:]+)`) - eigenclassRe = regexp.MustCompile(`^\s*class\s*<<\s*\w`) - classOneLineRe = regexp.MustCompile(`^\s*class\s+([\w:]+)(?:\s*<\s*([\w:]+))?`) - defRe = regexp.MustCompile(`^\s*def\s+(self\.)?([\w?!=]+)`) - requireRe = regexp.MustCompile(`^\s*require\s+['"]([^'"]+)['"]`) - requireRelRe = regexp.MustCompile(`^\s*require_relative\s+['"]([^'"]+)['"]`) - includeRe = regexp.MustCompile(`^\s*(?:include|extend|prepend)\s+([\w:]+)`) - mixinKindRe = regexp.MustCompile(`^\s*(include|extend|prepend)\s+`) - constantRe = regexp.MustCompile(`^\s*([A-Z][A-Z0-9_]+)\s*=\s*`) - attrRe = regexp.MustCompile(`^\s*attr_(reader|writer|accessor)\s+(.+)`) - symbolListRe = regexp.MustCompile(`:(\w+)`) - concernRe = regexp.MustCompile(`^\s*extend\s+ActiveSupport::Concern`) - visibilityRe = regexp.MustCompile(`^\s*(private|protected|public)\s*$`) - moduleFuncRe = regexp.MustCompile(`^\s*module_function\s*$`) - inlineEndRe = regexp.MustCompile(`;\s*end\s*$`) - heredocOpenRe = regexp.MustCompile(`<<[~-]?\s*['"]?([A-Z_]+)['"]?`) - endRe = regexp.MustCompile(`^\s*end\b`) - blockOpenerRe = regexp.MustCompile( - `(?:^\s*(?:if|unless|case|while|until|for|begin)\b)|` + - `\bdo\s*(?:\|[^|]*\|)?\s*$`) - - // openAPISpecPathRe matches openapi_spec_path declarations in Rails API controllers. - // Example: openapi_spec_path 'packages/items/api/openapi/api.yml' - openAPISpecPathRe = regexp.MustCompile(`^\s*openapi_spec_path\s+['"]([^'"]+)['"]`) - - // qualifiedCallRe matches calls where the receiver is a constant/class name - // (PascalCase or Namespace::Class). No trailing char required because Ruby - // method calls are valid without parentheses: Config.load_defaults - qualifiedCallRe = regexp.MustCompile(`\b([A-Z]\w*(?:::[A-Z]\w*)*)\.([\w?!=]+)`) - // receiverCallRe matches calls on lowercase receivers with explicit parens: object.method( - // Parens are required here to reduce noise from attribute reads. - receiverCallRe = regexp.MustCompile(`\b([a-z_]\w*)\.([\w?!=]+)\s*\(`) - // endlessMethodRe detects Ruby 3.0+ endless method syntax: - // def name = expr (no params) - // def name(args) = expr (with params) - // We require whitespace after = to distinguish from setter defs (def foo=(v)) - // and from == comparisons. RE2 has no lookahead, so we use \s as the guard. - endlessMethodRe = regexp.MustCompile(`\)\s*=\s|\bdef\s+(?:self\.)?[\w?!]+\s*=\s`) -) - -// scopeEntry tracks a class/module nesting level. -type scopeEntry struct { - name string - kind string // "class", "module", or "eigenclass" - depth int -} - -// methodEntry tracks an active method body for call accumulation. -type methodEntry struct { - name string - startDepth int -} - -// extractFile parses a single Ruby file and returns facts. -func extractFile(f *os.File, relFile string, isRails bool, exportedByPackwerk bool) []facts.Fact { - var result []facts.Fact - dir := filepath.Dir(relFile) - - scanner := bufio.NewScanner(f) - scanner.Buffer(make([]byte, 0, 256*1024), 1024*1024) - - var ( - lineNum int - depth int - scopeStack []scopeEntry - methodStack []methodEntry - visibility = "public" - isConcern bool - moduleFunction bool - heredocEnd string // non-empty when inside a heredoc - ) - callAccum := make(map[string][]string) - - for scanner.Scan() { - lineNum++ - line := scanner.Text() - trimmed := strings.TrimSpace(line) - - // Skip blank lines and comments. - if trimmed == "" || strings.HasPrefix(trimmed, "#") { - continue - } - - // Heredoc state: skip lines until terminator. - if heredocEnd != "" { - if trimmed == heredocEnd || strings.TrimSpace(trimmed) == heredocEnd { - heredocEnd = "" - } - continue - } - - // Check for heredoc opener on this line (process line normally first, then enter heredoc mode). - lineHasHeredoc := false - var heredocTerminator string - if m := heredocOpenRe.FindStringSubmatch(line); m != nil { - // Make sure it's actually a heredoc, not a left-shift operator. - // Heredocs use identifiers like SQL, TEXT, HEREDOC, JSON, MESSAGE, etc. - heredocTerminator = m[1] - if len(heredocTerminator) >= 2 { - lineHasHeredoc = true - } - } - - // Track end keywords to manage depth. - if trimmed == "end" { - depth-- - if depth < 0 { - depth = 0 - } - if len(scopeStack) > 0 && scopeStack[len(scopeStack)-1].depth == depth { - popped := scopeStack[len(scopeStack)-1] - scopeStack = scopeStack[:len(scopeStack)-1] - // Reset visibility when leaving a class/module scope. - if popped.kind != "eigenclass" { - visibility = "public" - moduleFunction = false - } - } - // Pop method stack when the end closes a method body. - if len(methodStack) > 0 && methodStack[len(methodStack)-1].startDepth == depth { - methodStack = methodStack[:len(methodStack)-1] - } - continue - } - - // Detect ActiveSupport::Concern. - if concernRe.MatchString(line) { - isConcern = true - } - - // Visibility section markers (bare private/protected/public on its own line). - if visibilityRe.MatchString(line) { - m := visibilityRe.FindStringSubmatch(line) - visibility = m[1] - continue - } - - // module_function -- subsequent defs become class methods. - if moduleFuncRe.MatchString(line) { - moduleFunction = true - continue - } - - // Module declarations. - if m := moduleRe.FindStringSubmatch(line); m != nil { - name := m[1] - qualName := qualifiedName(scopeStack, name) - - props := map[string]any{ - "symbol_kind": facts.SymbolInterface, - "exported": exportedByPackwerk, - "language": "ruby", - } - if isConcern { - props["concern"] = true - isConcern = false - } - if isRails { - props["framework"] = "rails" - } - - result = append(result, facts.Fact{ - Kind: facts.KindSymbol, - Name: qualName, - File: relFile, - Line: lineNum, - Props: props, - Relations: []facts.Relation{ - {Kind: facts.RelDeclares, Target: dir}, - }, - }) - - scopeStack = append(scopeStack, scopeEntry{name: name, kind: "module", depth: depth}) - depth++ - continue - } - - // Eigenclass: class << self (must be checked before class declarations). - if eigenclassRe.MatchString(line) { - scopeStack = append(scopeStack, scopeEntry{name: "", kind: "eigenclass", depth: depth}) - depth++ - continue - } - - // Class declarations. - if m := classOneLineRe.FindStringSubmatch(line); m != nil { - name := m[1] - superclass := m[2] - qualName := qualifiedName(scopeStack, name) - isInline := inlineEndRe.MatchString(line) - - exported := visibility == "public" && exportedByPackwerk - - props := map[string]any{ - "symbol_kind": facts.SymbolClass, - "exported": exported, - "language": "ruby", - } - if isRails { - props["framework"] = "rails" - } - if superclass != "" { - props["superclass"] = superclass - } - - rels := []facts.Relation{ - {Kind: facts.RelDeclares, Target: dir}, - } - if superclass != "" { - rels = append(rels, facts.Relation{ - Kind: facts.RelImplements, - Target: superclass, - }) - } - - result = append(result, facts.Fact{ - Kind: facts.KindSymbol, - Name: qualName, - File: relFile, - Line: lineNum, - Props: props, - Relations: rels, - }) - - // Only push to scope stack if this is NOT an inline class (class Foo < Bar; end). - if !isInline { - scopeStack = append(scopeStack, scopeEntry{name: name, kind: "class", depth: depth}) - depth++ - visibility = "public" - } - - if lineHasHeredoc { - heredocEnd = heredocTerminator - } - continue - } - - // Method definitions. - if m := defRe.FindStringSubmatch(line); m != nil { - isSelf := m[1] == "self." - methodName := m[2] - isInline := inlineEndRe.MatchString(line) - // Endless method: def name = expr or def name(args) = expr - isEndless := !isInline && endlessMethodRe.MatchString(line) - - // module_function makes subsequent defs into class methods. - if moduleFunction { - isSelf = true - } - - scopeName := qualifiedName(scopeStack, "") - var fullName string - if scopeName != "" { - if isSelf { - fullName = scopeName + "." + methodName - } else { - fullName = scopeName + "#" + methodName - } - } else { - fullName = dir + "." + methodName - } - - symbolKind := facts.SymbolMethod - if isSelf { - symbolKind = facts.SymbolFunc - } - - exported := visibility == "public" && exportedByPackwerk - - props := map[string]any{ - "symbol_kind": symbolKind, - "exported": exported, - "language": "ruby", - } - if isRails { - props["framework"] = "rails" - } - - // For endless methods, extract calls from the expression on this line. - var defLineCalls []string - if isEndless { - defLineCalls = extractRubyCalls(line) - } - - rels := []facts.Relation{{Kind: facts.RelDeclares, Target: dir}} - seen := make(map[string]bool) - for _, callee := range defLineCalls { - if !seen[callee] { - seen[callee] = true - rels = append(rels, facts.Relation{Kind: facts.RelCalls, Target: callee}) - } - } - - result = append(result, facts.Fact{ - Kind: facts.KindSymbol, - Name: fullName, - File: relFile, - Line: lineNum, - Props: props, - Relations: rels, - }) - - // Endless and inline one-liners have no body — don't push to stack or increment depth. - if !isInline && !isEndless { - methodStack = append(methodStack, methodEntry{name: fullName, startDepth: depth}) - depth++ - } - - if lineHasHeredoc { - heredocEnd = heredocTerminator - } - continue - } - - // Require / require_relative. - if m := requireRe.FindStringSubmatch(line); m != nil { - importPath := m[1] - result = append(result, facts.Fact{ - Kind: facts.KindDependency, - Name: dir + " -> " + importPath, - File: relFile, - Line: lineNum, - Props: map[string]any{ - "language": "ruby", - }, - Relations: []facts.Relation{ - {Kind: facts.RelImports, Target: importPath}, - }, - }) - continue - } - if m := requireRelRe.FindStringSubmatch(line); m != nil { - importPath := m[1] - result = append(result, facts.Fact{ - Kind: facts.KindDependency, - Name: dir + " -> " + importPath, - File: relFile, - Line: lineNum, - Props: map[string]any{ - "language": "ruby", - "require_relative": true, - }, - Relations: []facts.Relation{ - {Kind: facts.RelImports, Target: importPath}, - }, - }) - continue - } - - // Include / extend / prepend. - if m := includeRe.FindStringSubmatch(line); m != nil { - mixinName := m[1] - kindM := mixinKindRe.FindStringSubmatch(line) - mixinKind := "include" - if kindM != nil { - mixinKind = kindM[1] - } - - scopeName := qualifiedName(scopeStack, "") - if scopeName == "" { - scopeName = dir - } - - // Don't duplicate the ActiveSupport::Concern detection as a mixin. - if mixinName == "ActiveSupport::Concern" { - continue - } - - result = append(result, facts.Fact{ - Kind: facts.KindDependency, - Name: scopeName + " -> " + mixinName, - File: relFile, - Line: lineNum, - Props: map[string]any{ - "language": "ruby", - "mixin_kind": mixinKind, - }, - Relations: []facts.Relation{ - {Kind: facts.RelImplements, Target: mixinName}, - }, - }) - continue - } - - // openapi_spec_path 'path/to/spec.yml' — links a controller to its OpenAPI spec. - if m := openAPISpecPathRe.FindStringSubmatch(line); m != nil { - specFile := m[1] - scopeName := qualifiedName(scopeStack, "") - if scopeName == "" { - scopeName = dir - } - result = append(result, facts.Fact{ - Kind: facts.KindDependency, - Name: scopeName + " -> " + specFile, - File: relFile, - Line: lineNum, - Props: map[string]any{ - "language": "ruby", - "type": "openapi_spec", - "spec_file": specFile, - }, - Relations: []facts.Relation{ - {Kind: facts.RelDependsOn, Target: specFile}, - }, - }) - continue - } - - // Constants (ALL_CAPS = ...). - if m := constantRe.FindStringSubmatch(line); m != nil { - constName := m[1] - scopeName := qualifiedName(scopeStack, "") - var fullName string - if scopeName != "" { - fullName = scopeName + "::" + constName - } else { - fullName = dir + "." + constName - } - - result = append(result, facts.Fact{ - Kind: facts.KindSymbol, - Name: fullName, - File: relFile, - Line: lineNum, - Props: map[string]any{ - "symbol_kind": facts.SymbolConstant, - "exported": visibility == "public" && exportedByPackwerk, - "language": "ruby", - }, - Relations: []facts.Relation{ - {Kind: facts.RelDeclares, Target: dir}, - }, - }) - - if lineHasHeredoc { - heredocEnd = heredocTerminator - } - continue - } - - // attr_reader / attr_writer / attr_accessor. - if m := attrRe.FindStringSubmatch(line); m != nil { - attrKind := m[1] - symbolsStr := m[2] - symbols := symbolListRe.FindAllStringSubmatch(symbolsStr, -1) - - scopeName := qualifiedName(scopeStack, "") - if scopeName == "" { - scopeName = dir - } - - for _, sym := range symbols { - attrName := sym[1] - result = append(result, facts.Fact{ - Kind: facts.KindSymbol, - Name: scopeName + "#" + attrName, - File: relFile, - Line: lineNum, - Props: map[string]any{ - "symbol_kind": facts.SymbolVariable, - "exported": visibility == "public" && exportedByPackwerk, - "language": "ruby", - "attr_kind": attrKind, - }, - Relations: []facts.Relation{ - {Kind: facts.RelDeclares, Target: dir}, - }, - }) - } - continue - } - - // Accumulate method calls for any line inside an active method body. - if len(methodStack) > 0 { - mName := methodStack[len(methodStack)-1].name - callAccum[mName] = append(callAccum[mName], extractRubyCalls(line)...) - } - - // Track depth for other block openers (if/unless/case/while/do etc.). - // Note: module/class/def are already handled above and won't reach here. - if blockOpenerRe.MatchString(line) { - depth++ - } - - // Enter heredoc mode after processing this line. - if lineHasHeredoc { - heredocEnd = heredocTerminator - } - } - - // Attach accumulated RelCalls edges to each method/function fact. - seen := make(map[string]map[string]bool) - for i, f := range result { - sk, _ := f.Props["symbol_kind"].(string) - if f.Kind != facts.KindSymbol || - (sk != facts.SymbolMethod && sk != facts.SymbolFunc) { - continue - } - calls, ok := callAccum[f.Name] - if !ok { - continue - } - if seen[f.Name] == nil { - seen[f.Name] = make(map[string]bool) - } - for _, callee := range calls { - if seen[f.Name][callee] { - continue - } - seen[f.Name][callee] = true - result[i].Relations = append(result[i].Relations, - facts.Relation{Kind: facts.RelCalls, Target: callee}) - } - } - - return result -} - -// qualifiedName builds a fully-qualified Ruby name from the scope stack. -func qualifiedName(stack []scopeEntry, name string) string { - var parts []string - for _, entry := range stack { - // Skip eigenclass entries -- they don't contribute to the qualified name. - if entry.kind == "eigenclass" || entry.name == "" { - continue - } - parts = append(parts, entry.name) - } - if name != "" { - parts = append(parts, name) - } - return strings.Join(parts, "::") -} - -// extractRubyCalls returns callee names found on a single source line. -// It detects two tiers: -// - Qualified (high-confidence): ConstantName.method or Ns::Class.method -// - Receiver (medium-confidence): variable.method( -func extractRubyCalls(line string) []string { - var out []string - seen := make(map[string]bool) - add := func(s string) { - if !seen[s] { - seen[s] = true - out = append(out, s) - } - } - for _, m := range qualifiedCallRe.FindAllStringSubmatch(line, -1) { - add(m[1] + "." + m[2]) - } - for _, m := range receiverCallRe.FindAllStringSubmatch(line, -1) { - add(m[1] + "." + m[2]) - } - return out -} - // isRubyFile returns true if the file has a .rb extension. func isRubyFile(path string) bool { return strings.HasSuffix(strings.ToLower(path), ".rb") @@ -714,97 +144,3 @@ func isPublicAPI(relFile string, pkg *packwerkInfo) bool { publicDir := filepath.Join(ownerPkg, "app", "public") return strings.HasPrefix(relFile, publicDir+"/") || strings.HasPrefix(relFile, publicDir+"\\") } - -// extractAssociationsFromFile re-reads a file to extract ActiveRecord associations and scopes. -func extractAssociationsFromFile(absPath string, relFile string) []facts.Fact { - f, err := os.Open(absPath) - if err != nil { - return nil - } - defer f.Close() - - scanner := bufio.NewScanner(f) - scanner.Buffer(make([]byte, 0, 256*1024), 1024*1024) - - var lines []string - for scanner.Scan() { - lines = append(lines, scanner.Text()) - } - - // Find model classes in the file to determine context. - modelClasses := make(map[string]bool) - for _, line := range lines { - if m := classOneLineRe.FindStringSubmatch(line); m != nil { - superclass := m[2] - if isARBaseClass(superclass) { - modelClasses[m[1]] = true - } - } - } - - if len(modelClasses) == 0 { - return nil - } - - var result []facts.Fact - for lineNum, line := range lines { - // Association declarations. - if m := associationRe.FindStringSubmatch(line); m != nil { - assocKind := m[1] - assocName := m[2] - - targetModel := assocName - if assocKind == "has_many" || assocKind == "has_and_belongs_to_many" { - targetModel = singularize(assocName) - } - targetModel = snakeToCamel(targetModel) - - result = append(result, facts.Fact{ - Kind: facts.KindDependency, - Name: relFile + ":" + assocKind + " :" + assocName, - File: relFile, - Line: lineNum + 1, - Props: map[string]any{ - "language": "ruby", - "association_kind": assocKind, - }, - Relations: []facts.Relation{ - {Kind: facts.RelDependsOn, Target: targetModel}, - }, - }) - } - - // Scope declarations on models. - if m := scopeRe.FindStringSubmatch(line); m != nil { - result = append(result, facts.Fact{ - Kind: facts.KindSymbol, - Name: "scope:" + m[1], - File: relFile, - Line: lineNum + 1, - Props: map[string]any{ - "symbol_kind": facts.SymbolFunc, - "language": "ruby", - "scope": true, - }, - }) - } - - // Explicit table name: self.table_name = 'foo' - if m := tableNameRe.FindStringSubmatch(line); m != nil { - result = append(result, facts.Fact{ - Kind: facts.KindStorage, - Name: m[1], - File: relFile, - Line: lineNum + 1, - Props: map[string]any{ - "storage_kind": "table", - "language": "ruby", - "framework": "rails", - "explicit": true, - }, - }) - } - } - - return result -} diff --git a/internal/extractors/rubyextractor/ruby_ast.go b/internal/extractors/rubyextractor/ruby_ast.go new file mode 100644 index 0000000..6a145a9 --- /dev/null +++ b/internal/extractors/rubyextractor/ruby_ast.go @@ -0,0 +1,735 @@ +package rubyextractor + +import ( + "path/filepath" + "strings" + + "github.com/enola-labs/enola/internal/facts" + sitter "github.com/tree-sitter/go-tree-sitter" + ruby "github.com/tree-sitter/tree-sitter-ruby/bindings/go" +) + +// extractFileAST parses a Ruby file with tree-sitter and emits architectural +// facts. It replaces the former line-based regex scanner: every symbol, import, +// mixin, constant, attr, ActiveRecord storage/association, and RelCalls edge the +// regex produced is preserved here, with higher fidelity (heredocs, multi-line +// expressions, endless methods, and nested scopes are handled by the grammar). +func extractFileAST(src []byte, relFile string, isRails, exportedByPackwerk bool) []facts.Fact { + parser := sitter.NewParser() + defer parser.Close() + if err := parser.SetLanguage(sitter.NewLanguage(ruby.Language())); err != nil { + return nil + } + tree := parser.Parse(src, nil) + defer tree.Close() + + w := &rubyWalker{ + src: src, + relFile: relFile, + dir: filepath.Dir(relFile), + isRails: isRails, + exportedByPackwerk: exportedByPackwerk, + } + w.walkBody(tree.RootNode()) + return w.out +} + +// rubyScope tracks a class/module/eigenclass nesting level. +type rubyScope struct { + name string // simple (last) name; "" for an eigenclass (class << self) + kind string // "class", "module", or "eigenclass" + visibility string // "public" | "private" | "protected" + moduleFunc bool // module_function active: subsequent defs are class methods + isModel bool // ActiveRecord model: associations/scopes/table_name apply +} + +type rubyWalker struct { + src []byte + relFile string + dir string + isRails bool + exportedByPackwerk bool + + out []facts.Fact + scopeStack []rubyScope +} + +// --- scope helpers --- + +func (w *rubyWalker) push(s rubyScope) { w.scopeStack = append(w.scopeStack, s) } +func (w *rubyWalker) pop() { w.scopeStack = w.scopeStack[:len(w.scopeStack)-1] } + +func (w *rubyWalker) cur() *rubyScope { + if len(w.scopeStack) == 0 { + return nil + } + return &w.scopeStack[len(w.scopeStack)-1] +} + +// scopeQual joins the enclosing class/module names into a Ruby-qualified name. +// Eigenclass and anonymous entries do not contribute. +func (w *rubyWalker) scopeQual() string { + var parts []string + for _, s := range w.scopeStack { + if s.kind == "eigenclass" || s.name == "" { + continue + } + parts = append(parts, s.name) + } + return strings.Join(parts, "::") +} + +// curVisibility returns the visibility of the innermost type scope. +func (w *rubyWalker) curVisibility() string { + if s := w.cur(); s != nil && s.visibility != "" { + return s.visibility + } + return "public" +} + +// inEigenclass reports whether the innermost scope is an eigenclass. +func (w *rubyWalker) inEigenclass() bool { + s := w.cur() + return s != nil && s.kind == "eigenclass" +} + +func (w *rubyWalker) exported() bool { + return w.curVisibility() == "public" && w.exportedByPackwerk +} + +// --- body walking --- + +// walkBody iterates the statements of a program or body_statement, dispatching +// each. It is the single entry point for both top-level and nested scopes. +func (w *rubyWalker) walkBody(node *sitter.Node) { + if node == nil { + return + } + for i := uint(0); i < node.ChildCount(); i++ { + w.walkStatement(node.Child(i)) + } +} + +func (w *rubyWalker) walkStatement(node *sitter.Node) { + if node == nil || !node.IsNamed() { + return + } + switch node.Kind() { + case "module": + w.handleModule(node) + case "class": + w.handleClass(node) + case "singleton_class": + w.handleSingletonClass(node) + case "method": + w.handleMethod(node, w.classMethodContext()) + case "singleton_method": + w.handleMethod(node, true) + case "assignment": + w.handleAssignment(node) + case "call": + w.handleBodyCall(node) + // Descend into a trailing do/brace block so declarations inside + // included/class_methods/concerning/configure blocks are captured (the + // former line-based scanner was block-agnostic). + if body := blockBody(node); body != nil { + w.walkBody(body) + } + case "identifier": + // Bare statements: visibility markers and module_function. + switch rubyText(node, w.src) { + case "private", "protected", "public": + if s := w.cur(); s != nil { + s.visibility = rubyText(node, w.src) + } + case "module_function": + if s := w.cur(); s != nil { + s.moduleFunc = true + } + } + case "comment": + // ignore + default: + // Control-flow / grouping containers (if, unless, begin, case, while, + // modifiers, ...): descend so nested require/include/def/const + // declarations are captured, as the line-based scanner did. + for i := uint(0); i < node.ChildCount(); i++ { + w.walkStatement(node.Child(i)) + } + } +} + +// classMethodContext reports whether a plain `def` in the current scope should be +// treated as a class method (eigenclass body or after module_function). +func (w *rubyWalker) classMethodContext() bool { + if w.inEigenclass() { + return true + } + if s := w.cur(); s != nil && s.moduleFunc { + return true + } + return false +} + +// --- modules / classes --- + +func (w *rubyWalker) handleModule(node *sitter.Node) { + name := w.constName(node.ChildByFieldName("name")) + if name == "" { + return + } + qual := w.qualify(name) + body := node.ChildByFieldName("body") + + props := map[string]any{ + "symbol_kind": facts.SymbolInterface, + "exported": w.exportedByPackwerk, + "language": "ruby", + } + if bodyHasConcern(body, w.src) { + props["concern"] = true + } + if w.isRails { + props["framework"] = "rails" + } + w.out = append(w.out, facts.Fact{ + Kind: facts.KindSymbol, + Name: qual, + File: w.relFile, + Line: line(node), + Props: props, + Relations: []facts.Relation{{Kind: facts.RelDeclares, Target: w.dir}}, + }) + + w.push(rubyScope{name: name, kind: "module", visibility: "public"}) + w.walkBody(body) + w.pop() +} + +func (w *rubyWalker) handleClass(node *sitter.Node) { + name := w.constName(node.ChildByFieldName("name")) + if name == "" { + return + } + qual := w.qualify(name) + superclass := w.superclassName(node.ChildByFieldName("superclass")) + + props := map[string]any{ + "symbol_kind": facts.SymbolClass, + "exported": w.exported(), + "language": "ruby", + } + if w.isRails { + props["framework"] = "rails" + } + if superclass != "" { + props["superclass"] = superclass + } + rels := []facts.Relation{{Kind: facts.RelDeclares, Target: w.dir}} + if superclass != "" { + rels = append(rels, facts.Relation{Kind: facts.RelImplements, Target: superclass}) + } + w.out = append(w.out, facts.Fact{ + Kind: facts.KindSymbol, + Name: qual, + File: w.relFile, + Line: line(node), + Props: props, + Relations: rels, + }) + + // ActiveRecord model: emit a storage fact and flag the scope so the body + // scan picks up associations, scopes, and explicit table names. + isModel := isARBaseClass(superclass) + if isModel { + w.out = append(w.out, facts.Fact{ + Kind: facts.KindStorage, + Name: qual, + File: w.relFile, + Line: line(node), + Props: map[string]any{ + "storage_kind": "model", + "table": inferTableName(qual), + "language": "ruby", + "framework": "rails", + }, + Relations: []facts.Relation{{Kind: facts.RelDeclares, Target: w.dir}}, + }) + } + + w.push(rubyScope{name: name, kind: "class", visibility: "public", isModel: isModel}) + w.walkBody(node.ChildByFieldName("body")) + w.pop() +} + +func (w *rubyWalker) handleSingletonClass(node *sitter.Node) { + // class << self — methods inside become class (singleton) methods. The + // eigenclass entry carries no name and does not affect qualification. + w.push(rubyScope{name: "", kind: "eigenclass", visibility: "public"}) + w.walkBody(node.ChildByFieldName("body")) + w.pop() +} + +// --- methods --- + +func (w *rubyWalker) handleMethod(node *sitter.Node, isClassMethod bool) { + name := rubyText(node.ChildByFieldName("name"), w.src) + if name == "" { + return + } + + scope := w.scopeQual() + var fullName string + switch { + case scope == "": + fullName = w.dir + "." + name + case isClassMethod: + fullName = scope + "." + name + default: + fullName = scope + "#" + name + } + + symbolKind := facts.SymbolMethod + if isClassMethod { + symbolKind = facts.SymbolFunc + } + props := map[string]any{ + "symbol_kind": symbolKind, + "exported": w.exported(), + "language": "ruby", + } + if w.isRails { + props["framework"] = "rails" + } + + w.out = append(w.out, facts.Fact{ + Kind: facts.KindSymbol, + Name: fullName, + File: w.relFile, + Line: line(node), + Props: props, + Relations: []facts.Relation{{Kind: facts.RelDeclares, Target: w.dir}}, + }) + ownerIdx := len(w.out) - 1 + + // Accumulate RelCalls from the body onto this method (deduplicated). + seen := make(map[string]bool) + w.walkForCalls(node.ChildByFieldName("body"), ownerIdx, seen) +} + +// walkForCalls recursively scans a method body for call expressions and appends +// RelCalls edges to the owner fact. It does not descend into nested +// method/class/module definitions — those receive their own owner. +func (w *rubyWalker) walkForCalls(node *sitter.Node, ownerIdx int, seen map[string]bool) { + if node == nil { + return + } + switch node.Kind() { + case "method", "singleton_method", "class", "module", "singleton_class": + return + case "call": + if target := w.callTarget(node); target != "" && !seen[target] { + seen[target] = true + w.out[ownerIdx].Relations = append(w.out[ownerIdx].Relations, + facts.Relation{Kind: facts.RelCalls, Target: target}) + } + } + for i := uint(0); i < node.ChildCount(); i++ { + w.walkForCalls(node.Child(i), ownerIdx, seen) + } +} + +// callTarget resolves a call node to a RelCalls target string, preserving the +// two-tier convention of the former regex scanner: +// - constant / scope-resolution receiver -> "Const.method" / "Ns::Class.method" +// (always emitted, parentheses not required) +// - lowercase variable receiver with args -> "var.method" +// (only when arguments are present, to skip attribute reads) +// +// Bare calls without a receiver are not emitted (matching the regex behavior). +func (w *rubyWalker) callTarget(node *sitter.Node) string { + recv := node.ChildByFieldName("receiver") + method := node.ChildByFieldName("method") + if recv == nil || method == nil { + return "" + } + methodName := rubyText(method, w.src) + if methodName == "" { + return "" + } + + switch recv.Kind() { + case "constant", "scope_resolution": + return rubyText(recv, w.src) + "." + methodName + case "identifier": + if node.ChildByFieldName("arguments") != nil { + return rubyText(recv, w.src) + "." + methodName + } + case "call": + // Chained call, e.g. Rails.logger.info(x): use the inner call's method + // name as a pseudo-receiver when it is a lowercase identifier. + inner := recv.ChildByFieldName("method") + if inner != nil && inner.Kind() == "identifier" && node.ChildByFieldName("arguments") != nil { + return rubyText(inner, w.src) + "." + methodName + } + } + return "" +} + +// --- assignments (constants and explicit table names) --- + +func (w *rubyWalker) handleAssignment(node *sitter.Node) { + left := node.ChildByFieldName("left") + if left == nil { + return + } + + // CONSTANT = ... (all-caps only, matching the former regex). + if left.Kind() == "constant" { + constName := rubyText(left, w.src) + if !isAllCaps(constName) { + return + } + scope := w.scopeQual() + var fullName string + if scope != "" { + fullName = scope + "::" + constName + } else { + fullName = w.dir + "." + constName + } + w.out = append(w.out, facts.Fact{ + Kind: facts.KindSymbol, + Name: fullName, + File: w.relFile, + Line: line(node), + Props: map[string]any{ + "symbol_kind": facts.SymbolConstant, + "exported": w.exported(), + "language": "ruby", + }, + Relations: []facts.Relation{{Kind: facts.RelDeclares, Target: w.dir}}, + }) + return + } + + // self.table_name = "foo" on an ActiveRecord model. + if s := w.cur(); s != nil && s.isModel && left.Kind() == "call" { + if rubyText(left, w.src) == "self.table_name" { + if tbl := firstStringArg(node.ChildByFieldName("right"), w.src); tbl != "" { + w.out = append(w.out, facts.Fact{ + Kind: facts.KindStorage, + Name: tbl, + File: w.relFile, + Line: line(node), + Props: map[string]any{ + "storage_kind": "table", + "language": "ruby", + "framework": "rails", + "explicit": true, + }, + }) + } + } + } +} + +// --- body-level DSL calls --- + +func (w *rubyWalker) handleBodyCall(node *sitter.Node) { + // Only bare method calls (no receiver) are DSL declarations. + if node.ChildByFieldName("receiver") != nil { + return + } + method := rubyText(node.ChildByFieldName("method"), w.src) + args := node.ChildByFieldName("arguments") + + switch method { + case "require", "require_relative": + path := firstStringArg(args, w.src) + if path == "" { + return + } + props := map[string]any{"language": "ruby"} + if method == "require_relative" { + props["require_relative"] = true + } + w.out = append(w.out, facts.Fact{ + Kind: facts.KindDependency, + Name: w.dir + " -> " + path, + File: w.relFile, + Line: line(node), + Props: props, + Relations: []facts.Relation{{Kind: facts.RelImports, Target: path}}, + }) + + case "include", "extend", "prepend": + mixin := firstConstArg(args, w.src) + if mixin == "" || mixin == "ActiveSupport::Concern" { + return + } + scope := w.scopeQual() + if scope == "" { + scope = w.dir + } + w.out = append(w.out, facts.Fact{ + Kind: facts.KindDependency, + Name: scope + " -> " + mixin, + File: w.relFile, + Line: line(node), + Props: map[string]any{ + "language": "ruby", + "mixin_kind": method, + }, + Relations: []facts.Relation{{Kind: facts.RelImplements, Target: mixin}}, + }) + + case "attr_reader", "attr_writer", "attr_accessor": + attrKind := strings.TrimPrefix(method, "attr_") + scope := w.scopeQual() + if scope == "" { + scope = w.dir + } + for _, attr := range symbolArgs(args, w.src) { + w.out = append(w.out, facts.Fact{ + Kind: facts.KindSymbol, + Name: scope + "#" + attr, + File: w.relFile, + Line: line(node), + Props: map[string]any{ + "symbol_kind": facts.SymbolVariable, + "exported": w.exported(), + "language": "ruby", + "attr_kind": attrKind, + }, + Relations: []facts.Relation{{Kind: facts.RelDeclares, Target: w.dir}}, + }) + } + + case "openapi_spec_path": + spec := firstStringArg(args, w.src) + if spec == "" { + return + } + scope := w.scopeQual() + if scope == "" { + scope = w.dir + } + w.out = append(w.out, facts.Fact{ + Kind: facts.KindDependency, + Name: scope + " -> " + spec, + File: w.relFile, + Line: line(node), + Props: map[string]any{ + "language": "ruby", + "type": "openapi_spec", + "spec_file": spec, + }, + Relations: []facts.Relation{{Kind: facts.RelDependsOn, Target: spec}}, + }) + + case "has_many", "has_one", "belongs_to", "has_and_belongs_to_many": + if s := w.cur(); s == nil || !s.isModel { + return + } + assoc := firstSymbolArg(args, w.src) + if assoc == "" { + return + } + target := assoc + if method == "has_many" || method == "has_and_belongs_to_many" { + target = singularize(assoc) + } + target = snakeToCamel(target) + w.out = append(w.out, facts.Fact{ + Kind: facts.KindDependency, + Name: w.relFile + ":" + method + " :" + assoc, + File: w.relFile, + Line: line(node), + Props: map[string]any{ + "language": "ruby", + "association_kind": method, + }, + Relations: []facts.Relation{{Kind: facts.RelDependsOn, Target: target}}, + }) + + case "scope": + if s := w.cur(); s == nil || !s.isModel { + return + } + name := firstSymbolArg(args, w.src) + if name == "" { + return + } + w.out = append(w.out, facts.Fact{ + Kind: facts.KindSymbol, + Name: "scope:" + name, + File: w.relFile, + Line: line(node), + Props: map[string]any{ + "symbol_kind": facts.SymbolFunc, + "language": "ruby", + "scope": true, + }, + }) + } +} + +// --- AST value helpers --- + +// qualify joins the current scope with a simple name to form a Ruby-qualified name. +func (w *rubyWalker) qualify(name string) string { + if scope := w.scopeQual(); scope != "" { + return scope + "::" + name + } + return name +} + +// constName returns the name of a constant or scope_resolution node (e.g. "Foo" +// or "Foo::Bar"). +func (w *rubyWalker) constName(node *sitter.Node) string { + if node == nil { + return "" + } + switch node.Kind() { + case "constant", "scope_resolution": + return rubyText(node, w.src) + } + return "" +} + +// superclassName extracts the superclass name from a `superclass` node +// (the "< Base" part of a class declaration). +func (w *rubyWalker) superclassName(node *sitter.Node) string { + if node == nil { + return "" + } + for i := uint(0); i < node.ChildCount(); i++ { + c := node.Child(i) + switch c.Kind() { + case "constant", "scope_resolution": + return rubyText(c, w.src) + } + } + return "" +} + +// bodyHasConcern reports whether a class/module body directly contains +// `extend ActiveSupport::Concern`. +func bodyHasConcern(body *sitter.Node, src []byte) bool { + if body == nil { + return false + } + for i := uint(0); i < body.ChildCount(); i++ { + c := body.Child(i) + if c.Kind() != "call" || c.ChildByFieldName("receiver") != nil { + continue + } + if rubyText(c.ChildByFieldName("method"), src) != "extend" { + continue + } + if firstConstArg(c.ChildByFieldName("arguments"), src) == "ActiveSupport::Concern" { + return true + } + } + return false +} + +// firstStringArg returns the content of the first string in an argument_list (or +// the string node itself when passed directly). +func firstStringArg(node *sitter.Node, src []byte) string { + if node == nil { + return "" + } + var find func(n *sitter.Node) *sitter.Node + find = func(n *sitter.Node) *sitter.Node { + if n.Kind() == "string" { + return n + } + for i := uint(0); i < n.ChildCount(); i++ { + if r := find(n.Child(i)); r != nil { + return r + } + } + return nil + } + s := find(node) + if s == nil { + return "" + } + for i := uint(0); i < s.ChildCount(); i++ { + if s.Child(i).Kind() == "string_content" { + return rubyText(s.Child(i), src) + } + } + return "" +} + +// firstConstArg returns the first constant / scope_resolution argument's text. +func firstConstArg(args *sitter.Node, src []byte) string { + if args == nil { + return "" + } + for i := uint(0); i < args.ChildCount(); i++ { + c := args.Child(i) + switch c.Kind() { + case "constant", "scope_resolution": + return rubyText(c, src) + } + } + return "" +} + +// symbolArgs returns the names (without leading ':') of all simple_symbol args. +func symbolArgs(args *sitter.Node, src []byte) []string { + var out []string + if args == nil { + return out + } + for i := uint(0); i < args.ChildCount(); i++ { + c := args.Child(i) + if c.Kind() == "simple_symbol" { + out = append(out, strings.TrimPrefix(rubyText(c, src), ":")) + } + } + return out +} + +// firstSymbolArg returns the first simple_symbol name (without leading ':'). +func firstSymbolArg(args *sitter.Node, src []byte) string { + for _, s := range symbolArgs(args, src) { + return s + } + return "" +} + +// isAllCaps reports whether s is an ALL_CAPS constant name (letters uppercase, +// digits and underscores allowed). Matches the former constant regex. +func isAllCaps(s string) bool { + if s == "" { + return false + } + hasLetter := false + for _, r := range s { + switch { + case r >= 'A' && r <= 'Z': + hasLetter = true + case r >= '0' && r <= '9', r == '_': + default: + return false + } + } + return hasLetter +} + +// line returns the 1-based start line of a node. +func line(node *sitter.Node) int { + return int(node.StartPosition().Row) + 1 +} + +// rubyText returns the source text covered by a node (nil-safe). +func rubyText(node *sitter.Node, src []byte) string { + if node == nil { + return "" + } + return string(src[node.StartByte():node.EndByte()]) +} diff --git a/internal/extractors/rubyextractor/ruby_test.go b/internal/extractors/rubyextractor/ruby_test.go index b8734f2..c2a4a07 100644 --- a/internal/extractors/rubyextractor/ruby_test.go +++ b/internal/extractors/rubyextractor/ruby_test.go @@ -9,9 +9,29 @@ import ( "github.com/enola-labs/enola/internal/facts" ) +// symbolsByName indexes the symbol facts in a result by name (storage/dependency +// facts may share a class name, so they are excluded here). +func symbolsByName(result []facts.Fact) map[string]facts.Fact { + m := make(map[string]facts.Fact) + for _, f := range result { + if f.Kind == facts.KindSymbol { + m[f.Name] = f + } + } + return m +} + +// hasCall returns true if the fact has a RelCalls relation to target. +func hasCall(f facts.Fact, target string) bool { + for _, r := range f.Relations { + if r.Kind == facts.RelCalls && r.Target == target { + return true + } + } + return false +} + func TestExtractFile_BasicClassAndMethod(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "order.rb") src := `# frozen_string_literal: true module Orders @@ -26,53 +46,27 @@ module Orders end end ` - if err := os.WriteFile(path, []byte(src), 0o644); err != nil { - t.Fatal(err) - } - f, err := os.Open(path) - if err != nil { - t.Fatal(err) - } - defer f.Close() - - result := extractFile(f, "packages/orders/app/models/order.rb", true, false) + result := extractFileAST([]byte(src), "packages/orders/app/models/order.rb", true, false) + byName := symbolsByName(result) - // Collect by kind and name. - byName := make(map[string]facts.Fact) - for _, fact := range result { - byName[fact.Name] = fact - } - - // Module Orders. mod, ok := byName["Orders"] if !ok { t.Fatal("missing module Orders") } - if mod.Kind != facts.KindSymbol { - t.Errorf("Orders kind = %q, want symbol", mod.Kind) - } - sk, _ := mod.Props["symbol_kind"].(string) - if sk != facts.SymbolInterface { + if sk, _ := mod.Props["symbol_kind"].(string); sk != facts.SymbolInterface { t.Errorf("Orders symbol_kind = %q, want interface", sk) } - // Class Orders::Order. cls, ok := byName["Orders::Order"] if !ok { t.Fatal("missing class Orders::Order") } - if cls.Kind != facts.KindSymbol { - t.Errorf("Orders::Order kind = %q, want symbol", cls.Kind) - } - sk, _ = cls.Props["symbol_kind"].(string) - if sk != facts.SymbolClass { + if sk, _ := cls.Props["symbol_kind"].(string); sk != facts.SymbolClass { t.Errorf("Orders::Order symbol_kind = %q, want class", sk) } - superclass, _ := cls.Props["superclass"].(string) - if superclass != "ApplicationRecord" { - t.Errorf("superclass = %q, want ApplicationRecord", superclass) + if sc, _ := cls.Props["superclass"].(string); sc != "ApplicationRecord" { + t.Errorf("superclass = %q, want ApplicationRecord", sc) } - // Should have implements relation to ApplicationRecord. hasImpl := false for _, r := range cls.Relations { if r.Kind == facts.RelImplements && r.Target == "ApplicationRecord" { @@ -83,55 +77,43 @@ end t.Error("Orders::Order missing implements relation to ApplicationRecord") } - // Instance method Orders::Order#total. meth, ok := byName["Orders::Order#total"] if !ok { t.Fatal("missing method Orders::Order#total") } - sk, _ = meth.Props["symbol_kind"].(string) - if sk != facts.SymbolMethod { + if sk, _ := meth.Props["symbol_kind"].(string); sk != facts.SymbolMethod { t.Errorf("total symbol_kind = %q, want method", sk) } - // Class method Orders::Order.recent. cmeth, ok := byName["Orders::Order.recent"] if !ok { t.Fatal("missing class method Orders::Order.recent") } - sk, _ = cmeth.Props["symbol_kind"].(string) - if sk != facts.SymbolFunc { + if sk, _ := cmeth.Props["symbol_kind"].(string); sk != facts.SymbolFunc { t.Errorf("recent symbol_kind = %q, want function", sk) } } func TestStorageFacts_DeclaresTargetIsDirectory(t *testing.T) { relFile := "packages/items/app/models/item.rb" + src := `class Item < ApplicationRecord +end +` + result := extractFileAST([]byte(src), relFile, true, true) - fileFacts := []facts.Fact{ - { - Kind: facts.KindSymbol, - Name: "Item", - File: relFile, - Line: 3, - Props: map[string]any{ - "symbol_kind": facts.SymbolClass, - "superclass": "ApplicationRecord", - "language": "ruby", - }, - }, + var storageFact *facts.Fact + for i, f := range result { + if f.Kind == facts.KindStorage && f.Name == "Item" { + storageFact = &result[i] + break + } } - - result := extractStorageFacts(relFile, fileFacts) - if len(result) == 0 { - t.Fatal("expected at least one storage fact") + if storageFact == nil { + t.Fatal("expected a storage fact named Item") } - - storageFact := result[0] - if storageFact.Name != "Item" { - t.Errorf("storage fact name = %q, want Item", storageFact.Name) + if sk, _ := storageFact.Props["storage_kind"].(string); sk != "model" { + t.Errorf("storage_kind = %q, want model", sk) } - - // The declares target must be the directory, not the class name. if len(storageFact.Relations) == 0 { t.Fatal("storage fact has no relations") } @@ -146,37 +128,26 @@ func TestStorageFacts_DeclaresTargetIsDirectory(t *testing.T) { } func TestAssociationFactNames_IncludeFilePath(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "order.rb") + relFile := "packages/orders/app/models/order.rb" src := `class Order < ApplicationRecord belongs_to :user has_many :items end ` - if err := os.WriteFile(path, []byte(src), 0o644); err != nil { - t.Fatal(err) - } + result := extractFileAST([]byte(src), relFile, true, true) - relFile := "packages/orders/app/models/order.rb" - result := extractAssociationsFromFile(path, relFile) - - if len(result) == 0 { - t.Fatal("expected association facts") - } - - for _, fact := range result { - if fact.Kind != facts.KindDependency { + names := make(map[string]bool) + for _, f := range result { + if f.Kind != facts.KindDependency { continue } - if !strings.HasPrefix(fact.Name, relFile+":") { - t.Errorf("association fact name %q should start with file path %q", fact.Name, relFile+":") + if _, ok := f.Props["association_kind"]; !ok { + continue } - } - - // Verify specific associations. - names := make(map[string]bool) - for _, fact := range result { - names[fact.Name] = true + if !strings.HasPrefix(f.Name, relFile+":") { + t.Errorf("association fact name %q should start with file path %q", f.Name, relFile+":") + } + names[f.Name] = true } if !names[relFile+":belongs_to :user"] { t.Error("missing belongs_to :user with file prefix") @@ -184,20 +155,24 @@ end if !names[relFile+":has_many :items"] { t.Error("missing has_many :items with file prefix") } -} - -// --- RelCalls extraction tests --- -// hasCall returns true if the fact has a RelCalls relation to target. -func hasCall(f facts.Fact, target string) bool { - for _, r := range f.Relations { - if r.Kind == facts.RelCalls && r.Target == target { - return true + // has_many target is singularized + camelized; belongs_to is camelized as-is. + for _, f := range result { + if f.Name == relFile+":has_many :items" { + if f.Relations[0].Target != "Item" { + t.Errorf("has_many :items target = %q, want Item", f.Relations[0].Target) + } + } + if f.Name == relFile+":belongs_to :user" { + if f.Relations[0].Target != "User" { + t.Errorf("belongs_to :user target = %q, want User", f.Relations[0].Target) + } } } - return false } +// --- RelCalls extraction tests --- + func TestExtractFile_QualifiedClassMethodCall(t *testing.T) { src := `module Items class FetchService @@ -207,30 +182,13 @@ func TestExtractFile_QualifiedClassMethodCall(t *testing.T) { end end ` - dir := t.TempDir() - path := filepath.Join(dir, "fetch_service.rb") - if err := os.WriteFile(path, []byte(src), 0o644); err != nil { - t.Fatal(err) - } - f, err := os.Open(path) - if err != nil { - t.Fatal(err) - } - defer f.Close() - - result := extractFile(f, "packages/items/app/services/fetch_service.rb", false, true) - - byName := make(map[string]facts.Fact) - for _, fact := range result { - byName[fact.Name] = fact - } - - meth, ok := byName["Items::FetchService#call"] + result := extractFileAST([]byte(src), "packages/items/app/services/fetch_service.rb", false, true) + meth, ok := symbolsByName(result)["Items::FetchService#call"] if !ok { t.Fatal("missing method Items::FetchService#call") } if !hasCall(meth, "Items::Facade.fetch_item_fields") { - t.Errorf("Items::FetchService#call missing RelCalls -> Items::Facade.fetch_item_fields; relations = %v", meth.Relations) + t.Errorf("missing RelCalls -> Items::Facade.fetch_item_fields; relations = %v", meth.Relations) } } @@ -243,30 +201,13 @@ func TestExtractFile_MultiLevelNamespaceCall(t *testing.T) { end end ` - dir := t.TempDir() - path := filepath.Join(dir, "builder.rb") - if err := os.WriteFile(path, []byte(src), 0o644); err != nil { - t.Fatal(err) - } - f, err := os.Open(path) - if err != nil { - t.Fatal(err) - } - defer f.Close() - - result := extractFile(f, "packages/homepage_sources/app/builder.rb", false, true) - - byName := make(map[string]facts.Fact) - for _, fact := range result { - byName[fact.Name] = fact - } - - meth, ok := byName["HomepageSources::Builder#build"] + result := extractFileAST([]byte(src), "packages/homepage_sources/app/builder.rb", false, true) + meth, ok := symbolsByName(result)["HomepageSources::Builder#build"] if !ok { t.Fatal("missing method HomepageSources::Builder#build") } if !hasCall(meth, "HomepageSources::ItemDto.from_ids") { - t.Errorf("HomepageSources::Builder#build missing RelCalls -> HomepageSources::ItemDto.from_ids; relations = %v", meth.Relations) + t.Errorf("missing RelCalls -> HomepageSources::ItemDto.from_ids; relations = %v", meth.Relations) } } @@ -277,30 +218,13 @@ func TestExtractFile_ReceiverVariableCall(t *testing.T) { end end ` - dir := t.TempDir() - path := filepath.Join(dir, "order_processor.rb") - if err := os.WriteFile(path, []byte(src), 0o644); err != nil { - t.Fatal(err) - } - f, err := os.Open(path) - if err != nil { - t.Fatal(err) - } - defer f.Close() - - result := extractFile(f, "app/models/order_processor.rb", false, true) - - byName := make(map[string]facts.Fact) - for _, fact := range result { - byName[fact.Name] = fact - } - - meth, ok := byName["OrderProcessor#process"] + result := extractFileAST([]byte(src), "app/models/order_processor.rb", false, true) + meth, ok := symbolsByName(result)["OrderProcessor#process"] if !ok { t.Fatal("missing method OrderProcessor#process") } if !hasCall(meth, "service.call") { - t.Errorf("OrderProcessor#process missing RelCalls -> service.call; relations = %v", meth.Relations) + t.Errorf("missing RelCalls -> service.call; relations = %v", meth.Relations) } } @@ -312,29 +236,11 @@ func TestExtractFile_CallsDeduplication(t *testing.T) { end end ` - dir := t.TempDir() - path := filepath.Join(dir, "dispatcher.rb") - if err := os.WriteFile(path, []byte(src), 0o644); err != nil { - t.Fatal(err) - } - f, err := os.Open(path) - if err != nil { - t.Fatal(err) - } - defer f.Close() - - result := extractFile(f, "app/dispatcher.rb", false, true) - - byName := make(map[string]facts.Fact) - for _, fact := range result { - byName[fact.Name] = fact - } - - meth, ok := byName["Dispatcher#run"] + result := extractFileAST([]byte(src), "app/dispatcher.rb", false, true) + meth, ok := symbolsByName(result)["Dispatcher#run"] if !ok { t.Fatal("missing method Dispatcher#run") } - count := 0 for _, r := range meth.Relations { if r.Kind == facts.RelCalls && r.Target == "Items::Facade.fetch_item_fields" { @@ -342,53 +248,33 @@ end } } if count != 1 { - t.Errorf("expected exactly 1 RelCalls edge to Items::Facade.fetch_item_fields, got %d", count) + t.Errorf("expected exactly 1 RelCalls edge, got %d", count) } } func TestExtractFile_TopLevelMethodCalls(t *testing.T) { - // Ruby allows method calls without parentheses; qualifiedCallRe must capture - // them even when there is no trailing '(' character. + // Ruby allows method calls without parentheses; the qualified tier must still + // capture them. src := `def bootstrap Config.load_defaults Rails.application.initialize! end ` - dir := t.TempDir() - path := filepath.Join(dir, "init.rb") - if err := os.WriteFile(path, []byte(src), 0o644); err != nil { - t.Fatal(err) - } - f, err := os.Open(path) - if err != nil { - t.Fatal(err) - } - defer f.Close() - - result := extractFile(f, "config/init.rb", false, true) - - byName := make(map[string]facts.Fact) - for _, fact := range result { - byName[fact.Name] = fact - } - - meth, ok := byName["config.bootstrap"] + result := extractFileAST([]byte(src), "config/init.rb", false, true) + meth, ok := symbolsByName(result)["config.bootstrap"] if !ok { t.Fatal("missing top-level method config.bootstrap") } - // Config.load_defaults has no parens — qualifiedCallRe must still fire. if !hasCall(meth, "Config.load_defaults") { - t.Errorf("config.bootstrap missing RelCalls -> Config.load_defaults; relations = %v", meth.Relations) + t.Errorf("missing RelCalls -> Config.load_defaults; relations = %v", meth.Relations) } - // Rails.application.initialize! — qualifiedCallRe captures the first segment: Rails.application. if !hasCall(meth, "Rails.application") { - t.Errorf("config.bootstrap missing RelCalls -> Rails.application; relations = %v", meth.Relations) + t.Errorf("missing RelCalls -> Rails.application; relations = %v", meth.Relations) } } func TestExtractFile_EndlessMethodCall(t *testing.T) { // Ruby 3.0+ endless method: def name(args) = Expr.call(args) - // The call is on the same line as the def — must be captured directly. src := `module HomepageSources class ItemDto ITEM_FIELDS = %i[id title].freeze @@ -397,76 +283,280 @@ func TestExtractFile_EndlessMethodCall(t *testing.T) { end end ` - dir := t.TempDir() - path := filepath.Join(dir, "item_dto.rb") - if err := os.WriteFile(path, []byte(src), 0o644); err != nil { - t.Fatal(err) + result := extractFileAST([]byte(src), "packages/homepage_sources/app/public/homepage_sources/item_dto.rb", false, true) + byName := symbolsByName(result) + + meth, ok := byName["HomepageSources::ItemDto#fields_by_id"] + if !ok { + t.Fatal("missing method HomepageSources::ItemDto#fields_by_id") } - f, err := os.Open(path) - if err != nil { - t.Fatal(err) + if !hasCall(meth, "Items::Facade.fetch_item_fields") { + t.Errorf("missing RelCalls -> Items::Facade.fetch_item_fields; relations = %v", meth.Relations) + } + // The ALL-CAPS constant should be captured. + if _, ok := byName["HomepageSources::ItemDto::ITEM_FIELDS"]; !ok { + t.Error("missing constant HomepageSources::ItemDto::ITEM_FIELDS") + } +} + +func TestCallEdges_QualifiedAndReceiverAndChain(t *testing.T) { + src := `class Logger + def run(x) + Items::Facade.fetch_item_fields(x) + service.call(x) + Rails.logger.info("msg") + end +end +` + result := extractFileAST([]byte(src), "app/logger.rb", false, true) + meth := symbolsByName(result)["Logger#run"] + for _, want := range []string{ + "Items::Facade.fetch_item_fields", // scope-resolution receiver + "service.call", // lowercase receiver with args + "Rails.logger", // qualified inner of a chain + "logger.info", // chained receiver with args + } { + if !hasCall(meth, want) { + t.Errorf("missing RelCalls -> %s; relations = %v", want, meth.Relations) + } } - defer f.Close() +} - result := extractFile(f, "packages/homepage_sources/app/public/homepage_sources/item_dto.rb", false, true) +// --- AST-only coverage (cases the regex scanner handled poorly) --- - byName := make(map[string]facts.Fact) - for _, fact := range result { - byName[fact.Name] = fact +func TestAST_MultiLineCallArguments(t *testing.T) { + src := `class Svc + def run(ids) + Items::Facade.fetch_item_fields( + ids, + FIELDS, + ) + end +end +` + result := extractFileAST([]byte(src), "app/svc.rb", false, true) + meth := symbolsByName(result)["Svc#run"] + if !hasCall(meth, "Items::Facade.fetch_item_fields") { + t.Errorf("multi-line call not captured; relations = %v", meth.Relations) } +} - meth, ok := byName["HomepageSources::ItemDto#fields_by_id"] +func TestAST_HeredocContainingEnd(t *testing.T) { + // A heredoc body containing a bare "end" line used to corrupt the regex + // depth counter; the grammar treats it as string content. + src := "class Foo\n" + + " def bar\n" + + " sql = <<~SQL\n" + + " SELECT 1\n" + + " end\n" + + " SQL\n" + + " Other.call(sql)\n" + + " end\n" + + "end\n" + result := extractFileAST([]byte(src), "app/foo.rb", false, true) + byName := symbolsByName(result) + if _, ok := byName["Foo"]; !ok { + t.Fatal("missing class Foo") + } + meth, ok := byName["Foo#bar"] if !ok { - t.Fatal("missing method HomepageSources::ItemDto#fields_by_id") + t.Fatal("missing method Foo#bar (heredoc likely broke scope tracking)") } - if !hasCall(meth, "Items::Facade.fetch_item_fields") { - t.Errorf("fields_by_id missing RelCalls -> Items::Facade.fetch_item_fields; relations = %v", meth.Relations) + if !hasCall(meth, "Other.call") { + t.Errorf("call after heredoc not captured; relations = %v", meth.Relations) + } +} + +func TestAST_NestedModulesAndEigenclass(t *testing.T) { + src := `module A + module B + class C + class << self + def build + end + end + end + end +end +` + result := extractFileAST([]byte(src), "app/a.rb", false, true) + byName := symbolsByName(result) + if _, ok := byName["A::B::C"]; !ok { + t.Fatal("missing deeply nested class A::B::C") + } + build, ok := byName["A::B::C.build"] + if !ok { + t.Fatal("missing eigenclass method A::B::C.build") + } + if sk, _ := build.Props["symbol_kind"].(string); sk != facts.SymbolFunc { + t.Errorf("eigenclass method symbol_kind = %q, want func", sk) + } +} + +func TestAST_ConcernDetection(t *testing.T) { + src := `module Trackable + extend ActiveSupport::Concern +end +` + result := extractFileAST([]byte(src), "app/models/concerns/trackable.rb", true, true) + mod := symbolsByName(result)["Trackable"] + if c, _ := mod.Props["concern"].(bool); !c { + t.Errorf("Trackable should be flagged concern:true; props = %v", mod.Props) + } + // extend ActiveSupport::Concern must not be emitted as a mixin dependency. + for _, f := range result { + if f.Kind == facts.KindDependency { + for _, r := range f.Relations { + if r.Target == "ActiveSupport::Concern" { + t.Error("ActiveSupport::Concern should not be a mixin dependency") + } + } + } } } -func TestExtractRubyCalls_QualifiedAndReceiver(t *testing.T) { - cases := []struct { - line string - want []string - }{ - { - line: " Items::Facade.fetch_item_fields(ids, ITEM_FIELDS)", - want: []string{"Items::Facade.fetch_item_fields"}, - }, - { - line: " service.call(x)", - want: []string{"service.call"}, - }, - { - line: " Foo::Bar::Baz.do_thing(a, b)", - want: []string{"Foo::Bar::Baz.do_thing"}, - }, - { - // Chained call: qualifiedCallRe captures Rails.logger (first segment); - // receiverCallRe captures logger.info (lowercase receiver with parens). - line: " Rails.logger.info('msg')", - want: []string{"Rails.logger", "logger.info"}, - }, - } - - for _, tc := range cases { - got := extractRubyCalls(tc.line) - gotSet := make(map[string]bool) - for _, g := range got { - gotSet[g] = true +func TestAST_MixinsAndImports(t *testing.T) { + src := `class Account < ApplicationRecord + include Trackable + prepend Auditable + attr_accessor :name, :token + require "set" + require_relative "../helper" +end +` + result := extractFileAST([]byte(src), "app/models/account.rb", true, true) + + var includeKind, prependKind, reqRel string + attrs := map[string]bool{} + imports := map[string]bool{} + for _, f := range result { + if f.Kind == facts.KindDependency { + if mk, _ := f.Props["mixin_kind"].(string); mk != "" { + if f.Relations[0].Target == "Trackable" { + includeKind = mk + } + if f.Relations[0].Target == "Auditable" { + prependKind = mk + } + } + if rr, _ := f.Props["require_relative"].(bool); rr { + reqRel = f.Relations[0].Target + } + for _, r := range f.Relations { + if r.Kind == facts.RelImports { + imports[r.Target] = true + } + } } - for _, w := range tc.want { - if !gotSet[w] { - t.Errorf("extractRubyCalls(%q): missing %q in %v", tc.line, w, got) + if f.Kind == facts.KindSymbol { + if ak, _ := f.Props["attr_kind"].(string); ak == "accessor" { + attrs[f.Name] = true } } } + if includeKind != "include" { + t.Errorf("include mixin_kind = %q, want include", includeKind) + } + if prependKind != "prepend" { + t.Errorf("prepend mixin_kind = %q, want prepend", prependKind) + } + if reqRel != "../helper" { + t.Errorf("require_relative target = %q, want ../helper", reqRel) + } + if !imports["set"] { + t.Error("missing require 'set' import") + } + if !attrs["Account#name"] || !attrs["Account#token"] { + t.Errorf("missing attr_accessor symbols; got %v", attrs) + } +} + +// --- route tests --- + +func TestRoutes_NestedNamespaceResourcesMember(t *testing.T) { + src := `Rails.application.routes.draw do + namespace :admin do + resources :users, only: [:index, :show] do + member do + post "ban" + end + end + end + draw(:billing) +end +` + result := parseRouteFileAST([]byte(src), "config/routes.rb") + names := make(map[string]facts.Fact) + for _, f := range result { + if f.Kind == facts.KindRoute { + names[f.Name] = f + } + } + + for _, want := range []string{"/admin/users", "/admin/users/:id", "/admin/users/:id/ban", "/billing"} { + if _, ok := names[want]; !ok { + t.Errorf("missing route %q; got %v", want, keys(names)) + } + } + // only: [:index, :show] must exclude create/new/edit/destroy. + for _, absent := range []string{"/admin/users/new", "/admin/users/:id/edit"} { + if _, ok := names[absent]; ok { + t.Errorf("route %q should be excluded by only:", absent) + } + } + if f, ok := names["/admin/users/:id/ban"]; ok { + if m, _ := f.Props["method"].(string); m != "POST" { + t.Errorf("ban route method = %q, want POST", m) + } + } + if f, ok := names["/billing"]; ok { + if m, _ := f.Props["method"].(string); m != "DRAW" { + t.Errorf("draw route method = %q, want DRAW", m) + } + } +} + +func TestRoutes_VerbWithHandler(t *testing.T) { + src := `Rails.application.routes.draw do + root to: "home#index" + get "/health", to: "health#show" +end +` + result := parseRouteFileAST([]byte(src), "config/routes.rb") + byName := make(map[string]facts.Fact) + for _, f := range result { + byName[f.Name] = f + } + root, ok := byName["/"] + if !ok { + t.Fatal("missing root route") + } + if h, _ := root.Props["handler"].(string); h != "home#index" { + t.Errorf("root handler = %q, want home#index", h) + } + health, ok := byName["/health"] + if !ok { + t.Fatal("missing /health route") + } + if h, _ := health.Props["handler"].(string); h != "health#show" { + t.Errorf("/health handler = %q, want health#show", h) + } + if m, _ := health.Props["method"].(string); m != "GET" { + t.Errorf("/health method = %q, want GET", m) + } +} + +func keys(m map[string]facts.Fact) []string { + var out []string + for k := range m { + out = append(out, k) + } + return out } func TestPackwerk_RootDependencyNormalization(t *testing.T) { dir := t.TempDir() - // Create packwerk.yml. packwerkYml := `package_paths: - "." - "packages/*" @@ -474,15 +564,11 @@ func TestPackwerk_RootDependencyNormalization(t *testing.T) { if err := os.WriteFile(filepath.Join(dir, "packwerk.yml"), []byte(packwerkYml), 0o644); err != nil { t.Fatal(err) } - - // Root package.yml. rootPkg := `enforce_dependencies: true ` if err := os.WriteFile(filepath.Join(dir, "package.yml"), []byte(rootPkg), 0o644); err != nil { t.Fatal(err) } - - // A sub-package that depends on root ("."). pkgDir := filepath.Join(dir, "packages", "orders") if err := os.MkdirAll(pkgDir, 0o755); err != nil { t.Fatal(err) @@ -501,7 +587,6 @@ dependencies: t.Fatal("packwerk should be detected") } - // Find the orders module fact. var ordersFact *facts.Fact for i, f := range info.facts { if f.Name == "packages/orders" { @@ -513,7 +598,6 @@ dependencies: t.Fatal("missing packages/orders module fact") } - // The dependency on "." should be normalized to "root". hasDotTarget := false hasRootTarget := false for _, r := range ordersFact.Relations { @@ -533,7 +617,6 @@ dependencies: t.Error("expected dependency target 'root' after normalization") } - // The root module should be named "root", not ".". var rootFact *facts.Fact for i, f := range info.facts { if f.Name == "root" { diff --git a/internal/extractors/rubyextractor/storage.go b/internal/extractors/rubyextractor/storage.go index 0b902a4..23d3e8d 100644 --- a/internal/extractors/rubyextractor/storage.go +++ b/internal/extractors/rubyextractor/storage.go @@ -1,14 +1,10 @@ package rubyextractor import ( - "path/filepath" - "regexp" "strings" - - "github.com/enola-labs/enola/internal/facts" ) -// ActiveRecord patterns. +// ActiveRecord base-class detection. var ( // Base classes that indicate an ActiveRecord model. arBaseClasses = []string{ @@ -17,123 +13,8 @@ var ( } // Suffix convention for abstract base models (e.g. ItemsModel, ShippingModel). arModelSuffix = "Model" - - associationRe = regexp.MustCompile( - `^\s*(has_many|has_one|belongs_to|has_and_belongs_to_many)\s+:(\w+)`) - scopeRe = regexp.MustCompile(`^\s*scope\s+:(\w+)`) - validatesRe = regexp.MustCompile(`^\s*validates?\s+:(\w+)`) - tableNameRe = regexp.MustCompile(`^\s*self\.table_name\s*=\s*['"](\w+)['"]`) ) -// extractStorageFacts scans the file-level facts for ActiveRecord model classes -// and emits storage facts with associations, scopes, and table names. -func extractStorageFacts(relFile string, fileFacts []facts.Fact) []facts.Fact { - var result []facts.Fact - - // First, identify which classes in this file are ActiveRecord models. - modelClasses := make(map[string]bool) - for _, f := range fileFacts { - if f.Kind != facts.KindSymbol { - continue - } - sk, _ := f.Props["symbol_kind"].(string) - if sk != facts.SymbolClass { - continue - } - superclass, _ := f.Props["superclass"].(string) - if isARBaseClass(superclass) { - modelClasses[f.Name] = true - } - } - - if len(modelClasses) == 0 { - return nil - } - - // Re-scan the file to extract associations, scopes, validations, and table name. - // We do this by re-reading from the already-parsed facts plus scanning the source again. - // For efficiency, we extract what we can from a simple second pass of the file facts. - // However, associations/scopes/validates aren't captured as facts yet, so we need - // to read the source file. We'll use the fileFacts to identify model boundaries - // and build storage facts. - - dir := filepath.Dir(relFile) - - // For each model class, emit a storage fact. - for className := range modelClasses { - tableName := inferTableName(className) - - result = append(result, facts.Fact{ - Kind: facts.KindStorage, - Name: className, - File: relFile, - Props: map[string]any{ - "storage_kind": "model", - "table": tableName, - "language": "ruby", - "framework": "rails", - }, - Relations: []facts.Relation{ - {Kind: facts.RelDeclares, Target: dir}, - }, - }) - } - - return result -} - -// extractStorageDetailsFromFile does a second pass on an open file to extract -// associations, scopes, validations, and explicit table names for models. -// This is called from the main Extract loop. -func extractStorageDetailsFromFile(lines []string, modelClasses map[string]bool) []facts.Fact { - var result []facts.Fact - - for lineNum, line := range lines { - // Association declarations. - if m := associationRe.FindStringSubmatch(line); m != nil { - assocKind := m[1] - assocName := m[2] - - targetModel := singularize(assocName) - if assocKind == "has_many" || assocKind == "has_and_belongs_to_many" { - targetModel = singularize(assocName) - } else { - targetModel = assocName - } - targetModel = snakeToCamel(targetModel) - - result = append(result, facts.Fact{ - Kind: facts.KindDependency, - Name: assocKind + " :" + assocName, - Line: lineNum + 1, - Props: map[string]any{ - "language": "ruby", - "association_kind": assocKind, - }, - Relations: []facts.Relation{ - {Kind: facts.RelDependsOn, Target: targetModel}, - }, - }) - } - - // Scope declarations. - if m := scopeRe.FindStringSubmatch(line); m != nil { - result = append(result, facts.Fact{ - Kind: facts.KindSymbol, - Name: "scope:" + m[1], - Line: lineNum + 1, - Props: map[string]any{ - "symbol_kind": facts.SymbolFunc, - "language": "ruby", - "scope": true, - }, - }) - } - } - - return result -} - // isARBaseClass returns true if the superclass indicates an ActiveRecord model. func isARBaseClass(superclass string) bool { if superclass == "" {