WIP: Prism work, rebase only

.github/workflows/ci.yml

@@ -0,0 +1,34 @@
+name: CI
+
+on:
+  push:
+    branches: [prism]
+  pull_request:
+    branches: [prism]
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    name: Prism Parser Regression Tests
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.3.0
+      - name: Set up Bazel
+        uses: bazel-contrib/[email protected]
+        with:
+          # Avoid downloading Bazel every time.
+          bazelisk-cache: true
+          # Store build cache per workflow.
+          disk-cache: ${{ github.workflow }}
+          # Share repository cache between workflows.
+          repository-cache: true
+      - name: Verify parse trees
+        run: ./tools/scripts/verify_prism_regression_tests.sh
+      - name: Run tests
+        run: ./bazel test //test:prism_regression --config=dbg --test_output=errors
+      - name: Run location tests
+        run: ./bazel test //test:prism_location_tests --config=dbg --test_output=errors
+      - name: Run corpus tests
+        run: ./bazel test //test:test_corpus_prism --config=dbg --test_output=errors

.vscode/extensions.json

@@ -1,7 +1,9 @@
 {
   "recommendations": [
     // C++ intellisense with clangd.
-    "llvm-vs-code-extensions.vscode-clangd"
+    "llvm-vs-code-extensions.vscode-clangd",
+    // TODO: remove this before upstreaming
+    "vadimcn.vscode-lldb",
   ],
   "unwantedRecommendations": [
     // Microsoft C++ intellisense (doesn't work properly with Sorbet)

.vscode/launch.json

@@ -22,6 +22,43 @@
                     "ignoreFailures": false
                 }
             ]
+        },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug Prism test in LLDB",
+            "program": "${workspaceFolder}/bazel-bin/test/test_PosTests/prism_regression/${input:prism_regression_test_name}_prism.runfiles/com_stripe_ruby_typer/test/pipeline_test_runner",
+            "args": ["--single_test=${workspaceFolder}/test/prism_regression/${input:prism_regression_test_name}.rb", "--parser=prism"],
+            // We need to run all tests to generate the pipeline_test_runner files, which are needed by the LLDB debugger to execute the tests.
+            "preLaunchTask": "Run all Prism regression tests",
+            "stopOnEntry": false,
+            "sourceMap": {
+                "": "${workspaceFolder}",
+            },
+        },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug Sorbet pipeline in LLDB",
+            "program": "${workspaceFolder}/bazel-bin/main/sorbet",
+            "args": ["--parser=prism", "${input:file_path}"],
+            "preLaunchTask": "Build with Prism",
+            "stopOnEntry": false,
+            "sourceMap": {
+                "": "${workspaceFolder}",
+            },
+        },
+    ],
+    "inputs": [
+        {
+            "id": "prism_regression_test_name",
+            "type": "promptString",
+            "description": "Enter the test name, e.g. case for running test/prism_regression/case.rb",
+        },
+        {
+            "id": "file_path",
+            "type": "promptString",
+            "description": "Enter the path to the file to typecheck, e.g. test/prism_regression/case.rb",
         }
     ]
-}
+}

.vscode/tasks.json

@@ -0,0 +1,136 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "Build with Prism",
+            "type": "shell",
+            "command": "./bazel build //main:sorbet --config=dbg",
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            },
+            "presentation": {
+                "reveal": "always"
+            }
+        },
+        {
+            "label": "(Re)Generate Prism test",
+            "type": "shell",
+            "command": "TEST_NAME=${input:prism_regression_test_name}; $EDITOR -w test/prism_regression/$TEST_NAME.rb; ./bazel-bin/main/sorbet --stop-after=parser --print=parse-tree test/prism_regression/$TEST_NAME.rb > test/prism_regression/$TEST_NAME.parse-tree.exp; $EDITOR test/prism_regression/$TEST_NAME.parse-tree.exp test/prism_regression/$TEST_NAME.rb",
+            "group": {
+                "kind": "test",
+                "isDefault": true
+            },
+            "presentation": {
+                "reveal": "always"
+            }
+        },
+        {
+            "label": "Run a single Prism regression test",
+            "type": "shell",
+            "command": "./bazel test //test:test_PosTests/prism_regression/${input:prism_regression_test_name}_prism //test:prism_regression/${input:prism_regression_test_name}_location_test --config=dbg  --test_output=all",
+            "group": {
+                "kind": "test",
+                "isDefault": true
+            },
+            "presentation": {
+                "reveal": "always"
+            },
+            // With this option, when running this task with the `Tasks: Rerun Last Test` command, it'll
+            // reuse the previous run's input, rather than prompting for it again
+            "runOptions": {
+                "reevaluateOnRerun": false
+            }
+        },
+        {
+            "label": "Run all Prism regression tests",
+            "type": "shell",
+            "command": "./bazel test //test:prism_regression //test:prism_location_tests --config=dbg  --test_output=all",
+            "group": {
+                "kind": "test",
+                "isDefault": true
+            },
+            "presentation": {
+                "reveal": "always"
+            }
+        },
+        {
+            "label": "Run all Prism corpus tests",
+            "type": "shell",
+            "command": "./bazel test //test:test_corpus_prism --config=dbg  --test_output=all",
+            "group": {
+                "kind": "test",
+                "isDefault": true
+            },
+            "presentation": {
+                "reveal": "always"
+            }
+        },
+        {
+            "label": "Run a single Prism corpus test",
+            "type": "shell",
+            "command": "./bazel test //test:test_PosTests/testdata/${input:prism_corpus_test_name}_prism --config=dbg  --test_output=all",
+            "group": {
+                "kind": "test",
+                "isDefault": true
+            },
+            "presentation": {
+                "reveal": "always"
+            },
+            // With this option, when running this task with the `Tasks: Rerun Last Test` command, it'll
+            // reuse the previous run's input, rather than prompting for it again
+            "runOptions": {
+                "reevaluateOnRerun": false
+            }
+        },
+        {
+            "label": "Typecheck with Prism",
+            "type": "shell",
+            "command": "bazel-bin/main/sorbet --parser=prism ${input:file_path}",
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            },
+            "dependsOn": ["Build with Prism"],
+            "presentation": {
+                "reveal": "always"
+            }
+        },
+        {
+            "label": "Display Prism parse tree & errors for Ruby file",
+            "type": "shell",
+            "command": "tools/scripts/inspect_parsing_errors.rb ${file}",
+            "presentation": {
+                "reveal": "always"
+            },
+            "options": {
+                // Makes sure the shell can find the correct Ruby version
+                "shell": {
+                    "executable": "${env:SHELL}",
+                    "args": ["-c"]
+                }
+            },
+            "problemMatcher": []
+        }
+    ],
+    "inputs": [
+        {
+            "id": "prism_corpus_test_name",
+            "type": "promptString",
+            "description": "Enter the test name, e.g. parser/and_and_bug for running test/testdata/parser/and_and_bug.rb",
+        },
+        {
+            "id": "prism_regression_test_name",
+            "type": "promptString",
+            "description": "Enter the test name, e.g. case for running test/prism_regression/case.rb",
+        },
+        {
+            "id": "file_path",
+            "type": "promptString",
+            "default": "test.rb",
+            "description": "Enter the file path to typecheck, e.g. test.rb",
+        }
+    ]
+}

Gemfile

@@ -0,0 +1 @@
+source "https://rubygems.org"

Gemfile.lock

@@ -0,0 +1,12 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+
+PLATFORMS
+  arm64-darwin-23
+  ruby
+
+DEPENDENCIES
+
+BUNDLED WITH
+   2.5.17

main/options/options.cc

@@ -723,6 +723,9 @@ buildOptions(const vector<pipeline::semantic_extension::SemanticExtensionProvide
                                  cxxopts::value<vector<string>>()->implicit_value("all"), "SECTION");
     // }}}
 
+    options.add_options("dev")("parser", "Which parser to use", cxxopts::value<string>()->default_value("sorbet"),
+                               "{sorbet, prism}");
+
     for (auto &provider : semanticExtensionProviders) {
         provider->injectOptions(options);
     }
@@ -791,6 +794,22 @@ Phase extractStopAfter(cxxopts::ParseResult &raw, shared_ptr<spdlog::logger> log
     return Phase::INIT;
 }
 
+Parser extractParser(cxxopts::ParseResult &raw, shared_ptr<spdlog::logger> logger) {
+    string opt = raw["parser"].as<string>();
+    for (auto &known : parser_options) {
+        if (known.option == opt) {
+            return known.flag;
+        }
+    }
+    vector<string_view> allOptions;
+    for (auto &known : parser_options) {
+        allOptions.emplace_back(known.option);
+    }
+
+    logger->error("Unknown --parser option: {}\nValid values: {}", opt, fmt::join(allOptions, ", "));
+    return Parser::SORBET;
+}
+
 // Given a path, strips any trailing forward slashes (/) at the end of the path.
 string_view stripTrailingSlashes(string_view path) {
     while (path.back() == '/') {
@@ -954,6 +973,7 @@ void readOptions(Options &opts,
             throw EarlyReturnWithCode(1);
         }
         opts.stopAfterPhase = extractStopAfter(raw, logger);
+        opts.parser = extractParser(raw, logger);
 
         opts.silenceErrors = raw["quiet"].as<bool>();
         opts.autocorrect = raw["autocorrect"].as<bool>();
@@ -1346,6 +1366,7 @@ void readOptions(Options &opts,
             fmt::print("Sorbet typechecker {}\n", sorbet_full_version_string);
             throw EarlyReturnWithCode(0);
         }
+
     } catch (cxxopts::OptionParseException &e) {
         logger->info("{}. To see all available options pass `--help`.", e.what());
         throw EarlyReturnWithCode(1);

main/options/options.h

@@ -106,6 +106,28 @@ enum class Phase {
     INFERENCER,
 };
 
+enum class Parser {
+    PRISM,
+    SORBET,
+};
+
+struct ParserOptions {
+    std::string option;
+    Parser flag;
+};
+
+const std::vector<ParserOptions> parser_options({
+    {"sorbet", Parser::SORBET},
+    {"prism", Parser::PRISM},
+});
+
+struct AutogenConstCacheConfig {
+    // A file which contains a cache that can be used to potentially skip autogen
+    std::string cacheFile;
+    // A list of files which have changed since the last autogen run.
+    std::vector<std::string> changedFiles;
+};
+
 namespace {
 
 #if !defined(EMSCRIPTEN)
@@ -120,6 +142,7 @@ constexpr size_t MAX_CACHE_SIZE_BYTES = 1L * 1024 * 1024 * 1024; // 1 GiB
 struct Options {
     Printers print;
     Phase stopAfterPhase = Phase::INFERENCER;
+    Parser parser = Parser::SORBET;
     bool noStdlib = false;
 
     // Should we monitor STDOUT for HUP and exit if it hangs up. This is a

main/pipeline/BUILD

@@ -32,6 +32,7 @@ cc_library(
         "//namer",
         "//packager",
         "//parser",
+        "//parser/prism",
         "//payload/binary",
         "//payload/text",
         "//resolver",
@@ -70,6 +71,7 @@ cc_library(
         "//main/options",
         "//namer",
         "//parser",
+        "//parser/prism",
         "//payload/binary",
         "//payload/text",
         "//resolver",