Fix didopen for build files (#199)

#168 started
tracking build file changes via lifecycle methods, didOpen, etc. But it
didn't make a distinction between what was a build file, and what was a
Smithy file. There are two paths didOpen can take - the first is when
the file being opened is known to be part of a project. In this case,
the file is already tracked by its owning Project, so its basically a
no-op. The second path is when the file does not belong to any project,
in which case we created a "detached" project, which is a project with
no build files and just a single Smithy file. So if you opened a build
file that wasn't known to be part of a Project, the language server
tried to make a detached project containing the build file as a smithy
file. This is obviously wrong, but wasn't observable to clients AFAICT
because clients weren't set up to send requests to the server for build
files (specifically, you wouldn't get diagnostics or anything, only for
.smithy files). However, recent commits, including
#188, now want
to provide language support for smithy-build.json. In testing these new
commits with local changes to have VSCode send requests for
smithy-build.json, the issue could be observed. Specifically, the issue
happens when you open a new smithy-build.json before we receive the
didChangeWatchedFiles notification that tells us a new build file was
created. didChangeWatchedFiles is the way we actually updated the state
of projects to include new build files, or create new Projects. Since we
can receive didOpen for a build file before didChangeWatchedFiles, we
needed to do something with the build file on didOpen.

This commit addresses the problem by adding a new Project type,
UNRESOLVED, which is a project containing a single build file that no
existing projects are aware of. We do this by modifying the didOpen path
when the file isn't known to any project, checking if it is a build file
using a PathMatcher, and if it is, creating an unresolved project for
it. Then, when we load projects following a didChangeWatchedFiles, we
just drop any unresolved projects with the same path as any of the build
files in the newly loaded projects (see ServerState::resolveProjects).

I also made some (upgrades?) to FilePatterns to better handle the
discrepancy between matching behavior of PathMatchers and clients
(see #191).
Now there are (private) *PatternOptions enums that FilePatterns uses to
configure the pattern for different use cases. For example, the new
FilePatterns::getSmithyFileWatchPathMatchers provides a list of
PathMatchers which should match the same paths as the watcher patterns
we send back to clients, which is useful for testing.

I also fixed an issue where parsing an empty build file would cause an NPE
when trying to map validation events to ranges. Document::rangeBetween
would return null if the document was empty, but I wasn't checking for that
in ToSmithyNode (which creates parse events).

The reason the range is null is because Document.lineOfIndex returns
oob for an index of 0 into an empty document. Makes sense, as an empty
document has no lines. I updated a DocumentTest to clarify this
behavior.

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

Author: milesziemer

src/main/java/software/amazon/smithy/lsp/FilePatterns.java

@@ -9,6 +9,7 @@
 import java.nio.file.FileSystems;
 import java.nio.file.Path;
 import java.nio.file.PathMatcher;
+import java.util.EnumSet;
 import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -20,16 +21,37 @@
  * or build files in Projects and workspaces.
  */
 final class FilePatterns {
+    static final PathMatcher GLOBAL_BUILD_FILES_MATCHER = toPathMatcher(escapeBackslashes(
+            String.format("**%s{%s}", File.separator, String.join(",", BuildFileType.ALL_FILENAMES))));
+
     private FilePatterns() {
     }
 
+    private enum SmithyFilePatternOptions {
+        IS_WATCHER_PATTERN,
+        IS_PATH_MATCHER_PATTERN;
+
+        private static final EnumSet<SmithyFilePatternOptions> WATCHER = EnumSet.of(IS_WATCHER_PATTERN);
+        private static final EnumSet<SmithyFilePatternOptions> PATH_MATCHER = EnumSet.of(IS_PATH_MATCHER_PATTERN);
+        private static  final EnumSet<SmithyFilePatternOptions> ALL = EnumSet.allOf(SmithyFilePatternOptions.class);
+    }
+
+    private enum BuildFilePatternOptions {
+        IS_WORKSPACE_PATTERN,
+        IS_PATH_MATCHER_PATTERN;
+
+        private static final EnumSet<BuildFilePatternOptions> WORKSPACE = EnumSet.of(IS_WORKSPACE_PATTERN);
+        private static final EnumSet<BuildFilePatternOptions> PATH_MATCHER = EnumSet.of(IS_PATH_MATCHER_PATTERN);
+        private static final EnumSet<BuildFilePatternOptions> ALL = EnumSet.allOf(BuildFilePatternOptions.class);
+    }
+
     /**
      * @param project The project to get watch patterns for
      * @return A list of glob patterns used to watch Smithy files in the given project
      */
     static List<String> getSmithyFileWatchPatterns(Project project) {
         return Stream.concat(project.sources().stream(), project.imports().stream())
-                .map(path -> getSmithyFilePattern(path, true))
+                .map(path -> getSmithyFilePattern(path, SmithyFilePatternOptions.WATCHER))
                 .toList();
     }
 
@@ -39,25 +61,36 @@ static List<String> getSmithyFileWatchPatterns(Project project) {
      */
     static PathMatcher getSmithyFilesPathMatcher(Project project) {
         String pattern = Stream.concat(project.sources().stream(), project.imports().stream())
-                .map(path -> getSmithyFilePattern(path, false))
+                .map(path -> getSmithyFilePattern(path, SmithyFilePatternOptions.PATH_MATCHER))
                 .collect(Collectors.joining(","));
         return toPathMatcher("{" + pattern + "}");
     }
 
+    /**
+     * @param project The project to get a path matcher for
+     * @return A list of path matchers that match watched Smithy files in the given project
+     */
+    static List<PathMatcher> getSmithyFileWatchPathMatchers(Project project) {
+        return Stream.concat(project.sources().stream(), project.imports().stream())
+                .map(path -> getSmithyFilePattern(path, SmithyFilePatternOptions.ALL))
+                .map(FilePatterns::toPathMatcher)
+                .toList();
+    }
+
     /**
      * @param root The root to get the watch pattern for
      * @return A glob pattern used to watch build files in the given workspace
      */
     static String getWorkspaceBuildFilesWatchPattern(Path root) {
-        return getBuildFilesPattern(root, true);
+        return getBuildFilesPattern(root, BuildFilePatternOptions.WORKSPACE);
     }
 
     /**
      * @param root The root to get a path matcher for
      * @return A path matcher that can check if a file is a build file within the given workspace
      */
     static PathMatcher getWorkspaceBuildFilesPathMatcher(Path root) {
-        String pattern = getWorkspaceBuildFilesWatchPattern(root);
+        String pattern = getBuildFilesPattern(root, BuildFilePatternOptions.ALL);
         return toPathMatcher(pattern);
     }
 
@@ -66,35 +99,19 @@ static PathMatcher getWorkspaceBuildFilesPathMatcher(Path root) {
      * @return A path matcher that can check if a file is a build file belonging to the given project
      */
     static PathMatcher getProjectBuildFilesPathMatcher(Project project) {
-        String pattern = getBuildFilesPattern(project.root(), false);
+        String pattern = getBuildFilesPattern(project.root(), BuildFilePatternOptions.PATH_MATCHER);
         return toPathMatcher(pattern);
     }
 
     private static PathMatcher toPathMatcher(String globPattern) {
         return FileSystems.getDefault().getPathMatcher("glob:" + globPattern);
     }
 
-    // Patterns for the workspace need to match on all build files in all subdirectories,
-    // whereas patterns for projects only look at the top level (because project locations
-    // are defined by the presence of these build files).
-    private static String getBuildFilesPattern(Path root, boolean isWorkspacePattern) {
-        String rootString = root.toString();
-        if (!rootString.endsWith(File.separator)) {
-            rootString += File.separator;
-        }
-
-        if (isWorkspacePattern) {
-            rootString += "**" + File.separator;
-        }
-
-        return escapeBackslashes(rootString + "{" + String.join(",", BuildFileType.ALL_FILENAMES) + "}");
-    }
-
     // When computing the pattern used for telling the client which files to watch, we want
     // to only watch .smithy/.json files. We don't need it in the PathMatcher pattern because
     // we only need to match files, not listen for specific changes (and it is impossible anyway
     // because we can't have a nested pattern).
-    private static String getSmithyFilePattern(Path path, boolean isWatcherPattern) {
+    private static String getSmithyFilePattern(Path path, EnumSet<SmithyFilePatternOptions> options) {
         String glob = path.toString();
         if (glob.endsWith(".smithy") || glob.endsWith(".json")) {
             return escapeBackslashes(glob);
@@ -105,13 +122,38 @@ private static String getSmithyFilePattern(Path path, boolean isWatcherPattern)
         }
         glob += "**";
 
-        if (isWatcherPattern) {
-            glob += "/*.{smithy,json}";
+        if (options.contains(SmithyFilePatternOptions.IS_WATCHER_PATTERN)) {
+            // For some reason, the glob pattern matching works differently on vscode vs
+            // PathMatcher. See https://github.com/smithy-lang/smithy-language-server/issues/191
+            if (options.contains(SmithyFilePatternOptions.IS_PATH_MATCHER_PATTERN)) {
+                glob += ".{smithy,json}";
+            } else {
+                glob += "/*.{smithy,json}";
+            }
         }
 
         return escapeBackslashes(glob);
     }
 
+    // Patterns for the workspace need to match on all build files in all subdirectories,
+    // whereas patterns for projects only look at the top level (because project locations
+    // are defined by the presence of these build files).
+    private static String getBuildFilesPattern(Path root, EnumSet<BuildFilePatternOptions> options) {
+        String rootString = root.toString();
+        if (!rootString.endsWith(File.separator)) {
+            rootString += File.separator;
+        }
+
+        if (options.contains(BuildFilePatternOptions.IS_WORKSPACE_PATTERN)) {
+            rootString += "**";
+            if (!options.contains(BuildFilePatternOptions.IS_PATH_MATCHER_PATTERN)) {
+                rootString += File.separator;
+            }
+        }
+
+        return escapeBackslashes(rootString + "{" + String.join(",", BuildFileType.ALL_FILENAMES) + "}");
+    }
+
     // In glob patterns, '\' is an escape character, so it needs to escaped
     // itself to work as a separator (i.e. for windows)
     private static String escapeBackslashes(String pattern) {

src/main/java/software/amazon/smithy/lsp/ServerState.java

@@ -118,6 +118,17 @@ ProjectAndFile open(String uri, String text) {
         if (projectAndFile != null) {
             projectAndFile.file().document().applyEdit(null, text);
         } else {
+            // A newly created build file or smithy file may be opened before we receive the
+            // `didChangeWatchedFiles` notification, so we either need to load an unresolved
+            // project (build file), or load a detached project (smithy file). When we receive
+            // `didChangeWatchedFiles` we will move the file into a regular project, if applicable.
+            Path path = Path.of(LspAdapter.toPath(uri));
+            if (FilePatterns.GLOBAL_BUILD_FILES_MATCHER.matches(path)) {
+                Project project = ProjectLoader.loadUnresolved(path, text);
+                projects.put(uri, project);
+                return findProjectAndFile(uri);
+            }
+
             createDetachedProject(uri, text);
             projectAndFile = findProjectAndFile(uri); // Note: This will always be present
         }
@@ -129,13 +140,16 @@ void close(String uri) {
         managedUris.remove(uri);
 
         ProjectAndFile projectAndFile = findProjectAndFile(uri);
-        if (projectAndFile != null && projectAndFile.project().type() == Project.Type.DETACHED) {
-            // Only cancel tasks for detached projects, since we're dropping the project
+        if (projectAndFile != null && shouldDropOnClose(projectAndFile.project())) {
             lifecycleTasks.cancelTask(uri);
             projects.remove(uri);
         }
     }
 
+    private static boolean shouldDropOnClose(Project project) {
+        return project.type() == Project.Type.DETACHED || project.type() == Project.Type.UNRESOLVED;
+    }
+
     List<Exception> tryInitProject(Path root) {
         LOGGER.finest("Initializing project at " + root);
         lifecycleTasks.cancelAllTasks();
@@ -145,9 +159,9 @@ List<Exception> tryInitProject(Path root) {
             Project updatedProject = ProjectLoader.load(root, this);
 
             if (updatedProject.type() == Project.Type.EMPTY) {
-                removeProjectAndResolveDetached(projectName);
+                removeProjectAndResolve(projectName);
             } else {
-                resolveDetachedProjects(projects.get(projectName), updatedProject);
+                resolveProjects(projects.get(projectName), updatedProject);
                 projects.put(projectName, updatedProject);
             }
 
@@ -191,7 +205,7 @@ void removeWorkspace(WorkspaceFolder folder) {
         }
 
         for (String projectName : projectsToRemove) {
-            removeProjectAndResolveDetached(projectName);
+            removeProjectAndResolve(projectName);
         }
     }
 
@@ -230,14 +244,18 @@ List<Exception> applyFileEvents(List<FileEvent> events) {
         return errors;
     }
 
-    private void removeProjectAndResolveDetached(String projectName) {
+    private void removeProjectAndResolve(String projectName) {
         Project removedProject = projects.remove(projectName);
         if (removedProject != null) {
-            resolveDetachedProjects(removedProject, Project.empty(removedProject.root()));
+            resolveProjects(removedProject, Project.empty(removedProject.root()));
         }
     }
 
-    private void resolveDetachedProjects(Project oldProject, Project updatedProject) {
+    private void resolveProjects(Project oldProject, Project updatedProject) {
+        // There may be unresolved projects that have been resolved by the updated project, so
+        // we need to remove them here.
+        removeDetachedOrUnresolvedProjects(updatedProject.getAllBuildFilePaths());
+
         // This is a project reload, so we need to resolve any added/removed files
         // that need to be moved to or from detachedProjects projects.
         if (oldProject != null) {
@@ -246,10 +264,7 @@ private void resolveDetachedProjects(Project oldProject, Project updatedProject)
 
             Set<String> addedPaths = new HashSet<>(updatedProjectSmithyPaths);
             addedPaths.removeAll(currentProjectSmithyPaths);
-            for (String addedPath : addedPaths) {
-                String addedUri = LspAdapter.toUri(addedPath);
-                projects.remove(addedUri); // Remove any detached projects
-            }
+            removeDetachedOrUnresolvedProjects(addedPaths);
 
             Set<String> removedPaths = new HashSet<>(currentProjectSmithyPaths);
             removedPaths.removeAll(updatedProjectSmithyPaths);
@@ -261,6 +276,17 @@ private void resolveDetachedProjects(Project oldProject, Project updatedProject)
                     createDetachedProject(removedUri, removedDocument.copyText());
                 }
             }
+        } else {
+            // This is a new project, so there may be detached projects that are resolved by
+            // this new project.
+            removeDetachedOrUnresolvedProjects(updatedProject.getAllSmithyFilePaths());
+        }
+    }
+
+    private void removeDetachedOrUnresolvedProjects(Set<String> filePaths) {
+        for (String filePath : filePaths) {
+            String uri = LspAdapter.toUri(filePath);
+            projects.remove(uri);
         }
     }
 

src/main/java/software/amazon/smithy/lsp/project/BuildFiles.java

@@ -10,7 +10,9 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import software.amazon.smithy.lsp.ManagedFiles;
 import software.amazon.smithy.lsp.document.Document;
 import software.amazon.smithy.lsp.protocol.LspAdapter;
@@ -49,6 +51,10 @@ boolean isEmpty() {
         return buildFiles.isEmpty();
     }
 
+    Set<String> getAllPaths() {
+        return buildFiles.keySet();
+    }
+
     static BuildFiles of(Collection<BuildFile> buildFiles) {
         Map<String, BuildFile> buildFileMap = new HashMap<>(buildFiles.size());
         for (BuildFile buildFile : buildFiles) {
@@ -57,6 +63,18 @@ static BuildFiles of(Collection<BuildFile> buildFiles) {
         return new BuildFiles(buildFileMap);
     }
 
+    static BuildFiles of(Path path, Document document) {
+        for (BuildFileType type : BuildFileType.values()) {
+            if (path.endsWith(type.filename())) {
+                String pathString = path.toString();
+                BuildFile buildFile = BuildFile.create(pathString, document, type);
+                return new BuildFiles(Map.of(pathString, buildFile));
+            }
+        }
+
+        return BuildFiles.of(List.of());
+    }
+
     static BuildFiles load(Path root, ManagedFiles managedFiles) {
         Map<String, BuildFile> buildFiles = new HashMap<>(BuildFileType.values().length);
         for (BuildFileType type : BuildFileType.values()) {

src/main/java/software/amazon/smithy/lsp/project/Project.java

@@ -86,6 +86,16 @@ public enum Type {
          */
         DETACHED,
 
+        /**
+         * A project loaded from a single build file.
+         *
+         * <p>This occurs when a newly created build file is opened before we
+         * receive its `didChangeWatchedFiles` notification, which takes care
+         * of both adding new build files to an existing project, and creating
+         * a new project in a new root.
+         */
+        UNRESOLVED,
+
         /**
          * A project loaded with no source or build configuration files.
          */
@@ -156,6 +166,10 @@ public Set<String> getAllSmithyFilePaths() {
         return this.smithyFiles.keySet();
     }
 
+    public Set<String> getAllBuildFilePaths() {
+        return this.buildFiles.getAllPaths();
+    }
+
     /**
      * @return All the Smithy files loaded in the project.
      */

src/main/java/software/amazon/smithy/lsp/project/ProjectLoader.java

@@ -67,6 +67,37 @@ public static Project loadDetached(String uri, String text) {
         );
     }
 
+    /**
+     * Loads an unresolved (single config file) {@link Project} with the given file.
+     *
+     * @param path Path of the file to load into a project
+     * @param text Text of the file to load into a project
+     * @return The loaded project
+     */
+    public static Project loadUnresolved(Path path, String text) {
+        Document document = Document.of(text);
+        BuildFiles buildFiles = BuildFiles.of(path, document);
+
+        // An unresolved project is meant to be resolved at a later point, so we don't
+        // even try loading its configuration from the build file.
+        ProjectConfig config = ProjectConfig.empty();
+
+        // We aren't loading any smithy files in this project, so use a no-op ManagedFiles.
+        LoadModelResult result = doLoad((fileUri) -> null, config);
+
+        return new Project(
+                path,
+                config,
+                buildFiles,
+                result.smithyFiles(),
+                result.assemblerFactory(),
+                Project.Type.UNRESOLVED,
+                result.modelResult(),
+                result.rebuildIndex(),
+                List.of()
+        );
+    }
+
     /**
      * Loads a {@link Project} at the given root path, using any {@code managedDocuments}
      * instead of loading from disk.