Avoid duplicate SCIP documents

snapshots/input/overlapping-projects/package.json

@@ -0,0 +1,5 @@
+{
+  "name": "overlapping-projects",
+  "version": "1.0.0",
+  "private": true
+}

snapshots/input/overlapping-projects/parent/nested/shared.ts

@@ -0,0 +1,3 @@
+export function shared(): string {
+  return 'shared'
+}

snapshots/input/overlapping-projects/parent/nested/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "compilerOptions": {
+    "composite": true
+  },
+  "include": ["shared.ts"]
+}

snapshots/input/overlapping-projects/parent/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "compilerOptions": {
+    "composite": true
+  },
+  "include": ["**/*.ts"]
+}

snapshots/input/overlapping-projects/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  "files": [],
+  "references": [{ "path": "./parent" }, { "path": "./parent/nested" }]
+}

snapshots/output/overlapping-projects/parent/nested/shared.ts

@@ -0,0 +1,8 @@
+// language TypeScript
+// < definition . . nested/`shared.ts`/
+
+export function shared(): string {
+//              ^^^^^^ definition . . nested/`shared.ts`/shared().
+  return 'shared'
+}
+

src/CommandLineOptions.ts

@@ -35,6 +35,7 @@ export interface GlobalCache {
     [ts.SourceFile | undefined, ts.ScriptTarget | ts.CreateSourceFileOptions]
   >
   parsedCommandLines: Map<string, ts.ParsedCommandLine>
+  indexedFiles: Set<string>
 }
 
 export function mainCommand(

src/ProjectIndexer.test.ts

@@ -1,7 +1,17 @@
+import * as fs from 'fs'
+import * as os from 'os'
+import * as path from 'path'
+
+import * as ts from 'typescript'
 import { test } from 'uvu'
 import * as assert from 'uvu/assert'
 
-import { languageForFileName, prettyMilliseconds } from './ProjectIndexer'
+import { GlobalCache, ProjectOptions } from './CommandLineOptions'
+import {
+  languageForFileName,
+  prettyMilliseconds,
+  ProjectIndexer,
+} from './ProjectIndexer'
 
 function minute(x: number): number {
   return x * 60 * 1000
@@ -37,4 +47,48 @@ test('languageForFileName', () => {
   assert.is(languageForFileName('Component.svelte'), '')
 })
 
+test('only deduplicates documents after successful emission', () => {
+  const projectRoot = fs.mkdtempSync(
+    path.join(os.tmpdir(), 'scip-typescript-project-')
+  )
+  try {
+    const fileName = path.join(projectRoot, 'index.ts')
+    fs.writeFileSync(fileName, 'export const value = 1\n')
+    const cache: GlobalCache = {
+      sources: new Map(),
+      parsedCommandLines: new Map(),
+      indexedFiles: new Set(),
+    }
+    const options: ProjectOptions = {
+      cwd: projectRoot,
+      projectRoot,
+      projectDisplayName: projectRoot,
+      output: path.join(projectRoot, 'index.scip'),
+      inferTsconfig: false,
+      progressBar: false,
+      yarnWorkspaces: false,
+      yarnBerryWorkspaces: false,
+      pnpmWorkspaces: false,
+      globalCaches: false,
+      indexedProjects: new Set(),
+      writeIndex: () => {
+        throw new Error('emission failed')
+      },
+    }
+    const config: ts.ParsedCommandLine = {
+      options: {},
+      fileNames: [fileName],
+      errors: [],
+    }
+
+    assert.throws(
+      () => new ProjectIndexer(config, options, cache).index(),
+      /emission failed/
+    )
+    assert.not.ok(cache.indexedFiles.has(fileName))
+  } finally {
+    fs.rmSync(projectRoot, { recursive: true })
+  }
+})
+
 test.run()

src/ProjectIndexer.ts

@@ -73,6 +73,7 @@ export class ProjectIndexer {
   private symbolCache: Map<ts.Node, ScipSymbol> = new Map()
   private hasConstructor: Map<ts.ClassDeclaration, boolean> = new Map()
   private packages: Packages
+  private indexedFiles: Set<string>
   constructor(
     public readonly config: ts.ParsedCommandLine,
     public readonly options: ProjectOptions,
@@ -82,22 +83,33 @@ export class ProjectIndexer {
     this.program = ts.createProgram(config.fileNames, config.options, host)
     this.checker = this.program.getTypeChecker()
     this.packages = new Packages(options.projectRoot)
+    this.indexedFiles = cache.indexedFiles
   }
   public index(): void {
     const startTimestamp = Date.now()
     const sourceFiles = this.program.getSourceFiles()
 
     const filesToIndex: ts.SourceFile[] = []
+    let projectFileCount = 0
     // Visit every sourceFile in the program
     for (const sourceFile of sourceFiles) {
       const includes = this.config.fileNames.includes(sourceFile.fileName)
       if (!includes) {
         continue
       }
+      projectFileCount++
+      if (this.indexedFiles.has(sourceFile.fileName)) {
+        continue
+      }
       filesToIndex.push(sourceFile)
     }
 
     if (filesToIndex.length === 0) {
+      if (projectFileCount > 0) {
+        // Every source belongs to a project that was indexed earlier. SCIP
+        // requires document paths to be unique across a complete index.
+        return
+      }
       throw new Error(
         `no indexable files in project '${this.options.projectDisplayName}'`
       )
@@ -159,6 +171,10 @@ export class ProjectIndexer {
             documents: [visitor.document],
           })
         )
+        // Only suppress the file in later overlapping projects after its
+        // document has actually been emitted. If indexing or emission fails,
+        // another project that includes the file can still retry it.
+        this.indexedFiles.add(sourceFile.fileName)
       }
     }
     jobs?.terminate()

src/main.test.ts

@@ -74,6 +74,15 @@ for (const snapshotDirectory of snapshotDirectories) {
     if (index.documents.length === 0) {
       throw new Error('empty LSIF index')
     }
+    const documentPaths = new Set<string>()
+    const duplicateDocuments: string[] = []
+    for (const document of index.documents) {
+      if (documentPaths.has(document.relative_path)) {
+        duplicateDocuments.push(document.relative_path)
+      }
+      documentPaths.add(document.relative_path)
+    }
+    assert.equal(duplicateDocuments, [], 'SCIP document paths should be unique')
     for (const document of index.documents) {
       const symbols = new Set<string>()
       const duplicateSymbols: string[] = []

src/main.ts

@@ -54,6 +54,7 @@ export function indexCommand(
   const cache: GlobalCache = {
     sources: new Map(),
     parsedCommandLines: new Map(),
+    indexedFiles: new Set(),
   }
   try {
     writeIndex(