[glass] Optionally return full file content

Clients may want to update symbol positions to account for
local changes to the source file, and for this they need the original
file content as it was indexed. Currently the way to do that is to use
the `revision` returned by Glass and retrieve the file from source
control, but that might not be possible in general:

 * the indexed file might not be checked in (consider interactively
   indexing code in the background when using an IDE), or
 * the files might not be in source control at all (consider indexing
   packages you downloaded from a package manager).

This provides a way for the Glass client to ask Glass for the original
file content that was indexed. The content might not be available - it
relies on the indexer having produced the appropriate `src.FileContent`
facts, and only the Haskell indexer does this right now, but it could
become more widely supported in the future.

We could also have Glass try to get the content from the SCM; I haven't
implemented that.

Author: simonmar

glean/glass/Glean/Glass/Handler/Documents.hs

@@ -89,6 +89,7 @@ import ServiceData.GlobalStats as Stats
 import ServiceData.Types as Stats
 import Util.Time (elapsedTime, toDiffMillis)
 import Glean.Glass.Attributes.PerfUtils (parseGitDiff, applyLineMappingToDefs, applyLineMappingToRefs)
+import qualified Glean.Glass.Query as Query
 
 -- | Runner for methods that are keyed by a file path.
 -- Select the right Glean DBs and pass them to the function (via a GleanBackend)
@@ -199,7 +200,7 @@ translateMirroredRepoListXResult
   -> DocumentSymbolListXResult
   -> DocumentSymbolListXResult
 translateMirroredRepoListXResult
-  (DocumentSymbolsRequest repository path _ _ _) res =
+  (DocumentSymbolsRequest repository path _ _ _ _) res =
   case repoPathToMirror repository path of
     Just (Mirror mirror prefix origin) ->
       Utils.translateDocumentSymbolListXResult origin mirror prefix Nothing res
@@ -226,6 +227,7 @@ fetchSymbolsAndAttributesGlean
       (requestOptions_revision opts)
       (requestOptions_exact_revision opts)
       withContentHash
+      fetchContent
       ExtraSymbolOpts{..} be mlang dbInfo (requestOptions_attribute_opts opts)
 
   res1 <- if oAmendLinesOnRevisionMismatch
@@ -255,6 +257,7 @@ fetchSymbolsAndAttributesGlean
     oIncludeRefs = documentSymbolsRequest_include_refs req
     oLineRange = documentSymbolsRequest_range req
     withContentHash = shouldFetchContentHash opts
+    fetchContent = documentSymbolsRequest_include_content req
     oIncludeXlangRefs = documentSymbolsRequest_include_xlang_refs req
     oAmendLinesOnRevisionMismatch = maybe False
       featureFlags_amend_lines_on_revision_mismatch (requestOptions_feature_flags opts)
@@ -465,6 +468,7 @@ fromDocumentSymbolResult DocumentSymbolListXResult{..} = DocumentSymbols
   , srcFile = Nothing
   , offsets = Nothing
   , attributes = documentSymbolListXResult_attributes
+  , content = documentSymbolListXResult_content
   }
   where
     createDummyRefEntity :: ReferenceRangeSymbolX ->
@@ -591,15 +595,16 @@ fetchDocumentSymbols
   -> Maybe Revision
   -> Bool  -- ^ exact revision only?
   -> Bool  -- ^ fetch file content hash?
+  -> Bool  -- ^ fetch file content
   -> ExtraSymbolOpts
   -> GleanBackend b
   -> Maybe Language
   -> GleanDBInfo
   -> AttributeOptions
   -> IO (DocumentSymbols, QueryEachRepoLog, Maybe ErrorLogger)
 fetchDocumentSymbols env@Glass.Env{..} (FileReference scsrepo path)
-    repoPath mlimit wantedRevision exactRevision fetchContentHash extraOpts
-    b mlang dbInfo attrOpts = do
+    repoPath mlimit wantedRevision exactRevision fetchContentHash fetchContent
+    extraOpts b mlang dbInfo attrOpts = do
   backendRunHaxl b env $ do
     --
     -- we pick the first db in the list that has the full FileInfo{..}
@@ -644,6 +649,12 @@ fetchDocumentSymbols env@Glass.Env{..} (FileReference scsrepo path)
         (kinds, _, merr) <- withRepo fileRepo $
           documentSymbolKinds mlimit mlang fileId attrOpts
 
+        content <-
+          if fetchContent then
+            withRepo fileRepo $ Utils.fetchData $ Query.fileContent fileId
+          else
+            return Nothing
+
         let revision = getDBRevision (scmRevisions dbInfo) fileRepo scsrepo
         contentMatch <- case wantedRevision of
           Nothing -> return Nothing
@@ -796,12 +807,13 @@ data DocumentSymbols = DocumentSymbols
   , srcFile :: Maybe Src.File
   , offsets :: Maybe Range.LineOffsets
   , attributes :: Maybe AttributeList
+  , content :: Maybe Text
   }
 
 emptyDocumentSymbols :: Revision -> DocumentSymbols
 emptyDocumentSymbols revision =
   DocumentSymbols [] [] revision Nothing False Nothing mempty mempty Nothing
-    Nothing Nothing
+    Nothing Nothing Nothing
 
 -- | Drop any remnant entities after we are done with them
 toDocumentSymbolResult :: DocumentSymbols -> DocumentSymbolListXResult
@@ -815,6 +827,7 @@ toDocumentSymbolResult DocumentSymbols{..} = DocumentSymbolListXResult{..}
     documentSymbolListXResult_referenced_file_digests = xref_digests
     documentSymbolListXResult_content_match = contentMatch
     documentSymbolListXResult_attributes = attributes
+    documentSymbolListXResult_content = content
 
 
 
@@ -854,7 +867,8 @@ fetchDocumentSymbolIndex env latest req opts be
           documentSymbolListXResult_referenced_file_digests,
         documentSymbolIndex_content_match =
           documentSymbolListXResult_content_match,
-        documentSymbolIndex_attributes = documentSymbolListXResult_attributes
+        documentSymbolIndex_attributes = documentSymbolListXResult_attributes,
+        documentSymbolIndex_content = documentSymbolListXResult_content
       }
   return ((idxResult, status, gleanDataLog, attrLog), merr1)
 

glean/glass/Glean/Glass/Query.hs

@@ -14,6 +14,7 @@ module Glean.Glass.Query
     srcFile
   , fileInfo
   , fileDigests
+  , fileContent
 
   -- * Working with XRefs
   , fileEntityLocations
@@ -75,6 +76,13 @@ fileInfo (GleanPath path) = predicate @Glass.FileInfo $
     field @"file" (string path)
   end
 
+fileContent :: Glean.IdOf Src.File -> Angle Text
+fileContent fileId =
+  var $ \content ->
+    content `where_` [
+      stmt $ predicate @Src.FileContent $ factId fileId .-> content
+    ]
+
 -- | Given a file id, look up the index of line endings
 --
 -- Line ending tables are needed to do line:col conversions, but we want to

glean/glass/if/glass.thrift

@@ -191,6 +191,12 @@ struct DocumentSymbolsRequest {
   // resolved using latest version of target db. This may add latency
   // to the query. includes_refs must be set to true.
   5: bool include_xlang_refs = false;
+
+  // Include indexed file content in the response, if available. This
+  // may be useful if there is no SCM available to retrieve the
+  // content from, or the file was not indexed from a specific SCM
+  // revision.
+  6: bool include_content = false;
 }
 
 // response types
@@ -327,6 +333,9 @@ struct DocumentSymbolListXResult {
   // additional metadata associated with the file, non-symbol specific
   // e.g. list of available attributes / denominators for the file
   8: optional AttributeList attributes;
+
+  // optional file contents, if include_content = true in the request
+  9: optional string content;
 }
 
 // For cursor navigation in a file, it is useful to have a line indexed
@@ -359,6 +368,9 @@ struct DocumentSymbolIndex {
   // additional metadata associated with the file, non-symbol specific
   // e.g. list of available attributes / denominators for the file
   8: optional AttributeList attributes;
+
+  // optional file contents, if include_content = true in the request
+  9: optional string content;
 }
 
 // Generic server exception

glean/glass/test/regression/Glean/Glass/Regression/Haskell.hs

@@ -8,11 +8,17 @@
 
 module Glean.Glass.Regression.Haskell (main) where
 
+import System.Environment
 import Glean.Indexer.Haskell as Haskell ( indexer )
 import Glean.Glass.Regression.Snapshot ( mainGlassSnapshot )
 
 main :: IO ()
-main = mainGlassSnapshot testName testPath testIndexer (const [])
+main = do
+  args <- getArgs
+  withArgs ("--arg=--store-src" : args) $
+    -- tells the indexer to produce src.FileContent facts, which support
+    -- include_content=true in documentSymbolIndex calls.
+    mainGlassSnapshot testName testPath testIndexer (const [])
   where
     testName = "glass-regression-haskell"
     testPath = "glean/glass/test/regression/tests/haskell"

glean/glass/test/regression/lib/Glean/Glass/Regression/Snapshot.hs

@@ -317,23 +317,25 @@ instance DeterministicResponse (Either [Text] Cxx.SymbolEnv) where
 
 instance DeterministicResponse DocumentSymbolListXResult where
   det (DocumentSymbolListXResult refs defs _rev truncated digest fileMap
-      contentMatch attributes) =
+      contentMatch attributes content) =
     DocumentSymbolListXResult (det refs) (det defs) (Revision "testhash")
       truncated
       digest
       fileMap
       -- n.b. don't want to include any test group revision tags
       contentMatch
       attributes
+      content
 
 instance DeterministicResponse DocumentSymbolIndex where
   det (DocumentSymbolIndex syms _rev size truncated digest fileMap
-      contentMatch attributes) =
+      contentMatch attributes content) =
     DocumentSymbolIndex (Map.map sort syms) (Revision "testhash") size truncated
       digest
       fileMap
       contentMatch
       attributes
+      content
 
 instance DeterministicResponse SymbolSearchResult where
   det (SymbolSearchResult syms deets) =

glean/glass/test/regression/lib/Glean/Glass/Regression/Tests.hs

@@ -50,6 +50,7 @@ testDocumentSymbolListX path get =
             , documentSymbolsRequest_range = Nothing
             , documentSymbolsRequest_include_refs = True
             , documentSymbolsRequest_include_xlang_refs = False
+            , documentSymbolsRequest_include_content = False
             }
       res <- documentSymbolListX env req def
       assertBool "documentSymbolListX"

glean/glass/test/regression/tests/haskell/documentSymbolIndex.out

@@ -1,6 +1,7 @@
 [
     "@generated",
     {
+        "content": "{-\n  Copyright (c) Meta Platforms, Inc. and affiliates.\n  All rights reserved.\n\n  This source code is licensed under the BSD-style license found in the\n  LICENSE file in the root directory of this source tree.\n-}\n\n\nmodule A\n  ( a\n  , A\n  , T(..)\n  , R(..)\n  , C(..)\n  , zero\n  , f\n  ) where\n\nimport Data.Char (toLower, ord)\n\na :: String\na = map toLower \"A\"\n\ntype A = Int\n\ndata T a = C1 Float | C2 A | C3 a\n\ndata R a = R { f1 :: Char, f2 :: [a] }\n\nclass Eq a => C a where\n  m :: a -> Bool\n\ninstance C Int where\n  m x = x == x\n\nzero :: A\nzero = 0\n\nf :: R a -> T a\nf R{f2 = [x]} = C3 x\nf r | m (3::Int) = C2 (ord (f1 r { f1 = 'a' }))\n",
         "referenced_file_digests": {},
         "revision": "testhash",
         "size": 59,

glean/glass/test/regression/tests/haskell/documentSymbolIndex.query

@@ -1,3 +1,3 @@
 action: documentSymbolIndex
-args: {"include_refs":true,"filepath":"glean/lang/codemarkup/tests/haskell/code/A.hs","repository":"test"}
+args: {"include_refs":true,"filepath":"glean/lang/codemarkup/tests/haskell/code/A.hs","repository":"test","include_content":true}