[glass] Optionally return full file content

glean/glass/Glean/Glass/Handler/Documents.hs

@@ -89,6 +89,7 @@ import ServiceData.GlobalStats as Stats
 import ServiceData.Types as Stats
 import Util.Time (elapsedTime, toDiffMillis)
 import Glean.Glass.Attributes.PerfUtils (parseGitDiff, applyLineMappingToDefs, applyLineMappingToRefs)
+import qualified Glean.Glass.Query as Query
 
 -- | Runner for methods that are keyed by a file path.
 -- Select the right Glean DBs and pass them to the function (via a GleanBackend)
@@ -199,7 +200,7 @@ translateMirroredRepoListXResult
   -> DocumentSymbolListXResult
   -> DocumentSymbolListXResult
 translateMirroredRepoListXResult
-  (DocumentSymbolsRequest repository path _ _ _) res =
+  (DocumentSymbolsRequest repository path _ _ _ _) res =
   case repoPathToMirror repository path of
     Just (Mirror mirror prefix origin) ->
       Utils.translateDocumentSymbolListXResult origin mirror prefix Nothing res
@@ -226,6 +227,7 @@ fetchSymbolsAndAttributesGlean
       (requestOptions_revision opts)
       (requestOptions_exact_revision opts)
       withContentHash
+      fetchContent
       ExtraSymbolOpts{..} be mlang dbInfo (requestOptions_attribute_opts opts)
 
   res1 <- if oAmendLinesOnRevisionMismatch
@@ -255,6 +257,7 @@ fetchSymbolsAndAttributesGlean
     oIncludeRefs = documentSymbolsRequest_include_refs req
     oLineRange = documentSymbolsRequest_range req
     withContentHash = shouldFetchContentHash opts
+    fetchContent = documentSymbolsRequest_include_content req
     oIncludeXlangRefs = documentSymbolsRequest_include_xlang_refs req
     oAmendLinesOnRevisionMismatch = maybe False
       featureFlags_amend_lines_on_revision_mismatch (requestOptions_feature_flags opts)
@@ -465,6 +468,7 @@ fromDocumentSymbolResult DocumentSymbolListXResult{..} = DocumentSymbols
   , srcFile = Nothing
   , offsets = Nothing
   , attributes = documentSymbolListXResult_attributes
+  , content = documentSymbolListXResult_content
   }
   where
     createDummyRefEntity :: ReferenceRangeSymbolX ->
@@ -591,15 +595,16 @@ fetchDocumentSymbols
   -> Maybe Revision
   -> Bool  -- ^ exact revision only?
   -> Bool  -- ^ fetch file content hash?
+  -> Bool  -- ^ fetch file content
   -> ExtraSymbolOpts
   -> GleanBackend b
   -> Maybe Language
   -> GleanDBInfo
   -> AttributeOptions
   -> IO (DocumentSymbols, QueryEachRepoLog, Maybe ErrorLogger)
 fetchDocumentSymbols env@Glass.Env{..} (FileReference scsrepo path)
-    repoPath mlimit wantedRevision exactRevision fetchContentHash extraOpts
-    b mlang dbInfo attrOpts = do
+    repoPath mlimit wantedRevision exactRevision fetchContentHash fetchContent
+    extraOpts b mlang dbInfo attrOpts = do
   backendRunHaxl b env $ do
     --
     -- we pick the first db in the list that has the full FileInfo{..}
@@ -644,6 +649,12 @@ fetchDocumentSymbols env@Glass.Env{..} (FileReference scsrepo path)
         (kinds, _, merr) <- withRepo fileRepo $
           documentSymbolKinds mlimit mlang fileId attrOpts
 
+        content <-
+          if fetchContent then
+            withRepo fileRepo $ Utils.fetchData $ Query.fileContent fileId
+          else
+            return Nothing
+
         let revision = getDBRevision (scmRevisions dbInfo) fileRepo scsrepo
         contentMatch <- case wantedRevision of
           Nothing -> return Nothing
@@ -796,12 +807,13 @@ data DocumentSymbols = DocumentSymbols
   , srcFile :: Maybe Src.File
   , offsets :: Maybe Range.LineOffsets
   , attributes :: Maybe AttributeList
+  , content :: Maybe Text
   }
 
 emptyDocumentSymbols :: Revision -> DocumentSymbols
 emptyDocumentSymbols revision =
   DocumentSymbols [] [] revision Nothing False Nothing mempty mempty Nothing
-    Nothing Nothing
+    Nothing Nothing Nothing
 
 -- | Drop any remnant entities after we are done with them
 toDocumentSymbolResult :: DocumentSymbols -> DocumentSymbolListXResult
@@ -815,6 +827,7 @@ toDocumentSymbolResult DocumentSymbols{..} = DocumentSymbolListXResult{..}
     documentSymbolListXResult_referenced_file_digests = xref_digests
     documentSymbolListXResult_content_match = contentMatch
     documentSymbolListXResult_attributes = attributes
+    documentSymbolListXResult_content = content
 
 
 
@@ -854,7 +867,8 @@ fetchDocumentSymbolIndex env latest req opts be
           documentSymbolListXResult_referenced_file_digests,
         documentSymbolIndex_content_match =
           documentSymbolListXResult_content_match,
-        documentSymbolIndex_attributes = documentSymbolListXResult_attributes
+        documentSymbolIndex_attributes = documentSymbolListXResult_attributes,
+        documentSymbolIndex_content = documentSymbolListXResult_content
       }
   return ((idxResult, status, gleanDataLog, attrLog), merr1)
 

glean/glass/Glean/Glass/Query.hs

@@ -14,6 +14,7 @@ module Glean.Glass.Query
     srcFile
   , fileInfo
   , fileDigests
+  , fileContent
 
   -- * Working with XRefs
   , fileEntityLocations
@@ -75,6 +76,13 @@ fileInfo (GleanPath path) = predicate @Glass.FileInfo $
     field @"file" (string path)
   end
 
+fileContent :: Glean.IdOf Src.File -> Angle Text
+fileContent fileId =
+  var $ \content ->
+    content `where_` [
+      stmt $ predicate @Src.FileContent $ factId fileId .-> content
+    ]
+
 -- | Given a file id, look up the index of line endings
 --
 -- Line ending tables are needed to do line:col conversions, but we want to

glean/glass/if/glass.thrift

@@ -191,6 +191,12 @@ struct DocumentSymbolsRequest {
   // resolved using latest version of target db. This may add latency
   // to the query. includes_refs must be set to true.
   5: bool include_xlang_refs = false;
+
+  // Include indexed file content in the response, if available. This
+  // may be useful if there is no SCM available to retrieve the
+  // content from, or the file was not indexed from a specific SCM
+  // revision.
+  6: bool include_content = false;
 }
 
 // response types
@@ -327,6 +333,9 @@ struct DocumentSymbolListXResult {
   // additional metadata associated with the file, non-symbol specific
   // e.g. list of available attributes / denominators for the file
   8: optional AttributeList attributes;
+
+  // optional file contents, if include_content = true in the request
+  9: optional string content;
 }
 
 // For cursor navigation in a file, it is useful to have a line indexed
@@ -359,6 +368,9 @@ struct DocumentSymbolIndex {
   // additional metadata associated with the file, non-symbol specific
   // e.g. list of available attributes / denominators for the file
   8: optional AttributeList attributes;
+
+  // optional file contents, if include_content = true in the request
+  9: optional string content;
 }
 
 // Generic server exception

glean/glass/test/regression/Glean/Glass/Regression/Haskell.hs

@@ -8,11 +8,17 @@
 
 module Glean.Glass.Regression.Haskell (main) where
 
+import System.Environment
 import Glean.Indexer.Haskell as Haskell ( indexer )
 import Glean.Glass.Regression.Snapshot ( mainGlassSnapshot )
 
 main :: IO ()
-main = mainGlassSnapshot testName testPath testIndexer (const [])
+main = do
+  args <- getArgs
+  withArgs ("--arg=--store-src" : args) $
+    -- tells the indexer to produce src.FileContent facts, which support
+    -- include_content=true in documentSymbolIndex calls.
+    mainGlassSnapshot testName testPath testIndexer (const [])
   where
     testName = "glass-regression-haskell"
     testPath = "glean/glass/test/regression/tests/haskell"

glean/glass/test/regression/lib/Glean/Glass/Regression/Snapshot.hs

@@ -317,23 +317,25 @@ instance DeterministicResponse (Either [Text] Cxx.SymbolEnv) where
 
 instance DeterministicResponse DocumentSymbolListXResult where
   det (DocumentSymbolListXResult refs defs _rev truncated digest fileMap
-      contentMatch attributes) =
+      contentMatch attributes content) =
     DocumentSymbolListXResult (det refs) (det defs) (Revision "testhash")
       truncated
       digest
       fileMap
       -- n.b. don't want to include any test group revision tags
       contentMatch
       attributes
+      content
 
 instance DeterministicResponse DocumentSymbolIndex where
   det (DocumentSymbolIndex syms _rev size truncated digest fileMap
-      contentMatch attributes) =
+      contentMatch attributes content) =
     DocumentSymbolIndex (Map.map sort syms) (Revision "testhash") size truncated
       digest
       fileMap
       contentMatch
       attributes
+      content
 
 instance DeterministicResponse SymbolSearchResult where
   det (SymbolSearchResult syms deets) =

glean/glass/test/regression/lib/Glean/Glass/Regression/Tests.hs

@@ -50,6 +50,7 @@ testDocumentSymbolListX path get =
             , documentSymbolsRequest_range = Nothing
             , documentSymbolsRequest_include_refs = True
             , documentSymbolsRequest_include_xlang_refs = False
+            , documentSymbolsRequest_include_content = False
             }
       res <- documentSymbolListX env req def
       assertBool "documentSymbolListX"

glean/glass/test/regression/tests/haskell/documentSymbolIndex.out

@@ -1,6 +1,7 @@
 [
     "@generated",
     {
+        "content": "{-\n  Copyright (c) Meta Platforms, Inc. and affiliates.\n  All rights reserved.\n\n  This source code is licensed under the BSD-style license found in the\n  LICENSE file in the root directory of this source tree.\n-}\n\n\nmodule A\n  ( a\n  , A\n  , T(..)\n  , R(..)\n  , C(..)\n  , zero\n  , f\n  ) where\n\nimport Data.Char (toLower, ord)\n\na :: String\na = map toLower \"A\"\n\ntype A = Int\n\ndata T a = C1 Float | C2 A | C3 a\n\ndata R a = R { f1 :: Char, f2 :: [a] }\n\nclass Eq a => C a where\n  m :: a -> Bool\n\ninstance C Int where\n  m x = x == x\n\nzero :: A\nzero = 0\n\nf :: R a -> T a\nf R{f2 = [x]} = C3 x\nf r | m (3::Int) = C2 (ord (f1 r { f1 = 'a' }))\n",
         "referenced_file_digests": {},
         "revision": "testhash",
         "size": 59,

glean/glass/test/regression/tests/haskell/documentSymbolIndex.query

@@ -1,3 +1,3 @@
 action: documentSymbolIndex
-args: {"include_refs":true,"filepath":"glean/lang/codemarkup/tests/haskell/code/A.hs","repository":"test"}
+args: {"include_refs":true,"filepath":"glean/lang/codemarkup/tests/haskell/code/A.hs","repository":"test","include_content":true}