[glass] Add an "auto" repo mapping

When using AutoRepoMapping, repo names are assumed to correspond 1:1
with Glean DB names. This is convenient for a pre-built Glass binary
because we don't know the repos and DBs at build-time.

Author: simonmar

glean/glass/Glean/Glass/Base.hs

@@ -81,22 +81,32 @@ data GleanDBSelector = GleanDBSelector
   -- ^ Nothing means the db is default, can be used for any branch
   }
 
-data RepoMapping = RepoMapping
-  { gleanIndices :: Map.Map Glass.RepoName [GleanDBSelector]
-    -- ^ This should be in a config
+data RepoMapping
+  = AutoRepoMapping
+    -- ^ Repo name == Glean DB name. e.g. if we get a documentSymbols
+    -- request for myrepo/lang/My/File then we query the latest
+    -- "myrepo" Glean DB.
     --
-    -- This is the set of Glean dbs that should implement codemarkup.*
-    --
-    -- Note: the order here determines the order of search/lookup
-    -- if you have overlapping dbs , for file contents, the first in the order
-    -- with a src.File fact will win.
-    --
-    -- If you add/remove a db, consider if it needs to be present in
-    -- gleanRequiredIndices as well
+    -- In this case 'RepoMapping.allGleanRepos' can be 'Nothing' to
+    -- enable use of any Glean DB, or it can be set to a list to
+    -- restrict Glass to using particular Glean DBs.
 
-  , gleanAttrIndices :: Map.Map Glass.RepoName [GleanDBAttrName]
-    -- ^ Map of source db to attr db names & attribute key types
-    --
-    -- This pairs attribute Glean dbs with a key type to index the ToAttribute
-    -- class, that in turns knowns how to query and marshal the attributes
-  }
+  | RepoMapping
+    { gleanIndices :: Map.Map Glass.RepoName [GleanDBSelector]
+      -- ^ This should be in a config
+      --
+      -- This is the set of Glean dbs that should implement codemarkup.*
+      --
+      -- Note: the order here determines the order of search/lookup
+      -- if you have overlapping dbs , for file contents, the first in the order
+      -- with a src.File fact will win.
+      --
+      -- If you add/remove a db, consider if it needs to be present in
+      -- gleanRequiredIndices as well
+  
+    , gleanAttrIndices :: Map.Map Glass.RepoName [GleanDBAttrName]
+      -- ^ Map of source db to attr db names & attribute key types
+      --
+      -- This pairs attribute Glean dbs with a key type to index the ToAttribute
+      -- class, that in turns knowns how to query and marshal the attributes
+    }

glean/glass/Glean/Glass/RepoMapping.hs

@@ -18,71 +18,25 @@ module Glean.Glass.RepoMapping
 
 import Data.Set (Set)
 import qualified Data.Set as Set
-import qualified Data.Map.Strict as Map
+import Data.Text(Text)
 
 import Glean.Glass.Base
   ( GleanDBName(..)
   , RepoMapping(..)
-  , GleanDBSelector(..)
   )
-import Glean.Glass.Types ( Language(..), RepoName(..) )
-import Data.Text(Text)
+import Glean.Glass.Types ( RepoName(..) )
 
 getRepoMapping :: IO RepoMapping
-getRepoMapping = return RepoMapping
-  { gleanIndices = gleanIndices_
-  , gleanAttrIndices = Map.empty
-  }
+getRepoMapping = return AutoRepoMapping
 
 fixedRepoMapping :: RepoMapping
-fixedRepoMapping = RepoMapping
-  { gleanIndices = gleanIndices_
-  , gleanAttrIndices = Map.empty
-  }
-
--- example: the open source react repo.
-gleanIndices_ :: Map.Map RepoName [GleanDBSelector]
-gleanIndices_ = Map.fromList
-  -- demo
-  [ ( RepoName "react",
-      [ GleanDBSelector
-        { dbName = "react"
-        , language = Language_JavaScript
-        , branchName = Nothing
-        }
-      ]
-    )
-  -- for running tests with locally-indexed repos:
-  , ( RepoName "test",
-      [ testSelector Language_JavaScript
-      , testSelector Language_Hack
-      , testSelector Language_Haskell
-      , testSelector Language_Cpp
-      , testSelector Language_PreProcessor
-      , testSelector Language_Python
-      , testSelector Language_Thrift
-      , testSelector Language_Buck
-      , testSelector Language_Go
-      , testSelector Language_TypeScript
-      , testSelector Language_Rust
-      , testSelector Language_Java
-      , testSelector Language_Swift
-      ]
-    )
-  ]
-  where
-    testSelector language =
-      GleanDBSelector
-        { dbName = "test"
-        , language = language
-        , branchName = Nothing
-        }
+fixedRepoMapping = AutoRepoMapping
 
 -- | All the Glean db repo names we're aware of
--- We will only be able to query members of this set
-allGleanRepos :: Set GleanDBName
-allGleanRepos = Set.fromList $
-  map dbName (concat (Map.elems gleanIndices_))
+-- We will only be able to query members of this set.
+-- 'Nothing' means all existing Glean DBs can be used.
+allGleanRepos :: Maybe (Set GleanDBName)
+allGleanRepos = Nothing
 
 -- repos that are required
 gleanRequiredIndices :: Set.Set GleanDBName

glean/glass/Glean/Glass/Repos.hs

@@ -36,7 +36,6 @@ module Glean.Glass.Repos
   , getRepoHash
   , getRepoHashForLocation
   , getDBRevision
-  , getLatestRepos
   ) where
 
 import Control.Concurrent.Stream
@@ -104,6 +103,7 @@ data GleanDBInfo = GleanDBInfo
 
 -- return a RepoName if indexed by glean
 toRepoName :: RepoMapping -> Text -> Maybe RepoName
+toRepoName AutoRepoMapping repo = Just (RepoName repo)
 toRepoName RepoMapping{..} repo =
   case Map.lookup repoName gleanIndices of
     Just _ -> Just repoName
@@ -113,6 +113,7 @@ toRepoName RepoMapping{..} repo =
 
 -- | Additional metadata about files and methods in attribute dbs
 attrDBsForRepo :: RepoMapping -> RepoName -> [GleanDBAttrName]
+attrDBsForRepo AutoRepoMapping _ = []
 attrDBsForRepo RepoMapping{..} repo =
   Map.findWithDefault [] repo gleanAttrIndices
 
@@ -145,6 +146,12 @@ selectGleanDBs
   -> Maybe RepoName
   -> Set Language
   -> Either Text (Map.Map RepoName (Set GleanDBName))
+selectGleanDBs AutoRepoMapping (Just repoName) _langs =
+  Right (
+    Map.singleton repoName (
+      Set.singleton (GleanDBName (unRepoName repoName))))
+selectGleanDBs AutoRepoMapping Nothing _langs =
+  Right Map.empty
 selectGleanDBs repoMapping mRepoName langs0 =
   case map flatten (filter matches candidates) of
     [] -> Left err
@@ -182,6 +189,7 @@ normalizeLanguages l = l
 -- | Select universe of glean repo,(db/language) pairs.
 -- Either just the test dbs, or all the non-test dbs.
 listGleanIndices :: RepoMapping -> Bool -> [(RepoName, GleanDBSelector)]
+listGleanIndices AutoRepoMapping _ = [] -- handled earlier
 listGleanIndices RepoMapping{..} testsOnly =
   let testRepos = [RepoName "test", RepoName "test-xlang"]
       flatten (repo,langs) = map (repo,) langs
@@ -199,6 +207,8 @@ fromSCSRepo
   -> Maybe (Text -> IO Bool)
   -> Maybe Language
   -> IO [GleanDBName]
+fromSCSRepo AutoRepoMapping repo _branchFilter _mLanguage =
+  return [GleanDBName (unRepoName repo)]
 fromSCSRepo RepoMapping{..} repo branchFilter mLanguage
   | Just rs <- Map.lookup repo gleanIndices = do
     let filteredByLang = filterByLanguage mLanguage rs
@@ -279,7 +289,7 @@ fileLanguage (Path file)
     is a =  a `Text.isSuffixOf` file
 
 --
--- Operating on the latest repo statea
+-- Operating on the latest repo state
 --
 
 -- | Fetch all latest dbs we care for
@@ -295,7 +305,7 @@ getLatestRepos backend scm mlogger mretry = go mretry
     go :: Maybe Int -> IO GleanDBInfo
     go n = do
       latest <- Glean.getLatestRepos backend $ \name ->
-        GleanDBName name `Set.member` Mapping.allGleanRepos
+        maybe True (GleanDBName name `Set.member`) Mapping.allGleanRepos
 
       -- Filter unavailable DBs using hasDatabase. Glean.getLatestRepos does
       -- this for latestRepos but not allLatestRepos, in that case it's up to
@@ -317,21 +327,16 @@ getLatestRepos backend scm mlogger mretry = go mretry
             , scmRevisions = scmRevisions
             , dbByRevision = getDbByRevision dbs scmRevisions
             }
-      if required `Set.isSubsetOf` advertised
-        then return info
-        else do
+          required = Set.map unGleanDBName Mapping.gleanRequiredIndices
+          missing = required `Set.difference` advertised
+      if
+        | not (Set.null missing), Just m <- n, m > 1 -> do
           -- some required dbs are missing! this is transient? and bad
           -- in prod/full service mode this would be bad
-          let missing = required `Set.difference` advertised
+          -- if no retries allowed, give up
           logIt mlogger dbs missing n backend
-          case n of
-            Just n
-              | n > 1 -> do {- i.e. try more than 1 time -}
-                delay (seconds 1) >> go (Just (n-1))
-
-            _ -> return info -- if no retries allowed, give up
-
-    required = Set.map unGleanDBName Mapping.gleanRequiredIndices
+          delay (seconds 1) >> go (Just (m-1))
+        | otherwise -> return info
 
 -- | Log an entry in glean_glass_server_error_events if a logger is available,
 -- and locally (e.g. to stderr). Do not log otherwise (e.g. in test mode).