diff --git a/ChangeLog.md b/ChangeLog.md index be4678ed83..399bcd12d2 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -19,6 +19,9 @@ Behavior changes: `ls dependencies` command, so that the nodes of `stack dot --external --depth 0` are the same as the packages listed by `stack ls dependencies --depth 0`. +* When building GHC from source, on Windows, the default Hadrian build target is + `reloc-binary-dist` and the default path to the GHC built by Hadrian is + `_build/reloc-bindist`. Other enhancements: @@ -33,6 +36,10 @@ Other enhancements: specified). * Add option `-w` as synonym for `--stack-yaml`. * `stack new` now allows `codeberg:` as a service for template downloads +* In YAML configuration files, the `compiler-target` and + `compiler-bindist-path` keys are introduced to allow, when building GHC from + source, the Hadrian build target and Hadrian path to the built GHC to be + specified. Bug fixes: diff --git a/doc/topics/GHC_from_source.md b/doc/topics/GHC_from_source.md index 383c0cdd4e..fa55346b13 100644 --- a/doc/topics/GHC_from_source.md +++ b/doc/topics/GHC_from_source.md @@ -23,6 +23,10 @@ In the following example the commit ID is "5be7ad..." and the flavour is compiler: ghc-git-5be7ad7861c8d39f60b7101fd8d8e816ff50353a-quick ~~~ +The [`-j`, `--jobs` option](../configure/global_flags.md#-jobs-or-j-option) at +the command line or the [`jobs`](../configure/yaml/non-project.md#jobs) option +in a YAML configuraton file can be used to specify Hadrian's `-j[]` flag. + By default, the code is retrieved from the main GHC repository. If you want to select another repository, use the `compiler-repository` option in a YAML configuration file: @@ -33,6 +37,40 @@ compiler-repository: git://my/ghc/repository # compiler-repository: https://gitlab.haskell.org/ghc/ghc.git ~~~ +By default, the Hadrian build target is `reloc-binary-dist` on Windows and +`binary-dist` on other operating systems. If you want to specify another +Hadrian build target, use the `compiler-target` option in a YAML configuration +file: + +~~~yaml +compiler-target: binary-dist +# default (Windows) +# compiler-target: reloc-binary-dist +# default (non-Windows) +# compiler-target: binary-dist +~~~ + +By default, Stack assumes that the path to the binary distribution built by +Hadrian is `_build/reloc-bindist` on Windows and `_build/bindist` on other +operating systems. If you want to specify another path, use the +`compiler-bindist-path` option in a YAML configuration file: + +~~~yaml +compiler-bindist-path: _build/bindist +# default (Windows) +# compiler-bindist-path: _build/reloc-bindist +# default (non-Windows) +# compiler-bindist-path: _build/bindist +~~~ + +!!! note + + The Hadrian build target `reloc-binary-dist` was introduced with Git commit + id + [`fe23629b147d419053052e6e881f6e8ddfbf3bae`](https://gitlab.haskell.org/ghc/ghc/-/commit/fe23629b147d419053052e6e881f6e8ddfbf3bae). + + Once introduced, the target must be used on Windows. + Stack does not check the compiler version when it uses a compiler built from source. It is assumed that the built compiler is recent enough as Stack does not enable any known workaround to make older compilers work. @@ -65,13 +103,17 @@ fully managed by Stack. configure: error: GHC version 9.2 or later is required to compile GHC. ~~~ - The resolution is: (1) to specify an alternative snapshot (one that - specifies a sufficiently recent version of GHC) on the command line, using - Stack's option `--snapshot `. Stack will use that snapshot when - running GHC's `configure` script; and (2) to set the contents of the `STACK` - environment variable to be `stack --snapshot `. Hadrian's - `build-stack` script wil refer to that environment variable for the Stack - command it uses. + The resolution is: + + 1. to specify an alternative snapshot (one that specifies a sufficiently + recent version of GHC) on the command line, using Stack's option + `--snapshot `. Stack will use that snapshot when running GHC's + `configure` script; and + + 2. to set the contents of the `STACK` environment variable to be + `stack --snapshot `. If `` is a path to a local YAML + file, it needs to be an absolute one. Hadrian's `build-stack` script + will refer to that environment variable for the Stack command it uses. ### Hadrian prerequisites @@ -127,8 +169,10 @@ Stack will build and install `happy` and `alex`, if not already on the PATH. # documentation from a single source file, including `makeinfo`. stack exec -- pacman --sync mingw-w64-x86_64-ca-certificates # Common CA (certificate authority) certificates. - stack exec -- pip install -U sphinx + stack exec -- pacman -sync mingw-w64-x86_64-python-sphinx # Sphinx is the Python documentation generator. + stack exec -- pacman -sync mingw-w64-x86_64-texlive-full + # The TeX Live distribution. ~~~ Hadrian may require certain LaTeX packages and may prompt for these to be diff --git a/src/Stack/Config.hs b/src/Stack/Config.hs index b79c2c8b8a..0def9f8ca2 100644 --- a/src/Stack/Config.hs +++ b/src/Stack/Config.hs @@ -113,7 +113,10 @@ import Stack.Types.Build.Exception import Stack.Types.BuildConfig ( BuildConfig (..) ) import Stack.Types.BuildOpts ( BuildOpts (..) ) import Stack.Types.ColorWhen ( ColorWhen (..) ) -import Stack.Types.Compiler ( defaultCompilerRepository ) +import Stack.Types.Compiler + ( defaultCompilerBindistPath, defaultCompilerRepository + , defaultCompilerTarget + ) import Stack.Types.Config ( Config (..), HasConfig (..), askLatestSnapshotUrl , configProjectRoot, stackRootL, workDirL @@ -286,6 +289,12 @@ configFromConfigMonoid compilerRepository = fromFirst defaultCompilerRepository configMonoid.compilerRepository + compilerTarget = fromFirst + defaultCompilerTarget + configMonoid.compilerTarget + compilerBindistPath = fromFirst + defaultCompilerBindistPath + configMonoid.compilerBindistPath ghcBuild = getFirst configMonoid.ghcBuild installGHC = fromFirstTrue configMonoid.installGHC installMsys = fromFirst installGHC configMonoid.installMsys @@ -595,6 +604,8 @@ configFromConfigMonoid , msysEnvironment , compilerCheck , compilerRepository + , compilerTarget + , compilerBindistPath , localBin , fileWatchHook , requireStackVersion diff --git a/src/Stack/Setup.hs b/src/Stack/Setup.hs index b72be2c1f1..64f8c219be 100644 --- a/src/Stack/Setup.hs +++ b/src/Stack/Setup.hs @@ -131,8 +131,9 @@ import Stack.Types.BuildConfig ) import Stack.Types.BuildOptsCLI ( BuildOptsCLI (..) ) import Stack.Types.Compiler - ( ActualCompiler (..), CompilerException (..) - , CompilerRepository (..), WhichCompiler (..) + ( ActualCompiler (..), CompilerBindistPath (..) + , CompilerException (..), CompilerRepository (..) + , CompilerTarget (..), WhichCompiler (..) , compilerVersionText, getGhcVersion, isWantedCompiler , wantedToActual, whichCompiler, whichCompilerL ) @@ -1368,6 +1369,8 @@ ensureSandboxedCompiler sopts getSetupInfo' = do getSetupInfo' installed config.compilerRepository + config.compilerTarget + config.compilerBindistPath commitId flavour _ -> installGhcBindist sopts getSetupInfo' installed @@ -1569,118 +1572,130 @@ buildGhcFromSource :: => Memoized SetupInfo -> [Tool] -> CompilerRepository + -> CompilerTarget + -- ^ The Hadrian build target. + -> CompilerBindistPath + -- ^ The Hadrian path to the built binary distribution. -> Text -- ^ Commit ID. -> Text -- ^ Hadrain flavour. -> RIO (WithMSYS env) (Tool, CompilerBuild) -buildGhcFromSource getSetupInfo' installed (CompilerRepository url) commitId flavour = do - config <- view configL - let compilerTool = ToolGhcGit commitId flavour - -- detect when the correct GHC is already installed - if compilerTool `elem` installed - then pure (compilerTool, CompilerBuildStandard) - else - -- clone the repository and execute the given commands - withRepo (SimpleRepo url commitId RepoGit) $ do - -- withRepo is guaranteed to set workingDirL, so let's get it - mcwd <- traverse parseAbsDir =<< view workingDirL - cwd <- maybe (throwIO WorkingDirectoryInvalidBug) pure mcwd - let threads = config.jobs - relFileHadrianStackDotYaml' = toFilePath relFileHadrianStackDotYaml - ghcBootScriptPath = cwd ghcBootScript - boot = if osIsWindows - then proc "python3" ["boot"] runProcess_ - else - proc (toFilePath ghcBootScriptPath) [] runProcess_ - stack args = proc "stack" args'' runProcess_ - where - args'' = "--stack-yaml=" <> relFileHadrianStackDotYaml' : args' - -- If a snapshot is specified on the command line, Stack will - -- apply it. This allows the snapshot specified in Hadrian's - -- stack.yaml file to be overridden. - args' = maybe args addSnapshot config.snapshot - addSnapshot snapshot = "--snapshot=" <> show snapshot : args - happy = stack ["install", "happy"] - alex = stack ["install", "alex"] - -- Executed in the Stack environment, because GHC is required. - configure = stack ("exec" : "--" : ghcConfigure) - ghcConfigure - | osIsWindows = ghcConfigureWindows - | osIsMacOS = ghcConfigureMacOS - | otherwise = ghcConfigurePosix - hadrianScripts - | osIsWindows = hadrianScriptsWindows - | otherwise = hadrianScriptsPosix - hadrianArgs = fmap T.unpack - [ "-j" <> tshow threads -- parallel build - , "--flavour=" <> flavour -- selected flavour - , "binary-dist" - ] - foundHadrianPaths <- - filterM doesFileExist $ (cwd ) <$> hadrianScripts - hadrianPath <- maybe (prettyThrowIO HadrianScriptNotFound) pure $ - listToMaybe foundHadrianPaths - exists <- doesFileExist ghcBootScriptPath - unless exists $ prettyThrowIO GhcBootScriptNotFound - ensureConfigureScript cwd - logInfo "Running GHC boot script..." - boot - doesExecutableExist "happy" >>= \case - True -> logInfo "happy executable installed on the PATH." - False -> do - logInfo "Installing happy executable..." - happy - doesExecutableExist "alex" >>= \case - True -> logInfo "alex executable installed on the PATH." - False -> do - logInfo "Installing alex executable..." - alex - logInfo "Running GHC configure script..." - configure - logSticky $ - "Building GHC from source with `" - <> display flavour - <> "` flavour. It can take a long time (more than one hour)..." - -- We need to provide an absolute path to the script since the process - -- package only sets working directory _after_ discovering the - -- executable. - proc (toFilePath hadrianPath) hadrianArgs runProcess_ - - -- find the bindist and install it - bindistPath <- parseRelDir "_build/bindist" - (_,files) <- listDir (cwd bindistPath) - let isBindist p = do - extension <- fileExtension (filename p) - - pure $ - "ghc-" `isPrefixOf` toFilePath (filename p) - && extension == ".xz" - - filterM isBindist files >>= \case - [bindist] -> do - let bindist' = T.pack (toFilePath bindist) - dlinfo = DownloadInfo - { url = bindist' - -- we can specify a filepath instead of a URL - , contentLength = Nothing - , sha1 = Nothing - , sha256 = Nothing - } - ghcdlinfo = GHCDownloadInfo mempty mempty dlinfo - installer - | osIsWindows = installGHCWindows - | otherwise = installGHCPosix ghcdlinfo - si <- runMemoized getSetupInfo' - _ <- downloadAndInstallTool - config.localPrograms - dlinfo - compilerTool - (installer si) - pure (compilerTool, CompilerBuildStandard) - _ -> do - forM_ files (logDebug . fromString . (" - " ++) . toFilePath) - prettyThrowIO HadrianBindistNotFound +buildGhcFromSource + getSetupInfo' + installed + (CompilerRepository url) + (CompilerTarget hadrianBuildTarget) + (CompilerBindistPath hadrianBindistPath) + commitId + flavour + = do + config <- view configL + let compilerTool = ToolGhcGit commitId flavour + -- detect when the correct GHC is already installed + if compilerTool `elem` installed + then pure (compilerTool, CompilerBuildStandard) + else + -- clone the repository and execute the given commands + withRepo (SimpleRepo url commitId RepoGit) $ do + -- withRepo is guaranteed to set workingDirL, so let's get it + mcwd <- traverse parseAbsDir =<< view workingDirL + cwd <- maybe (throwIO WorkingDirectoryInvalidBug) pure mcwd + let threads = config.jobs + relFileHadrianStackDotYaml' = toFilePath relFileHadrianStackDotYaml + ghcBootScriptPath = cwd ghcBootScript + boot = if osIsWindows + then proc "python3" ["boot"] runProcess_ + else + proc (toFilePath ghcBootScriptPath) [] runProcess_ + stack args = proc "stack" args'' runProcess_ + where + args'' = "--stack-yaml=" <> relFileHadrianStackDotYaml' : args' + -- If a snapshot is specified on the command line, Stack will + -- apply it. This allows the snapshot specified in Hadrian's + -- stack.yaml file to be overridden. + args' = maybe args addSnapshot config.snapshot + addSnapshot snapshot = "--snapshot=" <> show snapshot : args + happy = stack ["install", "happy"] + alex = stack ["install", "alex"] + -- Executed in the Stack environment, because GHC is required. + configure = stack ("exec" : "--" : ghcConfigure) + ghcConfigure + | osIsWindows = ghcConfigureWindows + | osIsMacOS = ghcConfigureMacOS + | otherwise = ghcConfigurePosix + hadrianScripts + | osIsWindows = hadrianScriptsWindows + | otherwise = hadrianScriptsPosix + hadrianArgs = fmap T.unpack + [ "-j" <> tshow threads -- parallel build + , "--flavour=" <> flavour -- selected flavour + , hadrianBuildTarget + ] + foundHadrianPaths <- + filterM doesFileExist $ (cwd ) <$> hadrianScripts + hadrianPath <- maybe (prettyThrowIO HadrianScriptNotFound) pure $ + listToMaybe foundHadrianPaths + exists <- doesFileExist ghcBootScriptPath + unless exists $ prettyThrowIO GhcBootScriptNotFound + ensureConfigureScript cwd + logInfo "Running GHC boot script..." + boot + doesExecutableExist "happy" >>= \case + True -> logInfo "happy executable installed on the PATH." + False -> do + logInfo "Installing happy executable..." + happy + doesExecutableExist "alex" >>= \case + True -> logInfo "alex executable installed on the PATH." + False -> do + logInfo "Installing alex executable..." + alex + logInfo "Running GHC configure script..." + configure + logSticky $ + "Building GHC from source with `" + <> display flavour + <> "` flavour. It can take a long time (more than one hour)..." + -- We need to provide an absolute path to the script since the process + -- package only sets working directory _after_ discovering the + -- executable. + proc (toFilePath hadrianPath) hadrianArgs runProcess_ + + -- find the bindist and install it + bindistPath <- parseRelDir (T.unpack hadrianBindistPath) + (_,files) <- listDir (cwd bindistPath) + let isBindist p = do + extension <- fileExtension (filename p) + + pure $ + "ghc-" `isPrefixOf` toFilePath (filename p) + && extension == ".xz" + + filterM isBindist files >>= \case + [bindist] -> do + let bindist' = T.pack (toFilePath bindist) + dlinfo = DownloadInfo + { url = bindist' + -- we can specify a filepath instead of a URL + , contentLength = Nothing + , sha1 = Nothing + , sha256 = Nothing + } + ghcdlinfo = GHCDownloadInfo mempty mempty dlinfo + installer + | osIsWindows = installGHCWindows + | otherwise = installGHCPosix ghcdlinfo + si <- runMemoized getSetupInfo' + _ <- downloadAndInstallTool + config.localPrograms + dlinfo + compilerTool + (installer si) + pure (compilerTool, CompilerBuildStandard) + _ -> do + forM_ files (logDebug . fromString . (" - " ++) . toFilePath) + prettyThrowIO HadrianBindistNotFound -- | Determine which GHC builds to use depending on which shared libraries are -- available on the system. diff --git a/src/Stack/Types/Compiler.hs b/src/Stack/Types/Compiler.hs index 1237f93830..33a2c10faf 100644 --- a/src/Stack/Types/Compiler.hs +++ b/src/Stack/Types/Compiler.hs @@ -11,8 +11,12 @@ module Stack.Types.Compiler ( ActualCompiler (..) , WhichCompiler (..) , CompilerRepository (..) + , CompilerTarget (..) + , CompilerBindistPath (..) , CompilerException (..) , defaultCompilerRepository + , defaultCompilerTarget + , defaultCompilerBindistPath , getGhcVersion , whichCompiler , compilerVersionText @@ -31,9 +35,10 @@ import Data.Aeson import Database.Persist.Sql ( PersistField (..), PersistFieldSql (..), SqlType (..) ) import qualified Data.Text as T +import Distribution.Version ( mkVersion ) import Stack.Prelude import Stack.Types.Version ( VersionCheck, checkVersion ) -import Distribution.Version ( mkVersion ) +import System.Permissions ( osIsWindows ) -- | Type representing exceptions thrown by functions exported by the -- "Stack.Types.Compiler" module. @@ -147,5 +152,31 @@ defaultCompilerRepository :: CompilerRepository defaultCompilerRepository = CompilerRepository "https://gitlab.haskell.org/ghc/ghc.git" +-- | Target for Hadrian build +newtype CompilerTarget + = CompilerTarget Text + deriving Show + +instance FromJSON CompilerTarget where + parseJSON = withText "CompilerTarget" (pure . CompilerTarget) + +defaultCompilerTarget :: CompilerTarget +defaultCompilerTarget = if osIsWindows + then CompilerTarget "reloc-binary-dist" + else CompilerTarget "binary-dist" + +-- | Hadrian path to built binary distribution +newtype CompilerBindistPath + = CompilerBindistPath Text + deriving Show + +instance FromJSON CompilerBindistPath where + parseJSON = withText "CompilerBindistPath" (pure . CompilerBindistPath) + +defaultCompilerBindistPath :: CompilerBindistPath +defaultCompilerBindistPath = if osIsWindows + then CompilerBindistPath "_build/reloc-bindist" + else CompilerBindistPath "_build/bindist" + whichCompilerL :: Getting r ActualCompiler WhichCompiler whichCompilerL = to whichCompiler diff --git a/src/Stack/Types/Config.hs b/src/Stack/Types/Config.hs index bf7e00b856..a61112314f 100644 --- a/src/Stack/Types/Config.hs +++ b/src/Stack/Types/Config.hs @@ -40,7 +40,8 @@ import Stack.Types.ApplyGhcOptions ( ApplyGhcOptions (..) ) import Stack.Types.ApplyProgOptions ( ApplyProgOptions (..) ) import Stack.Types.BuildOpts ( BuildOpts ) import Stack.Types.CabalConfigKey ( CabalConfigKey ) -import Stack.Types.Compiler ( CompilerRepository ) +import Stack.Types.Compiler + ( CompilerBindistPath, CompilerRepository, CompilerTarget ) import Stack.Types.CompilerBuild ( CompilerBuild ) import Stack.Types.Docker ( DockerOpts ) import Stack.Types.DumpLogs ( DumpLogs ) @@ -114,6 +115,10 @@ data Config = Config -- ^ Specifies which versions of the compiler are acceptable. , compilerRepository :: !CompilerRepository -- ^ Specifies the repository containing the compiler sources + , compilerTarget :: !CompilerTarget + -- ^ Specifies the Hadrian build target + , compilerBindistPath :: !CompilerBindistPath + -- ^ Specifies the Hadrian path to built binary distribution , localBin :: !(Path Abs Dir) -- ^ Directory we should install executables into , fileWatchHook :: !(Maybe (Path Abs File)) diff --git a/src/Stack/Types/ConfigMonoid.hs b/src/Stack/Types/ConfigMonoid.hs index 13183fb846..b66f1d673e 100644 --- a/src/Stack/Types/ConfigMonoid.hs +++ b/src/Stack/Types/ConfigMonoid.hs @@ -44,7 +44,8 @@ import Stack.Types.BuildOptsMonoid ( BuildOptsMonoid ) import Stack.Types.Casa ( CasaOptsMonoid ) import Stack.Types.CabalConfigKey ( CabalConfigKey ) import Stack.Types.ColorWhen ( ColorWhen ) -import Stack.Types.Compiler ( CompilerRepository ) +import Stack.Types.Compiler + ( CompilerBindistPath, CompilerRepository, CompilerTarget ) import Stack.Types.CompilerBuild ( CompilerBuild ) import Stack.Types.Docker ( DockerOptsMonoid, VersionRangeJSON (..) ) import Stack.Types.DumpLogs ( DumpLogs ) @@ -101,6 +102,10 @@ data ConfigMonoid = ConfigMonoid -- ^ See: 'Stack.Types.Config.compilerCheck' , compilerRepository :: !(First CompilerRepository) -- ^ See: 'Stack.Types.Config.compilerRepository' + , compilerTarget :: !(First CompilerTarget) + -- ^ See: 'Stack.Types.Config.compilerTarget' + , compilerBindistPath :: !(First CompilerBindistPath) + -- ^ See: 'Stack.Types.Config.compilerBindistPath' , requireStackVersion :: !IntersectingVersionRange -- ^ See: 'Stack.Types.Config.requireStackVersion' , arch :: !(First String) @@ -289,6 +294,9 @@ parseConfigMonoidObject rootDir obj = do pure (First scmInit,fromMaybe M.empty params) compilerCheck <- First <$> obj ..:? configMonoidCompilerCheckName compilerRepository <- First <$> (obj ..:? configMonoidCompilerRepositoryName) + compilerTarget <- First <$> (obj ..:? configMonoidCompilerTargetName) + compilerBindistPath <- + First <$> (obj ..:? configMonoidCompilerBindistPathName) options <- Map.map (.ghcOptions) <$> obj ..:? configMonoidGhcOptionsName ..!= (mempty :: Map GhcOptionKey GhcOptions) @@ -381,6 +389,8 @@ parseConfigMonoidObject rootDir obj = do , msysEnvironment , compilerCheck , compilerRepository + , compilerTarget + , compilerBindistPath , requireStackVersion , arch , ghcVariant @@ -538,6 +548,12 @@ configMonoidCompilerCheckName = "compiler-check" configMonoidCompilerRepositoryName :: Text configMonoidCompilerRepositoryName = "compiler-repository" +configMonoidCompilerTargetName :: Text +configMonoidCompilerTargetName = "compiler-target" + +configMonoidCompilerBindistPathName :: Text +configMonoidCompilerBindistPathName = "compiler-bindist-path" + configMonoidGhcOptionsName :: Text configMonoidGhcOptionsName = "ghc-options"