Package management in the evaluator

Author: jneem

Cargo.lock

@@ -37,6 +37,7 @@ ansi_term = "0.12"
 anyhow = "1.0"
 assert_cmd = "2.0.11"
 assert_matches = "1.5.0"
+base16 = "0.2.1"
 bincode = "1.3.3"
 clap = "4.3"
 clap_complete = "4.3.2"

Cargo.toml

@@ -34,6 +34,7 @@ cxx-build = { workspace = true, optional = true }
 pkg-config = { workspace = true, optional = true }
 
 [dependencies]
+base16.workspace = true
 lalrpop-util.workspace = true
 regex.workspace = true
 simple-counter.workspace = true
@@ -79,6 +80,7 @@ tree-sitter-nickel = { workspace = true, optional = true }
 
 metrics = { workspace = true, optional = true }
 strsim = "0.10.0"
+directories.workspace = true
 
 [dev-dependencies]
 pretty_assertions.workspace = true

core/Cargo.toml

@@ -5,6 +5,7 @@ use crate::eval::cache::Cache as EvalCache;
 use crate::eval::Closure;
 #[cfg(feature = "nix-experimental")]
 use crate::nix_ffi;
+use crate::package::{self, LockedPackageSource, ResolvedLockFile};
 use crate::parser::{lexer::Lexer, ErrorTolerantParser};
 use crate::position::TermPos;
 use crate::program::FieldPath;
@@ -75,6 +76,7 @@ impl InputFormat {
 /// is, the operations that have been performed on this term) is stored in an [EntryState].
 #[derive(Debug, Clone)]
 pub struct Cache {
+    // TODO: associate packages to file ids
     /// The content of the program sources plus imports.
     files: Files<String>,
     file_paths: HashMap<FileId, SourcePath>,
@@ -86,6 +88,10 @@ pub struct Cache {
     rev_imports: HashMap<FileId, HashSet<FileId>>,
     /// The table storing parsed terms corresponding to the entries of the file database.
     terms: HashMap<FileId, TermEntry>,
+    /// A table mapping FileIds to the package that they belong to.
+    ///
+    /// Path dependencies have already been canonicalized to absolute paths.
+    package: HashMap<FileId, LockedPackageSource>,
     /// The list of ids corresponding to the stdlib modules
     stdlib_ids: Option<HashMap<StdlibModule, FileId>>,
     /// The inferred type of wildcards for each `FileId`.
@@ -94,6 +100,8 @@ pub struct Cache {
     error_tolerance: ErrorTolerance,
     import_paths: Vec<PathBuf>,
 
+    lock_file: ResolvedLockFile,
+
     #[cfg(debug_assertions)]
     /// Skip loading the stdlib, used for debugging purpose
     pub skip_stdlib: bool,
@@ -338,9 +346,11 @@ impl Cache {
             wildcards: HashMap::new(),
             imports: HashMap::new(),
             rev_imports: HashMap::new(),
+            package: HashMap::new(),
             stdlib_ids: None,
             error_tolerance,
             import_paths: Vec::new(),
+            lock_file: ResolvedLockFile::default(),
 
             #[cfg(debug_assertions)]
             skip_stdlib: false,
@@ -354,6 +364,10 @@ impl Cache {
         self.import_paths.extend(paths.map(PathBuf::from));
     }
 
+    pub fn set_lock_file(&mut self, lock_file: ResolvedLockFile) {
+        self.lock_file = lock_file;
+    }
+
     /// Same as [Self::add_file], but assume that the path is already normalized, and take the
     /// timestamp as a parameter.
     fn add_file_(&mut self, path: PathBuf, timestamp: SystemTime) -> io::Result<FileId> {
@@ -1313,6 +1327,7 @@ pub trait ImportResolver {
         &mut self,
         path: &OsStr,
         parent: Option<FileId>,
+        pkg: Option<&package::Name>,
         pos: &TermPos,
     ) -> Result<(ResolvedTerm, FileId), ImportError>;
 
@@ -1327,18 +1342,30 @@ impl ImportResolver for Cache {
         &mut self,
         path: &OsStr,
         parent: Option<FileId>,
+        pkg: Option<&package::Name>,
         pos: &TermPos,
     ) -> Result<(ResolvedTerm, FileId), ImportError> {
-        // `parent` is the file that did the import. We first look in its containing directory.
-        let mut parent_path = parent
-            .and_then(|p| self.get_path(p))
-            .map(PathBuf::from)
-            .unwrap_or_default();
-        parent_path.pop();
-
-        let possible_parents: Vec<PathBuf> = std::iter::once(parent_path)
-            .chain(self.import_paths.iter().cloned())
-            .collect();
+        let (possible_parents, pkg_id) = if let Some(pkg) = pkg {
+            let pkg_id = self
+                .lock_file
+                .get(parent.and_then(|p| self.package.get(&p)), pkg, pos)?;
+            (vec![pkg_id.local_path()], Some(pkg_id.clone()))
+        } else {
+            // `parent` is the file that did the import. We first look in its containing directory, followed by
+            // the directories in the import path.
+            let mut parent_path = parent
+                .and_then(|p| self.get_path(p))
+                .map(PathBuf::from)
+                .unwrap_or_default();
+            parent_path.pop();
+
+            (
+                std::iter::once(parent_path)
+                    .chain(self.import_paths.iter().cloned())
+                    .collect(),
+                None,
+            )
+        };
 
         // Try to import from all possibilities, taking the first one that succeeds.
         let (id_op, path_buf) = possible_parents
@@ -1374,6 +1401,10 @@ impl ImportResolver for Cache {
         self.parse_multi(file_id, format)
             .map_err(|err| ImportError::ParseErrors(err, *pos))?;
 
+        if let Some(pkg_id) = pkg_id {
+            self.package.insert(file_id, pkg_id);
+        }
+
         Ok((result, file_id))
     }
 
@@ -1414,7 +1445,7 @@ pub fn normalize_path(path: impl Into<PathBuf>) -> std::io::Result<PathBuf> {
 /// [`std::fs::canonicalize`] can be hard to use correctly, since it can often
 /// fail, or on Windows returns annoying device paths. This is a problem Cargo
 /// needs to improve on.
-fn normalize_abs_path(path: &Path) -> PathBuf {
+pub fn normalize_abs_path(path: &Path) -> PathBuf {
     use std::path::Component;
 
     let mut components = path.components().peekable();
@@ -1461,6 +1492,7 @@ pub mod resolvers {
             &mut self,
             _path: &OsStr,
             _parent: Option<FileId>,
+            _pkg: Option<&package::Name>,
             _pos: &TermPos,
         ) -> Result<(ResolvedTerm, FileId), ImportError> {
             panic!("cache::resolvers: dummy resolver should not have been invoked");
@@ -1502,6 +1534,7 @@ pub mod resolvers {
             &mut self,
             path: &OsStr,
             _parent: Option<FileId>,
+            _pkg: Option<&package::Name>,
             pos: &TermPos,
         ) -> Result<(ResolvedTerm, FileId), ImportError> {
             let file_id = self

core/src/cache.rs

@@ -2,6 +2,7 @@
 //!
 //! Define error types for different phases of the execution, together with functions to generate a
 //! [codespan](https://crates.io/crates/codespan-reporting) diagnostic from them.
+
 pub use codespan::{FileId, Files};
 pub use codespan_reporting::diagnostic::{Diagnostic, Label, LabelStyle};
 
@@ -18,6 +19,7 @@ use crate::{
         ty_path::{self, PathSpan},
         MergeKind, MergeLabel,
     },
+    package::{LockedPackageSource, Name},
     parser::{
         self,
         error::{InvalidRecordTypeError, LexicalError, ParseError as InternalParseError},
@@ -558,6 +560,8 @@ pub enum ParseError {
 /// An error occurring during the resolution of an import.
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub enum ImportError {
+    /// An unexpected internal error.
+    InternalError { msg: String, pos: TermPos },
     /// An IO error occurred during an import.
     IOError(
         /* imported file */ String,
@@ -569,6 +573,15 @@ pub enum ImportError {
         /* error */ ParseErrors,
         /* import position */ TermPos,
     ),
+    /// A package dependency was not found.
+    MissingDependency {
+        /// The package that tried to import the missing dependency, if there was one.
+        /// This will be `None` if the missing dependency was from the top-level
+        parent: Option<Box<(Name, LockedPackageSource)>>,
+        /// The name of the package that could not be resolved.
+        missing: Name,
+        pos: TermPos,
+    },
 }
 
 #[derive(Debug, PartialEq, Clone)]
@@ -2527,6 +2540,17 @@ impl IntoDiagnostics<FileId> for ImportError {
         stdlib_ids: Option<&Vec<FileId>>,
     ) -> Vec<Diagnostic<FileId>> {
         match self {
+            ImportError::InternalError { msg, pos } => {
+                let labels = pos
+                    .as_opt_ref()
+                    .map(|span| vec![primary(span).with_message("here")])
+                    .unwrap_or_default();
+
+                vec![Diagnostic::error()
+                    .with_message(format!("internal error: {msg}"))
+                    .with_labels(labels)
+                    .with_notes(vec![String::from(INTERNAL_ERROR_MSG)])]
+            }
             ImportError::IOError(path, error, span_opt) => {
                 let labels = span_opt
                     .as_opt_ref()
@@ -2552,6 +2576,26 @@ impl IntoDiagnostics<FileId> for ImportError {
 
                 diagnostic
             }
+            ImportError::MissingDependency {
+                parent,
+                missing,
+                pos,
+            } => {
+                let labels = pos
+                    .as_opt_ref()
+                    .map(|span| vec![primary(span).with_message("imported here")])
+                    .unwrap_or_default();
+                let msg = if let Some((parent_name, parent_source)) = parent.as_deref() {
+                    format!(
+                        "unknown package {missing}, imported from package {parent_name} at {}",
+                        parent_source.local_path().display()
+                    )
+                } else {
+                    format!("unknown package {missing}")
+                };
+
+                vec![Diagnostic::error().with_message(msg).with_labels(labels)]
+            }
         }
     }
 }

core/src/error/mod.rs

@@ -785,7 +785,7 @@ impl<R: ImportResolver, C: Cache> VirtualMachine<R, C> {
                         ));
                     }
                 }
-                Term::Import(path) => {
+                Term::Import(path, _pkg) => {
                     return Err(EvalError::InternalError(
                         format!("Unresolved import ({})", path.to_string_lossy()),
                         pos,
@@ -1159,7 +1159,7 @@ pub fn subst<C: Cache>(
         | v @ Term::Lbl(_)
         | v @ Term::SealingKey(_)
         | v @ Term::Enum(_)
-        | v @ Term::Import(_)
+        | v @ Term::Import(..)
         | v @ Term::ResolvedImport(_)
         // We could recurse here, because types can contain terms which would then be subject to
         // substitution. Not recursing should be fine, though, because a type in term position

core/src/eval/mod.rs

@@ -9,6 +9,7 @@ pub mod identifier;
 pub mod label;
 #[cfg(feature = "nix-experimental")]
 pub mod nix_ffi;
+pub mod package;
 pub mod parser;
 pub mod position;
 pub mod pretty;