Merge pull request #838 from viash-io/fix/broken_dependency_resolving

fix dependency resolving for dependencies of dependencies needing a local dependency

Author: hcannoodt

src/main/scala/io/viash/config/dependencies/Dependency.scala

@@ -24,6 +24,7 @@ import java.nio.file.Files
 import io.viash.ViashNamespace
 import io.viash.exceptions.MissingBuildYamlException
 import io.viash.config.ScopeEnum
+import io.viash.helpers.Logging
 
 @description(
   """Specifies a Viash component (script or executable) that should be made available for the code defined in the component.
@@ -140,7 +141,7 @@ case class Dependency(
   }.getOrElse(false)
 }
 
-object Dependency {
+object Dependency extends Logging {
 
   /**
     * Relativize the writtenPath info from a dependency to a source and destination path so it can be copied.
@@ -153,10 +154,39 @@ object Dependency {
     * @param mainDependency Top level dependency for which optionally dependencies of dependencies are being resolved. Used to relativize paths
     * @return Tuple with source and destination paths, relativized to current repository locations, ready to be copied
     */
-  def getSourceAndDestinationFromWrittenPath(dependencyPath: String, output: Path, repoPath: Path, mainDependency: Dependency): (Path, Path) = {
+  def getSourceAndDestinationFromWrittenPath(dependencyPath: String, output: Path, repoPath: Path, mainDependency: Dependency, remoteLocalDependencyResolver: Option[(Path, Path)]): (Path, Path) = {
     import scala.jdk.CollectionConverters._
 
-    val sourcePath = repoPath.resolve(dependencyPath)
+    val defaultSourcePath = repoPath.resolve(dependencyPath)
+    val sourcePath = if (defaultSourcePath.toFile().exists()) {
+      // If the dependencyPath is a valid path, use it as source
+      defaultSourcePath
+    } else if (remoteLocalDependencyResolver.isDefined) {
+      // This is empty if we're resolving the first level of dependencies.
+      // For the most part we should not end up here, but there is an edge case where we have to resolve a local dependency for a dependency of a dependency.
+      // In this case, we don't have the context of the location where the dependency is stored under the dependency folder, however we know where the dependant is stored,
+      // so we can use the remote local dependency resolver to find the source path.
+      logger.debug(s"Couldn't find sourcePath, using remote local dependency resolver for $dependencyPath")
+      logger.debug(s"Remote local dependency resolver: $remoteLocalDependencyResolver")
+      val alternativeSourcePath = remoteLocalDependencyResolver.get._1 // This is the path where the dependant is located
+      val alternativeTargetPath = remoteLocalDependencyResolver.get._2 // This is the relative path of the dependant in the target folder
+
+      // strips alternativeTargetPath from dependencyPath
+      // ie. `target` from `target/foo/bar` so that it can be added to alternativeSourcePath as it doesn't contain the `target` folder anymore at this point
+      val targetIter = alternativeTargetPath.iterator().asScala.toList.map(p => Some(p))
+      val depIter = Paths.get(dependencyPath).iterator().asScala.toList.map(p => Some(p))
+      val zipped = depIter.zipAll(targetIter, None, None).dropWhile {
+        case (depPart, targetPart) => depPart == targetPart
+      }
+      val relativePath = zipped.flatMap(_._1).reduce((p1, p2) => p1.resolve(p2))
+      logger.debug(s"relativePath: $relativePath")
+      val res = alternativeSourcePath.resolve(relativePath)
+      logger.debug(s"Using alternative source path: $res")
+      res
+    } else {
+      // Otherwise, throw an error. We shouldn't end up here.
+      throw new MissingBuildYamlException(defaultSourcePath, mainDependency)
+    }
     // Split the path into chunks so we can manipulate them more easily
     val pathParts = Paths.get(dependencyPath).iterator().asScala.toList.map(p => p.toString())
 

src/main/scala/io/viash/helpers/DependencyResolver.scala

@@ -265,8 +265,9 @@ object DependencyResolver extends Logging {
     }
   }
 
-  // Read a config file from a built target. Extract dependencies 'writtenPath'.
-  def getSparseDependencyInfo(configPath: String): List[String] = {
+  // Read a config file from a built target. Extract dependencies 'writtenPath' and the `output` path of the component.
+  // In 'legacy mode', the output path is not properly sanitized and thus useless, so we return an empty string instead.
+  def getSparseDependencyInfo(configPath: String): (List[String], String) = {
     try {
       val yamlText = IO.read(IO.uri(configPath))
       val json = Convert.textToJson(yamlText, configPath)
@@ -275,15 +276,18 @@ object DependencyResolver extends Logging {
       val dependencies =
         if (legacyMode) {
           val jsonVec = json.hcursor.downField("functionality").downField("dependencies").focus.flatMap(_.asArray).get
-          jsonVec.flatMap(_.hcursor.downField("writtenPath").as[String].toOption).toList
+          val depList = jsonVec.flatMap(_.hcursor.downField("writtenPath").as[String].toOption).toList
+          (depList, "")
         }
         else {
           val jsonVec = json.hcursor.downField("build_info").downField("dependencies").focus.flatMap(_.asArray).get
-          jsonVec.flatMap(_.hcursor.as[String].toOption).toList
+          val depList = jsonVec.flatMap(_.hcursor.as[String].toOption).toList
+          val outputPath = json.hcursor.downField("build_info").downField("output").as[String].toOption.get
+          (depList, outputPath)
         }
       dependencies
     } catch {
-      case _: Throwable => Nil
+      case _: Throwable => (Nil, "")
     }
   }
 
@@ -296,19 +300,42 @@ object DependencyResolver extends Logging {
   }
 
   // Handle dependencies of dependencies. For a given already built component, get their dependencies, copy them to our new target folder and recurse into these.
-  def recurseBuiltDependencies(output: Path, repoPath: Path, builtDependencyPath: String, dependency: Dependency, depth: Int = 0): Unit = {
+  def recurseBuiltDependencies(output: Path, repoPath: Path, builtDependencyPath: String, dependency: Dependency, dependencySourcePath: Option[Path] = None, depth: Int = 0): Unit = {
+    import scala.jdk.CollectionConverters._
 
     // Limit recursion depth to prevent infinite loops in e.g. cross dependencies (TODO)
     if (depth > 10)
       throw new RuntimeException("Copying dependencies traces too deep. Possibibly caused by a cross dependency.")
 
     // this returns paths relative to `repoPath` of dependencies to be copied to `output`
-    val dependencyPaths = getSparseDependencyInfo(builtDependencyPath + "/.config.vsh.yaml")
+    val (dependencyPaths, relativeOutput) = getSparseDependencyInfo(builtDependencyPath + "/.config.vsh.yaml")
+    logger.debug(s"Paths to relativize: dependencySourcePath: $dependencySourcePath, relativeOutput: $relativeOutput")
+    
+    // remove the trailing path parts as far as relativeOutputPath matches the dependencySourcePath
+    // dependencySourcePath: a/b/c/d/e
+    // relativeOutputPath: c'/d/e
+    // output: a/b/c & c'
+    // dependencySourcePath contains a tuple of:
+    //   - left: the the path where the dependency is stored down to common root, matching to 'target'
+    //   - right: the original 'target' folder name
+    // this is needed to relativize paths correctly when resolving a local dependency of this dependency
+    val dependencySourceParts = dependencySourcePath.map { dsp =>
+      val dspParts = dsp.iterator().asScala.toList.map(p => Some(p)).reverse
+      val relativeOutputPath = Paths.get(relativeOutput).iterator().asScala.toList.map(p => Some(p)).reverse
+      // Find the first part that is not in the relative output path
+      val commonParts = dspParts.zipAll(relativeOutputPath, None, None).dropWhile{ case (a, b) => a == b }
+
+      val leftPath = commonParts.flatMap(_._1).reverse.fold(dsp.getRoot())((p1, p2) => p1.resolve(p2))
+      val rightPath = commonParts.flatMap(_._2).reverse.reduce((p1, p2) => p1.resolve(p2))
+      
+      (leftPath, rightPath)
+    }
+    logger.debug(s"dependencySourceParts: $dependencySourceParts")
 
     for (dp <- dependencyPaths) {
       // Get the source & destination path for the dependency, functionality depends whether it was a previous dependency or not.
       // Paths are relativized depending the original dependency.
-      val (sourcePath, destPath) = Dependency.getSourceAndDestinationFromWrittenPath(dp, output, repoPath, dependency)
+      val (sourcePath, destPath) = Dependency.getSourceAndDestinationFromWrittenPath(dp, output, repoPath, dependency, dependencySourceParts)
 
       // Make sure the destination is clean so first remove the destination folder if it exists
       if (destPath.toFile().exists())
@@ -319,7 +346,7 @@ object DependencyResolver extends Logging {
       IO.copyFolder(sourcePath, destPath)
 
       // Check for more dependencies
-      recurseBuiltDependencies(output, repoPath, destPath.toString(), dependency, depth + 1)
+      recurseBuiltDependencies(output, repoPath, destPath.toString(), dependency, Some(sourcePath), depth + 1)
     }
   }