[Application] Support single-file source deployment with artifacts (mlrun#9244)

Author: Yacouby

mlrun/common/constants.py

@@ -27,6 +27,10 @@
 NUCLIO_LABEL_PREFIX = "nuclio.io/"
 RESERVED_TAG_NAME_LATEST = "latest"
 
+# Internal path for application runtime source artifacts (avoids user artifact conflicts)
+# "+/" prefix makes it relative to the project's default artifact path (see extend_artifact_path)
+MLRUN_INTERNAL_ARTIFACT_PATH = "+/.mlrun/sources"
+
 # Kubernetes DNS-1123 label name length limit
 K8S_DNS_1123_LABEL_MAX_LENGTH = 63
 
@@ -87,6 +91,8 @@ class MLRunInternalLabels:
     app_name = f"{MLRUN_LABEL_PREFIX}app-name"
     endpoint_id = f"{MLRUN_LABEL_PREFIX}endpoint-id"
     endpoint_name = f"{MLRUN_LABEL_PREFIX}endpoint-name"
+    function_name = f"{MLRUN_LABEL_PREFIX}function-name"
+    system_generated = f"{MLRUN_LABEL_PREFIX}system-generated"
     host = "host"
     job_type = "job-type"
     kind = "kind"

mlrun/model.py

@@ -571,13 +571,15 @@ def source(self):
     def source(self, source):
         if source and not (
             source.startswith("git://")
+            # Allow store artifact URIs for single-file sources
+            or mlrun.datastore.is_store_uri(source)
             # lenient check for file extension because we support many file types locally and remotely
             or pathlib.Path(source).suffix
             or source in [".", "./"]
         ):
             raise mlrun.errors.MLRunInvalidArgumentError(
                 f"source ({source}) must be a compressed (tar.gz / zip) file, a git repo, "
-                f"a file path or in the project's context (.)"
+                f"a file path, a store URI, or in the project's context (.)"
             )
 
         self._source = source

mlrun/projects/project.py

@@ -5934,7 +5934,17 @@ def _init_function_from_dict(
         )
 
     elif url.endswith(".py"):
-        if in_context and with_repo:
+        # For application runtime we set the source path directly, deploy will upload it as an artifact
+        if kind == mlrun.runtimes.RuntimeKinds.application:
+            func = new_function(
+                name,
+                image=image,
+                kind=kind,
+                handler=handler,
+                tag=tag,
+            )
+            func.spec.build.source = url
+        elif in_context and with_repo:
             # when load_source_on_run is used we allow not providing image as code will be loaded pre-run. ML-4994
             if (
                 not image

mlrun/runtimes/nuclio/application/application.py

@@ -12,13 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import copy
+import os
 import pathlib
 import typing
 
 import nuclio
 import nuclio.auth
 
+import mlrun.common.constants
 import mlrun.common.schemas as schemas
+import mlrun.datastore
 import mlrun.errors
 import mlrun.run
 import mlrun.runtimes.nuclio.api_gateway as nuclio_api_gateway
@@ -28,7 +31,7 @@
     ProbeTimeConfig,
     ProbeType,
 )
-from mlrun.utils import is_valid_port, logger, update_in
+from mlrun.utils import is_relative_path, is_valid_port, logger, update_in
 
 
 class ApplicationSpec(nuclio_function.NuclioSpec):
@@ -490,6 +493,8 @@ def deploy(
 
         :return: The default API gateway URL if created or True if the function is ready (deployed)
         """
+        # Upload local single-file source as artifact (if applicable)
+        self._upload_source_as_artifact()
 
         if (self.requires_build() and not self.spec.image) or force_build:
             self._fill_credentials()
@@ -1050,3 +1055,68 @@ def _validate_set_probes_input(params: dict):
             raise ValueError(
                 "Empty probe configuration: at least one parameter must be set"
             )
+
+    def _upload_source_as_artifact(self) -> None:
+        """
+        Upload local single-file source as an MLRun artifact.
+
+        If spec.build.source is a local file path, upload it to the artifact store
+        and update spec.build.source with the artifact URI.
+        """
+        source = self.spec.build.source
+        if not source:
+            return
+
+        # Only upload if it's a local single file
+        if not self._is_single_local_file(source):
+            return
+
+        project_name = self.metadata.project
+        if not project_name:
+            raise mlrun.errors.MLRunMissingProjectError(
+                "Project is required to upload source as artifact"
+            )
+        project = mlrun.get_or_create_project(project_name)
+
+        # Use function name as part of the artifact key for identification
+        artifact_key = f"{self.metadata.name}-source"
+
+        logger.info(
+            "Uploading local source file as artifact",
+            source=source,
+            artifact_key=artifact_key,
+            project=project_name,
+        )
+
+        # Upload the file as an artifact to an internal path with system-generated label
+        try:
+            artifact = project.log_artifact(
+                item=artifact_key,
+                local_path=source,
+                artifact_path=mlrun.common.constants.MLRUN_INTERNAL_ARTIFACT_PATH,
+                upload=True,
+                labels={
+                    mlrun.common.constants.MLRunInternalLabels.function_name: self.metadata.name,
+                    mlrun.common.constants.MLRunInternalLabels.system_generated: "true",
+                },
+            )
+        except Exception as exc:
+            raise mlrun.errors.MLRunRuntimeError(
+                f"Failed to upload source file '{source}' as artifact"
+            ) from exc
+
+        # Update the source to point to the artifact URI
+        self.spec.build.source = artifact.uri
+
+    @staticmethod
+    def _is_single_local_file(source: str) -> bool:
+        # Skip if the source is already a store URI
+        if mlrun.datastore.is_store_uri(source):
+            return False
+
+        # Skip if it's a remote URL (not a relative/local path)
+        if not (is_relative_path(source) or os.path.isabs(source)):
+            return False
+
+        # Check if it's a local file (not a directory)
+        return os.path.isfile(source)

tests/integration/sdk_api/run/test_main.py

@@ -376,7 +376,8 @@ def test_main_local_source(self):
             self._exec_run("./handler.py", args.split(), "test_main_local_source")
         assert (
             f"source ({examples_path}) must be a compressed (tar.gz / zip) file, "
-            f"a git repo, a file path or in the project's context (.)" in str(e.value)
+            f"a git repo, a file path, a store URI, or in the project's context (.)"
+            in str(e.value)
         )
 
     def test_main_run_archive_subdir(self):

tests/runtimes/test_application.py

@@ -14,10 +14,12 @@
 
 import base64
 import pathlib
+import unittest.mock
 
 import pytest
 
 import mlrun
+import mlrun.common.constants
 import mlrun.common.schemas
 import mlrun.runtimes
 import mlrun.utils
@@ -395,17 +397,6 @@ def test_deploy_reverse_proxy_image(rundb_mock, igz_version_mock):
     assert mlrun.runtimes.ApplicationRuntime.reverse_proxy_image
 
 
-def test_application_from_local_file_validation():
-    project = mlrun.get_or_create_project("test-application", allow_cross_project=True)
-    func_path = assets_path / "sample_function.py"
-    with pytest.raises(
-        mlrun.errors.MLRunInvalidArgumentError,
-        match="Embedding a code file is not supported for application runtime. "
-        "Code files should be specified via project/function source.",
-    ):
-        project.set_function(func=str(func_path), name="my-app", kind="application")
-
-
 def _assert_function_code(fn, file_path=None):
     file_path = (
         file_path or mlrun.runtimes.ApplicationRuntime.get_filename_and_handler()[0]
@@ -967,3 +958,153 @@ def test_enrich_sidecar_probe_ports_no_probes():
     assert ProbeType.READINESS.key not in sidecar
     assert ProbeType.LIVENESS.key not in sidecar
     assert ProbeType.STARTUP.key not in sidecar
+
+
+@pytest.mark.parametrize(
+    "source,setup_file,expected",
+    [
+        ("local_file.py", True, True),
+        ("directory_path", False, False),
+        ("", False, False),
+        ("store://artifacts/project/file", False, False),
+        ("https://example.com/file.py", False, False),
+        ("git://github.com/repo.git", False, False),
+        ("/non/existent/path.py", False, False),
+    ],
+)
+def test_is_single_local_file(tmp_path, source, setup_file, expected):
+    # Test _is_single_local_file identifies local files vs remote/invalid sources.
+    func_name = "application-test"
+    fn: mlrun.runtimes.ApplicationRuntime = mlrun.new_function(
+        func_name,
+        kind="application",
+        image="mlrun/mlrun",
+    )
+
+    if setup_file:
+        # Create a temporary file for local file case
+        file_path = tmp_path / source
+        file_path.write_text("def handler(): pass")
+        test_source = str(file_path)
+    elif source == "directory_path":
+        # Use tmp_path as directory
+        test_source = str(tmp_path)
+    else:
+        test_source = source
+
+    assert fn._is_single_local_file(test_source) is expected
+
+
+def test_upload_source_as_artifact(tmp_path):
+    # Test that local single file is uploaded as artifact
+    func_name = "application-test"
+    # Create a temporary source file
+    source_file = tmp_path / "handler.py"
+    source_file.write_text("def handler(): pass")
+
+    fn: mlrun.runtimes.ApplicationRuntime = mlrun.new_function(
+        func_name,
+        kind="application",
+        image="mlrun/mlrun",
+        project="test-project",
+    )
+    fn.spec.build.source = str(source_file)
+
+    mock_artifact = unittest.mock.MagicMock()
+    mock_artifact.uri = "store://artifacts/test-project/application-test-source"
+
+    mock_project = unittest.mock.MagicMock()
+    mock_project.log_artifact.return_value = mock_artifact
+
+    with unittest.mock.patch(
+        "mlrun.get_or_create_project", return_value=mock_project
+    ) as mock_get_project:
+        fn._upload_source_as_artifact()
+
+    # Verify project was retrieved
+    mock_get_project.assert_called_once_with("test-project")
+
+    # Verify artifact was logged with correct parameters
+    mock_project.log_artifact.assert_called_once_with(
+        item="application-test-source",
+        local_path=str(source_file),
+        artifact_path=mlrun.common.constants.MLRUN_INTERNAL_ARTIFACT_PATH,
+        upload=True,
+        labels={
+            mlrun.common.constants.MLRunInternalLabels.function_name: func_name,
+            mlrun.common.constants.MLRunInternalLabels.system_generated: "true",
+        },
+    )
+
+    # Verify source was updated to the artifact URI
+    assert (
+        fn.spec.build.source == "store://artifacts/test-project/application-test-source"
+    )
+
+
+@pytest.mark.parametrize(
+    "source",
+    [
+        "store://artifacts/test-project/existing-artifact",
+        "https://github.com/org/repo.git",
+        "s3://bucket/path/file.py",
+        "",
+    ],
+)
+def test_upload_source_as_artifact_skip_non_local(source):
+    # Test that non-local sources (store URIs, remote URLs) and empty path are not uploaded
+    func_name = "application-test"
+    fn: mlrun.runtimes.ApplicationRuntime = mlrun.new_function(
+        func_name,
+        kind="application",
+        image="mlrun/mlrun",
+        project="test-project",
+    )
+    fn.spec.build.source = source
+
+    with unittest.mock.patch("mlrun.get_or_create_project") as mock_get_project:
+        fn._upload_source_as_artifact()
+
+    # Verify project was not called (upload skipped)
+    mock_get_project.assert_not_called()
+
+    # Verify source remains unchanged
+    assert fn.spec.build.source == source
+
+
+def test_upload_source_as_artifact_no_project_error():
+    # Test that missing project raises an error
+    func_name = "application-test"
+    # Create a temporary source file
+    with unittest.mock.patch("os.path.isfile", return_value=True):
+        fn: mlrun.runtimes.ApplicationRuntime = mlrun.new_function(
+            func_name,
+            kind="application",
+            image="mlrun/mlrun",
+        )
+        fn.metadata.project = None
+        fn.spec.build.source = "/path/to/handler.py"
+
+        with pytest.raises(
+            mlrun.errors.MLRunMissingProjectError,
+            match="Project is required to upload source as artifact",
+        ):
+            fn._upload_source_as_artifact()
+
+
+def test_set_function_single_file_application(tmp_path):
+    # Test that set_function with single .py file works for application runtime
+    source_file = tmp_path / "handler.py"
+    source_file.write_text("def handler(): pass")
+
+    project = mlrun.get_or_create_project("test-proj", allow_cross_project=True)
+    fn = project.set_function(
+        str(source_file),
+        name="my-app",
+        kind="application",
+        image="mlrun/mlrun",
+    )
+
+    assert fn.kind == "application"
+    assert fn.metadata.project == "test-proj"
+    assert fn.spec.build.source == str(source_file)