tests written for injecting evaluation files

Author: aronchick

.flake8

@@ -107,7 +107,7 @@ max-line-length = 79
 max-complexity = 18
 per-file-ignores =
     src/exec_file.py:F821
-    tests/sample_exec_file.py:F821
+    tests/sample_*.py:F821
 # Specify the list of error codes you wish Flake8 to report.
 select = B,C,E,F,W,T4,B9
 

.parameters/schemas/package_result.yaml

@@ -29,6 +29,10 @@ tested_platforms:
   type: list_strings
   required: True
 
+package_size:
+  type: int
+  required: True
+
 model_source:
   # model_source: (Optional) - (Required if servable is true)
   type: nested

Pipfile

@@ -9,6 +9,7 @@ pylint = "*"
 black = "*"
 ipython = "*"
 flake8 = "*"
+rope = "*"
 
 [packages]
 pyyaml = "*"

Pipfile.lock

@@ -0,0 +1,9 @@
+schema_version: 9999.0.1
+schema_type: data_process_run
+base_image: ubuntu1804
+machine_type: NDv2
+nodes: 5
+cpu_per_node: 8
+ram_per_node: 16Gi
+gpu_required: True
+output_root_path: 'https://contoso.s3.us-west-2.amazonaws.com/ml/processed_data'

tests/.parameters/process_data/execution/execution.yaml

@@ -0,0 +1,7 @@
+schema_version: 9999.0.1
+schema_type: data_source
+run_id: 6a9a5931-1c1d-47cc-aaf3-ad8b03f70575
+step_id: 433b4d99-042d-47c5-8c2f-6a4f5094510e
+run_date: 1970-01-08 00:00:00.00000
+source_id: 894b0239-e616-43fa-914b-de55894ad67d
+source_uri: 'https://data.vision.ee.ethz.ch/cvl/rrothe/imdb-wiki/'

tests/.parameters/process_data/input/input.yaml

@@ -0,0 +1,37 @@
+schema_type: workflow
+schema_version: 9999.0.1
+steps:
+  process_data:
+    input:
+      schema_type: data_source
+      schema_version: 9999.0.1
+    execution:
+      schema_type: data_process_run
+      schema_version: 9999.0.1
+    output:
+      schema_type: data_result
+      schema_version: 9999.0.1
+    next: train
+  train:
+    previous: process_data
+    input:
+      schema_type: data_result
+      schema_version: 9999.0.1
+    execution:
+      schema_type: training_run
+      schema_version: 9999.0.1
+    output:
+      schema_type: training_result
+      schema_version: 9999.0.1
+    next: package
+  package:
+    previous: train
+    input:
+      schema_type: training_result
+      schema_version: 9999.0.1
+    execution:
+      schema_type: package_run
+      schema_version: 9999.0.1
+    output:
+      schema_type: package_result
+      schema_version: 9999.0.1

tests/.parameters/workflow.yaml

@@ -4,7 +4,7 @@ mlspec_base_type:
 
 mlspec_schema_version:
   # Identifies the version of this schema
-  meta: 999.0.1
+  meta: 9999.0.1
 
 mlspec_schema_type:
   # Identifies the type of this schema

tests/bad_schemas/bad_base/data_result.yaml

@@ -0,0 +1,40 @@
+from mlspeclib import MLSchema, MLObject
+from random import randint, random, randrange
+from pathlib import Path
+import uuid
+
+results_ml_object.set_type(
+    schema_type=result_ml_object_schema_type, schema_version=result_ml_object_schema_version,
+)
+
+# Mocked up results
+return_dict = YAML.safe_load(
+    """
+servable: True
+package_size: 1029310298
+tested_platforms: ['kubeflow', 'azureml', 'sagemaker']
+model_source:
+    servable_model:
+        data_store: 's3'
+        bucket: 'nlp-bucket'
+        path: 'a231653454ca8e07f42adc7941aeec6b'
+serving_container_image:
+    container_image_url: 'https://hub.docker.com/repository/docker/contoso/nlp-base-images'
+"""
+)
+
+results_ml_object.servable = return_dict["servable"]
+results_ml_object.tested_platforms = return_dict["tested_platforms"]
+results_ml_object.package_size = return_dict["package_size"]
+results_ml_object.model_source.servable_model.data_store = return_dict["model_source"][
+    "servable_model"
+]["data_store"]
+results_ml_object.model_source.servable_model.bucket = return_dict["model_source"][
+    "servable_model"
+]["bucket"]
+results_ml_object.model_source.servable_model.path = return_dict["model_source"][
+    "servable_model"
+]["path"]
+results_ml_object.serving_container_image.container_image_url = return_dict[
+    "serving_container_image"
+]["container_image_url"]

tests/sample_package_execution.py

@@ -0,0 +1,20 @@
+from mlspeclib import MLSchema, MLObject
+from random import randint, random
+from pathlib import Path
+
+results_ml_object.set_type(
+    schema_type=result_ml_object_schema_type,  # noqa
+    schema_version=result_ml_object_schema_version,  # noqa
+)
+
+return_dict = {
+    "data_output_path": str(Path("tests/data/data_output.csv").absolute()),
+    "data_statistics_path": str(Path("tests/data/data_stats.csv").absolute()),
+    "data_schemas_path": str(Path("tests/data/data_schemas.yaml").absolute()),
+    "feature_file_path": str(Path("tests/data/feature_file.yaml").absolute()),
+}
+
+results_ml_object.data_output_path = return_dict["data_output_path"]
+results_ml_object.data_statistics_path = return_dict["data_statistics_path"]
+results_ml_object.data_schemas_path = return_dict["data_schemas_path"]
+results_ml_object.feature_file_path = return_dict["feature_file_path"]