Milestone 1: Updating Oppia-ML pipeline (#32)

* Milestone 1: Updating Oppia-ML pipeline

* Disable lint (no-name-in-module) for importing generated proto files

* Refactor protobuf implementation and address review comments

* Fix lint tests

* Addressed review comments

* Fix __init__.py inclusion

* Address review comments

* Address Review Comments

* Correct doc string

* Nit changes

Author: prasanna08

.gitignore

@@ -3,3 +3,5 @@
 *.swp
 *.swo
 third_party/*
+core/domain/proto/*.py
+!core/domain/proto/__init__.py

core/classifiers/CodeClassifier/CodeClassifier.py

@@ -538,6 +538,22 @@ def __init__(self):
         # text into a feature a vector.
         self.count_vector = None
 
+    @property
+    def name_in_job_result_proto(self):
+        # This property needs to be defined as it is defined as
+        # abstract property in BaseClassifier. However, since code classifier
+        # is currently disabled, the proto message definitions for code
+        # classifier will not be added until it is re-implemented and enabled.
+        return 'code_classifier'
+
+    @property
+    def type_in_job_result_proto(self):
+        # This property needs to be defined as it is defined as
+        # abstract property in BaseClassifier. However, since code classifier
+        # is currently disabled, the proto message definitions for code
+        # classifier will not be added until it is re-implemented and enabled.
+        return '%sFrozenModel' % self.__class__.__name__
+
     def to_dict(self):
         """Returns a dict representing this classifier.
 

core/classifiers/TextClassifier/TextClassifier.py

@@ -35,6 +35,7 @@ class TextClassifier(base.BaseClassifier):
     Support Vector Classifier (SVC) to obtain the best model using the linear
     kernel.
     """
+
     def __init__(self):
         super(TextClassifier, self).__init__()
         # sklearn.svm.SVC classifier object.
@@ -54,6 +55,14 @@ def __init__(self):
         # Time taken to train the classifier
         self.exec_time = None
 
+    @property
+    def name_in_job_result_proto(self):
+        return 'text_classifier'
+
+    @property
+    def type_in_job_result_proto(self):
+        return '%sFrozenModel' % (self.__class__.__name__)
+
     def train(self, training_data):
         """Trains classifier using given training_data.
 

core/classifiers/base.py

@@ -36,6 +36,22 @@ class BaseClassifier(object):
     def __init__(self):
         pass
 
+    @property
+    @abc.abstractproperty
+    def name_in_job_result_proto(self):
+        """A property that identifies the attribute in job result proto message
+        which will store this classifier's classifier data.
+        """
+        raise NotImplementedError
+
+    @property
+    @abc.abstractproperty
+    def type_in_job_result_proto(self):
+        """The type of the property in job result proto message which stores
+        this classifier's classifier data.
+        """
+        raise NotImplementedError
+
     @abc.abstractmethod
     def to_dict(self):
         """Returns a dict representing this classifier.

core/services/__init__.py core/domain/__init__.pycore/services/__init__.py renamed to core/domain/__init__.py

@@ -18,7 +18,7 @@
 
 from core.classifiers import algorithm_registry
 from core.classifiers import classifier_utils
-from core.services import remote_access_services
+from core.domain import remote_access_services
 
 # pylint: disable=too-many-branches
 def _validate_job_data(job_data):

core/services/job_services.py core/domain/job_services.pycore/services/job_services.py renamed to core/domain/job_services.py

@@ -0,0 +1,23 @@
+// coding: utf-8
+//
+// Copyright 2020 The Oppia Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS-IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+message TextClassifierFrozenModel {
+  // The parameters of a trained text classifier model which are necessary
+  // for inference.
+  string model_json = 1;
+}

core/domain/proto/__init__.py

@@ -0,0 +1,44 @@
+// coding: utf-8
+//
+// Copyright 2020 The Oppia Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS-IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+import "core/domain/proto/text_classifier.proto";
+
+// Training job response payload contains job result of the training job
+// along with other metadata items such as vm_id (to identify which VM executed
+// this job) and signature of the payload for security purpose.
+message TrainingJobResponsePayload {
+  // Job result of the training job. Job result contains the ID of the Job and
+  // trained model (frozen model) of the job.
+  message JobResult {
+    // Id of the training job whose data is being stored.
+    string job_id = 1;
+    
+    // Each of the classifier algorithms' proto message must be present in
+    // the oneof classifier_data field.
+    oneof classifier_frozen_model {
+      TextClassifierFrozenModel text_classifier = 2;
+    }
+  }
+  JobResult job_result = 1;
+
+  // Id of the VM instance that trained the job.
+  string vm_id = 2;
+  
+  // Signature of the job data for authenticated communication.
+  string signature = 3;
+}