[Cherry-Pick-Main][UI][Server]Fixed docker_runner in codegen and cleaned up some config and files (#58)

Author: mkaliberda

docker-compose.yml

@@ -77,15 +77,6 @@ services:
     depends_on:
       - db
       - redis
-      - sensiml.databse.initalize
-  nginx:
-    image: nginx:latest
-    ports:
-      - "80:80"
-    volumes:
-      - ./nginx/conf.d:/etc/nginx/conf.d
-    depends_on:
-      - sensiml.cloud      
   sensiml.pipelines:
     #mem_limit: "2000M"
     image: sensiml/base

src/server/codegen/docker_runner.py

@@ -116,13 +116,8 @@ def build_code_bin(self, build_type, application):
             }
         )
 
-        if settings.INSIDE_LOCAL_DOCKER:  # self.is_inside_docker():
-            bind = {
-                "piccolo_server_data": {
-                    "bind": "/data",
-                    "mode": "rw",
-                }
-            }
+        if settings.INSIDE_LOCAL_DOCKER:
+            bind = [f"{settings.LOCAL_DOCKER_SERVER_DATA_VOLUME_NAME}:/data"]
         else:
             bind = {
                 self.root_dir: {
@@ -218,6 +213,10 @@ def build_code_bin(self, build_type, application):
             finally:
                 # Remove container on success. Maybe just do it anyway.
                 client.api.remove_container(resource_id=container_to_run.get("Id"))
+        elif container_returns["StatusCode"] == 127 and settings.INSIDE_LOCAL_DOCKER:
+            raise ExitCodeNonZeroException(
+                f"Model data has missing some files. Please, check the data volume name."
+            )
         else:
             raise ExitCodeNonZeroException(container_returns)
 

src/server/datamanager/migrations/0087_lock_schema.py src/server/datamanager/migrations/0005_lock_schema.py

@@ -6,7 +6,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ("datamanager", "0086_label_info"),
+        ("datamanager", "0004_label_info"),
     ]
 
     operations = [

src/server/datamanager/tasks.py

@@ -321,7 +321,7 @@ def pipeline_async(self, user_id, project_id, sandbox_id, err_queue=deque()):
             team_member=user.teammember,
         )
         engine._temp.clean_up_temp()
-        engine.update_execution_status(
+        engine.update_team_cpu_usage(
             execution_type=PipelineExecution.ExecutionTypeEnum.PIPELINE,
             status=PipelineExecution.ExecutionStatusEnum.FAILED,
             wall_time=time.time() - start,
@@ -331,7 +331,7 @@ def pipeline_async(self, user_id, project_id, sandbox_id, err_queue=deque()):
         e.args = (engine._current_step_info,)
         raise e
 
-    engine.update_execution_status(
+    engine.update_team_cpu_usage(
         execution_type=PipelineExecution.ExecutionTypeEnum.PIPELINE,
         status=PipelineExecution.ExecutionStatusEnum.SUCCESS,
         wall_time=time.time() - start,
@@ -570,7 +570,7 @@ def gridsearch_async(
             }
         )
         engine._temp.clean_up_temp()
-        engine.update_execution_status(
+        engine.update_team_cpu_usage(
             execution_type=PipelineExecution.ExecutionTypeEnum.GRIDSEARCH,
             status=PipelineExecution.ExecutionStatusEnum.FAILED,
             wall_time=time.time() - start,
@@ -580,7 +580,7 @@ def gridsearch_async(
         e.args = (engine._current_step_info,)
         raise e
 
-    engine.update_execution_status(
+    engine.update_team_cpu_usage(
         execution_type=PipelineExecution.ExecutionTypeEnum.GRIDSEARCH,
         status=PipelineExecution.ExecutionStatusEnum.SUCCESS,
         wall_time=time.time() - start,
@@ -721,7 +721,7 @@ def auto_async(
             team_member=user.teammember,
         )
         engine._temp.clean_up_temp()
-        engine.update_execution_status(
+        engine.update_team_cpu_usage(
             execution_type=PipelineExecution.ExecutionTypeEnum.AUTOML,
             status=PipelineExecution.ExecutionStatusEnum.FAILED,
             wall_time=time.time() - start,
@@ -731,7 +731,7 @@ def auto_async(
         e.args = (engine._current_step_info,)
         raise e
 
-    engine.update_execution_status(
+    engine.update_team_cpu_usage(
         execution_type=PipelineExecution.ExecutionTypeEnum.AUTOML,
         status=PipelineExecution.ExecutionStatusEnum.SUCCESS,
         wall_time=time.time() - start,
@@ -820,7 +820,7 @@ def autosegment_async(
         )
 
         engine._temp.clean_up_temp()
-        engine.update_execution_status(
+        engine.update_team_cpu_usage(
             execution_type=PipelineExecution.ExecutionTypeEnum.AUTOSEG,
             status=PipelineExecution.ExecutionStatusEnum.FAILED,
             wall_time=time.time() - start,
@@ -829,7 +829,7 @@ def autosegment_async(
         e.args = ({"error": e.args[0]},)
         raise e
 
-    engine.update_execution_status(
+    engine.update_team_cpu_usage(
         execution_type=PipelineExecution.ExecutionTypeEnum.AUTOSEG,
         status=PipelineExecution.ExecutionStatusEnum.SUCCESS,
         wall_time=time.time() - start,

src/server/datamanager/tests/test_api_views.py

@@ -257,199 +257,6 @@ def test_labelvalue_create_update(self, client):
         assert response.status_code == status.HTTP_400_BAD_REQUEST
 
 
-@pytest.mark.usefixtures("authenticate")
-class TestUser:
-    @pytest.fixture(autouse=True)
-    def setup(self, client, request):
-        self.user = user = TeamMember.objects.get(email="[email protected]").user
-        client.force_authenticate(user=user)
-
-    def test_get_team_subscription(self, client):
-        response = client.get(reverse("team-subscription"))
-
-        assert response.status_code == status.HTTP_200_OK
-
-        assert len(response.json()) == 6
-
-        data = response.json()
-
-        assert data["team"] == "SensimlDevTeam"
-        assert data["subscription"] == "ENTERPRISE"
-        assert data["active"] == True
-        assert data["expires"] == "2084-12-24T17:21:28Z"
-
-        testuser = TeamMember.objects.get(email="[email protected]").user
-        client.force_authenticate(user=testuser)
-
-        response = client.get(reverse("team-subscription"))
-
-        assert response.status_code == status.HTTP_200_OK
-
-        assert len(response.json()) == 6
-
-        data = response.json()
-
-        assert data["team"] == "StarterTeam"
-        assert data["subscription"] == "STARTER"
-        assert data["active"] == True
-        assert data["expires"] == "2014-12-24T17:21:28Z"
-
-        client.force_authenticate(user=self.user)
-
-    def test_get_team_subscriptionv2(self, client):
-        response = client.get(reverse("team-subscription-v2"))
-
-        assert response.status_code == status.HTTP_200_OK
-
-        assert len(response.json()) == 6
-
-        data = response.json()
-
-        assert data["team"] == "SensimlDevTeam"
-        assert data["subscription"]["name"] == "ENTERPRISE"
-        assert data["active"] == True
-        assert data["subscription"]["is_read_only"] == False
-        assert data["subscription"]["can_edit_offline"] == True
-        assert data["subscription"]["max_project_segments"] == None
-        assert data["expires"] == "2084-12-24T17:21:28Z"
-
-        testuser = TeamMember.objects.get(email="[email protected]").user
-        client.force_authenticate(user=testuser)
-
-        response = client.get(reverse("team-subscription-v2"))
-
-        assert response.status_code == status.HTTP_200_OK
-
-        assert len(response.json()) == 6
-
-        data = response.json()
-
-        assert data["team"] == "StarterTeam"
-        assert data["subscription"]["name"] == "STARTER"
-        assert data["subscription"]["uuid"] == "56eada75-e049-47bc-bf73-535b8666c91d"
-        assert data["subscription"]["display_name"] == "Community Edition"
-        assert data["subscription"]["is_read_only"] == False
-        assert data["subscription"]["can_edit_offline"] == False
-        assert data["subscription"]["max_project_segments"] == 2500
-        assert data["active"] == True
-        assert data["expires"] == "2014-12-24T17:21:28Z"
-
-        client.force_authenticate(user=self.user)
-
-    def test_get_options(self, client):
-        response = client.options(reverse("user-list"))
-        assert response.data["actions"]["POST"]["email"]["required"] == True
-        assert response.data["actions"]["POST"]["password"]["required"] == True
-
-    def test_create_user(self, client):
-        data = {"email": "[email protected]", "password": "testpassword"}
-        response = client.post(reverse("user-list"), data=data)
-        assert response.status_code == status.HTTP_201_CREATED
-        response = client.get(response.data["url"])
-        assert response.status_code == status.HTTP_200_OK
-        assert response.data["team"] == TEAM_NAME
-        TeamMember.objects.get(email="[email protected]")
-
-    def test_create_user_with_new_team(self, client):
-        data = {
-            "email": "[email protected]",
-            "password": "testpassword",
-            "team": "new_team",
-        }
-        response = client.post(reverse("user-list"), data=data)
-        assert response.status_code == status.HTTP_201_CREATED
-        assert response.data["team"] == "new_team"
-        # check team manager
-        response.data["email"]
-        response = client.get(reverse("team-list"))
-        assert response.status_code == status.HTTP_200_OK
-        team = next(
-            (team for team in response.data if team["name"] == "new_team"), None
-        )
-        assert team is not None
-
-    def test_modify_user_password(self, client):
-        data = {"email": "[email protected]", "password": "testpassword"}
-        response = client.post(reverse("user-list"), data=data)
-        assert response.status_code == status.HTTP_201_CREATED
-        response = client.patch(response.data["url"], data={"password": "newpassword"})
-        assert response.status_code == status.HTTP_200_OK
-        user = User.objects.get(email="[email protected]")
-        assert user.check_password("newpassword")
-
-
-class TestActivationCode:
-    def test_activation_code_creation(self, client):
-        response = client.post(
-            reverse("activation-code"),
-            data={
-                "authentication": settings.ACTIVATION_CODE_AUTH,
-                "code": "11111" * 5,
-                "duration": "90 day",
-                "subscription": "STARTER",
-            },
-        )
-
-        assert response.status_code == status.HTTP_201_CREATED
-
-        act = ActivationCode.objects.get(code="11111" * 5)
-        assert act.subscription.id == 1
-
-    def test_activation_code_creation_no_duration(self, client):
-        response = client.post(
-            reverse("activation-code"),
-            data={
-                "authentication": settings.ACTIVATION_CODE_AUTH,
-                "code": "11111" * 5,
-                "subscription": "STARTER",
-            },
-        )
-
-        assert response.status_code == status.HTTP_201_CREATED
-
-        act = ActivationCode.objects.get(code="11111" * 5)
-        assert act.subscription.id == 1
-
-    def test_activation_code_invalid(self, client):
-        response = client.post(
-            reverse("activation-code"),
-            data={
-                "authentication": settings.ACTIVATION_CODE_AUTH,
-                "code": "11111",
-                "duration": "90 day",
-                "subscription": "STARTER",
-            },
-        )
-
-        assert response.status_code == status.HTTP_400_BAD_REQUEST
-
-    def test_subsciption_invalid(self, client):
-        response = client.post(
-            reverse("activation-code"),
-            data={
-                "authentication": settings.ACTIVATION_CODE_AUTH,
-                "code": "11111" * 5,
-                "duration": "90 day",
-                "subscription": "BLAH",
-            },
-        )
-
-        assert response.status_code == status.HTTP_400_BAD_REQUEST
-
-    def test_authentication_invalid(self, client):
-        response = client.post(
-            reverse("activation-code"),
-            data={
-                "authentication": "555",
-                "code": "11111" * 5,
-                "duration": "90 day",
-                "subscription": "BLAH",
-            },
-        )
-
-        assert response.status_code == status.HTTP_400_BAD_REQUEST
-
-
 class TestSegmenterViews:
     @pytest.fixture(autouse=True)
     def setup(self, client, authenticate):

src/server/datamanager/tests/views/test_capture.py

@@ -227,7 +227,7 @@ def test_create_csv_with_timestamp(self, client, project):
         from engine.base import pipeline_utils
         from datamanager.models import TeamMember
 
-        user = TeamMember.objects.get(email="unittest@sensiml.com").user
+        user = TeamMember.objects.get(email="unittest@piccolo.com").user
         capture_df_pipeline_utils, _, _ = pipeline_utils.get_capturefile(
             user, project.uuid, capture.name
         )

src/server/server/settings.py

@@ -159,7 +159,7 @@ def load_env(filename=".env", dir=os.path.dirname(__file__)):
     CELERY_RDB_PORT=(int, 6899),
     DOCKER_HOST_SML_DATA_DIR=(str, ""),
     INSIDE_LOCAL_DOCKER=(bool, True),
-    LOCAL_DOCKER_SERVER_DATA_VOLUMNE_NAME=(str, "sensiml_server_data"),
+    LOCAL_DOCKER_SERVER_DATA_VOLUME_NAME=(str, "sensiml_server_data"),
     FOUNDATION_MODEL_STORE_DOMAIN=(str, "sensiml"),
     FOUNDATION_MODEL_STORE_PROJECT=(str, "sensiml"),
     ADMIN_SITE_HEADER=(str, "SensiML Account Management"),
@@ -236,7 +236,8 @@ def load_env(filename=".env", dir=os.path.dirname(__file__)):
     print("No environment {} found".format(os.environ.get("DJANGO_ENV")))
 
 
-LOCAL_DOCKER_SERVER_DATA_VOLUMNE_NAME = env("LOCAL_DOCKER_SERVER_DATA_VOLUMNE_NAME")
+LOCAL_DOCKER_SERVER_DATA_VOLUME_NAME = env("LOCAL_DOCKER_SERVER_DATA_VOLUME_NAME")
+SERVER_NAME = env("SERVER_NAME")
 ENVIRONMENT = env("ENVIRONMENT")
 FREE_TEAM_PK = env("FREE_TEAM_PK")
 SENSIML_LIBRARY_PACK = None

src/ui/src/containers/BuildModel/components/PiplineBuilderCustomTraining.jsx

@@ -20,6 +20,7 @@ License along with SensiML Piccolo AI. If not, see <https://www.gnu.org/licenses
 import React, { useMemo } from "react";
 
 import { DISABLED_FOR_CUSTOM_STEPS } from "store/autoML/const";
+// eslint-disable-next-line import/extensions
 import PipelineBuilder from "./PipelineBuilder";
 
 const PipelineBuilderCustomTraining = (props) => {

src/ui/src/containers/DownloadModel/TheDownloadModel.jsx

@@ -280,7 +280,7 @@ const DownloadModel = ({
       fileDownload(response.data, response.filename);
       stopDownloadStatusCheck();
       setCompleted(100);
-      showMessage("success", t("download.success-dowload", { filename: response.filename }));
+      showMessage("success", t("download.success-download", { filename: response.filename }));
       setDownloadingCode(response?.code);
       stopDownloadStatusCheck();
     } else if (response.status === CANCELED) {