add monitoring, save new tweets in mongo db, add tests, prediction in worker in server

Author: Wonters

Dockerfile

@@ -2,10 +2,12 @@ FROM python:3.10-slim
 RUN apt-get update && apt-get install -y procps
 LABEL authors="shift"
 WORKDIR /app
-COPY src .
 COPY requirements.txt .
 RUN pip install -r requirements.txt
+COPY src src/
+COPY scripts/entrypoint.sh .
+COPY supervisord.conf .
 EXPOSE 5000
 EXPOSE 5001
 
-ENTRYPOINT "./entrypoint.sh"
+ENTRYPOINT ["./entrypoint.sh"]

checkpoints/Logistic_regression_tokenizer.pkl

@@ -1,18 +1,28 @@
 "0","1467810369","Mon Apr 06 22:19:45 PDT 2009","NO_QUERY","_TheSpecialOne_","@switchfoot http://twitpic.com/2y1zl - Awww, that's a bummer.  You shoulda got David Carr of Third Day to do it. ;D"
+"4","1467822272","Mon Apr 06 22:22:45 PDT 2009","NO_QUERY","ersle","I LOVE @Health4UandPets u guys r the best!! "
 "0","1467810672","Mon Apr 06 22:19:49 PDT 2009","NO_QUERY","scotthamilton","is upset that he can't update his Facebook by texting it... and might cry as a result  School today also. Blah!"
+"4","1467822273","Mon Apr 06 22:22:45 PDT 2009","NO_QUERY","becca210","im meeting up with one of my besties tonight! Cant wait!!  - GIRL TALK!!"
+"4","1467822287","Mon Apr 06 22:22:46 PDT 2009","NO_QUERY","katarinka","Being sick can be really cheap when it hurts too much to eat real food  Plus, your friends make you soup"
 "0","1467810917","Mon Apr 06 22:19:53 PDT 2009","NO_QUERY","mattycus","@Kenichan I dived many times for the ball. Managed to save 50%  The rest go out of bounds"
 "0","1467811184","Mon Apr 06 22:19:57 PDT 2009","NO_QUERY","ElleCTF","my whole body feels itchy and like its on fire "
 "0","1467811193","Mon Apr 06 22:19:57 PDT 2009","NO_QUERY","Karoli","@nationwideclass no, it's not behaving at all. i'm mad. why am i here? because I can't see you all over there. "
+"4","1467822283","Mon Apr 06 22:22:46 PDT 2009","NO_QUERY","Wingman29","@DaRealSunisaKim Thanks for the Twitter add, Sunisa! I got to meet you once at a HIN show here in the DC area and you were a sweetheart. "
 "0","1467811372","Mon Apr 06 22:20:00 PDT 2009","NO_QUERY","joy_wolf","@Kwesidei not the whole crew "
 "0","1467811592","Mon Apr 06 22:20:03 PDT 2009","NO_QUERY","mybirch","Need a hug "
+"4","1467822391","Mon Apr 06 22:22:47 PDT 2009","NO_QUERY","ajarofalmonds","@ProductOfFear You can tell him that I just burst out laughing really loud because of that  Thanks for making me come out of my sulk!"
 "0","1467811594","Mon Apr 06 22:20:03 PDT 2009","NO_QUERY","coZZ","@LOLTrish hey  long time no see! Yes.. Rains a bit ,only a bit  LOL , I'm fine thanks , how's you ?"
 "0","1467811795","Mon Apr 06 22:20:05 PDT 2009","NO_QUERY","2Hood4Hollywood","@Tatiana_K nope they didn't have it "
 "0","1467812025","Mon Apr 06 22:20:09 PDT 2009","NO_QUERY","mimismo","@twittera que me muera ? "
+"4","1467822293","Mon Apr 06 22:22:46 PDT 2009","NO_QUERY","_EmilyYoung","@LovesBrooklyn2 he has that effect on everyone "
 "0","1467812416","Mon Apr 06 22:20:16 PDT 2009","NO_QUERY","erinx3leannexo","spring break in plain city... it's snowing "
 "0","1467812579","Mon Apr 06 22:20:17 PDT 2009","NO_QUERY","pardonlauren","I just re-pierced my ears "
 "0","1467812723","Mon Apr 06 22:20:19 PDT 2009","NO_QUERY","TLeC","@caregiving I couldn't bear to watch it.  And I thought the UA loss was embarrassing . . . . ."
+"4","1467822447","Mon Apr 06 22:22:51 PDT 2009","NO_QUERY","vmdavinci","@r_keith_hill Thans for your response. Ihad already find this answer "
+"4","1467822465","Mon Apr 06 22:22:48 PDT 2009","NO_QUERY","jessicavaliyi","@KeepinUpWKris I am so jealous, hope you had a great time in vegas! how did you like the ACM's?! LOVE YOUR SHOW!! "
 "0","1467812771","Mon Apr 06 22:20:19 PDT 2009","NO_QUERY","robrobbierobert","@octolinz16 It it counts, idk why I did either. you never talk to me anymore "
 "0","1467812784","Mon Apr 06 22:20:20 PDT 2009","NO_QUERY","bayofwolves","@smarrison i would've been the first, but i didn't have a gun.    not really though, zac snyder's just a doucheclown."
+"4","1467822489","Mon Apr 06 22:22:49 PDT 2009","NO_QUERY","emmasaur28","@tommcfly ah, congrats mr fletcher for finally joining twitter "
 "0","1467812799","Mon Apr 06 22:20:20 PDT 2009","NO_QUERY","HairByJess","@iamjazzyfizzle I wish I got to watch it with you!! I miss you and @iamlilnicki  how was the premiere?!"
 "0","1467812964","Mon Apr 06 22:20:22 PDT 2009","NO_QUERY","lovesongwriter","Hollis' death scene will hurt me severely to watch on film  wry is directors cut not out now?"
-"0","1467813137","Mon Apr 06 22:20:25 PDT 2009","NO_QUERY","armotley","about to file taxes "
+"0","1467813137","Mon Apr 06 22:20:25 PDT 2009","NO_QUERY","armotley","about to file taxes "
+"4","1467822496","Mon Apr 06 22:22:49 PDT 2009","NO_QUERY","SherylBreuker","@e4VoIP I RESPONDED  Stupid cat is helping me type. Forgive errors "

checkpoints/logistic_regression.pkl

@@ -4,21 +4,76 @@ services:
     container_name: sentimental_analyses_backend
     ports:
       - "5000:5000"
-      -  "5001:5001"
+      - "5001:5001"
     volumes:
-      - ./mlflow:/app/mlruns/
-    entrypoint: "./entrypoint.sh"
+      - ./mlruns:/app/mlruns/
+      - ./src:/app/src/
+    stdin_open: true
+    tty: true
+    # entrypoint: "./entrypoint.sh"
     environment:
       - MLFLOW_TRACKING_URI=http://0.0.0.0:5001
       - HOST_DEPLOY = 0.0.0.0
 
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: sentimental_analyses_prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: sentimental_analyses_grafana
+    ports:
+      - "3000:3000"
+    depends_on:
+      - loki
+    volumes:
+      - grafana-storage:/var/lib/grafana
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    restart: unless-stopped
+
+  loki:
+    image: grafana/loki:2.9.4
+    container_name: sentimental_analyses_loki
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+    restart: unless-stopped
+
+  db:
+    image: mongo:4.4
+    container_name: sentimental_analyses_db
+    ports:
+      - "27017:27017"
+    restart: unless-stopped
+
+  promtail:
+    image: grafana/promtail:2.9.4
+    container_name: sentimental_analyses_promtail
+    volumes:
+      - ./log:/var/log
+      - ./promtail-config.yaml:/etc/promtail/config.yaml
+    command: -config.file=/etc/promtail/config.yaml
+    depends_on:
+      - loki
+    restart: unless-stopped
   nginx:
     image: nginx
     container_name: sentimental_analyses_nginx
     ports:
-      - "80:80"
-      - '443:443'
+      - "81:80"
+      - '444:443'
     volumes:
       - ./nginx.conf:/etc/nginx/conf.d/default.conf
     depends_on:
-      - backend
+      - backend
+
+volumes:
+  grafana-storage:

data/tweets_test_train.csv

@@ -1,12 +1,4 @@
 
-upstream backend {
-    server backend:5000;
-}
-
-upstream mlflow {
-    server backend:5000;
-}
-
 server {
     listen 80;
     listen [::]:80;

docker-compose.yml

@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'fastapi-backend'
+    static_configs:
+      - targets: ['backend:5000']

nginx.conf

@@ -0,0 +1,18 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: system
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: varlogs
+          __path__: /var/log/*.log

prometheus.yml

@@ -4,5 +4,5 @@ echo "start supervisor"
 supervisord -c supervisord.conf
 
 echo "start server"
-uvicorn server:app --host 0.0.0.0 --port 5000 --reload
+uvicorn src.server:app --host 0.0.0.0 --port 5000 --reload
 

promtail-config.yaml

@@ -0,0 +1,81 @@
+import mlflow
+import torch
+from tqdm import tqdm
+import gc
+import time
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class TorchModelTrainMixin:
+    """
+    Mixin to use with BaseModelABC
+    """
+
+    checkpoint: str = ""
+    lr: float = 2e-5
+    device: torch.device
+
+    def _train_batch(self, x, y):
+        inputs = self.tokenizer(x, return_tensors="pt", truncation=True, padding=True)
+        inputs.to(self.device)
+        labels = y.to(self.device)
+        self.optimizer.zero_grad()
+        outputs = self.model(**inputs)
+        try:
+            loss = self.criterion(outputs.logits, labels)
+        except AttributeError:
+            loss = self.criterion(outputs, labels)
+        loss.backward()
+        self.optimizer.step()
+        logger.info(loss.item())
+        mlflow.log_metric("loss", loss.item())
+        mlflow.log_metric("time", time.time())
+        del inputs, labels, outputs, loss
+        gc.collect()
+        if torch.backends.mps.is_available():
+            torch.mps.empty_cache()
+        time.sleep(0.2)
+
+    def train(self):
+        self.init_mlflow()
+        self.model.train()
+        self.model.to(self.device)
+        for epoch in tqdm(range(self.epoch)):
+            for tweets, labels in tqdm(self.dataloader):
+                try:
+                    self._train_batch(tweets, labels.float())
+                except RuntimeError as e:
+                    logger.error(e)
+                    del tweets, labels, self.optimizer
+                    gc.collect()
+                    if torch.backends.mps.is_available():
+                        torch.mps.empty_cache()
+                    time.sleep(0.2)
+                    self.save()
+                    self.load_checkpoint()
+                    self.model.to(self.device)
+                    self.model.train()
+                    continue
+                if torch.backends.mps.is_available():
+                    logger.info(
+                        f"MPS allocated memory: {torch.mps.driver_allocated_memory()}"
+                    )
+                if torch.cuda.is_available():
+                    logger.info(
+                        f"CUDA allocated memory: {torch.cuda.memory_allocated()}"
+                    )
+            self.scheduler.step()
+        self.save()
+        super().train()
+
+
+class SklearnModelTrainMixin:
+    """
+    Mixin to use with BaseModelABC
+    """
+
+    def train(self):
+        self.model.fit(self.x_train, self.y_train)
+        super().train()