cmf kubernetes initial commit

docker-compose-server.yml

@@ -22,7 +22,7 @@ services:
       - "6006:6006"
     volumes:
     # directory path should be updated as per user's environment
-      - /home/xxxx/cmf-server/data/tensorboard-logs:/logs
+      - $HOME/cmf-server/data/tensorboard-logs:/logs
     container_name: tensorboard
   server:
     image: server:latest

kubernetes/docker/clean_uninstall.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+#
+#This is the script to perform clean docker removal including data. 
+#Backup import data first and run this script with caution.
+
+
+sudo rm $HOME/neo4j -rf
+sudo rm $HOME/minio -rf
+sudo rm $HOME/cmf-server -rf
+
+
+docker-compose -f docker-compose-neo4j.yml down
+docker-compose -f docker-compose-minio.yml down
+docker-compose -f ../../docker-compose-server.yml down

kubernetes/docker/docker-compose-minio.yml

@@ -0,0 +1,20 @@
+version: '3.8'
+
+services:
+  minio:
+    image: minio/minio:RELEASE.2021-02-11T08-23-43Z
+    container_name: s3
+    ports:
+      - "9000:9000"
+    command: minio server data/
+
+  aws:
+    image: amazon/aws-cli
+    container_name: aws-cli
+    command: -c "sleep 2 && aws --endpoint-url http://${IP}:9000 s3 mb s3://${bucket_name} --region eu-west-1 || exit 0"
+    entrypoint: [/bin/bash]
+    environment:
+      AWS_ACCESS_KEY_ID: ${access_key_id}
+      AWS_SECRET_ACCESS_KEY: ${secret_key}
+    depends_on:
+      - minio

kubernetes/docker/docker-compose-neo4j.yml

@@ -0,0 +1,18 @@
+version: "3.8"
+
+services:
+  neo4j:
+    image: neo4j:latest
+    container_name: testneo4j
+    ports:
+      - "7474:7474"   # HTTP port
+      - "7687:7687"   # Bolt port
+    volumes:
+      - $HOME/neo4j/data:/data        # Data storage
+      - $HOME/neo4j/logs:/logs        # Log storage
+      - $HOME/neo4j/import:/var/lib/neo4j/import   # Import folder
+      - $HOME/neo4j/plugins:/plugins  # Plugins folder
+    environment:
+      - NEO4J_AUTH=${neo4j_user}/${neo4j_password}     # Username and password
+    restart: unless-stopped
+

kubernetes/docker/setup_script.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+###
+# Copyright (2023) Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###
+
+#setup cmf server
+
+export IP=$MYIP
+export hostname=""
+
+original_docker_file="../../server/Dockerfile"
+new_docker_file="../../server/Dockerfile_new"
+cp "$original_docker_file" "$new_docker_file"
+sed -i 's/--port, "80"/--port, "8080"/g' "$new_docker_file"
+
+original_compose_file="../../docker-compose-server.yml"
+new_compose_file="../../docker-compose-server-new.yml"
+cp "$original_compose_file" "$new_compose_file"
+sed -i 's/8080:80/8080:8080/g' "$new_compose_file"
+sed -i 's/dockerfile: ./server\/Dockerfile/dockerfile: ../server\/Dockerfile-new/g' "$new_compose_file"
+
+docker-compose --verbose -f $new_compose_file build
+rm $new_docker_file
+rm $new_compose_file
+
+
+#setup minio server
+export bucket_name="dvc-art"
+export access_key_id="minioadmin"
+export secret_key="minioadmin"
+
+docker-compose --verbose -f docker-compose-minio.yml build
+
+#setup neo4j serverA
+#
+export neo4j_user="neo4j"
+export neo4j_password="test1234"
+
+docker-compose --verbose -f docker-compose-neo4j.yml build

kubernetes/minio/aws.yaml

@@ -0,0 +1,33 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: aws-cli
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: aws-cli
+  template:
+    metadata:
+      labels:
+        app: aws-cli
+    spec:
+      containers:
+      - name: aws-cli
+        image: amazon/aws-cli
+        env:
+        - name: AWS_ACCESS_KEY_ID
+          value: "minioadmin"
+        - name: AWS_SECRET_ACCESS_KEY
+          value: "minioadmin"
+        command: ["/bin/sh", "-c"]
+        args:
+        - |
+          echo "Waiting for MinIO to be ready..."
+          until curl -s http://minio:9000/minio/health/live; do
+            sleep 3
+          done
+          echo "MinIO is ready. Creating bucket dvc-art..."
+          aws --endpoint-url http://minio:9000 s3 mb s3://dvc-art --region eu-west-1 || exit 0
+      restartPolicy: Always
+

kubernetes/minio/minio.yaml

@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minio
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: minio
+  template:
+    metadata:
+      labels:
+        app: minio
+    spec:
+      containers:
+      - name: minio
+        image: minio/minio:RELEASE.2021-02-11T08-23-43Z
+        args:
+        - server
+        - /data
+        ports:
+        - containerPort: 9000
+        env:
+        - name: MINIO_ROOT_USER
+          value: "minioadmin"
+        - name: MINIO_ROOT_PASSWORD
+          value: "minioadmin"
+        volumeMounts:
+        - name: data
+          mountPath: /data
+      volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: pvc-minio-data
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pvc-minio-data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio
+spec:
+  type: LoadBalancer
+  ports:
+  - port: 9000
+    targetPort: 9000
+  selector:
+    app: minio
+

kubernetes/neo4j/neo4j-pv.yaml

@@ -0,0 +1,51 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: pv-neo4j-data
+spec:
+  capacity:
+    storage: 10Gi
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: /mnt/neo4j/data  
+
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: pv-neo4j-logs
+spec:
+  capacity:
+    storage: 5Gi  
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: /mnt/neo4j/logs 
+
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: pv-neo4j-import
+spec:
+  capacity:
+    storage: 5Gi  
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: /mnt/neo4j/import 
+
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: pv-neo4j-plugins
+spec:
+  capacity:
+    storage: 5Gi  
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: /mnt/neo4j/plugins  
+

kubernetes/neo4j/neo4j-pvc.yaml

@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pvc-neo4j-data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi  # Should match the size of the corresponding PV
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pvc-neo4j-logs
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi  # Should match the size of the corresponding PV
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pvc-neo4j-import
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi  # Should match the size of the corresponding PV
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pvc-neo4j-plugins
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi  # Should match the size of the corresponding PV
+

kubernetes/neo4j/neo4j.yaml

@@ -0,0 +1,68 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: neo4j
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: neo4j
+  template:
+    metadata:
+      labels:
+        app: neo4j
+    spec:
+      containers:
+      - name: neo4
+        image: neo4j:latest
+        ports:
+        - containerPort: 7474  
+        - containerPort: 7687  
+        volumeMounts:
+        - name: data-volume
+          mountPath: /data
+        - name: logs-volume
+          mountPath: /logs
+        - name: import-volume
+          mountPath: /var/lib/neo4j/import
+        - name: plugins-volume
+          mountPath: /plugins
+        env:
+        - name: NEO4J_AUTH
+          value: "neo4j/test1234"  
+        - name: NEO4J_dbms_connector_http_listen__address
+          value: "0.0.0.0:7474"  
+        - name: NEO4J_dbms_connector_bolt_listen__address
+          value: "0.0.0.0:7687" 
+        - name: NEO4J_server_config_strict__validation_enabled
+          value: "false"
+      volumes:
+      - name: data-volume
+        persistentVolumeClaim:
+          claimName: pvc-neo4j-data
+      - name: logs-volume
+        persistentVolumeClaim:
+          claimName: pvc-neo4j-logs
+      - name: import-volume
+        persistentVolumeClaim:
+          claimName: pvc-neo4j-import
+      - name: plugins-volume
+        persistentVolumeClaim:
+          claimName: pvc-neo4j-plugins
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: neo4j
+spec:
+  type: LoadBalancer
+  ports:
+  - port: 7474    
+    targetPort: 7474
+    name: http
+  - port: 7687  
+    targetPort: 7687
+    name: bolt
+  selector:
+    app: neo4j
+