diff --git a/docker-compose-server.yml b/docker-compose-server.yml index 626b678a9..ab4cbffbc 100644 --- a/docker-compose-server.yml +++ b/docker-compose-server.yml @@ -22,7 +22,7 @@ services: - "6006:6006" volumes: # directory path should be updated as per user's environment - - /home/xxxx/cmf-server/data/tensorboard-logs:/logs + - $HOME/cmf-server/data/tensorboard-logs:/logs container_name: tensorboard server: image: server:latest diff --git a/kubernetes/docker/clean_uninstall.sh b/kubernetes/docker/clean_uninstall.sh new file mode 100755 index 000000000..4f7ba0443 --- /dev/null +++ b/kubernetes/docker/clean_uninstall.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# +#This is the script to perform clean docker removal including data. +#Backup import data first and run this script with caution. + + +sudo rm $HOME/neo4j -rf +sudo rm $HOME/minio -rf +sudo rm $HOME/cmf-server -rf + + +docker-compose -f docker-compose-neo4j.yml down +docker-compose -f docker-compose-minio.yml down +docker-compose -f ../../docker-compose-server.yml down diff --git a/kubernetes/docker/docker-compose-minio.yml b/kubernetes/docker/docker-compose-minio.yml new file mode 100644 index 000000000..46921b177 --- /dev/null +++ b/kubernetes/docker/docker-compose-minio.yml @@ -0,0 +1,20 @@ +version: '3.8' + +services: + minio: + image: minio/minio:RELEASE.2021-02-11T08-23-43Z + container_name: s3 + ports: + - "9000:9000" + command: minio server data/ + + aws: + image: amazon/aws-cli + container_name: aws-cli + command: -c "sleep 2 && aws --endpoint-url http://${IP}:9000 s3 mb s3://${bucket_name} --region eu-west-1 || exit 0" + entrypoint: [/bin/bash] + environment: + AWS_ACCESS_KEY_ID: ${access_key_id} + AWS_SECRET_ACCESS_KEY: ${secret_key} + depends_on: + - minio diff --git a/kubernetes/docker/docker-compose-neo4j.yml b/kubernetes/docker/docker-compose-neo4j.yml new file mode 100644 index 000000000..1c22fe787 --- /dev/null +++ b/kubernetes/docker/docker-compose-neo4j.yml @@ -0,0 +1,18 @@ +version: "3.8" + +services: + neo4j: + image: neo4j:latest + container_name: testneo4j + ports: + - "7474:7474" # HTTP port + - "7687:7687" # Bolt port + volumes: + - $HOME/neo4j/data:/data # Data storage + - $HOME/neo4j/logs:/logs # Log storage + - $HOME/neo4j/import:/var/lib/neo4j/import # Import folder + - $HOME/neo4j/plugins:/plugins # Plugins folder + environment: + - NEO4J_AUTH=${neo4j_user}/${neo4j_password} # Username and password + restart: unless-stopped + diff --git a/kubernetes/docker/setup_script.sh b/kubernetes/docker/setup_script.sh new file mode 100755 index 000000000..987b2263d --- /dev/null +++ b/kubernetes/docker/setup_script.sh @@ -0,0 +1,51 @@ +#!/bin/bash +### +# Copyright (2023) Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +### + +#setup cmf server + +export IP=$MYIP +export hostname="" + +original_docker_file="../../server/Dockerfile" +new_docker_file="../../server/Dockerfile_new" +cp "$original_docker_file" "$new_docker_file" +sed -i 's/--port, "80"/--port, "8080"/g' "$new_docker_file" + +original_compose_file="../../docker-compose-server.yml" +new_compose_file="../../docker-compose-server-new.yml" +cp "$original_compose_file" "$new_compose_file" +sed -i 's/8080:80/8080:8080/g' "$new_compose_file" +sed -i 's/dockerfile: ./server\/Dockerfile/dockerfile: ../server\/Dockerfile-new/g' "$new_compose_file" + +docker-compose --verbose -f $new_compose_file build +rm $new_docker_file +rm $new_compose_file + + +#setup minio server +export bucket_name="dvc-art" +export access_key_id="minioadmin" +export secret_key="minioadmin" + +docker-compose --verbose -f docker-compose-minio.yml build + +#setup neo4j serverA +# +export neo4j_user="neo4j" +export neo4j_password="test1234" + +docker-compose --verbose -f docker-compose-neo4j.yml build diff --git a/kubernetes/minio/aws.yaml b/kubernetes/minio/aws.yaml new file mode 100644 index 000000000..c0191f37b --- /dev/null +++ b/kubernetes/minio/aws.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: aws-cli +spec: + replicas: 1 + selector: + matchLabels: + app: aws-cli + template: + metadata: + labels: + app: aws-cli + spec: + containers: + - name: aws-cli + image: amazon/aws-cli + env: + - name: AWS_ACCESS_KEY_ID + value: "minioadmin" + - name: AWS_SECRET_ACCESS_KEY + value: "minioadmin" + command: ["/bin/sh", "-c"] + args: + - | + echo "Waiting for MinIO to be ready..." + until curl -s http://minio:9000/minio/health/live; do + sleep 3 + done + echo "MinIO is ready. Creating bucket dvc-art..." + aws --endpoint-url http://minio:9000 s3 mb s3://dvc-art --region eu-west-1 || exit 0 + restartPolicy: Always + diff --git a/kubernetes/minio/minio.yaml b/kubernetes/minio/minio.yaml new file mode 100644 index 000000000..7e8889944 --- /dev/null +++ b/kubernetes/minio/minio.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: minio/minio:RELEASE.2021-02-11T08-23-43Z + args: + - server + - /data + ports: + - containerPort: 9000 + env: + - name: MINIO_ROOT_USER + value: "minioadmin" + - name: MINIO_ROOT_PASSWORD + value: "minioadmin" + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: pvc-minio-data +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-minio-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: minio +spec: + type: LoadBalancer + ports: + - port: 9000 + targetPort: 9000 + selector: + app: minio + diff --git a/kubernetes/neo4j/neo4j-pv.yaml b/kubernetes/neo4j/neo4j-pv.yaml new file mode 100644 index 000000000..c9937d96d --- /dev/null +++ b/kubernetes/neo4j/neo4j-pv.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-neo4j-data +spec: + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /mnt/neo4j/data + +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-neo4j-logs +spec: + capacity: + storage: 5Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /mnt/neo4j/logs + +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-neo4j-import +spec: + capacity: + storage: 5Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /mnt/neo4j/import + +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-neo4j-plugins +spec: + capacity: + storage: 5Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /mnt/neo4j/plugins + diff --git a/kubernetes/neo4j/neo4j-pvc.yaml b/kubernetes/neo4j/neo4j-pvc.yaml new file mode 100644 index 000000000..47d070bf7 --- /dev/null +++ b/kubernetes/neo4j/neo4j-pvc.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-neo4j-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi # Should match the size of the corresponding PV + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-neo4j-logs +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi # Should match the size of the corresponding PV + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-neo4j-import +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi # Should match the size of the corresponding PV + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-neo4j-plugins +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi # Should match the size of the corresponding PV + diff --git a/kubernetes/neo4j/neo4j.yaml b/kubernetes/neo4j/neo4j.yaml new file mode 100644 index 000000000..d4c6a304b --- /dev/null +++ b/kubernetes/neo4j/neo4j.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: neo4j +spec: + replicas: 1 + selector: + matchLabels: + app: neo4j + template: + metadata: + labels: + app: neo4j + spec: + containers: + - name: neo4 + image: neo4j:latest + ports: + - containerPort: 7474 + - containerPort: 7687 + volumeMounts: + - name: data-volume + mountPath: /data + - name: logs-volume + mountPath: /logs + - name: import-volume + mountPath: /var/lib/neo4j/import + - name: plugins-volume + mountPath: /plugins + env: + - name: NEO4J_AUTH + value: "neo4j/test1234" + - name: NEO4J_dbms_connector_http_listen__address + value: "0.0.0.0:7474" + - name: NEO4J_dbms_connector_bolt_listen__address + value: "0.0.0.0:7687" + - name: NEO4J_server_config_strict__validation_enabled + value: "false" + volumes: + - name: data-volume + persistentVolumeClaim: + claimName: pvc-neo4j-data + - name: logs-volume + persistentVolumeClaim: + claimName: pvc-neo4j-logs + - name: import-volume + persistentVolumeClaim: + claimName: pvc-neo4j-import + - name: plugins-volume + persistentVolumeClaim: + claimName: pvc-neo4j-plugins +--- +apiVersion: v1 +kind: Service +metadata: + name: neo4j +spec: + type: LoadBalancer + ports: + - port: 7474 + targetPort: 7474 + name: http + - port: 7687 + targetPort: 7687 + name: bolt + selector: + app: neo4j + diff --git a/kubernetes/server/cmf-deployment.yaml b/kubernetes/server/cmf-deployment.yaml new file mode 100644 index 000000000..c4591fab9 --- /dev/null +++ b/kubernetes/server/cmf-deployment.yaml @@ -0,0 +1,72 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cmf-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: cmf + template: + metadata: + labels: + app: cmf + spec: + containers: + - name: cmf-server + image: localhost:5000/cmf-server:latest + ports: + - containerPort: 8080 # CMF server port + env: + - name: MYIP + value: "192.168.30.116" + volumeMounts: + - mountPath: /cmf-server/data + name: server-data + - mountPath: /cmf-server/data/static + name: server-static + - mountPath: /cmf-server/data/tensorboard-logs + name: tensorboard-logs + readinessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + # Add resources and volume mounts if needed + - name: cmf-ui + image: localhost:5000/cmf-ui:latest + ports: + - containerPort: 3000 # CMF UI port + # Add resources and volume mounts if needed + env: + - name: REACT_APP_MY_IP + value: "192.168.30.116" + - name: REACT_APP_MY_HOSTNAME + value: "" + volumes: + - name: server-data + persistentVolumeClaim: + claimName: pvc-server-data + - name: server-static + persistentVolumeClaim: + claimName: pvc-server-static + - name: tensorboard-logs + persistentVolumeClaim: + claimName: pvc-tensorboard-logs +--- +apiVersion: v1 +kind: Service +metadata: + name: cmf-service +spec: + type: LoadBalancer + ports: + - name: cmf-server-port + port: 8080 # External port for CMF Server access + targetPort: 8080 # CMF server container port + - name: cmf-ui-port + port: 3000 # External port for CMF UI access + targetPort: 3000 # CMF UI container port + selector: + app: cmf diff --git a/kubernetes/server/cmfserver-pv.yaml b/kubernetes/server/cmfserver-pv.yaml new file mode 100644 index 000000000..876ef5b33 --- /dev/null +++ b/kubernetes/server/cmfserver-pv.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-server-data +spec: + capacity: + storage: 40Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/mnt/cmf/data" +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-server-static +spec: + capacity: + storage: 20Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/mnt/cmf/data/static" +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-tensorboard-logs +spec: + capacity: + storage: 20Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/mnt/cmf/data/tensorboard-logs" diff --git a/kubernetes/server/cmfserver-pvc.yaml b/kubernetes/server/cmfserver-pvc.yaml new file mode 100644 index 000000000..cf80a0d35 --- /dev/null +++ b/kubernetes/server/cmfserver-pvc.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-server-data +spec: + storageClassName: "" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 40Gi + volumeName: pv-server-data +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-server-static +spec: + storageClassName: "" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + volumeName: pv-server-static +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-tensorboard-logs +spec: + storageClassName: "" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + volumeName: pv-tensorboard-logs + diff --git a/kubernetes/server/tensorboard.yaml b/kubernetes/server/tensorboard.yaml new file mode 100644 index 000000000..3c8da5a24 --- /dev/null +++ b/kubernetes/server/tensorboard.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tensorboard +spec: + replicas: 1 + selector: + matchLabels: + app: tensorboard + template: + metadata: + labels: + app: tensorboard + spec: + initContainers: + - name: wait-for-cmf-server + image: busybox + command: ['sh', '-c', 'until nc -z cmf-service 8080; do echo waiting for cmf-service; sleep 2; done;'] + containers: + - name: tensorboard + image: tensorflow/tensorflow + command: ["tensorboard", "--logdir", "/logs", "--host", "0.0.0.0"] + ports: + - containerPort: 6006 + volumeMounts: + - mountPath: /logs + name: tensorboard-logs + volumes: + - name: tensorboard-logs + persistentVolumeClaim: + claimName: pvc-tensorboard-logs +--- +apiVersion: v1 +kind: Service +metadata: + name: tensorboard-service +spec: + selector: + app: tensorboard + ports: + - protocol: TCP + port: 6006 + targetPort: 6006 + type: LoadBalancer