Merge pman (#149)

* Fix dev environment and add pman dependencies

* Refactor storage modules into their own package

* Add pman's code packages and modules

* Fix pfcon tests

* Make docker the default container environment

* Remove reference to pman

* Add pman's automated tests

* Integrate pman's README

* Use plain docker environment for CI instead of docker swarm

Author: jbernal0019

.gitattributes

@@ -17,15 +17,12 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
-      - name: Start Docker Swarm
-        run: docker swarm init --advertise-addr 127.0.0.1
       - name: Build and run tests
         run: ./make.sh -i
       - name: Tear down
         run: |
           ./unmake.sh
           sudo rm -fr ./CHRIS_REMOTE_FS
-          docker swarm leave --force
   test-cube:
     name: tests (CUBE)
     runs-on: ubuntu-24.04

.github/workflows/ci.yml

@@ -37,20 +37,28 @@ spec:
         app: pfcon
         env: development
     spec:
-      initContainers:
-        - name: init-pfcon
-          image: busybox:1.32
-          command: [ 'sh', '-c', "until wget --spider -S -T 2 http://pman:5010/api/v1/ 2>&1 | grep '200 OK'; do echo waiting for pman; done" ]
       containers:
         - image: localhost:5000/fnndsc/pfcon:dev
           name: pfcon
           stdin: true
           tty: true
           ports:
             - containerPort: 5005
+          # We need to mount a physical dir in the HOST onto the key store in pfcon. This dir
+          # is given by the STOREBASE env variable substitution. The keystore can be specified
+          # by the --storeBase flag during development.
+          # Since pfcon spins off containers of its own it also needs to mount storeBase dir
+          # (where it shares the data) into the spawned container. This directory is passed
+          # in the STOREBASE env variable.
           env:
             - name: APPLICATION_MODE
               value: development
+            - name: COMPUTE_VOLUME_TYPE
+            - value: host
+            - name: STOREBASE
+              value: ${STOREBASE}
+            - name: CONTAINER_ENV
+              value: kubernetes
           command: ["python"]
           args: ["-m", "pfcon"]
           volumeMounts:
@@ -70,60 +78,3 @@ spec:
         - name: "pfcon-tests"
           hostPath:
             path: ${SOURCEDIR}/tests
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  name: pman
-  labels:
-    app: pman
-    env: production
-spec:
-  selector:
-    app: pman
-    env: production
-  ports:
-    - port: 5010
-      targetPort: 5010
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: pman
-  labels:
-    app: pman
-    env: production
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: pman
-      env: production
-  template:
-    metadata:
-      name: pman
-      labels:
-        app: pman
-        env: production
-    spec:
-      containers:
-        - image: fnndsc/pman
-          name: pman
-          ports:
-            - containerPort: 5010
-          # Since pman spins off containers of its own it needs to mount storeBase dir
-          # (where pfcon shares the data) into the spawned container. This directory is
-          # passed in the STOREBASE env variable.
-          env:
-            - name: STORAGE_TYPE
-              value: host
-            - name: SECRET_KEY
-              value: "anysu^l=@pnsf!5piqz6!!5kdcdpo79y6jebbp+2244yjm*#+k"
-            - name: STOREBASE
-              value: ${STOREBASE}
-            - name: CONTAINER_ENV
-              value: kubernetes

README.rst

@@ -38,9 +38,6 @@ spec:
         env: development
     spec:
       initContainers:
-        - name: init-pfcon
-          image: busybox:1.32
-          command: [ 'sh', '-c', "until wget --spider -S -T 2 http://pman:5010/api/v1/ 2>&1 | grep '200 OK'; do echo waiting for pman; done" ]
         - name: init-swift
           image: busybox:1.32
           command: [ "sh", "-c", "until wget --spider -S -T 2 http://swift:8080/info 2>&1 | grep '200 OK'; do echo waiting for Swift storage; sleep2; done" ]
@@ -58,6 +55,12 @@ spec:
               value: true
             - name: STORAGE_ENV
               value: swift
+            - name: COMPUTE_VOLUME_TYPE
+            - value: ${COMPUTE_VOLUME_TYPE}
+            - name: STOREBASE
+              value: ${STOREBASE}
+            - name: CONTAINER_ENV
+              value: kubernetes
           command: ["python"]
           args: ["-m", "pfcon"]
           volumeMounts:
@@ -80,63 +83,6 @@ spec:
 
 ---
 
-apiVersion: v1
-kind: Service
-metadata:
-  name: pman
-  labels:
-    app: pman
-    env: production
-spec:
-  selector:
-    app: pman
-    env: production
-  ports:
-    - port: 5010
-      targetPort: 5010
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: pman
-  labels:
-    app: pman
-    env: production
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: pman
-      env: production
-  template:
-    metadata:
-      name: pman
-      labels:
-        app: pman
-        env: production
-    spec:
-      containers:
-        - image: fnndsc/pman
-          name: pman
-          ports:
-            - containerPort: 5010
-          # Since pman spins off containers of its own it needs to mount storeBase dir
-          # (where pfcon shares the data) into the spawned container. This directory is
-          # passed in the STOREBASE env variable.
-          env:
-            - name: STORAGE_TYPE
-              value: host
-            - name: SECRET_KEY
-              value: "anysu^l=@pnsf!5piqz6!!5kdcdpo79y6jebbp+2244yjm*#+k"
-            - name: STOREBASE
-              value: ${STOREBASE}
-            - name: CONTAINER_ENV
-              value: kubernetes
-
----
-
 apiVersion: v1
 kind: Service
 metadata:

kubernetes/pfcon_dev.yaml

@@ -37,10 +37,6 @@ spec:
         app: pfcon
         env: development
     spec:
-      initContainers:
-        - name: init-pfcon
-          image: busybox:1.32
-          command: [ 'sh', '-c', "until wget --spider -S -T 2 http://pman:5010/api/v1/ 2>&1 | grep '200 OK'; do echo waiting for pman; done" ]
       containers:
         - image: localhost:5000/fnndsc/pfcon:dev
           name: pfcon
@@ -54,7 +50,13 @@ spec:
             - name: PFCON_INNETWORK
               value: true
             - name: STORAGE_ENV
-              value: filesystem
+              value: ${STORAGE_ENV}
+            - name: COMPUTE_VOLUME_TYPE
+            - value: ${COMPUTE_VOLUME_TYPE}
+            - name: STOREBASE
+              value: ${STOREBASE}
+            - name: CONTAINER_ENV
+              value: kubernetes
           command: ["python"]
           args: ["-m", "pfcon"]
           volumeMounts:
@@ -74,60 +76,3 @@ spec:
         - name: "pfcon-tests"
           hostPath:
             path: ${SOURCEDIR}/tests
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  name: pman
-  labels:
-    app: pman
-    env: production
-spec:
-  selector:
-    app: pman
-    env: production
-  ports:
-    - port: 5010
-      targetPort: 5010
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: pman
-  labels:
-    app: pman
-    env: production
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: pman
-      env: production
-  template:
-    metadata:
-      name: pman
-      labels:
-        app: pman
-        env: production
-    spec:
-      containers:
-        - image: fnndsc/pman
-          name: pman
-          ports:
-            - containerPort: 5010
-          # Since pman spins off containers of its own it needs to mount storeBase dir
-          # (where pfcon shares the data) into the spawned container. This directory is
-          # passed in the STOREBASE env variable.
-          env:
-            - name: STORAGE_TYPE
-              value: host
-            - name: SECRET_KEY
-              value: "anysu^l=@pnsf!5piqz6!!5kdcdpo79y6jebbp+2244yjm*#+k"
-            - name: STOREBASE
-              value: ${STOREBASE}
-            - name: CONTAINER_ENV
-              value: kubernetes