add clf-vepac tools to materials-galaxy

charts/materials-galaxy/Chart.yaml

@@ -10,4 +10,4 @@ dependencies:
     repository: https://oauth2-proxy.github.io/manifests
     version: 7.12.13
 name: materials-galaxy
-version: 2.0.1
+version: 2.1.0

charts/materials-galaxy/README.md

@@ -56,24 +56,37 @@ We use [Init Containers](https://kubernetes.io/docs/concepts/workloads/pods/init
 
 To add a new set of galaxy tools - you can add a new init container definition under `extraInitContainers` like so:
 
-We recommend creating a container image to install all your tools instead of using git image like below
+We recommend using [git-sync](https://github.com/kubernetes/git-sync) as a initcontainer to sync github repos
 
 ```yaml
 galaxy:
   ...
   extraInitContainers:
     ...
-    - name: clone-my-tools # or name of your tools
+    - name: clone-my-tools
       applyToJob: false
-      applyToWeb: true # this should just apply to one pod - galaxy-web
+      applyToWeb: true
       applyToWorkflow: false
-      image: "alpine/git:latest" # or your setup image
-
-      # this is an example of how to clone your tools using git
-      command: ['sh', '-c', 'git clone https://github.com/me/my-tools.git --depth 1 --branch main {{.Values.persistence.mountPath}}/my-tools || true']
-      volumeMounts:
+      image: "registry.k8s.io/git-sync/git-sync:v4.4.0"
+      securityContext: # so we can read/write to galaxy-data volume
+        runAsGroup: 0
+        runAsUser: 0
+      volumeMounts:  
         - name: galaxy-data
           mountPath: "{{.Values.persistence.mountPath}}"
+      env:
+        - name: GITSYNC_ROOT
+          value: "{{.Values.persistence.mountPath}}/git-sync/my-tools" # what git-sync will use to track changes
+        - name: GITSYNC_LINK
+          value: "{{.Values.persistence.mountPath}}/tool-data/my-tools" # what materials-galaxy will see (always the latest changes) 
+        - name: GITSYNC_ONE_TIME
+          value: "true" 
+        - name: GITSYNC_DEPTH
+          value: "1"
+        - name: GITSYNC_REPO
+          value: https://github.com/MaterialsGalaxy/larch-tools.git
+        - name: GITSYNC_REF
+          value: main
 ```
 `{{.Values.persistence.mountPath}}` is a reference to the filepath for the mounted shared volume - same on all containers
 
@@ -82,7 +95,76 @@ Then edit `galaxy.configs.tool_conf.xml` to make it available to users - add a x
  ```xml
  <tool file="{{.Values.persistence.mountPath}}/my-tools/my-tool-1/my-tool-1.xml>
 ``` 
-where `file` is a filepath to the galaxy tool config you want to make available
+where `file` is a filepath to the galaxy tool config you want to make available 
+
+
+### Private repos
+
+To deploy tools from a private repo - you need to create a git deploy key for that repo so that we can access it
+more about deploy keys here - https://docs.github.com/en/authentication/connecting-to-github-with-ssh/managing-deploy-keys#deploy-keys
+Deploy keys are useful as they only grant access to a single repository, limiting attack vectors, additionally, we can set them to be read-only which is recommended for this use-case.
+
+Once you create a deploy key, you need to add it to the secrets file under - be careful not to publish this as plaintext
+```yaml
+gitRepos:
+  - name: repo-name
+    deployKey: |-
+    -----BEGIN OPENSSH PRIVATE KEY-----
+    ... private key content for repo1 ...
+    -----END OPENSSH PRIVATE KEY-----
+```
+
+then you'll need to create a init container to clone private repo like so:
+```yaml
+galaxy:
+  ...
+  extraInitContainers:
+  ...
+    - name: clone-my-tools
+      applyToJob: false
+      applyToWeb: true
+      applyToWorkflow: false
+      image: "registry.k8s.io/git-sync/git-sync:v4.4.0"
+      command:  # copies ssh-key from env variable into file, sets permissions and runs git-sync
+        - sh
+        - -c 
+        - |
+          mkdir -p /root/.ssh &&\ 
+          echo "${SSH_PRIVATE_KEY}" > /root/.ssh/id_rsa &&\
+          chmod 600 /root/.ssh/id_rsa &&\
+          exec /git-sync
+      securityContext:
+        runAsGroup: 0
+        runAsUser: 0
+      volumeMounts:
+        - name: galaxy-data
+          mountPath: "{{.Values.persistence.mountPath}}"
+      env: # grab the key from secret - this is pre-generated from config above
+        - name: SSH_PRIVATE_KEY
+          valueFrom:
+            secretKeyRef:
+              name: git-deploy-keys # name is hardcoded
+              key: clf-vepac-key
+        - name: GITSYNC_ROOT
+          value: "{{.Values.persistence.mountPath}}/git-sync/my-tools"
+        - name: GITSYNC_LINK
+          value: "{{.Values.persistence.mountPath}}/tool-data/my-tools"
+        - name: GITSYNC_ONE_TIME
+          value: "true"
+        - name: GITSYNC_DEPTH
+          value: "1"
+        - name: GITSYNC_REPO
+          value: [email protected]:user/my-private-repo.git
+        - name: GITSYNC_REF
+          value: dev
+        # add the following - its required and boilerplate
+        - name: GITSYNC_SSH 
+          value: "true"
+        - name: GITSYNC_SSH_KEY_FILE 
+          value: /root/.ssh/id_rsa
+        - name: GITSYNC_SSH_KNOWN_HOSTS
+          value: "false"
+```
 
 ## Configuring main page
 

charts/materials-galaxy/secret-values.yaml.template

@@ -33,4 +33,10 @@ galaxy:
         # comma spaced list of admin emails
         admin_users: "[email protected],[email protected]"
 
-
+  # any git repo deploy keys - to access private repos
+  gitRepos:
+    - name: first-repo
+      deployKey: |-
+        -----BEGIN OPENSSH PRIVATE KEY-----
+        ... private key content for repo1 ...
+        -----END OPENSSH PRIVATE KEY-----

charts/materials-galaxy/templates/repo-deploy-keys-secret.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: git-deploy-keys
+  namespace: {{ .Release.Namespace }}
+type: Opaque
+data:
+  {{- range $index, $repo := .Values.gitRepos }}
+  {{ $repo.name }}-key: {{ $repo.deployKey | b64enc }}
+  {{- end }}

charts/materials-galaxy/values.yaml

@@ -35,35 +35,6 @@ galaxy:
     downloadToolConfs:
       enabled: false
 
-  #- Allow users to specify extra init containers
-  # We use init containers to install tools which aren't available via galaxy toolshed
-  # TODO: find a better way to do this so we can change tool versions easily
-  extraInitContainers: 
-    - name: clone-muon-tools
-      applyToJob: false
-      applyToWeb: true
-      applyToWorkflow: false
-      image: "alpine/git:latest"
-      command: ['sh', '-c', 'git clone https://github.com/muon-spectroscopy-computational-project/muon-galaxy-tools.git --depth 1 --branch main {{.Values.persistence.mountPath}}/tool-data/muon-galaxy-tools || true']
-      volumeMounts:
-        - name: galaxy-data
-          mountPath: "{{.Values.persistence.mountPath}}"
-      securityContext:
-        runAsGroup: 0
-        runAsUser: 0
-    - name: clone-larch-tools
-      applyToJob: false
-      applyToWeb: true
-      applyToWorkflow: false
-      image: "alpine/git:latest"
-      command: ['sh', '-c', 'git clone https://github.com/MaterialsGalaxy/larch-tools.git --depth 1 --branch main {{.Values.persistence.mountPath}}/tool-data/larch-tools || true']
-      volumeMounts:
-        - name: galaxy-data
-          mountPath: "{{.Values.persistence.mountPath}}"
-      securityContext:
-        runAsGroup: 0
-        runAsUser: 0
-
   ingress:
     # used in galaxy configuration
     path: "/"
@@ -126,78 +97,6 @@ galaxy:
         # service info
         organization_name: "galaxy cloud cape"
         organization_url: null
-
-    integrated_tool_panel.xml: |-
-      <?xml version='1.0' encoding='utf-8'?>
-      <toolbox monitor="true">
-        <section id="get_data" name="Get Data">
-        </section>
-        <label id="muon_label" text="Muons" />
-        <section id="muspinsim" name="MuSpinSim">
-        </section>
-        <section id="muon_stopping_sites" name="Muon Stopping Sites">
-        </section>
-        <section id="muon_other" name="Other Muon Tools">
-        </section>
-        <label id="xas_label" text="xas" />
-        <label id="other_tools" text="Other Tools" />
-        <section id="file_conversion" name="File Conversion">
-        </section>
-        <section id="collection_operations" name="Collection Operations">
-        </section>
-      </toolbox>
-
-    tool_conf.xml: |-
-      <?xml version='1.0' encoding='utf-8'?>
-      <toolbox monitor="true">
-        <section id="get_data" name="Get Data">
-          <tool file="data_source/upload.xml" />
-        </section>
-        <label id="muon_label" text="Muons" />
-        <section id="muspinsim" name="MuSpinSim">
-          <tool file="{{.Values.persistence.mountPath}}/tool-data/muon-galaxy-tools/muspinsim_combine/muspinsim_combine.xml"/>
-          <tool file="{{.Values.persistence.mountPath}}/tool-data/muon-galaxy-tools/muspinsim_config/muspinsim_config.xml"/>
-          <tool file="{{.Values.persistence.mountPath}}/tool-data/muon-galaxy-tools/muspinsim_plot/muspinsim_plot.xml"/>
-          <tool file="{{.Values.persistence.mountPath}}/tool-data/muon-galaxy-tools/muspinsim/muspinsim.xml"/>
-        </section>
-        <section id="muon_stopping_sites" name="Muon Stopping Sites">
-          <tool file="{{.Values.persistence.mountPath}}/tool-data/muon-galaxy-tools/pm_muairss_read/pm_muairss_read.xml"/>
-          <tool file="{{.Values.persistence.mountPath}}/tool-data/muon-galaxy-tools/pm_uep_opt/pm_uep_opt.xml"/>
-          <tool file="{{.Values.persistence.mountPath}}/tool-data/muon-galaxy-tools/pm_symmetry/pm_symmetry.xml"/>
-        </section>
-        <section id="muon_other" name="Other Muon Tools">
-          <tool file="{{.Values.persistence.mountPath}}/tool-data/muon-galaxy-tools/mudirac/mudirac.xml"/>
-          <tool file="{{.Values.persistence.mountPath}}/tool-data/muon-galaxy-tools/pm_asephonons/pm_asephonons.xml"/>
-          <tool file="{{.Values.persistence.mountPath}}/tool-data/muon-galaxy-tools/pm_nq/pm_nq.xml"/>
-        </section>
-        <label id="xas_label" text="xas" />
-        <tool file="{{.Values.persistence.mountPath}}/tool-data/larch-tools/larch_select_paths/larch_select_paths.xml" />
-        <tool file="{{.Values.persistence.mountPath}}/tool-data/larch-tools/larch_plot/larch_plot.xml" />
-        <tool file="{{.Values.persistence.mountPath}}/tool-data/larch-tools/larch_athena/larch_athena.xml" />
-        <tool file="{{.Values.persistence.mountPath}}/tool-data/larch-tools/larch_artemis/larch_artemis.xml" />
-        <tool file="{{.Values.persistence.mountPath}}/tool-data/larch-tools/larch_feff/larch_feff.xml" />
-        <tool file="{{.Values.persistence.mountPath}}/tool-data/larch-tools/larch_lcf/larch_lcf.xml" />
-        <tool file="{{.Values.persistence.mountPath}}/tool-data/larch-tools/larch_criteria_report/larch_criteria_report.xml" />
-        <label id="other_tools" text="Other Tools" />
-        <section id="file_conversion" name="File Conversion">
-          <tool file="{{.Values.persistence.mountPath}}/tool-data/muon-galaxy-tools/cif2cell/cif2cell.xml" />
-        </section>
-        <section id="collection_operations" name="Collection Operations">
-          <tool file="${model_tools_path}/tool-data/unzip_collection.xml" />
-          <tool file="${model_tools_path}/tool-data/zip_collection.xml" />
-          <tool file="${model_tools_path}/tool-data/filter_failed_collection.xml" />
-          <tool file="${model_tools_path}/tool-data/filter_empty_collection.xml" />
-          <tool file="${model_tools_path}/tool-data/flatten_collection.xml" />
-          <tool file="${model_tools_path}/tool-data/merge_collection.xml" />
-          <tool file="${model_tools_path}/tool-data/relabel_from_file.xml" />
-          <tool file="${model_tools_path}/tool-data/filter_from_file.xml" />
-          <tool file="${model_tools_path}/tool-data/sort_collection_list.xml" />
-          <tool file="${model_tools_path}/tool-data/tag_collection_from_file.xml" />
-          <tool file="${model_tools_path}/tool-data/apply_rules.xml" />
-          <tool file="${model_tools_path}/tool-data/build_list.xml" />
-          <tool file="${model_tools_path}/tool-data/extract_dataset.xml" />
-        </section>
-      </toolbox>
 
   extraFileMappings:
     /galaxy/server/static/welcome.html: