Minimal first version

.gitignore

@@ -0,0 +1,3 @@
+kind_cluster_config.yml
+conf_env.yml
+dispatcher_potgresql_secret.yml

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "cdci-data-analysis"]
+	path = cdci_data_analysis
+	url = https://github.com/oda-hub/dispatcher-app.git
+	branch = ivoa_helper-endpoint

Dockerfile

@@ -0,0 +1,32 @@
+FROM integralsw/osa-python:11.2-2-g667521a3-20220403-190332-refcat-43.0-heasoft-6.32.1-python-3.10.11
+
+ENV NB_USER=jovyan
+ENV NB_UID=1000
+ENV HOME=/home/${NB_USER}
+
+SHELL [ "bash", "-c" ]
+
+ENV DISPATCHER_CONFIG_FILE=/dispatcher/conf/conf.d/osa_data_server_conf.yml
+
+ADD requirements.txt /requirements.txt
+ADD cdci_data_analysis /cdci_data_analysis
+
+RUN yum -y install epel-release
+RUN yum -y update
+RUN yum -y install postgresql-devel
+
+RUN source /init.sh && \
+	pip install pip --upgrade && \
+    pip install -r /requirements.txt \
+                /cdci_data_analysis
+
+USER root
+RUN mkdir -pv $HOME
+RUN adduser --uid ${NB_UID} ${NB_USER}
+RUN chown -R ${NB_UID}:${NB_UID} ${HOME}
+
+USER ${NB_USER}
+WORKDIR ${HOME}
+
+ADD entrypoint.sh /dispatcher/entrypoint.sh
+ENTRYPOINT ["bash", "/dispatcher/entrypoint.sh"]

Dockerfile_postgresql

@@ -0,0 +1,39 @@
+# Use the official PostgreSQL 16.4 image as the base image
+FROM  bitnami/postgresql:16.4.0-debian-12-r28
+
+USER root
+
+RUN apt-get update && apt-get install -y vim \
+                                         build-essential \ 
+                                         software-properties-common \
+                                         git \
+                                         make \
+                                         gcc \
+                                         postgresql-server-dev-all \
+                                         bison \
+                                         byacc \
+                                         clang-13 \
+                                         sbcl \
+                                         curl \
+                                         gawk \ 
+                                         freetds-dev \
+                                         libzip-dev
+
+
+# Install the PostgreSQL extension for pgsphere, getting a specific version from a specific branch
+RUN git clone --branch aiprdbms16 https://github.com/kimakan/pgsphere.git /pgsphere
+
+RUN cd /pgsphere && \
+make USE_PGXS=1 PG_CONFIG=/opt/bitnami/postgresql/bin/pg_config
+RUN cd /pgsphere && \
+make USE_PGXS=1 PG_CONFIG=/opt/bitnami/postgresql/bin/pg_config install
+
+# Install the pgloader tool, to use it to load the data from a MySQL DB onto a Postgres DB
+RUN git clone https://github.com/dimitri/pgloader.git /pgloader
+
+RUN cd /pgloader && make save
+
+# add the pgloader binary to the PATH
+ENV PATH="/pgloader/build/bin:${PATH}"
+
+USER 1001

Makefile

@@ -0,0 +1,36 @@
+version ?= $(shell git describe --always)
+
+IMAGE=odahub/vo-interface:$(version)
+IMAGE_LATEST=odahub/vo-interface:latest
+
+DISPATCHER_CONFIG_DIR?="$(PWD)/conf"
+DISPATCHER_CONFIG_FILE?="/dispatcher/conf/conf.d/osa_data_server_conf.yml"
+
+run: build
+	docker run \
+		-it \
+		-v $(DISPATCHER_CONFIG_DIR):/dispatcher/conf/conf.d \
+		-e DISPATCHER_GUNICORN=yes \
+		-e DISPATCHER_CONFIG_FILE=$(DISPATCHER_CONFIG_FILE) \
+		--rm \
+		--name dev-oda-dispatcher \
+		$(IMAGE) 
+
+run-local: build
+		docker run \
+			-it \
+			-v $(DISPATCHER_CONFIG_DIR):/dispatcher/conf/conf.d \
+			-e DISPATCHER_GUNICORN=no \
+			-e DISPATCHER_CONFIG_FILE=$(DISPATCHER_CONFIG_FILE) \
+			--network=host \
+			--rm \
+			--name dev-oda-dispatcher \
+			$(IMAGE) 
+
+build: Dockerfile
+	docker build . -t $(IMAGE)
+	docker build . -t $(IMAGE_LATEST)
+
+build-no-cache: Dockerfile
+	docker build . --no-cache -t $(IMAGE)
+	docker build . --no-cache -t $(IMAGE_LATEST)

deployment_config.yaml

@@ -0,0 +1,77 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dev-oda-dispatcher
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: dev-oda-dispatcher
+  template:
+    metadata:
+      labels:
+        run: dev-oda-dispatcher
+    spec:
+      containers:
+      - name: dev-oda-dispatcher
+        image: odahub/vo-interface:latest
+        imagePullPolicy: Never  # or IfNotPresent
+        env:
+        - name: DISPATCHER_GUNICORN
+          value: "no"
+        - name: DISPATCHER_CONFIG_FILE
+          value: "/dispatcher/conf/conf.d/osa_data_server_conf.yml"
+        volumeMounts:
+        - mountPath: /dispatcher/conf/conf.d/osa_data_server_conf.yml
+          name: dispatcher-conf-secret
+          subPath: osa_data_server_conf.yml
+          readOnly: true
+      - name: dispatcher-postgresql
+        image: odahub/dispatcher-postgresql:latest
+        imagePullPolicy: Never  # or IfNotPresent
+        env:
+        - name: POSTGRES_USER
+          valueFrom:
+            secretKeyRef:
+              name: dispatcher-postgresql
+              key: postgres-user
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: dispatcher-postgresql
+              key: postgres-password
+        volumeMounts:
+        - name: postgres-data
+          mountPath: /var/lib/postgresql/data
+      volumes:
+      - name: dispatcher-conf-secret
+        secret:
+          secretName: dispatcher-conf
+      - name: postgres-data
+        persistentVolumeClaim:
+          claimName: dispatcher-postgresql-pvc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: dev-oda-dispatcher-service
+  labels:
+    run: dev-oda-dispatcher
+spec:
+  ports:
+  - port: 8001
+    targetPort: http
+    protocol: TCP
+  selector:
+    run: dev-oda-dispatcher
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: dispatcher-postgresql-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

entrypoint.sh

@@ -0,0 +1,46 @@
+export PYTHONUNBUFFERED=TRUE
+
+export XDG_CACHE_HOME=/tmp/xdg_cache_home
+mkdir -pv $XDG_CACHE_HOME/astropy
+
+ls -tlroa /var/log/containers/
+
+
+(
+    export HOME_OVERRRIDE=$PWD/runtime-home
+    if [ ! -d "$HOME_OVERRRIDE" ]; then
+        echo -e "\033[31mHOME_OVERRRIDE $HOME_OVERRRIDE to be created\033[0m"
+        mkdir -pv $HOME_OVERRRIDE
+    else
+        echo -e "\033[31mHOME_OVERRRIDE $HOME_OVERRRIDE already exists\033[0m"
+        ls -l $HOME_OVERRRIDE
+    fi
+
+    source /init.sh
+
+    # export TMPDIR=/scratch/tmpdir
+    # mkdir -pv TMPDIR
+
+    # python -c 'import xspec; print(xspec)'
+
+   # it might be better not to change directory, if jobs need to be preserved between restarts
+   # WORK_DIR=$PWD/$(date +%Y-%m-%d-%H-%M-%S)-$$
+   # mkdir -pv $WORK_DIR
+   # cd $WORK_DIR
+    echo -e "\033[31mDISPATCHER_CONFIG_FILE $DISPATCHER_CONFIG_FILE\033[0m"
+
+    if [ ${DISPATCHER_GUNICORN:-no} == "yes" ]; then
+        gunicorn \
+            "cdci_data_analysis.flask_app.app:conf_app(${DISPATCHER_CONFIG_FILE})" \
+            --bind 0.0.0.0:8000 \
+            --workers 8 \
+            --preload \
+            --timeout 900 \
+            --limit-request-line 0 \
+            --log-level debug
+    else
+        python /cdci_data_analysis/bin/run_osa_cdci_server.py \
+            -conf_file ${DISPATCHER_CONFIG_FILE} \
+            -debug
+    fi
+)

entrypoint_postgresql.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
+	CREATE DATABASE mmoda_pg_prod;
+    \c mmoda_pg_prod;
+    CREATE SCHEMA IF NOT EXISTS mmoda_pg_prod;
+    SET search_path = mmoda_pg_prod, public;
+    CREATE EXTENSION pg_sphere;
+EOSQL

init_postgres_db.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+set -e
+
+export PGPASSWORD=$POSTGRESQL_PASSWORD
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRESQL_USER" <<-EOSQL
+    DROP DATABASE IF EXISTS $POSTGRESQL_DB_NAME;
+	CREATE DATABASE $POSTGRESQL_DB_NAME;
+    \c $POSTGRESQL_DB_NAME;
+    CREATE SCHEMA IF NOT EXISTS $POSTGRESQL_DB_SCHEMA;
+    ALTER DATABASE $POSTGRESQL_DB_NAME SET search_path = $POSTGRESQL_DB_SCHEMA, public;
+    CREATE EXTENSION pg_sphere;
+EOSQL
+
+cd /pgloader && \
+    ./build/bin/pgloader mysql://$GALLERY_DB_USER:$GALLERY_DB_PASSWORD@$GALLERY_DB_HOST:$GALLERY_DB_PORT/$GALLERY_DB_NAME \
+                         postgresql://$POSTGRESQL_USER:$POSTGRESQL_PASSWORD@$POSTGRESQL_HOST:$POSTGRESQL_PORT/$POSTGRESQL_DB_NAME
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRESQL_USER" -d $POSTGRESQL_DB_NAME <<-EOSQL
+    -- Create the view
+    CREATE VIEW $POSTGRESQL_DB_SCHEMA.data_product_table_view_v AS
+    SELECT 
+        node_field_data.title AS title,
+        path_alias.alias AS path_alias,
+        path_alias.path AS path,
+        node__field_product_id.field_product_id_value AS product_id,
+        node__field_e1_kev.field_e1_kev_value AS e1_kev,
+        node__field_e2_kev.field_e2_kev_value AS e2_kev,
+        node__field_ra.field_ra_value AS ra,
+        node__field_dec.field_dec_value AS dec,
+        node__field_time_bin.field_time_bin_value AS time_bin,
+        tax1.name AS instrument_name,
+        tax1.description__value AS instrument_description_value,
+        tax2.name AS product_type_name,
+        tax2.description__value AS product_type_description_value,
+        node__field_timerange.field_timerange_value AS timerange,
+        node__field_timerange.field_timerange_end_value AS timerange_end,
+        string_agg(node__field_obsid.field_obsid_value::text, ', ') AS proposal_id,
+        string_agg(node__field_source_name.field_source_name_value::text, ', ') AS sources,
+        string_agg(SUBSTRING(f_m_1.uri FROM 10), ', ') AS file_uri,
+        string_agg(SUBSTRING(f_m_2.uri FROM 10), ', ') AS image_uri
+    FROM
+        node_field_data
+        LEFT JOIN node__field_time_bin ON node_field_data.nid = node__field_time_bin.entity_id
+        LEFT JOIN node__field_product_id ON node_field_data.nid = node__field_product_id.entity_id
+        LEFT JOIN node__field_e1_kev ON node_field_data.nid = node__field_e1_kev.entity_id
+        LEFT JOIN node__field_e2_kev ON node_field_data.nid = node__field_e2_kev.entity_id
+        LEFT JOIN node__field_ra ON node_field_data.nid = node__field_ra.entity_id
+        LEFT JOIN node__field_dec ON node_field_data.nid = node__field_dec.entity_id
+        LEFT JOIN node__field_instrumentused ON node_field_data.nid = node__field_instrumentused.entity_id
+        LEFT JOIN taxonomy_term_field_data tax1 ON node__field_instrumentused.field_instrumentused_target_id = tax1.tid
+        LEFT JOIN node__field_data_product_type ON node_field_data.nid = node__field_data_product_type.entity_id
+        LEFT JOIN taxonomy_term_field_data tax2 ON node__field_data_product_type.field_data_product_type_target_id = tax2.tid
+        LEFT JOIN node__field_derived_from_observation ON node_field_data.nid = node__field_derived_from_observation.entity_id
+        LEFT JOIN node_field_data obs ON obs.nid = node__field_derived_from_observation.field_derived_from_observation_target_id
+        LEFT JOIN node__field_timerange ON obs.nid = node__field_timerange.entity_id
+        LEFT JOIN node__field_obsid ON obs.nid = node__field_obsid.entity_id
+        LEFT JOIN node__field_describes_astro_entity ON node_field_data.nid = node__field_describes_astro_entity.entity_id
+        LEFT JOIN node_field_data sources ON sources.nid = node__field_describes_astro_entity.field_describes_astro_entity_target_id
+        LEFT JOIN node__field_source_name ON sources.nid = node__field_source_name.entity_id
+        LEFT JOIN path_alias ON path_alias.path::text LIKE ('/node/'::text || node_field_data.nid::text)
+        LEFT JOIN node__field_fits_file ON node_field_data.nid = node__field_fits_file.entity_id
+        LEFT JOIN file_managed AS f_m_1 ON node__field_fits_file.field_fits_file_target_id = f_m_1.fid
+        LEFT JOIN node__field_image_png ON node_field_data.nid = node__field_image_png.entity_id
+        LEFT JOIN file_managed AS f_m_2 ON node__field_image_png.field_image_png_target_id = f_m_2.fid
+    WHERE
+        node_field_data.status = '1'::smallint
+        AND node_field_data.type::text = 'data_product'::text
+        AND f_m_1.fid IS NOT NULL 
+        AND f_m_2.fid IS NOT NULL
+    GROUP BY 
+        node_field_data.nid, 
+        node_field_data.title, 
+        path_alias.alias, 
+        path_alias.path, 
+        node__field_product_id.field_product_id_value, 
+        node__field_e1_kev.field_e1_kev_value, 
+        node__field_e2_kev.field_e2_kev_value, 
+        node__field_ra.field_ra_value, 
+        node__field_dec.field_dec_value, 
+        node__field_time_bin.field_time_bin_value, 
+        tax1.name, 
+        tax1.description__value, 
+        tax2.name, 
+        tax2.description__value, 
+        node__field_timerange.field_timerange_value, 
+        node__field_timerange.field_timerange_end_value;
+EOSQL

requirements.txt

@@ -0,0 +1,3 @@
+flask>=2.0.3
+flask-restplus
+flask-restx>=0.2.0

setup_kind_cluster.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+cat <<EOF > kind_cluster_config.yml
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    extraMounts:
+      - hostPath: ${DISPATCHER_CONFIG_DIR}
+        containerPath: /dispatcher/conf/conf.d
+EOF
+
+kind create cluster --config kind_cluster_config.yml