[ENH] postgres monitor (#1093)

* wip: adding semantic search

* add pghero to docker compose

* add missing sql file

* replace store_pgsql to store-pgsql

* fix configuration

* change hardcoding into variables

* Revert "wip: adding semantic search"

This reverts commit 6b0d6bf.

Author: jdkent

.github/workflows/deploy-to-staging.yml

@@ -69,7 +69,7 @@ jobs:
           docker compose up -d
 
           # Wait for PostgreSQL to be ready
-          until docker compose exec store_pgsql pg_isready; do sleep 5; done
+          until docker compose exec store-pgsql pg_isready; do sleep 5; done
 
           docker compose exec neurostore flask db migrate
           docker compose exec neurostore flask db upgrade

.github/workflows/workflow.yml

@@ -18,7 +18,7 @@ jobs:
   clean:
     strategy:
       matrix:
-        package: ['neurosynth_compose', 'compose_nginx', 'compose_pgsql', 'compose_worker', 'neurostore', 'store_nginx', 'store_pgsql']
+        package: ['neurosynth_compose', 'compose_nginx', 'compose_pgsql', 'compose_worker', 'neurostore', 'store_nginx', 'store-pgsql']
     runs-on: ubuntu-latest
 
     steps:
@@ -126,10 +126,10 @@ jobs:
               store_nginx.cache-from=type=registry,ref=ghcr.io/${{ github.repository_owner }}/store_nginx:${{ hashFiles('**/store/nginx/**') }}
               store_nginx.cache-from=type=gha,scope=cached-store-nginx
               store_nginx.cache-to=type=gha,scope=cached-store-nginx,mode=max,ignore-error=true
-              store_pgsql.tags=ghcr.io/${{ github.repository_owner }}/store_pgsql:${{ hashFiles('**/store/postgres/**') }}
-              store_pgsql.cache-from=type=registry,ref=ghcr.io/${{ github.repository_owner }}/store_pgsql:${{ hashFiles('**/store/postgres/**') }}
-              store_pgsql.cache-from=type=gha,scope=cached-store-pgsql
-              store_pgsql.cache-to=type=gha,scope=cached-store-pgsql,mode=max,ignore-error=true
+              store-pgsql.tags=ghcr.io/${{ github.repository_owner }}/store-pgsql:${{ hashFiles('**/store/postgres/**') }}
+              store-pgsql.cache-from=type=registry,ref=ghcr.io/${{ github.repository_owner }}/store-pgsql:${{ hashFiles('**/store/postgres/**') }}
+              store-pgsql.cache-from=type=gha,scope=cached-store-pgsql
+              store-pgsql.cache-to=type=gha,scope=cached-store-pgsql,mode=max,ignore-error=true
 
 
   neurostore_backend_tests:
@@ -164,7 +164,7 @@ jobs:
           set: |
               neurostore.cache-from=type=gha,scope=cached-neurostore
               store_nginx.cache-from=type=gha,scope=cached-store-nginx
-              store_pgsql.cache-from=type=gha,scope=cached-store-pgsql
+              store-pgsql.cache-from=type=gha,scope=cached-store-pgsql
       - 
         name: spin up backend
         run: |
@@ -178,7 +178,7 @@ jobs:
         name: Create Test Database
         run: |
           docker compose exec -T \
-          store_pgsql \
+          store-pgsql \
           psql -U postgres -c "create database test_db"
       -
         name: Backend Tests
@@ -418,7 +418,7 @@ jobs:
         set: |
             neurostore.cache-from=type=gha,scope=cached-neurostore
             store_nginx.cache-from=type=gha,scope=cached-store-nginx
-            store_pgsql.cache-from=type=gha,scope=cached-store-pgsql
+            store-pgsql.cache-from=type=gha,scope=cached-store-pgsql
     -
       name: Spin up compose backend
       run: |
@@ -452,7 +452,7 @@ jobs:
         cd store
 
         docker compose exec -T \
-        store_pgsql \
+        store-pgsql \
         psql -U postgres -c "create database test_db"
     -
       name: Initialize Compose Database

store/.env.example

@@ -3,8 +3,10 @@ FLASK_APP=manage
 ACE_DIR=/media
 FILE_DIR=/tmp
 COMPOSE_CONVERT_WINDOWS_PATHS=1
-POSTGRES_HOST=store_pgsql
+POSTGRES_HOST=store-pgsql
+POSTGRES_USER=postgres
 POSTGRES_PASSWORD=example
+POSTGRES_DB=neurostore
 BEARERINFO_FUNC=neurostore.resources.auth.decode_token
 AUTH0_CLIENT_ID=YOUR_CLIENT_ID
 AUTH0_CLIENT_SECRET=YOUR_CLIENT_SECRET

store/backend/README.md

@@ -27,7 +27,7 @@ The server should now be running at http://localhost/
 
 Create the database for neurostore:
 
-    docker-compose exec store_pgsql psql -U postgres -c "create database neurostore"
+    docker-compose exec store-pgsql psql -U postgres -c "create database neurostore"
 
 Next, migrate and upgrade the database migrations.
 
@@ -64,11 +64,37 @@ If you need to upgrade the db after changing any models:
 ## Running tests
 To run tests, after starting services, create a test database:
 
-    docker-compose exec store_pgsql psql -U postgres -c "create database test_db"
+    docker-compose exec store-pgsql psql -U postgres -c "create database test_db"
 
 **NOTE**: This command will ask you for the postgres password which is defined
 in the `.env` file.
 
 and execute:
 
     docker-compose run -e "APP_SETTINGS=neurostore.config.DockerTestConfig" --rm -w /neurostore neurostore python -m pytest neurostore/tests
+
+## pgHero Service
+
+[pgHero](https://github.com/ankane/pghero) is a PostgreSQL monitoring tool that provides insights into database performance and queries through a web UI.
+
+### Starting pgHero with Docker Compose
+
+To start the pgHero service, ensure Docker Compose is set up and run:
+
+```sh
+docker-compose up -d pghero
+```
+
+### Accessing the pgHero Web UI
+
+Once running, access the pgHero dashboard at: [http://localhost/pghero](http://localhost/pghero) (replace "localhost" with your server's address as needed).
+
+The `/pghero` route is proxied by nginx to the pgHero service, so you do not need to specify a port.
+
+### Required Environment Variables
+
+Set the following environment variables (typically in your `.env` file):
+
+- `PGHERO_DATABASE_URL` — The connection string for your PostgreSQL database (e.g., `postgres://postgres:password@store-pgsql:5432/neurostore`).
+
+Refer to the `docker-compose.yml` for additional configuration options.

store/docker-compose.yml

@@ -7,7 +7,7 @@ services:
       context: ./backend
       dockerfile: ./Dockerfile
     depends_on:
-      store_pgsql:
+      store-pgsql:
         condition: service_healthy
       store_redis:
         condition: service_healthy
@@ -47,8 +47,8 @@ services:
       - VIRTUAL_HOST=${V_HOST}
       - LETSENCRYPT_HOST=${V_HOST}
 
-  store_pgsql:
-    image: store_pgsql
+  store-pgsql:
+    image: store-pgsql
     restart: always
     build: ./postgres
     # shm_size: '256MB'  # if I'm going to increase work_mem in postgres, also increase this
@@ -58,12 +58,26 @@ services:
       - '5432'
     env_file:
       - .env
-    command: ["postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"]  # Use the custom config
+    command: ["postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"]
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 3s
       timeout: 3s
       retries: 10
+    
+  pghero:
+    image: ankane/pghero
+    restart: always
+    ports:
+      - "8080:8080"
+    env_file:
+      - .env
+    environment:
+      DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${POSTGRES_DB}"
+      RAILS_RELATIVE_URL_ROOT: "/pghero"
+    depends_on:
+      store-pgsql:
+        condition: service_healthy
 
 volumes:
   postgres_data:

store/nginx/nginx.conf

@@ -93,5 +93,15 @@ http {
 			return 301 https://neurostuff.github.io/;
 		}
 
+		location /pghero/ {
+			proxy_pass http://pghero:8080/pghero/;
+			proxy_set_header Host $host;
+			proxy_set_header X-Real-IP $remote_addr;
+			proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+			proxy_set_header X-Forwarded-Proto $scheme;
+			proxy_http_version 1.1;
+			proxy_set_header Upgrade $http_upgrade;
+			proxy_set_header Connection "upgrade";
+		}
 	}
 }

store/postgres/Dockerfile

@@ -11,4 +11,5 @@ RUN dos2unix /home/pg_dump-to-s3.sh
 RUN dos2unix /home/s3-autodelete.sh
 
 COPY postgresql.conf /etc/postgresql/postgresql.conf
+COPY pg_stat_statements.sql /docker-entrypoint-initdb.d/
 

store/postgres/pg_stat_statements.sql

@@ -0,0 +1 @@
+CREATE EXTENSION IF NOT EXISTS pg_stat_statements;

store/postgres/postgresql.conf

@@ -1,3 +1,5 @@
 shared_buffers = 128MB
 work_mem = 8MB
 listen_addresses = '*'
+shared_preload_libraries = 'pg_stat_statements'
+pg_stat_statements.track = all