Skip to content

Commit 7e84236

Browse files
jdkentjdkent
authored
[MNT] upgrade postgres database (#1095)
* upgrade postgres database * update the build instructions * setup first stage of upgrade * update calls in workflow
1 parent 3bb3018 commit 7e84236

File tree

7 files changed

+90
-8
lines changed

7 files changed

+90
-8
lines changed

‎.github/workflows/workflow.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,10 @@ jobs:
7777
compose_pgsql.cache-from=type=registry,ref=ghcr.io/${{ github.repository_owner }}/compose_pgsql:${{ hashFiles('**/compose/postgres/**') }}
7878
compose_pgsql.cache-from=type=gha,scope=cached-compose-pgsql
7979
compose_pgsql.cache-to=type=gha,scope=cached-compose-pgsql,mode=max,ignore-error=true
80+
compose_pgsql17.tags=ghcr.io/${{ github.repository_owner }}/compose_pgsql17:${{ hashFiles('**/compose/postgres/**') }}
81+
compose_pgsql17.cache-from=type=registry,ref=ghcr.io/${{ github.repository_owner }}/compose_pgsql17:${{ hashFiles('**/compose/postgres/**') }}
82+
compose_pgsql17.cache-from=type=gha,scope=cached-compose-pgsql17
83+
compose_pgsql17.cache-to=type=gha,scope=cached-compose-pgsql17,mode=max,ignore-error=true
8084
compose_worker.tags=ghcr.io/${{ github.repository_owner }}/compose_worker:${{ hashFiles('**/compose/backend/neurosynth_compose/**') }}
8185
compose_worker.cache-from=type=registry,ref=ghcr.io/${{ github.repository_owner }}/compose_worker:${{ hashFiles('**/compose/backend/neurosynth_compose/**') }}
8286
compose_worker.cache-from=type=gha,scope=cached-compose-worker
@@ -130,6 +134,10 @@ jobs:
130134
store-pgsql.cache-from=type=registry,ref=ghcr.io/${{ github.repository_owner }}/store-pgsql:${{ hashFiles('**/store/postgres/**') }}
131135
store-pgsql.cache-from=type=gha,scope=cached-store-pgsql
132136
store-pgsql.cache-to=type=gha,scope=cached-store-pgsql,mode=max,ignore-error=true
137+
store-pgsql17.tags=ghcr.io/${{ github.repository_owner }}/store-pgsql17:${{ hashFiles('**/store/postgres/**') }}
138+
store-pgsql17.cache-from=type=registry,ref=ghcr.io/${{ github.repository_owner }}/store-pgsql17:${{ hashFiles('**/store/postgres/**') }}
139+
store-pgsql17.cache-from=type=gha,scope=cached-store-pgsql17
140+
store-pgsql17.cache-to=type=gha,scope=cached-store-pgsql17,mode=max,ignore-error=true
133141
134142
135143
neurostore_backend_tests:
@@ -165,6 +173,7 @@ jobs:
165173
neurostore.cache-from=type=gha,scope=cached-neurostore
166174
store_nginx.cache-from=type=gha,scope=cached-store-nginx
167175
store-pgsql.cache-from=type=gha,scope=cached-store-pgsql
176+
store-pgsql17.cache-from=type=gha,scope=cached-store-pgsql17
168177
-
169178
name: spin up backend
170179
run: |
@@ -233,6 +242,7 @@ jobs:
233242
compose.cache-from=type=gha,scope=cached-compose
234243
compose_nginx.cache-from=type=gha,scope=cached-compose-nginx
235244
compose_pgsql.cache-from=type=gha,scope=cached-compose-pgsql
245+
compose_pgsql17.cache-from=type=gha,scope=cached-compose-pgsql17
236246
compose_worker.cache-from=type=gha,scope=cached-compose-worker
237247
-
238248
name: Spin up backend
@@ -323,6 +333,7 @@ jobs:
323333
compose.cache-from=type=gha,scope=cached-compose
324334
compose_nginx.cache-from=type=gha,scope=cached-compose-nginx
325335
compose_pgsql.cache-from=type=gha,scope=cached-compose-pgsql
336+
compose_pgsql17.cache-from=type=gha,scope=cached-compose-pgsql17
326337
compose_worker.cache-from=type=gha,scope=cached-compose-worker
327338
-
328339
name: Spin up backend
@@ -405,6 +416,7 @@ jobs:
405416
compose.cache-from=type=gha,scope=cached-compose
406417
compose_nginx.cache-from=type=gha,scope=cached-compose-nginx
407418
compose_pgsql.cache-from=type=gha,scope=cached-compose-pgsql
419+
compose_pgsql17.cache-from=type=gha,scope=cached-compose-pgsql17
408420
compose_worker.cache-from=type=gha,scope=cached-compose-worker
409421
-
410422
name: load store images
@@ -419,6 +431,7 @@ jobs:
419431
neurostore.cache-from=type=gha,scope=cached-neurostore
420432
store_nginx.cache-from=type=gha,scope=cached-store-nginx
421433
store-pgsql.cache-from=type=gha,scope=cached-store-pgsql
434+
store-pgsql17.cache-from=type=gha,scope=cached-store-pgsql17
422435
-
423436
name: Spin up compose backend
424437
run: |

‎compose/docker-compose.yml‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,9 @@ services:
6363
compose_pgsql:
6464
image: compose_pgsql
6565
restart: always
66-
build: ./postgres
66+
build:
67+
context: ./postgres
68+
dockerfile: ./Dockerfile.old
6769
volumes:
6870
- postgres_data:/var/lib/postgresql/data
6971
expose:
@@ -75,6 +77,24 @@ services:
7577
interval: 3s
7678
timeout: 3s
7779
retries: 10
80+
81+
compose_pgsql17:
82+
image: compose_pgsql17
83+
restart: always
84+
build:
85+
context: ./postgres
86+
dockerfile: ./Dockerfile
87+
volumes:
88+
- postgres17_data:/var/lib/postgresql/data
89+
expose:
90+
- '5432'
91+
env_file:
92+
- .env
93+
healthcheck:
94+
test: ["CMD-SHELL", "pg_isready -U postgres"]
95+
interval: 3s
96+
timeout: 3s
97+
retries: 10
7898

7999
# compose_monitor:
80100
# image: compose_monitor
@@ -94,6 +114,7 @@ services:
94114

95115
volumes:
96116
postgres_data:
117+
postgres17_data:
97118

98119
networks:
99120
default:

‎compose/postgres/Dockerfile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
FROM postgres:12.12
1+
FROM postgres:17.6
22
RUN apt-get update && apt-get install -y dos2unix
3-
RUN apt-get install -yq python3-pip python-dev build-essential
3+
RUN apt-get install -yq python3-pip python-dev-is-python3 build-essential
44
RUN apt-get install -yq cron
5-
RUN pip3 install awscli
5+
RUN pip3 install --break-system-packages awscli
66
COPY pg_dump-to-s3 /home
77
RUN chmod +x /home/pg_dump-to-s3.sh /home/s3-autodelete.sh
88
RUN crontab /home/backup.txt

‎compose/postgres/Dockerfile.old‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
FROM postgres:12.12
2+
RUN apt-get update && apt-get install -y dos2unix
3+
RUN apt-get install -yq python3-pip python-dev build-essential
4+
RUN apt-get install -yq cron
5+
RUN pip3 install awscli
6+
COPY pg_dump-to-s3 /home
7+
RUN chmod +x /home/pg_dump-to-s3.sh /home/s3-autodelete.sh
8+
RUN crontab /home/backup.txt
9+
RUN service cron start
10+
RUN dos2unix /home/pg_dump-to-s3.sh
11+
RUN dos2unix /home/s3-autodelete.sh

‎store/docker-compose.yml‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,9 @@ services:
5050
store-pgsql:
5151
image: store-pgsql
5252
restart: always
53-
build: ./postgres
53+
build:
54+
context: ./postgres
55+
dockerfile: ./Dockerfile.old
5456
shm_size: '256MB'
5557
volumes:
5658
- postgres_data:/var/lib/postgresql/data
@@ -64,6 +66,26 @@ services:
6466
interval: 3s
6567
timeout: 3s
6668
retries: 10
69+
70+
store-pgsql17:
71+
image: store-pgsql17
72+
restart: always
73+
build:
74+
context: ./postgres
75+
dockerfile: ./Dockerfile
76+
shm_size: '256MB' # if I'm going to increase work_mem in postgres, also increase this
77+
volumes:
78+
- postgres17_data:/var/lib/postgresql/data
79+
expose:
80+
- '5432'
81+
env_file:
82+
- .env
83+
command: ["postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"]
84+
healthcheck:
85+
test: ["CMD-SHELL", "pg_isready -U postgres"]
86+
interval: 3s
87+
timeout: 3s
88+
retries: 10
6789

6890
pghero:
6991
image: ankane/pghero
@@ -104,6 +126,7 @@ services:
104126
volumes:
105127
postgres_data:
106128
grafana_data:
129+
postgres17_data:
107130

108131
networks:
109132
default:

‎store/postgres/Dockerfile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
FROM postgres:12.12
1+
FROM postgres:17.6
22
RUN apt-get update && apt-get install -y dos2unix
3-
RUN apt-get install -yq python3-pip python-dev build-essential
3+
RUN apt-get install -yq python3-pip python-dev-is-python3 build-essential
44
RUN apt-get install -yq cron
5-
RUN pip3 install awscli
5+
RUN pip3 install --break-system-packages awscli
66
COPY pg_dump-to-s3 /home
77
RUN chmod +x /home/pg_dump-to-s3.sh /home/s3-autodelete.sh
88
RUN crontab /home/backup.txt

‎store/postgres/Dockerfile.old‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
FROM postgres:12.12
2+
RUN apt-get update && apt-get install -y dos2unix
3+
RUN apt-get install -yq python3-pip python-dev build-essential
4+
RUN apt-get install -yq cron
5+
RUN pip3 install awscli
6+
COPY pg_dump-to-s3 /home
7+
RUN chmod +x /home/pg_dump-to-s3.sh /home/s3-autodelete.sh
8+
RUN crontab /home/backup.txt
9+
RUN service cron start
10+
RUN dos2unix /home/pg_dump-to-s3.sh
11+
RUN dos2unix /home/s3-autodelete.sh
12+
13+
COPY postgresql.conf /etc/postgresql/postgresql.conf
14+
COPY pg_stat_statements.sql /docker-entrypoint-initdb.d/

0 commit comments

Comments
 (0)