Skip to content

Commit a3b24f6

Browse files
mighty-spongemighty-sponge
authored
Merge pull request #187 from anyproto/consolidate-any-sync-init-service
refactor: consolidate generateconfig services into single any-sync-init
2 parents 9f4e270 + 1c8e43b commit a3b24f6

8 files changed

Lines changed: 138 additions & 159 deletions

Dockerfile-generateconfig-anyconf renamed to Dockerfile-any-sync-init

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,18 @@ FROM ghcr.io/anyproto/any-sync-tools
44
ARG TARGETARCH
55

66
RUN set -eux; \
7-
apt-get update; \
8-
apt-get install -y --no-install-recommends \
7+
apt update; \
8+
apt install -y --no-install-recommends \
99
bash \
1010
curl \
11-
ca-certificates; \
11+
ca-certificates \
12+
perl \
13+
python3 \
14+
python3-yaml \
15+
python-is-python3; \
1216
curl -L "https://github.com/mikefarah/yq/releases/latest/download/yq_linux_${TARGETARCH}" \
1317
-o /usr/local/bin/yq; \
1418
chmod +x /usr/local/bin/yq; \
1519
rm -rf /var/lib/apt/lists/*
1620
WORKDIR /code
17-
ENTRYPOINT ["bash", "/code/docker-generateconfig/anyconf.sh"]
21+
ENTRYPOINT ["bash", "/code/docker-generateconfig/init.sh"]

‎Dockerfile-generateconfig-env‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
# syntax=docker/dockerfile:1
2-
FROM python:3.11-alpine
2+
FROM python:3.14-alpine
33
WORKDIR /code
4-
COPY docker-generateconfig/env-requirements.txt requirements.txt
5-
RUN pip install -r requirements.txt
4+
RUN pip install requests==2.32.5
65
ENTRYPOINT ["python", "/code/docker-generateconfig/env.py"]

‎Dockerfile-generateconfig-processing‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎docker-compose.yml‎

Lines changed: 7 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,10 @@
11
services:
2-
# generate configs using the anyconf utility
3-
generateconfig-anyconf:
2+
# generate and process any-sync configs
3+
any-sync-init:
44
restart: "no"
55
build:
66
context: .
7-
dockerfile: Dockerfile-generateconfig-anyconf
8-
volumes:
9-
- ./.env:/code/.env:ro,Z
10-
- ./docker-generateconfig:/code/docker-generateconfig:ro,Z
11-
- "${STORAGE_DIR}:/code/storage:Z"
12-
13-
# processing any-sync-* configs
14-
generateconfig-processing:
15-
restart: "no"
16-
depends_on:
17-
generateconfig-anyconf:
18-
condition: service_completed_successfully
19-
build:
20-
context: .
21-
dockerfile: Dockerfile-generateconfig-processing
7+
dockerfile: Dockerfile-any-sync-init
228
volumes:
239
- ./.env:/code/.env:ro,Z
2410
- ./docker-generateconfig:/code/docker-generateconfig:ro,Z
@@ -27,7 +13,7 @@ services:
2713

2814
mongo-1:
2915
depends_on:
30-
generateconfig-processing:
16+
any-sync-init:
3117
condition: service_completed_successfully
3218
image: "mongo:${MONGO_VERSION}"
3319
restart: unless-stopped
@@ -43,7 +29,7 @@ services:
4329

4430
redis:
4531
depends_on:
46-
generateconfig-processing:
32+
any-sync-init:
4733
condition: service_completed_successfully
4834
image: "redis/redis-stack-server:${REDIS_VERSION}"
4935
restart: unless-stopped
@@ -94,7 +80,7 @@ services:
9480
any-sync-coordinator_bootstrap:
9581
image: "ghcr.io/anyproto/any-sync-coordinator:${ANY_SYNC_COORDINATOR_VERSION}"
9682
depends_on:
97-
generateconfig-processing:
83+
any-sync-init:
9884
condition: service_completed_successfully
9985
mongo-1:
10086
condition: service_healthy
@@ -106,7 +92,7 @@ services:
10692
any-sync-coordinator:
10793
image: "ghcr.io/anyproto/any-sync-coordinator:${ANY_SYNC_COORDINATOR_VERSION}"
10894
depends_on:
109-
generateconfig-processing:
95+
any-sync-init:
11096
condition: service_completed_successfully
11197
mongo-1:
11298
condition: service_healthy

‎docker-generateconfig/anyconf.sh‎

Lines changed: 0 additions & 58 deletions
This file was deleted.

‎docker-generateconfig/env-requirements.txt‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎docker-generateconfig/init.sh‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
#!/bin/bash
2+
set -e
3+
4+
echo "INFO: loading .env file"
5+
source .env
6+
7+
# =============================================================================
8+
echo "INFO: === Phase 1: anyconf ==="
9+
# =============================================================================
10+
11+
echo "INFO: create persistent config dir='./storage/docker-generateconfig'"
12+
install -d ./storage/docker-generateconfig
13+
14+
(
15+
cd ./storage/docker-generateconfig
16+
17+
if [[ -s .networkId ]]; then
18+
echo "INFO: saved networkId found, skipping"
19+
else
20+
echo "INFO: saved networkId not found, creating"
21+
anyconf create-network
22+
grep '^networkId:' nodes.yml | awk '{print $NF}' > .networkId
23+
yq '.account.signingKey' account.yml > .networkSigningKey
24+
fi
25+
NETWORK_ID=$(cat .networkId)
26+
NETWORK_SIGNING_KEY=$(cat .networkSigningKey)
27+
28+
if [[ -s account0.yml ]]; then
29+
echo "INFO: saved nodes and accounts configuration found, skipping"
30+
else
31+
echo "INFO: saved nodes and accounts not found, creating"
32+
anyconf generate-nodes \
33+
--t tree \
34+
--t tree \
35+
--t tree \
36+
--t coordinator \
37+
--t file \
38+
--t consensus \
39+
--addresses ${ANY_SYNC_NODE_1_ADDRESSES} \
40+
--addresses ${ANY_SYNC_NODE_2_ADDRESSES} \
41+
--addresses ${ANY_SYNC_NODE_3_ADDRESSES} \
42+
--addresses ${ANY_SYNC_COORDINATOR_ADDRESSES} \
43+
--addresses ${ANY_SYNC_FILENODE_ADDRESSES} \
44+
--addresses ${ANY_SYNC_CONSENSUSNODE_ADDRESSES}
45+
fi
46+
47+
echo "INFO: yq processing yml files"
48+
yq --indent 2 --inplace 'del(.creationTime)' nodes.yml
49+
yq --indent 2 --inplace ".networkId |= \"${NETWORK_ID}\"" nodes.yml
50+
yq --indent 2 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account3.yml
51+
yq --indent 2 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account5.yml
52+
)
53+
54+
# =============================================================================
55+
echo "INFO: === Phase 2: processing ==="
56+
# =============================================================================
57+
58+
DEST_PATH="./etc"
59+
NETWORK_FILE="./storage/docker-generateconfig/network.yml"
60+
61+
echo "INFO: Create directories for all node types"
62+
for NODE_TYPE in node-1 node-2 node-3 filenode coordinator consensusnode; do
63+
mkdir -p "${DEST_PATH}/any-sync-${NODE_TYPE}"
64+
done
65+
66+
echo "INFO: Create directory for S3 credentials"
67+
mkdir -p "${DEST_PATH}/.aws"
68+
69+
echo "INFO: Configure external listen host"
70+
python ./docker-generateconfig/setListenIp.py \
71+
"./storage/docker-generateconfig/nodes.yml" \
72+
"./storage/docker-generateconfig/nodesProcessed.yml"
73+
74+
echo "INFO: Generate client.yml"
75+
cp "./storage/docker-generateconfig/nodesProcessed.yml" "${DEST_PATH}/client.yml"
76+
77+
echo "INFO: Generate network file"
78+
yq eval '. as $item | {"network": $item}' --indent 2 \
79+
./storage/docker-generateconfig/nodesProcessed.yml > "${NETWORK_FILE}"
80+
81+
echo "INFO: Generate config files for any-sync-nodes - x3"
82+
for i in {0..2}; do
83+
cat \
84+
"${NETWORK_FILE}" \
85+
docker-generateconfig/etc/common.yml \
86+
storage/docker-generateconfig/account${i}.yml \
87+
docker-generateconfig/etc/node-$((i+1)).yml \
88+
> "${DEST_PATH}/any-sync-node-$((i+1))/config.yml"
89+
done
90+
91+
echo "INFO: Generate config files for coordinator, filenode, consensusnode"
92+
declare -A SERVICE_ACCOUNTS=([coordinator]=3 [filenode]=4 [consensusnode]=5)
93+
for SERVICE in coordinator filenode consensusnode; do
94+
cat \
95+
"${NETWORK_FILE}" \
96+
docker-generateconfig/etc/common.yml \
97+
storage/docker-generateconfig/account${SERVICE_ACCOUNTS[$SERVICE]}.yml \
98+
docker-generateconfig/etc/${SERVICE}.yml \
99+
> "${DEST_PATH}/any-sync-${SERVICE}/config.yml"
100+
done
101+
102+
echo "INFO: Copy network file to coordinator directory"
103+
cp "storage/docker-generateconfig/nodesProcessed.yml" "${DEST_PATH}/any-sync-coordinator/network.yml"
104+
105+
echo "INFO: Copy S3 credentials config"
106+
cp "docker-generateconfig/etc/aws-credentials" "${DEST_PATH}/.aws/credentials"
107+
108+
echo "INFO: Replace variables from .env file"
109+
for PLACEHOLDER in $(perl -ne 'print "$1\n" if /^([A-z0-9_-]+)=/' .env); do
110+
perl -i -pe "s|%${PLACEHOLDER}%|${!PLACEHOLDER}|g" \
111+
"${DEST_PATH}/.aws/credentials" \
112+
"${NETWORK_FILE}" \
113+
"${DEST_PATH}"/*/*.yml
114+
done
115+
116+
echo "INFO: fix indent in yml files"
117+
for FILE in $(find ${DEST_PATH}/ -name "*.yml"); do
118+
yq --inplace --indent=2 "${FILE}"
119+
done
120+
121+
echo "INFO: === any-sync-init done ==="

‎docker-generateconfig/processing.sh‎

Lines changed: 0 additions & 67 deletions
This file was deleted.

0 commit comments

Comments
 (0)