Skip to content

Commit b0f7bf0

Browse files
fengmk2fengmk2
authored
chore: enable elasticsearch on local dev env (#757)
1 parent 02a1ee3 commit b0f7bf0

6 files changed

+205
-30
lines changed

‎.env.example

+41
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,44 @@
77
# CNPMCORE_DATABASE_USER=postgres
88
# CNPMCORE_DATABASE_PASSWORD=postgres
99
# CNPMCORE_DATABASE_NAME=cnpmcore
10+
11+
# CNPMCORE_CONFIG_ENABLE_ES=true
12+
# CNPMCORE_CONFIG_ES_CLIENT_NODE=http://localhost:9200
13+
# CNPMCORE_CONFIG_ES_CLIENT_AUTH_USERNAME=elastic
14+
# CNPMCORE_CONFIG_ES_CLIENT_AUTH_PASSWORD=abcdef
15+
16+
# https://github.com/cnpm/cnpmcore/blob/next/docs/elasticsearch-setup.md#%E6%96%B0%E5%BB%BA-env-%E6%96%87%E4%BB%B6
17+
# Password for the 'elastic' user (at least 6 characters)
18+
ELASTIC_PASSWORD="abcdef"
19+
20+
# Password for the 'kibana_system' user (at least 6 characters)
21+
KIBANA_PASSWORD="abcdef"
22+
23+
# Version of Elastic products
24+
STACK_VERSION=8.7.1
25+
# enable for arm64
26+
# STACK_VERSION_ARM64=-arm64
27+
# STACK_PLATFORM=linux/arm64
28+
29+
# Set the cluster name
30+
CLUSTER_NAME=docker-cluster
31+
32+
# Set to 'basic' or 'trial' to automatically start the 30-day trial
33+
LICENSE=basic
34+
#LICENSE=trial
35+
36+
# Port to expose Elasticsearch HTTP API to the host
37+
ES_PORT=9200
38+
#ES_PORT=127.0.0.1:9200
39+
40+
# Port to expose Kibana to the host
41+
KIBANA_PORT=5601
42+
#KIBANA_PORT=80
43+
44+
# Increase or decrease based on the available host memory (in bytes)
45+
ES_MEM_LIMIT=1073741824
46+
KB_MEM_LIMIT=1073741824
47+
LS_MEM_LIMIT=1073741824
48+
49+
# SAMPLE Predefined Key only to be used in POC environments
50+
ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2

‎docker-compose-es.yml

+142
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
name: cnpmcore_dev_services_es
2+
3+
volumes:
4+
certs:
5+
driver: local
6+
esdata01:
7+
driver: local
8+
kibanadata:
9+
driver: local
10+
11+
services:
12+
setup:
13+
image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
14+
volumes:
15+
- certs:/usr/share/elasticsearch/config/certs
16+
user: "0"
17+
command: >
18+
bash -c '
19+
if [ x${ELASTIC_PASSWORD} == x ]; then
20+
echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
21+
exit 1;
22+
elif [ x${KIBANA_PASSWORD} == x ]; then
23+
echo "Set the KIBANA_PASSWORD environment variable in the .env file";
24+
exit 1;
25+
fi;
26+
if [ ! -f config/certs/ca.zip ]; then
27+
echo "Creating CA";
28+
bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
29+
unzip config/certs/ca.zip -d config/certs;
30+
fi;
31+
if [ ! -f config/certs/certs.zip ]; then
32+
echo "Creating certs";
33+
echo -ne \
34+
"instances:\n"\
35+
" - name: es01\n"\
36+
" dns:\n"\
37+
" - es01\n"\
38+
" - localhost\n"\
39+
" ip:\n"\
40+
" - 127.0.0.1\n"\
41+
" - name: kibana\n"\
42+
" dns:\n"\
43+
" - kibana\n"\
44+
" - localhost\n"\
45+
" ip:\n"\
46+
" - 127.0.0.1\n"\
47+
> config/certs/instances.yml;
48+
bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
49+
unzip config/certs/certs.zip -d config/certs;
50+
fi;
51+
echo "Setting file permissions"
52+
chown -R root:root config/certs;
53+
find . -type d -exec chmod 750 \{\} \;;
54+
find . -type f -exec chmod 640 \{\} \;;
55+
echo "Waiting for Elasticsearch availability";
56+
until curl -s --cacert config/certs/ca/ca.crt http://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
57+
echo "Setting kibana_system password";
58+
until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" http://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
59+
echo "All done!";
60+
'
61+
healthcheck:
62+
test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
63+
interval: 1s
64+
timeout: 5s
65+
retries: 120
66+
67+
es01:
68+
depends_on:
69+
setup:
70+
condition: service_healthy
71+
image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}${STACK_VERSION_ARM64}
72+
platform: ${STACK_PLATFORM}
73+
labels:
74+
co.elastic.logs/module: elasticsearch
75+
volumes:
76+
- certs:/usr/share/elasticsearch/config/certs
77+
- esdata01:/usr/share/elasticsearch/data
78+
ports:
79+
- ${ES_PORT}:9200
80+
environment:
81+
- node.name=es01
82+
- cluster.name=${CLUSTER_NAME}
83+
- discovery.type=single-node
84+
- ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
85+
- bootstrap.memory_lock=true
86+
- xpack.security.enabled=true
87+
- xpack.security.http.ssl.enabled=false
88+
- xpack.security.http.ssl.key=certs/es01/es01.key
89+
- xpack.security.http.ssl.certificate=certs/es01/es01.crt
90+
- xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
91+
- xpack.security.transport.ssl.enabled=false
92+
- xpack.security.transport.ssl.key=certs/es01/es01.key
93+
- xpack.security.transport.ssl.certificate=certs/es01/es01.crt
94+
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
95+
- xpack.security.transport.ssl.verification_mode=certificate
96+
- xpack.license.self_generated.type=${LICENSE}
97+
mem_limit: ${ES_MEM_LIMIT}
98+
ulimits:
99+
memlock:
100+
soft: -1
101+
hard: -1
102+
healthcheck:
103+
test:
104+
[
105+
"CMD-SHELL",
106+
"curl -s --cacert config/certs/ca/ca.crt http://localhost:9200 | grep -q 'missing authentication credentials'",
107+
]
108+
interval: 10s
109+
timeout: 10s
110+
retries: 120
111+
112+
kibana:
113+
depends_on:
114+
es01:
115+
condition: service_healthy
116+
image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
117+
labels:
118+
co.elastic.logs/module: kibana
119+
volumes:
120+
- certs:/usr/share/kibana/config/certs
121+
- kibanadata:/usr/share/kibana/data
122+
ports:
123+
- ${KIBANA_PORT}:5601
124+
environment:
125+
- SERVERNAME=kibana
126+
- ELASTICSEARCH_HOSTS=http://es01:9200
127+
- ELASTICSEARCH_USERNAME=kibana_system
128+
- ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
129+
- ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
130+
- XPACK_SECURITY_ENCRYPTIONKEY=${ENCRYPTION_KEY}
131+
- XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY}
132+
- XPACK_REPORTING_ENCRYPTIONKEY=${ENCRYPTION_KEY}
133+
mem_limit: ${KB_MEM_LIMIT}
134+
healthcheck:
135+
test:
136+
[
137+
"CMD-SHELL",
138+
"curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
139+
]
140+
interval: 10s
141+
timeout: 10s
142+
retries: 120

‎docker-compose-postgres.yml

+10-10
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,15 @@
11
name: cnpmcore_dev_services_postgres
22

3+
volumes:
4+
cnpm-redis:
5+
cnpm-postgres:
6+
cnpm-pgadmin:
7+
8+
networks:
9+
cnpm-postgres:
10+
name: cnpm-postgres
11+
driver: bridge
12+
313
services:
414
redis:
515
env_file:
@@ -49,13 +59,3 @@ services:
4959
- cnpm-postgres
5060
depends_on:
5161
- postgres
52-
53-
volumes:
54-
cnpm-redis:
55-
cnpm-postgres:
56-
cnpm-pgadmin:
57-
58-
networks:
59-
cnpm-postgres:
60-
name: cnpm-postgres
61-
driver: bridge

‎docker-compose.yml

+7-15
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
name: cnpmcore_dev_services_mysql
22

3+
volumes:
4+
cnpm-redis:
5+
cnpm-mysql:
6+
7+
networks:
8+
cnpm-mysql:
9+
310
services:
411
redis:
512
env_file:
@@ -11,8 +18,6 @@ services:
1118
- cnpm-redis:/data
1219
ports:
1320
- 6379:6379
14-
networks:
15-
- cnpm-mysql
1621

1722
mysql:
1823
env_file:
@@ -28,8 +33,6 @@ services:
2833
- cnpm-mysql:/var/lib/mysql
2934
ports:
3035
- 3306:3306
31-
networks:
32-
- cnpm-mysql
3336

3437
# database explorer
3538
phpmyadmin:
@@ -46,16 +49,5 @@ services:
4649
PMA_HOST: 'mysql'
4750
ports:
4851
- 8080:80
49-
networks:
50-
- cnpm-mysql
5152
depends_on:
5253
- mysql
53-
54-
volumes:
55-
cnpm-redis:
56-
cnpm-mysql:
57-
58-
networks:
59-
cnpm-mysql:
60-
name: cnpm-mysql
61-
driver: bridge

‎docs/elasticsearch-setup.md

+4-5
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,6 @@ path.logs: ./logs
5858
新建文件 `docker-compose.yaml`， 复制如下的 `docker-compose.yaml`
5959

6060
```yaml
61-
version: "3.8"
62-
6361
volumes:
6462
certs:
6563
driver: local
@@ -142,6 +140,7 @@ services:
142140
- discovery.type=single-node
143141
- ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
144142
- bootstrap.memory_lock=true
143+
- bootstrap.system_call_filter=false
145144
- xpack.security.enabled=true
146145
- xpack.security.http.ssl.enabled=false
147146
- xpack.security.http.ssl.key=certs/es01/es01.key
@@ -252,7 +251,7 @@ docker compose up
252251

253252
### 访问 Elastic
254253

255-
浏览器打开 http://localhost:5601/app/dev_tools#/console，默认账号为 `elastic` 密码为 .env 文件中定义的 `abcdef`
254+
浏览器打开 http://localhost:5601/app/dev_tools#/console ，默认账号为 `elastic` 密码为 .env 文件中定义的 `abcdef`
256255

257256
## 创建索引
258257

@@ -262,7 +261,7 @@ ES 可以通过 Kibana devtool 进行数据的写入和查询操作。下面创
262261
PUT cnpmcore_packages
263262
{
264263
"settings": ${settings} // copy 下方 settings
265-
"mappings": ${mappings} // copy 下方 settings
264+
"mappings": ${mappings} // copy 下方 mappings
266265
}
267266
```
268267

@@ -975,7 +974,7 @@ config: {
975974
### 同步一条数据
976975

977976
```bash
978-
$ curl -X PUT https://r.cnpmjs.org/-/v1/search/sync/${pkgName}
977+
curl -X PUT http://localhost:7001/-/v1/search/sync/${pkgName}
979978
```
980979

981980
### 删除一条数据

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
}
3737
},
3838
"scripts": {
39+
"predev": "npm run clean",
3940
"dev": "egg-bin dev",
4041
"dev:postgresql": "CNPMCORE_DATABASE_TYPE=PostgreSQL egg-bin dev",
4142
"lint": "eslint --cache --ext .ts .",

0 commit comments

Comments
 (0)