Skip to content

Commit 2d0e15c

Browse files
general-kroll-4-lifegeneral-kroll-4-life
committed
simple-libpq-capture
Summary: - Crude `libpq`-based notice capture. - Support for regression testing. - Robot test working. - Manual test documented. - Robot testing contiunues crude integration with shared core resources. - Regression testing in CI.
1 parent 0d9993c commit 2d0e15c

14 files changed

Lines changed: 947 additions & 304 deletions

File tree

‎.github/workflows/regression.yml‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
name: Integration Testing and Analysis
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- main
7+
- develop
8+
- version*
9+
push:
10+
branches:
11+
- main
12+
- develop
13+
- version*
14+
tags:
15+
- robot*
16+
- regression*
17+
- integration*
18+
19+
env:
20+
GO_VERSION: '^1.22'
21+
STACKQL_CORE_REPOSITORY: ${{ vars.STACKQL_CORE_REPOSITORY != '' && vars.STACKQL_CORE_REPOSITORY || 'stackql/stackql' }}
22+
STACKQL_CORE_REF: ${{ vars.STACKQL_CORE_REF != '' && vars.STACKQL_CORE_REF || 'main' }}
23+
STACKQL_ANY_SDK_REPOSITORY: ${{ vars.STACKQL_ANY_SDK_REPOSITORY != '' && vars.STACKQL_ANY_SDK_REPOSITORY || 'stackql/any-sdk' }}
24+
STACKQL_ANY_SDK_REF: ${{ vars.STACKQL_ANY_SDK_REF != '' && vars.STACKQL_ANY_SDK_REF || 'main' }}
25+
26+
jobs:
27+
build-and-deploy:
28+
name: build-and-deploy
29+
runs-on: ubuntu-latest
30+
permissions:
31+
id-token: write
32+
contents: read
33+
env:
34+
AWS_DEFAULT_REGION: us-west-1
35+
REG_MAX_VERSIONS: 3
36+
REG_MAX_AGE_MONTHS: 6
37+
REG_WEBSITE_DIR: _deno_website
38+
REG_PROVIDER_PATH: providers/dist
39+
REG_ARTIFACT_REPO_BUCKET: stackql-registry-artifacts
40+
REG_DENO_DEPLOY_ASSET_REPO: deno-deploy-registry
41+
REG_DENO_DEPLOY_API_DEV: stackql-dev-registry
42+
REG_DENO_DEPLOY_API_PROD: stackql-registry
43+
44+
steps:
45+
- uses: actions/checkout@v4.1.1
46+
name: "[SETUP] checkout repo"
47+
with:
48+
fetch-depth: 0
49+
50+
- name: Set up Go
51+
uses: actions/setup-go@v5.0.0
52+
with:
53+
go-version: ${{ env.GO_VERSION }}
54+
check-latest: true
55+
cache: true
56+
id: go
57+
58+
- name: Build rust release artifact
59+
if: success()
60+
run: |
61+
cargo build --release --bin client_test_harness
62+
63+
- name: Download core
64+
uses: actions/checkout@v4.1.1
65+
with:
66+
repository: ${{ env.STACKQL_CORE_REPOSITORY }}
67+
ref: ${{ env.STACKQL_CORE_REF }}
68+
token: ${{ secrets.CI_STACKQL_PACKAGE_DOWNLOAD_TOKEN }}
69+
path: stackql-core
70+
71+
- name: Download any-sdk
72+
uses: actions/checkout@v4.1.1
73+
with:
74+
repository: ${{ env.STACKQL_ANY_SDK_REPOSITORY }}
75+
ref: ${{ env.STACKQL_ANY_SDK_REF }}
76+
token: ${{ secrets.CI_STACKQL_PACKAGE_DOWNLOAD_TOKEN }}
77+
path: stackql-any-sdk
78+
79+
- name: Setup Python
80+
uses: actions/setup-python@v5.0.0
81+
with:
82+
python-version: '3.11'
83+
84+
- name: Add dependencies
85+
working-directory: stackql-core
86+
run: |
87+
sudo apt-get install -y jq
88+
pip3 install -r cicd/requirements.txt
89+
90+
- name: Build stackql from core source
91+
working-directory: stackql-core
92+
run: |
93+
go get ./...
94+
python3 cicd/python/build.py --build
95+
96+
- name: Generate rewritten registry for simulations
97+
working-directory: stackql-core
98+
run: |
99+
python3 test/python/registry-rewrite.py
100+
101+
- name: Parse tag
102+
id: parse_tag
103+
run: |
104+
tag_obj="$(python3 stackql-core/cicd/python/tag_parse.py '${{ github.ref_name }}' --parse-registry-tag)"
105+
echo "tag_obj: $tag_obj"
106+
{
107+
echo "PARSED_TAG_IS_ROBOT=$(echo $tag_obj | jq -r '.is_robot')"
108+
echo "PARSED_TAG_IS_REGRESSION=$(echo $tag_obj | jq -r '.is_regression')"
109+
} | tee -a "$GITHUB_ENV"
110+
111+
112+
- name: Prepare load balancing materials
113+
working-directory: stackql-core
114+
run: |
115+
sudo cp /etc/hosts /etc/hosts.bak
116+
python3 test/python/tcp_lb.py --generate-hosts-entries | sudo tee -a /etc/hosts
117+
python3 test/python/tcp_lb.py --generate-nginx-lb > test/tcp/reverse-proxy/nginx/dynamic-sni-proxy.conf
118+
119+
120+
- name: Install and run nginx load balancer
121+
working-directory: stackql-core
122+
run: |
123+
sudo apt-get install -y curl gnupg2 ca-certificates lsb-release ubuntu-keyring
124+
curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor \
125+
| sudo tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null
126+
gpg --dry-run --quiet --no-keyring --import --import-options import-show /usr/share/keyrings/nginx-archive-keyring.gpg
127+
echo "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] \
128+
http://nginx.org/packages/ubuntu $(lsb_release -cs) nginx" \
129+
| sudo tee /etc/apt/sources.list.d/nginx.list
130+
sudo apt-get update
131+
sudo apt-get install nginx
132+
sudo nginx -c "$(pwd)/test/tcp/reverse-proxy/nginx/dynamic-sni-proxy.conf"
133+
134+
- name: Create materials for core tests
135+
working-directory: stackql-core
136+
run: |
137+
openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365
138+
openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365
139+
openssl req -x509 -keyout test/server/mtls/credentials/pg_rubbish_key.pem -out test/server/mtls/credentials/pg_rubbish_cert.pem -config test/server/mtls/openssl.cnf -days 365
140+
141+
- name: Start Core Test Mocks
142+
working-directory: stackql-core
143+
run: |
144+
pgrep -f flask | xargs kill -9 || true
145+
flask --app=./test/python/flask/github/app run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1093 &
146+
147+
- name: Run core robot functional tests
148+
if: success()
149+
run: |
150+
providerRoot="$(realpath $(pwd)/stackql-core/test/registry-mocked)"
151+
sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'"}'
152+
robot \
153+
--variable "${sundryCfg}" \
154+
--variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
155+
-d test/robot/reports \
156+
test/robot/functional
157+
158+
- name: Output from functional tests
159+
if: always()
160+
run: |
161+
cat test/robot/reports/output.xml
162+
163+
- name: Post core test cleanup
164+
if: always()
165+
run: |
166+
pgrep -f flask | xargs kill -9 || true
167+

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,8 @@ stackql*.zip
66
stackql*.pkg
77
stackql_history.txt
88
stackql.log
9+
stackql-core/
10+
stackql-any-sdk/
11+
stackql-stackql-provider-registry/
912
.env
1013
nohup.out

‎.vscode/launch.json‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{
2+
"configurations": [
3+
{
4+
"type": "lldb",
5+
"request": "launch",
6+
"name": "Debug Rust",
7+
"cargo": {
8+
"args": ["build", "--manifest-path", "${workspaceFolder}/Cargo.toml"]
9+
},
10+
"args": [
11+
"${input:queryStr}",
12+
"${input:dsnStr}"
13+
],
14+
"cwd": "${workspaceFolder}"
15+
}
16+
],
17+
"inputs": [
18+
{
19+
"type": "pickString",
20+
"id": "queryStr",
21+
"description": "Enter the name of the Rust executable",
22+
"options": [
23+
"SELECT repo, count(*) as has_starred FROM github.activity.repo_stargazers WHERE owner = 'stackql' and repo in ('stackql', 'stackql-deploy') and login = 'generalkroll0' GROUP BY repo;"
24+
]
25+
},
26+
27+
{
28+
"type": "pickString",
29+
"id": "dsnStr",
30+
"description": "Enter the name of the Rust executable",
31+
"options": [
32+
"host=localhost port=5444"
33+
]
34+
}
35+
]
36+
}

0 commit comments

Comments
 (0)