Skip to content

Commit 30c6f59

Browse files
st1020st1020
authored
Merge pull request #1893 from HXSecurity/beta
Beta to Main
2 parents ec9ae25 + a7050a8 commit 30c6f59

File tree

81 files changed

+3671
-1008
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

81 files changed

+3671
-1008
lines changed

‎.github/actions/setup-python/action.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: Setup Python
2+
description: Setup Python
3+
4+
inputs:
5+
python-version:
6+
description: Python version
7+
required: false
8+
default: "3.10"
9+
10+
runs:
11+
using: "composite"
12+
steps:
13+
- uses: actions/setup-python@v4
14+
with:
15+
python-version: ${{ inputs.python-version }}
16+
cache: "pip"
17+
18+
- name: Install dependencies
19+
shell: bash
20+
run: |
21+
sudo apt-get install build-essential cmake ninja-build python3-dev cython3 pybind11-dev libre2-dev libsasl2-dev python3-dev libldap2-dev libssl-dev -y
22+
python -m pip install --upgrade pip
23+
pip install wheel maturin
24+
pip install -r requirements.txt
25+
curl -L https://github.com/HXSecurity/tantivy-py/releases/download/0.21.0/tantivy-0.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -o /tmp/tantivy-0.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
26+
pip install /tmp/tantivy-0.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

‎.github/workflows/deploy-dev.yaml‎

Lines changed: 15 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -37,33 +37,24 @@ jobs:
3737
- name: Login to Aliyun Registry
3838
uses: docker/login-action@v1
3939
with:
40-
registry: ${{ secrets.ALIYUN_REGISTRY_HONGKONG }}
41-
username: ${{ secrets.ALIYUN_DOCKERHUB_USER }}
42-
password: ${{ secrets.ALIYUN_DOCKERHUB_PASSWORD }}
40+
registry: ${{ secrets.TEN_REGISTRY }}
41+
username: ${{ secrets.TEN_DOCKERHUB_USER }}
42+
password: ${{ secrets.TEN_DOCKERHUB_PASSWORD }}
4343

4444
- name: Generate version file
4545
run: |
4646
#bash .github/workflows/config_update.sh
4747
#bash .github/workflows/config_update.sh "${{ steps.release.outputs.VERSION }}"
4848
49-
- name: Setup Ossutil
50-
uses: manyuanrong/[email protected]
49+
- name: Upload COS
50+
if: ${{ steps.version.outputs.GITHUB_REF }} == develop
51+
uses: zkqiang/[email protected]
5152
with:
52-
endpoint: ${{ secrets.CHART_OSS_ENDPOINT }}
53-
access-key-id: ${{ secrets.CHART_OSS_ACCESS_KEY_ID }}
54-
access-key-secret: ${{ secrets.CHART_OSS_ACCESS_KEY_SECRET }}
55-
56-
- name: Download Agent dev
57-
# if: ${{ steps.version.outputs.GITHUB_REF }} == develop || ${{ steps.version.outputs.GITHUB_REF }} == beta || ${{ steps.version.outputs.GITHUB_REF }} == main
58-
run: |
59-
if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then ossutil cp oss://dongtai-helm-charts/agent_test/java/latest/ ./ --include "*.jar" -r
60-
else ossutil cp oss://dongtai-helm-charts/agent_${{ steps.version.outputs.GITHUB_REF }}/java/latest/ ./ --include "*.jar" -r
61-
fi
62-
63-
- name: Download Agent ${{ inputs.agent_version }}
64-
if: ${{ inputs.agent_version }}
65-
run: |
66-
ossutil cp oss://dongtai/agent/java/${{ inputs.agent_version }}/ ./ --include "*.jar" -r
53+
args: download -rs /agent_test/java/latest/ ./ --include "*.jar"
54+
secret_id: ${{ secrets.TENSECRET_ID }}
55+
secret_key: ${{ secrets.TENSECRET_KEY }}
56+
bucket: dongtai-helm-charts-1251882848
57+
region: ap-hongkong
6758

6859
- name: Generate version file
6960
run: |
@@ -76,12 +67,12 @@ jobs:
7667
context: .
7768
push: true
7869
platforms: linux/amd64
79-
tags: |
80-
registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server:${{ steps.version.outputs.GITHUB_REF }}-latest
81-
registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server:${{ steps.version.outputs.GITHUB_REF }}-1.0.${{github.run_number}}
70+
tags: |
71+
wukong.tencentcloudcr.com/dongtai/dongtai-server:${{ steps.version.outputs.GITHUB_REF }}-latest
72+
wukong.tencentcloudcr.com/dongtai/dongtai-server:${{ steps.version.outputs.GITHUB_REF }}-1.0.${{github.run_number}}
8273
8374
- name: deploy to cluster
84-
uses: tscuite/kubectl-helm-action@main
75+
uses: HXSecurity/kubectl-helm-action@main
8576
env:
8677
MAX: false
8778
PROJECT: server

‎.github/workflows/nightly.yaml‎

Lines changed: 5 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -37,18 +37,11 @@ jobs:
3737
steps:
3838
- name: Checkout
3939
uses: actions/checkout@v2
40-
- uses: actions/setup-python@v4
41-
with:
42-
python-version: "3.10"
43-
cache: "pip"
40+
- name: Setup Python
41+
uses: ./.github/actions/setup-python
4442

4543
- name: Install dependencies
46-
run: |
47-
sudo apt-get install build-essential cmake ninja-build python3-dev cython3 pybind11-dev libre2-dev -y
48-
python -m pip install --upgrade pip
49-
pip install wheel
50-
pip install coverage
51-
pip install -r requirements.txt
44+
run: pip install coverage
5245

5346
- name: Django Unit Testing
5447
run: |
@@ -92,17 +85,11 @@ jobs:
9285
steps:
9386
- name: Checkout
9487
uses: actions/checkout@v2
95-
- uses: actions/setup-python@v4
96-
with:
97-
python-version: "3.10"
98-
cache: "pip"
88+
- name: Setup Python
89+
uses: ./.github/actions/setup-python
9990

10091
- name: Install dependencies
10192
run: |
102-
sudo apt-get install build-essential cmake ninja-build python3-dev cython3 pybind11-dev libre2-dev -y
103-
python -m pip install --upgrade pip
104-
pip install wheel
105-
pip install -r requirements.txt
10693
pip install schemathesis
10794
pip install httpx
10895

‎.github/workflows/release_dongtai.yml‎

Lines changed: 44 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -60,29 +60,37 @@ jobs:
6060
- name: Login to Aliyun Registry
6161
uses: docker/login-action@v1
6262
with:
63-
registry: ${{ secrets.ALIYUN_REGISTRY }}
63+
registry: ${{ secrets.ALIYUN_REGISTRY_HONGKONG }}
6464
username: ${{ secrets.ALIYUN_DOCKERHUB_USER }}
6565
password: ${{ secrets.ALIYUN_DOCKERHUB_PASSWORD }}
6666

67-
- name: Setup Ossutil
68-
uses: manyuanrong/[email protected]
69-
with:
70-
endpoint: ${{ secrets.ALIYUN_OSS_ENDPOINT }}
71-
access-key-id: ${{ secrets.ALIYUN_OSS_KEY_ID }}
72-
access-key-secret: ${{ secrets.ALIYUN_OSS_KEY_SECRET }}
7367
- run: |
74-
echo "${{ github.event.repository.name }},version,${{ env.iast_version }}" >> version.txt && \
75-
echo "${{ github.event.repository.name }},commit_hash,${GITHUB_SHA}" >> version.txt && \
76-
cat version.txt
77-
ossutil cp -rf version.txt oss://huoqi-public/iast/release-version/${{ github.event.repository.name }}/${{ env.iast_version }}/version.txt
78-
ossutil cp oss://dongtai/agent/java/${{ inputs.agent_version }}/ ./ --include "*.jar" -r
79-
[ ! -f ./dongtai-agent.jar ] && echo "$FILE does not exist."
80-
[ ! -f ./dongtai-agent.jar ] && ossutil cp oss://dongtai/agent/java/latest/ ./ --include "*.jar" -r
81-
[ ! -f ./dongtai-agent.jar ] && echo "$FILE does not exist."
82-
ossutil cp oss://dongtai/agent/python/ ./ --include "*.tar.gz" -r
83-
ossutil cp oss://dongtai/agent/php/ ./ --include "*.tar.gz" -r
8468
echo "REPLACE INTO project_version_control (version, component_name, component_version_hash) VALUES('${{ env.iast_version }}', '${{ github.event.repository.name }}', '${GITHUB_SHA}');" >> ./deploy/docker/version.sql
8569
70+
- name: Upload COS java
71+
uses: zkqiang/[email protected]
72+
with:
73+
args: download -rs /agent/java/latest/ ./ --include "*.jar"
74+
secret_id: ${{ secrets.TENSECRET_ID }}
75+
secret_key: ${{ secrets.TENSECRET_KEY }}
76+
bucket: dongtai-helm-charts-1251882848
77+
region: ap-hongkong
78+
- name: Upload COS python
79+
uses: zkqiang/[email protected]
80+
with:
81+
args: download -rs /agent/python/ ./ --include "*.tar.gz"
82+
secret_id: ${{ secrets.TENSECRET_ID }}
83+
secret_key: ${{ secrets.TENSECRET_KEY }}
84+
bucket: dongtai-helm-charts-1251882848
85+
region: ap-hongkong
86+
- name: Upload COS php
87+
uses: zkqiang/[email protected]
88+
with:
89+
args: download -rs /agent/php/ ./ --include "*.tar.gz"
90+
secret_id: ${{ secrets.TENSECRET_ID }}
91+
secret_key: ${{ secrets.TENSECRET_KEY }}
92+
bucket: dongtai-helm-charts-1251882848
93+
region: ap-hongkong
8694
- name: Setup QEMU
8795
uses: docker/setup-qemu-action@v2
8896
- name: Setup Docker Buildx
@@ -136,24 +144,32 @@ jobs:
136144
id: get_version
137145
run: echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/}
138146

139-
- name: Setup ossutil
140-
uses: manyuanrong/[email protected]
141-
with:
142-
endpoint: ${{ secrets.CHART_OSS_ENDPOINT }}
143-
access-key-id: ${{ secrets.CHART_OSS_ACCESS_KEY_ID }}
144-
access-key-secret: ${{ secrets.CHART_OSS_ACCESS_KEY_SECRET }}
145147

146148
- name: Download existed repo files
147149
run: |
148150
sed -i "s#tag: latest#tag: ${{ env.iast_version }}#g" deploy/kubernetes/helm/values.yaml
149-
ossutil cp -rf oss://dongtai-helm-charts/iast/ ~/helm/repo/ --include dongtai-iast-*.tgz --exclude "index.yaml"
151+
- name: Upload COS php
152+
uses: zkqiang/[email protected]
153+
with:
154+
args: download -rs /iast/ ~/helm/repo/ --include dongtai-iast-*.tgz --ignore "index.yaml"
155+
secret_id: ${{ secrets.TENSECRET_ID }}
156+
secret_key: ${{ secrets.TENSECRET_KEY }}
157+
bucket: dongtai-helm-charts-1251882848
158+
region: ap-hongkong
150159

151160
- name: Create helm package
152161
run: |
153162
helm package deploy/kubernetes/helm -d ~/helm/repo --app-version ${{ env.iast_version }} --version ${{ env.iast_version }}
154163
helm repo index ~/helm/repo/ --url ${{ secrets.DONGTAI_IAST_CHART_REPO_URL }}
155164
156-
- name: Push helm chart to repo
157-
run: |
158-
ossutil cp -rf ~/helm/repo/dongtai-iast-${{ env.iast_version }}.tgz oss://dongtai-helm-charts/iast/
159-
ossutil cp -rf ~/helm/repo/index.yaml oss://dongtai-helm-charts/iast/
165+
166+
- name: Upload COS 2
167+
uses: zkqiang/[email protected]
168+
with:
169+
args: upload -rs ~/helm/repo/dongtai-iast-${{ env.iast_version }}.tgz /iast/ && upload -rs ~/helm/repo/index.yaml /iast/
170+
secret_id: ${{ secrets.TENSECRET_ID }}
171+
secret_key: ${{ secrets.TENSECRET_KEY }}
172+
bucket: dongtai-helm-charts-1251882848
173+
region: ap-hongkong
174+
175+

‎.github/workflows/teststate.yml‎

Lines changed: 12 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -55,17 +55,11 @@ jobs:
5555
steps:
5656
- name: Checkout
5757
uses: actions/checkout@v2
58-
- uses: actions/setup-python@v4
59-
with:
60-
python-version: "3.10"
61-
cache: "pip"
58+
- name: Setup Python
59+
uses: ./.github/actions/setup-python
6260

6361
- name: Install dependencies
6462
run: |
65-
sudo apt-get install build-essential cmake ninja-build python3-dev cython3 pybind11-dev libre2-dev -y
66-
python -m pip install --upgrade pip
67-
pip install wheel
68-
pip install -r requirements.txt
6963
pip install schemathesis
7064
pip install httpx
7165
@@ -124,18 +118,8 @@ jobs:
124118
steps:
125119
- name: Checkout
126120
uses: actions/checkout@v2
127-
- uses: actions/setup-python@v4
128-
with:
129-
python-version: "3.10"
130-
cache: "pip"
131-
132-
- name: Install dependencies
133-
run: |
134-
sudo apt-get install build-essential cmake ninja-build python3-dev cython3 pybind11-dev libre2-dev -y
135-
python -m pip install --upgrade pip
136-
pip install wheel
137-
pip install -r requirements.txt
138-
121+
- name: Setup Python
122+
uses: ./.github/actions/setup-python
139123
- name: Django Unit Testing
140124
run: |
141125
mypy --show-error-codes --ignore-missing-imports --no-incremental --show-error-codes --check-untyped-defs --disable-error-code var-annotated --disable-error-code list-item --disable-error-code attr-defined --disable-error-code assignment --disable-error-code misc --disable-error-code union-attr --disable-error-code index --disable-error-code call-overload --disable-error-code dict-item --disable-error-code truthy-function --disable-error-code operator --disable-error-code name-defined .
@@ -168,18 +152,11 @@ jobs:
168152
steps:
169153
- name: Checkout
170154
uses: actions/checkout@v2
171-
- uses: actions/setup-python@v4
172-
with:
173-
python-version: "3.10"
174-
cache: "pip"
155+
- name: Setup Python
156+
uses: ./.github/actions/setup-python
175157

176158
- name: Install dependencies
177-
run: |
178-
sudo apt-get install build-essential cmake ninja-build python3-dev cython3 pybind11-dev libre2-dev -y
179-
python -m pip install --upgrade pip
180-
pip install wheel
181-
pip install -r requirements.txt
182-
pip install bandit
159+
run: pip install bandit
183160
- name: Django Unit Testing
184161
run: |
185162
bandit -iii -lll -r .
@@ -215,18 +192,11 @@ jobs:
215192
steps:
216193
- name: Checkout
217194
uses: actions/checkout@v2
218-
- uses: actions/setup-python@v4
219-
with:
220-
python-version: "3.10"
221-
cache: "pip"
195+
- name: Setup Python
196+
uses: ./.github/actions/setup-python
222197

223198
- name: Install dependencies
224-
run: |
225-
sudo apt-get install build-essential cmake ninja-build python3-dev cython3 pybind11-dev libre2-dev -y
226-
python -m pip install --upgrade pip
227-
pip install wheel
228-
pip install coverage
229-
pip install -r requirements.txt
199+
run: pip install coverage
230200

231201
- name: Django Unit Testing
232202
run: |
@@ -246,18 +216,8 @@ jobs:
246216
steps:
247217
- name: Checkout
248218
uses: actions/checkout@v2
249-
- uses: actions/setup-python@v4
250-
with:
251-
python-version: "3.10"
252-
cache: "pip"
253-
254-
- name: Install dependencies
255-
run: |
256-
sudo apt-get install build-essential cmake ninja-build python3-dev cython3 pybind11-dev libre2-dev -y
257-
python -m pip install --upgrade pip
258-
pip install wheel
259-
pip install coverage
260-
pip install -r requirements.txt
219+
- name: Setup Python
220+
uses: ./.github/actions/setup-python
261221

262222
- name: Check Schema
263223
run: python3 manage.py check_schema

‎Dockerfile‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,25 @@ ENV LANGUAGE=en_US.UTF-8
77
ENV TZ=Asia/Shanghai
88

99
RUN apt-get update -y \
10-
&& apt install -y gettext gcc make cmake libmariadb-dev curl libc6-dev libxrender1 libxtst6 libxi6 unzip cron \
11-
fonts-wqy-microhei vim build-essential ninja-build cython3 pybind11-dev libre2-dev locales \
12-
# htop sysstat net-tools iproute2 procps lsof \
13-
zip libjpeg62 \
14-
&& sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen \
15-
&& ALIMARCH=`arch` && curl -L https://charts.dongtai.io/apk/${ALIMARCH}/wkhtmltopdf -o /usr/bin/wkhtmltopdf \
16-
&& chmod +x /usr/bin/wkhtmltopdf
10+
&& apt install -y gettext gcc make cmake libmariadb-dev curl libc6-dev libxrender1 libxtst6 libxi6 unzip cron \
11+
fonts-wqy-microhei vim build-essential ninja-build cython3 pybind11-dev libre2-dev locales \
12+
libsasl2-dev python3-dev libldap2-dev libssl-dev \
13+
# htop sysstat net-tools iproute2 procps lsof \
14+
zip libjpeg62 \
15+
&& sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen \
16+
&& ALIMARCH=`arch` && curl -L https://charts.dongtai.io/apk/${ALIMARCH}/wkhtmltopdf -o /usr/bin/wkhtmltopdf \
17+
&& chmod +x /usr/bin/wkhtmltopdf \
18+
&& if [ "aarch64" = "$ALIMARCH" ] ; then curl -L https://github.com/HXSecurity/tantivy-py/releases/download/0.21.0/tantivy-0.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl -o /tmp/tantivy-0.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl; \
19+
else curl -L https://github.com/HXSecurity/tantivy-py/releases/download/0.21.0/tantivy-0.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -o /tmp/tantivy-0.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl; fi
1720

1821
COPY Pipfile .
1922
COPY Pipfile.lock .
2023
RUN pip install -U pip && pip install pipenv wheel && python3 -m pipenv sync --system -v
24+
RUN ALIMARCH=`arch` && if [ "aarch64" = "$ALIMARCH" ] ; then pip install /tmp/tantivy-0.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl; else pip install /tmp/tantivy-0.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl; fi
2125

2226
COPY . /opt/dongtai
2327
WORKDIR /opt/dongtai
2428

2529
RUN /bin/bash -c 'mkdir -p /tmp/{logstash/{batchagent,report/{img,word,pdf,excel,html}},iast_cache/package}' \
26-
&& mv /opt/dongtai/*.jar /tmp/iast_cache/package/ || true && mv /opt/dongtai/*.tar.gz /tmp/ || true
30+
&& mv /opt/dongtai/*.jar /tmp/iast_cache/package/ || true && mv /opt/dongtai/*.tar.gz /tmp/ || true
2731
ENTRYPOINT ["/bin/bash","/opt/dongtai/deploy/docker/entrypoint.sh"]

0 commit comments

Comments
 (0)