From 3774f4432bb7ac9a077bb025d6896e3fcf12c98a Mon Sep 17 00:00:00 2001 From: Manuel Schmid Date: Sat, 29 Mar 2025 19:11:05 +0100 Subject: [PATCH 1/5] fix: correctly implement webhook secret for both github and gitlab --- api.py | 41 ++++++++++++++++++++--------- biz/utils/webhook_secret_checker.py | 34 ++++++++++++++++++++++++ conf/.env.dist | 2 ++ 3 files changed, 64 insertions(+), 13 deletions(-) create mode 100644 biz/utils/webhook_secret_checker.py diff --git a/api.py b/api.py index ef40870..d982279 100644 --- a/api.py +++ b/api.py @@ -19,6 +19,8 @@ from biz.utils.reporter import Reporter from biz.utils.config_checker import check_config +from biz.utils.webhook_secret_checker import verify_gitlab_webhook_secret_token, verify_github_signature + load_dotenv("conf/.env") api_app = Flask(__name__) @@ -122,18 +124,26 @@ def handle_webhook(): return jsonify({'message': 'Invalid data format'}), 400 def handle_github_webhook(event_type, data): + # 打印整个payload数据 + logger.info(f'Received GitHub event: {event_type}') + logger.info(f'Payload: {json.dumps(data)}') + # 获取GitHub配置 - github_token = os.getenv('GITHUB_ACCESS_TOKEN') or request.headers.get('X-GitHub-Token') + github_token = os.getenv('GITHUB_ACCESS_TOKEN') if not github_token: return jsonify({'message': 'Missing GitHub access token'}), 400 - + + # 获取GitHub Webhook Secret Token + payload_body = request.get_data() + github_webhook_secret_token_env = os.getenv('GITHUB_WEBHOOK_SECRET_TOKEN') + github_webhook_secret_token_request = request.headers.get('X-Hub-Signature-256') + if not verify_github_signature(payload_body, github_webhook_secret_token_env, github_webhook_secret_token_request): + logger.error(f"GitHub Webhook Secret Token mismatch") + return jsonify({'message': 'GitHub Webhook Secret Token mismatch'}), 403 + github_url = os.getenv('GITHUB_URL') or 'https://github.com' github_url_slug = slugify_url(github_url) - - # 打印整个payload数据 - logger.info(f'Received GitHub event: {event_type}') - logger.info(f'Payload: {json.dumps(data)}') - + if event_type == "pull_request": # 使用handle_queue进行异步处理 handle_queue(handle_github_pull_request_event, data, github_token, github_url, github_url_slug) @@ -167,17 +177,22 @@ def handle_gitlab_webhook(data): except Exception as e: return jsonify({"error": f"Failed to parse homepage URL: {str(e)}"}), 400 - # 优先从环境变量获取，如果没有，则从请求头获取 - gitlab_token = os.getenv('GITLAB_ACCESS_TOKEN') or request.headers.get('X-Gitlab-Token') + # 打印整个payload数据，或根据需求进行处理 + logger.info(f'Received event: {object_kind}') + logger.info(f'Payload: {json.dumps(data)}') + + gitlab_token = os.getenv('GITLAB_ACCESS_TOKEN') # 如果gitlab_token为空，返回错误 if not gitlab_token: return jsonify({'message': 'Missing GitLab access token'}), 400 - gitlab_url_slug = slugify_url(gitlab_url) + gitlab_webhook_secret_token_env = os.getenv('GITLAB_WEBHOOK_SECRET_TOKEN') + gitlab_webhook_secret_token_request = request.headers.get('X-Gitlab-Token') + if not verify_gitlab_webhook_secret_token(gitlab_webhook_secret_token_env, gitlab_webhook_secret_token_request): + logger.error(f"GitLab Webhook Secret Token mismatch") + return jsonify({'message': 'GitLab Webhook Secret Token mismatch'}), 403 - # 打印整个payload数据，或根据需求进行处理 - logger.info(f'Received event: {object_kind}') - logger.info(f'Payload: {json.dumps(data)}') + gitlab_url_slug = slugify_url(gitlab_url) # 处理Merge Request Hook if object_kind == "merge_request": diff --git a/biz/utils/webhook_secret_checker.py b/biz/utils/webhook_secret_checker.py new file mode 100644 index 0000000..7581667 --- /dev/null +++ b/biz/utils/webhook_secret_checker.py @@ -0,0 +1,34 @@ +import hashlib +import hmac +from http.client import HTTPException + + +# from https://docs.github.com/en/webhooks/using-webhooks/validating-webhook-deliveries#python-example +def verify_github_signature(payload_body, secret_token, signature_header): + """Verify that the payload was sent from GitHub by validating SHA256. + + Raise and return 403 if not authorized. + + Args: + payload_body: original request body to verify (request.body()) + secret_token: GitHub app webhook token (WEBHOOK_SECRET) + signature_header: header received from GitHub (x-hub-signature-256) + """ + # 仅当设置了环境变量时才验证秘密令牌 + if not secret_token: + return True + if not signature_header: + raise HTTPException(status_code=403, detail="x-hub-signature-256 header is missing!") + hash_object = hmac.new(secret_token.encode('utf-8'), msg=payload_body, digestmod=hashlib.sha256) + expected_signature = "sha256=" + hash_object.hexdigest() + if not hmac.compare_digest(expected_signature, signature_header): + raise HTTPException(status_code=403, detail="Request signatures didn't match!") + return True + + +def verify_gitlab_webhook_secret_token(secret_token_env, secret_token_request): + # 仅当设置了环境变量时才验证秘密令牌 + if secret_token_env: + if secret_token_env != secret_token_request: + return False + return True diff --git a/conf/.env.dist b/conf/.env.dist index 5857290..9e9d3b7 100644 --- a/conf/.env.dist +++ b/conf/.env.dist @@ -55,9 +55,11 @@ REPORT_CRONTAB_EXPRESSION=0 18 * * 1-5 #Gitlab配置 #GITLAB_URL={YOUR_GITLAB_URL} #部分老版本Gitlab webhook不传递URL，需要开启此配置，示例：https://gitlab.example.com #GITLAB_ACCESS_TOKEN={YOUR_GITLAB_ACCESS_TOKEN} #系统会优先使用此GITLAB_ACCESS_TOKEN，如果未配置，则使用Webhook 传递的Secret Token +#GITLAB_WEBHOOK_SECRET_TOKEN={YOUR_GITLAB_WEBHOOK_SECRET_TOKEN} #在 webhook 中使用相同的秘密令牌来验证其签名 #Github配置(如果使用 Github 作为代码托管平台，需要配置此项) #GITHUB_ACCESS_TOKEN={YOUR_GITHUB_ACCESS_TOKEN} +#GITHUB_WEBHOOK_SECRET_TOKEN={YOUR_GITHUB_WEBHOOK_SECRET_TOKEN} #在 webhook 中使用相同的秘密令牌来验证其签名 # 开启Push Review功能(如果不需要push事件触发Code Review，设置为0) PUSH_REVIEW_ENABLED=1 From 7f24014b9b422b4e6905ef8d230eac2eecc26724 Mon Sep 17 00:00:00 2001 From: Manuel Schmid Date: Sat, 29 Mar 2025 20:01:07 +0100 Subject: [PATCH 2/5] fix: return boolean and log message instead of incorrectly using HTTPException --- api.py | 1 - biz/utils/webhook_secret_checker.py | 9 ++++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/api.py b/api.py index d982279..9067702 100644 --- a/api.py +++ b/api.py @@ -138,7 +138,6 @@ def handle_github_webhook(event_type, data): github_webhook_secret_token_env = os.getenv('GITHUB_WEBHOOK_SECRET_TOKEN') github_webhook_secret_token_request = request.headers.get('X-Hub-Signature-256') if not verify_github_signature(payload_body, github_webhook_secret_token_env, github_webhook_secret_token_request): - logger.error(f"GitHub Webhook Secret Token mismatch") return jsonify({'message': 'GitHub Webhook Secret Token mismatch'}), 403 github_url = os.getenv('GITHUB_URL') or 'https://github.com' diff --git a/biz/utils/webhook_secret_checker.py b/biz/utils/webhook_secret_checker.py index 7581667..d0fbb11 100644 --- a/biz/utils/webhook_secret_checker.py +++ b/biz/utils/webhook_secret_checker.py @@ -1,6 +1,7 @@ import hashlib import hmac -from http.client import HTTPException + +from biz.utils.log import logger # from https://docs.github.com/en/webhooks/using-webhooks/validating-webhook-deliveries#python-example @@ -18,11 +19,13 @@ def verify_github_signature(payload_body, secret_token, signature_header): if not secret_token: return True if not signature_header: - raise HTTPException(status_code=403, detail="x-hub-signature-256 header is missing!") + logger.error("x-hub-signature-256 header is missing!") + return False hash_object = hmac.new(secret_token.encode('utf-8'), msg=payload_body, digestmod=hashlib.sha256) expected_signature = "sha256=" + hash_object.hexdigest() if not hmac.compare_digest(expected_signature, signature_header): - raise HTTPException(status_code=403, detail="Request signatures didn't match!") + logger.error("Request signatures didn't match!") + return False return True From 2f601fff9ac1ddcfbb878d78ff94e80cf3cf807b Mon Sep 17 00:00:00 2001 From: Manuel Schmid Date: Sat, 29 Mar 2025 20:03:36 +0100 Subject: [PATCH 3/5] qa: remove comment --- biz/utils/webhook_secret_checker.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/biz/utils/webhook_secret_checker.py b/biz/utils/webhook_secret_checker.py index d0fbb11..8841f44 100644 --- a/biz/utils/webhook_secret_checker.py +++ b/biz/utils/webhook_secret_checker.py @@ -8,8 +8,6 @@ def verify_github_signature(payload_body, secret_token, signature_header): """Verify that the payload was sent from GitHub by validating SHA256. - Raise and return 403 if not authorized. - Args: payload_body: original request body to verify (request.body()) secret_token: GitHub app webhook token (WEBHOOK_SECRET) From 7f33078f9ad60e4ad37fcca51e93f15fea378d13 Mon Sep 17 00:00:00 2001 From: Manuel Schmid Date: Sat, 29 Mar 2025 20:30:59 +0100 Subject: [PATCH 4/5] refactor: move functions to corresponding subfolders --- api.py | 6 ++-- biz/github/webhook_handler.py | 44 ++++++++++++++++++++++------- biz/gitlab/webhook_handler.py | 7 +++++ biz/utils/webhook_secret_checker.py | 35 ----------------------- 4 files changed, 44 insertions(+), 48 deletions(-) delete mode 100644 biz/utils/webhook_secret_checker.py diff --git a/api.py b/api.py index 9067702..8f4facd 100644 --- a/api.py +++ b/api.py @@ -10,7 +10,8 @@ from dotenv import load_dotenv from flask import Flask, request, jsonify -from biz.gitlab.webhook_handler import slugify_url +from biz.github.webhook_handler import verify_github_signature +from biz.gitlab.webhook_handler import slugify_url, verify_gitlab_webhook_secret_token from biz.queue.worker import handle_merge_request_event, handle_push_event, handle_github_pull_request_event, handle_github_push_event from biz.service.review_service import ReviewService from biz.utils.im import notifier @@ -19,7 +20,6 @@ from biz.utils.reporter import Reporter from biz.utils.config_checker import check_config -from biz.utils.webhook_secret_checker import verify_gitlab_webhook_secret_token, verify_github_signature load_dotenv("conf/.env") api_app = Flask(__name__) @@ -115,7 +115,7 @@ def handle_webhook(): # 判断是GitLab还是GitHub的webhook webhook_source = request.headers.get('X-GitHub-Event') - + if webhook_source: # GitHub webhook return handle_github_webhook(webhook_source, data) else: # GitLab webhook diff --git a/biz/github/webhook_handler.py b/biz/github/webhook_handler.py index f2948e7..d21771c 100644 --- a/biz/github/webhook_handler.py +++ b/biz/github/webhook_handler.py @@ -1,4 +1,5 @@ -import json +import hashlib +import hmac import os import re import time @@ -23,7 +24,7 @@ def filter_changes(changes: list): if change.get('status') == 'removed': logger.info(f"Detected file deletion via status field: {change.get('new_path')}") continue - + # 如果没有status字段或status不为"removed"，继续检查diff模式 diff = change.get('diff', '') if diff: @@ -34,12 +35,12 @@ def filter_changes(changes: list): if all(line.startswith('-') or not line for line in diff_lines): logger.info(f"Detected file deletion via diff pattern: {change.get('new_path')}") continue - + not_deleted_changes.append(change) - + logger.info(f"SUPPORTED_EXTENSIONS: {SUPPORTED_EXTENSIONS}") logger.info(f"After filtering deleted files: {not_deleted_changes}") - + # 过滤 `new_path` 以支持的扩展名结尾的元素, 仅保留diff和new_path字段 filtered_changes = [ { @@ -53,6 +54,29 @@ def filter_changes(changes: list): return filtered_changes +# from https://docs.github.com/en/webhooks/using-webhooks/validating-webhook-deliveries#python-example +def verify_github_signature(payload_body, secret_token, signature_header): + """Verify that the payload was sent from GitHub by validating SHA256. + + Args: + payload_body: original request body to verify (request.body()) + secret_token: GitHub app webhook token (WEBHOOK_SECRET) + signature_header: header received from GitHub (x-hub-signature-256) + """ + # 仅当设置了环境变量时才验证秘密令牌 + if not secret_token: + return True + if not signature_header: + logger.error("x-hub-signature-256 header is missing!") + return False + hash_object = hmac.new(secret_token.encode('utf-8'), msg=payload_body, digestmod=hashlib.sha256) + expected_signature = "sha256=" + hash_object.hexdigest() + if not hmac.compare_digest(expected_signature, signature_header): + logger.error("Request signatures didn't match!") + return False + return True + + class PullRequestHandler: def __init__(self, webhook_data: dict, github_token: str, github_url: str): self.pull_request_number = None @@ -133,7 +157,7 @@ def get_pull_request_commits(self) -> list: } response = requests.get(url, headers=headers) logger.debug(f"Get commits response from GitHub: {response.status_code}, {response.text}") - + # 检查请求是否成功 if response.status_code == 200: # 将GitHub的commits转换为GitLab格式的commits @@ -330,12 +354,12 @@ def get_push_changes(self) -> list: elif self.webhook_data.get('deleted', False): # 删除分支处理 return [] - + return self.repository_compare(before, after) else: # 如果before和after不存在，尝试通过commits获取 logger.info("before or after not found in webhook data, trying to get changes from commits.") - + changes = [] for commit in self.commit_list: commit_id = commit.get('id') @@ -344,5 +368,5 @@ def get_push_changes(self) -> list: if parent_id: commit_changes = self.repository_compare(parent_id, commit_id) changes.extend(commit_changes) - - return changes \ No newline at end of file + + return changes diff --git a/biz/gitlab/webhook_handler.py b/biz/gitlab/webhook_handler.py index ade6e98..3af6d73 100644 --- a/biz/gitlab/webhook_handler.py +++ b/biz/gitlab/webhook_handler.py @@ -47,6 +47,13 @@ def slugify_url(original_url: str) -> str: return target +def verify_gitlab_webhook_secret_token(secret_token_env, secret_token_request): + # 仅当设置了环境变量时才验证秘密令牌 + if secret_token_env: + if secret_token_env != secret_token_request: + return False + return True + class MergeRequestHandler: def __init__(self, webhook_data: dict, gitlab_token: str, gitlab_url: str): diff --git a/biz/utils/webhook_secret_checker.py b/biz/utils/webhook_secret_checker.py deleted file mode 100644 index 8841f44..0000000 --- a/biz/utils/webhook_secret_checker.py +++ /dev/null @@ -1,35 +0,0 @@ -import hashlib -import hmac - -from biz.utils.log import logger - - -# from https://docs.github.com/en/webhooks/using-webhooks/validating-webhook-deliveries#python-example -def verify_github_signature(payload_body, secret_token, signature_header): - """Verify that the payload was sent from GitHub by validating SHA256. - - Args: - payload_body: original request body to verify (request.body()) - secret_token: GitHub app webhook token (WEBHOOK_SECRET) - signature_header: header received from GitHub (x-hub-signature-256) - """ - # 仅当设置了环境变量时才验证秘密令牌 - if not secret_token: - return True - if not signature_header: - logger.error("x-hub-signature-256 header is missing!") - return False - hash_object = hmac.new(secret_token.encode('utf-8'), msg=payload_body, digestmod=hashlib.sha256) - expected_signature = "sha256=" + hash_object.hexdigest() - if not hmac.compare_digest(expected_signature, signature_header): - logger.error("Request signatures didn't match!") - return False - return True - - -def verify_gitlab_webhook_secret_token(secret_token_env, secret_token_request): - # 仅当设置了环境变量时才验证秘密令牌 - if secret_token_env: - if secret_token_env != secret_token_request: - return False - return True From d30dc7952034285f946e77b8e1c3aba77b19db5c Mon Sep 17 00:00:00 2001 From: Manuel Schmid Date: Thu, 3 Apr 2025 23:03:20 +0200 Subject: [PATCH 5/5] feat: add env vars to (de)activate secret token validation --- api.py | 20 +++++++++++--------- conf/.env.dist | 2 ++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/api.py b/api.py index 8f4facd..33ac782 100644 --- a/api.py +++ b/api.py @@ -135,10 +135,11 @@ def handle_github_webhook(event_type, data): # 获取GitHub Webhook Secret Token payload_body = request.get_data() - github_webhook_secret_token_env = os.getenv('GITHUB_WEBHOOK_SECRET_TOKEN') - github_webhook_secret_token_request = request.headers.get('X-Hub-Signature-256') - if not verify_github_signature(payload_body, github_webhook_secret_token_env, github_webhook_secret_token_request): - return jsonify({'message': 'GitHub Webhook Secret Token mismatch'}), 403 + if os.getenv('VALIDATE_GITHUB_WEBHOOK_SECRET_TOKEN', '0') == '1': + github_webhook_secret_token_env = os.getenv('GITHUB_WEBHOOK_SECRET_TOKEN') + github_webhook_secret_token_request = request.headers.get('X-Hub-Signature-256') + if not verify_github_signature(payload_body, github_webhook_secret_token_env, github_webhook_secret_token_request): + return jsonify({'message': 'GitHub Webhook Secret Token mismatch'}), 403 github_url = os.getenv('GITHUB_URL') or 'https://github.com' github_url_slug = slugify_url(github_url) @@ -185,11 +186,12 @@ def handle_gitlab_webhook(data): if not gitlab_token: return jsonify({'message': 'Missing GitLab access token'}), 400 - gitlab_webhook_secret_token_env = os.getenv('GITLAB_WEBHOOK_SECRET_TOKEN') - gitlab_webhook_secret_token_request = request.headers.get('X-Gitlab-Token') - if not verify_gitlab_webhook_secret_token(gitlab_webhook_secret_token_env, gitlab_webhook_secret_token_request): - logger.error(f"GitLab Webhook Secret Token mismatch") - return jsonify({'message': 'GitLab Webhook Secret Token mismatch'}), 403 + if os.getenv('VALIDATE_GITLAB_WEBHOOK_SECRET_TOKEN', '0') == '1': + gitlab_webhook_secret_token_env = os.getenv('GITLAB_WEBHOOK_SECRET_TOKEN') + gitlab_webhook_secret_token_request = request.headers.get('X-Gitlab-Token') + if not verify_gitlab_webhook_secret_token(gitlab_webhook_secret_token_env, gitlab_webhook_secret_token_request): + logger.error(f"GitLab Webhook Secret Token mismatch") + return jsonify({'message': 'GitLab Webhook Secret Token mismatch'}), 403 gitlab_url_slug = slugify_url(gitlab_url) diff --git a/conf/.env.dist b/conf/.env.dist index 9e9d3b7..6bd7111 100644 --- a/conf/.env.dist +++ b/conf/.env.dist @@ -55,10 +55,12 @@ REPORT_CRONTAB_EXPRESSION=0 18 * * 1-5 #Gitlab配置 #GITLAB_URL={YOUR_GITLAB_URL} #部分老版本Gitlab webhook不传递URL，需要开启此配置，示例：https://gitlab.example.com #GITLAB_ACCESS_TOKEN={YOUR_GITLAB_ACCESS_TOKEN} #系统会优先使用此GITLAB_ACCESS_TOKEN，如果未配置，则使用Webhook 传递的Secret Token +#VALIDATE_GITLAB_WEBHOOK_SECRET_TOKEN=1 #GITLAB_WEBHOOK_SECRET_TOKEN={YOUR_GITLAB_WEBHOOK_SECRET_TOKEN} #在 webhook 中使用相同的秘密令牌来验证其签名 #Github配置(如果使用 Github 作为代码托管平台，需要配置此项) #GITHUB_ACCESS_TOKEN={YOUR_GITHUB_ACCESS_TOKEN} +#VALIDATE_GITHUB_ACCESS_TOKEN=1 #GITHUB_WEBHOOK_SECRET_TOKEN={YOUR_GITHUB_WEBHOOK_SECRET_TOKEN} #在 webhook 中使用相同的秘密令牌来验证其签名 # 开启Push Review功能(如果不需要push事件触发Code Review，设置为0)