Skip to content

Feature/add grace period minutes #60

Feature/add grace period minutes

Feature/add grace period minutes #60

Workflow file for this run

# =============================================================================
# E2E Integration Test Pipeline for Simili Bot
# =============================================================================
# Triggers on every PR to main. Creates a test repo under the bot user account,
# seeds it with issues, indexes them, installs the PR version of Simili,
# then verifies duplicate detection and loop safety.
#
# Required secrets:
# BOT_PAT – Bot user PAT (repo, workflow scopes)
# BOT_USERNAME – Bot user's GitHub username (must start with "gh-simili")
# QDRANT_URL – Qdrant cluster URL
# QDRANT_API_KEY – Qdrant API key
# GEMINI_API_KEY – Gemini API key
# =============================================================================
name: Simili E2E Test
on:
pull_request_target:
types: [labeled]
workflow_dispatch:
inputs:
sha:
description: 'Commit SHA to test (e.g. from a PR head)'
required: true
pr_number:
description: 'PR Number (for reporting)'
required: true
concurrency:
group: e2e-${{ github.event.pull_request.number || inputs.pr_number || github.sha }}
cancel-in-progress: false
permissions:
contents: read
pull-requests: write
env:
TEST_REPO_NAME: "simili-e2e-${{ github.run_id }}"
TEST_COLLECTION: "simili-e2e-test-${{ github.run_id }}"
# Bot username — not sensitive (visible in all URLs/logs).
# Declared as plain env var so GitHub does NOT mask it (it would
# be masked if derived from the BOT_USERNAME secret).
BOT_NAME: "gh-simili-bot"
jobs:
e2e-test:
name: E2E Integration Test
runs-on: ubuntu-latest
timeout-minutes: 15
# Only run if the 'e2e' label was added (or manual trigger)
if: github.event.label.name == 'e2e' || github.event_name == 'workflow_dispatch'
steps:
# ----- 0. Validate secrets -----
- name: Validate secrets
run: |
missing=""
[ -z "${{ secrets.BOT_PAT }}" ] && missing="$missing BOT_PAT"
[ -z "${{ secrets.BOT_USERNAME }}" ] && missing="$missing BOT_USERNAME"
[ -z "${{ secrets.QDRANT_URL }}" ] && missing="$missing QDRANT_URL"
[ -z "${{ secrets.QDRANT_API_KEY }}" ] && missing="$missing QDRANT_API_KEY"
[ -z "${{ secrets.GEMINI_API_KEY }}" ] && missing="$missing GEMINI_API_KEY"
if [ -n "$missing" ]; then
echo "::error::Missing secrets:$missing"
exit 1
fi
echo "✅ All required secrets present"
# ----- 1. Create test repo (public, so devs can inspect) -----
- name: Create test repository
env:
B_PAT: ${{ secrets.BOT_PAT }}
run: |
echo "$B_PAT" | gh auth login --with-token
gh auth setup-git
gh repo create "${{ env.BOT_NAME }}/${{ env.TEST_REPO_NAME }}" \
--private \
--description "Simili E2E test – PR #${{ github.event.pull_request.number }}" \
--clone
# ----- 2. Seed config files (NO workflow yet) -----
- name: Seed test repository
run: |
cd "${{ env.TEST_REPO_NAME }}"
git config user.name "${{ env.BOT_NAME }}"
git config user.email "makerslab.foss@gmail.com"
# README
cat > README.md << 'READMEEOF'
# Simili E2E Test Repository
Auto-created by the Simili Bot E2E pipeline. Kept for inspection.
READMEEOF
# .github/simili.yaml
mkdir -p .github
COLLECTION="${{ env.TEST_COLLECTION }}"
cat > .github/simili.yaml << 'CFGEOF'
qdrant:
url: ${QDRANT_URL}
api_key: ${QDRANT_API_KEY}
collection: %COLLECTION%
embedding:
provider: gemini
api_key: ${GEMINI_API_KEY}
model: gemini-embedding-001
dimensions: 768
llm:
provider: gemini
api_key: ${GEMINI_API_KEY}
model: gemini-2.5-flash
defaults:
similarity_threshold: 0.65
max_similar_to_show: 3
CFGEOF
sed -i "s/%COLLECTION%/$COLLECTION/g" .github/simili.yaml
git add -A
git commit -m "chore: seed E2E test config"
git push origin HEAD:main
# ----- 3. Copy secrets into test repo -----
- name: Set secrets on test repository
env:
GH_TOKEN: ${{ secrets.BOT_PAT }}
QDR_URL: ${{ secrets.QDRANT_URL }}
QDR_KEY: ${{ secrets.QDRANT_API_KEY }}
GEM_KEY: ${{ secrets.GEMINI_API_KEY }}
B_PAT: ${{ secrets.BOT_PAT }}
B_NAME: ${{ env.BOT_NAME }}
run: |
REPO="${{ env.BOT_NAME }}/${{ env.TEST_REPO_NAME }}"
gh secret set QDRANT_URL --repo "$REPO" --body "$QDR_URL"
gh secret set QDRANT_API_KEY --repo "$REPO" --body "$QDR_KEY"
gh secret set GEMINI_API_KEY --repo "$REPO" --body "$GEM_KEY"
gh secret set GH_PAT --repo "$REPO" --body "$B_PAT"
gh secret set BOT_USERNAME --repo "$REPO" --body "$B_NAME"
# ----- 4. Create 5 seed issues -----
- name: Create seed issues
env:
GH_TOKEN: ${{ secrets.BOT_PAT }}
run: |
REPO="${{ secrets.BOT_USERNAME }}/${{ env.TEST_REPO_NAME }}"
gh issue create --repo "$REPO" \
--title "App crashes on login with invalid credentials" \
--body "When entering a wrong password, the application crashes with an unhandled exception instead of displaying a user-friendly error message. Steps to reproduce: 1. Open the app 2. Enter valid username 3. Enter incorrect password 4. Click login. Expected: Error message. Actual: App crashes."
gh issue create --repo "$REPO" \
--title "Add dark mode support" \
--body "Users have requested a dark mode theme for better readability in low-light environments. This should include a toggle in settings and automatic detection of system preferences."
gh issue create --repo "$REPO" \
--title "Performance degradation on large datasets" \
--body "Loading time exceeds 30 seconds when the dataset has more than 10,000 rows. The UI becomes unresponsive during data loading. We need pagination or virtual scrolling."
gh issue create --repo "$REPO" \
--title "Update documentation for API v2" \
--body "Current documentation still references deprecated API v1 endpoints. All examples, curl commands, and SDK snippets need to be updated to reflect the v2 API changes."
gh issue create --repo "$REPO" \
--title "Mobile responsive layout broken on iOS Safari" \
--body "The navigation menu overlaps with the main content area on iPhone 12 and below when using Safari. The hamburger menu does not collapse properly after selecting a menu item."
# ----- 5. Build CLI from PR & index seed issues -----
# ----- 5. Build CLI from PR & index seed issues -----
- name: Checkout PR code
uses: actions/checkout@v4
with:
# Explicitly checkout the PR code
# If triggered by review: pull_request.head.sha
# If manual: inputs.sha
ref: ${{ github.event.pull_request.head.sha || inputs.sha }}
path: simili-src
persist-credentials: false
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.23'
- name: Build simili CLI
run: |
cd simili-src
go build -o "$RUNNER_TEMP/simili-cli" ./cmd/simili/main.go
- name: Index seed issues
env:
GITHUB_TOKEN: ${{ secrets.BOT_PAT }}
QDRANT_URL: ${{ secrets.QDRANT_URL }}
QDRANT_API_KEY: ${{ secrets.QDRANT_API_KEY }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
TEST_COLLECTION: ${{ env.TEST_COLLECTION }}
run: |
REPO="${{ env.BOT_NAME }}/${{ env.TEST_REPO_NAME }}"
cat > "$RUNNER_TEMP/index-cfg.yaml" << 'IDXEOF'
qdrant:
url: ${QDRANT_URL}
api_key: ${QDRANT_API_KEY}
collection: ${TEST_COLLECTION}
embedding:
provider: gemini
api_key: ${GEMINI_API_KEY}
model: gemini-embedding-001
dimensions: 768
llm:
provider: gemini
api_key: ${GEMINI_API_KEY}
model: gemini-2.5-flash
IDXEOF
"$RUNNER_TEMP/simili-cli" index \
--repo "$REPO" \
--config "$RUNNER_TEMP/index-cfg.yaml" \
--workers 2
# ----- 6. Let the index settle -----
- name: Wait for index
run: sleep 15
# ----- 7. Push triage workflow to test repo -----
- name: Enable Simili workflow
run: |
rm -rf test-repo-wf
gh repo clone "${{ env.BOT_NAME }}/${{ env.TEST_REPO_NAME }}" test-repo-wf
cd test-repo-wf
git config user.name "${{ env.BOT_NAME }}"
git config user.email "makerslab.foss@gmail.com"
mkdir -p .github/workflows
cat > .github/workflows/triage.yml << 'WFEOF'
name: Simili Triage
on:
issues:
types: [opened, edited, reopened]
issue_comment:
types: [created]
concurrency:
group: simili-${{ github.event.issue.number || github.event.pull_request.number }}
cancel-in-progress: false
permissions:
contents: read
issues: write
jobs:
triage:
runs-on: ubuntu-latest
# Skip only when the bot itself posts a comment (prevents feedback loop).
# Always run for issues: opened/edited events.
if: >
github.event_name != 'issue_comment' ||
(github.actor != 'gh-simili-bot' && github.actor != 'github-actions[bot]')
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Run Simili Bot
uses: similigh/simili-bot@__PR_SHA__
env:
QDRANT_URL: ${{ secrets.QDRANT_URL }}
QDRANT_API_KEY: ${{ secrets.QDRANT_API_KEY }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
SIMILI_E2E_TEST_USER: ${{ secrets.BOT_USERNAME }}
with:
config_path: .github/simili.yaml
command: process
WFEOF
# Generate workflow from template
# Uses the PR SHA so we test the PR code, not main
# Fallback to inputs.sha for manual dispatch
SHA="${{ github.event.pull_request.head.sha || inputs.sha }}"
sed -i "s|__PR_SHA__|$SHA|g" .github/workflows/triage.yml
git add -A
git commit -m "chore: enable Simili Triage workflow"
git push origin HEAD:main
echo "Waiting 15s for GitHub to register the workflow…"
sleep 15
# ----- 7b. Create auto-closer test issue -----
# Created BEFORE the trigger issue so it ages long enough to exceed the
# 1-minute grace-period override we use in the auto-closer E2E step below.
# We manually ensure the "potential-duplicate" label exists and apply it
# immediately, independent of what the triage bot decides.
- name: Setup auto-closer test issue
id: ac_setup
env:
GH_TOKEN: ${{ secrets.BOT_PAT }}
run: |
REPO="${{ env.BOT_NAME }}/${{ env.TEST_REPO_NAME }}"
# Ensure the potential-duplicate label exists (bot creates it lazily;
# this step runs before the trigger issue is triaged).
gh label create "potential-duplicate" \
--repo "$REPO" \
--color "FBCA04" \
--description "Potential duplicate — pending grace period" \
2>/dev/null || true
# Create the AC test issue
URL=$(gh issue create --repo "$REPO" \
--title "Auto-closer grace period and human-activity E2E test" \
--body "This issue is created by the E2E pipeline to verify the auto-closer detects expired grace periods and correctly skips issues with human activity signals.")
AC_NUM=$(echo "$URL" | grep -oE '[0-9]+$')
echo "ac_issue_number=$AC_NUM" >> "$GITHUB_OUTPUT"
echo "AC test issue: #$AC_NUM"
# Apply potential-duplicate label immediately so the grace period clock starts
gh issue edit "$AC_NUM" --repo "$REPO" --add-label "potential-duplicate"
echo "✅ Labeled #$AC_NUM as potential-duplicate"
# ----- 8. Create trigger issue (should match seed #1) -----
- name: Create trigger issue
id: trigger
env:
GH_TOKEN: ${{ secrets.BOT_PAT }}
run: |
REPO="${{ env.BOT_NAME }}/${{ env.TEST_REPO_NAME }}"
URL=$(gh issue create --repo "$REPO" \
--title "Application crashes when logging in with wrong password" \
--body "The application throws an unhandled exception when invalid credentials are entered during the login flow. The crash occurs immediately after clicking the submit button. No error dialog is shown to the user.")
NUM=$(echo "$URL" | grep -oE '[0-9]+$')
echo "issue_number=$NUM" >> "$GITHUB_OUTPUT"
echo "Trigger issue created: #$NUM"
# ----- 9. Poll for bot comment -----
- name: Wait for bot response
id: poll
env:
GH_TOKEN: ${{ secrets.BOT_PAT }}
run: |
REPO="${{ env.BOT_NAME }}/${{ env.TEST_REPO_NAME }}"
ISSUE="${{ steps.trigger.outputs.issue_number }}"
# Comments are posted via github.token, so the author is github-actions[bot]
BOT="github-actions[bot]"
MAX=30 # 30 × 10 s = 5 min
for i in $(seq 1 $MAX); do
echo " Poll $i/$MAX …"
COUNT=$(gh api "repos/$REPO/issues/$ISSUE/comments" \
--jq "[.[] | select(.user.login == \"$BOT\")] | length")
if [ "$COUNT" -gt 0 ]; then
echo "✅ Bot commented!"
BODY=$(gh api "repos/$REPO/issues/$ISSUE/comments" \
--jq "[.[] | select(.user.login == \"$BOT\")][0].body")
echo "comment_body<<CBEOF" >> "$GITHUB_OUTPUT"
echo "$BODY" >> "$GITHUB_OUTPUT"
echo "CBEOF" >> "$GITHUB_OUTPUT"
echo "bot_commented=true" >> "$GITHUB_OUTPUT"
exit 0
fi
sleep 10
done
echo "bot_commented=false" >> "$GITHUB_OUTPUT"
echo "❌ Bot did not respond within 5 min"
gh run list --repo "$REPO" --limit 5 || true
exit 1
- name: Verify similarity detection
if: steps.poll.outputs.bot_commented == 'true'
env:
COMMENT_BODY: ${{ steps.poll.outputs.comment_body }}
run: |
if echo "$COMMENT_BODY" | grep -qiE "(similar|duplicate|related|#1)"; then
echo "✅ Similarity detected correctly"
else
echo "⚠️ Comment posted but no similarity reference found"
echo "$COMMENT_BODY"
fi
# ----- 11. Verify no loop -----
- name: Verify no infinite loop
if: steps.poll.outputs.bot_commented == 'true'
env:
GH_TOKEN: ${{ secrets.BOT_PAT }}
run: |
REPO="${{ env.BOT_NAME }}/${{ env.TEST_REPO_NAME }}"
ISSUE="${{ steps.trigger.outputs.issue_number }}"
BOT="github-actions[bot]"
echo "Waiting 60 s for potential re-triggers…"
sleep 60
N=$(gh api "repos/$REPO/issues/$ISSUE/comments" \
--jq "[.[] | select(.user.login == \"$BOT\")] | length")
echo "Bot comments on trigger issue: $N"
if [ "$N" -gt 1 ]; then
echo "❌ LOOP DETECTED – bot commented $N times"
exit 1
fi
echo "✅ No loop – single comment"
# ----- 11b. Auto-closer E2E test -----
# By this point the AC test issue (step 7b) and the trigger issue (step 8)
# have both been labeled "potential-duplicate" for 6–8 minutes, well past
# the 1-minute grace-period override used here.
# In dry-run mode the closer reports what it WOULD do, so no issues are
# actually closed. We verify:
# • The CLI runs without error and emits valid JSON.
# • At least the AC test issue is processed (processed >= 1).
# • Issues past grace with no human activity are reported as "closed"
# (dry-run), confirming Checks A, B, C found no human signals.
- name: Run auto-closer E2E test
id: ac_test
if: always() && steps.ac_setup.outputs.ac_issue_number != ''
env:
GITHUB_TOKEN: ${{ secrets.BOT_PAT }}
GH_TOKEN: ${{ secrets.BOT_PAT }}
run: |
REPO="${{ env.BOT_NAME }}/${{ env.TEST_REPO_NAME }}"
echo "Running auto-closer dry-run (grace-period-minutes=1)…"
# Use the config from the cloned test-repo working tree.
# Qdrant/Gemini sections are ignored by the auto-closer.
AC_OUTPUT=$("$RUNNER_TEMP/simili-cli" auto-close \
--repo "$REPO" \
--config "test-repo-wf/.github/simili.yaml" \
--dry-run \
--grace-period-minutes 1 \
--verbose \
2>/tmp/ac-stderr.txt) || AC_EXIT=$?
echo "--- stderr ---"
cat /tmp/ac-stderr.txt || true
echo "--- stdout (JSON) ---"
echo "$AC_OUTPUT"
# Parse summary fields from JSON output
PROCESSED=$(echo "$AC_OUTPUT" | jq -r '.processed // 0' 2>/dev/null || echo "0")
CLOSED=$(echo "$AC_OUTPUT" | jq -r '.closed // 0' 2>/dev/null || echo "0")
SKIPPED_GRACE=$(echo "$AC_OUTPUT" | jq -r '.skipped_grace_period // 0' 2>/dev/null || echo "0")
SKIPPED_HUMAN=$(echo "$AC_OUTPUT" | jq -r '.skipped_human_activity // 0' 2>/dev/null || echo "0")
echo "ac_processed=$PROCESSED" >> "$GITHUB_OUTPUT"
echo "ac_closed=$CLOSED" >> "$GITHUB_OUTPUT"
echo "ac_skipped_grace=$SKIPPED_GRACE" >> "$GITHUB_OUTPUT"
echo "ac_skipped_human=$SKIPPED_HUMAN" >> "$GITHUB_OUTPUT"
echo ""
echo "Auto-closer summary: processed=$PROCESSED closed=$CLOSED skipped_grace=$SKIPPED_GRACE skipped_human=$SKIPPED_HUMAN"
if [ "${AC_EXIT:-0}" -ne 0 ]; then
echo "❌ auto-close command failed (exit $AC_EXIT)"
exit 1
fi
if [ "$PROCESSED" -gt 0 ]; then
echo "✅ Auto-closer processed $PROCESSED issue(s) — dry-run OK"
else
echo "⚠️ Auto-closer found no issues to process (AC test issue may not be labeled yet)"
fi
# ----- 12. Cleanup Qdrant (always) -----
- name: Cleanup Qdrant collection
if: always()
run: |
CODE=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \
"${{ secrets.QDRANT_URL }}/collections/${{ env.TEST_COLLECTION }}" \
-H "api-key: ${{ secrets.QDRANT_API_KEY }}" \
-H "Content-Type: application/json")
echo "Qdrant cleanup: HTTP $CODE"
# ----- 13. Update test-repo README with results -----
- name: Update test repo README
if: always()
env:
GH_TOKEN: ${{ secrets.BOT_PAT }}
# Fallback to manual inputs
PR_NUM: ${{ github.event.pull_request.number || inputs.pr_number }}
CO_SHA: ${{ github.event.pull_request.head.sha || inputs.sha }}
run: |
rm -rf test-repo-results
gh repo clone "${{ env.BOT_NAME }}/${{ env.TEST_REPO_NAME }}" test-repo-results || exit 0
cd test-repo-results
git config user.name "${{ env.BOT_NAME }}"
git config user.email "makerslab.foss@gmail.com"
BOT_OK="${{ steps.poll.outputs.bot_commented }}"
cat > README.md << RMEOF
# Simili E2E Test Repository
| Detail | Value |
|--------|-------|
| **PR** | [#${PR_NUM}](https://github.com/${{ github.repository }}/pull/${PR_NUM}) |
| **Commit** | \`${CO_SHA:0:7}\` |
| **Bot responded** | $([ "$BOT_OK" = "true" ] && echo "✅" || echo "❌") |
| **Date** | $(date -u '+%Y-%m-%d %H:%M UTC') |
Check the **Issues** tab to see the bot's triage comment.
RMEOF
git add -A
git diff --cached --quiet || {
git commit -m "docs: update README with E2E results"
git push origin HEAD:main
}
# ----- 14. PR comment with link to test repo -----
- name: Comment on PR
if: always() && (github.event_name == 'pull_request_target' || github.event_name == 'workflow_dispatch')
env:
GH_TOKEN: ${{ secrets.BOT_PAT }}
PR_NUM: ${{ github.event.pull_request.number || inputs.pr_number }}
CO_SHA: ${{ github.event.pull_request.head.sha || inputs.sha }}
run: |
REPO="${{ env.BOT_NAME }}/${{ env.TEST_REPO_NAME }}"
OK="${{ steps.poll.outputs.bot_commented }}"
ICON=$([ "$OK" = "true" ] && echo "✅" || echo "❌")
AC_PROC="${{ steps.ac_test.outputs.ac_processed }}"
AC_CLOSED="${{ steps.ac_test.outputs.ac_closed }}"
AC_GRACE="${{ steps.ac_test.outputs.ac_skipped_grace }}"
AC_HUMAN="${{ steps.ac_test.outputs.ac_skipped_human }}"
if [ -n "$AC_PROC" ]; then
AC_LINE="| **Auto-closer (dry-run)** | processed: ${AC_PROC} closed: ${AC_CLOSED} grace: ${AC_GRACE} human: ${AC_HUMAN} |"
else
AC_LINE="| **Auto-closer** | skipped (AC setup failed) |"
fi
gh pr comment "$PR_NUM" \
--repo "${{ github.repository }}" \
--body "## 🧪 E2E Test
${ICON} **Bot responded:** $([ "$OK" = "true" ] && echo "yes" || echo "no")
${AC_LINE}
**Test repo →** [${REPO}](https://github.com/${REPO})
**Run →** [logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
<sub>Auto-generated by E2E pipeline</sub>"
# ----- 15. Summary -----
- name: Test summary
if: always()
run: |
{
echo "## E2E Test Summary"
echo ""
echo "| Check | Result |"
echo "|-------|--------|"
echo "| Repo created | ✅ |"
if [ "${{ steps.poll.outputs.bot_commented }}" = "true" ]; then
echo "| Bot responded | ✅ |"
else
echo "| Bot responded | ❌ |"
fi
echo "| Test repo | [${{ env.BOT_NAME }}/${{ env.TEST_REPO_NAME }}](https://github.com/${{ env.BOT_NAME }}/${{ env.TEST_REPO_NAME }}) |"
AC="${{ steps.ac_test.outputs.ac_processed }}"
if [ -n "$AC" ]; then
echo "| Auto-closer (dry-run) | processed: $AC closed: ${{ steps.ac_test.outputs.ac_closed }} grace: ${{ steps.ac_test.outputs.ac_skipped_grace }} human: ${{ steps.ac_test.outputs.ac_skipped_human }} |"
else
echo "| Auto-closer | ⚠️ no output |"
fi
} >> "$GITHUB_STEP_SUMMARY"