Skip to content

Commit ae5efbc

Browse files
authored
Merge branch 'main' into code-interpreter-handle-large-files
2 parents 06b58fe + 5cdeb84 commit ae5efbc

File tree

241 files changed

+16106
-1877
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

241 files changed

+16106
-1877
lines changed

.github/workflows/deployment.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ jobs:
151151
fetch-depth: 0
152152

153153
- name: Setup uv
154-
uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # ratchet:astral-sh/setup-uv@v7
154+
uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # ratchet:astral-sh/setup-uv@v7
155155
with:
156156
version: "0.9.9"
157157
# NOTE: This isn't caching much and zizmor suggests this could be poisoned, so disable.

.github/workflows/post-merge-beta-cherry-pick.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ jobs:
7070

7171
- name: Install the latest version of uv
7272
if: steps.gate.outputs.should_cherrypick == 'true'
73-
uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # ratchet:astral-sh/setup-uv@v7
73+
uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # ratchet:astral-sh/setup-uv@v7
7474
with:
7575
enable-cache: false
7676
version: "0.9.9"

.github/workflows/pr-playwright-tests.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -468,7 +468,7 @@ jobs:
468468

469469
- name: Install the latest version of uv
470470
if: always()
471-
uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # ratchet:astral-sh/setup-uv@v7
471+
uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # ratchet:astral-sh/setup-uv@v7
472472
with:
473473
enable-cache: false
474474
version: "0.9.9"
@@ -707,7 +707,7 @@ jobs:
707707
pull-requests: write
708708
steps:
709709
- name: Download visual diff summaries
710-
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
710+
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3
711711
with:
712712
pattern: screenshot-diff-summary-*
713713
path: summaries/

.github/workflows/pr-quality-checks.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ jobs:
2828
with:
2929
python-version: "3.11"
3030
- name: Setup Terraform
31-
uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # ratchet:hashicorp/setup-terraform@v3
31+
uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # ratchet:hashicorp/setup-terraform@v4.0.0
3232
- name: Setup node
3333
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # ratchet:actions/setup-node@v6
3434
with: # zizmor: ignore[cache-poisoning]

.github/workflows/release-devtools.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ jobs:
2626
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
2727
with:
2828
persist-credentials: false
29-
- uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # ratchet:astral-sh/setup-uv@v7
29+
- uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # ratchet:astral-sh/setup-uv@v7
3030
with:
3131
enable-cache: false
3232
version: "0.9.9"
Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: Storybook Deploy
2+
env:
3+
VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
4+
VERCEL_PROJECT_ID: prj_sG49mVsA25UsxIPhN2pmBJlikJZM
5+
VERCEL_CLI: vercel@50.14.1
6+
VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
7+
8+
concurrency:
9+
group: storybook-deploy-production
10+
cancel-in-progress: true
11+
12+
on:
13+
workflow_dispatch:
14+
push:
15+
branches:
16+
- main
17+
paths:
18+
- "web/lib/opal/**"
19+
- "web/src/refresh-components/**"
20+
- "web/.storybook/**"
21+
- "web/package.json"
22+
- "web/package-lock.json"
23+
permissions:
24+
contents: read
25+
jobs:
26+
Deploy-Storybook:
27+
runs-on: ubuntu-latest
28+
timeout-minutes: 30
29+
steps:
30+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v4
31+
with:
32+
persist-credentials: false
33+
34+
- name: Setup node
35+
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # ratchet:actions/setup-node@v4
36+
with:
37+
node-version: 22
38+
cache: "npm"
39+
cache-dependency-path: ./web/package-lock.json
40+
41+
- name: Install dependencies
42+
working-directory: web
43+
run: npm ci
44+
45+
- name: Build Storybook
46+
working-directory: web
47+
run: npm run storybook:build
48+
49+
- name: Deploy to Vercel (Production)
50+
working-directory: web
51+
run: npx --yes "$VERCEL_CLI" deploy storybook-static/ --prod --yes
52+
53+
notify-slack-on-failure:
54+
needs: Deploy-Storybook
55+
if: always() && needs.Deploy-Storybook.result == 'failure'
56+
runs-on: ubuntu-latest
57+
timeout-minutes: 10
58+
steps:
59+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v4
60+
with:
61+
persist-credentials: false
62+
sparse-checkout: .github/actions/slack-notify
63+
64+
- name: Send Slack notification
65+
uses: ./.github/actions/slack-notify
66+
with:
67+
webhook-url: ${{ secrets.MONITOR_DEPLOYMENTS_WEBHOOK }}
68+
failed-jobs: "• Deploy-Storybook"
69+
title: "🚨 Storybook Deploy Failed"

.github/workflows/zizmor.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
persist-credentials: false
2525

2626
- name: Install the latest version of uv
27-
uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # ratchet:astral-sh/setup-uv@v7
27+
uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # ratchet:astral-sh/setup-uv@v7
2828
with:
2929
enable-cache: false
3030
version: "0.9.9"

backend/Dockerfile

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,9 @@ RUN apt-get update && \
4646
pkg-config \
4747
gcc \
4848
nano \
49-
vim && \
49+
vim \
50+
libjemalloc2 \
51+
&& \
5052
rm -rf /var/lib/apt/lists/* && \
5153
apt-get clean
5254

@@ -141,6 +143,7 @@ COPY --chown=onyx:onyx ./scripts/debugging /app/scripts/debugging
141143
COPY --chown=onyx:onyx ./scripts/force_delete_connector_by_id.py /app/scripts/force_delete_connector_by_id.py
142144
COPY --chown=onyx:onyx ./scripts/supervisord_entrypoint.sh /app/scripts/supervisord_entrypoint.sh
143145
COPY --chown=onyx:onyx ./scripts/setup_craft_templates.sh /app/scripts/setup_craft_templates.sh
146+
COPY --chown=onyx:onyx ./scripts/reencrypt_secrets.py /app/scripts/reencrypt_secrets.py
144147
RUN chmod +x /app/scripts/supervisord_entrypoint.sh /app/scripts/setup_craft_templates.sh
145148

146149
# Run Craft template setup at build time when ENABLE_CRAFT=true
@@ -164,6 +167,13 @@ ENV PYTHONPATH=/app
164167
ARG ONYX_VERSION=0.0.0-dev
165168
ENV ONYX_VERSION=${ONYX_VERSION}
166169

170+
# Use jemalloc instead of glibc malloc to reduce memory fragmentation
171+
# in long-running Python processes (API server, Celery workers).
172+
# The soname is architecture-independent; the dynamic linker resolves
173+
# the correct path from standard library directories.
174+
# Placed after all RUN steps so build-time processes are unaffected.
175+
ENV LD_PRELOAD=libjemalloc.so.2
176+
167177
# Default command which does nothing
168178
# This container is used by api server and background which specify their own CMD
169179
CMD ["tail", "-f", "/dev/null"]
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
"""add hierarchy_node_by_connector_credential_pair table
2+
3+
Revision ID: b5c4d7e8f9a1
4+
Revises: a3b8d9e2f1c4
5+
Create Date: 2026-03-04
6+
7+
"""
8+
9+
import sqlalchemy as sa
10+
from alembic import op
11+
12+
revision = "b5c4d7e8f9a1"
13+
down_revision = "a3b8d9e2f1c4"
14+
branch_labels = None
15+
depends_on = None
16+
17+
18+
def upgrade() -> None:
19+
op.create_table(
20+
"hierarchy_node_by_connector_credential_pair",
21+
sa.Column("hierarchy_node_id", sa.Integer(), nullable=False),
22+
sa.Column("connector_id", sa.Integer(), nullable=False),
23+
sa.Column("credential_id", sa.Integer(), nullable=False),
24+
sa.ForeignKeyConstraint(
25+
["hierarchy_node_id"],
26+
["hierarchy_node.id"],
27+
ondelete="CASCADE",
28+
),
29+
sa.ForeignKeyConstraint(
30+
["connector_id", "credential_id"],
31+
[
32+
"connector_credential_pair.connector_id",
33+
"connector_credential_pair.credential_id",
34+
],
35+
ondelete="CASCADE",
36+
),
37+
sa.PrimaryKeyConstraint("hierarchy_node_id", "connector_id", "credential_id"),
38+
)
39+
op.create_index(
40+
"ix_hierarchy_node_cc_pair_connector_credential",
41+
"hierarchy_node_by_connector_credential_pair",
42+
["connector_id", "credential_id"],
43+
)
44+
45+
46+
def downgrade() -> None:
47+
op.drop_index(
48+
"ix_hierarchy_node_cc_pair_connector_credential",
49+
table_name="hierarchy_node_by_connector_credential_pair",
50+
)
51+
op.drop_table("hierarchy_node_by_connector_credential_pair")

backend/alembic_tenants/versions/3b9f09038764_add_read_only_kg_user.py

Lines changed: 40 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
from alembic import op
1212
from onyx.configs.app_configs import DB_READONLY_PASSWORD
1313
from onyx.configs.app_configs import DB_READONLY_USER
14-
from shared_configs.configs import MULTI_TENANT
1514

1615

1716
# revision identifiers, used by Alembic.
@@ -22,59 +21,52 @@
2221

2322

2423
def upgrade() -> None:
25-
if MULTI_TENANT:
24+
# Enable pg_trgm extension if not already enabled
25+
op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
2626

27-
# Enable pg_trgm extension if not already enabled
28-
op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
27+
# Create the read-only db user if it does not already exist.
28+
if not (DB_READONLY_USER and DB_READONLY_PASSWORD):
29+
raise Exception("DB_READONLY_USER or DB_READONLY_PASSWORD is not set")
2930

30-
# Create read-only db user here only in multi-tenant mode. For single-tenant mode,
31-
# the user is created in the standard migration.
32-
if not (DB_READONLY_USER and DB_READONLY_PASSWORD):
33-
raise Exception("DB_READONLY_USER or DB_READONLY_PASSWORD is not set")
34-
35-
op.execute(
36-
text(
37-
f"""
38-
DO $$
39-
BEGIN
40-
-- Check if the read-only user already exists
41-
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '{DB_READONLY_USER}') THEN
42-
-- Create the read-only user with the specified password
43-
EXECUTE format('CREATE USER %I WITH PASSWORD %L', '{DB_READONLY_USER}', '{DB_READONLY_PASSWORD}');
44-
-- First revoke all privileges to ensure a clean slate
45-
EXECUTE format('REVOKE ALL ON DATABASE %I FROM %I', current_database(), '{DB_READONLY_USER}');
46-
-- Grant only the CONNECT privilege to allow the user to connect to the database
47-
-- but not perform any operations without additional specific grants
48-
EXECUTE format('GRANT CONNECT ON DATABASE %I TO %I', current_database(), '{DB_READONLY_USER}');
49-
END IF;
50-
END
51-
$$;
52-
"""
53-
)
54-
)
55-
56-
57-
def downgrade() -> None:
58-
if MULTI_TENANT:
59-
# Drop read-only db user here only in single tenant mode. For multi-tenant mode,
60-
# the user is dropped in the alembic_tenants migration.
61-
62-
op.execute(
63-
text(
64-
f"""
31+
op.execute(
32+
text(
33+
f"""
6534
DO $$
6635
BEGIN
67-
IF EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '{DB_READONLY_USER}') THEN
68-
-- First revoke all privileges from the database
36+
-- Check if the read-only user already exists
37+
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '{DB_READONLY_USER}') THEN
38+
-- Create the read-only user with the specified password
39+
EXECUTE format('CREATE USER %I WITH PASSWORD %L', '{DB_READONLY_USER}', '{DB_READONLY_PASSWORD}');
40+
-- First revoke all privileges to ensure a clean slate
6941
EXECUTE format('REVOKE ALL ON DATABASE %I FROM %I', current_database(), '{DB_READONLY_USER}');
70-
-- Then revoke all privileges from the public schema
71-
EXECUTE format('REVOKE ALL ON SCHEMA public FROM %I', '{DB_READONLY_USER}');
72-
-- Then drop the user
73-
EXECUTE format('DROP USER %I', '{DB_READONLY_USER}');
42+
-- Grant only the CONNECT privilege to allow the user to connect to the database
43+
-- but not perform any operations without additional specific grants
44+
EXECUTE format('GRANT CONNECT ON DATABASE %I TO %I', current_database(), '{DB_READONLY_USER}');
7445
END IF;
7546
END
7647
$$;
77-
"""
78-
)
48+
"""
49+
)
50+
)
51+
52+
53+
def downgrade() -> None:
54+
op.execute(
55+
text(
56+
f"""
57+
DO $$
58+
BEGIN
59+
IF EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '{DB_READONLY_USER}') THEN
60+
-- First revoke all privileges from the database
61+
EXECUTE format('REVOKE ALL ON DATABASE %I FROM %I', current_database(), '{DB_READONLY_USER}');
62+
-- Then revoke all privileges from the public schema
63+
EXECUTE format('REVOKE ALL ON SCHEMA public FROM %I', '{DB_READONLY_USER}');
64+
-- Then drop the user
65+
EXECUTE format('DROP USER %I', '{DB_READONLY_USER}');
66+
END IF;
67+
END
68+
$$;
69+
"""
7970
)
80-
op.execute(text("DROP EXTENSION IF EXISTS pg_trgm"))
71+
)
72+
op.execute(text("DROP EXTENSION IF EXISTS pg_trgm"))

0 commit comments

Comments
 (0)