Skip to content

Commit 6a18514

Browse files
bodinsamuelbodinsamuel
authored
feat: in product metrics (NangoHQ#2541)
## Describe your changes Fixes https://linear.app/nango/issue/NAN-1466/empty-state-when-elasticsearch-is-not-present Fixes https://linear.app/nango/issue/NAN-1467/empty-state-when-there-is-no-data Fixes https://linear.app/nango/issue/NAN-1465/endpointquery-to-get-metrics Fixes https://linear.app/nango/issue/NAN-1464/code-the-ui - New endpoint `POST /api/v1/logs/insights` Retrieve insights by operations type, depending on the performance I might add some Redis cache. - Dashboard homepage The UI now displays the homepage by default, if you have finished your interactive demo. ## Test > Currently deployed in staging <img width="1512" alt="Screenshot 2024-07-25 at 10 25 11" src="https://github.com/user-attachments/assets/44cde003-dc9c-4f3b-957d-199f5d877587">
1 parent 31bdffd commit 6a18514

File tree

20 files changed

+1202
-75
lines changed

20 files changed

+1202
-75
lines changed

‎package-lock.json

+325
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/logs/lib/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@ export * from './client.js';
33
export * from './models/helpers.js';
44
export * from './models/logContextGetter.js';
55
export * as model from './models/messages.js';
6+
export * as modelOperations from './models/insights.js';
67
export { envs, defaultOperationExpiration } from './env.js';

‎packages/logs/lib/models/insights.ts

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import type { estypes } from '@elastic/elasticsearch';
2+
import { indexMessages } from '../es/schema.js';
3+
import { client } from '../es/client.js';
4+
import type { InsightsHistogramEntry } from '@nangohq/types';
5+
6+
export async function retrieveInsights(opts: { accountId: number; environmentId: number; type: string }) {
7+
const query: estypes.QueryDslQueryContainer = {
8+
bool: {
9+
must: [{ term: { accountId: opts.accountId } }, { term: { environmentId: opts.environmentId } }, { term: { 'operation.type': opts.type } }],
10+
must_not: { exists: { field: 'parentId' } },
11+
should: []
12+
}
13+
};
14+
15+
const res = await client.search<
16+
never,
17+
{
18+
histogram: estypes.AggregationsDateHistogramAggregate;
19+
}
20+
>({
21+
index: indexMessages.index,
22+
size: 0,
23+
sort: [{ createdAt: 'desc' }, 'id'],
24+
track_total_hits: true,
25+
aggs: {
26+
histogram: {
27+
date_histogram: { field: 'createdAt', calendar_interval: '1d', format: 'yyyy-MM-dd' },
28+
aggs: {
29+
state_agg: {
30+
terms: { field: 'state' }
31+
}
32+
}
33+
}
34+
},
35+
query
36+
});
37+
38+
const agg = res.aggregations!['histogram'];
39+
40+
const computed: InsightsHistogramEntry[] = [];
41+
for (const item of agg.buckets as any[]) {
42+
const success = (item.state_agg.buckets as { key: string; doc_count: number }[]).find((i) => i.key === 'success');
43+
const failure = (item.state_agg.buckets as { key: string; doc_count: number }[]).find((i) => i.key === 'failed');
44+
computed.push({ key: item.key_as_string, total: item.doc_count, success: success?.doc_count || 0, failure: failure?.doc_count || 0 });
45+
}
46+
47+
return {
48+
items: computed
49+
};
50+
}

‎packages/server/lib/controllers/v1/logs/postInsights.integration.test.ts

+88
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import { migrateLogsMapping } from '@nangohq/logs';
2+
import { multipleMigrations } from '@nangohq/database';
3+
import { seeders } from '@nangohq/shared';
4+
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
5+
import { isSuccess, runServer, shouldBeProtected, shouldRequireQueryEnv } from '../../../utils/tests.js';
6+
7+
let api: Awaited<ReturnType<typeof runServer>>;
8+
describe('POST /logs/insights', () => {
9+
beforeAll(async () => {
10+
await multipleMigrations();
11+
await migrateLogsMapping();
12+
13+
api = await runServer();
14+
});
15+
afterAll(() => {
16+
api.server.close();
17+
});
18+
19+
it('should be protected', async () => {
20+
const res = await api.fetch('/api/v1/logs/insights', { method: 'POST', query: { env: 'dev' }, body: { type: 'action' } });
21+
22+
shouldBeProtected(res);
23+
});
24+
25+
it('should enforce env query params', async () => {
26+
const { env } = await seeders.seedAccountEnvAndUser();
27+
const res = await api.fetch(
28+
'/api/v1/logs/insights',
29+
// @ts-expect-error missing query on purpose
30+
{
31+
method: 'POST',
32+
token: env.secret_key,
33+
body: { type: 'action' }
34+
}
35+
);
36+
37+
shouldRequireQueryEnv(res);
38+
});
39+
40+
it('should validate body', async () => {
41+
const { env } = await seeders.seedAccountEnvAndUser();
42+
const res = await api.fetch('/api/v1/logs/insights', {
43+
method: 'POST',
44+
query: {
45+
env: 'dev',
46+
// @ts-expect-error on purpose
47+
foo: 'bar'
48+
},
49+
token: env.secret_key,
50+
body: {
51+
// @ts-expect-error on purpose
52+
type: 'foobar'
53+
}
54+
});
55+
56+
expect(res.json).toStrictEqual<typeof res.json>({
57+
error: {
58+
code: 'invalid_query_params',
59+
errors: [
60+
{
61+
code: 'unrecognized_keys',
62+
message: "Unrecognized key(s) in object: 'foo'",
63+
path: []
64+
}
65+
]
66+
}
67+
});
68+
expect(res.res.status).toBe(400);
69+
});
70+
71+
it('should get empty result', async () => {
72+
const { env } = await seeders.seedAccountEnvAndUser();
73+
const res = await api.fetch('/api/v1/logs/insights', {
74+
method: 'POST',
75+
query: { env: 'dev' },
76+
token: env.secret_key,
77+
body: { type: 'sync' }
78+
});
79+
80+
isSuccess(res.json);
81+
expect(res.res.status).toBe(200);
82+
expect(res.json).toStrictEqual<typeof res.json>({
83+
data: {
84+
histogram: []
85+
}
86+
});
87+
});
88+
});

‎packages/server/lib/controllers/v1/logs/postInsights.ts

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import { z } from 'zod';
2+
import { asyncWrapper } from '../../../utils/asyncWrapper.js';
3+
import { requireEmptyQuery, zodErrorToHTTP } from '@nangohq/utils';
4+
import type { PostInsights } from '@nangohq/types';
5+
import { envs, modelOperations } from '@nangohq/logs';
6+
7+
const validation = z
8+
.object({
9+
type: z.enum(['sync', 'action', 'proxy', 'webhook_external'])
10+
})
11+
.strict();
12+
13+
export const postInsights = asyncWrapper<PostInsights>(async (req, res) => {
14+
if (!envs.NANGO_LOGS_ENABLED) {
15+
res.status(404).send({ error: { code: 'feature_disabled' } });
16+
return;
17+
}
18+
19+
const emptyQuery = requireEmptyQuery(req, { withEnv: true });
20+
if (emptyQuery) {
21+
res.status(400).send({ error: { code: 'invalid_query_params', errors: zodErrorToHTTP(emptyQuery.error) } });
22+
return;
23+
}
24+
25+
const val = validation.safeParse(req.body);
26+
if (!val.success) {
27+
res.status(400).send({
28+
error: { code: 'invalid_body', errors: zodErrorToHTTP(val.error) }
29+
});
30+
return;
31+
}
32+
33+
const env = res.locals['environment'];
34+
const body: PostInsights['Body'] = val.data;
35+
const insights = await modelOperations.retrieveInsights({
36+
accountId: env.account_id,
37+
environmentId: env.id,
38+
type: body.type
39+
});
40+
41+
res.status(200).send({
42+
data: {
43+
histogram: insights.items
44+
}
45+
});
46+
});

‎packages/server/lib/routes.ts

+3-1
Original file line numberDiff line numberDiff line change
@@ -63,11 +63,12 @@ import { deleteInvite } from './controllers/v1/invite/deleteInvite.js';
6363
import { deleteTeamUser } from './controllers/v1/team/users/deleteTeamUser.js';
6464
import { getUser } from './controllers/v1/user/getUser.js';
6565
import { patchUser } from './controllers/v1/user/patchUser.js';
66+
import { postInsights } from './controllers/v1/logs/postInsights.js';
6667
import { getInvite } from './controllers/v1/invite/getInvite.js';
6768
import { declineInvite } from './controllers/v1/invite/declineInvite.js';
6869
import { acceptInvite } from './controllers/v1/invite/acceptInvite.js';
69-
import { securityMiddlewares } from './middleware/security.js';
7070
import { getMeta } from './controllers/v1/meta/getMeta.js';
71+
import { securityMiddlewares } from './middleware/security.js';
7172
import { postManagedSignup } from './controllers/v1/account/managed/postSignup.js';
7273
import { getManagedCallback } from './controllers/v1/account/managed/getCallback.js';
7374
import { getEnvJs } from './controllers/v1/getEnvJs.js';
@@ -281,6 +282,7 @@ web.route('/api/v1/logs/operations').post(webAuth, searchOperations);
281282
web.route('/api/v1/logs/messages').post(webAuth, searchMessages);
282283
web.route('/api/v1/logs/filters').post(webAuth, searchFilters);
283284
web.route('/api/v1/logs/operations/:operationId').get(webAuth, getOperation);
285+
web.route('/api/v1/logs/insights').post(webAuth, postInsights);
284286

285287
// Hosted signin
286288
if (!isCloud && !isEnterprise) {

‎packages/types/lib/api.endpoints.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import type { EndpointMethod } from './api';
2-
import type { GetOperation, SearchFilters, SearchMessages, SearchOperations } from './logs/api';
2+
import type { GetOperation, PostInsights, SearchFilters, SearchMessages, SearchOperations } from './logs/api';
33
import type { GetOnboardingStatus } from './onboarding/api';
44
import type { SetMetadata, UpdateMetadata } from './connection/api/metadata';
55
import type { PostDeploy, PostDeployConfirmation } from './deploy/api';
@@ -18,6 +18,7 @@ export type APIEndpoints =
1818
| PostInvite
1919
| DeleteInvite
2020
| DeleteTeamUser
21+
| PostInsights
2122
| PostForgotPassword
2223
| PutResetPassword
2324
| SearchOperations

‎packages/webapp/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@
7272
"react-scripts": "5.0.1",
7373
"react-toastify": "9.1.1",
7474
"react-use": "17.5.0",
75+
"recharts": "2.12.7",
7576
"swr": "2.2.5",
7677
"tailwind-merge": "2.3.0",
7778
"tailwindcss": "3.4.3",

‎packages/webapp/src/App.tsx

+4-2
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,14 @@ import { VerifyEmail } from './pages/Account/VerifyEmail';
2727
import { VerifyEmailByExpiredToken } from './pages/Account/VerifyEmailByExpiredToken';
2828
import { EmailVerified } from './pages/Account/EmailVerified';
2929
import AuthLink from './pages/AuthLink';
30-
import { Homepage } from './pages/Homepage';
30+
import { Homepage } from './pages/Homepage/Show';
3131
import { NotFound } from './pages/NotFound';
3232
import { LogsSearch } from './pages/Logs/Search';
3333
import { TooltipProvider } from '@radix-ui/react-tooltip';
3434
import { SentryRoutes } from './utils/sentry';
3535
import { TeamSettings } from './pages/Team/Settings';
3636
import { UserSettings } from './pages/User/Settings';
37+
import { Root } from './pages/Root';
3738
import { globalEnv } from './utils/env';
3839

3940
const theme = createTheme({
@@ -69,8 +70,9 @@ const App = () => {
6970
}}
7071
>
7172
<SentryRoutes>
72-
<Route path="/" element={<Homepage />} />
73+
<Route path="/" element={<Root />} />
7374
<Route element={<PrivateRoute />} key={env}>
75+
<Route path="/:env" element={<Homepage />} />
7476
{showInteractiveDemo && (
7577
<Route path="/dev/interactive-demo" element={<PrivateRoute />}>
7678
<Route path="/dev/interactive-demo" element={<InteractiveDemo />} />

0 commit comments

Comments
 (0)