Skip to content

Commit 2bd1064

Browse files
duyetduyet
authored
Merge pull request #208 from duyet/chore/new-features
feat(cluster): expand parts count on each replicas, initial filterPreset
2 parents eed62cf + 0d83a4c commit 2bd1064

File tree

22 files changed

+492
-71
lines changed

22 files changed

+492
-71
lines changed

‎app/[query]/more/settings.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,22 @@ export const settingsConfig: QueryConfig = {
77
SELECT *
88
FROM system.settings
99
ORDER BY name
10-
`,
10+
`,
1111
columns: ['name', 'value', 'changed', 'description', 'default'],
1212
columnFormats: {
1313
name: ColumnFormat.Code,
1414
changed: ColumnFormat.Boolean,
1515
value: ColumnFormat.Code,
1616
default: ColumnFormat.Code,
1717
},
18+
defaultParams: {
19+
changed: '0',
20+
},
21+
filterParamPresets: [
22+
{
23+
name: 'Changed only',
24+
key: 'changed',
25+
sql: 'changed = 1',
26+
},
27+
],
1828
}

‎app/[query]/page.tsx‎

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,52 @@ export default async function Page({
3131
return notFound()
3232
}
3333

34+
// Get valid query params from URL (existing on QueryConfig['defaultParams'])
35+
const validQueryParams = Object.entries(searchParams).filter(([key, _]) => {
36+
return config.defaultParams && config.defaultParams[key] !== undefined
37+
})
38+
const validQueryParamsObj = Object.fromEntries(validQueryParams)
39+
40+
// Filter presets
41+
let condition = []
42+
const searchParamsKeys = ('' + searchParams['__presets'])
43+
.split(',')
44+
.map((key) => key.trim())
45+
.filter((key) => key !== '')
46+
for (const key of searchParamsKeys) {
47+
const preset = config.filterParamPresets?.find(
48+
(preset) => preset.key === key
49+
)
50+
if (preset) {
51+
condition.push(preset.sql)
52+
}
53+
}
54+
55+
let sql = config.sql
56+
if (condition.length > 0) {
57+
// Adding condition to the query after WHERE (if WHERE exists)
58+
// or after FROM (if WHERE doesn't exist)
59+
const whereIndex = sql.indexOf('WHERE')
60+
const fromIndex = sql.indexOf('FROM')
61+
const index = whereIndex !== -1 ? whereIndex : fromIndex
62+
sql =
63+
sql.slice(0, index) +
64+
' WHERE ' +
65+
condition.join(' AND ') +
66+
' AND ' +
67+
sql.slice(index)
68+
}
69+
70+
console.log('========', sql)
71+
3472
// Fetch the data from ClickHouse
3573
try {
3674
const queryParams = {
37-
...searchParams,
3875
...config.defaultParams,
76+
...validQueryParamsObj,
3977
}
4078
const data = await fetchData<RowData[]>({
41-
query: config.sql,
79+
query: sql,
4280
format: 'JSONEachRow',
4381
query_params: queryParams,
4482
})
@@ -47,13 +85,11 @@ export default async function Page({
4785
<div className="flex flex-col">
4886
<RelatedCharts relatedCharts={config.relatedCharts} />
4987

50-
<div>
51-
<DataTable
52-
title={query.replaceAll('-', ' ')}
53-
config={config}
54-
data={data}
55-
/>
56-
</div>
88+
<DataTable
89+
title={query.replaceAll('-', ' ')}
90+
config={config}
91+
data={data}
92+
/>
5793
</div>
5894
)
5995
} catch (error) {

‎app/clusters/[cluster]/parts-across-replicas/page.tsx‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import { ColumnFormat } from '@/components/data-table/column-defs'
2+
import { DataTable } from '@/components/data-table/data-table'
3+
import { fetchData } from '@/lib/clickhouse'
4+
5+
import type { QueryConfig } from '@/lib/types/query-config'
6+
7+
interface PageProps {
8+
params: {
9+
cluster: string
10+
}
11+
}
12+
13+
export default async function Page({ params: { cluster } }: PageProps) {
14+
const replicas = await fetchData<{ replica: string }[]>({
15+
query: `SELECT hostName() as replica FROM clusterAllReplicas({cluster: String}) ORDER BY 1`,
16+
query_params: { cluster },
17+
})
18+
console.log('Replicas', replicas)
19+
20+
if (replicas.length === 0) {
21+
return <div>No replicas found in cluster: {cluster}</div>
22+
}
23+
24+
const query = `
25+
SELECT database, table,
26+
${replicas.map(({ replica }) => `countIf(active AND hostName() = '${replica}') as \`${replica}\``).join(', ')},
27+
${replicas.map(({ replica }) => `formatReadableQuantity(\`${replica}\`) as \`readable_${replica}\``).join(', ')},
28+
${replicas.map(({ replica }) => `(100 * \`${replica}\` / max(\`${replica}\`) OVER ()) as \`pct_${replica}\``).join(', ')}
29+
30+
FROM clusterAllReplicas({cluster: String}, system.parts)
31+
WHERE database NOT IN ('system')
32+
GROUP BY 1, 2
33+
ORDER BY 3 DESC
34+
`
35+
36+
const config: QueryConfig = {
37+
name: 'count-across-replicas',
38+
description: 'Part count across replicas',
39+
sql: query,
40+
columns: [
41+
'database',
42+
'table',
43+
...replicas.map(({ replica }) => `readable_${replica}`),
44+
],
45+
columnFormats: {
46+
...replicas
47+
.map(({ replica }) => ({
48+
[`readable_${replica}`]: ColumnFormat.BackgroundBar,
49+
}))
50+
.reduce((acc, val) => ({ ...acc, ...val }), {}),
51+
},
52+
}
53+
54+
const rows = await fetchData<
55+
{
56+
database: string
57+
table: string
58+
[replica: string]: string | number
59+
}[]
60+
>({
61+
query: config.sql,
62+
query_params: { cluster },
63+
})
64+
65+
return (
66+
<DataTable
67+
title={`Count of active parts across replicas in the '${cluster}' cluster`}
68+
config={config}
69+
data={rows}
70+
/>
71+
)
72+
}

‎app/clusters/[cluster]/replicas-status/config.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ export const config: QueryConfig = {
2626
formatReadableSize(sum(bytes)) AS readable_total_bytes,
2727
(100 * total_bytes / max(total_bytes) OVER ()) AS pct_total_bytes,
2828
29+
countDistinct(database) AS database_count,
30+
countDistinct(database || table) AS table_count,
31+
2932
countIf(active) as active_part_count,
3033
(100 * active_part_count / max(active_part_count) OVER ()) AS pct_active_part_count,
3134
@@ -44,7 +47,7 @@ export const config: QueryConfig = {
4447
4548
FROM clusterAllReplicas({cluster: String}, system.parts)
4649
WHERE active
47-
GROUP BY 1
50+
GROUP BY host
4851
ORDER BY
4952
1 ASC,
5053
2 DESC
@@ -58,6 +61,8 @@ export const config: QueryConfig = {
5861
'active_part_count',
5962
'all_part_count',
6063
'last_modification_time',
64+
'database_count',
65+
'table_count',
6166
],
6267
columnFormats: {
6368
readable_total_rows: ColumnFormat.BackgroundBar,
@@ -66,5 +71,7 @@ export const config: QueryConfig = {
6671
readable_marks_bytes: ColumnFormat.BackgroundBar,
6772
active_part_count: ColumnFormat.BackgroundBar,
6873
all_part_count: ColumnFormat.BackgroundBar,
74+
database_count: ColumnFormat.BackgroundBar,
75+
table_count: ColumnFormat.BackgroundBar,
6976
},
7077
}

‎app/clusters/[cluster]/replicas-status/page.tsx‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import { DataTable } from '@/components/data-table/data-table'
22
import { fetchData } from '@/lib/clickhouse'
33

4+
import { Button } from '@/components/ui/button'
5+
import Link from 'next/link'
46
import { config, type Row } from './config'
57

68
interface PageProps {
@@ -9,7 +11,7 @@ interface PageProps {
911
}
1012
}
1113

12-
export default async function ClustersPage({ params: { cluster } }: PageProps) {
14+
export default async function Page({ params: { cluster } }: PageProps) {
1315
const tables = await fetchData<Row[]>({
1416
query: config.sql,
1517
query_params: { cluster },
@@ -20,6 +22,18 @@ export default async function ClustersPage({ params: { cluster } }: PageProps) {
2022
title={`Row counts across '${cluster}' cluster`}
2123
config={config}
2224
data={tables}
25+
topRightToolbarExtras={<TopRightToolbarExtras cluster={cluster} />}
2326
/>
2427
)
2528
}
29+
30+
const TopRightToolbarExtras = ({ cluster }: PageProps['params']) => (
31+
<Link href={`/clusters/${cluster}/parts-across-replicas`}>
32+
<Button
33+
variant="outline"
34+
className="flex flex-row gap-2 text-muted-foreground"
35+
>
36+
Parts on each tables
37+
</Button>
38+
</Link>
39+
)

‎app/database/[database]/[table]/page.tsx‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,10 @@ export default async function ColumnsPage({
126126
<DataTable
127127
title={`Table: ${database}.${table}`}
128128
description={description}
129-
extras={<Extras database={database} table={table} />}
130-
toolbarExtras={<ToolbarExtras database={database} table={table} />}
129+
toolbarExtras={<Extras database={database} table={table} />}
130+
topRightToolbarExtras={
131+
<TopRightToolbarExtras database={database} table={table} />
132+
}
131133
config={config}
132134
data={columns}
133135
/>
@@ -168,7 +170,7 @@ const Extras = ({ database, table }: { database: string; table: string }) => (
168170
</div>
169171
)
170172

171-
const ToolbarExtras = ({
173+
const TopRightToolbarExtras = ({
172174
database,
173175
table,
174176
}: {

‎app/database/[database]/page.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ export default async function TableListPage({
5757
title={`Database: ${database}`}
5858
config={config}
5959
data={tables}
60-
toolbarExtras={<ToolbarExtras database={database} />}
60+
topRightToolbarExtras={<ToolbarExtras database={database} />}
6161
/>
6262
)
6363
}

‎app/replica/[replica]/tables/config.ts‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import { ColumnFormat } from '@/components/data-table/column-defs'
2+
import { type QueryConfig } from '@/lib/types/query-config'
3+
4+
export type Row = {
5+
host: string
6+
readable_total_rows: string
7+
readable_total_bytes: string
8+
readable_primary_key_size: string
9+
readable_marks_bytes: string
10+
part_count: number
11+
last_modification_time: string
12+
}
13+
14+
export const config: QueryConfig = {
15+
name: 'count-in-replicas',
16+
description: 'Count across replicas for all tables in the cluster',
17+
sql: `
18+
SELECT
19+
hostName() AS host,
20+
21+
sum(rows) AS total_rows,
22+
formatReadableQuantity(total_rows) AS readable_total_rows,
23+
(100 * total_rows / max(total_rows) OVER ()) AS pct_total_rows,
24+
25+
sum(bytes) AS total_bytes,
26+
formatReadableSize(sum(bytes)) AS readable_total_bytes,
27+
(100 * total_bytes / max(total_bytes) OVER ()) AS pct_total_bytes,
28+
29+
countDistinct(database) AS database_count,
30+
countDistinct(database || table) AS table_count,
31+
32+
countIf(active) as active_part_count,
33+
(100 * active_part_count / max(active_part_count) OVER ()) AS pct_active_part_count,
34+
35+
count() as all_part_count,
36+
(100 * all_part_count / max(all_part_count) OVER ()) AS pct_all_part_count,
37+
38+
sum(primary_key_size) AS primary_key_size,
39+
formatReadableSize(primary_key_size) AS readable_primary_key_size,
40+
(100 * primary_key_size / max(primary_key_size) OVER ()) AS pct_primary_key_size,
41+
42+
sum(marks_bytes) AS marks_bytes,
43+
formatReadableSize(marks_bytes) AS readable_marks_bytes,
44+
(100 * marks_bytes / max(marks_bytes) OVER ()) AS pct_marks_bytes,
45+
46+
max(modification_time) AS last_modification_time
47+
48+
FROM clusterAllReplicas({cluster: String}, system.parts)
49+
WHERE active
50+
GROUP BY 1
51+
ORDER BY
52+
1 ASC,
53+
2 DESC
54+
`,
55+
columns: [
56+
'host',
57+
'readable_total_rows',
58+
'readable_total_bytes',
59+
'readable_primary_key_size',
60+
'readable_marks_bytes',
61+
'active_part_count',
62+
'all_part_count',
63+
'last_modification_time',
64+
'database_count',
65+
'table_count',
66+
],
67+
columnFormats: {
68+
readable_total_rows: ColumnFormat.BackgroundBar,
69+
readable_total_bytes: ColumnFormat.BackgroundBar,
70+
readable_primary_key_size: ColumnFormat.BackgroundBar,
71+
readable_marks_bytes: ColumnFormat.BackgroundBar,
72+
active_part_count: ColumnFormat.BackgroundBar,
73+
all_part_count: ColumnFormat.BackgroundBar,
74+
database_count: ColumnFormat.BackgroundBar,
75+
table_count: ColumnFormat.BackgroundBar,
76+
},
77+
}

‎app/replica/[replica]/tables/page.tsx‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import { DataTable } from '@/components/data-table/data-table'
2+
import { fetchData } from '@/lib/clickhouse'
3+
4+
import { config, type Row } from './config'
5+
6+
interface PageProps {
7+
params: {
8+
replica: string
9+
}
10+
}
11+
12+
export default async function ClustersPage({ params: { replica } }: PageProps) {
13+
const tables = await fetchData<Row[]>({
14+
query: config.sql,
15+
query_params: { replica },
16+
})
17+
18+
return (
19+
<DataTable
20+
title={`Tables in replica - ${replica}`}
21+
config={config}
22+
data={tables}
23+
/>
24+
)
25+
}

‎components/data-table/cell.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,6 @@ export const formatCell = <TData extends RowData, TValue>(
8686
)
8787

8888
default:
89-
return <span className='text-nowrap'>{value as string}</span>
89+
return <span className="text-nowrap">{value as string}</span>
9090
}
9191
}

0 commit comments

Comments
 (0)