Skip to content

Commit d9748d1

Browse files
authored
Unify metadata UI for Quilt package, parquet and h5ad (new) tables (#4665)
1 parent d6dd34c commit d9748d1

6 files changed

Lines changed: 359 additions & 132 deletions

File tree

catalog/CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ where verb is one of
1818

1919
## Changes
2020

21+
- [Added] H5AD file preview ([#4665](https://github.com/quiltdata/quilt/pull/4665))
22+
- [Changed] Unified metadata table design for H5AD, Parquet, and Quilt package previews ([#4665](https://github.com/quiltdata/quilt/pull/4665))
2123
- [Changed] Migrate test framework from Jest to Vitest with improved performance ([#4657](https://github.com/quiltdata/quilt/pull/4657))
2224
- [Changed] Add `crc64Checksums` config property indicating CRC64/NVMe checksums are enabled ([#4632](https://github.com/quiltdata/quilt/pull/4632))
2325
- [Changed] Remove outdated `chunkedChecksums` config property, and remove related dead code ([#4632](https://github.com/quiltdata/quilt/pull/4632))

catalog/app/components/Preview/loaders/Manifest.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ export const Loader = function ManifestLoader({ gated, handle, children }: Loade
5858
})),
5959
entries,
6060
)
61-
return PreviewData.Perspective({ packageMeta, data: packageEntries })
61+
return PreviewData.Perspective({ meta: packageMeta, data: packageEntries })
6262
} catch (e) {
6363
if (e instanceof SyntaxError) {
6464
const head = data.head.join('\n')

catalog/app/components/Preview/loaders/Tabular.tsx

Lines changed: 46 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,14 @@ const isParquet = R.anyPass([
2929

3030
const isTsv = utils.extIn(['.tsv', '.tab'])
3131

32+
const isH5ad = utils.extIs('.h5ad')
33+
3234
export const detect = R.pipe(
3335
utils.stripCompression,
34-
R.anyPass([isCsv, isExcel, isJsonl, isParquet, isTsv]),
36+
R.anyPass([isCsv, isExcel, isJsonl, isParquet, isTsv, isH5ad]),
3537
)
3638

37-
type TabularType = 'csv' | 'jsonl' | 'excel' | 'parquet' | 'tsv'
39+
type TabularType = 'csv' | 'jsonl' | 'excel' | 'parquet' | 'tsv' | 'h5ad'
3840

3941
const detectTabularType: (type: string) => TabularType = R.pipe(
4042
utils.stripCompression,
@@ -44,11 +46,12 @@ const detectTabularType: (type: string) => TabularType = R.pipe(
4446
[isJsonl, R.always('jsonl')],
4547
[isParquet, R.always('parquet')],
4648
[isTsv, R.always('tsv')],
49+
[isH5ad, R.always('h5ad')],
4750
[R.T, R.always('csv')],
4851
]),
4952
)
5053

51-
interface ParquetMetadataBackend {
54+
export interface ParquetMetadata {
5255
created_by: string
5356
format_version: string
5457
num_row_groups: number
@@ -59,20 +62,37 @@ interface ParquetMetadataBackend {
5962
shape: [number, number] // rows, columns
6063
}
6164

62-
export interface ParquetMetadata {
63-
createdBy: string
64-
formatVersion: string
65-
numRowGroups: number
65+
export interface H5adMetadata {
66+
created_by: string
67+
format_version: string
68+
num_row_groups: number
6669
schema: {
6770
names: string[]
6871
}
69-
serializedSize: number
70-
shape: { rows: number; columns: number }
72+
serialized_size: number
73+
shape: [number, number] // rows, columns
74+
h5ad_obs_keys: string[]
75+
h5ad_var_keys: string[]
76+
h5ad_uns_keys: string[]
77+
h5ad_obsm_keys: string[]
78+
h5ad_varm_keys: string[]
79+
h5ad_layers_keys: string[]
80+
anndata_version?: string
81+
n_cells: number
82+
n_genes: number
83+
matrix_type: string
84+
has_raw: boolean
85+
}
86+
87+
export interface PackageMetadata {
88+
version?: string
89+
workflow?: any
90+
message?: string
7191
}
7292

7393
function getQuiltInfo(
7494
headers: Headers,
75-
): { meta?: ParquetMetadataBackend; truncated: boolean } | null {
95+
): { meta?: ParquetMetadata | H5adMetadata; truncated: boolean } | null {
7696
try {
7797
const header = headers.get('x-quilt-info')
7898
return header ? JSON.parse(header) : null
@@ -99,15 +119,6 @@ async function getCsvFromResponse(r: Response): Promise<ArrayBuffer | string> {
99119
return isArrow ? r.arrayBuffer() : r.text()
100120
}
101121

102-
export const parseParquetData = (data: ParquetMetadataBackend): ParquetMetadata => ({
103-
createdBy: data.created_by,
104-
formatVersion: data.format_version,
105-
numRowGroups: data.num_row_groups,
106-
schema: data.schema,
107-
serializedSize: data.serialized_size,
108-
shape: { rows: data.shape[0], columns: data.shape[1] },
109-
})
110-
111122
interface LoadTabularDataArgs {
112123
compression?: 'gz' | 'bz2'
113124
handle: Model.S3.S3ObjectLocation
@@ -118,7 +129,7 @@ interface LoadTabularDataArgs {
118129

119130
interface TabularDataOutput {
120131
csv: ArrayBuffer | string
121-
parquetMeta: ParquetMetadata | null
132+
meta: ParquetMetadata | H5adMetadata | null
122133
size: number | null
123134
truncated: boolean
124135
}
@@ -151,7 +162,7 @@ const loadTabularData = async ({
151162

152163
return {
153164
csv,
154-
parquetMeta: quiltInfo?.meta ? parseParquetData(quiltInfo?.meta) : null,
165+
meta: quiltInfo?.meta || null,
155166
size: contentLength,
156167
truncated: !!quiltInfo?.truncated,
157168
}
@@ -188,11 +199,21 @@ export const Loader = function TabularLoader({
188199
const [gated, setGated] = React.useState(true)
189200
const sign = AWS.Signer.useS3Signer()
190201
const type = React.useMemo(() => detectTabularType(handle.key), [handle.key])
191-
const onLoadMore = React.useCallback(() => setGated(false), [setGated])
202+
const onLoadMore = React.useCallback(() => setGated(false), [])
192203
const size = React.useMemo(
193204
() => getNeededSize(options.context, gated),
194205
[options.context, gated],
195206
)
207+
const showLoadMore = React.useCallback(
208+
(truncated) =>
209+
// There is more data to show
210+
truncated &&
211+
// We explicitly had shown less data
212+
size !== 'large' &&
213+
// tabular-preview is able to show more
214+
!isH5ad(handle.key),
215+
[size, handle.key],
216+
)
196217

197218
const compression = utils.getCompression(handle.key)
198219
const data = Data.use(loadTabularData, {
@@ -205,13 +226,13 @@ export const Loader = function TabularLoader({
205226
// TODO: get correct sizes from API
206227
const processed = utils.useProcessing(
207228
data.result,
208-
({ csv, parquetMeta, truncated }: TabularDataOutput) =>
229+
({ csv, meta, truncated }: TabularDataOutput) =>
209230
PreviewData.Perspective({
210231
data: csv,
211232
handle,
212233
modes: [FileType.Tabular, FileType.Text],
213-
parquetMeta,
214-
onLoadMore: truncated && size !== 'large' ? onLoadMore : null,
234+
meta,
235+
onLoadMore: showLoadMore(truncated) ? onLoadMore : null,
215236
truncated,
216237
}),
217238
)
Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
import * as React from 'react'
2+
import { render, screen } from '@testing-library/react'
3+
import { describe, expect, it, vi } from 'vitest'
4+
import { ThemeOptions, ThemeProvider, createMuiTheme } from '@material-ui/core/styles'
5+
6+
import type { Json } from 'utils/types'
7+
8+
import type {
9+
ParquetMetadata,
10+
H5adMetadata,
11+
PackageMetadata,
12+
} from '../../loaders/Tabular'
13+
14+
import Metadata from './Metadata'
15+
16+
vi.mock('constants/config', () => ({ default: {} }))
17+
18+
vi.mock('components/JsonDisplay', () => ({
19+
default: ({ value }: { value: Json }) => (
20+
<div data-testid="json">{JSON.stringify(value)}</div>
21+
),
22+
}))
23+
24+
const theme = createMuiTheme({
25+
typography: {
26+
monospace: {
27+
fontFamily: 'monospace',
28+
},
29+
} as ThemeOptions['typography'],
30+
})
31+
32+
function renderWithTheme(component: React.ReactElement) {
33+
return render(<ThemeProvider theme={theme}>{component}</ThemeProvider>)
34+
}
35+
36+
describe('components/Preview/renderers/Perspective/Metadata', () => {
37+
const mockParquetMetadata: ParquetMetadata = {
38+
created_by: 'Apache Parquet Writer v1.0',
39+
format_version: '1.0',
40+
num_row_groups: 5,
41+
schema: {
42+
names: ['id', 'name', 'value'],
43+
},
44+
serialized_size: 1024000,
45+
shape: [10000, 3],
46+
}
47+
48+
const mockH5adMetadata: H5adMetadata = {
49+
created_by: 'scanpy==1.9.1',
50+
format_version: 'h5ad',
51+
num_row_groups: 1,
52+
schema: {
53+
names: ['gene_id', 'gene_name', 'expression'],
54+
},
55+
serialized_size: 2048000,
56+
shape: [5000, 2000],
57+
h5ad_obs_keys: ['cell_type', 'tissue'],
58+
h5ad_var_keys: ['gene_symbol', 'chromosome'],
59+
h5ad_uns_keys: ['pca', 'neighbors'],
60+
h5ad_obsm_keys: ['X_pca', 'X_umap'],
61+
h5ad_varm_keys: ['PCs'],
62+
h5ad_layers_keys: ['raw', 'normalized'],
63+
anndata_version: '0.8.0',
64+
n_cells: 5000,
65+
n_genes: 2000,
66+
matrix_type: 'sparse',
67+
has_raw: true,
68+
}
69+
70+
const mockPackageMetadata: PackageMetadata = {
71+
version: '1.2.3',
72+
workflow: {
73+
is_valid: true,
74+
config: {
75+
name: 'test-workflow',
76+
metadata_schema: {},
77+
},
78+
},
79+
message: 'Initial package version with comprehensive data',
80+
}
81+
82+
it('should render ParquetMetadata with caption and fields', () => {
83+
renderWithTheme(<Metadata className="pq" metadata={mockParquetMetadata} />)
84+
85+
// Check caption is rendered
86+
expect(screen.getByText('Apache Parquet Writer v1.0')).toBeTruthy()
87+
88+
// Check specific fields are rendered
89+
expect(screen.getByText('Format version:')).toBeTruthy()
90+
expect(screen.getByText('1.0')).toBeTruthy()
91+
92+
expect(screen.getByText('# row groups:')).toBeTruthy()
93+
expect(screen.getByText('5')).toBeTruthy()
94+
95+
expect(screen.getByText('Shape:')).toBeTruthy()
96+
expect(screen.getByText('10000 rows × 3 columns')).toBeTruthy()
97+
98+
expect(screen.getByText('Serialized size:')).toBeTruthy()
99+
expect(screen.getByText('1024000')).toBeTruthy()
100+
101+
// Check JsonDisplay is used for schema
102+
expect(screen.getByText('Schema:')).toBeTruthy()
103+
expect(screen.getByTestId('json')).toBeTruthy()
104+
})
105+
106+
it('should render H5adMetadata with H5AD-specific fields', () => {
107+
renderWithTheme(<Metadata className="h5ad" metadata={mockH5adMetadata} />)
108+
109+
// Check caption
110+
expect(screen.getByText('scanpy==1.9.1')).toBeTruthy()
111+
112+
// Check H5AD-specific fields
113+
expect(screen.getByText('Cells:')).toBeTruthy()
114+
expect(screen.getByText('5000')).toBeTruthy()
115+
116+
expect(screen.getByText('Genes:')).toBeTruthy()
117+
expect(screen.getByText('2000')).toBeTruthy()
118+
119+
expect(screen.getByText('Matrix type:')).toBeTruthy()
120+
expect(screen.getByText('sparse')).toBeTruthy()
121+
122+
expect(screen.getByText('Has raw data:')).toBeTruthy()
123+
expect(screen.getByText('✓')).toBeTruthy()
124+
125+
expect(screen.getByText('AnnData version:')).toBeTruthy()
126+
expect(screen.getByText('0.8.0')).toBeTruthy()
127+
128+
// Check array fields are rendered with JsonDisplay
129+
expect(screen.getByText('Cell metadata keys:')).toBeTruthy()
130+
expect(screen.getByText('Gene metadata keys:')).toBeTruthy()
131+
expect(screen.getByText('Cell embeddings:')).toBeTruthy()
132+
expect(screen.getByText('Expression layers:')).toBeTruthy()
133+
134+
// Should have multiple JsonDisplay components for arrays
135+
const jsonDisplays = screen.getAllByTestId('json')
136+
expect(jsonDisplays.length).toBe(7)
137+
})
138+
139+
it('should render PackageMetadata without caption', () => {
140+
renderWithTheme(<Metadata className="package" metadata={mockPackageMetadata} />)
141+
142+
// Should not have a caption since PackageMetadata doesn't have created_by
143+
expect(screen.queryByRole('caption')).toBeFalsy()
144+
145+
// Check package-specific fields
146+
expect(screen.getByText('Manifest version:')).toBeTruthy()
147+
expect(screen.getByText('1.2.3')).toBeTruthy()
148+
149+
expect(screen.getByText('Workflow:')).toBeTruthy()
150+
expect(screen.getByTestId('json')).toBeTruthy()
151+
152+
expect(screen.getByText('Message:')).toBeTruthy()
153+
expect(
154+
screen.getByText('Initial package version with comprehensive data'),
155+
).toBeTruthy()
156+
})
157+
158+
it('should only render fields that are not null/undefined', () => {
159+
const partialMetadata: Partial<H5adMetadata> = {
160+
created_by: 'test',
161+
n_cells: 1000,
162+
// Other fields are undefined/null and should not be rendered
163+
}
164+
165+
renderWithTheme(
166+
<Metadata className="test-class" metadata={partialMetadata as H5adMetadata} />,
167+
)
168+
169+
expect(screen.getByText('Cells:')).toBeTruthy()
170+
expect(screen.getByText('1000')).toBeTruthy()
171+
172+
// Fields that are null/undefined should not be rendered
173+
expect(screen.queryByText('Genes:')).toBeFalsy()
174+
expect(screen.queryByText('Format version:')).toBeFalsy()
175+
})
176+
})

0 commit comments

Comments
 (0)