Skip to content

Commit c367ca7

Browse files
mmwintherdapla-bot[bot]
mmwinther
and
dapla-bot[bot]
authored
Move fields from dataset to variable level (#69)
* Format json-schema with 2 spaces * Remove contains_personal_data * Remove unit_type * Remove data_source * Remove temporality_type * Add unit_type to variable model * Update description of data_source * Update description of temporality_type * [GENERATE] Pydantic models from JSON Schema * [GENERATE] Java classes from JSON Schema * Fix tests * Make fields required * [GENERATE] Pydantic models from JSON Schema * [GENERATE] Java classes from JSON Schema --------- Co-authored-by: dapla-bot[bot] <143391972+dapla-bot[bot]@users.noreply.github.com>
1 parent 4f81da0 commit c367ca7

File tree

7 files changed

+657
-847
lines changed

7 files changed

+657
-847
lines changed

‎generated/java/datadoc-model/src/main/java/no/ssb/dapla/metadata/datadoc/Dataset.java‎

Lines changed: 27 additions & 255 deletions
Large diffs are not rendered by default.

‎generated/java/datadoc-model/src/main/java/no/ssb/dapla/metadata/datadoc/Variable.java‎

Lines changed: 129 additions & 25 deletions
Large diffs are not rendered by default.

‎generated/python/datadoc_model/datadoc_model/all_optional/model.py‎

Lines changed: 16 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
# generated by datamodel-codegen:
22
# filename: metadata-container-json-schema.json
3-
# timestamp: 2025-05-23T12:45:42+00:00
3+
# timestamp: 2025-09-02T11:54:53+00:00
44

55
from __future__ import annotations
66

7-
from datetime import date, datetime
7+
from datetime import date
88
from enum import Enum
99
from typing import Any, Literal, Optional, Union
1010
from uuid import UUID
1111

1212
from datadoc_model.datadoc_base_model import DatadocBaseModel
13-
from pydantic import AnyUrl, Field, RootModel
13+
from pydantic import AnyUrl, AwareDatetime, Field, RootModel
1414

1515

1616
class Assessment(str, Enum):
@@ -67,7 +67,7 @@ class VariableRole(str, Enum):
6767

6868

6969
class Pseudonymization(DatadocBaseModel):
70-
pseudonymization_time: Optional[datetime] = Field(
70+
pseudonymization_time: Optional[AwareDatetime] = Field(
7171
None,
7272
description="Time at which the variable was pseudonymized. In ISO 8601 format.",
7373
title="Variable pseudo time",
@@ -164,11 +164,6 @@ class Dataset(DatadocBaseModel):
164164
description: Optional[LanguageStringType] = Field(
165165
None, description="Free text description of the data set", title="Description"
166166
)
167-
data_source: Optional[str] = Field(
168-
None,
169-
description="Data source. Set either for the data set or instance variable.",
170-
title="Data source",
171-
)
172167
population_description: Optional[LanguageStringType] = Field(
173168
None,
174169
description="Description of the population covered in the data set. Includes unit type, spatial coverage and period of time.",
@@ -182,16 +177,6 @@ class Dataset(DatadocBaseModel):
182177
description="Dataset version information in the form of descriptive text",
183178
title="Version description",
184179
)
185-
unit_type: Optional[str] = Field(
186-
None,
187-
description="Unit Type for data file, table or data set. See Definitions of Unit Types https://www.ssb.no/en/metadata/definisjoner-av-statistiske-enheter",
188-
title="Unit type",
189-
)
190-
temporality_type: Optional[TemporalityTypeType] = Field(
191-
None,
192-
description="Temporality type. Either for the instance variable or the data set",
193-
title="Temporality type",
194-
)
195180
subject_field: Optional[str] = Field(
196181
None,
197182
description="Primary area of statistics in which the data set is included",
@@ -207,15 +192,10 @@ class Dataset(DatadocBaseModel):
207192
description="Description of the data set's spatial coverage",
208193
title="Spatial coverage description",
209194
)
210-
contains_personal_data: Optional[bool] = Field(
211-
None,
212-
description="Is there any personal data amongst this data (data set)?",
213-
title="Contains personal data",
214-
)
215195
use_restriction: Optional[UseRestriction] = Field(
216196
None, description="Data set use restriction", title="Use restriction"
217197
)
218-
use_restriction_date: Optional[datetime] = Field(
198+
use_restriction_date: Optional[AwareDatetime] = Field(
219199
None,
220200
description="Use restriction date, eg. the date (deadline) for when data must be deleted/anonymised.",
221201
title="Use restriction date",
@@ -238,15 +218,15 @@ class Dataset(DatadocBaseModel):
238218
description="The file path contains the data set's name and the path to where it is stored",
239219
title="File path",
240220
)
241-
metadata_created_date: Optional[datetime] = Field(
221+
metadata_created_date: Optional[AwareDatetime] = Field(
242222
None,
243223
description="Created date for metadata about the data set",
244224
title="Metadata created date",
245225
)
246226
metadata_created_by: Optional[str] = Field(
247227
None, description="Created by identifiable person.", title="Metadata created by"
248228
)
249-
metadata_last_updated_date: Optional[datetime] = Field(
229+
metadata_last_updated_date: Optional[AwareDatetime] = Field(
250230
None,
251231
description="Last updated date for metadata about the dataset",
252232
title="Metadata last updated date",
@@ -273,10 +253,10 @@ class OtherSpecialValue(DatadocBaseModel):
273253
name: Optional[LanguageStringType] = Field(
274254
None, description="Other value name (text)", title="Name"
275255
)
276-
valid_from: Optional[datetime] = Field(
256+
valid_from: Optional[AwareDatetime] = Field(
277257
None, description="Other value valid from date", title="Valid from"
278258
)
279-
valid_until: Optional[datetime] = Field(
259+
valid_until: Optional[AwareDatetime] = Field(
280260
None, description="Other value until date", title="Valid until"
281261
)
282262

@@ -327,9 +307,14 @@ class Variable(DatadocBaseModel):
327307
pseudonymization: Optional[Pseudonymization] = Field(
328308
None, description="A pseudonymized variable in the dataset."
329309
)
310+
unit_type: Optional[str] = Field(
311+
None,
312+
description="Unit Type which this variable pertains to. Values shall be codes from https://www.ssb.no/klass/klassifikasjoner/702. See Definitions of Unit Types https://www.ssb.no/en/metadata/definisjoner-av-statistiske-enheter",
313+
title="Unit type",
314+
)
330315
data_source: Optional[str] = Field(
331316
None,
332-
description="Data source. Set at data set level, but can be overwritten at variable instance level.",
317+
description="The source of the data contained in this variable. Values shall be codes from https://www.ssb.no/klass/klassifikasjoner/712.",
333318
title="Data source",
334319
)
335320
population_description: Optional[LanguageStringType] = Field(
@@ -343,9 +328,7 @@ class Variable(DatadocBaseModel):
343328
title="Comment",
344329
)
345330
temporality_type: Optional[TemporalityTypeType] = Field(
346-
None,
347-
description="Temporality type. Set either for variable instance or dataset.",
348-
title="Temporality type",
331+
None, description="Temporality type of this variable.", title="Temporality type"
349332
)
350333
measurement_unit: Optional[str] = Field(
351334
None, description="Measurement unit", title="Measurement unit"

‎generated/python/datadoc_model/datadoc_model/required/model.py‎

Lines changed: 19 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
# generated by datamodel-codegen:
22
# filename: metadata-container-json-schema.json
3-
# timestamp: 2025-05-23T12:45:44+00:00
3+
# timestamp: 2025-09-02T11:54:55+00:00
44

55
from __future__ import annotations
66

7-
from datetime import date, datetime
7+
from datetime import date
88
from enum import Enum
99
from typing import Any, Literal, Optional, Union
1010
from uuid import UUID
1111

1212
from datadoc_model.datadoc_base_model import DatadocBaseModel
13-
from pydantic import AnyUrl, Field, RootModel
13+
from pydantic import AnyUrl, AwareDatetime, Field, RootModel
1414

1515

1616
class Assessment(str, Enum):
@@ -67,7 +67,7 @@ class VariableRole(str, Enum):
6767

6868

6969
class Pseudonymization(DatadocBaseModel):
70-
pseudonymization_time: Optional[datetime] = Field(
70+
pseudonymization_time: Optional[AwareDatetime] = Field(
7171
None,
7272
description="Time at which the variable was pseudonymized. In ISO 8601 format.",
7373
title="Variable pseudo time",
@@ -160,11 +160,6 @@ class Dataset(DatadocBaseModel):
160160
description: LanguageStringType = Field(
161161
..., description="Free text description of the data set", title="Description"
162162
)
163-
data_source: Optional[str] = Field(
164-
None,
165-
description="Data source. Set either for the data set or instance variable.",
166-
title="Data source",
167-
)
168163
population_description: LanguageStringType = Field(
169164
...,
170165
description="Description of the population covered in the data set. Includes unit type, spatial coverage and period of time.",
@@ -176,16 +171,6 @@ class Dataset(DatadocBaseModel):
176171
description="Dataset version information in the form of descriptive text",
177172
title="Version description",
178173
)
179-
unit_type: str = Field(
180-
...,
181-
description="Unit Type for data file, table or data set. See Definitions of Unit Types https://www.ssb.no/en/metadata/definisjoner-av-statistiske-enheter",
182-
title="Unit type",
183-
)
184-
temporality_type: Optional[TemporalityTypeType] = Field(
185-
None,
186-
description="Temporality type. Either for the instance variable or the data set",
187-
title="Temporality type",
188-
)
189174
subject_field: str = Field(
190175
...,
191176
description="Primary area of statistics in which the data set is included",
@@ -201,15 +186,10 @@ class Dataset(DatadocBaseModel):
201186
description="Description of the data set's spatial coverage",
202187
title="Spatial coverage description",
203188
)
204-
contains_personal_data: bool = Field(
205-
...,
206-
description="Is there any personal data amongst this data (data set)?",
207-
title="Contains personal data",
208-
)
209189
use_restriction: Optional[UseRestriction] = Field(
210190
None, description="Data set use restriction", title="Use restriction"
211191
)
212-
use_restriction_date: Optional[datetime] = Field(
192+
use_restriction_date: Optional[AwareDatetime] = Field(
213193
None,
214194
description="Use restriction date, eg. the date (deadline) for when data must be deleted/anonymised.",
215195
title="Use restriction date",
@@ -232,15 +212,15 @@ class Dataset(DatadocBaseModel):
232212
description="The file path contains the data set's name and the path to where it is stored",
233213
title="File path",
234214
)
235-
metadata_created_date: datetime = Field(
215+
metadata_created_date: AwareDatetime = Field(
236216
...,
237217
description="Created date for metadata about the data set",
238218
title="Metadata created date",
239219
)
240220
metadata_created_by: str = Field(
241221
..., description="Created by identifiable person.", title="Metadata created by"
242222
)
243-
metadata_last_updated_date: datetime = Field(
223+
metadata_last_updated_date: AwareDatetime = Field(
244224
...,
245225
description="Last updated date for metadata about the dataset",
246226
title="Metadata last updated date",
@@ -267,10 +247,10 @@ class OtherSpecialValue(DatadocBaseModel):
267247
name: Optional[LanguageStringType] = Field(
268248
None, description="Other value name (text)", title="Name"
269249
)
270-
valid_from: Optional[datetime] = Field(
250+
valid_from: Optional[AwareDatetime] = Field(
271251
None, description="Other value valid from date", title="Valid from"
272252
)
273-
valid_until: Optional[datetime] = Field(
253+
valid_until: Optional[AwareDatetime] = Field(
274254
None, description="Other value until date", title="Valid until"
275255
)
276256

@@ -319,9 +299,14 @@ class Variable(DatadocBaseModel):
319299
pseudonymization: Optional[Pseudonymization] = Field(
320300
None, description="A pseudonymized variable in the dataset."
321301
)
322-
data_source: Optional[str] = Field(
323-
None,
324-
description="Data source. Set at data set level, but can be overwritten at variable instance level.",
302+
unit_type: str = Field(
303+
...,
304+
description="Unit Type which this variable pertains to. Values shall be codes from https://www.ssb.no/klass/klassifikasjoner/702. See Definitions of Unit Types https://www.ssb.no/en/metadata/definisjoner-av-statistiske-enheter",
305+
title="Unit type",
306+
)
307+
data_source: str = Field(
308+
...,
309+
description="The source of the data contained in this variable. Values shall be codes from https://www.ssb.no/klass/klassifikasjoner/712.",
325310
title="Data source",
326311
)
327312
population_description: Optional[LanguageStringType] = Field(
@@ -334,10 +319,8 @@ class Variable(DatadocBaseModel):
334319
description="Further clarification of the variables definition",
335320
title="Comment",
336321
)
337-
temporality_type: Optional[TemporalityTypeType] = Field(
338-
None,
339-
description="Temporality type. Set either for variable instance or dataset.",
340-
title="Temporality type",
322+
temporality_type: TemporalityTypeType = Field(
323+
..., description="Temporality type of this variable.", title="Temporality type"
341324
)
342325
measurement_unit: Optional[str] = Field(
343326
None, description="Measurement unit", title="Measurement unit"

‎generated/python/datadoc_model/tests/test_datadoc_model_required.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,16 +51,14 @@ def test_instantiate_all_required_values_datadoc():
5151
],
5252
),
5353
version="1",
54-
unit_type="20",
5554
subject_field="20",
56-
contains_personal_data=True,
5755
id=uuid4(),
5856
owner="team-metadata",
5957
file_path="path/to/dataset",
6058
metadata_created_by="mmw",
61-
metadata_created_date=datetime.now(),
59+
metadata_created_date=datetime.now().astimezone(),
6260
metadata_last_updated_by="mmw",
63-
metadata_last_updated_date=datetime.now(),
61+
metadata_last_updated_date=datetime.now().astimezone(),
6462
assessment="PROTECTED",
6563
dataset_status="DRAFT",
6664
dataset_state="PROCESSED_DATA",

‎generated/python/datadoc_model/uv.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)