diff --git a/clickhouse_search/management/commands/set_saved_variant_key.py b/clickhouse_search/management/commands/set_saved_variant_key.py
index c76769effd..0928b6d728 100644
--- a/clickhouse_search/management/commands/set_saved_variant_key.py
+++ b/clickhouse_search/management/commands/set_saved_variant_key.py
@@ -6,7 +6,7 @@
from clickhouse_search.search import get_clickhouse_key_lookup
from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37
-from seqr.models import SavedVariant, Sample
+from seqr.models import SavedVariant, Dataset
from seqr.utils.xpos_utils import parse_variant_id
logger = logging.getLogger(__name__)
@@ -22,7 +22,7 @@ def handle(self, *args, **options):
saved_variant_json__populations__isnull=False, # Omit manual variants
).values_list('variant_id', flat=True).distinct()
ids_by_dataset_type = {
- Sample.DATASET_TYPE_VARIANT_CALLS: [], Sample.DATASET_TYPE_MITO_CALLS: [], Sample.DATASET_TYPE_SV_CALLS: [],
+ Dataset.DATASET_TYPE_VARIANT_CALLS: [], Dataset.DATASET_TYPE_MITO_CALLS: [], Dataset.DATASET_TYPE_SV_CALLS: [],
}
for variant_id in variant_ids:
parsed_id = parse_variant_id(variant_id)
@@ -31,23 +31,23 @@ def handle(self, *args, **options):
parsed_id = parse_variant_id(variant_id[:-1])
if parsed_id:
is_mito = parsed_id[0].replace('chr', '').startswith('M')
- dataset_type = Sample.DATASET_TYPE_MITO_CALLS if is_mito else Sample.DATASET_TYPE_VARIANT_CALLS
+ dataset_type = Dataset.DATASET_TYPE_MITO_CALLS if is_mito else Dataset.DATASET_TYPE_VARIANT_CALLS
else:
- dataset_type = Sample.DATASET_TYPE_SV_CALLS
+ dataset_type = Dataset.DATASET_TYPE_SV_CALLS
ids_by_dataset_type[dataset_type].append(variant_id)
- no_key_mito = self._set_variant_keys(ids_by_dataset_type[Sample.DATASET_TYPE_MITO_CALLS], Sample.DATASET_TYPE_MITO_CALLS)
+ no_key_mito = self._set_variant_keys(ids_by_dataset_type[Dataset.DATASET_TYPE_MITO_CALLS], Dataset.DATASET_TYPE_MITO_CALLS)
no_key_snv_indel = self._set_variant_keys(
- ids_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] + list(no_key_mito), Sample.DATASET_TYPE_VARIANT_CALLS,
+ ids_by_dataset_type[Dataset.DATASET_TYPE_VARIANT_CALLS] + list(no_key_mito), Dataset.DATASET_TYPE_VARIANT_CALLS,
)
if no_key_snv_indel:
self._resolve_missing_variants(no_key_snv_indel, GENOME_VERSION_GRCh38)
no_keys_svs = self._set_variant_keys(
- ids_by_dataset_type[Sample.DATASET_TYPE_SV_CALLS], f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WGS}',
+ ids_by_dataset_type[Dataset.DATASET_TYPE_SV_CALLS], f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WGS}',
)
- no_keys_svs = self._set_variant_keys(list(no_keys_svs), f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}')
+ no_keys_svs = self._set_variant_keys(list(no_keys_svs), f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WES}')
if no_keys_svs:
self._resolve_reloaded_svs(no_keys_svs)
@@ -55,7 +55,7 @@ def handle(self, *args, **options):
key__isnull=True, family__project__genome_version=GENOME_VERSION_GRCh37,
).values_list('variant_id', flat=True).distinct()
- no_keys_37 = self._set_variant_keys(variant_ids_37, Sample.DATASET_TYPE_VARIANT_CALLS, genome_version=GENOME_VERSION_GRCh37)
+ no_keys_37 = self._set_variant_keys(variant_ids_37, Dataset.DATASET_TYPE_VARIANT_CALLS, genome_version=GENOME_VERSION_GRCh37)
if no_keys_37:
self._resolve_missing_variants(no_keys_37, GENOME_VERSION_GRCh37)
@@ -103,7 +103,7 @@ def _query_missing_variants(cls, variant_ids, variant_fields, genome_version=GEN
variant_id__in=variant_ids, family__project__genome_version=genome_version,
)
num_missing = missing_variants.count()
- missing_with_data_qs = missing_variants.filter(family__individual__sample__is_active=True).distinct()
+ missing_with_data_qs = missing_variants.filter(family__individual__active_datasets__isnull=False).distinct()
missing_with_search_data = missing_with_data_qs.values(
'variant_id', *variant_fields,
).annotate(family_ids=ArrayAgg('family__family_id', distinct=True)).order_by('variant_id')
@@ -128,7 +128,7 @@ def _resolve_missing_variants(cls, variant_ids, genome_version):
@classmethod
def _resolve_reloaded_svs(cls, variant_ids):
missing_with_search_data, num_missing = cls._query_missing_variants(
- list(variant_ids), ['family__individual__sample__sample_type'],
+ list(variant_ids), ['family__individual__active_datasets__sample_type'],
)
logger.info(
f'{num_missing} SV variants have no key, {num_missing - len(missing_with_search_data)} of which have no search data'
@@ -139,7 +139,7 @@ def _resolve_reloaded_svs(cls, variant_ids):
missing_by_sample_type = defaultdict(list)
for variant in missing_with_search_data:
variant_id = variant['variant_id']
- sample_type = variant['family__individual__sample__sample_type']
+ sample_type = variant['family__individual__active_datasets__sample_type']
missing_by_sample_type[sample_type].append(f"{variant_id} - {'; '.join(variant['family_ids'])}" )
for sample_type, variants in missing_by_sample_type.items():
diff --git a/clickhouse_search/management/tests/set_saved_variant_key_tests.py b/clickhouse_search/management/tests/set_saved_variant_key_tests.py
index e53bdf1a2b..9b3d07f479 100644
--- a/clickhouse_search/management/tests/set_saved_variant_key_tests.py
+++ b/clickhouse_search/management/tests/set_saved_variant_key_tests.py
@@ -1,7 +1,7 @@
from django.core.management import call_command
import mock
-from seqr.models import Project, Sample, SavedVariant
+from seqr.models import Project, Dataset, SavedVariant
from seqr.views.utils.test_utils import AnvilAuthenticationTestCase
MOCK_GCNV_DATA = [
@@ -18,7 +18,10 @@ class SetSavedVariantKeyTest(AnvilAuthenticationTestCase):
@classmethod
def setUpTestData(cls):
Project.objects.filter(id=3).update(genome_version='38')
- Sample.objects.filter(guid='S000154_na20889').update(dataset_type='SV', is_active=True)
+ dataset = Dataset.objects.get(guid='S000154_na20889')
+ dataset.dataset_type = 'SV'
+ dataset.save()
+ dataset.active_individuals.set({17})
for sv in SavedVariant.objects.filter(key__isnull=False):
sv.saved_variant_json = {
'genotypes': sv.genotypes, 'populations': {'gnomad': {'af': 0.01}},
diff --git a/clickhouse_search/models/gt_stats_models.py b/clickhouse_search/models/gt_stats_models.py
index b6a12f2fb2..86ccb1e05a 100644
--- a/clickhouse_search/models/gt_stats_models.py
+++ b/clickhouse_search/models/gt_stats_models.py
@@ -5,7 +5,7 @@
from clickhouse_search.backend.table_models import RefreshableMaterializedView, RefreshableMaterializedViewMeta, \
IncrementalMaterializedView, Dictionary
from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37
-from seqr.models import Sample
+from seqr.models import Dataset
class BaseProjectGtStats(models.ClickhouseModel):
@@ -308,10 +308,10 @@ class Meta(GtStatsDictMeta):
layout = 'FLAT(MAX_ARRAY_SIZE 5000000)'
PROJECT_GT_STATS_VIEW_CLASS_MAP = {
- GENOME_VERSION_GRCh37: {Sample.DATASET_TYPE_VARIANT_CALLS: ProjectsToGtStatsGRCh37SnvIndel},
+ GENOME_VERSION_GRCh37: {Dataset.DATASET_TYPE_VARIANT_CALLS: ProjectsToGtStatsGRCh37SnvIndel},
GENOME_VERSION_GRCh38: {
- Sample.DATASET_TYPE_VARIANT_CALLS: ProjectsToGtStatsSnvIndel,
- Sample.DATASET_TYPE_MITO_CALLS: ProjectsToGtStatsMito,
- Sample.DATASET_TYPE_SV_CALLS: ProjectsToGtStatsSv,
+ Dataset.DATASET_TYPE_VARIANT_CALLS: ProjectsToGtStatsSnvIndel,
+ Dataset.DATASET_TYPE_MITO_CALLS: ProjectsToGtStatsMito,
+ Dataset.DATASET_TYPE_SV_CALLS: ProjectsToGtStatsSv,
},
}
diff --git a/clickhouse_search/models/search_models.py b/clickhouse_search/models/search_models.py
index 5891057573..fb2653401c 100644
--- a/clickhouse_search/models/search_models.py
+++ b/clickhouse_search/models/search_models.py
@@ -10,7 +10,7 @@
from clickhouse_search.models.reference_data_models import GnomadNonCodingConstraintDict, BaseSpliceAi, \
ScreenDict
from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37
-from seqr.models import Sample
+from seqr.models import Dataset
from seqr.utils.xpos_utils import CHROMOSOME_CHOICES
from settings import CLICKHOUSE_IN_MEMORY_DIR, CLICKHOUSE_DATA_DIR
@@ -414,7 +414,7 @@ class Meta(BaseEntries.Meta):
)
class EntriesSv(BaseEntries):
- SAMPLE_TYPE = Sample.SAMPLE_TYPE_WGS
+ SAMPLE_TYPE = Dataset.SAMPLE_TYPE_WGS
CALL_FIELDS = [
('sampleId', models.StringField()),
('gt', models.Enum8Field(null=True, blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')])),
@@ -436,7 +436,7 @@ class Meta(BaseEntries.Meta):
db_table = 'GRCh38/SV/entries'
class EntriesGcnv(BaseEntries):
- SAMPLE_TYPE = Sample.SAMPLE_TYPE_WES
+ SAMPLE_TYPE = Dataset.SAMPLE_TYPE_WES
CALL_FIELDS = [
('sampleId', models.StringField()),
('gt', models.Enum8Field(null=True, blank=True, choices=[(0, 'REF'), (1, 'HET'), (2, 'HOM')])),
@@ -621,21 +621,21 @@ class Meta:
ENTRY_CLASS_MAP = {
- GENOME_VERSION_GRCh37: {Sample.DATASET_TYPE_VARIANT_CALLS: EntriesGRCh37SnvIndel},
+ GENOME_VERSION_GRCh37: {Dataset.DATASET_TYPE_VARIANT_CALLS: EntriesGRCh37SnvIndel},
GENOME_VERSION_GRCh38: {
- Sample.DATASET_TYPE_VARIANT_CALLS: EntriesSnvIndel,
- Sample.DATASET_TYPE_MITO_CALLS: EntriesMito,
- f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WGS}': EntriesSv,
- f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}': EntriesGcnv,
+ Dataset.DATASET_TYPE_VARIANT_CALLS: EntriesSnvIndel,
+ Dataset.DATASET_TYPE_MITO_CALLS: EntriesMito,
+ f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WGS}': EntriesSv,
+ f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WES}': EntriesGcnv,
},
}
VARIANTS_CLASS_MAP = {
- GENOME_VERSION_GRCh37: {Sample.DATASET_TYPE_VARIANT_CALLS: VariantsGRCh37SnvIndel},
+ GENOME_VERSION_GRCh37: {Dataset.DATASET_TYPE_VARIANT_CALLS: VariantsGRCh37SnvIndel},
GENOME_VERSION_GRCh38: {
- Sample.DATASET_TYPE_VARIANT_CALLS: VariantsSnvIndel,
- Sample.DATASET_TYPE_MITO_CALLS: VariantsMito,
- f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WGS}': VariantsSv,
- f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}': VariantsGcnv,
+ Dataset.DATASET_TYPE_VARIANT_CALLS: VariantsSnvIndel,
+ Dataset.DATASET_TYPE_MITO_CALLS: VariantsMito,
+ f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WGS}': VariantsSv,
+ f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WES}': VariantsGcnv,
},
}
VARIANT_DETAILS_CLASS_MAP = {
diff --git a/clickhouse_search/search.py b/clickhouse_search/search.py
index a1093606f7..2c45bec171 100644
--- a/clickhouse_search/search.py
+++ b/clickhouse_search/search.py
@@ -18,7 +18,7 @@
from clickhouse_search.models.search_models import BaseVariants, BaseVariantsSvGcnv, EntriesSnvIndel, \
ENTRY_CLASS_MAP, VARIANTS_CLASS_MAP, VARIANT_DETAILS_CLASS_MAP
from reference_data.models import GeneInfo, GeneConstraint, Omim, GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh38, GENOME_VERSION_GRCh37
-from seqr.models import Sample, PhenotypePrioritization, Family, Individual
+from seqr.models import Dataset, PhenotypePrioritization, Family, Individual
from seqr.utils.gene_utils import parse_locus_list_items
from seqr.utils.logging_utils import SeqrLogger
from clickhouse_search.constants import MAX_VARIANTS, XPOS_SORT_KEY, PATHOGENICTY_SORT_KEY, PATHOGENICTY_HGMD_SORT_KEY, \
@@ -63,7 +63,7 @@ def get_clickhouse_variants(families, user, genome_version=None, sort=None, samp
except InvalidDatasetTypeException:
continue
- if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS and no_access_project_genome_version:
+ if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS and no_access_project_genome_version:
results += _get_no_access_search_results(
entry_qs, variants_qs, has_comp_het, user, **search, **parsed_filters,
exclude_projects=sample_data_by_dataset_type[dataset_type].get('project_guids'), inheritance_mode=inheritance_mode,
@@ -93,7 +93,7 @@ def get_clickhouse_variants(families, user, genome_version=None, sort=None, samp
dataset_results += _get_data_type_comp_het_results_queryset(
entry_qs, variants_qs, sample_data, user, parsed_filters, **search,
exclude_key_pairs=(exclude_key_pairs or {}).get(dataset_type),
- is_x_chrom=has_x_chrom_comp_het and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS,
+ is_x_chrom=has_x_chrom_comp_het and dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS,
)
if 'samples' not in sample_data:
@@ -101,7 +101,7 @@ def get_clickhouse_variants(families, user, genome_version=None, sort=None, samp
results += dataset_results
searched_dataset_types.add(dataset_type)
- if has_comp_het and any(dt.startswith(Sample.DATASET_TYPE_SV_CALLS) for dt in VARIANTS_CLASS_MAP[genome_version]):
+ if has_comp_het and any(dt.startswith(Dataset.DATASET_TYPE_SV_CALLS) for dt in VARIANTS_CLASS_MAP[genome_version]):
results += _get_multi_data_type_comp_het_results(genome_version, families, sample_data_by_dataset_type, user, exclude_key_pairs or {}, searched_dataset_types, **search)
if not searched_dataset_types:
@@ -198,8 +198,8 @@ def _raise_dataset_type_errors(sample_data_errors, sample_data_by_dataset_type):
if sample_data_errors:
raise InvalidSearchException(next(iter(sample_data_errors)))
no_data_type = next(data_type for data_type, data in sample_data_by_dataset_type.items() if not data)
- if no_data_type.startswith(Sample.DATASET_TYPE_SV_CALLS):
- no_data_type = Sample.DATASET_TYPE_SV_CALLS
+ if no_data_type.startswith(Dataset.DATASET_TYPE_SV_CALLS):
+ no_data_type = Dataset.DATASET_TYPE_SV_CALLS
raise InvalidSearchException(f'Unable to search against dataset type "{no_data_type}"')
@@ -250,29 +250,29 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d
}
try:
- snv_indel_entry_qs = ENTRY_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS].objects.filter_locus(**search_kwargs)
- snv_indel_variants_qs = VARIANTS_CLASS_MAP[genome_version][Sample.DATASET_TYPE_VARIANT_CALLS].objects
+ snv_indel_entry_qs = ENTRY_CLASS_MAP[genome_version][Dataset.DATASET_TYPE_VARIANT_CALLS].objects.filter_locus(**search_kwargs)
+ snv_indel_variants_qs = VARIANTS_CLASS_MAP[genome_version][Dataset.DATASET_TYPE_VARIANT_CALLS].objects
snv_indel_parsed_filters = snv_indel_variants_qs.get_parsed_annotations_filters(annotations=annotations, **search_kwargs)
- sv_variants_cls = VARIANTS_CLASS_MAP[genome_version][f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}']
+ sv_variants_cls = VARIANTS_CLASS_MAP[genome_version][f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WES}']
sv_parsed_filters = sv_variants_cls.objects.get_parsed_annotations_filters(annotations=annotations, **search_kwargs)
except InvalidDatasetTypeException:
return []
- if Sample.DATASET_TYPE_VARIANT_CALLS not in sample_data_by_dataset_type:
- sample_data_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS] = _get_sample_data(
+ if Dataset.DATASET_TYPE_VARIANT_CALLS not in sample_data_by_dataset_type:
+ sample_data_by_dataset_type[Dataset.DATASET_TYPE_VARIANT_CALLS] = _get_sample_data(
all_families,
- Sample.DATASET_TYPE_VARIANT_CALLS,
+ Dataset.DATASET_TYPE_VARIANT_CALLS,
inheritance_mode=COMPOUND_HET,
inheritance_filter=search_kwargs.get('inheritance_filter'),
)
- snv_indel_sample_data = sample_data_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS]
+ snv_indel_sample_data = sample_data_by_dataset_type[Dataset.DATASET_TYPE_VARIANT_CALLS]
if not (snv_indel_sample_data or {}).get('num_families'):
return []
snv_indel_families = set().union(*snv_indel_sample_data['sample_type_families'].values())
results = []
- for sample_type in [Sample.SAMPLE_TYPE_WES, Sample.SAMPLE_TYPE_WGS]:
- sv_dataset_type = f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}'
+ for sample_type in [Dataset.SAMPLE_TYPE_WES, Dataset.SAMPLE_TYPE_WGS]:
+ sv_dataset_type = f'{Dataset.DATASET_TYPE_SV_CALLS}_{sample_type}'
if sv_dataset_type not in sample_data_by_dataset_type:
sample_data_by_dataset_type[sv_dataset_type] = _get_sample_data(
all_families,
@@ -285,7 +285,7 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d
families = snv_indel_families.intersection(sv_families)
if not families:
continue
- logger.info(f'Loading {Sample.DATASET_TYPE_VARIANT_CALLS}/{sv_dataset_type} data for {len(families)} families', user)
+ logger.info(f'Loading {Dataset.DATASET_TYPE_VARIANT_CALLS}/{sv_dataset_type} data for {len(families)} families', user)
snv_indel_sample_type_families = {
sample_type: set(familes).intersection(sv_families)
@@ -320,7 +320,7 @@ def _get_multi_data_type_comp_het_results(genome_version, all_families, sample_d
result_q = _get_comp_het_results_queryset(
snv_indel_variants_qs, snv_indel_q, sv_q, len(families),
- exclude_key_pairs=exclude_key_pairs.get(f'{Sample.DATASET_TYPE_VARIANT_CALLS},{sv_dataset_type}'),
+ exclude_key_pairs=exclude_key_pairs.get(f'{Dataset.DATASET_TYPE_VARIANT_CALLS},{sv_dataset_type}'),
)
dataset_results = _evaluate_results(result_q, is_comp_het=True)
if not sv_sample_data['samples']:
@@ -448,9 +448,9 @@ def _add_individual_guids(results):
for r in (result if isinstance(result, list) else [result]):
families.update(r.get('familyGenotypes', {}).keys())
sample_map = {
- (family_guid, sample_id): individual_guid for family_guid, individual_guid, sample_id in Sample.objects.filter(
- individual__family__guid__in=families, is_active=True,
- ).values_list('individual__family__guid', 'individual__guid', 'sample_id')
+ (family_guid, sample_id): individual_guid for family_guid, individual_guid, sample_id in Individual.objects.filter(
+ family__guid__in=families, active_datasets__isnull=False,
+ ).values_list('family__guid', 'guid', 'individual_id')
}
for result in results:
if isinstance(result, list):
@@ -488,7 +488,7 @@ def format_clickhouse_export_results(results):
genome_version = formatted_results[0]['genomeVersion']
keys_with_no_details = {result['key'] for result in formatted_results if not 'transcripts' in result}
- detail_qs = get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details)
+ detail_qs = get_variant_details_queryset(genome_version, Dataset.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details)
details_by_key = {
detail['key']: detail for detail in detail_qs.values(
'key', 'rsid', mainTranscript=F('transcripts__0'), variantId=F('variant_id'),
@@ -523,7 +523,7 @@ def format_clickhouse_results(results):
}
details_by_key = {
detail['key']: detail for detail in
- get_variant_details_queryset(genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details).result_values()
+ get_variant_details_queryset(genome_version, Dataset.DATASET_TYPE_VARIANT_CALLS, keys_with_no_details).result_values()
}
formatted_results = []
@@ -577,66 +577,66 @@ def _is_matched_minimal_transcript(transcript, minimal_transcript):
def _get_valid_samples(families, dataset_type, sample_type, allow_no_samples):
- samples = Sample.objects.filter(individual__family__in=families, is_active=True)
- if not samples.exists():
+ individuals = Individual.objects.filter(family__in=families, active_datasets__isnull=False)
+ if not individuals.exists():
if allow_no_samples:
return None
raise InvalidSearchException(f'No search data found for families {", ".join([f.family_id for f in families])}')
- samples = samples.filter(dataset_type=dataset_type)
+ individuals = individuals.filter(active_datasets__dataset_type=dataset_type)
if sample_type:
- samples = samples.filter(sample_type=sample_type)
+ individuals = individuals.filter(active_datasets__sample_type=sample_type)
- mismatch_affected_samples = samples.values('sample_id').annotate(
- projects=ArrayAgg('individual__family__project__name', distinct=True),
- affected=ArrayAgg('individual__affected', distinct=True),
+ mismatch_affected_samples = individuals.values('individual_id').annotate(
+ projects=ArrayAgg('family__project__name', distinct=True),
+ affected=ArrayAgg('affected', distinct=True),
).filter(affected__len__gt=1)
if mismatch_affected_samples:
raise InvalidSearchException(
'The following samples are incorrectly configured and have different affected statuses in different projects: ' +
- ', '.join([f'{agg["sample_id"]} ({"/ ".join(agg["projects"])})' for agg in mismatch_affected_samples]),
+ ', '.join([f'{agg["individual_id"]} ({"/ ".join(agg["projects"])})' for agg in mismatch_affected_samples]),
)
- return samples
+ return individuals
-def _get_sample_metadata(samples, affected_family_only, annotate_affected_males):
- skip_individual_guid = samples.values('individual__family__project_id').distinct().count() > 1
+def _get_sample_metadata(individuals, affected_family_only, annotate_affected_males):
+ skip_individual_guid = individuals.values('family__project_id').distinct().count() > 1
family_array_kwargs = {'distinct': True}
if affected_family_only:
- family_array_kwargs['filter'] = Q(individual__affected=Individual.AFFECTED_STATUS_AFFECTED)
+ family_array_kwargs['filter'] = Q(affected=Individual.AFFECTED_STATUS_AFFECTED)
annotations = {
- 'project_guids': ArrayAgg('individual__family__project__guid', distinct=True),
- 'family_guids': ArrayAgg('individual__family__guid', **family_array_kwargs),
+ 'project_guids': ArrayAgg('family__project__guid', distinct=True),
+ 'family_guids': ArrayAgg('family__guid', **family_array_kwargs),
}
if skip_individual_guid:
annotations['num_unaffected'] = Count(
- 'individual_id', distinct=True, filter=Q(individual__affected=Individual.AFFECTED_STATUS_UNAFFECTED),
+ 'id', distinct=True, filter=Q(affected=Individual.AFFECTED_STATUS_UNAFFECTED),
)
if annotate_affected_males:
- annotations['affected_male_family_guids'] = ArrayAgg('individual__family__guid', distinct=True, filter=Q(
- individual__affected=Individual.AFFECTED_STATUS_AFFECTED, individual__sex__in=Individual.MALE_SEXES,
+ annotations['affected_male_family_guids'] = ArrayAgg('family__guid', distinct=True, filter=Q(
+ affected=Individual.AFFECTED_STATUS_AFFECTED, sex__in=Individual.MALE_SEXES,
))
else:
annotations['samples'] = ArrayAgg(JSONObject(
- affected='individual__affected', sex='individual__sex', sample_id='sample_id', sample_type='sample_type',
- family_guid=F('individual__family__guid'), individual_guid=F('individual__guid'),
- ))
+ affected='affected', sex='sex', sample_id='individual_id', sample_type='active_datasets__sample_type',
+ family_guid=F('family__guid'), individual_guid=F('guid'),
+ ), distinct=True)
- return samples.aggregate(**annotations)
+ return individuals.aggregate(**annotations)
def _get_sample_data(families, dataset_type, annotate_affected_males=False, allow_no_samples=False, inheritance_mode=None, inheritance_filter=None, has_location_filter=False):
sample_type = None
- if dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS):
+ if dataset_type.startswith(Dataset.DATASET_TYPE_SV_CALLS):
dataset_type, sample_type = dataset_type.split('_')
- samples = _get_valid_samples(families, dataset_type, sample_type, allow_no_samples)
- if not samples:
+ individuals = _get_valid_samples(families, dataset_type, sample_type, allow_no_samples)
+ if not individuals:
return {}
individual_affected_status = (inheritance_filter or {}).get('affected')
affected_family_only = inheritance_mode and not individual_affected_status
- sample_data = _get_sample_metadata(samples, affected_family_only, annotate_affected_males)
+ sample_data = _get_sample_metadata(individuals, affected_family_only, annotate_affected_males)
family_guids = set(sample_data.pop('family_guids'))
if not has_location_filter:
@@ -657,8 +657,8 @@ def _get_sample_data(families, dataset_type, annotate_affected_males=False, allo
sample_data['sample_type_families'][sample['sample_type']].add(sample['family_guid'])
else:
sample_data['sample_type_families'] = dict(
- samples.filter(individual__family__guid__in=family_guids).values('sample_type').values_list(
- 'sample_type', ArrayAgg('individual__family__guid', distinct=True),
+ individuals.filter(family__guid__in=family_guids).values('active_datasets__sample_type').values_list(
+ 'active_datasets__sample_type', ArrayAgg('family__guid', distinct=True),
)
)
if len(sample_data['sample_type_families']) == 2:
@@ -671,12 +671,12 @@ def _get_sample_data(families, dataset_type, annotate_affected_males=False, allo
if set(families) - multi_families
}
sample_data['sample_type_families']['multi'] = multi_families
- _add_missing_multi_type_samples(samples, sample_data)
+ _add_missing_multi_type_samples(individuals, sample_data)
return sample_data
-def _add_missing_multi_type_samples(samples, data):
+def _add_missing_multi_type_samples(individuals, data):
data['family_missing_type_samples'] = defaultdict(lambda: defaultdict(list))
if 'samples' in data:
individual_sample_types = defaultdict(list)
@@ -687,13 +687,13 @@ def _add_missing_multi_type_samples(samples, data):
for samples in individual_sample_types.values() if len(samples) == 1
]
else:
- individual_samples = samples.filter(individual__family__guid__in=data['sample_type_families']['multi'],
- ).values('individual_id', family_guid=F('individual__family__guid')).annotate(
- samples=ArrayAgg(JSONObject(sample_id='sample_id', sample_type='sample_type'))
+ individual_samples = individuals.filter(family__guid__in=data['sample_type_families']['multi'],
+ ).values('id', family_guid=F('family__guid')).annotate(
+ samples=ArrayAgg(JSONObject(sample_id='individual_id', sample_type='active_datasets__sample_type'))
).filter(samples__len=1)
for agg in individual_samples:
sample = agg['samples'][0]
- missing_type = Sample.SAMPLE_TYPE_WES if sample['sample_type'] == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS
+ missing_type = Dataset.SAMPLE_TYPE_WES if sample['sample_type'] == Dataset.SAMPLE_TYPE_WGS else Dataset.SAMPLE_TYPE_WGS
data['family_missing_type_samples'][agg['family_guid']][missing_type].append(sample['sample_id'])
@@ -907,7 +907,7 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff
)
except InvalidDatasetTypeException:
continue
- if dataset_type.startswith(Sample.DATASET_TYPE_SV_CALLS):
+ if dataset_type.startswith(Dataset.DATASET_TYPE_SV_CALLS):
if not sample_type:
raise InvalidSearchException('Sample type must be specified to look up a structural variant')
elif not dataset_type.endswith(sample_type):
@@ -945,7 +945,7 @@ def clickhouse_variant_lookup(user, variant_id, sample_type, genome_version, aff
if variant.get('svType') in {'DEL', 'DUP'}:
other_sample_type, other_entry_class = next(
(dt, cls) for dt, cls in ENTRY_CLASS_MAP[genome_version].items()
- if dt != data_type and dt.startswith(Sample.DATASET_TYPE_SV_CALLS)
+ if dt != data_type and dt.startswith(Dataset.DATASET_TYPE_SV_CALLS)
)
other_variants_cls = VARIANTS_CLASS_MAP[genome_version][other_sample_type]
@@ -1000,7 +1000,7 @@ def get_variants_queryset(genome_version, dataset_type, keys, variant_ids=None):
def get_variant_details_queryset(genome_version, dataset_type, keys):
- if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS:
+ if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS:
return VARIANT_DETAILS_CLASS_MAP[genome_version].objects.filter(key__in=keys)
return get_variants_queryset(genome_version, dataset_type, keys)
@@ -1029,12 +1029,12 @@ def _main_transcript(selected_transcript_id, sorted_transcripts):
def delete_clickhouse_project(project, dataset_type, sample_type=None):
- if dataset_type == Sample.DATASET_TYPE_SV_CALLS and sample_type == Sample.SAMPLE_TYPE_WES:
+ if dataset_type == Dataset.DATASET_TYPE_SV_CALLS and sample_type == Dataset.SAMPLE_TYPE_WES:
dataset_type = 'GCNV'
table_base = f'{GENOME_VERSION_LOOKUP[project.genome_version]}/{dataset_type}'
partition_id = f"'{project.guid}'"
partition_ids = [partition_id]
- if project.genome_version == GENOME_VERSION_GRCh38 and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS:
+ if project.genome_version == GENOME_VERSION_GRCh38 and dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS:
partition_ids = [
f'({partition_id}, {pid})' for pid in
EntriesSnvIndel.objects.filter(project_guid=project.guid).values_list('partition_id', flat=True).distinct()
diff --git a/clickhouse_search/search_tests.py b/clickhouse_search/search_tests.py
index 8681ac739d..6801b65b4b 100644
--- a/clickhouse_search/search_tests.py
+++ b/clickhouse_search/search_tests.py
@@ -38,7 +38,7 @@
DEFAULT_PROJECT_FAMILIES, SINGLE_FAMILY_PROJECT_FAMILIES, SV_PROJECT_FAMILIES, MULTI_PROJECT_PROJECT_FAMILIES, \
format_cached_variant
from reference_data.models import Omim
-from seqr.models import Project, Family, Sample, VariantSearch, VariantSearchResults, SavedVariant, Individual
+from seqr.models import Project, Family, Dataset, VariantSearch, VariantSearchResults, SavedVariant, Individual
from seqr.views.apis.data_manager_api import trigger_delete_project
from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, GENE_VARIANT_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, \
SAVED_VARIANT_DETAIL_FIELDS, FUNCTIONAL_FIELDS, TAG_FIELDS, FAMILY_FIELDS, INDIVIDUAL_FIELDS, IGV_SAMPLE_FIELDS, \
@@ -139,7 +139,7 @@ def assert_cached_results(self, expected_results, cache_key):
self.mock_redis.expire.reset_mock()
def _execute_search(self, sort='xpos', inheritance_mode=None, inheritance_filter=None, quality_filter=None, project_families=None, request_body=None, check_login=None, query_params=None, search_hash=None, **search_kwargs):
- search_hash = search_hash or random.randint(1000, 10000) # nosec
+ search_hash = search_hash or random.randint(1000, 9000000) # nosec
self.mock_results_guid.return_value = f'VRS{search_hash:07d}'
url = reverse(query_variants_handler, args=[search_hash])
@@ -250,8 +250,6 @@ def _get_cached_variant(cls, variant, cached_variant_fields):
def _set_grch37_search(self):
Project.objects.filter(id=1).update(genome_version='37')
- Sample.objects.filter(sample_id='HG00732').update(is_active=False)
- Sample.objects.exclude(dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS).update(is_active=False)
def test_single_family_search(self):
variant_gene_counts = {
@@ -326,7 +324,7 @@ def test_single_project_search(self):
gene_counts={**variant_gene_counts, **GCNV_GENE_COUNTS, 'ENSG00000277258': {'total': 2, 'families': {'F000002_2': 2}}}
)
- self._add_sample_type_samples('WES', dataset_type='SV', guid__in=['S000135_na20870'])
+ self._add_sample_type_samples('WES', dataset_type='SV', guid__in=['S000129_na19675'])
self._assert_expected_search(
[GCNV_MULTI_FAMILY_VARIANT1, GCNV_MULTI_FAMILY_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4], gene_counts={
'ENSG00000129562': {'total': 1, 'families': {'F000002_2': 1, 'F000003_3': 1}},
@@ -410,17 +408,21 @@ def test_all_project_search(self):
)
def test_both_sample_types_search(self):
- Sample.objects.filter(dataset_type='MITO').update(is_active=False)
+ Dataset.objects.get(guid='S000149_hg00733').active_individuals.clear()
# One family (F000011_11) in a multi-project search has identical exome and genome data.
- self._add_sample_type_samples('WES', individual__family__guid='F000011_11')
+ self._add_sample_type_samples('WES', active_individuals__family__guid='F000011_11')
self._assert_expected_search(
MULTI_PROJECT_BOTH_SAMPLE_TYPE_VARIANTS, gene_counts=GENE_COUNTS, locus={'rawItems': 'chr1:1-100000000'},
project_families=MULTI_PROJECT_PROJECT_FAMILIES, check_login=self.check_collaborator_login,
)
- self._add_sample_type_samples('WGS', guid__in=['S000132_hg00731'])
+ dataset = Dataset.objects.get(guid='S000129_na19675')
+ dataset.pk = None
+ dataset.sample_type = 'WGS'
+ dataset.save()
+ dataset.active_individuals.add(4)
# Variant 1 is de novo in exome but inherited and homozygous in genome.
# Variant 2 is inherited and homozygous in exome and de novo and homozygous in genome, so it fails de-novo inheritance when parental data is missing in genome.
@@ -437,12 +439,13 @@ def test_both_sample_types_search(self):
inheritance_mode='any_affected', quality_filter={'min_gq': 40, 'min_qs': 20}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES,
)
+ self.maxDiff = None
self._assert_expected_search(
[VARIANT1_BOTH_SAMPLE_TYPES, VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1],
inheritance_mode='de_novo', quality_filter=None, project_families=SINGLE_FAMILY_PROJECT_FAMILIES,
)
- self._add_sample_type_samples('WGS', guid__in=['S000133_hg00732', 'S000134_hg00733'])
+ dataset.active_individuals.add(5, 6)
self._assert_expected_search(
[VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES, VARIANT3_BOTH_SAMPLE_TYPES,
VARIANT4_BOTH_SAMPLE_TYPES, GCNV_VARIANT1, GCNV_VARIANT2, GCNV_VARIANT3, GCNV_VARIANT4],
@@ -459,7 +462,6 @@ def test_both_sample_types_search(self):
inheritance_mode='de_novo', quality_filter={'min_gq': 40}, project_families=SINGLE_FAMILY_PROJECT_FAMILIES,
)
- self.maxDiff = None
self._assert_expected_search(
[VARIANT1_BOTH_SAMPLE_TYPES, VARIANT2_BOTH_SAMPLE_TYPES,
[{**VARIANT2_BOTH_SAMPLE_TYPES, 'selectedMainTranscriptId': 'ENST00000450625'}, GCNV_VARIANT4],
@@ -474,12 +476,14 @@ def test_both_sample_types_search(self):
@staticmethod
def _add_sample_type_samples(sample_type, dataset_type=None, **sample_filter):
- for sample in Sample.objects.filter(**sample_filter):
- sample.pk = None
- sample.sample_type = sample_type
+ for dataset in Dataset.objects.filter(**sample_filter):
+ individuals = dataset.active_individuals.all()
+ dataset.pk = None
+ dataset.sample_type = sample_type
if dataset_type:
- sample.dataset_type = dataset_type
- sample.save()
+ dataset.dataset_type = dataset_type
+ dataset.save()
+ dataset.active_individuals.set(individuals)
def test_inheritance_filter(self):
inheritance_mode = 'any_affected'
@@ -647,10 +651,12 @@ def test_inheritance_filter(self):
)
# Test deletion in trans with hom alt snp/indel
- for sample in Sample.objects.filter(individual__family_id=14):
- sample.pk = None
- sample.dataset_type = 'SNV_INDEL'
- sample.save()
+ dataset = Dataset.objects.get(guid='S000147_na21234')
+ dataset_individuals = dataset.active_individuals.all()
+ dataset.pk = None
+ dataset.dataset_type = 'SNV_INDEL'
+ dataset.save()
+ dataset.active_individuals.set(dataset_individuals)
self._assert_expected_search(
[[SV_VARIANT1, SV_VARIANT2], [SV_VARIANT1, PROJECT_4_COMP_HET_VARIANT], PROJECT_4_COMP_HET_VARIANT, SV_VARIANT4],
inheritance_mode=inheritance_mode, inheritance_filter=sv_affected, **COMP_HET_ALL_PASS_FILTERS,
@@ -845,7 +851,7 @@ def test_location_search(self):
],
)
- self._add_sample_type_samples('WES', individual__family__guid='F000014_14')
+ self._add_sample_type_samples('WES', active_individuals__family__guid='F000014_14')
self._assert_expected_search(
[SV_VARIANT1, SV_VARIANT2, MULTI_PROJECT_GCNV_VARIANT3, GCNV_VARIANT4], locus=sv_locus,
project_families=[*SINGLE_FAMILY_PROJECT_FAMILIES, *SV_PROJECT_FAMILIES],
@@ -968,7 +974,7 @@ def test_invalid_search(self):
self._assert_expected_search_error('Location must be specified to search across multiple projects', project_families=MULTI_PROJECT_PROJECT_FAMILIES)
- Sample.objects.filter(guid='S000143_na20885').update(sample_id='HG00732')
+ Individual.objects.filter(guid='I000015_na20885').update(individual_id='HG00732')
self._assert_expected_search_error(
'The following samples are incorrectly configured and have different affected statuses in different projects: '
'HG00732 (1kg project nåme with uniçøde/ Test Reprocessed Project)',
@@ -1852,7 +1858,7 @@ def test_secondary_annotations_filter(self):
],
)
- self._add_sample_type_samples('WES', individual__family__guid='F000014_14')
+ self._add_sample_type_samples('WES', active_individuals__family__guid='F000014_14')
self._assert_expected_search(
[MULTI_DATA_TYPE_COMP_HET_VARIANT2, [MULTI_DATA_TYPE_COMP_HET_VARIANT2, GCNV_VARIANT4], MULTI_PROJECT_GCNV_VARIANT3, [GCNV_VARIANT3, GCNV_VARIANT4]],
inheritance_mode='recessive',
@@ -1876,7 +1882,6 @@ def test_secondary_annotations_filter(self):
'familyGuids': ['F000002_2_x'],
'genotypes': {k: {**v, 'familyGuid': 'F000002_2_x'} for k, v in MULTI_DATA_TYPE_COMP_HET_VARIANT2['genotypes'].items()}
}
- Sample.objects.filter(guid='S000146_hg00732').update(is_active=False)
Family.objects.filter(guid='F000002_2').update(guid='F000002_2_x')
self._assert_expected_search(
[[missing_gt_comp_het_variant, missing_gt_gcnv_variant]],
@@ -2467,9 +2472,10 @@ def test_trigger_delete_project(self):
23: {'ac_wes': 0, 'ac_wgs': 0, 'hom_wes': 0, 'hom_wgs': 0, 'ac_affected': 0, 'hom_affected': 0},
})
- project_samples = Sample.objects.filter(individual__family__project__guid='R0001_1kg', is_active=True)
- self.assertEqual(project_samples.filter(dataset_type='SNV_INDEL').count(), 0)
- self.assertEqual(project_samples.count(), 4)
+ project_datasets = Dataset.objects.filter(active_individuals__family__project__guid='R0001_1kg')
+ self.assertEqual(project_datasets.filter(dataset_type='SNV_INDEL').count(), 0)
+ self.assertEqual(project_datasets.count(), 4)
+ self.assertEqual(Dataset.objects.get(guid='S000129_na19675').inactive_individuals.count(), 7)
body['datasetType'] = 'SV'
response = self.client.post(url, content_type='application/json', data=json.dumps(body))
@@ -2480,7 +2486,7 @@ def test_trigger_delete_project(self):
'Deleted all GCNV search data for project 1kg project n\xe5me with uni\xe7\xf8de',
],
})
- self.assertEqual(project_samples.filter(dataset_type='SV').count(), 0)
- self.assertEqual(project_samples.count(), 1)
+ self.assertEqual(project_datasets.filter(dataset_type='SV').count(), 0)
+ self.assertEqual(project_datasets.count(), 1)
diff --git a/seqr/admin.py b/seqr/admin.py
index 2ac70cc8e6..09d595c0be 100644
--- a/seqr/admin.py
+++ b/seqr/admin.py
@@ -1,12 +1,12 @@
from copy import deepcopy
from django.contrib import admin
from matchmaker.models import MatchmakerSubmission, MatchmakerIncomingQuery, MatchmakerResult, MatchmakerContactNotes
-from seqr.models import Project, Family, Individual, Sample, LocusList, LocusListGene, LocusListInterval, VariantNote, \
+from seqr.models import Project, Family, Individual, Dataset, LocusList, LocusListGene, LocusListInterval, VariantNote, \
VariantTag, VariantTagType, VariantFunctionalData, SavedVariant, GeneNote, AnalysisGroup, ProjectCategory, \
FamilyAnalysedBy, VariantSearch, VariantSearchResults, IgvSample, UserPolicy, WarningMessage, FamilyNote, DynamicAnalysisGroup
for model_class in [
- Project, Family, Individual, Sample, IgvSample, LocusList, LocusListGene, LocusListInterval, VariantNote, VariantTag,
+ Project, Family, Individual, Dataset, IgvSample, LocusList, LocusListGene, LocusListInterval, VariantNote, VariantTag,
VariantTagType, VariantFunctionalData, SavedVariant, GeneNote, AnalysisGroup, ProjectCategory, FamilyAnalysedBy,
VariantSearch, VariantSearchResults, MatchmakerSubmission, MatchmakerIncomingQuery, MatchmakerResult,
MatchmakerContactNotes, FamilyNote, DynamicAnalysisGroup,
diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json
index 005f532f82..68a415bcdc 100644
--- a/seqr/fixtures/1kg_project.json
+++ b/seqr/fixtures/1kg_project.json
@@ -910,38 +910,33 @@
}
},
{
- "model": "seqr.sample",
+ "model": "seqr.dataset",
"pk": 129,
"fields": {
"guid": "S000129_na19675",
"created_date": "2017-02-05T06:42:55.397Z",
"created_by": null,
"last_modified_date": "2017-03-13T09:07:49.744Z",
-
- "individual": 1,
+ "active_individuals": [1, 4, 5, 6, 7, 9],
+ "inactive_individuals": [3],
"sample_type": "WES",
"dataset_type": "SNV_INDEL",
- "sample_id": "NA19675",
- "is_active": true,
- "elasticsearch_index": "test_index",
+ "data_source": "test_index",
"loaded_date": "2017-02-05T06:12:55.397Z"
}
},
{
- "model": "seqr.sample",
+ "model": "seqr.dataset",
"pk": 130,
"fields": {
"guid": "S000130_na19678",
"created_date": "2017-02-05T06:42:55.397Z",
"created_by": null,
"last_modified_date": "2017-03-13T09:07:49.744Z",
-
- "individual": 2,
+ "active_individuals": [2],
"sample_type": "WES",
"dataset_type": "SNV_INDEL",
- "sample_id": "NA19678",
- "is_active": true,
- "elasticsearch_index": "test_index_old",
+ "data_source": "test_index_old",
"loaded_date": "2017-02-05T06:13:55.397Z"
}
},
@@ -961,365 +956,91 @@
}
},
{
- "model": "seqr.sample",
- "pk": 131,
- "fields": {
- "guid": "S000131_na19679",
- "created_date": "2017-02-05T06:42:55.397Z",
- "created_by": null,
- "last_modified_date": "2017-03-13T09:07:49.800Z",
-
- "sample_id": "NA19679",
- "sample_type": "WES",
- "is_active": false,
- "elasticsearch_index": "test_index",
- "individual": 3,
- "dataset_type": "SNV_INDEL",
- "loaded_date": "2017-02-05T06:15:55.397Z"
- }
-},
-{
- "model": "seqr.sample",
- "pk": 132,
- "fields": {
- "guid": "S000132_hg00731",
- "created_date": "2017-02-05T06:42:55.397Z",
- "created_by": null,
- "last_modified_date": "2017-03-13T09:07:49.937Z",
- "elasticsearch_index": "test_index",
- "sample_id": "HG00731",
- "sample_type": "WES",
- "is_active": true,
- "individual": 4,
- "dataset_type": "SNV_INDEL",
- "loaded_date": "2017-02-05T06:16:55.397Z"
- }
-},
-{
- "model": "seqr.sample",
- "pk": 133,
- "fields": {
- "guid": "S000133_hg00732",
- "created_date": "2017-02-05T06:42:55.397Z",
- "created_by": null,
- "last_modified_date": "2017-03-13T09:07:49.963Z",
- "elasticsearch_index": "test_index",
- "sample_id": "HG00732",
- "sample_type": "WES",
- "is_active": true,
- "individual": 5,
- "dataset_type": "SNV_INDEL",
- "loaded_date": "2017-02-05T06:17:55.397Z"
- }
-},
-{
- "model": "seqr.sample",
- "pk": 134,
- "fields": {
- "guid": "S000134_hg00733",
- "created_date": "2017-02-05T06:42:55.397Z",
- "created_by": null,
- "last_modified_date": "2017-03-13T09:07:49.990Z",
- "elasticsearch_index": "test_index",
- "sample_id": "HG00733",
- "sample_type": "WES",
- "is_active": true,
- "individual": 6,
- "dataset_type": "SNV_INDEL",
- "loaded_date": "2017-02-05T06:18:55.397Z"
- }
-},
-{
- "model": "seqr.sample",
- "pk": 135,
- "fields": {
- "guid": "S000135_na20870",
- "created_date": "2017-02-05T06:42:55.397Z",
- "created_by": null,
- "last_modified_date": "2017-03-13T09:07:50.022Z",
- "elasticsearch_index": "test_index",
- "sample_id": "NA20870",
- "sample_type": "WES",
- "is_active": true,
- "individual": 7,
- "dataset_type": "SNV_INDEL",
- "loaded_date": "2017-02-05T06:19:55.397Z"
- }
-},
-{
- "model": "seqr.sample",
+ "model": "seqr.dataset",
"pk": 136,
"fields": {
"guid": "S000136_na20872",
"created_date": "2017-02-05T06:42:55.397Z",
"created_by": null,
"last_modified_date": "2017-03-13T09:07:50.052Z",
- "sample_id": "NA20872",
"sample_type": "WES",
- "is_active": false,
- "individual": 8,
+ "inactive_individuals": [8, 10, 11, 12, 14],
"dataset_type": "SNV_INDEL",
- "elasticsearch_index": "1kg.vcf.gz",
+ "data_source": "1kg.vcf.gz",
"loaded_date": "2017-02-05T06:20:55.397Z"
}
},
{
- "model": "seqr.sample",
- "pk": 137,
- "fields": {
- "guid": "S000137_na20874",
- "created_date": "2017-02-05T06:42:55.397Z",
- "created_by": null,
- "last_modified_date": "2017-03-13T09:07:50.079Z",
- "elasticsearch_index": "test_index",
- "sample_id": "NA20874",
- "sample_type": "WES",
- "is_active": true,
- "individual": 9,
- "dataset_type": "SNV_INDEL",
- "loaded_date": "2017-02-05T06:21:55.397Z"
- }
-},
-{
- "model": "seqr.sample",
- "pk": 138,
- "fields": {
- "guid": "S000138_na20875",
- "created_date": "2017-02-05T06:42:55.397Z",
- "created_by": null,
- "last_modified_date": "2017-03-13T09:07:50.111Z",
- "sample_id": "NA20875",
- "sample_type": "WES",
- "is_active": false,
- "individual": 10,
- "dataset_type": "SNV_INDEL",
- "elasticsearch_index": "1kg.vcf.gz",
- "loaded_date": "2017-02-05T06:22:55.397Z"
- }
-},
-{
- "model": "seqr.sample",
- "pk": 139,
- "fields": {
- "guid": "S000139_na20876",
- "created_date": "2017-02-05T06:42:55.397Z",
- "created_by": null,
- "last_modified_date": "2017-03-13T09:07:50.136Z",
-
- "sample_id": "NA20876",
- "sample_type": "WES",
- "is_active": false,
- "individual": 11,
- "dataset_type": "SNV_INDEL",
- "elasticsearch_index": "1kg.vcf.gz",
- "loaded_date": "2017-02-05T06:23:55.397Z"
- }
-},
-{
- "model": "seqr.sample",
- "pk": 140,
- "fields": {
- "guid": "S000140_na20877",
- "created_date": "2017-02-05T06:42:55.397Z",
- "created_by": null,
- "last_modified_date": "2017-03-13T09:07:50.165Z",
-
- "sample_id": "NA19678",
- "sample_type": "WES",
- "is_active": false,
- "individual": 12,
- "dataset_type": "SNV_INDEL",
- "elasticsearch_index": "1kg.vcf.gz",
- "loaded_date": "2017-02-05T06:24:55.397Z"
- }
-},
-{
- "model": "seqr.sample",
- "pk": 142,
- "fields": {
- "guid": "S000142_na20881",
- "created_date": "2017-02-05T06:42:55.397Z",
- "created_by": null,
- "last_modified_date": "2017-03-13T09:07:50.220Z",
-
- "sample_id": "NA20881",
- "sample_type": "WES",
- "is_active": false,
- "individual": 14,
- "dataset_type": "SNV_INDEL",
- "elasticsearch_index": "1kg.vcf.gz",
- "loaded_date": "2017-02-05T06:25:55.397Z"
- }
-},
-{
- "model": "seqr.sample",
+ "model": "seqr.dataset",
"pk": 143,
"fields": {
"guid": "S000143_na20885",
"created_date": "2020-02-05T06:42:55.397Z",
"created_by": null,
"last_modified_date": "2020-03-13T09:07:50.247Z",
- "elasticsearch_index": "test_index_second",
- "sample_id": "NA20885",
+ "data_source": "test_index_second",
"sample_type": "WGS",
- "is_active": true,
- "individual": 15,
+ "active_individuals": [15],
"dataset_type": "SNV_INDEL",
"loaded_date": "2020-02-05T06:26:55.397Z"
}
},
{
- "model": "seqr.sample",
+ "model": "seqr.dataset",
"pk": 144,
"fields": {
"guid": "S000144_na20888",
"created_date": "2017-02-05T06:42:55.397Z",
"created_by": null,
"last_modified_date": "2017-03-13T09:07:50.277Z",
-
- "sample_id": "NA20888",
"sample_type": "WGS",
- "is_active": false,
- "individual": 16,
+ "inactive_individuals": [16],
"dataset_type": "SNV_INDEL",
- "elasticsearch_index": "1kg.vcf.gz",
+ "data_source": "1kg.vcf.gz",
"loaded_date": "2017-02-05T06:27:55.397Z"
}
},
{
- "model": "seqr.sample",
- "pk": 154,
- "fields": {
- "guid": "S000154_na20889",
- "created_date": "2017-02-05T06:42:55.397Z",
- "created_by": null,
- "last_modified_date": "2017-03-13T09:07:50.277Z",
-
- "sample_id": "NA20889",
- "sample_type": "WES",
- "is_active": false,
- "individual": 17,
- "dataset_type": "SNV_INDEL",
- "elasticsearch_index": "1kg.vcf.gz",
- "data_source": "auto__2023-08-08",
- "loaded_date": "2017-02-05T06:28:55.397Z"
- }
-},
-{
- "model": "seqr.sample",
+ "model": "seqr.dataset",
"pk": 145,
"fields": {
"guid": "S000145_hg00731",
"created_date": "2018-02-05T06:42:55.397Z",
"created_by": null,
"last_modified_date": "2018-03-13T09:07:49.937Z",
- "elasticsearch_index": "test_index_sv",
- "sample_id": "HG00731",
+ "data_source": "test_index_sv",
"sample_type": "WES",
- "is_active": true,
- "individual": 4,
+ "active_individuals": [4, 5, 6],
"dataset_type": "SV",
"loaded_date": "2018-02-05T06:29:55.397Z"
}
},
{
- "model": "seqr.sample",
- "pk": 146,
- "fields": {
- "guid": "S000146_hg00732",
- "created_date": "2018-02-05T06:42:55.397Z",
- "created_by": null,
- "last_modified_date": "2018-03-13T09:07:49.963Z",
- "elasticsearch_index": "test_index_sv",
- "sample_id": "HG00732",
- "sample_type": "WES",
- "is_active": true,
- "individual": 5,
- "dataset_type": "SV",
- "loaded_date": "2018-02-05T06:30:55.397Z"
- }
-},
-{
- "model": "seqr.sample",
+ "model": "seqr.dataset",
"pk": 147,
"fields": {
"guid": "S000147_na21234",
"created_date": "2018-02-05T06:42:55.397Z",
"created_by": null,
"last_modified_date": "2018-03-13T09:07:49.937Z",
- "elasticsearch_index": "test_index_sv_wgs",
- "sample_id": "NA21234",
- "sample_type": "WGS",
- "is_active": true,
- "individual": 18,
- "dataset_type": "SV",
- "loaded_date": "2018-02-05T06:31:55.397Z"
- }
-},
-{
- "model": "seqr.sample",
- "pk": 173,
- "fields": {
- "guid": "S000173_na21987",
- "created_date": "2018-02-05T06:42:55.397Z",
- "created_by": null,
- "last_modified_date": "2018-03-13T09:07:49.937Z",
- "elasticsearch_index": "test_index_sv_wgs",
- "sample_id": "NA21987",
- "sample_type": "WGS",
- "is_active": true,
- "individual": 19,
- "dataset_type": "SV",
- "loaded_date": "2018-02-05T06:31:55.397Z"
- }
-},
-{
- "model": "seqr.sample",
- "pk": 174,
- "fields": {
- "guid": "S000174_na21654",
- "created_date": "2018-02-05T06:42:55.397Z",
- "created_by": null,
- "last_modified_date": "2018-03-13T09:07:49.937Z",
- "elasticsearch_index": "test_index_sv_wgs",
- "sample_id": "NA21654",
+ "data_source": "test_index_sv_wgs",
"sample_type": "WGS",
- "is_active": true,
- "individual": 21,
+ "active_individuals": [18, 19, 21],
"dataset_type": "SV",
"loaded_date": "2018-02-05T06:31:55.397Z"
}
},
{
- "model": "seqr.sample",
- "pk": 148,
- "fields": {
- "guid": "S000148_hg00733",
- "created_date": "2018-02-05T06:42:55.397Z",
- "created_by": null,
- "last_modified_date": "2018-03-13T09:07:49.963Z",
- "elasticsearch_index": "test_index_sv",
- "sample_id": "HG00733",
- "sample_type": "WES",
- "is_active": true,
- "individual": 6,
- "dataset_type": "SV",
- "loaded_date": "2018-02-05T06:32:55.397Z"
- }
-},
-{
- "model": "seqr.sample",
+ "model": "seqr.dataset",
"pk": 149,
"fields": {
"guid": "S000149_hg00733",
"created_date": "2022-02-05T06:42:55.397Z",
"created_by": null,
"last_modified_date": "2022-03-13T09:07:49.937Z",
- "elasticsearch_index": "test_index_mito_wgs",
- "sample_id": "HG00731",
+ "data_source": "test_index_mito_wgs",
"sample_type": "WES",
- "is_active": true,
- "individual": 4,
+ "active_individuals": [4],
"dataset_type": "MITO",
"loaded_date": "2022-02-05T06:33:55.397Z"
}
@@ -1354,6 +1075,21 @@
"data_type": "S"
}
},
+{
+ "model": "seqr.dataset",
+ "pk": 154,
+ "fields": {
+ "guid": "S000154_na20889",
+ "created_date": "2017-02-05T06:42:55.397Z",
+ "created_by": null,
+ "last_modified_date": "2017-03-13T09:07:50.277Z",
+ "sample_type": "WES",
+ "inactive_individuals": [17],
+ "dataset_type": "SNV_INDEL",
+ "data_source": "auto__2023-08-08",
+ "loaded_date": "2017-02-05T06:28:55.397Z"
+ }
+},
{
"model": "seqr.rnasample",
"pk": 161,
diff --git a/seqr/management/commands/check_for_new_samples_from_pipeline.py b/seqr/management/commands/check_for_new_samples_from_pipeline.py
index abe2fd3ecf..444473d486 100644
--- a/seqr/management/commands/check_for_new_samples_from_pipeline.py
+++ b/seqr/management/commands/check_for_new_samples_from_pipeline.py
@@ -10,12 +10,13 @@
from clickhouse_search.search import get_clickhouse_genotypes
from reference_data.models import GENOME_VERSION_LOOKUP
-from seqr.models import Family, Sample, Project, Individual, SavedVariant
+from seqr.models import Family, Dataset, Project, Individual, SavedVariant
from seqr.utils.communication_utils import safe_post_to_slack, send_project_email
from seqr.utils.file_utils import file_iter, list_files, is_google_bucket_file_path
from seqr.utils.add_data_utils import notify_search_data_loaded, update_airtable_loading_tracking_status
from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS
from seqr.views.utils.export_utils import write_multiple_files
+from seqr.views.utils.json_to_orm_utils import create_model_from_json
from seqr.views.utils.permissions_utils import is_internal_anvil_project, project_has_anvil
from seqr.views.utils.variant_utils import reset_cached_search_results
from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, PIPELINE_DATA_DIR, ANVIL_UI_URL, IS_ANVIL_LOADING_DELAY, \
@@ -30,10 +31,10 @@
ERRORS_REPORTED_FILE_NAME = '_ERRORS_REPORTED'
RUN_PATH_FIELDS = ['genome_version', 'dataset_type', 'run_version', 'file_name']
-DATASET_TYPE_MAP = {'GCNV': Sample.DATASET_TYPE_SV_CALLS}
+DATASET_TYPE_MAP = {'GCNV': Dataset.DATASET_TYPE_SV_CALLS}
CLICKHOUSE_DATASET_TYPE_MAP = {
- 'GCNV': f'{Sample.DATASET_TYPE_SV_CALLS}_WES',
- Sample.DATASET_TYPE_SV_CALLS: f'{Sample.DATASET_TYPE_SV_CALLS}_WGS',
+ 'GCNV': f'{Dataset.DATASET_TYPE_SV_CALLS}_WES',
+ Dataset.DATASET_TYPE_SV_CALLS: f'{Dataset.DATASET_TYPE_SV_CALLS}_WGS',
}
RELATEDNESS_CHECK_NAME = 'relatedness_check'
@@ -76,7 +77,7 @@ def handle(self, *args, **options):
def _load_success_runs(self, runs, success_run_dirs):
- loaded_runs = set(Sample.objects.filter(data_source__isnull=False).values_list('data_source', flat=True))
+ loaded_runs = {source for sources in Dataset.objects.values_list('data_source', flat=True) for source in sources.split(',')}
new_runs = {
run_dir: run_details for run_dir, run_details in runs.items()
if run_dir in success_run_dirs and run_details['run_version'] not in loaded_runs
@@ -206,18 +207,29 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers
family_project_map = {f.guid: f.project for f in families.select_related('project')}
families_by_project = defaultdict(list)
samples_by_project = defaultdict(list)
+ num_samples = 0
for family_guid, sample_ids in metadata['family_samples'].items():
project = family_project_map[family_guid]
families_by_project[project].append(family_guid)
samples_by_project[project] += sample_ids
+ num_samples += len(sample_ids)
- sample_project_tuples = []
+ individuals_by_project = {}
+ missing_samples = set()
+ all_individual_ids = set()
invalid_genome_version_projects = []
for project, sample_ids in samples_by_project.items():
- sample_project_tuples += [(sample_id, project.id) for sample_id in sample_ids]
project_genome_version = GENOME_VERSION_LOOKUP.get(project.genome_version, project.genome_version)
if project_genome_version != genome_version:
invalid_genome_version_projects.append((project.guid, project_genome_version))
+ continue
+
+ matched_individuals = dict(
+ Individual.objects.filter(family__project=project, individual_id__in=sample_ids).values_list('id', 'individual_id')
+ )
+ missing_samples.update(set(sample_ids) - set(matched_individuals.values()))
+ individuals_by_project[project] = matched_individuals
+ all_individual_ids.update(matched_individuals.keys())
if invalid_genome_version_projects:
raise CommandError(
@@ -226,14 +238,18 @@ def _load_new_samples(cls, metadata_path, genome_version, dataset_type, run_vers
)
sample_type = metadata['sample_type']
- logger.info(f'Loading {len(sample_project_tuples)} {sample_type} {dataset_type} samples in {len(samples_by_project)} projects')
- new_samples = cls._match_and_update_search_samples(
- sample_project_tuples, sample_type, dataset_type, data_source=run_version, elasticsearch_index=';'.join(metadata['callsets']),
- )
+ logger.info(f'Loading {num_samples} {sample_type} {dataset_type} samples in {len(samples_by_project)} projects')
+ if missing_samples:
+ sample_ids = ', '.join(sorted(missing_samples))
+ raise ValueError(f'Matches not found for sample ids: {sample_ids}')
- new_samples_by_project = dict(new_samples.values('individual__family__project').annotate(
- samples=ArrayAgg('sample_id', distinct=True),
- ).values_list('individual__family__project', 'samples'))
+ cls._update_matched_families(all_individual_ids, dataset_type, sample_type)
+
+ new_samples_by_project = {}
+ for project, individuals in individuals_by_project.items():
+ new_samples_by_project[project.id] = cls._match_and_update_search_datasets(
+ individuals, sample_type, dataset_type, data_source=run_version,
+ )
split_project_pdos = cls._report_loading_success(
dataset_type, sample_type, run_version, samples_by_project, new_samples_by_project,
@@ -271,7 +287,7 @@ def _report_loading_success(cls, dataset_type, sample_type, run_version, samples
project, is_internal, dataset_type, sample_type, new_samples_by_project.get(project.id, []),
num_samples=len(sample_ids),
)
- if session and is_internal and dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS:
+ if session and is_internal and dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS:
split_project_pdos[project.name] = cls._update_pdos(session, project.guid, sample_ids)
except Exception as e:
logger.error(f'Error reporting loading success for project {project.name} in {run_version}: {e}')
@@ -412,72 +428,38 @@ def _update_project_saved_variant_genotypes(project, family_guids, dataset_type)
return updates
@classmethod
- def _match_and_update_search_samples(cls, sample_project_tuples, sample_type, dataset_type, **sample_data):
- individual_ids_by_keys = cls._get_matched_individuals(sample_project_tuples)
- individual_ids = individual_ids_by_keys.values()
- matched_samples = Sample.objects.filter(is_active=True, dataset_type=dataset_type, sample_type=sample_type)
- cls._update_matched_families(individual_ids, matched_samples)
-
- sample_guids_by_individual = dict(Sample.objects.filter(
- individual_id__in=individual_ids, sample_type=sample_type, dataset_type=dataset_type, **sample_data,
- ).values_list('individual_id', 'guid'))
- remaining_key_individuals = {
- key: individual_id for key, individual_id in individual_ids_by_keys.items()
- if individual_id not in sample_guids_by_individual
- }
+ def _match_and_update_search_datasets(cls, individuals, sample_type, dataset_type, data_source):
loaded_date = timezone.now()
- created_sample_guids = cls._create_samples(
- remaining_key_individuals, loaded_date=loaded_date, sample_type=sample_type, dataset_type=dataset_type,
- **sample_data,
- ) if remaining_key_individuals else []
-
- sample_guids = list(sample_guids_by_individual.values()) + created_sample_guids
-
- activated_sample_guids = Sample.bulk_update(user=None, update_json={
- 'is_active': True,
- 'loaded_date': loaded_date,
- **sample_data,
- }, guid__in=sample_guids, is_active=False)
-
- inactivate_samples = matched_samples.filter(individual_id__in=individual_ids).exclude(guid__in=sample_guids)
- inactivated_sample_guids = Sample.bulk_update(
- user=None, update_json={'is_active': False}, queryset=inactivate_samples,
- )
-
- previous_loaded_individuals = set(Sample.objects.filter(guid__in=inactivated_sample_guids).values_list('individual_id', flat=True))
- return Sample.objects.filter(guid__in=activated_sample_guids).exclude(individual_id__in=previous_loaded_individuals)
+ dataset = create_model_from_json(Dataset, {
+ 'dataset_type': dataset_type, 'sample_type': sample_type, 'data_source': data_source, 'loaded_date': loaded_date,
+ }, user=None)
+ dataset.active_individuals.set(individuals.keys())
+
+ inactivate_datasets = Dataset.objects.filter(
+ dataset_type=dataset_type, sample_type=sample_type, active_individuals__id__in=individuals.keys(),
+ ).exclude(id=dataset.id)
+ loaded_individuals = set()
+ for dataset in inactivate_datasets:
+ inactivate_individuals = dataset.active_individuals.filter(id__in=individuals.keys())
+ loaded_individuals.update(inactivate_individuals.values_list('id', flat=True))
+ dataset.inactive_individuals.add(*inactivate_individuals)
+ dataset.active_individuals.remove(*inactivate_individuals)
+
+ return [sample_id for individual_id, sample_id in individuals.items() if individual_id not in loaded_individuals]
@staticmethod
- def _get_matched_individuals(sample_project_tuples):
- individual_ids_by_keys = {
- (individual_id, project_id): individual_db_id
- for individual_db_id, individual_id, project_id in Individual.objects.filter(
- family__project_id__in={project_id for _, project_id in sample_project_tuples},
- individual_id__in={sample_id for sample_id, _ in sample_project_tuples},
- ).values_list('id', 'individual_id', 'family__project_id')
- if (individual_id, project_id) in sample_project_tuples
- }
-
- missing_keys = set(sample_project_tuples) - set(individual_ids_by_keys.keys())
- if missing_keys:
- sample_ids = ', '.join(sorted([sample_id for sample_id, _ in missing_keys]))
- raise ValueError(f'Matches not found for sample ids: {sample_ids}')
-
- return individual_ids_by_keys
-
- @staticmethod
- def _update_matched_families(individual_ids, matched_samples):
+ def _update_matched_families(individual_ids, dataset_type, sample_type):
included_families = dict(
Family.objects.filter(individual__id__in=individual_ids).values_list('id', 'analysis_status')
)
- missing_individuals = matched_samples.filter(
- individual__family_id__in=included_families,
- ).exclude(individual_id__in=individual_ids).values(
- 'individual__family__family_id',
- ).annotate(individual_ids=ArrayAgg('individual__individual_id', ordering='individual__individual_id'))
+ missing_individuals = Individual.objects.filter(
+ family_id__in=included_families, active_datasets__dataset_type=dataset_type, active_datasets__sample_type=sample_type,
+ ).exclude(id__in=individual_ids).values(
+ 'family__family_id',
+ ).annotate(individual_ids=ArrayAgg('individual_id', ordering='individual_id'))
if missing_individuals:
missing_summary = ', '.join(sorted([
- f"{agg['individual__family__family_id']} ({', '.join(agg['individual_ids'])})" for agg in missing_individuals
+ f"{agg['family__family_id']} ({', '.join(agg['individual_ids'])})" for agg in missing_individuals
]))
raise ValueError(
f'The following families are included in the callset but are missing some family members: {missing_summary}'
@@ -491,17 +473,6 @@ def _update_matched_families(individual_ids, matched_samples):
user=None, update_json={'analysis_status': Family.ANALYSIS_STATUS_ANALYSIS_IN_PROGRESS}, id__in=family_ids_to_update,
)
- @staticmethod
- def _create_samples(remaining_key_individuals, **sample_params):
- new_samples = [
- Sample(
- individual_id=individual_id,
- sample_id=sample_key[0],
- **sample_params,
- ) for sample_key, individual_id in remaining_key_individuals.items()]
- new_sample_models = Sample.bulk_create(user=None, new_models=new_samples)
- return [s.guid for s in new_sample_models]
-
update_individuals_sample_qc = Command._update_individuals_sample_qc
get_pipeline_runs = Command._get_runs
diff --git a/seqr/management/commands/reload_saved_variant_genotypes.py b/seqr/management/commands/reload_saved_variant_genotypes.py
index 3cd146ac40..0af32763aa 100644
--- a/seqr/management/commands/reload_saved_variant_genotypes.py
+++ b/seqr/management/commands/reload_saved_variant_genotypes.py
@@ -1,7 +1,7 @@
import logging
from django.core.management.base import BaseCommand
from seqr.management.commands.check_for_new_samples_from_pipeline import update_project_saved_variant_genotypes
-from seqr.models import Project, Family, Sample
+from seqr.models import Project, Family, Dataset
logger = logging.getLogger(__name__)
@@ -18,12 +18,12 @@ def handle(self, *args, **options):
family_guid = options['family_guid']
family_guids = [family_guid] if family_guid else Family.objects.filter(project=project).values_list('guid', flat=True)
- samples = Sample.objects.filter(individual__family__project_id=project.id, is_active=True)
+ datasets = Dataset.objects.filter(active_individuals__family__project_id=project.id)
if family_guid:
- samples = samples.filter(individual__family__guid=family_guid)
+ datasets = datasets.filter(active_individuals__family__guid=family_guid)
dataset_types = {
- f'{dataset_type}_{sample_type}' if dataset_type == Sample.DATASET_TYPE_SV_CALLS else dataset_type
- for dataset_type, sample_type in samples.values_list('dataset_type', 'sample_type').distinct()
+ f'{dataset_type}_{sample_type}' if dataset_type == Dataset.DATASET_TYPE_SV_CALLS else dataset_type
+ for dataset_type, sample_type in datasets.values_list('dataset_type', 'sample_type').distinct()
}
for dataset_type in sorted(dataset_types):
update_project_saved_variant_genotypes(project, family_guids, dataset_type)
diff --git a/seqr/management/commands/tag_seqr_prioritized_variants.py b/seqr/management/commands/tag_seqr_prioritized_variants.py
index c59138e3f2..477bc2851b 100644
--- a/seqr/management/commands/tag_seqr_prioritized_variants.py
+++ b/seqr/management/commands/tag_seqr_prioritized_variants.py
@@ -9,7 +9,7 @@
from clickhouse_search.search import get_clickhouse_variants, get_search_genes, ENTRY_CLASS_MAP
from panelapp.models import PaLocusListGene
from reference_data.models import GENOME_VERSION_GRCh38
-from seqr.models import Project, Family, Individual, Sample, LocusList
+from seqr.models import Project, Family, Individual, Dataset, LocusList
from seqr.utils.communication_utils import send_project_notification
from clickhouse_search.constants import ANY_AFFECTED, HOMOZYGOUS_RECESSIVE, X_LINKED_RECESSIVE_MALE_AFFECTED, DE_NOVO, COMPOUND_HET
from seqr.views.utils.json_utils import DjangoJSONEncoderWithSets
@@ -482,7 +482,7 @@ def handle(self, *args, **options):
updates = {update: set() for update in ['matched_families', 'new_tag_keys', 'update_tag_keys', 'skipped_tag_keys']}
search_counts = {}
samples_by_dataset_type = {}
- sample_qs = Sample.objects.filter(individual__family__project=project, is_active=True)
+ sample_qs = Individual.objects.filter(family__project=project)
for dataset_type, searches in SEARCHES.items():
self._run_dataset_type_searches(
dataset_type, searches, sample_qs, updates, search_counts, samples_by_dataset_type, family_guid_map,
@@ -517,25 +517,25 @@ def handle(self, *args, **options):
@classmethod
def _run_dataset_type_searches(cls, dataset_type, searches, sample_qs, updates, search_counts, samples_by_dataset_type, family_guid_map, project, exclude_intervals, gene_by_moi):
- is_sv = dataset_type == Sample.DATASET_TYPE_SV_CALLS
- sample_qs = sample_qs.filter(dataset_type=dataset_type)
+ is_sv = dataset_type == Dataset.DATASET_TYPE_SV_CALLS
+ sample_qs = sample_qs.filter(active_datasets__dataset_type=dataset_type)
if is_sv:
sample_qs = sample_qs.exclude(
- individual__sv_flags__contains=['outlier_num._calls'], individual__affected=Individual.AFFECTED_STATUS_AFFECTED,
+ sv_flags__contains=['outlier_num._calls'], affected=Individual.AFFECTED_STATUS_AFFECTED,
)
- sample_types = list(sample_qs.values_list('sample_type', flat=True).distinct())
+ sample_types = list(sample_qs.values_list('active_datasets__sample_type', flat=True).distinct())
if len(sample_types) > 1:
raise CommandError('Variant prioritization not supported for projects with multiple sample types')
sample_type = sample_types[0]
if is_sv:
dataset_type = f'{dataset_type}_{sample_type}'
samples_by_family = {
- agg['individual__family__guid']: agg for agg in sample_qs.values('individual__family__guid').annotate(
+ agg['family__guid']: agg for agg in sample_qs.values('family__guid').annotate(
affecteds=ArrayAgg(
- JSONObject(maternal_guid='individual__mother__guid', paternal_guid='individual__father__guid', sex='individual__sex'),
- filter=Q(individual__affected=Individual.AFFECTED_STATUS_AFFECTED),
+ JSONObject(maternal_guid='mother__guid', paternal_guid='father__guid', sex='sex'),
+ filter=Q(affected=Individual.AFFECTED_STATUS_AFFECTED),
),
- unaffected_guids=ArrayAgg('individual__guid', filter=Q(individual__affected=Individual.AFFECTED_STATUS_UNAFFECTED)),
+ unaffected_guids=ArrayAgg('guid', filter=Q(affected=Individual.AFFECTED_STATUS_UNAFFECTED)),
).filter(affecteds__len__gt=0)
}
samples_by_dataset_type[dataset_type] = samples_by_family
@@ -608,12 +608,12 @@ def wrapped(new_variant_keys, family_variant_data):
def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by_dataset_type, family_guid_map, project, genes):
sv_dataset_type = next(dt for dt in samples_by_dataset_type.keys() if dt.startswith('SV'))
sample_type = sv_dataset_type.split('_')[-1]
- families = set(samples_by_dataset_type[sv_dataset_type].keys()).intersection(samples_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS].keys())
+ families = set(samples_by_dataset_type[sv_dataset_type].keys()).intersection(samples_by_dataset_type[Dataset.DATASET_TYPE_VARIANT_CALLS].keys())
sv_samples_by_family = {
guid: sample_data for guid, sample_data in samples_by_dataset_type[sv_dataset_type].items() if guid in families
}
snv_indel_samples_by_family = {
- guid: sample_data for guid, sample_data in samples_by_dataset_type[Sample.DATASET_TYPE_VARIANT_CALLS].items()
+ guid: sample_data for guid, sample_data in samples_by_dataset_type[Dataset.DATASET_TYPE_VARIANT_CALLS].items()
if guid in families
}
family_variant_data = defaultdict(lambda: {'matched_searches': set(), 'matched_comp_het_searches': set(), 'support_vars': set()})
@@ -627,7 +627,7 @@ def _run_multi_data_type_comp_het_search(cls, updates, search_counts, samples_by
project, sample_type, snv_indel_samples_by_family, config_search.get('family_filter'),
)
sample_data_by_dataset_type = {
- Sample.DATASET_TYPE_VARIANT_CALLS: snv_indel_sample_data, sv_dataset_type: sv_sample_data,
+ Dataset.DATASET_TYPE_VARIANT_CALLS: snv_indel_sample_data, sv_dataset_type: sv_sample_data,
}
num_results = cls._execute_search(
sample_data_by_dataset_type, search_name, family_variant_data, family_guid_map,
diff --git a/seqr/management/commands/transfer_families_to_different_project.py b/seqr/management/commands/transfer_families_to_different_project.py
index 93ac4ba1ae..5c5e401b73 100644
--- a/seqr/management/commands/transfer_families_to_different_project.py
+++ b/seqr/management/commands/transfer_families_to_different_project.py
@@ -1,6 +1,7 @@
from django.core.management.base import BaseCommand
+from django.db.models import Q
-from seqr.models import Project, Family, VariantTag, VariantTagType
+from seqr.models import Project, Family, VariantTag, VariantTagType, Dataset
from seqr.utils.add_data_utils import trigger_delete_families_search
import logging
@@ -36,6 +37,19 @@ def handle(self, *args, **options):
trigger_delete_families_search(from_project, list(families.values_list('guid', flat=True)))
+ remaining_families = Family.objects.filter(project=from_project).exclude(id__in={f.id for f in families})
+ split_dataset = Dataset.objects.filter(inactive_individuals__family__in=families).filter(
+ Q(inactive_individuals__family__in=remaining_families) | Q(active_individuals__family__in=remaining_families)
+ ).distinct()
+ logger.info(f'Splitting {split_dataset.count()} datasets')
+ for dataset in split_dataset:
+ individuals = dataset.inactive_individuals.filter(family__in=families)
+ dataset.inactive_individuals.remove(*individuals)
+ new_dataset = dataset
+ new_dataset.pk = None
+ new_dataset.save()
+ new_dataset.inactive_individuals.set(individuals)
+
for variant_tag_type in VariantTagType.objects.filter(project=from_project):
variant_tags = VariantTag.objects.filter(saved_variants__family__in=families, variant_tag_type=variant_tag_type)
if variant_tags:
diff --git a/seqr/management/commands/update_individuals_sample_qc.py b/seqr/management/commands/update_individuals_sample_qc.py
index fcb3e7fcf5..eb9e5d6889 100644
--- a/seqr/management/commands/update_individuals_sample_qc.py
+++ b/seqr/management/commands/update_individuals_sample_qc.py
@@ -4,7 +4,7 @@
from django.core.management.base import BaseCommand, CommandError
from reference_data.models import GENOME_VERSION_LOOKUP, GENOME_VERSION_GRCh38
-from seqr.models import Sample
+from seqr.models import Dataset
from seqr.management.commands.check_for_new_samples_from_pipeline import update_individuals_sample_qc, get_pipeline_runs
from seqr.utils.file_utils import file_iter
@@ -13,7 +13,7 @@ class Command(BaseCommand):
help = 'Ingest sample qc data for a particular pipeline run'
def add_arguments(self, parser):
- parser.add_argument('dataset_type', choices={Sample.DATASET_TYPE_VARIANT_CALLS})
+ parser.add_argument('dataset_type', choices={Dataset.DATASET_TYPE_VARIANT_CALLS})
parser.add_argument('genome_version', choices={GENOME_VERSION_LOOKUP[GENOME_VERSION_GRCh38]})
parser.add_argument('run_version')
diff --git a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py
index 86bf7e08af..7ae407db96 100644
--- a/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py
+++ b/seqr/management/tests/check_for_new_samples_from_pipeline_tests.py
@@ -6,27 +6,20 @@
import mock
import responses
-from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, AuthenticationTestCase, DifferentDbTransactionSupportMixin
-from seqr.models import Project, Family, Individual, Sample, SavedVariant
+from seqr.views.utils.test_utils import AnvilAuthenticationTestCase, AuthenticationTestCase
+from seqr.models import Project, Family, Individual, Dataset, SavedVariant
SEQR_URL = 'https://seqr.broadinstitute.org/'
PROJECT_GUID = 'R0003_test'
EXTERNAL_PROJECT_GUID = 'R0004_non_analyst_project'
-GUID_ID = 54321
-GCNV_GUID_ID = 12345
-NEW_SAMPLE_GUID_P3 = f'S00000{GUID_ID}_na20888'
-NEW_SAMPLE_GUID_P4 = f'S00000{GUID_ID}_na21234'
-REPLACED_SAMPLE_GUID = f'S00000{GUID_ID}_na20885'
-EXISTING_INACTIVE_SAMPLE_GUID = 'S000154_na20889'
-ACTIVE_SAMPLE_GUID = f'S00000{GUID_ID}_na20889'
-EXISTING_WGS_SAMPLE_GUID = 'S000144_na20888'
-EXISTING_SV_SAMPLE_GUID = 'S000147_na21234'
-SAMPLE_GUIDS = [ACTIVE_SAMPLE_GUID, REPLACED_SAMPLE_GUID, NEW_SAMPLE_GUID_P3, NEW_SAMPLE_GUID_P4]
-GCNV_SAMPLE_GUID = f'S00000{GCNV_GUID_ID}_na20889'
-EXISTING_GCNV_SAMPLE_GUIDS = ['S000145_hg00731', 'S000146_hg00732', 'S000148_hg00733']
-GCNV_SAMPLE_GUIDS = [f'S00000{GCNV_GUID_ID}_hg00731', f'S00000{GCNV_GUID_ID}_hg00732', f'S00000{GCNV_GUID_ID}_hg00733', GCNV_SAMPLE_GUID]
-OLD_DATA_SAMPLE_GUID = 'S000143_na20885'
+EXISTING_INACTIVE_DATASET_GUID = 'S000154_na20889'
+EXISTING_WGS_DATASET_GUID = 'S000144_na20888'
+EXISTING_SV_DATASET_GUID = 'S000147_na21234'
+EXISTING_GCNV_DATASET_GUID = 'S000145_hg00731'
+OLD_DATA_DATASET_GUID = 'S000143_na20885'
+NEW_DATASET_GUIDS = ['D0000155_snv_indel_wes_2025_09', 'D0000156_snv_indel_wes_2025_09']
+NEW_GCNV_DATASET_GUIDS = ['D0000157_sv_wes_2025_09_23_000', 'D0000158_sv_wes_2025_09_23_000']
namespace_path = 'ext-data/anvil-non-analyst-project 1000 Genomes Demo'
anvil_link = f'{namespace_path}'
@@ -384,14 +377,14 @@ def set_up(self):
self.mock_redis = patcher.start()
self.mock_redis.return_value.keys.side_effect = lambda pattern: [pattern]
self.addCleanup(patcher.stop)
- patcher = mock.patch('seqr.models.random.randint')
- mock_rand_int = patcher.start()
- mock_rand_int.side_effect = [GUID_ID, GUID_ID, GUID_ID, GUID_ID, GCNV_GUID_ID, GCNV_GUID_ID, GCNV_GUID_ID, GCNV_GUID_ID, GUID_ID, GUID_ID, GUID_ID, GUID_ID]
- self.addCleanup(patcher.stop)
patcher = mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.PIPELINE_DATA_DIR')
mock_data_dir = patcher.start()
mock_data_dir.__str__.return_value = self.MOCK_DATA_DIR
self.addCleanup(patcher.stop)
+ patcher = mock.patch('seqr.management.commands.check_for_new_samples_from_pipeline.timezone.now')
+ mock_now = patcher.start()
+ mock_now.return_value = datetime(2025, 9, 23)
+ self.addCleanup(patcher.stop)
patcher = mock.patch('seqr.views.utils.export_utils.TemporaryDirectory')
mock_temp_dir = patcher.start()
mock_temp_dir.return_value.__enter__.return_value = '/mock/tmp'
@@ -421,7 +414,7 @@ def set_up(self):
patcher.start()
self.addCleanup(patcher.stop)
- Sample.objects.filter(guid=OLD_DATA_SAMPLE_GUID).update(sample_type='WES')
+ Dataset.objects.filter(guid=OLD_DATA_DATASET_GUID).update(sample_type='WES')
def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=6):
self._set_loading_files()
@@ -463,16 +456,16 @@ def _test_call(self, error_logs=None, run_loading_logs=None, num_runs=6):
num_calls = self._assert_expected_airtable_calls(bool(run_loading_logs), single_call)
self.assertEqual(len(responses.calls), num_calls)
- def _test_success_call(self, anvil_email_calls):
+ def _test_success_call(self, anvil_email_calls, next_dataset_id=155):
Project.objects.filter(id__in=[1, 3]).update(genome_version=38)
+ self.maxDiff = None
self._test_call(run_loading_logs={
'GRCh38/SNV_INDEL': [
('Loading 4 WES SNV_INDEL samples in 2 projects', None),
('update 2 Familys', {'dbUpdate': mock.ANY}),
- ('create 4 Samples', {'dbUpdate': mock.ANY}),
- ('update 4 Samples', {'dbUpdate': mock.ANY}),
- ('update 1 Samples', {'dbUpdate': mock.ANY}),
+ (f'create Dataset D0000{next_dataset_id}_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}),
+ (f'create Dataset D0000{next_dataset_id+1}_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}),
] + self.AIRTABLE_LOGS + [
('update 3 Familys', {'dbUpdate': mock.ANY}),
] + self.UPDATE_SAMPLE_LOGS,
@@ -482,9 +475,8 @@ def _test_success_call(self, anvil_email_calls):
'GRCh38/SV': [
('Loading 4 WES SV samples in 2 projects', None),
('update 1 Familys', {'dbUpdate': mock.ANY}),
- ('create 4 Samples', {'dbUpdate': mock.ANY}),
- ('update 4 Samples', {'dbUpdate': mock.ANY}),
- ('update 3 Samples', {'dbUpdate': mock.ANY}),
+ (f'create Dataset D0000{next_dataset_id+2}_sv_wes_2025_09_23_000', {'dbUpdate': mock.ANY}),
+ (f'create Dataset D0000{next_dataset_id+3}_sv_wes_2025_09_23_000', {'dbUpdate': mock.ANY}),
('Reloading saved variants in 2 projects', None),
('Updated 0 variants in 1 families for project 1kg project nåme with uniçøde', None),
('Updated 0 variants in 1 families for project Test Reprocessed Project', None),
@@ -561,11 +553,15 @@ def _test_success_call(self, anvil_email_calls):
self.assertDictEqual(self.mock_email.return_value.merge_data, {})
self.assertEqual(self.manager_user.notifications.count(), 5)
- self.assertEqual(
- str(self.manager_user.notifications.first()), 'Test Reprocessed Project Loaded 1 new WES SV samples 0 minutes ago')
+ manager_notification = self.manager_user.notifications.order_by('-id').first()
+ self.assertEqual(manager_notification.actor.name, 'Test Reprocessed Project')
+ self.assertEqual(manager_notification.verb, 'Loaded 1 new WES SV samples')
+ self.assertEqual(manager_notification.timestamp.isoformat(), '2025-09-23T00:00:00+00:00')
self.assertEqual(self.collaborator_user.notifications.count(), 2)
- self.assertEqual(
- str(self.collaborator_user.notifications.first()), 'Non-Analyst Project Loaded 1 new WES samples 0 minutes ago')
+ collaborator_notification = self.collaborator_user.notifications.first()
+ self.assertEqual(collaborator_notification.actor.name, 'Non-Analyst Project')
+ self.assertEqual(collaborator_notification.verb, 'Loaded 1 new WES samples')
+ self.assertEqual(collaborator_notification.timestamp.isoformat(), '2025-09-23T00:00:00+00:00')
def _additional_loading_logs(self, data_type, version):
return []
@@ -596,7 +592,7 @@ def test_command(self):
'auto__2024-09-14': 'Data has genome version GRCh38 but the following projects have conflicting versions: R0001_1kg (GRCh37), R0003_test (GRCh37)',
}
self._test_call(error_logs=error_logs)
- self.assertEqual(Sample.objects.filter(guid__in=SAMPLE_GUIDS + GCNV_SAMPLE_GUIDS).count(), 0)
+ self.assertEqual(Dataset.objects.filter(guid__in=NEW_DATASET_GUIDS + NEW_GCNV_DATASET_GUIDS).count(), 0)
# Update fixture data to allow testing edge cases
svs = SavedVariant.objects.filter(guid__in=['SV0000002_1248367227_r0390_100', 'SV0000006_1248367227_r0003_tes', 'SV0000007_prefix_19107_DEL_r00'])
@@ -610,55 +606,71 @@ def test_command(self):
self._test_success_call(self._anvil_email_calls())
# Tests Sample models created/updated
- snv_indel_samples = Sample.objects.filter(data_source='auto__2023-08-09')
- gcnv_samples = Sample.objects.filter(data_source='auto__2024-09-14')
- updated_sample_models = snv_indel_samples | gcnv_samples
- self.assertSetEqual({'WES'}, set(updated_sample_models.values_list('sample_type', flat=True)))
- self.assertSetEqual({True}, set(updated_sample_models.values_list('is_active', flat=True)))
+ snv_indel_datasets = Dataset.objects.filter(data_source='auto__2023-08-09')
+ gcnv_datasets = Dataset.objects.filter(data_source='auto__2024-09-14')
+ new_dataset_models = snv_indel_datasets | gcnv_datasets
+ self.assertSetEqual({'WES'}, set(new_dataset_models.values_list('sample_type', flat=True)))
self.assertSetEqual(
- {datetime.now().strftime('%Y-%m-%d')},
- {date.strftime('%Y-%m-%d') for date in updated_sample_models.values_list('loaded_date', flat=True)}
+ {'2025-09-23'},
+ {date.strftime('%Y-%m-%d') for date in new_dataset_models.values_list('loaded_date', flat=True)}
)
- self.assertSetEqual(set(snv_indel_samples.values_list('guid', flat=True)), set(SAMPLE_GUIDS))
- self.assertSetEqual({'SNV_INDEL'}, set(snv_indel_samples.values_list('dataset_type', flat=True)))
- self.assertSetEqual({'1kg.vcf.gz;new_samples.vcf.gz'}, set(snv_indel_samples.values_list('elasticsearch_index', flat=True)))
+ self.assertSetEqual(set(snv_indel_datasets.values_list('guid', flat=True)), set(NEW_DATASET_GUIDS))
+ self.assertSetEqual({'SNV_INDEL'}, set(snv_indel_datasets.values_list('dataset_type', flat=True)))
- self.assertSetEqual(set(gcnv_samples.values_list('guid', flat=True)), set(GCNV_SAMPLE_GUIDS))
- self.assertSetEqual({'SV'}, set(gcnv_samples.values_list('dataset_type', flat=True)))
- self.assertSetEqual({'gcnv.bed.gz'}, set(gcnv_samples.values_list('elasticsearch_index', flat=True)))
+ self.assertSetEqual(set(gcnv_datasets.values_list('guid', flat=True)), set(NEW_GCNV_DATASET_GUIDS))
+ self.assertSetEqual({'SV'}, set(gcnv_datasets.values_list('dataset_type', flat=True)))
- self.assertFalse(Sample.objects.get(guid=OLD_DATA_SAMPLE_GUID).is_active)
+ old_dataset = Dataset.objects.get(guid=OLD_DATA_DATASET_GUID)
+ self.assertListEqual(list(old_dataset.active_individuals.values_list('id', flat=True)), [])
+ self.assertListEqual(list(old_dataset.inactive_individuals.values_list('id', flat=True)), [15])
- previous_gcnv_samples = Sample.objects.filter(guid__in=EXISTING_GCNV_SAMPLE_GUIDS)
- self.assertEqual(len(previous_gcnv_samples), len(EXISTING_GCNV_SAMPLE_GUIDS))
- self.assertFalse(any(previous_gcnv_samples.values_list('is_active', flat=True)))
+ old_gcnv_dataset = Dataset.objects.get(guid=EXISTING_GCNV_DATASET_GUID)
+ self.assertListEqual(list(old_gcnv_dataset.active_individuals.values_list('id', flat=True)), [])
+ self.assertListEqual(list(old_gcnv_dataset.inactive_individuals.values_list('id', flat=True)), [4, 5, 6])
# Previously loaded WGS data should be unchanged by loading WES data
self.assertEqual(
- Sample.objects.get(guid=EXISTING_WGS_SAMPLE_GUID).last_modified_date.strftime('%Y-%m-%d'), '2017-03-13')
+ Dataset.objects.get(guid=EXISTING_WGS_DATASET_GUID).last_modified_date.strftime('%Y-%m-%d'), '2017-03-13')
# Previously loaded SV data should be unchanged by loading SNV_INDEL data
- sv_sample = Sample.objects.get(guid=EXISTING_SV_SAMPLE_GUID)
- self.assertEqual(sv_sample.last_modified_date.strftime('%Y-%m-%d'), '2018-03-13')
- self.assertTrue(sv_sample.is_active)
+ sv_dataset= Dataset.objects.get(guid=EXISTING_SV_DATASET_GUID)
+ self.assertEqual(sv_dataset.last_modified_date.strftime('%Y-%m-%d'), '2018-03-13')
+ self.assertListEqual(list(sv_dataset.active_individuals.values_list('id', flat=True)), [18, 19, 21])
+ self.assertListEqual(list(sv_dataset.inactive_individuals.values_list('id', flat=True)), [])
# Test Individual models properly associated with Samples
self.assertSetEqual(
- set(Individual.objects.get(guid='I000015_na20885').sample_set.values_list('guid', flat=True)),
- {REPLACED_SAMPLE_GUID, OLD_DATA_SAMPLE_GUID}
+ set(Individual.objects.get(guid='I000015_na20885').active_datasets.values_list('guid', flat=True)),
+ {NEW_DATASET_GUIDS[0]}
+ )
+ self.assertSetEqual(
+ set(Individual.objects.get(guid='I000015_na20885').inactive_datasets.values_list('guid', flat=True)),
+ {OLD_DATA_DATASET_GUID}
+ )
+ self.assertSetEqual(
+ set(Individual.objects.get(guid='I000016_na20888').active_datasets.values_list('guid', flat=True)),
+ {NEW_DATASET_GUIDS[0]}
+ )
+ self.assertSetEqual(
+ set(Individual.objects.get(guid='I000016_na20888').inactive_datasets.values_list('guid', flat=True)),
+ {EXISTING_WGS_DATASET_GUID}
+ )
+ self.assertSetEqual(
+ set(Individual.objects.get(guid='I000017_na20889').active_datasets.values_list('guid', flat=True)),
+ {NEW_DATASET_GUIDS[0], NEW_GCNV_DATASET_GUIDS[1]}
)
self.assertSetEqual(
- set(Individual.objects.get(guid='I000016_na20888').sample_set.values_list('guid', flat=True)),
- {EXISTING_WGS_SAMPLE_GUID, NEW_SAMPLE_GUID_P3}
+ set(Individual.objects.get(guid='I000017_na20889').inactive_datasets.values_list('guid', flat=True)),
+ {EXISTING_INACTIVE_DATASET_GUID}
)
self.assertSetEqual(
- set(Individual.objects.get(guid='I000017_na20889').sample_set.values_list('guid', flat=True)),
- {EXISTING_INACTIVE_SAMPLE_GUID, ACTIVE_SAMPLE_GUID, GCNV_SAMPLE_GUID}
+ set(Individual.objects.get(guid='I000018_na21234').active_datasets.values_list('guid', flat=True)),
+ {NEW_DATASET_GUIDS[1], EXISTING_SV_DATASET_GUID}
)
self.assertSetEqual(
- set(Individual.objects.get(guid='I000018_na21234').sample_set.values_list('guid', flat=True)),
- {EXISTING_SV_SAMPLE_GUID, NEW_SAMPLE_GUID_P4}
+ set(Individual.objects.get(guid='I000018_na21234').inactive_datasets.values_list('guid', flat=True)),
+ set(),
)
# Test Individual model properly updated with sample qc results
@@ -707,26 +719,26 @@ def test_command(self):
self.mock_email.reset_mock()
self.mock_send_slack.reset_mock()
self.mock_redis.reset_mock()
- sample_last_modified = Sample.objects.filter(
+ dataset_last_modified = Dataset.objects.filter(
last_modified_date__isnull=False).values_list('last_modified_date', flat=True).order_by('-last_modified_date')[0]
call_command('check_for_new_samples_from_pipeline')
self.assert_json_logs(user=None, expected=self.LIST_FILE_LOGS[:1] + [('Data already loaded for all 2 runs', None)])
self.mock_email.assert_not_called()
self.mock_send_slack.assert_not_called()
- self.assertFalse(Sample.objects.filter(last_modified_date__gt=sample_last_modified).exists())
+ self.assertFalse(Dataset.objects.filter(last_modified_date__gt=dataset_last_modified).exists())
self.mock_redis.return_value.delete.assert_not_called()
# Test reloading shared annotations is skipped if too many saved variants
- snv_indel_samples.delete()
+ snv_indel_datasets.delete()
airtable_logs = self.AIRTABLE_LOGS[:-1]
if self.AIRTABLE_LOGS:
airtable_logs.append(('Fetched 1 AnVIL Seqr Loading Requests Tracking records from airtable', None))
self._test_call(num_runs=2, run_loading_logs={
'GRCh38/SNV_INDEL': [
('Loading 4 WES SNV_INDEL samples in 2 projects', None),
- ('create 4 Samples', {'dbUpdate': mock.ANY}),
- ('update 4 Samples', {'dbUpdate': mock.ANY}),
+ ('create Dataset D0000159_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}),
+ ('create Dataset D0000160_snv_indel_wes_2025_09', {'dbUpdate': mock.ANY}),
] + airtable_logs + self.UPDATE_SAMPLE_LOGS,
})
@@ -1030,4 +1042,4 @@ def test_loading_delay_command(self):
self._test_success_call(self._anvil_email_calls(
email_text=ANVIL_ERROR_TEXT_EMAIL_TEMPLATE.format(error='\n'+ANVIL_ERROR_DELAY),
email_html=ANVIL_ERROR_HTML_EMAIL_TEMPLATE.format(error='
'+ANVIL_ERROR_DELAY),
- ))
\ No newline at end of file
+ ), next_dataset_id=161)
\ No newline at end of file
diff --git a/seqr/management/tests/reload_saved_variant_genotypes_tests.py b/seqr/management/tests/reload_saved_variant_genotypes_tests.py
index 4f7f8cf3e9..50ec2d6003 100644
--- a/seqr/management/tests/reload_saved_variant_genotypes_tests.py
+++ b/seqr/management/tests/reload_saved_variant_genotypes_tests.py
@@ -1,7 +1,7 @@
from django.core.management import call_command
from seqr.views.utils.test_utils import AnvilAuthenticationTestCase
-from seqr.models import SavedVariant, Sample
+from seqr.models import SavedVariant, Dataset
class ReloadSavedVariantGenotypesTest(AnvilAuthenticationTestCase):
@@ -9,7 +9,8 @@ class ReloadSavedVariantGenotypesTest(AnvilAuthenticationTestCase):
def test_command(self):
# Update fixture data
- Sample.objects.filter(id__in=[143, 149]).update(individual_id=18, sample_id='NA21234')
+ for dataset in Dataset.objects.filter(id__in=[143, 149]):
+ dataset.active_individuals.set([18])
call_command('reload_saved_variant_genotypes', 'R0004_non_analyst_project')
self.assert_json_logs(user=None, expected=[
diff --git a/seqr/management/tests/transfer_families_to_different_project_tests.py b/seqr/management/tests/transfer_families_to_different_project_tests.py
index b2feeecd53..d346420ad0 100644
--- a/seqr/management/tests/transfer_families_to_different_project_tests.py
+++ b/seqr/management/tests/transfer_families_to_different_project_tests.py
@@ -1,7 +1,7 @@
from django.core.management import call_command
import responses
-from seqr.models import Family, VariantTagType, VariantTag, Sample
+from seqr.models import Family, VariantTagType, VariantTag, Dataset
from seqr.views.utils.test_utils import AnvilAuthenticationTestCase
@@ -26,6 +26,7 @@ def test_command(self):
('Found 3 out of 4 families. No match for: 12.', None),
('Skipping 1 families with analysis groups in the project: 5 (Test Group 1)', None),
*self.LOGS,
+ ('Splitting 2 datasets', None),
('Updating "Excluded" tags', None),
('Updating families', None),
('Done.', None),
@@ -45,9 +46,21 @@ def test_command(self):
self.assertEqual(len(new_tags), 1)
self.assertEqual(new_tags[0].saved_variants.first().family, family)
- samples = Sample.objects.filter(individual__family=family)
- self.assertEqual(samples.count(), 7)
- self.assertEqual(samples.filter(is_active=True).count(), 0)
+ existing_dataset = Dataset.objects.get(guid='S000129_na19675')
+ self.assertListEqual(list(existing_dataset.active_individuals.order_by('id').values_list('id', flat=True)),[1, 7, 9])
+ self.assertListEqual(list(existing_dataset.inactive_individuals.order_by('id').values_list('id', flat=True)), [3])
+ self.assertEqual(Dataset.objects.filter(active_individuals__family=family).count(), 0)
+ datasets = {d.guid: d for d in Dataset.objects.filter(inactive_individuals__family=family).distinct()}
+ self.assertEqual(len(datasets), 3)
+ previous_guids = {'S000145_hg00731', 'S000149_hg00733'}
+ self.assertTrue(previous_guids.issubset(set(datasets.keys())))
+ split_dataset = next(d for guid, d in datasets.items() if guid not in previous_guids)
+ self.assertListEqual(list(split_dataset.inactive_individuals.order_by('id').values_list('id', flat=True)), [4, 5, 6])
+ self.assertEqual(split_dataset.active_individuals.count(), 0)
+ self.assertEqual(split_dataset.sample_type, 'WES')
+ self.assertEqual(split_dataset.dataset_type, 'SNV_INDEL')
+ self.assertEqual(split_dataset.data_source, 'test_index')
+ self.assertEqual(split_dataset.loaded_date.isoformat(), '2017-02-05T06:12:55.397000+00:00')
family = Family.objects.get(family_id='4')
self.assertEqual(family.project.guid, 'R0003_test')
diff --git a/seqr/migrations/0087_dataset.py b/seqr/migrations/0087_dataset.py
new file mode 100644
index 0000000000..1ae541adf6
--- /dev/null
+++ b/seqr/migrations/0087_dataset.py
@@ -0,0 +1,117 @@
+# Generated by Django 4.2.24 on 2026-04-06 16:39
+
+from django.conf import settings
+from django.contrib.postgres.aggregates import ArrayAgg
+from django.db import migrations, models
+from django.db.models.functions import Trim, TruncDate
+import django.db.models.deletion
+import django.utils.timezone
+
+
+def set_individual_id(apps, schema_editor):
+ Sample = apps.get_model('seqr', 'Sample')
+ Individual = apps.get_model('seqr', 'Individual')
+
+ db_alias = schema_editor.connection.alias
+
+ mismatch_samples = Sample.objects.using(db_alias).filter(is_active=True).exclude(sample_id=models.F('individual__individual_id'))
+ individual_ids = []
+ for sample in mismatch_samples:
+ individual = sample.individual
+ individual.display_name = individual.individual_id
+ # Since some of these are sample swamps, set a padded id to avoid conflicts and then trim whitespace for clean up
+ individual.individual_id = f' {sample.sample_id} '
+ individual.save()
+ individual_ids.append(individual.id)
+
+ updated = Individual.objects.using(db_alias).filter(id__in=individual_ids).update(individual_id=Trim('individual_id'))
+ if updated:
+ print(f'Updated individual_id for {updated} individuals to match their active sample_id')
+
+
+def populate_datasets(apps, schema_editor):
+ Dataset = apps.get_model('seqr', 'Dataset')
+ Sample = apps.get_model('seqr', 'Sample')
+ db_alias = schema_editor.connection.alias
+
+ dataset_aggs = Sample.objects.using(db_alias).values(
+ 'sample_type', 'dataset_type', project=models.F('individual__family__project'), group_loaded_date=TruncDate('loaded_date'),
+ ).annotate(
+ guid=models.Max('guid'),
+ loaded_date=models.Max('loaded_date'),
+ created_date=models.Max('created_date'),
+ last_modified_date=models.Max('last_modified_date'),
+ created_by_id=models.Min('created_by_id'),
+ data_sources=ArrayAgg('data_source', distinct=True, filter=models.Q(data_source__isnull=False)),
+ es_indices=ArrayAgg('elasticsearch_index', distinct=True, filter=models.Q(elasticsearch_index__isnull=False)),
+ active_individuals=ArrayAgg('individual__id', distinct=True, filter=models.Q(is_active=True)),
+ inactive_individuals=ArrayAgg('individual__id', distinct=True, filter=models.Q(is_active=False)),
+ )
+
+ datasets = []
+ active_individuals = []
+ inactive_individuals = []
+ for dataset_agg in dataset_aggs:
+ del dataset_agg['project']
+ del dataset_agg['group_loaded_date']
+ active_individuals.append(dataset_agg.pop('active_individuals'))
+ inactive_individuals.append(dataset_agg.pop('inactive_individuals'))
+ data_sources = dataset_agg.pop('data_sources')
+ es_indices = dataset_agg.pop('es_indices')
+ datasets.append(Dataset(**dataset_agg, data_source=','.join(data_sources or es_indices)))
+
+ created = Dataset.objects.using(db_alias).bulk_create(datasets)
+ if created:
+ print(f'Created {len(created)} datasets.')
+
+ ActiveRelation = Dataset.active_individuals.through
+ InactiveRelation = Dataset.inactive_individuals.through
+
+ active_relations = []
+ for dataset, individual_ids in zip(created, active_individuals):
+ active_relations.extend(ActiveRelation(dataset_id=dataset.id, individual_id=ind_id) for ind_id in individual_ids)
+ ActiveRelation.objects.using(db_alias).bulk_create(active_relations)
+ if active_relations:
+ print(f'Created {len(active_relations)} active samples relations')
+
+ inactive_relations = []
+ for dataset, individual_ids in zip(created, inactive_individuals):
+ inactive_relations.extend(InactiveRelation(dataset_id=dataset.id, individual_id=ind_id) for ind_id in individual_ids)
+ InactiveRelation.objects.using(db_alias).bulk_create(inactive_relations)
+ if inactive_relations:
+ print(f'Created {len(inactive_relations)} inactive samples relations')
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ('seqr', '0086_alter_rnasample_tissue_type'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='Dataset',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('guid', models.CharField(db_index=True, max_length=30, unique=True)),
+ ('created_date', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
+ ('last_modified_date', models.DateTimeField(blank=True, db_index=True, null=True)),
+ ('sample_type', models.CharField(choices=[('WES', 'Exome'), ('WGS', 'Whole Genome')], max_length=10)),
+ ('dataset_type', models.CharField(choices=[('SNV_INDEL', 'Variant Calls'), ('SV', 'SV Calls'), ('MITO', 'Mitochondria calls')], max_length=13)),
+ ('data_source', models.TextField()),
+ ('loaded_date', models.DateTimeField()),
+ ('active_individuals', models.ManyToManyField(related_name='active_datasets', to='seqr.individual')),
+ ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)),
+ ('inactive_individuals', models.ManyToManyField(related_name='inactive_datasets', to='seqr.individual')),
+ ],
+ options={
+ 'json_fields': ['guid', 'sample_type', 'dataset_type', 'loaded_date'],
+ },
+ ),
+ migrations.RunPython(set_individual_id),
+ migrations.RunPython(populate_datasets),
+ migrations.DeleteModel(
+ name='Sample',
+ ),
+ ]
diff --git a/seqr/models.py b/seqr/models.py
index 4b625edc7a..7942ec7bf4 100644
--- a/seqr/models.py
+++ b/seqr/models.py
@@ -616,7 +616,6 @@ class Individual(ModelWithGUID):
sex = models.CharField(max_length=3, choices=SEX_CHOICES, default='U')
affected = models.CharField(max_length=1, choices=AFFECTED_STATUS_CHOICES, default=AFFECTED_STATUS_UNKNOWN)
- # TODO once sample and individual ids are fully decoupled no reason to maintain this field
display_name = models.TextField(default="", blank=True)
notes = models.TextField(blank=True, null=True)
@@ -695,14 +694,7 @@ class Meta:
audit_fields = {'case_review_status'}
-class Sample(ModelWithGUID):
- """This model represents a single data type (eg. Variant Calls, or SV Calls) that's generated from a single
- biological sample (eg. WES, WGS).
-
- It stores metadata on both the dataset (fields: dataset_type, loaded_date, etc.) and the underlying sample
- (fields: sample_type, sample_id etc.)
- """
-
+class Dataset(ModelWithGUID):
SAMPLE_TYPE_WES = 'WES'
SAMPLE_TYPE_WGS = 'WGS'
SAMPLE_TYPE_CHOICES = (
@@ -721,31 +713,22 @@ class Sample(ModelWithGUID):
)
DATASET_TYPE_LOOKUP = dict(DATASET_TYPE_CHOICES)
- individual = models.ForeignKey('Individual', on_delete=models.PROTECT)
+ active_individuals = models.ManyToManyField('Individual', related_name='active_datasets')
+ inactive_individuals = models.ManyToManyField('Individual', related_name='inactive_datasets')
sample_type = models.CharField(max_length=10, choices=SAMPLE_TYPE_CHOICES)
dataset_type = models.CharField(max_length=13, choices=DATASET_TYPE_CHOICES)
- # The sample's id in the underlying dataset (eg. the VCF Id for variant callsets).
- sample_id = models.TextField(db_index=True)
-
- elasticsearch_index = models.TextField(db_index=True, null=True)
- data_source = models.TextField(null=True)
-
- # sample status
- is_active = models.BooleanField(default=False)
+ data_source = models.TextField()
loaded_date = models.DateTimeField()
def __unicode__(self):
- return self.sample_id.strip()
+ return f'{self.dataset_type}_{self.sample_type}_{self.loaded_date}'
- GUID_PREFIX = 'S'
- GUID_PRECISION = 10
+ GUID_PREFIX = 'D'
class Meta:
- json_fields = [
- 'guid', 'created_date', 'sample_type', 'dataset_type', 'sample_id', 'is_active', 'loaded_date',
- ]
+ json_fields = ['guid', 'sample_type', 'dataset_type', 'loaded_date']
class RnaSample(ModelWithGUID):
@@ -828,10 +811,10 @@ class Meta:
class SavedVariant(ModelWithGUID):
DATASET_TYPE_CHOICES = (
- (Sample.DATASET_TYPE_VARIANT_CALLS, 'Variant Calls'),
- (Sample.DATASET_TYPE_MITO_CALLS, 'Mitochondria calls'),
- (f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WGS}', 'SV WGS Calls'),
- (f'{Sample.DATASET_TYPE_SV_CALLS}_{Sample.SAMPLE_TYPE_WES}', 'gCNV Calls'),
+ (Dataset.DATASET_TYPE_VARIANT_CALLS, 'Variant Calls'),
+ (Dataset.DATASET_TYPE_MITO_CALLS, 'Mitochondria calls'),
+ (f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WGS}', 'SV WGS Calls'),
+ (f'{Dataset.DATASET_TYPE_SV_CALLS}_{Dataset.SAMPLE_TYPE_WES}', 'gCNV Calls'),
)
family = models.ForeignKey('Family', on_delete=models.CASCADE)
diff --git a/seqr/urls.py b/seqr/urls.py
index ab0b4b461f..4b2e11a7d9 100644
--- a/seqr/urls.py
+++ b/seqr/urls.py
@@ -144,7 +144,7 @@
from seqr.views.apis.project_api import create_project_handler, update_project_handler, delete_project_handler, \
project_page_data, project_families, project_overview, project_mme_submisssions, project_individuals, \
project_analysis_groups, update_project_workspace, project_family_notes, project_collaborators, project_locus_lists, \
- project_samples, project_notifications, mark_read_project_notifications, subscribe_project_notifications, \
+ project_notifications, mark_read_project_notifications, subscribe_project_notifications, \
update_project_rna_seq, load_rna_seq_sample_data
from seqr.views.apis.project_categories_api import update_project_categories_handler
from seqr.views.apis.anvil_workspace_api import anvil_workspace_page, create_project_from_workspace, \
@@ -212,7 +212,6 @@
'project/(?P[^/]+)/details': project_page_data,
'project/(?P[^/]+)/get_families': project_families,
'project/(?P[^/]+)/get_individuals': project_individuals,
- 'project/(?P[^/]+)/get_samples': project_samples,
'project/(?P[^/]+)/get_family_notes': project_family_notes,
'project/(?P[^/]+)/get_mme_submissions': project_mme_submisssions,
'project/(?P[^/]+)/get_analysis_groups': project_analysis_groups,
diff --git a/seqr/utils/add_data_utils.py b/seqr/utils/add_data_utils.py
index a7a5c1a2c9..2feefa04fa 100644
--- a/seqr/utils/add_data_utils.py
+++ b/seqr/utils/add_data_utils.py
@@ -5,7 +5,7 @@
import requests
from reference_data.models import GeneInfo, GENOME_VERSION_LOOKUP
-from seqr.models import Sample, Individual, Project
+from seqr.models import Dataset, Individual, Project
from seqr.utils.communication_utils import send_project_notification, safe_post_to_slack
from seqr.utils.file_utils import does_file_exist
from seqr.utils.logging_utils import SeqrLogger
@@ -21,7 +21,7 @@
def basic_notify_search_data_loaded(project, dataset_type, sample_type, new_samples, email_template=None, is_internal=True):
- msg_dataset_type = '' if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS else f' {dataset_type}'
+ msg_dataset_type = '' if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS else f' {dataset_type}'
num_new_samples = len(new_samples)
sample_summary = f'{num_new_samples} new {sample_type}{msg_dataset_type} samples'
@@ -64,12 +64,17 @@ def update_airtable_loading_tracking_status(project, status, additional_update=N
)
def trigger_delete_families_search(project, family_guids, user=None):
- search_samples = Sample.objects.filter(is_active=True, individual__family__guid__in=family_guids)
+ num_updated = 0
+ updated_families = set()
+ for dataset in Dataset.objects.filter(active_individuals__family__guid__in=family_guids).distinct():
+ active_individuals = dataset.active_individuals.filter(family__guid__in=family_guids)
+ num_updated += len(active_individuals)
+ updated_families.update(active_individuals.values_list('family__family_id', flat=True).distinct())
+ dataset.inactive_individuals.add(*active_individuals)
+ dataset.active_individuals.remove(*active_individuals)
info = []
- if search_samples:
- updated_families = search_samples.values_list("individual__family__family_id", flat=True).distinct()
+ if num_updated:
family_summary = ", ".join(sorted(updated_families))
- num_updated = search_samples.update(is_active=False)
message = f'Disabled search for {num_updated} samples in the following {len(updated_families)} families: {family_summary}'
info.append(message)
logger.info(message, user)
@@ -142,7 +147,7 @@ def _enqueue_pipeline_request(name: str, variables: dict, user: User, raise_erro
def _loading_dataset_type(sample_type: str, dataset_type: str):
- return 'GCNV' if dataset_type == Sample.DATASET_TYPE_SV_CALLS and sample_type == Sample.SAMPLE_TYPE_WES \
+ return 'GCNV' if dataset_type == Dataset.DATASET_TYPE_SV_CALLS and sample_type == Dataset.SAMPLE_TYPE_WES \
else dataset_type
diff --git a/seqr/utils/vcf_utils.py b/seqr/utils/vcf_utils.py
index 892985a490..312df0064f 100644
--- a/seqr/utils/vcf_utils.py
+++ b/seqr/utils/vcf_utils.py
@@ -5,7 +5,7 @@
from seqr.utils.middleware import ErrorsWarningsException
from seqr.utils.file_utils import file_iter, does_file_exist, list_files
-from seqr.models import Sample
+from seqr.models import Dataset
BLOCK_SIZE = 65536
@@ -20,12 +20,12 @@
}
DATA_TYPE_FORMAT_FIELDS = {
- Sample.DATASET_TYPE_SV_CALLS: BASE_EXPECTED_FORMAT_FIELDS,
+ Dataset.DATASET_TYPE_SV_CALLS: BASE_EXPECTED_FORMAT_FIELDS,
}
DATA_TYPE_FILE_EXTS = {
- Sample.DATASET_TYPE_MITO_CALLS: ('.mt',),
- Sample.DATASET_TYPE_SV_CALLS: ('.bed', '.bed.gz'),
+ Dataset.DATASET_TYPE_MITO_CALLS: ('.mt',),
+ Dataset.DATASET_TYPE_SV_CALLS: ('.bed', '.bed.gz'),
}
REQUIRED_HEADERS = ['#CHROM', 'POS', 'ID', 'REF', 'ALT', 'QUAL', 'FILTER', 'INFO', 'FORMAT']
diff --git a/seqr/views/apis/anvil_workspace_api.py b/seqr/views/apis/anvil_workspace_api.py
index 24c2152510..4b08c4e2af 100644
--- a/seqr/views/apis/anvil_workspace_api.py
+++ b/seqr/views/apis/anvil_workspace_api.py
@@ -10,7 +10,7 @@
from django.shortcuts import redirect
from reference_data.models import GENOME_VERSION_LOOKUP
-from seqr.models import Project, Family, CAN_EDIT, Sample, IgvSample
+from seqr.models import Project, Family, CAN_EDIT, Dataset, IgvSample
from seqr.views.react_app import render_app_html
from seqr.views.utils.airtable_utils import AirtableSession, ANVIL_REQUEST_TRACKING_TABLE
from seqr.views.utils.json_to_orm_utils import create_model_from_json
@@ -222,7 +222,7 @@ def add_workspace_data(request, project_guid):
error = 'Field(s) "{}" are required'.format(', '.join(missing_fields))
return create_json_response({'error': error}, status=400, reason=error)
- pedigree_records, loaded_individual_ids, sample_type = _parse_uploaded_pedigree(request_json, project=project, search_dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS)
+ pedigree_records, loaded_individual_ids, sample_type = _parse_uploaded_pedigree(request_json, project=project, search_dataset_type=Dataset.DATASET_TYPE_VARIANT_CALLS)
loading_families = {record[JsonConstants.FAMILY_ID_COLUMN] for record in pedigree_records}
pending_families = Family.objects.filter(
@@ -302,7 +302,7 @@ def _trigger_add_workspace_data(project, pedigree_records, user, data_path, samp
f"{data_path} to seqr project <{_get_seqr_project_url(project)}|*{project.name}*> (guid: {project.guid})"
)
trigger_success = trigger_data_loading(
- [project], individual_ids, sample_type, Sample.DATASET_TYPE_VARIANT_CALLS, project.genome_version, data_path, user=user, success_message=success_message,
+ [project], individual_ids, sample_type, Dataset.DATASET_TYPE_VARIANT_CALLS, project.genome_version, data_path, user=user, success_message=success_message,
success_slack_channel=SEQR_SLACK_ANVIL_DATA_LOADING_CHANNEL, error_message=f'ERROR triggering AnVIL loading for project {project.guid}',
)
AirtableSession(user, base=AirtableSession.ANVIL_BASE).safe_create_records(
diff --git a/seqr/views/apis/dashboard_api.py b/seqr/views/apis/dashboard_api.py
index 1c20f6216b..d11a4d9bc1 100644
--- a/seqr/views/apis/dashboard_api.py
+++ b/seqr/views/apis/dashboard_api.py
@@ -2,8 +2,9 @@
APIs used by the main seqr dashboard page
"""
from django.db import models
+from django.db.models.functions import Coalesce
-from seqr.models import ProjectCategory, Sample, RnaSample, Family, Project
+from seqr.models import ProjectCategory, Individual, RnaSample, Family, Project
from seqr.views.utils.individual_utils import check_project_individuals_deletable
from seqr.views.utils.json_utils import create_json_response
from seqr.views.utils.orm_to_json_utils import get_json_for_projects
@@ -59,13 +60,19 @@ def _get_projects_json(user):
projects_by_guid[project_guid]['analysisStatusCounts'] = {}
projects_by_guid[project_guid]['analysisStatusCounts'][agg['analysis_status']] = agg['count']
- sample_type_status_counts = _sample_type_counts(
- Sample.objects.filter(individual__family__project__in=projects, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS)
- ) + _sample_type_counts(
- RnaSample.objects.filter(individual__family__project__in=projects).annotate(sample_type=models.Value('RNA'))
+ sample_type_status_counts = list(
+ Individual.objects.filter(family__project__in=projects).annotate(sample_type=Coalesce(
+ models.F('active_datasets__sample_type'), models.F('inactive_datasets__sample_type'),
+ )).filter(sample_type__isnull=False).values('sample_type', project_guid=models.F('family__project__guid')).annotate(
+ count=models.Count('id', distinct=True)
+ )
+ ) + list(
+ RnaSample.objects.filter(individual__family__project__in=projects).values(
+ project_guid=models.F('individual__family__project__guid')
+ ).annotate(sample_type=models.Value('RNA'), count=models.Count('individual_id', distinct=True))
)
for agg in sample_type_status_counts:
- project_guid = agg['individual__family__project__guid']
+ project_guid = agg['project_guid']
if 'sampleTypeCounts' not in projects_by_guid[project_guid]:
projects_by_guid[project_guid]['sampleTypeCounts'] = {}
projects_by_guid[project_guid]['sampleTypeCounts'][agg['sample_type']] = agg['count']
@@ -73,11 +80,6 @@ def _get_projects_json(user):
return projects_by_guid
-def _sample_type_counts(sample_q):
- return list(sample_q.values(
- 'individual__family__project__guid', 'sample_type',
- ).annotate(count=models.Count('individual_id', distinct=True)))
-
def _retrieve_project_categories_by_guid(project_guids):
"""Retrieves project categories from the database, and returns a 'project_categories_by_guid' dictionary,
while also adding a 'projectCategoryGuids' attribute to each project dict in 'projects_by_guid'.
diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py
index 422a1b2aec..4e1b386b21 100644
--- a/seqr/views/apis/data_manager_api.py
+++ b/seqr/views/apis/data_manager_api.py
@@ -23,7 +23,7 @@
from seqr.views.utils.permissions_utils import data_manager_required, pm_or_data_manager_required, get_internal_projects
from seqr.views.utils.terra_api_utils import anvil_enabled
-from seqr.models import Sample, RnaSample, Individual, Project, PhenotypePrioritization
+from seqr.models import Dataset, RnaSample, Individual, Project, PhenotypePrioritization
from settings import LOADING_DATASETS_DIR, LUIGI_UI_SERVICE_HOSTNAME, LUIGI_UI_SERVICE_PORT
@@ -219,17 +219,17 @@ def get_loaded_projects(request, genome_version, sample_type, dataset_type):
except ValueError as e:
return create_json_response({'error': str(e)}, status=400)
projects = projects.filter(guid__in=project_samples.keys())
- if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS:
- exclude_sample_type = Sample.SAMPLE_TYPE_WES if sample_type == Sample.SAMPLE_TYPE_WGS else Sample.SAMPLE_TYPE_WGS
+ if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS:
+ exclude_sample_type = Dataset.SAMPLE_TYPE_WES if sample_type == Dataset.SAMPLE_TYPE_WGS else Dataset.SAMPLE_TYPE_WGS
# Include projects with either the matched sample type OR with no loaded data
- projects = projects.exclude(family__individual__sample__sample_type=exclude_sample_type)
+ projects = projects.exclude(family__individual__active_datasets__sample_type=exclude_sample_type)
else:
# All other data types can only be loaded to projects which already have loaded data
- projects = projects.filter(family__individual__sample__sample_type=sample_type)
+ projects = projects.filter(family__individual__active_datasets__sample_type=sample_type)
projects = projects.distinct().order_by('name').values('name', projectGuid=F('guid'), dataTypeLastLoaded=Max(
- 'family__individual__sample__loaded_date',
- filter=Q(family__individual__sample__dataset_type=dataset_type) & Q(family__individual__sample__sample_type=sample_type),
+ 'family__individual__active_datasets__loaded_date',
+ filter=Q(family__individual__active_datasets__dataset_type=dataset_type) & Q(family__individual__active_datasets__sample_type=sample_type),
))
if project_samples:
@@ -240,10 +240,10 @@ def get_loaded_projects(request, genome_version, sample_type, dataset_type):
AIRTABLE_CALLSET_FIELDS = {
- (Sample.DATASET_TYPE_MITO_CALLS, Sample.SAMPLE_TYPE_WES): 'MITO_WES_CallsetPath',
- (Sample.DATASET_TYPE_MITO_CALLS, Sample.SAMPLE_TYPE_WGS): 'MITO_WGS_CallsetPath',
- (Sample.DATASET_TYPE_SV_CALLS, Sample.SAMPLE_TYPE_WES): 'gCNV_CallsetPath',
- (Sample.DATASET_TYPE_SV_CALLS, Sample.SAMPLE_TYPE_WGS): 'SV_CallsetPath',
+ (Dataset.DATASET_TYPE_MITO_CALLS, Dataset.SAMPLE_TYPE_WES): 'MITO_WES_CallsetPath',
+ (Dataset.DATASET_TYPE_MITO_CALLS, Dataset.SAMPLE_TYPE_WGS): 'MITO_WGS_CallsetPath',
+ (Dataset.DATASET_TYPE_SV_CALLS, Dataset.SAMPLE_TYPE_WES): 'gCNV_CallsetPath',
+ (Dataset.DATASET_TYPE_SV_CALLS, Dataset.SAMPLE_TYPE_WGS): 'SV_CallsetPath',
}
@@ -271,7 +271,7 @@ def load_data(request):
request_json = json.loads(request.body)
vcf_samples = request_json['vcfSamples']
sample_type = request_json['sampleType']
- dataset_type = request_json.get('datasetType', Sample.DATASET_TYPE_VARIANT_CALLS)
+ dataset_type = request_json.get('datasetType', Dataset.DATASET_TYPE_VARIANT_CALLS)
projects = [json.loads(project) for project in request_json['projects']]
project_samples = {p['projectGuid']: p.get('sampleIds') for p in projects}
@@ -400,12 +400,17 @@ def trigger_delete_project(request):
project_guid = request_json.pop('project')
dataset_type = request_json.get('datasetType')
project = Project.objects.get(guid=project_guid)
- samples = Sample.objects.filter(individual__family__project=project, dataset_type=dataset_type, is_active=True)
+ datasets = Dataset.objects.filter(active_individuals__family__project=project, dataset_type=dataset_type)
sample_types = list(
- samples.values_list('sample_type', flat=True).distinct()
- ) if dataset_type == Sample.DATASET_TYPE_SV_CALLS else [None]
- updated = Sample.bulk_update(user=request.user, update_json={'is_active': False}, queryset=samples)
- info = [f'Deactivated search for {len(updated)} individuals']
+ datasets.values_list('sample_type', flat=True).distinct()
+ ) if dataset_type == Dataset.DATASET_TYPE_SV_CALLS else [None]
+ updated = 0
+ for dataset in datasets:
+ active_individuals = dataset.active_individuals.all()
+ updated += len(active_individuals)
+ dataset.inactive_individuals.add(*active_individuals)
+ dataset.active_individuals.clear()
+ info = [f'Deactivated search for {updated} individuals']
for sample_type in sample_types:
info.append(delete_clickhouse_project(project, dataset_type=dataset_type, sample_type=sample_type))
return create_json_response({'info': info})
@@ -416,7 +421,6 @@ def trigger_delete_family(request):
request_json = json.loads(request.body)
family_guid = request_json.pop('family')
project = Project.objects.get(family__guid=family_guid)
- samples = Sample.objects.filter(individual__family__guid=family_guid)
info = trigger_delete_families_search(project, [family_guid], request.user)
return create_json_response({'info': info})
diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py
index 4c5295c56d..00d6500c25 100644
--- a/seqr/views/apis/data_manager_api_tests.py
+++ b/seqr/views/apis/data_manager_api_tests.py
@@ -11,7 +11,7 @@
get_loaded_projects, load_data, trigger_delete_family
from seqr.views.utils.orm_to_json_utils import _get_json_for_models
from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, AirtableTest
-from seqr.models import Individual, Sample, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier, RnaSample, Project, PhenotypePrioritization
+from seqr.models import Dataset, Individual, RnaSeqOutlier, RnaSeqTpm, RnaSeqSpliceOutlier, RnaSample, Project, PhenotypePrioritization
from settings import SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL
PROJECT_GUID = 'R0001_1kg'
@@ -1279,8 +1279,7 @@ def test_trigger_delete_family(self):
],
})
- family_samples = Sample.objects.filter(individual__family_id=2, is_active=True)
- self.assertEqual(family_samples.count(), 0)
+ self.assertEqual(Dataset.objects.filter(active_individuals__family_id=2).count(), 0)
self.assertEqual(len(responses.calls), 1)
self.assertDictEqual(json.loads(responses.calls[-1].request.body), {
@@ -1299,7 +1298,7 @@ class LocalDataManagerAPITest(AuthenticationTestCase, DataManagerAPITest):
PROJECT_OPTION = PROJECT_OPTION
WGS_PROJECT_OPTIONS = [EMPTY_PROJECT_OPTION]
WES_PROJECT_OPTIONS = [
- {'name': '1kg project nåme with uniçøde', 'projectGuid': 'R0001_1kg', 'dataTypeLastLoaded': '2017-02-05T06:25:55.397Z'},
+ {'name': '1kg project nåme with uniçøde', 'projectGuid': 'R0001_1kg', 'dataTypeLastLoaded': '2017-02-05T06:13:55.397Z'},
EMPTY_PROJECT_OPTION,
]
PROJECT_OPTIONS = [{'projectGuid': 'R0001_1kg'}, PROJECT_OPTION]
diff --git a/seqr/views/apis/family_api.py b/seqr/views/apis/family_api.py
index d3254d5203..0b934912cf 100644
--- a/seqr/views/apis/family_api.py
+++ b/seqr/views/apis/family_api.py
@@ -17,11 +17,11 @@
from seqr.views.utils.json_to_orm_utils import update_family_from_json, update_model_from_json, create_model_from_json
from seqr.views.utils.json_utils import create_json_response
from seqr.views.utils.note_utils import create_note_handler, update_note_handler, delete_note_handler
-from seqr.views.utils.orm_to_json_utils import _get_json_for_model, get_json_for_family_note, get_json_for_samples, \
+from seqr.views.utils.orm_to_json_utils import _get_json_for_model, get_json_for_family_note, get_json_for_datasets, \
get_json_for_matchmaker_submissions, get_json_for_analysis_groups, _get_json_for_families, get_json_for_queryset
from seqr.views.utils.project_context_utils import add_families_context, families_discovery_tags, add_project_tag_types, \
MME_TAG_NAME
-from seqr.models import Family, FamilyAnalysedBy, Individual, FamilyNote, Sample, VariantTag, AnalysisGroup, RnaSeqTpm, \
+from seqr.models import Family, FamilyAnalysedBy, Individual, FamilyNote, Dataset, VariantTag, AnalysisGroup, RnaSeqTpm, \
PhenotypePrioritization, Project, RnaSample
from seqr.views.utils.permissions_utils import check_project_permissions, get_project_and_check_pm_permissions, \
login_and_policies_required, user_is_analyst, has_case_review_permissions, external_anvil_project_can_edit, \
@@ -45,12 +45,12 @@ def family_page_data(request, family_guid):
is_analyst = user_is_analyst(request.user)
has_case_review_perm = has_case_review_permissions(project, request.user)
- sample_models = Sample.objects.filter(individual__family=family)
- samples = get_json_for_samples(
- sample_models, project_guid=project.guid, family_guid=family_guid, is_analyst=is_analyst
- )
+ dataset_models = Dataset.objects.filter(
+ Q(active_individuals__family=family) | Q(inactive_individuals__family=family)
+ ).distinct()
+ datasets = get_json_for_datasets(dataset_models, project.guid)
response = {
- 'samplesByGuid': {s['sampleGuid']: s for s in samples}
+ 'datasetsByGuid': {d['datasetGuid']: d for d in datasets}
}
add_families_context(response, families, project.guid, request.user, is_analyst, has_case_review_perm)
@@ -63,7 +63,7 @@ def family_page_data(request, family_guid):
gene_ids = {gene_id for variant in discovery_variants for gene_id in variant['gene_ids']}
discovery_variant_intervals = [dict(zip(
['chrom', 'start', 'end_chrom', 'end', 'svType', 'hasSvType'],
- [*get_chrom_pos(v['xpos']), *get_chrom_pos(v['xpos_end']), v['svType'], (v.get('dataset_type') or '').startswith(Sample.DATASET_TYPE_SV_CALLS)]
+ [*get_chrom_pos(v['xpos']), *get_chrom_pos(v['xpos_end']), v['svType'], (v.get('dataset_type') or '').startswith(Dataset.DATASET_TYPE_SV_CALLS)]
)) for v in discovery_variants]
omims = Omim.objects.filter(
get_omim_intervals_query(discovery_variant_intervals) | Q(gene__gene_id__in=gene_ids)
diff --git a/seqr/views/apis/family_api_tests.py b/seqr/views/apis/family_api_tests.py
index 67fc881917..381a29e323 100644
--- a/seqr/views/apis/family_api_tests.py
+++ b/seqr/views/apis/family_api_tests.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
import json
+
import mock
from copy import deepcopy
from datetime import datetime
@@ -14,9 +15,9 @@
family_variant_tag_summary, update_family_analysis_groups, get_family_rna_seq_data, get_family_phenotype_gene_scores
from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, \
FAMILY_NOTE_FIELDS, FAMILY_FIELDS, IGV_SAMPLE_FIELDS, \
- SAMPLE_FIELDS, INDIVIDUAL_FIELDS, INTERNAL_INDIVIDUAL_FIELDS, INTERNAL_FAMILY_FIELDS, CASE_REVIEW_FAMILY_FIELDS, \
+ DATASET_FIELDS, INDIVIDUAL_FIELDS, INTERNAL_INDIVIDUAL_FIELDS, INTERNAL_FAMILY_FIELDS, CASE_REVIEW_FAMILY_FIELDS, \
MATCHMAKER_SUBMISSION_FIELDS, TAG_TYPE_FIELDS, CASE_REVIEW_INDIVIDUAL_FIELDS
-from seqr.models import FamilyAnalysedBy, AnalysisGroup, Sample
+from seqr.models import FamilyAnalysedBy, AnalysisGroup, Individual
FAMILY_GUID = 'F000001_1'
FAMILY_GUID2 = 'F000002_2'
@@ -48,7 +49,7 @@ def test_family_page_data(self):
response_json = response.json()
response_keys = {
- 'familiesByGuid', 'individualsByGuid', 'familyNotesByGuid', 'samplesByGuid', 'igvSamplesByGuid',
+ 'familiesByGuid', 'individualsByGuid', 'familyNotesByGuid', 'datasetsByGuid', 'igvSamplesByGuid',
'mmeSubmissionsByGuid',
}
self.assertSetEqual(set(response_json.keys()), response_keys)
@@ -73,7 +74,7 @@ def test_family_page_data(self):
self.assertEqual(len(response_json['individualsByGuid']), 3)
individual = response_json['individualsByGuid'][INDIVIDUAL_GUID]
- individual_fields = {'sampleGuids', 'igvSampleGuids', 'mmeSubmissionGuid', 'phenotypePrioritizationTools', 'rnaSample'}
+ individual_fields = {'igvSampleGuids', 'mmeSubmissionGuid', 'phenotypePrioritizationTools', 'rnaSample'}
individual_fields.update(INDIVIDUAL_FIELDS)
self.assertSetEqual(set(individual.keys()), individual_fields)
self.assertListEqual([
@@ -95,12 +96,15 @@ def test_family_page_data(self):
self.assertSetEqual({PROJECT_GUID}, {i['projectGuid'] for i in response_json['individualsByGuid'].values()})
self.assertSetEqual({FAMILY_GUID}, {i['familyGuid'] for i in response_json['individualsByGuid'].values()})
- self.assertEqual(len(response_json['samplesByGuid']), 3)
- self.assertSetEqual(set(next(iter(response_json['samplesByGuid'].values())).keys()), SAMPLE_FIELDS)
- self.assertSetEqual({PROJECT_GUID}, {s['projectGuid'] for s in response_json['samplesByGuid'].values()})
- self.assertSetEqual({FAMILY_GUID}, {s['familyGuid'] for s in response_json['samplesByGuid'].values()})
- self.assertEqual(len(individual['sampleGuids']), 1)
- self.assertTrue(set(individual['sampleGuids']).issubset(set(response_json['samplesByGuid'].keys())))
+ self.assertEqual(len(response_json['datasetsByGuid']), 2)
+ dataset = response_json['datasetsByGuid']['S000129_na19675']
+ self.assertSetEqual(set(dataset.keys()), DATASET_FIELDS)
+ self.assertSetEqual({PROJECT_GUID}, {s['projectGuid'] for s in response_json['datasetsByGuid'].values()})
+ self.assertListEqual(
+ dataset['activeIndividuals'],
+ ['I000001_na19675', 'I000004_hg00731', 'I000005_hg00732', 'I000006_hg00733', 'I000007_na20870', 'I000009_na20874'],
+ )
+ self.assertListEqual(dataset['inactiveIndividuals'], ['I000003_na19679'])
self.assertEqual(len(response_json['igvSamplesByGuid']), 1)
self.assertSetEqual(set(next(iter(response_json['igvSamplesByGuid'].values())).keys()), IGV_SAMPLE_FIELDS)
@@ -209,7 +213,7 @@ def test_family_page_data(self):
self.assertEqual(family['projectGuid'], PROJECT_GUID)
self.assertSetEqual(set(family['individualGuids']), set(response_json['individualsByGuid'].keys()))
self.assertEqual(len(response_json['individualsByGuid']), 3)
- self.assertEqual(len(response_json['samplesByGuid']), 7)
+ self.assertEqual(len(response_json['datasetsByGuid']), 3)
self.assertEqual(len(response_json['igvSamplesByGuid']), 0)
self.assertEqual(len(response_json['mmeSubmissionsByGuid']), 0)
self.assertEqual(len(response_json['familyNotesByGuid']), 0)
@@ -324,7 +328,7 @@ def test_delete_families_handler(self, mock_pm_group, mock_internal_namespaces):
# Test success
MatchmakerSubmission.objects.update(deleted_date=datetime.now())
- Sample.objects.update(is_active=False)
+ Individual.active_datasets.through.objects.all().delete()
response = self.client.post(url, content_type='application/json', data=json.dumps(req_values))
self.assertEqual(response.status_code, 200)
diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py
index 94cb1c80fa..00a0bf7a0f 100644
--- a/seqr/views/apis/individual_api_tests.py
+++ b/seqr/views/apis/individual_api_tests.py
@@ -12,7 +12,7 @@
from io import BytesIO
from openpyxl import load_workbook
-from seqr.models import Individual, Sample, SavedVariant, VariantTag
+from seqr.models import Individual, Dataset, SavedVariant, VariantTag
from seqr.views.apis.individual_api import edit_individuals_handler, update_individual_handler, \
delete_individuals_handler, receive_individuals_table_handler, save_individuals_table_handler, \
receive_individuals_metadata_handler, save_individuals_metadata_table_handler, update_individual_hpo_terms, \
@@ -206,7 +206,9 @@ def test_edit_individuals(self, mock_pm_group):
])
# send valid request
- Sample.objects.filter(guid__in=['S000130_na19678', 'S000135_na20870']).update(is_active=False)
+ for dataset in Dataset.objects.filter(guid__in=['S000130_na19678', 'S000129_na19675']):
+ dataset.inactive_individuals.set(dataset.active_individuals.all())
+ dataset.active_individuals.set(set())
response = self.client.post(edit_individuals_url, content_type='application/json', data=json.dumps({
'individuals': [INDIVIDUAL_IDS_UPDATE_DATA, INDIVIDUAL_FAMILY_UPDATE_DATA]
}))
@@ -369,7 +371,9 @@ def test_delete_individuals(self, mock_pm_group):
}))
self.assertEqual(response.status_code, 400)
self.assertListEqual(response.json()['errors'], ['Unable to delete individuals with active search sample: NA19678'])
- Sample.objects.filter(guid__in=['S000130_na19678', 'S000143_na20885', 'S000173_na21987']).update(is_active=False)
+ for dataset in Dataset.objects.filter(guid__in=['S000130_na19678', 'S000143_na20885', 'S000147_na21234']):
+ dataset.inactive_individuals.set(dataset.active_individuals.all())
+ dataset.active_individuals.set(set())
# send valid requests
response = self.client.post(individuals_url, content_type='application/json', data=json.dumps({
@@ -385,6 +389,7 @@ def test_delete_individuals(self, mock_pm_group):
})
self.assertFalse('I000002_na19678' in response_json['familiesByGuid']['F000001_1']['individualGuids'])
self.assertIsNone(response_json['familiesByGuid']['F000001_1']['pedigreeImage'])
+ self.assertFalse(Dataset.objects.filter(guid='S000130_na19678').exists())
# Test PM permission
pm_required_delete_individuals_url = reverse(delete_individuals_handler, args=[PM_REQUIRED_PROJECT_GUID])
@@ -433,6 +438,11 @@ def test_delete_individuals(self, mock_pm_group):
'individuals': [EXTERNAL_WORKSPACE_INDIVIDUAL_UPDATE_DATA]
}))
self.assertEqual(response.status_code, 200 if self.HAS_EXTERNAL_PROJECT_ACCESS else 403)
+ if self.HAS_EXTERNAL_PROJECT_ACCESS:
+ self.assertListEqual(
+ list(Dataset.objects.get(guid='S000147_na21234').inactive_individuals.order_by('id').values_list('guid', flat=True)),
+ ['I000018_na21234', 'I000021_na21654'],
+ )
def test_individuals_table_handler_errors(self):
individuals_url = reverse(receive_individuals_table_handler, args=[PROJECT_GUID])
diff --git a/seqr/views/apis/project_api.py b/seqr/views/apis/project_api.py
index b5793f8db7..4641d331c9 100644
--- a/seqr/views/apis/project_api.py
+++ b/seqr/views/apis/project_api.py
@@ -12,7 +12,7 @@
from notifications.models import Notification
from matchmaker.models import MatchmakerSubmission
-from seqr.models import Project, Family, Individual, Sample, RnaSample, FamilyNote, PhenotypePrioritization, CAN_EDIT
+from seqr.models import Project, Family, Individual, Dataset, RnaSample, FamilyNote, PhenotypePrioritization, CAN_EDIT
from seqr.utils.file_utils import file_iter
from seqr.utils.logging_utils import SeqrLogger
from seqr.views.utils.airtable_utils import AirtableSession, ANVIL_REQUEST_TRACKING_TABLE
@@ -21,7 +21,7 @@
from seqr.views.utils.individual_utils import delete_individuals
from seqr.views.utils.json_utils import create_json_response, _to_snake_case, _to_camel_case
from seqr.views.utils.json_to_orm_utils import update_project_from_json, create_model_from_json, update_model_from_json
-from seqr.views.utils.orm_to_json_utils import _get_json_for_project, get_json_for_samples, \
+from seqr.views.utils.orm_to_json_utils import _get_json_for_project, get_json_for_datasets, \
get_json_for_project_collaborator_list, get_json_for_matchmaker_submissions, \
get_json_for_family_notes, _get_json_for_individuals, get_json_for_project_collaborator_groups, \
FAMILY_ADDITIONAL_VALUES
@@ -265,25 +265,24 @@ def project_families(request, project_guid):
def project_overview(request, project_guid):
project = get_project_and_check_permissions(project_guid, request.user)
- sample_load_counts, sample_models = _sample_load_counts(
- Sample, project, 'sample_type', 'dataset_type', loadedDate=TruncDate('loaded_date'),
- )
- rna_sample_load_counts, _ = _sample_load_counts(
- RnaSample, project, sample_type=Value('RNA'), dataset_type=F('data_type'), loadedDate=TruncDate('created_date'),
- )
-
- first_loaded_samples = sample_models.order_by('individual__family', 'loaded_date').distinct('individual__family').values_list('id', flat=True)
- samples = sample_models.filter(Q(is_active=True) | Q(id__in=first_loaded_samples))
- samples_by_guid = {s['sampleGuid']: s for s in get_json_for_samples(samples, project_guid=project_guid)}
+ datasets = Dataset.objects.filter(
+ Q(active_individuals__family__project=project) | Q(inactive_individuals__family__project=project)
+ ).distinct()
+ datasets_by_guid = {d['datasetGuid']: d for d in get_json_for_datasets(datasets, project_guid)}
+ rna_sample_load_counts = RnaSample.objects.filter(
+ individual__family__project=project,
+ ).values('data_type', loadedDate=TruncDate('created_date')).order_by('loadedDate').annotate(
+ familyCounts=ArrayAgg('individual__family__guid'),
+ )
grouped_sample_counts = defaultdict(list)
- for s in sample_load_counts + rna_sample_load_counts:
+ for s in rna_sample_load_counts:
s['familyCounts'] = {f: s['familyCounts'].count(f) for f in s['familyCounts']}
- grouped_sample_counts[f'{s.pop("sample_type")}__{s.pop("dataset_type")}'].append(s)
+ grouped_sample_counts[s.pop('data_type')].append(s)
- project_json = {'projectGuid': project_guid, 'sampleCounts': grouped_sample_counts}
+ project_json = {'projectGuid': project_guid, 'rnaSampleCounts': grouped_sample_counts}
response = {
- 'samplesByGuid': samples_by_guid,
+ 'datasetsByGuid': datasets_by_guid,
}
add_project_tag_type_counts(project, response, project_json=project_json)
@@ -299,13 +298,6 @@ def project_overview(request, project_guid):
return create_json_response(response)
-def _sample_load_counts(sample_cls, project, *args, **kwargs):
- sample_models = sample_cls.objects.filter(individual__family__project=project)
- return list(sample_models.values(*args, **kwargs).order_by('loadedDate').annotate(
- familyCounts=ArrayAgg('individual__family__guid'))
- ), sample_models
-
-
@login_and_policies_required
def project_collaborators(request, project_guid):
project = get_project_and_check_permissions(project_guid, request.user)
@@ -330,16 +322,6 @@ def project_individuals(request, project_guid):
})
-@login_and_policies_required
-def project_samples(request, project_guid):
- project = get_project_and_check_permissions(project_guid, request.user)
- samples = Sample.objects.filter(individual__family__project=project)
-
- return create_json_response({
- 'samplesByGuid': {s['sampleGuid']: s for s in get_json_for_samples(samples, project_guid=project_guid)},
- })
-
-
@login_and_policies_required
def project_analysis_groups(request, project_guid):
project = get_project_and_check_permissions(project_guid, request.user)
@@ -448,9 +430,7 @@ def _delete_project(project_guid, user):
project = Project.objects.get(guid=project_guid)
check_user_created_object_permissions(project, user)
- individual_guids_to_delete = Individual.objects.filter(
- family__project__guid=project_guid).values_list('guid', flat=True)
- delete_individuals(project, individual_guids_to_delete, user)
+ delete_individuals(project, individual_guids=None, user=user)
Family.bulk_delete(user, project=project)
diff --git a/seqr/views/apis/project_api_tests.py b/seqr/views/apis/project_api_tests.py
index 43b4fcf6ad..96b5538d38 100644
--- a/seqr/views/apis/project_api_tests.py
+++ b/seqr/views/apis/project_api_tests.py
@@ -12,7 +12,7 @@
from seqr.views.apis.project_api import create_project_handler, delete_project_handler, update_project_handler, \
project_page_data, project_families, project_overview, project_mme_submisssions, project_individuals, \
project_analysis_groups, update_project_workspace, project_family_notes, project_collaborators, project_locus_lists, \
- project_samples, project_notifications, mark_read_project_notifications, subscribe_project_notifications, \
+ project_notifications, mark_read_project_notifications, subscribe_project_notifications, \
update_project_rna_seq, load_rna_seq_sample_data
from seqr.views.apis.data_manager_api_tests import RNA_OUTLIER_SAMPLE_DATA, RNA_OUTLIER_MUSCLE_SAMPLE_GUID, RNA_TPM_SAMPLE_DATA, \
RNA_TPM_MUSCLE_SAMPLE_GUID, RNA_SPLICE_SAMPLE_DATA, RNA_SPLICE_SAMPLE_GUID, PLACEHOLDER_GUID, \
@@ -20,7 +20,7 @@
from seqr.views.utils.terra_api_utils import TerraAPIException, TerraRefreshTokenFailedException
from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, \
PROJECT_FIELDS, LOCUS_LIST_FIELDS, PA_LOCUS_LIST_FIELDS, NO_INTERNAL_CASE_REVIEW_INDIVIDUAL_FIELDS, \
- SAMPLE_FIELDS, SUMMARY_FAMILY_FIELDS, INTERNAL_INDIVIDUAL_FIELDS, INDIVIDUAL_FIELDS, TAG_TYPE_FIELDS, \
+ DATASET_FIELDS, SUMMARY_FAMILY_FIELDS, INTERNAL_INDIVIDUAL_FIELDS, INDIVIDUAL_FIELDS, TAG_TYPE_FIELDS, \
FAMILY_NOTE_FIELDS, MATCHMAKER_SUBMISSION_FIELDS, ANALYSIS_GROUP_FIELDS, \
EXT_WORKSPACE_NAMESPACE, TEST_EMPTY_PROJECT_WORKSPACE, DYNAMIC_ANALYSIS_GROUP_FIELDS
@@ -341,12 +341,12 @@ def test_project_overview(self):
response_json = response.json()
response_keys = {
- 'projectsByGuid', 'samplesByGuid', 'familyTagTypeCounts',
+ 'projectsByGuid', 'datasetsByGuid', 'familyTagTypeCounts',
}
self.assertSetEqual(set(response_json.keys()), response_keys)
project_fields = {
- 'variantTagTypes', 'variantFunctionalTagTypes', 'sampleCounts',
+ 'variantTagTypes', 'variantFunctionalTagTypes', 'rnaSampleCounts',
'projectGuid', 'mmeDeletedSubmissionCount', 'mmeSubmissionCount',
}
project_response = response_json['projectsByGuid'][PROJECT_GUID]
@@ -374,25 +374,16 @@ def test_project_overview(self):
'order': 99,
'numTags': 1,
})
- self.assertDictEqual(project_response['sampleCounts'], {
- 'WES__SNV_INDEL': [{
- 'familyCounts': {
- 'F000001_1': 3, 'F000002_2': 3, 'F000003_3': 1, 'F000004_4': 1, 'F000005_5': 1, 'F000006_6': 1,
- 'F000007_7': 1, 'F000008_8': 1, 'F000010_10': 1,
- },
- 'loadedDate': '2017-02-05',
- }],
- 'WES__SV': [{'familyCounts': {'F000002_2': 3}, 'loadedDate': '2018-02-05'}],
- 'WES__MITO': [{'familyCounts': {'F000002_2': 1}, 'loadedDate': '2022-02-05'}],
- 'RNA__S': [{'familyCounts': {'F000001_1': 2}, 'loadedDate': '2017-02-05'}],
- 'RNA__T': [{'familyCounts': {'F000001_1': 2}, 'loadedDate': '2017-02-05'}],
- 'RNA__E': [{'familyCounts': {'F000001_1': 1}, 'loadedDate': '2017-02-05'}],
+ self.assertDictEqual(project_response['rnaSampleCounts'], {
+ 'S': [{'familyCounts': {'F000001_1': 2}, 'loadedDate': '2017-02-05'}],
+ 'T': [{'familyCounts': {'F000001_1': 2}, 'loadedDate': '2017-02-05'}],
+ 'E': [{'familyCounts': {'F000001_1': 1}, 'loadedDate': '2017-02-05'}],
})
self.assertEqual(project_response['mmeSubmissionCount'], 1)
self.assertEqual(project_response['mmeDeletedSubmissionCount'], 0)
- self.assertEqual(len(response_json['samplesByGuid']), 16)
- self.assertSetEqual(set(next(iter(response_json['samplesByGuid'].values())).keys()), SAMPLE_FIELDS)
+ self.assertEqual(len(response_json['datasetsByGuid']), 5)
+ self.assertSetEqual(set(next(iter(response_json['datasetsByGuid'].values())).keys()), DATASET_FIELDS)
self.assertDictEqual(response_json['familyTagTypeCounts'], {
'F000001_1': {'Review': 1, 'Tier 1 - Novel gene and phenotype': 1, 'MME Submission': 1},
'F000002_2': {'AIP': 1, 'Excluded': 1, 'Known gene for phenotype': 1},
@@ -543,24 +534,6 @@ def test_project_individuals(self):
NO_INTERNAL_CASE_REVIEW_INDIVIDUAL_FIELDS,
)
- def test_project_samples(self):
- url = reverse(project_samples, args=[PROJECT_GUID])
- self.check_collaborator_login(url)
-
- response = self.client.get(url)
- self.assertEqual(response.status_code, 200)
-
- response_json = response.json()
- response_keys = {'samplesByGuid'}
- self.assertSetEqual(set(response_json.keys()), response_keys)
-
- self.assertEqual(len(response_json['samplesByGuid']), 17)
- self.assertSetEqual(set(next(iter(response_json['samplesByGuid'].values())).keys()), SAMPLE_FIELDS)
-
- # Test empty project
- empty_url = reverse(project_samples, args=[EMPTY_PROJECT_GUID])
- self._check_empty_project(empty_url, response_keys)
-
def test_project_analysis_groups(self):
url = reverse(project_analysis_groups, args=[PROJECT_GUID])
self.check_collaborator_login(url)
diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py
index 165cd931bc..3d2589a250 100644
--- a/seqr/views/apis/report_api.py
+++ b/seqr/views/apis/report_api.py
@@ -24,7 +24,7 @@
from seqr.views.utils.terra_api_utils import anvil_enabled
from seqr.views.utils.variant_utils import DISCOVERY_CATEGORY
-from seqr.models import Project, Family, FamilyAnalysedBy, Sample, RnaSample, Individual
+from seqr.models import Project, Family, FamilyAnalysedBy, Dataset, RnaSample, Individual
from settings import GREGOR_DATA_MODEL_URL
@@ -60,10 +60,13 @@ def seqr_stats(request):
grouped_sample_counts = defaultdict(dict)
for project_key, projects in project_models.items():
- samples_counts = _get_sample_counts(Sample.objects.filter(individual__family__project__in=projects))
+ samples_counts = _get_sample_counts(
+ Dataset.objects.filter(active_individuals__family__project__in=projects),
+ count=Count('active_individuals'),
+ )
samples_counts.update(_get_sample_counts(
- RnaSample.objects.filter(individual__family__project__in=projects).annotate(sample_type=Value('RNA')),
- data_type_key='data_type')
+ RnaSample.objects.filter(individual__family__project__in=projects, is_active=True).annotate(sample_type=Value('RNA')),
+ data_type_key='data_type', count=Count('*'))
)
for k, v in samples_counts.items():
grouped_sample_counts[k][project_key] = v
@@ -80,8 +83,8 @@ def seqr_stats(request):
})
-def _get_sample_counts(sample_q, data_type_key='dataset_type'):
- samples_agg = sample_q.filter(is_active=True).values('sample_type', data_type_key).annotate(count=Count('*'))
+def _get_sample_counts(sample_q, count=None, data_type_key='dataset_type'):
+ samples_agg = sample_q.values('sample_type', data_type_key).annotate(count=count)
return {
f'{sample_agg["sample_type"]}__{sample_agg[data_type_key]}': sample_agg['count'] for sample_agg in samples_agg
}
@@ -176,8 +179,8 @@ def _add_row(row, family_id, row_type):
'congenital_status': Individual.ONSET_AGE_LOOKUP[individual.onset_age] if individual.onset_age else 'Unknown',
**anvil_export_airtable_fields(airtable_metadata, has_dbgap_submission),
},
- get_additional_sample_fields=lambda sample, *args: {
- 'entity:sample_id': sample.individual.individual_id,
+ get_additional_sample_fields=lambda individual, *args: {
+ 'entity:sample_id': individual.individual_id,
'sequencing_center': 'Broad',
},
family_fields={'phenotype_group': {
@@ -379,11 +382,11 @@ def gregor_export(request):
grouped_data_type_individuals = _get_individual_data_types(projects)
# If multiple individual records, prefer WGS
- individual_lookup = {
+ individual_lookup = [
next(data_type_individuals[data_type.upper()] for data_type in GREGOR_DATA_TYPES
- if data_type_individuals.get(data_type.upper())): None
+ if data_type_individuals.get(data_type.upper()))
for data_type_individuals in grouped_data_type_individuals.values()
- }
+ ]
participant_rows = []
family_map = {}
@@ -402,7 +405,7 @@ def _add_row(row, family_id, row_type):
parse_anvil_metadata(
projects,
user=request.user,
- individual_samples=individual_lookup,
+ individuals=individual_lookup,
individual_data_types=grouped_data_type_individuals,
add_row=_add_row,
format_id=_format_gregor_id,
@@ -513,20 +516,18 @@ def _process_participant_row(participant, phenotype_rows, missing_participant_id
def _get_individual_data_types(projects):
- sample_types = Sample.objects.filter(individual__family__project__in=projects).values_list('individual_id', 'sample_type')
- individual_data_types = defaultdict(set)
- for individual_db_id, sample_type in sample_types:
- individual_data_types[individual_db_id].add(sample_type)
- for individual_db_id in RnaSample.objects.filter(individual__family__project__in=projects).values_list('individual_id', flat=True):
- individual_data_types[individual_db_id].add('RNA')
- individuals = Individual.objects.filter(id__in=individual_data_types).prefetch_related(
- 'family__project', 'mother', 'father')
+ rna_individuals = set(RnaSample.objects.filter(individual__family__project__in=projects).values_list('individual_id', flat=True))
+ individuals = Individual.objects.filter(family__project__in=projects).annotate(
+ active_sample_types=ArrayAgg('active_datasets__sample_type', distinct=True, filter=Q(active_datasets__isnull=False)),
+ inactive_sample_types=ArrayAgg('inactive_datasets__sample_type', distinct=True, filter=Q(inactive_datasets__isnull=False)),
+ ).prefetch_related('family__project', 'mother', 'father')
grouped_data_type_individuals = defaultdict(dict)
for i in individuals:
participant_id = _format_gregor_id(i.individual_id)
- grouped_data_type_individuals[participant_id].update(
- {data_type: i for data_type in individual_data_types[i.id]})
+ data_types = {*i.active_sample_types, *i.inactive_sample_types} | ({'RNA'} if i.id in rna_individuals else set())
+ if data_types:
+ grouped_data_type_individuals[participant_id].update({data_type: i for data_type in data_types})
return grouped_data_type_individuals
@@ -995,7 +996,8 @@ def variant_metadata(request, project_guid):
individuals = Individual.objects.filter(
family__project__in=projects, family__savedvariant__varianttag__variant_tag_type__category=DISCOVERY_CATEGORY,
).distinct().annotate(
- data_types=ArrayAgg('sample__sample_type', distinct=True, filter=Q(sample__isnull=False))
+ active_data_types=ArrayAgg('active_datasets__sample_type', distinct=True),
+ inactive_data_types=ArrayAgg('inactive_datasets__sample_type', distinct=True),
)
families_by_id = {}
@@ -1021,8 +1023,8 @@ def _add_row(row, family_id, row_type):
parse_anvil_metadata(
projects,
user=request.user,
- individual_samples={i: None for i in individuals},
- individual_data_types={i.individual_id: i.data_types for i in individuals},
+ individuals=individuals,
+ individual_data_types={i.individual_id: {*i.active_data_types, *i.inactive_data_types} for i in individuals},
add_row=_add_row,
mme_value=ArrayAgg('matchmakersubmissiongenes__saved_variant__variant_id'),
include_family_name_display=True,
@@ -1069,6 +1071,6 @@ def _get_clickhouse_metadata(dataset_type, genome_version, keys, include_clinvar
if include_clinvar:
fields.append('clinvar')
qs = qs.join_clinvar()
- if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS:
+ if dataset_type == Dataset.DATASET_TYPE_VARIANT_CALLS:
fields.append('caid')
return {(dataset_type, genome_version, v.pop('key')): v for v in qs.values(*fields)}
diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py
index 729b3ad93d..60d214a668 100644
--- a/seqr/views/apis/report_api_tests.py
+++ b/seqr/views/apis/report_api_tests.py
@@ -18,7 +18,7 @@
"id": "rec2B6OGmQpAkQW3s",
"fields": {
"SeqrCollaboratorSampleID": "VCGS_FAM203_621_D1",
- "CollaboratorSampleID": "NA19675",
+ "CollaboratorSampleID": "NA19675_1",
"Collaborator": ["recW24C2CJW5lT64K"],
"dbgap_study_id": "dbgap_stady_id_1",
"dbgap_subject_id": "dbgap_subject_id_1",
@@ -746,7 +746,7 @@ def _check_anvil_export_response(self, response, mock_zip, no_analyst_project_ur
'entity:sample_id', '01-subject_id', '02-sample_id', '03-dbgap_sample_id', '04-sequencing_center',
'05-sample_source', '06-tissue_affected_status',])
self.assertIn(
- ['NA19675_1', 'NA19675_1', 'NA19675', 'SM-A4GQ4', 'Broad', '-', '-'],
+ ['NA19675_1', 'NA19675_1', 'NA19675_1', 'SM-A4GQ4', 'Broad', '-', '-'],
sample_file,
)
@@ -767,7 +767,7 @@ def _check_anvil_export_response(self, response, mock_zip, no_analyst_project_ur
'1_248367227_HG00731', 'HG00731', 'HG00731', 'RP11', 'Known', 'paternal',
'Homozygous', 'GRCh37', '1', '248367227', 'TC', 'T', '-', '-', 'ENST00000371839', '-', '-', '-', '-'], discovery_file)
self.assertIn([
- '21_3343353_NA19675_1', 'NA19675_1', 'NA19675', 'RP11', 'Candidate', 'de novo',
+ '21_3343353_NA19675_1', 'NA19675_1', 'NA19675_1', 'RP11', 'Candidate', 'de novo',
'Heterozygous', 'GRCh37', '21', '3343353', 'GAGA', 'G', 'c.375_377delTCT', 'p.Leu126del', 'ENST00000258436.5',
'-', '-', '-', 'This individual is published in PMID34415322'], discovery_file)
self.assertIn([
diff --git a/seqr/views/apis/saved_variant_api.py b/seqr/views/apis/saved_variant_api.py
index 6daa725fc7..81a3c0c713 100644
--- a/seqr/views/apis/saved_variant_api.py
+++ b/seqr/views/apis/saved_variant_api.py
@@ -4,7 +4,7 @@
from django.db.models import Q
from seqr.models import SavedVariant, VariantTagType, VariantTag, VariantNote, VariantFunctionalData,\
- Family, GeneNote, Project, Sample
+ Family, GeneNote, Project, Dataset
from seqr.utils.xpos_utils import get_xpos
from seqr.views.utils.json_to_orm_utils import update_model_from_json, get_or_create_model_from_json, \
create_model_from_json
@@ -83,7 +83,7 @@ def create_manual_saved_variant_handler(request, family_guid):
variant_json['saved_variant_json'] = {**variant_json}
variant_json.update({
'key': None,
- 'dataset_type': Sample.DATASET_TYPE_SV_CALLS if variant_json.get('svName') else Sample.DATASET_TYPE_VARIANT_CALLS,
+ 'dataset_type': Dataset.DATASET_TYPE_SV_CALLS if variant_json.get('svName') else Dataset.DATASET_TYPE_VARIANT_CALLS,
})
model_json = parse_saved_variant_json(variant_json, family.id)
diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py
index 6aaf14cb6a..6316d0167b 100644
--- a/seqr/views/apis/summary_data_api.py
+++ b/seqr/views/apis/summary_data_api.py
@@ -11,7 +11,7 @@
get_mme_metrics, get_hpo_terms_by_id
from matchmaker.models import MatchmakerSubmission
from reference_data.models import HumanPhenotypeOntology
-from seqr.models import Project, Family, Individual, Sample, VariantTagType, SavedVariant, FamilyAnalysedBy
+from seqr.models import Project, Family, Individual, Dataset, VariantTagType, SavedVariant, FamilyAnalysedBy
from seqr.views.utils.airtable_utils import AirtableSession
from seqr.views.utils.file_utils import load_uploaded_file
from seqr.utils.communication_utils import safe_post_to_slack
@@ -275,7 +275,7 @@ def _get_clickhouse_variants(families_by_id: dict[int, dict], family_variant_ids
families_by_project[family['project__guid']].append(family['guid'])
for project_guid, family_guids in families_by_project.items():
genotype_keys = get_clickhouse_genotypes(
- project_guid, family_guids, genome_version, Sample.DATASET_TYPE_VARIANT_CALLS, variants_by_key.keys(),
+ project_guid, family_guids, genome_version, Dataset.DATASET_TYPE_VARIANT_CALLS, variants_by_key.keys(),
additional_fields=['xpos']
)
for key, entry_data in genotype_keys.items():
diff --git a/seqr/views/apis/variant_search_api.py b/seqr/views/apis/variant_search_api.py
index 76564c5d00..0d78858e59 100644
--- a/seqr/views/apis/variant_search_api.py
+++ b/seqr/views/apis/variant_search_api.py
@@ -17,7 +17,7 @@
from clickhouse_search.search import get_clickhouse_variants, format_clickhouse_results, format_clickhouse_export_results, \
get_sorted_search_results, clickhouse_variant_lookup, InvalidSearchException
from reference_data.models import GENOME_VERSION_GRCh38, GENOME_VERSION_LOOKUP
-from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Sample
+from seqr.models import Project, Family, Individual, SavedVariant, VariantSearch, VariantSearchResults, ProjectCategory, Dataset
from seqr.views.utils.export_utils import export_table
from seqr.utils.gene_utils import get_genes_for_variant_display
from seqr.utils.logging_utils import SeqrLogger
@@ -138,8 +138,8 @@ def _get_or_create_results_model(search_hash, search_context, user):
if search_context.get('unsolvedFamiliesOnly'):
families = families.exclude(analysis_status__in=Family.SOLVED_ANALYSIS_STATUSES)
if search_context.get('trioFamiliesOnly'):
- families = families.annotate(search_sample_count=Count('individual__sample__id', filter=Q(
- individual__sample__is_active=True, individual__sample__dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS,
+ families = families.annotate(search_sample_count=Count('individual__id', filter=Q(
+ individual__active_datasets__dataset_type=Dataset.DATASET_TYPE_VARIANT_CALLS,
))).filter(
search_sample_count__gte=3, individual__mother__isnull=False, individual__father__isnull=False,
).distinct()
@@ -439,16 +439,16 @@ def search_context_handler(request):
projectGuid=Value(project_guid) if project_guid else F('project__guid'),
familyGuid=F('guid'),
analysisStatus=F('analysis_status'),
+ sampleTypes=ArrayAgg(
+ JSONObject(sampleType='individual__active_datasets__sample_type', datasetType='individual__active_datasets__dataset_type', isActive=Value(True)),
+ distinct=True, filter=Q(individual__active_datasets__dataset_type__isnull=False),
+ ),
**FAMILY_ADDITIONAL_VALUES,
)}
- family_sample_types = Sample.objects.filter(individual__family__project__in=projects, is_active=True).values('individual__family__guid').annotate(
- samples=ArrayAgg(JSONObject(sampleType='sample_type', datasetType='dataset_type', isActive=Value(True)), distinct=True))
project_dataset_types = defaultdict(set)
- for agg in family_sample_types:
- family = response['familiesByGuid'][agg['individual__family__guid']]
- family['sampleTypes'] = agg['samples']
- project_dataset_types[family['projectGuid']].update([s['datasetType'] for s in agg['samples']])
+ for family in response['familiesByGuid'].values():
+ project_dataset_types[family['projectGuid']].update([s['datasetType'] for s in family['sampleTypes']])
for project_guid, dataset_types in project_dataset_types.items():
response['projectsByGuid'][project_guid]['datasetTypes'] = list(dataset_types)
diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py
index 5d9d860ab0..53ea3b4bcd 100644
--- a/seqr/views/utils/anvil_metadata_utils.py
+++ b/seqr/views/utils/anvil_metadata_utils.py
@@ -1,8 +1,8 @@
from collections import defaultdict
from datetime import datetime
-from django.db.models import F, Q, Value, Case, When, CharField, Aggregate
-from django.db.models.functions import Replace
+from django.db.models import F, Q, Value, Case, When, CharField, Aggregate, Min
+from django.db.models.functions import Replace, Coalesce
from django.contrib.auth.models import User
from django.contrib.postgres.aggregates import ArrayAgg
import requests
@@ -10,7 +10,7 @@
from matchmaker.models import MatchmakerSubmission
from reference_data.models import HumanPhenotypeOntology, Omim, GENOME_VERSION_LOOKUP
-from seqr.models import Project, Family, Individual, Sample, SavedVariant, VariantTagType
+from seqr.models import Project, Family, Individual, Dataset, SavedVariant, VariantTagType
from seqr.views.utils.airtable_utils import AirtableSession
from seqr.utils.gene_utils import get_genes
from seqr.utils.middleware import ErrorsWarningsException
@@ -96,8 +96,8 @@
}
METHOD_MAP = {
- Sample.SAMPLE_TYPE_WES: 'SR-ES',
- Sample.SAMPLE_TYPE_WGS: 'SR-GS',
+ Dataset.SAMPLE_TYPE_WES: 'SR-ES',
+ Dataset.SAMPLE_TYPE_WGS: 'SR-GS',
}
FAMILY_INDIVIDUAL_FIELDS = ['family_id', 'internal_project_id', 'phenotype_description', 'pmid_id', 'solve_status']
@@ -163,31 +163,31 @@ def _get_variant_json(saved_variants, fields, annotations):
def parse_anvil_metadata(
projects: Iterable[Project], user: User, add_row: Callable[[dict, str, str], None],
max_loaded_date: str = None, family_fields: dict = None, format_id: Callable[[str], str] = lambda s: s,
- get_additional_sample_fields: Callable[[Sample, dict], dict] = None,
+ get_additional_sample_fields: Callable[[Individual, dict], dict] = None,
get_additional_individual_fields: Callable[[Individual, dict], dict] = None,
- individual_samples: dict[Individual, Sample] = None, individual_data_types: dict[str, Iterable[str]] = None,
+ individuals: list[Individual] = None, individual_data_types: dict[str, Iterable[str]] = None,
airtable_fields: Iterable[str] = None, mme_value: Aggregate = None,
get_variant_json: Callable[[Iterable[SavedVariant], list[str], dict], dict] = _get_variant_json, post_process_variant: Callable[[dict, list[dict]], dict] = None,
include_no_individual_families: bool = False, omit_airtable: bool = False, include_family_name_display: bool = False, include_family_sample_metadata: bool = False,
include_discovery_sample_id: bool = False, include_mondo: bool = False, omit_parent_mnvs: bool = False,
proband_only_variants: bool = False):
- individual_samples = individual_samples or (_get_loaded_before_date_project_individual_samples(projects, max_loaded_date) \
- if max_loaded_date else _get_all_project_individual_samples(projects))
+ if not individuals:
+ individuals = _get_sample_annotated_individuals(projects, max_loaded_date)
family_data_by_id = _get_family_metadata(
- {'project__in': projects} if include_no_individual_families else {'individual__in': individual_samples},
+ {'project__in': projects} if include_no_individual_families else {'individual__in': individuals},
family_fields, include_family_name_display, include_family_sample_metadata, include_mondo, format_id
)
individuals_by_family_id = defaultdict(list)
individual_ids_map = {}
sample_ids = set()
- for individual, sample in individual_samples.items():
+ for individual in individuals:
individuals_by_family_id[individual.family_id].append(individual)
individual_ids_map[individual.id] = (individual.individual_id, individual.guid)
- if sample:
- sample_ids.add(sample.sample_id)
+ if getattr(individual, 'sample_type', None):
+ sample_ids.add(individual.individual_id)
saved_variants_by_family = _get_parsed_saved_discovery_variants_by_family(
list(family_data_by_id.keys()), bool(mme_value), get_variant_json,
@@ -199,7 +199,7 @@ def parse_anvil_metadata(
list(sample_ids) or [i[0] for i in individual_ids_map.values()], user, airtable_fields)
matchmaker_individuals = {m['individual_id']: m['value'] for m in MatchmakerSubmission.objects.filter(
- individual__in=individual_samples).values('individual_id', value=mme_value)} if mme_value else {}
+ individual__in=individuals).values('individual_id', value=mme_value)} if mme_value else {}
for family_id, family_subject_row in family_data_by_id.items():
saved_variants = saved_variants_by_family[family_id]
@@ -223,15 +223,14 @@ def parse_anvil_metadata(
add_row(family_row, family_id, FAMILY_ROW_TYPE)
for individual in family_individuals:
- sample = individual_samples[individual]
-
airtable_metadata = None
has_dbgap_submission = None
+ sample_type = getattr(individual, 'sample_type', None)
if sample_airtable_metadata is not None:
- if sample:
- airtable_metadata = sample_airtable_metadata.get(sample.sample_id, {})
+ if sample_type:
+ airtable_metadata = sample_airtable_metadata.get(individual.individual_id, {})
dbgap_submission = airtable_metadata.get('dbgap_submission') or set()
- has_dbgap_submission = sample.sample_type in dbgap_submission
+ has_dbgap_submission = sample_type in dbgap_submission
elif not sample_ids:
airtable_metadata = sample_airtable_metadata.get(individual.individual_id, {})
@@ -249,8 +248,8 @@ def parse_anvil_metadata(
add_row(subject_row, family_id, SUBJECT_ROW_TYPE)
participant_id = subject_row['participant_id']
- if sample:
- sample_row = _get_sample_row(sample, participant_id, has_dbgap_submission, airtable_metadata, include_family_sample_metadata, get_additional_sample_fields)
+ if sample_type:
+ sample_row = _get_sample_row(individual, participant_id, has_dbgap_submission, airtable_metadata, include_family_sample_metadata, get_additional_sample_fields)
add_row(sample_row, family_id, SAMPLE_ROW_TYPE)
if proband_only_variants and individual.proband_relationship != Individual.SELF_RELATIONSHIP:
@@ -260,7 +259,7 @@ def parse_anvil_metadata(
format_id=format_id, omit_parent_mnvs=omit_parent_mnvs,
individual_data_types=(individual_data_types or {}).get(participant_id),
family_individuals=family_individuals if proband_only_variants else None,
- sample=sample if include_discovery_sample_id else None,
+ sample_id=individual.individual_id if include_discovery_sample_id else None,
post_process_variant=post_process_variant,
)
add_row(discovery_row, family_id, DISCOVERY_ROW_TYPE)
@@ -270,22 +269,24 @@ def _get_nested_variant_name(v):
return v['sv_name'] or f"{v['chrom']}-{v['pos']}-{v['ref']}-{v['alt']}"
-def _get_loaded_before_date_project_individual_samples(projects, max_loaded_date):
- max_loaded_date = datetime.strptime(max_loaded_date, '%Y-%m-%d')
- loaded_samples = _get_sorted_search_samples(projects).filter(
- loaded_date__lte=max_loaded_date).select_related('individual')
- # Only return the oldest sample for each individual
- return {sample.individual: sample for sample in loaded_samples}
-
-
-def _get_all_project_individual_samples(projects):
- samples_by_individual_id = {s.individual_id: s for s in _get_sorted_search_samples(projects)}
+def _get_sample_annotated_individuals(projects, max_loaded_date):
individuals = Individual.objects.filter(family__project__in=projects)
- return {i: samples_by_individual_id.get(i.id) for i in individuals}
-
-
-def _get_sorted_search_samples(projects):
- return Sample.objects.filter(individual__family__project__in=projects).order_by('-loaded_date')
+ active_filter = inactive_filter = None
+ if max_loaded_date:
+ max_loaded_date = datetime.strptime(max_loaded_date, '%Y-%m-%d')
+ active_filter = Q(active_datasets__loaded_date__lte=max_loaded_date)
+ inactive_filter = Q(inactive_datasets__loaded_date__lte=max_loaded_date)
+ individuals = individuals.filter(active_filter | inactive_filter)
+ return individuals.annotate(
+ sample_type=Coalesce(
+ Min('active_datasets__sample_type', filter=active_filter),
+ Min('inactive_datasets__sample_type', filter=inactive_filter),
+ ),
+ loaded_date=Coalesce(
+ Min('inactive_datasets__loaded_date', filter=inactive_filter),
+ Min('active_datasets__loaded_date', filter=active_filter),
+ ),
+ )
HET = 'Heterozygous'
@@ -458,27 +459,27 @@ def anvil_export_airtable_fields(airtable_metadata, has_dbgap_submission):
}
-def _get_sample_row(sample, participant_id, has_dbgap_submission, airtable_metadata, include_family_sample_metadata, get_additional_sample_fields=None):
+def _get_sample_row(individual, participant_id, has_dbgap_submission, airtable_metadata, include_family_sample_metadata, get_additional_sample_fields=None):
sample_row = {
'participant_id': participant_id,
- 'sample_id': sample.sample_id,
+ 'sample_id': individual.individual_id,
}
if has_dbgap_submission:
sample_row['dbgap_sample_id'] = airtable_metadata.get('dbgap_sample_id', '')
if include_family_sample_metadata:
sample_row.update({
- 'data_type': sample.sample_type,
- 'date_data_generation': sample.loaded_date.strftime('%Y-%m-%d'),
+ 'data_type': individual.sample_type,
+ 'date_data_generation': individual.loaded_date.strftime('%Y-%m-%d'),
})
if get_additional_sample_fields:
- sample_row.update(get_additional_sample_fields(sample, airtable_metadata))
+ sample_row.update(get_additional_sample_fields(individual, airtable_metadata))
return sample_row
def _get_genetic_findings_rows(rows: list[dict], individual: Individual, family_row: dict, participant_id: str,
individual_data_types: Iterable[str], family_individuals: dict[str, str],
post_process_variant: Callable[[dict, list[dict]], dict],
- format_id: Callable[[str], str], omit_parent_mnvs: bool, sample: Sample) -> list[dict]:
+ format_id: Callable[[str], str], omit_parent_mnvs: bool, sample_id: str) -> list[dict]:
parsed_rows = []
variants_by_gene = defaultdict(list)
for row in (rows or []):
@@ -510,10 +511,10 @@ def _get_genetic_findings_rows(rows: list[dict], individual: Individual, family_
])
if individual_data_types is not None:
parsed_row['method_of_discovery'] = '|'.join([
- METHOD_MAP.get(data_type) for data_type in individual_data_types if data_type in Sample.SAMPLE_TYPE_LOOKUP
+ METHOD_MAP.get(data_type) for data_type in individual_data_types if data_type in Dataset.SAMPLE_TYPE_LOOKUP
])
- if sample is not None:
- parsed_row['sample_id'] = sample.sample_id
+ if sample_id is not None:
+ parsed_row['sample_id'] = sample_id
parsed_rows.append(parsed_row)
variants_by_gene[row[GENE_COLUMN]].append({**parsed_row, 'individual_genotype': individual_genotype})
diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py
index d12cf24b06..8c6c296fec 100644
--- a/seqr/views/utils/individual_utils.py
+++ b/seqr/views/utils/individual_utils.py
@@ -5,7 +5,7 @@
from clickhouse_search.models.postgres_dicts import SexDict
from matchmaker.models import MatchmakerSubmission, MatchmakerResult
-from seqr.models import Sample, IgvSample, RnaSample, Individual, Family, FamilyNote
+from seqr.models import Dataset, IgvSample, RnaSample, Individual, Family, FamilyNote
from seqr.utils.middleware import ErrorsWarningsException
from seqr.utils.add_data_utils import trigger_rebuild_gt_stats
from seqr.views.utils.json_to_orm_utils import update_individual_from_json, update_individual_parents, create_model_from_json, \
@@ -193,7 +193,15 @@ def delete_individuals(project, individual_guids, user):
if errors:
raise ErrorsWarningsException(errors)
- Sample.bulk_delete(user, individual__in=individuals_to_delete)
+ datasets = Dataset.objects.filter(inactive_individuals__in=individuals_to_delete)
+ if individual_guids is None:
+ Dataset.bulk_delete(user, queryset=datasets)
+ else:
+ for dataset in datasets:
+ dataset.inactive_individuals.remove(*individuals_to_delete)
+ if not dataset.inactive_individuals.exists():
+ dataset.delete_model(user, user_can_delete=True)
+
IgvSample.bulk_delete(user, individual__in=individuals_to_delete)
RnaSample.bulk_delete(user, individual__in=individuals_to_delete)
MatchmakerResult.bulk_delete(user, submission__individual__in=individuals_to_delete, submission__deleted_date__isnull=False)
@@ -231,7 +239,7 @@ def _validate_no_sumissions_no_search_samples(individuals_to_delete):
individuals_to_delete, 'MME submission',
dict(matchmakersubmission__isnull=False, matchmakersubmission__deleted_date__isnull=True)
) + _validate_delete_individuals(
- individuals_to_delete, 'search sample', dict(sample__is_active=True)
+ individuals_to_delete, 'search sample', dict(active_datasets__isnull=False)
)
@@ -244,7 +252,7 @@ def _get_updated_pedigree_json(updated_individuals, updated_families, updated_no
individual['individualGuid']: individual for individual in
_get_json_for_individuals(Individual.objects.filter(id__in=[
i.id for i in updated_individuals
- ]), user, add_sample_guids_field=True)
+ ]), user)
}
families_by_guid = {
family['familyGuid']: family for family in
diff --git a/seqr/views/utils/orm_to_json_utils.py b/seqr/views/utils/orm_to_json_utils.py
index 3f2c153c22..01513afab9 100644
--- a/seqr/views/utils/orm_to_json_utils.py
+++ b/seqr/views/utils/orm_to_json_utils.py
@@ -221,7 +221,7 @@ def _get_case_review_fields(model_cls, has_case_review_perm):
createdBy=_user_expr('familyanalysedby__created_by'),
dataType='familyanalysedby__data_type',
lastModifiedDate='familyanalysedby__last_modified_date',
- ), filter=Q(familyanalysedby__isnull=False)),
+ ), filter=Q(familyanalysedby__isnull=False), distinct=True),
'assignedAnalyst': Case(
When(assigned_analyst__isnull=False, then=JSONObject(
fullName=_full_name_expr('assigned_analyst'), email=F('assigned_analyst__email'),
@@ -265,7 +265,7 @@ def get_json_for_family_note(note):
INDIVIDUAL_DISPLAY_NAME_EXPR = Coalesce(NullIf('display_name', Value('')), 'individual_id', output_field=CharField())
-def _get_json_for_individuals(individuals, user=None, project_guid=None, add_sample_guids_field=False,
+def _get_json_for_individuals(individuals, user=None, project_guid=None,
add_hpo_details=False, is_analyst=None, has_case_review_perm=False):
additional_model_fields = _get_case_review_fields(individuals.model, has_case_review_perm)
nested_fields = [
@@ -284,11 +284,6 @@ def _get_json_for_individuals(individuals, user=None, project_guid=None, add_sam
'paternalId': F('father__individual_id'),
'displayName': INDIVIDUAL_DISPLAY_NAME_EXPR,
}
- if add_sample_guids_field:
- additional_values.update({
- f'{field}Guids': ArrayAgg(f'{field.lower()}__guid', filter=Q(**{f'{field.lower()}__isnull': False}))
- for field in ['sample', 'igvSample']
- })
parsed_individuals = get_json_for_queryset(
individuals, user=user, is_analyst=is_analyst, additional_values=additional_values,
@@ -339,6 +334,14 @@ def get_json_for_samples(samples, **kwargs):
return get_json_for_queryset(samples, **_get_sample_json_kwargs(**kwargs))
+def get_json_for_datasets(datasets, project_guid):
+ return get_json_for_queryset(datasets, additional_values={
+ 'projectGuid': Value(project_guid),
+ 'activeIndividuals': ArrayAgg('active_individuals__guid', distinct=True, filter=Q(active_individuals__isnull=False)),
+ 'inactiveIndividuals': ArrayAgg('inactive_individuals__guid', distinct=True, filter=Q(inactive_individuals__isnull=False)),
+ })
+
+
def get_json_for_sample(sample, **kwargs):
"""Returns a JSON representation of the given Sample.
diff --git a/seqr/views/utils/orm_to_json_utils_tests.py b/seqr/views/utils/orm_to_json_utils_tests.py
index f5a68c23a0..8f344bb717 100644
--- a/seqr/views/utils/orm_to_json_utils_tests.py
+++ b/seqr/views/utils/orm_to_json_utils_tests.py
@@ -1,12 +1,12 @@
from django.contrib.auth.models import User
import mock
from copy import deepcopy
-from seqr.models import Project, Sample, IgvSample, SavedVariant, VariantNote, LocusList, VariantSearch
+from seqr.models import Project, IgvSample, SavedVariant, VariantNote, LocusList, VariantSearch
from seqr.views.utils.orm_to_json_utils import get_json_for_user, _get_json_for_project, \
get_json_for_sample, get_json_for_variant_note, get_json_for_locus_list, \
get_json_for_saved_searches, get_json_for_saved_variants_with_tags, get_json_for_current_user
from seqr.views.utils.test_utils import AuthenticationTestCase, AnvilAuthenticationTestCase, \
- PROJECT_FIELDS, SAMPLE_FIELDS, SAVED_VARIANT_FIELDS, \
+ PROJECT_FIELDS, SAVED_VARIANT_FIELDS, \
FUNCTIONAL_FIELDS, SAVED_SEARCH_FIELDS, LOCUS_LIST_DETAIL_FIELDS, PA_LOCUS_LIST_FIELDS, IGV_SAMPLE_FIELDS, \
TAG_FIELDS, VARIANT_NOTE_FIELDS
@@ -95,12 +95,6 @@ def test_json_for_project(self):
self.assertSetEqual(set(json.keys()), PROJECT_FIELDS)
- def test_json_for_sample(self):
- sample = Sample.objects.first()
- json = get_json_for_sample(sample)
-
- self.assertSetEqual(set(json.keys()), SAMPLE_FIELDS)
-
def test_json_for_igv_sample(self):
sample = IgvSample.objects.first()
json = get_json_for_sample(sample)
diff --git a/seqr/views/utils/pedigree_info_utils.py b/seqr/views/utils/pedigree_info_utils.py
index f29f8a2cc1..b87573e76a 100644
--- a/seqr/views/utils/pedigree_info_utils.py
+++ b/seqr/views/utils/pedigree_info_utils.py
@@ -15,7 +15,7 @@
from seqr.utils.middleware import ErrorsWarningsException
from seqr.views.utils.json_utils import _to_snake_case, _to_title_case
from seqr.views.utils.permissions_utils import user_is_pm, get_pm_user_emails
-from seqr.models import Individual, Sample
+from seqr.models import Individual
logger = SeqrLogger(__name__)
@@ -361,22 +361,24 @@ def get_validated_related_individuals(project, records_by_id, errors, related_gu
affected = records_by_id[individual_id].get(JsonConstants.AFFECTED_COLUMN, Individual.AFFECTED_STATUS_UNKNOWN)
affected_status_by_family[family_id].append(affected)
- search_samples = Sample.objects.filter(individual__family__project=project, is_active=True)
+ search_individuals = Individual.objects.filter(family__project=project)
sample_type = None
if search_dataset_type:
- search_samples = search_samples.filter(dataset_type=search_dataset_type)
+ search_individuals = search_individuals.filter(active_datasets__dataset_type=search_dataset_type)
if search_sample_type:
- search_samples = search_samples.filter(sample_type=search_sample_type)
- elif search_samples:
- sample_type = search_samples.first().sample_type
+ search_individuals = search_individuals.filter(active_datasets__sample_type=search_sample_type)
+ elif search_individuals:
+ sample_type = search_individuals.first().active_datasets.first().sample_type
+ else:
+ search_individuals = search_individuals.filter(active_datasets__dataset_type__isnull=False)
previous_loaded_individuals = {
i[JsonConstants.INDIVIDUAL_ID_COLUMN]: i
- for i in search_samples.values(
- 'individual_id', **{
- JsonConstants.INDIVIDUAL_ID_COLUMN: F('individual__individual_id'),
- JsonConstants.FAMILY_ID_COLUMN: F('individual__family__family_id'),
- })
+ for i in search_individuals.values(
+ **{
+ JsonConstants.INDIVIDUAL_ID_COLUMN: F('individual_id'),
+ JsonConstants.FAMILY_ID_COLUMN: F('family__family_id'),
+ }).annotate(individual_id=F('id'))
}
if validate_expected_samples:
diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py
index a8af246857..02335af03e 100644
--- a/seqr/views/utils/test_utils.py
+++ b/seqr/views/utils/test_utils.py
@@ -703,9 +703,8 @@ def _get_list_param(call, param):
INTERNAL_INDIVIDUAL_FIELDS = deepcopy(NO_INTERNAL_CASE_REVIEW_INDIVIDUAL_FIELDS)
INTERNAL_INDIVIDUAL_FIELDS.update(CORE_INTERNAL_INDIVIDUAL_FIELDS)
-SAMPLE_FIELDS = {
- 'projectGuid', 'familyGuid', 'individualGuid', 'sampleGuid', 'createdDate', 'sampleType', 'sampleId', 'isActive',
- 'loadedDate', 'datasetType',
+DATASET_FIELDS = {
+ 'projectGuid', 'datasetGuid', 'sampleType', 'loadedDate', 'datasetType', 'activeIndividuals', 'inactiveIndividuals',
}
IGV_SAMPLE_FIELDS = {
diff --git a/seqr/views/utils/variant_utils.py b/seqr/views/utils/variant_utils.py
index 4446836be6..19088364e6 100644
--- a/seqr/views/utils/variant_utils.py
+++ b/seqr/views/utils/variant_utils.py
@@ -12,7 +12,7 @@
from matchmaker.models import MatchmakerSubmissionGenes, MatchmakerSubmission
from reference_data.models import TranscriptInfo, Omim, GENOME_VERSION_GRCh38
from seqr.models import SavedVariant, VariantSearchResults, Family, LocusList, LocusListInterval, LocusListGene, \
- RnaSeqTpm, PhenotypePrioritization, Project, Sample, RnaSample, VariantTag, VariantTagType
+ RnaSeqTpm, PhenotypePrioritization, Project, Dataset, RnaSample, VariantTag, VariantTagType
from seqr.utils.gene_utils import get_genes_for_variants
from seqr.utils.xpos_utils import parse_variant_id
from seqr.views.utils.json_to_orm_utils import create_model_from_json
@@ -65,9 +65,9 @@ def parse_saved_variant_json(variant_json, family_id):
def variant_dataset_type(variant):
if not parse_variant_id(variant['variantId']):
- sample_type = Sample.SAMPLE_TYPE_WGS if 'endChrom' in variant else Sample.SAMPLE_TYPE_WES
- return f'{Sample.DATASET_TYPE_SV_CALLS}_{sample_type}'
- return Sample.DATASET_TYPE_MITO_CALLS if 'mitomapPathogenic' in variant else Sample.DATASET_TYPE_VARIANT_CALLS
+ sample_type = Dataset.SAMPLE_TYPE_WGS if 'endChrom' in variant else Dataset.SAMPLE_TYPE_WES
+ return f'{Dataset.DATASET_TYPE_SV_CALLS}_{sample_type}'
+ return Dataset.DATASET_TYPE_MITO_CALLS if 'mitomapPathogenic' in variant else Dataset.DATASET_TYPE_VARIANT_CALLS
def _transcript_sort(gene_id, saved_variant_json, main_transcript_id):
@@ -178,7 +178,7 @@ def _set_updated_tags(key: tuple[int, str], metadata: dict[str, dict], comp_het_
def get_saved_variant_annotations(variant_keys: abc.Iterable[tuple[int, str]], genome_version: str, primary_id_field: str = 'variant_id', group_by_field=None, dataset_type: str = None) -> dict[str, dict]:
- dataset_type = dataset_type or Sample.DATASET_TYPE_VARIANT_CALLS
+ dataset_type = dataset_type or Dataset.DATASET_TYPE_VARIANT_CALLS
variant_ids = {variant_id for _, variant_id in variant_keys}
keys = None
if primary_id_field == 'key':
@@ -497,9 +497,9 @@ def _set_response_gene_scores(response, family_genes, gene_ids):
def _add_sample_count_stats(response, genome_versions):
- sample_counts = Sample.objects.filter(
- is_active=True, individual__family__project__is_demo=False, individual__family__project__genome_version__in=genome_versions,
- ).values('sample_type', 'dataset_type').annotate(count=Count('*'))
+ sample_counts = Dataset.objects.filter(
+ active_individuals__family__project__is_demo=False, active_individuals__family__project__genome_version__in=genome_versions,
+ ).values('sample_type', 'dataset_type').annotate(count=Count('active_individuals', distinct=True))
counts_by_dataset_type = defaultdict(dict)
for sample_type, dataset_type, count in sample_counts.values_list('sample_type', 'dataset_type', 'count'):
counts_by_dataset_type[dataset_type][sample_type] = count
diff --git a/ui/pages/Project/components/ProjectOverview.jsx b/ui/pages/Project/components/ProjectOverview.jsx
index 82ef203dad..be47568655 100644
--- a/ui/pages/Project/components/ProjectOverview.jsx
+++ b/ui/pages/Project/components/ProjectOverview.jsx
@@ -29,6 +29,7 @@ import {
getProjectAnalysisGroupFamilySizeHistogram,
getProjectAnalysisGroupDataLoadedFamilySizeHistogram,
getProjectAnalysisGroupSamplesByTypes,
+ getProjectAnalysisGroupRnaSamplesByTypes,
getProjectAnalysisGroupMmeSubmissionDetails,
getMmeSubmissionsLoading,
} from '../selectors'
@@ -303,15 +304,19 @@ class DatasetSection extends React.PureComponent {
}
-const Dataset = React.memo(({ showLoadWorkspaceData, hasAnvil, samplesByType, user }) => {
- const datasetSections = samplesByType.map(([sampleTypeKey, loadedSampleCounts]) => {
+const Dataset = React.memo(({ showLoadWorkspaceData, hasAnvil, samplesByType, rnaSamplesByType, user }) => {
+ const datasetSections = [...Object.entries(samplesByType).map(([sampleTypeKey, loadedSampleCounts]) => {
const [sampleType, datasetType] = sampleTypeKey.split('__')
return {
key: sampleTypeKey,
title: `${SAMPLE_TYPE_LOOKUP[sampleType] || sampleType}${DATASET_TITLE_LOOKUP[datasetType] || ''} Datasets`,
content: ,
}
- }).sort((a, b) => a.title.localeCompare(b.title))
+ }), ...rnaSamplesByType.map(([dataType, loadedSampleCounts]) => ({
+ key: dataType,
+ title: `RNA${DATASET_TITLE_LOOKUP[dataType]} Datasets`,
+ content: ,
+ }))].sort((a, b) => a.title.localeCompare(b.title))
const noLoadedData = !datasetSections.length
if (noLoadedData) {
@@ -358,6 +363,7 @@ const Dataset = React.memo(({ showLoadWorkspaceData, hasAnvil, samplesByType, us
Dataset.propTypes = {
samplesByType: PropTypes.object.isRequired,
+ rnaSamplesByType: PropTypes.arrayOf(PropTypes.any).isRequired,
hasAnvil: PropTypes.bool,
showLoadWorkspaceData: PropTypes.bool,
user: PropTypes.object.isRequired,
@@ -366,6 +372,7 @@ Dataset.propTypes = {
const mapDatasetStateToProps = (state, ownProps) => ({
user: getUser(state),
samplesByType: getProjectAnalysisGroupSamplesByTypes(state, ownProps),
+ rnaSamplesByType: getProjectAnalysisGroupRnaSamplesByTypes(state, ownProps),
})
const DatasetOverview = connect(mapDatasetStateToProps)(Dataset)
diff --git a/ui/pages/Project/fixtures.js b/ui/pages/Project/fixtures.js
index 033fc74b57..6689bcb1e0 100644
--- a/ui/pages/Project/fixtures.js
+++ b/ui/pages/Project/fixtures.js
@@ -121,7 +121,7 @@ export const STATE1 = {
sex: 'M',
},
},
- samplesByGuid: {},
+ datasetsByGuid: {},
mmeSubmissionsByGuid: {},
project: {
createdDate: '2016-05-16T05:37:08.634Z',
@@ -310,19 +310,14 @@ export const STATE_WITH_2_FAMILIES = {
sex: 'M',
},
},
- samplesByGuid: {
+ datasetsByGuid: {
S2310656_wal_mc16200_mc16203: {
- createdDate: "2018-03-30T11:50:40.079Z",
- datasetFilePath: "gs://seqr-datasets/GRCh37/cmg_sankaran_wes/CMG_MYOSEQ.vcf.gz",
- datasetName: null,
datasetType: "SNV_INDEL",
- familyGuid: 'F011652_2',
- individualGuid: "I021476_na19678_2",
+ activeIndividuals: ["I021476_na19678_2"],
+ inactiveIndividuals: [],
loadedDate: "2018-03-13T13:25:21.551Z",
projectGuid: "R0237_1000_genomes_demo",
- sampleGuid: "S2310656_wal_mc16200_mc16203",
- sampleId: "WAL_MC16200_MC16203",
- isActive: true,
+ datasetGuid: "S2310656_wal_mc16200_mc16203",
sampleType: "WES",
},
},
diff --git a/ui/pages/Project/reducers.js b/ui/pages/Project/reducers.js
index c2181b8dfe..9d0d355724 100644
--- a/ui/pages/Project/reducers.js
+++ b/ui/pages/Project/reducers.js
@@ -56,11 +56,8 @@ export const loadMmeSubmissions = () => loadCurrentProjectChildEntities('mme sub
const loadFamilyNotes = () => loadCurrentProjectChildEntities('family notes', REQUEST_FAMILIES, RECEIVE_FAMILIES)
-const loadSamples = () => loadCurrentProjectChildEntities('samples', REQUEST_INDIVIDUALS)
-
export const loadProjectExportData = () => (dispatch, getState) => Promise.all([
loadIndividuals()(dispatch, getState),
- loadSamples()(dispatch, getState),
loadFamilyNotes()(dispatch, getState),
])
diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js
index 7b5b560511..9d533feb31 100644
--- a/ui/pages/Project/selectors.js
+++ b/ui/pages/Project/selectors.js
@@ -15,6 +15,7 @@ import {
SIMPLIFIED_SEX_LOOKUP,
} from 'shared/utils/constants'
import { toCamelcase, toSnakecase, snakecaseToTitlecase } from 'shared/utils/stringUtils'
+import { compareObjects } from 'shared/utils/sortUtils'
import {
getProjectsByGuid, getFamiliesGroupedByProjectGuid, getIndividualsByGuid, getGenesById, getUser,
@@ -23,7 +24,7 @@ import {
getVariantTagsByGuid, getUserOptionsByUsername, getNotesByFamilyType,
getVariantTagNotesByFamilyVariants, getPhenotypeGeneScoresByIndividual, getActiveDatasetsByIndividual,
getRnaSeqDataByIndividual, familyPassesFilters, getAnalysisGroupGuid, getCurrentAnalysisGroupFamilyGuids,
- getDatasetsByIndividual, getActiveDatasetsByFamily, getMinMaxDatasetsByFamily,
+ getDatasetsByIndividual, getActiveDatasetsByFamily, getMinMaxDatasetsByFamily, getDatasetsGroupedByProjectGuid,
} from 'redux/selectors'
import {
@@ -170,9 +171,32 @@ export const getProjectAnalysisGroupIndividualsByGuid = createSelector(
)
export const getProjectAnalysisGroupSamplesByTypes = createSelector(
+ getProjectGuid,
+ getCurrentAnalysisGroupFamilyGuids,
+ getProjectAnalysisGroupIndividualsByGuid,
+ getDatasetsGroupedByProjectGuid,
+ (projectGuid, analysisGroupFamilyGuids, analysisGroupIndividuals, datasetsByProject) => Object.values(
+ datasetsByProject[projectGuid] || {},
+ ).sort(compareObjects('loadedDate')).reduce((acc, { sampleType, datasetType, loadedDate, activeIndividuals, inactiveIndividuals }) => {
+ const key = `${sampleType}__${datasetType}`
+ if (!acc[key]) {
+ acc[key] = []
+ }
+ const individualGuids = [...(activeIndividuals || []), ...(inactiveIndividuals || [])]
+ const count = !analysisGroupFamilyGuids ? individualGuids.length : individualGuids.filter(
+ individualGuid => individualGuid in analysisGroupIndividuals,
+ ).length
+ if (count) {
+ acc[key].push({ loadedDate, count })
+ }
+ return acc
+ }, {}),
+)
+
+export const getProjectAnalysisGroupRnaSamplesByTypes = createSelector(
getCurrentProject,
getCurrentAnalysisGroupFamilyGuids,
- (project, analysisGroupFamilyGuids) => Object.entries(project.sampleCounts || {}).map(
+ (project, analysisGroupFamilyGuids) => Object.entries(project.rnaSampleCounts || {}).map(
([key, typeCounts]) => ([key, typeCounts.map(({ familyCounts, ...data }) => ({
...data,
count: Object.entries(familyCounts).reduce((total, [familyGuid, count]) => (
diff --git a/ui/pages/Search/fixtures.js b/ui/pages/Search/fixtures.js
index 49bbe7a4f2..013130e72e 100644
--- a/ui/pages/Search/fixtures.js
+++ b/ui/pages/Search/fixtures.js
@@ -158,59 +158,33 @@ export const STATE = {
paternalId: '',
},
},
- samplesByGuid: {
+ datasetsByGuid: {
S2310658_wal_mc16200_mc16203: {
- createdDate: "2018-03-30T11:50:40.079Z",
- elasticsearchIndex: "CMG_MYOSEQ",
datasetType: "SNV_INDEL",
- individualGuid: "I021476_na19678",
+ activeIndividuals: ["I021476_na19678"],
+ inactiveIndividuals: [],
loadedDate: "2018-03-13T13:25:21.551Z",
projectGuid: PROJECT_GUID,
- familyGuid: FAMILY_GUID,
- sampleGuid: "S2310656_wal_mc16200_mc16203",
- sampleId: "WAL_MC16200_MC16203",
- isActive: true,
+ datasetGuid: "S2310658_wal_mc16200_mc16203",
sampleType: "WES",
},
S2310657_wal_mc16200_mc16203: {
- createdDate: "2018-03-30T11:50:40.079Z",
- elasticsearchIndex: "CMG_MYOSEQ",
datasetType: "SV",
- individualGuid: "I021476_na19678",
+ activeIndividuals: ["I021476_na19678"],
+ inactiveIndividuals: null,
loadedDate: "2018-03-13T13:25:21.551Z",
projectGuid: PROJECT_GUID,
- sampleGuid: "S2310656_wal_mc16200_mc16203",
- sampleId: "WAL_MC16200_MC16203",
- isActive: true,
+ datasetGuid: "S2310657_wal_mc16200_mc16203",
sampleType: "WES",
},
S2310656_wal_mc16200_mc16203: {
- createdDate: "2018-03-30T11:50:40.079Z",
- elasticsearchIndex: "CMG_MYOSEQ",
datasetType: "SNV_INDEL",
- individualGuid: "I021476_na19678",
+ inactiveIndividuals: ["I021476_na19678"],
loadedDate: "2018-03-13T13:25:21.551Z",
projectGuid: PROJECT_GUID,
- familyGuid: FAMILY_GUID,
- sampleGuid: "S2310656_wal_mc16200_mc16203",
- sampleId: "WAL_MC16200_MC16203",
- isActive: false,
+ datasetGuid: "S2310656_wal_mc16200_mc16203",
sampleType: "WES",
},
- S2310659_wal_mc16200_mc16203: {
- createdDate: "2018-03-30T11:50:40.079Z",
- elasticsearchIndex: null,
- datasetName: null,
- datasetType: "SNV_INDEL",
- individualGuid: "I021476_na19678",
- loadedDate: "2018-03-13T13:25:21.551Z",
- projectGuid: PROJECT_GUID,
- familyGuid: FAMILY_GUID,
- sampleGuid: "S2310656_wal_mc16200_mc16203",
- sampleId: "WAL_MC16200_MC16203",
- isActive: true,
- sampleType: "RNA",
- },
},
analysisGroupsByGuid: {
[ANALYSIS_GROUP_GUID]: {
diff --git a/ui/redux/rootReducer.js b/ui/redux/rootReducer.js
index 99bf5f2136..6b694015bb 100644
--- a/ui/redux/rootReducer.js
+++ b/ui/redux/rootReducer.js
@@ -262,7 +262,7 @@ const rootReducer = combineReducers({
familyNotesByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'familyNotesByGuid'),
familyDetailsLoading: createSingleObjectReducer(REQUEST_FAMILY_DETAILS),
individualsByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'individualsByGuid'),
- samplesByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'samplesByGuid'),
+ datasetsByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'datasetsByGuid'),
igvSamplesByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'igvSamplesByGuid'),
analysisGroupsByGuid: createObjectsByIdReducer(RECEIVE_DATA, 'analysisGroupsByGuid'),
analysisGroupsLoading: loadingReducer(REQUEST_ANALYSIS_GROUPS, RECEIVE_ANALYSIS_GROUPS),
diff --git a/ui/redux/selectors.js b/ui/redux/selectors.js
index 97ef046a96..e9b293b467 100644
--- a/ui/redux/selectors.js
+++ b/ui/redux/selectors.js
@@ -11,7 +11,7 @@ export const getFamiliesByGuid = state => state.familiesByGuid
export const getFamilyNotesByGuid = state => state.familyNotesByGuid
export const getFamilyDetailsLoading = state => state.familyDetailsLoading
export const getIndividualsByGuid = state => state.individualsByGuid
-const getSamplesByGuid = state => state.samplesByGuid
+const getDatasetsByGuid = state => state.datasetsByGuid
export const getIgvSamplesByGuid = state => state.igvSamplesByGuid
export const getAnalysisGroupsByGuid = state => state.analysisGroupsByGuid
export const getAnalysisGroupIsLoading = state => state.analysisGroupsLoading.isLoading
@@ -58,6 +58,7 @@ const groupEntitiesByProjectGuid = entities => Object.entries(entities).reduce((
}, {})
export const getFamiliesGroupedByProjectGuid = createSelector(getFamiliesByGuid, groupEntitiesByProjectGuid)
export const getAnalysisGroupsGroupedByProjectGuid = createSelector(getAnalysisGroupsByGuid, groupEntitiesByProjectGuid)
+export const getDatasetsGroupedByProjectGuid = createSelector(getDatasetsByGuid, groupEntitiesByProjectGuid)
const groupByFamilyGuid = objs => objs.reduce((acc, o) => {
if (!acc[o.familyGuid]) {
@@ -69,14 +70,14 @@ const groupByFamilyGuid = objs => objs.reduce((acc, o) => {
export const getProjectDatasetTypes = createSelector(
getProjectsByGuid,
- getSamplesByGuid,
- (projectsByGuid, samplesByGuid) => {
+ getDatasetsByGuid,
+ (projectsByGuid, datasetsByGuid) => {
const projectDatasetTypes = Object.values(projectsByGuid).reduce(
(acc, { projectGuid, datasetTypes }) => ({ ...acc, [projectGuid]: datasetTypes }), {},
)
- const sampleDatasetTypes = Object.values(samplesByGuid).reduce(
- (acc, { projectGuid, datasetType, isActive }) => {
- if (projectDatasetTypes[projectGuid] || !isActive) {
+ const sampleDatasetTypes = Object.values(datasetsByGuid).reduce(
+ (acc, { projectGuid, datasetType, activeIndividuals }) => {
+ if (projectDatasetTypes[projectGuid] || !(activeIndividuals || []).length) {
return acc
}
if (!acc[projectGuid]) {
@@ -91,15 +92,25 @@ export const getProjectDatasetTypes = createSelector(
)
export const getDatasetsByIndividual = createSelector(
- getSamplesByGuid,
- samplesByGuid => Object.values(samplesByGuid).sort(
+ getDatasetsByGuid,
+ datasetsByGuid => Object.values(datasetsByGuid).sort(
(a, b) => a.loadedDate.localeCompare(b.loadedDate),
- ).reduce((acc, sample) => {
- const { individualGuid, isActive, sampleType, datasetType, loadedDate } = sample
- if (!acc[individualGuid]) {
- acc[individualGuid] = []
- }
- acc[individualGuid].push({ isActive, sampleType, datasetType, loadedDate: loadedDate.split('T')[0] })
+ ).reduce((acc, { activeIndividuals, inactiveIndividuals, sampleType, datasetType, loadedDate }) => {
+ const parsedLoadedDate = loadedDate.split('T')[0]
+ const activeIndivs = activeIndividuals || []
+ activeIndivs.forEach((individualGuid) => {
+ if (!acc[individualGuid]) {
+ acc[individualGuid] = []
+ }
+ acc[individualGuid].push({ isActive: true, sampleType, datasetType, loadedDate: parsedLoadedDate })
+ })
+ const inactiveIndivs = inactiveIndividuals || []
+ inactiveIndivs.forEach((individualGuid) => {
+ if (!acc[individualGuid]) {
+ acc[individualGuid] = []
+ }
+ acc[individualGuid].push({ isActive: false, sampleType, datasetType, loadedDate: parsedLoadedDate })
+ })
return acc
}, {}),
)
diff --git a/ui/shared/components/panel/fixtures.js b/ui/shared/components/panel/fixtures.js
index 4bca6989a7..f05658a765 100644
--- a/ui/shared/components/panel/fixtures.js
+++ b/ui/shared/components/panel/fixtures.js
@@ -374,7 +374,7 @@ export const STATE1 = {
},
},
analysisGroupsByGuid: {},
- samplesByGuid: {},
+ datasetsByGuid: {},
igvSamplesByGuid: {
S2310656_wal_mc16200_mc16203: {
projectGuid: 'R0237_1000_genomes_demo',