diff --git a/CHANGELOG.md b/CHANGELOG.md index bc3a6933b..ecfbd2a9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,7 @@ DecodingParameters().alter() - Remove pre-existing `Units` from created analysis nwb files #1453 - Allow multiple VideoFile entries during ingestion #1462 - Handle epoch formats with varying zero-padding #1459 + - Reduce lock conflicts between users during ingestion #1483 - Decoding - Ensure results directory is created if it doesn't exist #1362 - Change BLOB fields to LONGBLOB in DecodingParameters #1463 diff --git a/src/spyglass/utils/mixins/ingestion.py b/src/spyglass/utils/mixins/ingestion.py index 6956a7eda..aa94370ac 100644 --- a/src/spyglass/utils/mixins/ingestion.py +++ b/src/spyglass/utils/mixins/ingestion.py @@ -253,11 +253,15 @@ def insert_from_nwbfile( # validate that new entries are consistent with existing entries if self._expected_duplicates: - self.validate_duplicates(entries) + entries_to_insert = self.validate_duplicates(entries) + else: + entries_to_insert = entries # run insertions if not dry_run: - self._run_nwbfile_insert(entries, nwb_file_name=nwb_file_name) + self._run_nwbfile_insert( + entries_to_insert, nwb_file_name=nwb_file_name + ) return entries @@ -279,7 +283,7 @@ def expect_dupes(tbl): for table, table_entries in entries.items(): table.insert( table_entries, - skip_duplicates=expect_dupes(table), + skip_duplicates=False, allow_direct_insert=True, ) self._insert_logline(nwb_file_name, len(table_entries), table) @@ -348,10 +352,23 @@ def validate_duplicates(self, entry_dict: Dict[dj.Table, List[dict]]): entry_dict : dict or Dict[dj.Table, List[dict]] The new entry or dict of table entries to validate against existing entries in the database. + + Returns + ------- + dict or Dict[dj.Table, List[dict]] + The new entries to insert after validation. Avoids need to flag + skip_duplicates """ + entries_to_insert = dict() for table, table_entries in entry_dict.items(): - for entry in self._adjust_keys_for_entry(table_entries): + entries_to_insert[table] = [] + for entry, table_entry in zip( + self._adjust_keys_for_entry(table_entries), table_entries + ): self.validate1_duplicate(table, entry) + if not (table & entry): + entries_to_insert[table].append(table_entry) + return entries_to_insert def validate1_duplicate(self, tbl, new_key): """If matching primary key, check for consistency in secondary keys.