Repository navigation
Expand file tree
/
Copy pathextract_human_interpro.py
More file actions
234 lines (204 loc) · 8.27 KB
/
Copy pathextract_human_interpro.py
File metadata and controls
234 lines (204 loc) · 8.27 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
#!/usr/bin/env python3
"""
Extract a species' protein annotations from the full protein2ipr.dat.gz file.
This creates a much smaller species-specific file for faster processing. The
name is kept for backward compatibility, but the script works for any organism
via ``--species`` (default: human), mirroring ``run_dcgo_human.py``.
uv run python extract_human_interpro.py # human (default)
uv run python extract_human_interpro.py --species mouse # any organism
A subset for a temporal benchmark is cut from an archived protein2ipr and
selected by the union of several GAFs — the training snapshot *and* the later
one whose newly annotated proteins are scored, so both have architectures:
uv run python extract_human_interpro.py \
--source data/raw/interpro_archive/85.0/protein2ipr.dat.gz \
--gaf data/raw/goa_archive/goa_human.gaf.205.gz \
--gaf data/raw/goa_annotations/goa_human.gaf.gz \
--evidence-filter all \
--output data/interim/protein2ipr_human_t0_ipr85.dat.gz
"""
import argparse
import gzip
from pathlib import Path
from loguru import logger
import sys
logger.remove()
logger.add(sys.stderr, level="INFO")
def extract_species_proteins(
protein_set_file: Path, interpro_file: Path, output_file: Path
) -> tuple[int, int]:
"""
Extract lines from protein2ipr.dat for specific proteins.
Args:
protein_set_file: Text file with one protein ID per line
interpro_file: Full protein2ipr.dat.gz file
output_file: Output file (will be gzipped)
Returns:
``(n_selecting_accessions, n_matched_lines)``, for the caller's
provenance marker (see :mod:`src.universe_provenance`).
"""
# Load protein set
logger.info(f"Loading protein IDs from {protein_set_file}")
with open(protein_set_file) as f:
protein_ids = set(line.strip() for line in f if line.strip())
logger.info(f" Loaded {len(protein_ids):,} protein IDs")
# Extract matching lines
logger.info(f"Extracting annotations from {interpro_file}")
logger.info(" This will take ~10 minutes to scan the 20GB file")
matched_lines = 0
total_lines = 0
with gzip.open(interpro_file, "rt") as fin, gzip.open(output_file, "wt") as fout:
for line in fin:
total_lines += 1
if total_lines % 10000000 == 0:
logger.info(
f" Processed {total_lines:,} lines, found {matched_lines:,} matches"
)
# Quick check: does the line start with any of our protein IDs?
protein_id = line.split("\t")[0] if "\t" in line else ""
if protein_id in protein_ids:
fout.write(line)
matched_lines += 1
logger.info("✓ Extraction complete!")
logger.info(f" Total lines scanned: {total_lines:,}")
logger.info(f" Matching lines found: {matched_lines:,}")
logger.info(f" Output file: {output_file}")
logger.info(f" Output size: {output_file.stat().st_size / 1e6:.1f} MB")
return len(protein_ids), matched_lines
def main():
parser = argparse.ArgumentParser(
description="Extract a species' subset of protein2ipr.dat.gz for a fast dcGO run."
)
parser.add_argument(
"--species",
default="human",
help="Species to extract: 'human', 'mouse', etc. Must match the "
"goa_<species>.gaf.gz file name (default: human)",
)
parser.add_argument(
"--evidence-filter",
default="manual",
choices=["all", "manual", "experimental"],
help="Evidence filter used to select the protein set (default: manual)",
)
parser.add_argument(
"--gaf",
type=Path,
action="append",
help="GAF whose proteins select the subset; repeat to take the union "
"(default: data/raw/goa_annotations/goa_<species>.gaf.gz)",
)
parser.add_argument(
"--source",
type=Path,
default=None,
help="protein2ipr file to filter, e.g. an archived release "
"(default: data/raw/interpro_mappings/protein2ipr.dat.gz)",
)
parser.add_argument(
"--output",
type=Path,
default=None,
help="Output path (default: data/interim/protein2ipr_<species>.dat.gz). "
"The selecting accession list is written beside it. Required with "
"--source or --gaf",
)
parser.add_argument(
"--force",
action="store_true",
help="Overwrite an existing extract even if its provenance marker "
"records a different selection rule (e.g. an idmapping-selected "
"extract from scripts/extract_species_interpro.py)",
)
args = parser.parse_args()
# The default extract is every run's and evaluator's default --interpro;
# a subset of another release or selection must not replace it unasked.
if (args.source or args.gaf) and not args.output:
parser.error("--source and --gaf require an explicit --output")
# Parse GOA to get this species' protein IDs
from src.goa_parser import parse_goa
from src.run_manifest import sha256_file
from src.universe_provenance import (
ProvenanceConflictError,
ensure_overwrite_allowed,
write_marker,
)
logger.info(f"Step 1: Parsing GOA to get {args.species} protein IDs...")
goa_files = args.gaf or [
Path(f"data/raw/goa_annotations/goa_{args.species}.gaf.gz")
]
for goa_file in goa_files:
if not goa_file.exists():
logger.error(f"GOA file not found: {goa_file}")
logger.error(
f"Download it first, e.g. "
f"uv run python scripts/download_data.py --species {args.species}"
)
return 1
protein_ids: set[str] = set()
for goa_file in goa_files:
protein_ids.update(
parse_goa(
goa_file, evidence_filter=args.evidence_filter, aspects={"P", "F", "C"}
)
)
output_file = args.output or Path(f"data/interim/protein2ipr_{args.species}.dat.gz")
# Beside a custom --output, so a second subset never overwrites the
# species' own list.
protein_list_file = (
output_file.with_name(
output_file.name.removesuffix(".dat.gz") + "_proteins.txt"
)
if args.output
else Path(f"data/interim/{args.species}_proteins.txt")
)
protein_list_file.parent.mkdir(parents=True, exist_ok=True)
logger.info(f"Writing {len(protein_ids):,} protein IDs to {protein_list_file}")
with open(protein_list_file, "w") as f:
for protein_id in sorted(protein_ids):
f.write(f"{protein_id}\n")
# Extract species annotations
logger.info("")
logger.info(
f"Step 2: Extracting {args.species} protein annotations from InterPro..."
)
interpro_file = args.source or Path("data/raw/interpro_mappings/protein2ipr.dat.gz")
if not interpro_file.exists():
logger.error(f"InterPro mappings file not found: {interpro_file}")
logger.error(
"Download it first: uv run python scripts/download_data.py "
"--datasets interpro_mappings"
)
return 1
# An extract selected by a different rule (idmapping / accession list, via
# scripts/extract_species_interpro.py) must not be silently clobbered.
try:
ensure_overwrite_allowed(output_file, "goa", force=args.force)
except ProvenanceConflictError as exc:
logger.error(str(exc))
return 1
n_accessions, n_matched = extract_species_proteins(
protein_list_file, interpro_file, output_file
)
write_marker(
output_file,
selection_rule="goa",
selection_sources=goa_files,
selection_sha256=[sha256_file(goa_file) for goa_file in goa_files],
interpro_source=interpro_file,
evidence_filter=args.evidence_filter,
n_accessions=n_accessions,
n_matched_lines=n_matched,
tool="extract_human_interpro.py",
)
logger.info("")
logger.info("=" * 60)
logger.info(f"SUCCESS! {args.species.title()}-specific InterPro file created")
logger.info("=" * 60)
logger.info(f"You can now use: {output_file}")
logger.info(
f"This file contains only the {len(protein_ids):,} "
f"{args.species} proteins from GOA"
)
return 0
if __name__ == "__main__":
sys.exit(main() or 0)