Skip to content

Commit 20d44d1

Browse files
SHINTEKISHINTEKI
committed
use paginated response format in all interop get mothods
1 parent 19fa4a4 commit 20d44d1

4 files changed

Lines changed: 117 additions & 32 deletions

File tree

‎gene_function/models.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,18 @@ def get_all_strains():
176176
cursor = GenomeInfo.objects.aggregate(pipeline)
177177
return [doc["genome_id"] for doc in cursor if doc.get("genome_id")]
178178

179+
@staticmethod
180+
def get_all_strains_paginated(skip=0, limit=10000):
181+
"""
182+
Return paginated distinct genome_id values.
183+
Uses distinct() for fast retrieval, then slices in Python.
184+
"""
185+
col = GenomeInfo.objects.collection
186+
all_ids = sorted(col.distinct("genome_id"))
187+
total = len(all_ids)
188+
page = all_ids[skip:skip + limit]
189+
return {"strains": page, "total": total}
190+
179191

180192
def get_genome_and_isolation_info_pipeline(genome_match):
181193
return [

‎interop_query/views.py‎

Lines changed: 85 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -15,72 +15,125 @@
1515

1616
@extend_schema(
1717
tags=["Genes"],
18-
summary="List all genes",
19-
description="Return all genes with species and PanKB URLs.",
18+
summary="List all genes (paginated)",
19+
description=(
20+
"Return distinct (gene, species) pairs with PanKB URLs. "
21+
"Supports cursor-based pagination via skip/limit query parameters."
22+
),
23+
parameters=[
24+
OpenApiParameter(name="skip", type=int, location="query", description="Number of records to skip (default 0)"),
25+
OpenApiParameter(name="limit", type=int, location="query", description="Max records to return (default 10000, max 50000)"),
26+
],
2027
responses={
2128
200: {
22-
"type": "array",
23-
"items": {
24-
"type": "object",
25-
"properties": {
26-
"gene": {"type": "string"},
27-
"species": {"type": "string"},
28-
"url": {"type": "string", "format": "uri"},
29+
"type": "object",
30+
"properties": {
31+
"genes": {
32+
"type": "array",
33+
"items": {
34+
"type": "object",
35+
"properties": {
36+
"gene": {"type": "string"},
37+
"species": {"type": "string"},
38+
"url": {"type": "string", "format": "uri"},
39+
},
40+
},
2941
},
42+
"total": {"type": "integer"},
43+
"skip": {"type": "integer"},
44+
"limit": {"type": "integer"},
45+
"has_more": {"type": "boolean"},
3046
},
3147
}
3248
},
3349
)
3450
@api_view(["GET"])
3551
def genes(request):
36-
"""Return all genes with species and PanKB URLs."""
52+
"""Return all genes with species and PanKB URLs (paginated)."""
3753
try:
38-
gene_list = GeneAnnotations.get_all_genes()
39-
result = []
54+
skip = int(request.GET.get("skip", 0))
55+
limit = min(int(request.GET.get("limit", 10000)), 50000)
56+
57+
result = GeneAnnotations.get_all_genes_paginated(skip=skip, limit=limit)
58+
gene_list = result["genes"]
59+
4060
for gene_data in gene_list:
4161
gene = gene_data.get("gene")
4262
species = gene_data.get("pangenome_analysis")
4363
if gene and species:
44-
result.append({
45-
"gene": gene,
46-
"species": species,
47-
"url": f"{settings.PANKB_BASE_URL}/gene_function/gene_info/?species={quote(species)}&gene={quote(gene)}",
48-
})
49-
return Response(result)
64+
gene_data["species"] = species
65+
gene_data["url"] = f"{settings.PANKB_BASE_URL}/gene_function/gene_info/?species={quote(species)}&gene={quote(gene)}"
66+
gene_data.pop("pangenome_analysis", None)
67+
68+
return Response({
69+
"genes": gene_list,
70+
"total": result["total"],
71+
"skip": skip,
72+
"limit": limit,
73+
"has_more": skip + len(gene_list) < result["total"],
74+
})
5075
except Exception as e:
5176
logger.exception("list_all_genes failed")
5277
return Response({"message": f"Error: {e}"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
5378

5479

5580
@extend_schema(
5681
tags=["Strains"],
57-
summary="List all strains",
58-
description="Return all strains (genome IDs) with PanKB URLs.",
82+
summary="List all strains (paginated)",
83+
description=(
84+
"Return all strains (genome IDs) with PanKB URLs. "
85+
"Supports cursor-based pagination via skip/limit query parameters."
86+
),
87+
parameters=[
88+
OpenApiParameter(name="skip", type=int, location="query", description="Number of records to skip (default 0)"),
89+
OpenApiParameter(name="limit", type=int, location="query", description="Max records to return (default 10000, max 50000)"),
90+
],
5991
responses={
6092
200: {
61-
"type": "array",
62-
"items": {
63-
"type": "object",
64-
"properties": {
65-
"strain": {"type": "string"},
66-
"url": {"type": "string", "format": "uri"},
93+
"type": "object",
94+
"properties": {
95+
"strains": {
96+
"type": "array",
97+
"items": {
98+
"type": "object",
99+
"properties": {
100+
"strain": {"type": "string"},
101+
"url": {"type": "string", "format": "uri"},
102+
},
103+
},
67104
},
105+
"total": {"type": "integer"},
106+
"skip": {"type": "integer"},
107+
"limit": {"type": "integer"},
108+
"has_more": {"type": "boolean"},
68109
},
69110
}
70111
},
71112
)
72113
@api_view(["GET"])
73114
def strains(request):
74-
"""Return all strains (genome IDs) with PanKB URLs."""
115+
"""Return all strains (genome IDs) with PanKB URLs (paginated)."""
75116
try:
76-
strain_ids = GenomeInfo.get_all_strains()
77-
result = []
117+
skip = int(request.GET.get("skip", 0))
118+
limit = min(int(request.GET.get("limit", 10000)), 50000)
119+
120+
result = GenomeInfo.get_all_strains_paginated(skip=skip, limit=limit)
121+
strain_ids = result["strains"]
122+
123+
strain_list = []
78124
for strain_id in strain_ids:
79-
result.append({
125+
strain_list.append({
80126
"strain": strain_id,
81127
"url": f"{settings.PANKB_BASE_URL}/gene_function/genome_info/?genome_id={quote(strain_id)}",
82128
})
83-
return Response(result)
129+
130+
return Response({
131+
"strains": strain_list,
132+
"total": result["total"],
133+
"skip": skip,
134+
"limit": limit,
135+
"has_more": skip + len(strain_list) < result["total"],
136+
})
84137
except Exception as e:
85138
logger.exception("list_all_strains failed")
86139
return Response({"message": f"Error: {e}"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
@@ -266,7 +319,7 @@ def query_by_pair(request):
266319
examples=[
267320
OpenApiExample(
268321
"Example request",
269-
value={"ids": ["COQ3_1", "COQ3_2", "COQ3_3"]},
322+
value={"ids": ["AAH1"]},
270323
request_only=True,
271324
)
272325
],

‎nginx/default-prod.conf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,5 +39,8 @@ server {
3939

4040
location / {
4141
proxy_pass http://django;
42+
proxy_connect_timeout 300;
43+
proxy_send_timeout 300;
44+
proxy_read_timeout 300;
4245
}
4346
}

‎pangenome_analyses/models.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,23 @@ def get_all_genes():
2929
]
3030
return result
3131

32+
@staticmethod
33+
def get_all_genes_paginated(skip=0, limit=10000):
34+
"""
35+
Return paginated (gene, pangenome_analysis) pairs.
36+
Uses indexed find+sort+skip+limit for performance on large collections.
37+
"""
38+
col = GeneAnnotations.objects.collection
39+
cursor = col.find(
40+
{"gene": {"$ne": None}, "pangenome_analysis": {"$ne": None}},
41+
{"_id": 0, "gene": 1, "pangenome_analysis": 1},
42+
).sort([("pangenome_analysis", 1), ("gene", 1)]).skip(skip).limit(limit)
43+
genes = list(cursor)
44+
45+
total = col.estimated_document_count()
46+
47+
return {"genes": genes, "total": total}
48+
3249
def get_gene_analysis_pairs(genes):
3350
"""
3451
Return all distinct (gene, pangenome_analysis) tuples

0 commit comments

Comments
 (0)