Skip to content

Commit 7019269

Browse files
mdrxymdrxy
authored
fix(model-profiles): sort generated profiles by model ID for stable diffs (langchain-ai#35344)
- Sort model profiles alphabetically by model ID (the top-level `_PROFILES` dictionary keys, e.g. `claude-3-5-haiku-20241022`, `gpt-4o-mini`) before writing `_profiles.py`, so that regenerating profiles only shows actual data changes in diffs — not random reordering from the models.dev API response order - Regenerate all 10 partner profile files with the new sorted ordering
1 parent 8aa7a3f commit 7019269

12 files changed

Lines changed: 1520 additions & 1433 deletions

File tree

‎libs/model-profiles/langchain_model_profiles/cli.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,7 @@ def refresh(provider: str, data_dir: Path) -> None: # noqa: C901, PLR0915
310310
print(f"Writing to {output_file}...")
311311
module_content = [f'"""{MODULE_ADMONITION}"""\n\n', "from typing import Any\n\n"]
312312
module_content.append("_PROFILES: dict[str, dict[str, Any]] = ")
313-
json_str = json.dumps(profiles, indent=4)
313+
json_str = json.dumps(dict(sorted(profiles.items())), indent=4)
314314
json_str = (
315315
json_str.replace("true", "True")
316316
.replace("false", "False")

‎libs/model-profiles/tests/unit_tests/test_cli.py‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,62 @@ def test_refresh_includes_models_defined_only_in_augmentations(
216216
)
217217

218218

219+
def test_refresh_generates_sorted_profiles(
220+
tmp_path: Path, mock_models_dev_response: dict
221+
) -> None:
222+
"""Test that profiles are sorted alphabetically by model ID."""
223+
data_dir = tmp_path / "data"
224+
data_dir.mkdir()
225+
226+
# Inject models in reverse-alphabetical order so the API response
227+
# is NOT already sorted.
228+
mock_models_dev_response["anthropic"]["models"] = {
229+
"z-model": {
230+
"id": "z-model",
231+
"name": "Z Model",
232+
"tool_call": True,
233+
"limit": {"context": 100000, "output": 2048},
234+
"modalities": {"input": ["text"], "output": ["text"]},
235+
},
236+
"a-model": {
237+
"id": "a-model",
238+
"name": "A Model",
239+
"tool_call": True,
240+
"limit": {"context": 100000, "output": 2048},
241+
"modalities": {"input": ["text"], "output": ["text"]},
242+
},
243+
"m-model": {
244+
"id": "m-model",
245+
"name": "M Model",
246+
"tool_call": True,
247+
"limit": {"context": 100000, "output": 2048},
248+
"modalities": {"input": ["text"], "output": ["text"]},
249+
},
250+
}
251+
252+
mock_response = Mock()
253+
mock_response.json.return_value = mock_models_dev_response
254+
mock_response.raise_for_status = Mock()
255+
256+
with (
257+
patch("langchain_model_profiles.cli.httpx.get", return_value=mock_response),
258+
patch("builtins.input", return_value="y"),
259+
):
260+
refresh("anthropic", data_dir)
261+
262+
profiles_file = data_dir / "_profiles.py"
263+
spec = importlib.util.spec_from_file_location(
264+
"generated_profiles_sorted", profiles_file
265+
)
266+
assert spec
267+
assert spec.loader
268+
module = importlib.util.module_from_spec(spec)
269+
spec.loader.exec_module(module) # type: ignore[union-attr]
270+
271+
model_ids = list(module._PROFILES.keys()) # type: ignore[attr-defined]
272+
assert model_ids == sorted(model_ids), f"Profile keys are not sorted: {model_ids}"
273+
274+
219275
def test_model_data_to_profile_text_modalities() -> None:
220276
"""Test that text input/output modalities are correctly mapped."""
221277
# Model with text in both input and output

‎libs/partners/anthropic/langchain_anthropic/data/_profiles.py‎

Lines changed: 48 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@
1616
from typing import Any
1717

1818
_PROFILES: dict[str, dict[str, Any]] = {
19-
"claude-opus-4-5-20251101": {
19+
"claude-3-5-haiku-20241022": {
2020
"max_input_tokens": 200000,
21-
"max_output_tokens": 64000,
21+
"max_output_tokens": 8192,
2222
"text_inputs": True,
2323
"image_inputs": True,
2424
"audio_inputs": False,
@@ -28,7 +28,7 @@
2828
"image_outputs": False,
2929
"audio_outputs": False,
3030
"video_outputs": False,
31-
"reasoning_output": True,
31+
"reasoning_output": False,
3232
"tool_calling": True,
3333
"image_url_inputs": True,
3434
"pdf_tool_message": True,
@@ -54,9 +54,9 @@
5454
"image_tool_message": True,
5555
"structured_output": False,
5656
},
57-
"claude-opus-4-1": {
57+
"claude-3-5-sonnet-20240620": {
5858
"max_input_tokens": 200000,
59-
"max_output_tokens": 32000,
59+
"max_output_tokens": 8192,
6060
"text_inputs": True,
6161
"image_inputs": True,
6262
"audio_inputs": False,
@@ -66,12 +66,12 @@
6666
"image_outputs": False,
6767
"audio_outputs": False,
6868
"video_outputs": False,
69-
"reasoning_output": True,
69+
"reasoning_output": False,
7070
"tool_calling": True,
7171
"image_url_inputs": True,
7272
"pdf_tool_message": True,
7373
"image_tool_message": True,
74-
"structured_output": True,
74+
"structured_output": False,
7575
},
7676
"claude-3-5-sonnet-20241022": {
7777
"max_input_tokens": 200000,
@@ -92,9 +92,9 @@
9292
"image_tool_message": True,
9393
"structured_output": False,
9494
},
95-
"claude-3-sonnet-20240229": {
95+
"claude-3-7-sonnet-20250219": {
9696
"max_input_tokens": 200000,
97-
"max_output_tokens": 4096,
97+
"max_output_tokens": 64000,
9898
"text_inputs": True,
9999
"image_inputs": True,
100100
"audio_inputs": False,
@@ -104,16 +104,16 @@
104104
"image_outputs": False,
105105
"audio_outputs": False,
106106
"video_outputs": False,
107-
"reasoning_output": False,
107+
"reasoning_output": True,
108108
"tool_calling": True,
109109
"image_url_inputs": True,
110110
"pdf_tool_message": True,
111111
"image_tool_message": True,
112112
"structured_output": False,
113113
},
114-
"claude-opus-4-6": {
114+
"claude-3-7-sonnet-latest": {
115115
"max_input_tokens": 200000,
116-
"max_output_tokens": 128000,
116+
"max_output_tokens": 64000,
117117
"text_inputs": True,
118118
"image_inputs": True,
119119
"audio_inputs": False,
@@ -130,9 +130,9 @@
130130
"image_tool_message": True,
131131
"structured_output": False,
132132
},
133-
"claude-sonnet-4-6": {
133+
"claude-3-haiku-20240307": {
134134
"max_input_tokens": 200000,
135-
"max_output_tokens": 64000,
135+
"max_output_tokens": 4096,
136136
"text_inputs": True,
137137
"image_inputs": True,
138138
"audio_inputs": False,
@@ -142,16 +142,16 @@
142142
"image_outputs": False,
143143
"audio_outputs": False,
144144
"video_outputs": False,
145-
"reasoning_output": True,
145+
"reasoning_output": False,
146146
"tool_calling": True,
147147
"image_url_inputs": True,
148148
"pdf_tool_message": True,
149149
"image_tool_message": True,
150150
"structured_output": False,
151151
},
152-
"claude-sonnet-4-0": {
152+
"claude-3-opus-20240229": {
153153
"max_input_tokens": 200000,
154-
"max_output_tokens": 64000,
154+
"max_output_tokens": 4096,
155155
"text_inputs": True,
156156
"image_inputs": True,
157157
"audio_inputs": False,
@@ -161,16 +161,16 @@
161161
"image_outputs": False,
162162
"audio_outputs": False,
163163
"video_outputs": False,
164-
"reasoning_output": True,
164+
"reasoning_output": False,
165165
"tool_calling": True,
166166
"image_url_inputs": True,
167167
"pdf_tool_message": True,
168168
"image_tool_message": True,
169169
"structured_output": False,
170170
},
171-
"claude-opus-4-20250514": {
171+
"claude-3-sonnet-20240229": {
172172
"max_input_tokens": 200000,
173-
"max_output_tokens": 32000,
173+
"max_output_tokens": 4096,
174174
"text_inputs": True,
175175
"image_inputs": True,
176176
"audio_inputs": False,
@@ -180,14 +180,14 @@
180180
"image_outputs": False,
181181
"audio_outputs": False,
182182
"video_outputs": False,
183-
"reasoning_output": True,
183+
"reasoning_output": False,
184184
"tool_calling": True,
185185
"image_url_inputs": True,
186186
"pdf_tool_message": True,
187187
"image_tool_message": True,
188188
"structured_output": False,
189189
},
190-
"claude-sonnet-4-5-20250929": {
190+
"claude-haiku-4-5": {
191191
"max_input_tokens": 200000,
192192
"max_output_tokens": 64000,
193193
"text_inputs": True,
@@ -206,9 +206,9 @@
206206
"image_tool_message": True,
207207
"structured_output": False,
208208
},
209-
"claude-opus-4-0": {
209+
"claude-haiku-4-5-20251001": {
210210
"max_input_tokens": 200000,
211-
"max_output_tokens": 32000,
211+
"max_output_tokens": 64000,
212212
"text_inputs": True,
213213
"image_inputs": True,
214214
"audio_inputs": False,
@@ -225,9 +225,9 @@
225225
"image_tool_message": True,
226226
"structured_output": False,
227227
},
228-
"claude-3-5-haiku-20241022": {
228+
"claude-opus-4-0": {
229229
"max_input_tokens": 200000,
230-
"max_output_tokens": 8192,
230+
"max_output_tokens": 32000,
231231
"text_inputs": True,
232232
"image_inputs": True,
233233
"audio_inputs": False,
@@ -237,16 +237,16 @@
237237
"image_outputs": False,
238238
"audio_outputs": False,
239239
"video_outputs": False,
240-
"reasoning_output": False,
240+
"reasoning_output": True,
241241
"tool_calling": True,
242242
"image_url_inputs": True,
243243
"pdf_tool_message": True,
244244
"image_tool_message": True,
245245
"structured_output": False,
246246
},
247-
"claude-3-5-sonnet-20240620": {
247+
"claude-opus-4-1": {
248248
"max_input_tokens": 200000,
249-
"max_output_tokens": 8192,
249+
"max_output_tokens": 32000,
250250
"text_inputs": True,
251251
"image_inputs": True,
252252
"audio_inputs": False,
@@ -256,16 +256,16 @@
256256
"image_outputs": False,
257257
"audio_outputs": False,
258258
"video_outputs": False,
259-
"reasoning_output": False,
259+
"reasoning_output": True,
260260
"tool_calling": True,
261261
"image_url_inputs": True,
262262
"pdf_tool_message": True,
263263
"image_tool_message": True,
264-
"structured_output": False,
264+
"structured_output": True,
265265
},
266-
"claude-3-7-sonnet-latest": {
266+
"claude-opus-4-1-20250805": {
267267
"max_input_tokens": 200000,
268-
"max_output_tokens": 64000,
268+
"max_output_tokens": 32000,
269269
"text_inputs": True,
270270
"image_inputs": True,
271271
"audio_inputs": False,
@@ -282,9 +282,9 @@
282282
"image_tool_message": True,
283283
"structured_output": False,
284284
},
285-
"claude-3-7-sonnet-20250219": {
285+
"claude-opus-4-20250514": {
286286
"max_input_tokens": 200000,
287-
"max_output_tokens": 64000,
287+
"max_output_tokens": 32000,
288288
"text_inputs": True,
289289
"image_inputs": True,
290290
"audio_inputs": False,
@@ -301,9 +301,9 @@
301301
"image_tool_message": True,
302302
"structured_output": False,
303303
},
304-
"claude-3-haiku-20240307": {
304+
"claude-opus-4-5": {
305305
"max_input_tokens": 200000,
306-
"max_output_tokens": 4096,
306+
"max_output_tokens": 64000,
307307
"text_inputs": True,
308308
"image_inputs": True,
309309
"audio_inputs": False,
@@ -313,14 +313,14 @@
313313
"image_outputs": False,
314314
"audio_outputs": False,
315315
"video_outputs": False,
316-
"reasoning_output": False,
316+
"reasoning_output": True,
317317
"tool_calling": True,
318318
"image_url_inputs": True,
319319
"pdf_tool_message": True,
320320
"image_tool_message": True,
321321
"structured_output": False,
322322
},
323-
"claude-haiku-4-5-20251001": {
323+
"claude-opus-4-5-20251101": {
324324
"max_input_tokens": 200000,
325325
"max_output_tokens": 64000,
326326
"text_inputs": True,
@@ -339,9 +339,9 @@
339339
"image_tool_message": True,
340340
"structured_output": False,
341341
},
342-
"claude-haiku-4-5": {
342+
"claude-opus-4-6": {
343343
"max_input_tokens": 200000,
344-
"max_output_tokens": 64000,
344+
"max_output_tokens": 128000,
345345
"text_inputs": True,
346346
"image_inputs": True,
347347
"audio_inputs": False,
@@ -358,7 +358,7 @@
358358
"image_tool_message": True,
359359
"structured_output": False,
360360
},
361-
"claude-opus-4-5": {
361+
"claude-sonnet-4-0": {
362362
"max_input_tokens": 200000,
363363
"max_output_tokens": 64000,
364364
"text_inputs": True,
@@ -377,9 +377,9 @@
377377
"image_tool_message": True,
378378
"structured_output": False,
379379
},
380-
"claude-3-opus-20240229": {
380+
"claude-sonnet-4-20250514": {
381381
"max_input_tokens": 200000,
382-
"max_output_tokens": 4096,
382+
"max_output_tokens": 64000,
383383
"text_inputs": True,
384384
"image_inputs": True,
385385
"audio_inputs": False,
@@ -389,7 +389,7 @@
389389
"image_outputs": False,
390390
"audio_outputs": False,
391391
"video_outputs": False,
392-
"reasoning_output": False,
392+
"reasoning_output": True,
393393
"tool_calling": True,
394394
"image_url_inputs": True,
395395
"pdf_tool_message": True,
@@ -415,7 +415,7 @@
415415
"image_tool_message": True,
416416
"structured_output": True,
417417
},
418-
"claude-sonnet-4-20250514": {
418+
"claude-sonnet-4-5-20250929": {
419419
"max_input_tokens": 200000,
420420
"max_output_tokens": 64000,
421421
"text_inputs": True,
@@ -434,9 +434,9 @@
434434
"image_tool_message": True,
435435
"structured_output": False,
436436
},
437-
"claude-opus-4-1-20250805": {
437+
"claude-sonnet-4-6": {
438438
"max_input_tokens": 200000,
439-
"max_output_tokens": 32000,
439+
"max_output_tokens": 64000,
440440
"text_inputs": True,
441441
"image_inputs": True,
442442
"audio_inputs": False,

0 commit comments

Comments
 (0)