feat: default answer to unparsed jsonAnswer

.vscode/launch.json

@@ -0,0 +1,24 @@
+{
+    "configurations": [
+        {
+            "name": "entrypoint (extended scraper)",
+            "type": "pwa-node",
+            "request": "launch",
+            "cwd": "${workspaceFolder}/actors/extended-gpt-scraper",
+            "program": "src/main.ts",
+            "runtimeArgs": [
+              "--no-warnings",
+              "--experimental-specifier-resolution=node",
+              "--loader",
+              "../../module_loader.js"
+            ],
+            "runtimeExecutable": "node",
+            "console": "integratedTerminal",
+            "internalConsoleOptions": "neverOpen",
+            "skipFiles": [
+                "<node_internals>/**",
+                "**/node_modules/**"
+            ]
+        }
+    ]
+}

packages/gpt-scraper-core/src/models/openai.ts

@@ -11,7 +11,7 @@
 import { GeneralModelHandler } from './model.js';
 
 // TODO: refactor: make this error modular with other non-OpenAI models
 export const tryWrapInOpenaiError = (error: any) => {
     if (error?.response?.data?.error) {
         return new OpenaiAPIError(error.response.data.error.message || error.response.data.error.code);
     }
@@ -47,7 +47,9 @@
         const possibleJsonAnswer = functionArguments || answer;
         const jsonAnswer = possibleJsonAnswer ? tryToParseJsonFromString(possibleJsonAnswer) : null;
 
-        return { answer, jsonAnswer, usage };
+        // it may return just function arguments (jsonAnswer), but if jsonAnswer is present, then `answer` should be too,
+        // so default `answer` to the unparsed value of `jsonAnswer`
+        return { answer: answer ?? possibleJsonAnswer, jsonAnswer, usage };
     }
 
     /**
@@ -56,10 +58,10 @@
      * // TODO: refactor: perhaps we can achieve this with only crawlee?
      */
     processInstructionsWithRetry = (options: ProcessInstructionsOptions<OpenAIModelSettings>) => {
         const process: RetryFunction<any> = async (stopTrying: (e: Error) => void, attempt: number) => {
             try {
                 return await this.processInstructions(options);
             } catch (error: any) {
                 // NOTE: OpenAI API returns 429 with insufficient_quota, user needs to buy a plan.
                 if (error?.response?.status === 429 && error.response?.data?.error?.type === 'insufficient_quota') {
                     stopTrying(error);
@@ -102,7 +104,7 @@
      * Parses the function arguments from the OpenAI LLM's output.
      */
     private parseFunctionArguments = (functionOutput: LLMResult): string | null => {
         const generations = functionOutput.generations as any;
         const firstFunction = generations?.[0]?.[0]?.message;
         const functionArguments = firstFunction?.lc_kwargs?.additional_kwargs?.function_call?.arguments;