Merge pull request #10 from StarRocks/ast

Ast

Author: DanRoscigno

README.md

@@ -25,12 +25,27 @@ This code and most of the README are from the team at [PlayCanvas](https://githu
                           preserve structure)
   --suffix <suffix>       Custom suffix for output files (default: language
                           name)
-   --completeness <mode>   Completeness check mode: warn, fail, or off (default:
-                                       warn)
-   --log-chunk-metadata    Log API metadata for each chunk (and on mismatches)
+   --log-chunk-metadata    Log API metadata for each chunk
   -h, --help              display help for command
 ```
 
+The translator now uses the AST pipeline by default.
+
+### Interpreting AST parse failures
+
+In AST mode, each chunk asks the model to return a strict JSON array of `{ id, text }` items.
+
+- Parse errors such as `Expected ',' or '}'` or `Expected ':' after property name` usually mean the model returned malformed JSON for that chunk.
+- These are response-format failures, not semantic translation failures.
+- `finishReason: STOP` with parse errors means the output completed, but the JSON structure was invalid.
+- When you see `json repair retry`, the tool requested a strict JSON retry and recovered automatically.
+- When you see `split fallback recovered X/Y missing ids`, the tool retried unresolved IDs in smaller sub-batches and merged recovered results back into the chunk.
+
+How to read the outcome:
+
+- `AST completeness check: Translated IDs N/N - ✅ PASS` means the chunk is fully recovered, even if repair notes are present.
+- Missing IDs after all retries are the only case that indicates unresolved chunk-level translation for those specific items.
+
 ## Quick Start
 
 1. cd into the root of this repo
@@ -153,6 +168,9 @@ md-translate translate -i docs/guide.md -l French -o docs/guide_fr.md
 
 # Translate using API key argument
 md-translate translate -i file.md -l German --key your-api-key
+
+# Translate with AST mode (default)
+md-translate translate -i examples/External_table.md -l Japanese
 ```
 
 ### Batch Processing
@@ -189,8 +207,7 @@ Options:
   -k, --key <apikey>       Google Gemini API key (optional)
   --flat                   Use flat structure in output directory (default: preserve structure)
   --suffix <suffix>        Custom suffix for output files (default: language name)
-   --completeness <mode>    Completeness check mode: warn, fail, or off (default: warn)
-   --log-chunk-metadata     Log API metadata for each chunk (and on mismatches)
+   --log-chunk-metadata     Log API metadata for each chunk
 ```
 
 #### `languages` - List supported languages

bin/cli.js

@@ -8,7 +8,7 @@ import fs from 'fs-extra';
 import ora from 'ora';
 
 
-import MarkdownTranslator from '../src/translator.js';
+import AstMarkdownTranslator from '../src/translator_ast_mvp.js';
 
 const program = new Command();
 
@@ -29,8 +29,8 @@ program
 .command('translate')
 .description('Translate markdown files to specified language')
 .requiredOption('-i, --input <pattern>', 'Input file path or glob pattern (e.g., "*.md", "docs/**/*.md")')
-    .requiredOption('-l, --language <lang>', 'Target language (e.g., Spanish, French, German)')
-    .option('-s, --source <lang>', 'Source language (default: English)')
+.requiredOption('-l, --language <lang>', 'Target language (e.g., Spanish, French, German)')
+.option('-s, --source <lang>', 'Source language (default: English)')
 .option('-o, --output <file>', 'Output file path (for single file translation)')
 .option('-d, --output-dir <dir>', 'Output directory (for batch translation or single file)')
 .option('-k, --key <apikey>', 'Google Gemini API key (or set GEMINI_API_KEY env var)')
@@ -51,7 +51,7 @@ program
         }
 
         // Initialize translator
-        const translator = new MarkdownTranslator(apiKey);
+        const translator = new AstMarkdownTranslator(apiKey);
 
         // Check if input is a glob pattern (contains wildcards or multiple matches)
         const inputPattern = options.input;
@@ -78,6 +78,7 @@ program
             console.log(chalk.gray(`   Source:   ${options.source || 'English'}`));
             console.log(chalk.gray(`   Language: ${options.language}`));
             console.log(chalk.gray(`   Structure: ${options.flat ? 'Flat' : 'Preserved'}`));
+            console.log(chalk.gray('   Mode: AST'));
             console.log('');
 
             // Create progress handler
@@ -174,6 +175,7 @@ program
             console.log(chalk.gray(`   Output:   ${outputPath}`));
             console.log(chalk.gray(`   Source:   ${options.source || 'English'}`));
             console.log(chalk.gray(`   Language: ${options.language}`));
+            console.log(chalk.gray('   Mode:     AST'));
             console.log('');
 
             // Create progress spinner
@@ -230,7 +232,7 @@ program
     console.log(chalk.blue('🌍 Supported Languages:'));
     console.log('');
 
-    const languages = MarkdownTranslator.getSupportedLanguages();
+    const languages = AstMarkdownTranslator.getSupportedLanguages();
     const columns = 3;
     const rows = Math.ceil(languages.length / columns);