Release 2.0.10

jellesiderius · jellesiderius · commit 07db2ab5772c · 2025-11-12T11:30:13.000+01:00
diff --git a/.gitignore b/.gitignore
@@ -274,3 +274,4 @@ config/databases/commands/*
 config/databases/custom-commands
 
 dist
+.cursor/
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "mage-db-sync",
-    "version": "2.0.9",
+    "version": "2.0.10",
     "description": "Database synchronizer for Magento, based on Magerun",
     "license": "MIT",
     "author": {
diff --git a/src/tasks/DownloadTask.ts b/src/tasks/DownloadTask.ts
@@ -302,7 +302,7 @@ class DownloadTask {
                         // Uses official magerun2 table groups: https://github.com/netz98/n98-magerun2
                         const customStripParts: string[] = [];
                         const keepOptions = config.settings.stripOptions || [];
-                        
+
                         // Always strip these for development (safe to remove)
                         customStripParts.push(
                             '@log',              // Log tables
@@ -312,41 +312,41 @@ class DownloadTask {
                             '@replica',          // Replica tables
                             '@newrelic_reporting' // New Relic tables
                         );
-                        
+
                         // Strip based on what user wants to KEEP (unchecked = strip)
                         if (!keepOptions.includes('customers')) {
                             customStripParts.push('@customers');
                         }
-                        
+
                         if (!keepOptions.includes('admin')) {
                             customStripParts.push('@admin', '@oauth', '@2fa');
                         }
-                        
+
                         if (!keepOptions.includes('sales')) {
                             customStripParts.push('@sales');
                         }
-                        
+
                         if (!keepOptions.includes('quotes')) {
                             customStripParts.push('@quotes');
                         }
-                        
+
                         if (!keepOptions.includes('search')) {
                             customStripParts.push('@search', '@idx');
                         }
-                        
+
                         if (!keepOptions.includes('dotmailer')) {
                             customStripParts.push('@dotmailer', '@mailchimp');
                         }
-                        
+
                         // Note: We don't strip config even if unchecked, as it would break the database
                         // Configuration data (core_config_data) is not in any strip group - it's always kept
                         if (!keepOptions.includes('config')) {
                             logger.warn('Configuration settings are always kept as they are required for Magento to function');
                         }
-                        
+
                         const customStripString = customStripParts.join(' ');
                         stripOptions = customStripString ? `--strip="${customStripString}"` : '';
-                        
+
                         logger.info('Using custom strip configuration', {
                             keepOptions,
                             stripGroups: customStripParts,
@@ -357,26 +357,30 @@ class DownloadTask {
                         const keepCustomerOptions = staticSettings.settings?.databaseStripKeepCustomerData || '';
                         stripOptions = keepCustomerOptions ? `--strip="${keepCustomerOptions}"` : '';
                     } else if (config.settings.strip === 'full and human readable') {
-                        const staticSettings = this.services.getConfig().getStaticSettings();
-                        const fullStripOptions = staticSettings.settings?.databaseStripDevelopment || '';
-                        stripOptions = fullStripOptions ? `--strip="${fullStripOptions}"` : '';
+                        // FULL dump with human-readable format - NO stripping
+                        stripOptions = '';
                         humanReadable = '--human-readable';
+                        logger.info('Using full database dump with human-readable format (no stripping)');
                     } else if (config.settings.strip === 'full') {
-                        const staticSettings = this.services.getConfig().getStaticSettings();
-                        const fullStripOptions = staticSettings.settings?.databaseStripDevelopment || '';
-                        stripOptions = fullStripOptions ? `--strip="${fullStripOptions}"` : '';
+                        // FULL dump - NO stripping
+                        stripOptions = '';
+                        logger.info('Using full database dump (no stripping)');
                     } else {
+                        // Default: apply development strip options
                         const staticSettings = this.services.getConfig().getStaticSettings();
                         const developmentStripOptions = staticSettings.settings?.databaseStripDevelopment || '';
                         stripOptions = developmentStripOptions ? `--strip="${developmentStripOptions}"` : '';
                     }
 
+                    // Escape filename for shell usage
+                    const escapedFileName = shellEscape(databaseFileName);
+
                     // Build compression command based on what's available
                     if (compression.type === 'gzip') {
-                        dumpCommand = `db:dump --stdout -n --no-tablespaces ${humanReadable} ${stripOptions} | gzip ${compression.level} > ${databaseFileName}`;
+                        dumpCommand = `db:dump --stdout -n --no-tablespaces ${humanReadable} ${stripOptions} | gzip ${compression.level} > ${escapedFileName}`;
                     } else {
                         // No compression - just dump to file
-                        dumpCommand = `db:dump --stdout -n --no-tablespaces ${humanReadable} ${stripOptions} > ${databaseFileName}`;
+                        dumpCommand = `db:dump --stdout -n --no-tablespaces ${humanReadable} ${stripOptions} > ${escapedFileName}`;
                     }
 
                     logger.info('Using compression for database dump', {
@@ -387,27 +391,80 @@ class DownloadTask {
                     });
 
                     const fullCommand = sshMagentoRootFolderMagerunCommand(
-                        `${dumpCommand}; mv ${databaseFileName} ~`,
+                        `${dumpCommand}; mv ${escapedFileName} ~`,
                         config
                     );
 
-                    task.output = 'Dumping database (this may take a minute)...';
+                    task.output = 'Starting database dump...';
                     logger.info('Starting database dump', {
                         database: config.serverVariables.databaseName,
                         stripType
                     });
 
-                    await ssh.execCommand(fullCommand).then(function (result: any) {
+                    // Start the dump command (non-blocking)
+                    const dumpPromise = ssh.execCommand(fullCommand);
+
+                    // Monitor file size in real-time
+                    const startTime = Date.now();
+                    let lastSize = 0;
+                    let lastSizeTime = Date.now();
+
+                    const sizeCheckInterval = setInterval(async () => {
+                        try {
+                            // Check file size on server (in Magento root first, then home)
+                            const sizeCommand = sshMagentoRootFolderMagerunCommand(
+                                `stat -f%z ${escapedFileName} 2>/dev/null || stat -c%s ${escapedFileName} 2>/dev/null || echo "0"`,
+                                config
+                            );
+
+                            const sizeResult = await ssh.execCommand(sizeCommand);
+                            const currentSize = parseInt(sizeResult.stdout.trim() || '0');
+
+                            if (currentSize > 0) {
+                                const elapsed = Math.floor((Date.now() - startTime) / 1000);
+                                const elapsedMinutes = Math.floor(elapsed / 60);
+                                const elapsedSeconds = elapsed % 60;
+                                const timeStr = elapsedMinutes > 0
+                                    ? `${elapsedMinutes}m ${elapsedSeconds}s`
+                                    : `${elapsedSeconds}s`;
+
+                                // Calculate speed since last check
+                                const timeDiff = (Date.now() - lastSizeTime) / 1000;
+                                const sizeDiff = currentSize - lastSize;
+                                const speed = timeDiff > 0 ? sizeDiff / timeDiff : 0;
+
+                                lastSize = currentSize;
+                                lastSizeTime = Date.now();
+
+                                const sizeStr = ProgressDisplay.formatBytes(currentSize);
+                                const speedStr = speed > 0 ? ` ${chalk.cyan('~' + ProgressDisplay.formatSpeed(speed))}` : '';
+
+                                task.output = `Dumping database... ${chalk.bold.cyan(sizeStr)}${speedStr} ${chalk.gray(`(${timeStr} elapsed)`)}`;
+                            }
+                        } catch (err) {
+                            // Ignore errors during size check (file might not exist yet)
+                        }
+                    }, 2000); // Check every 2 seconds
+
+                    await dumpPromise.then(function (result: any) {
+                        clearInterval(sizeCheckInterval);
+
                         if (result.code && result.code !== 0) {
                             throw new Error(
                                 `Database dump failed\n[TIP] Check database permissions and disk space\nError: ${result.stderr}`
                             );
                         }
-                        task.output = '✓ Database dump completed';
+
+                        const elapsed = Math.floor((Date.now() - startTime) / 1000);
+                        const finalSizeStr = lastSize > 0 ? ` (${ProgressDisplay.formatBytes(lastSize)})` : '';
+                        task.output = `✓ Database dump completed${finalSizeStr} in ${elapsed}s`;
+                    }).catch((err: Error) => {
+                        clearInterval(sizeCheckInterval);
+                        throw err;
                     });
 
                     const duration = PerformanceMonitor.end('database-dump');
-                    logger.info('Database dump complete', { duration });
+                    logger.info('Database dump complete', { duration, finalSize: lastSize });
                     task.title = `Dumped database`;
                 }
             });
diff --git a/src/tasks/ImportTask.ts b/src/tasks/ImportTask.ts
@@ -1,4 +1,4 @@
-import { localhostMagentoRootExec } from '../utils/Console';
+import { localhostMagentoRootExec, shellEscape } from '../utils/Console';
 import { Listr } from 'listr2';
 import { ServiceContainer } from '../core/ServiceContainer';
 import { ProgressDisplay } from '../utils/ProgressDisplay';
@@ -45,8 +45,10 @@ class ImportTask {
             compressionType
         });
         
-        // Estimate import duration (~10MB/s for compressed, ~5MB/s for uncompressed)
-        const speed = isCompressed ? 10 * 1024 * 1024 : 5 * 1024 * 1024;
+        // More conservative speed estimates based on database write operations
+        // Compressed: ~3-5MB/s (decompression + write overhead)
+        // Uncompressed: ~2-3MB/s (raw SQL parsing and execution)
+        const speed = isCompressed ? 4 * 1024 * 1024 : 2.5 * 1024 * 1024;
         const estimatedDuration = sqlFileSize > 0 ? (sqlFileSize / speed) * 1000 : 60000;
         
         // Show file info
@@ -57,10 +59,26 @@ class ImportTask {
         // Start progress estimation
         let progressInterval: NodeJS.Timeout;
         let lastPercentage = 0;
+        let isInFinalPhase = false;
         
         progressInterval = setInterval(() => {
             const elapsed = Date.now() - startTime;
-            let percentage = Math.min(95, Math.round((elapsed / estimatedDuration) * 100));
+            let percentage = Math.round((elapsed / estimatedDuration) * 100);
+            
+            // More realistic progress stages:
+            // 0-80%: Normal progression based on estimate
+            // 80-90%: Slow down (database operations are slower near end)
+            // 90%+: Final phase - show elapsed time instead
+            if (percentage <= 80) {
+                // Normal progression
+            } else if (percentage <= 90) {
+                // Slow down between 80-90%
+                percentage = 80 + Math.min(10, (percentage - 80) * 0.5);
+            } else {
+                // After 90%, don't show percentage, show elapsed time
+                percentage = 90;
+                isInFinalPhase = true;
+            }
             
             // Avoid going backwards
             if (percentage > lastPercentage) {
@@ -73,8 +91,18 @@ class ImportTask {
             const estimatedBytes = Math.min(sqlFileSize, (percentage / 100) * sqlFileSize);
             const avgSpeed = elapsed > 0 ? (estimatedBytes / (elapsed / 1000)) : 0;
             
-            const statusText = percentage >= 90 ? chalk.yellow('(finishing...)') : '';
-            task.output = `${progressBar} ${chalk.bold.cyan(percentage + '%')} ${chalk.gray(ProgressDisplay.formatBytes(estimatedBytes) + ' / ' + sizeInfo)}${compressionInfo} ${chalk.cyan('~' + ProgressDisplay.formatSpeed(avgSpeed))} ${statusText}`;
+            // Show different status based on phase
+            let statusText = '';
+            if (isInFinalPhase) {
+                const elapsedSeconds = Math.floor(elapsed / 1000);
+                statusText = chalk.yellow(`(finalizing database... ${elapsedSeconds}s elapsed)`);
+                task.output = `${progressBar} ${chalk.bold.cyan(percentage + '%')} ${statusText}`;
+            } else if (percentage >= 80) {
+                statusText = chalk.yellow('(processing indexes and constraints...)');
+                task.output = `${progressBar} ${chalk.bold.cyan(percentage + '%')} ${chalk.gray(ProgressDisplay.formatBytes(estimatedBytes) + ' / ' + sizeInfo)}${compressionInfo} ${chalk.cyan('~' + ProgressDisplay.formatSpeed(avgSpeed))} ${statusText}`;
+            } else {
+                task.output = `${progressBar} ${chalk.bold.cyan(percentage + '%')} ${chalk.gray(ProgressDisplay.formatBytes(estimatedBytes) + ' / ' + sizeInfo)}${compressionInfo} ${chalk.cyan('~' + ProgressDisplay.formatSpeed(avgSpeed))}`;
+            }
         }, 1000);
         
         try {
@@ -96,8 +124,8 @@ class ImportTask {
             
             importCommand += ` --drop` +  // Drop and recreate database
                            ` --force` + // Continue on SQL errors
-                           ` --skip-authorization-entry-creation` + // We'll add them later
-                           ` -q`; // Quiet mode
+                           ` --skip-authorization-entry-creation`; // We'll add them later
+            // NOTE: Removed -q flag to allow magerun2's progress output
             
             logger.info('Executing magerun2 db:import', { 
                 command: importCommand,
@@ -106,7 +134,62 @@ class ImportTask {
                 hasOptimize: !isCompressed // Only optimize uncompressed files
             });
             
-            await localhostMagentoRootExec(importCommand, config);
+            // Execute with streaming output to capture real progress
+            const { spawn } = require('child_process');
+            const escapedFolder = shellEscape(config.settings.currentFolder);
+            
+            await new Promise<void>((resolve, reject) => {
+                // Use shell to execute the cd + command
+                const proc = spawn('sh', ['-c', `cd ${escapedFolder} && ${importCommand}`]);
+                
+                let lastOutput = '';
+                
+                // Capture stdout for progress
+                proc.stdout.on('data', (data: Buffer) => {
+                    const output = data.toString();
+                    
+                    // Check if magerun2 is showing progress (it uses \r for updates)
+                    if (output.includes('\r') || output.includes('%')) {
+                        clearInterval(progressInterval);
+                        
+                        // Parse magerun2's progress output
+                        // magerun2 typically outputs: "Importing... XX%" or progress bars
+                        const lines = output.split('\r').filter(l => l.trim());
+                        if (lines.length > 0) {
+                            lastOutput = lines[lines.length - 1].trim();
+                            
+                            // Try to extract percentage from magerun2 output
+                            const percentMatch = lastOutput.match(/(\d+)%/);
+                            if (percentMatch) {
+                                const realPercentage = parseInt(percentMatch[1]);
+                                const progressBar = EnhancedProgress.createProgressBar(realPercentage, 20);
+                                task.output = `${progressBar} ${chalk.bold.cyan(realPercentage + '%')} ${chalk.gray(sizeInfo)}${compressionInfo} ${chalk.green('[magerun2]')}`;
+                            } else {
+                                // Show raw magerun2 output if no percentage found
+                                task.output = lastOutput;
+                            }
+                        }
+                    }
+                });
+                
+                // Capture stderr (magerun2 might output there too)
+                proc.stderr.on('data', (data: Buffer) => {
+                    const output = data.toString();
+                    logger.debug('magerun2 stderr', { output });
+                });
+                
+                proc.on('close', (code: number) => {
+                    if (code === 0) {
+                        resolve();
+                    } else {
+                        reject(new Error(`magerun2 db:import failed with code ${code}`));
+                    }
+                });
+                
+                proc.on('error', (err: Error) => {
+                    reject(err);
+                });
+            });
             
             clearInterval(progressInterval);
             

Original file line number	Diff line number	Diff line change
`@@ -274,3 +274,4 @@ config/databases/commands/*`
`274`	`274`	`config/databases/custom-commands`
`275`	`275`
`276`	`276`	`dist`
	`277`	`+.cursor/`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "mage-db-sync",`
`3`		`- "version": "2.0.9",`
	`3`	`+ "version": "2.0.10",`
`4`	`4`	`"description": "Database synchronizer for Magento, based on Magerun",`
`5`	`5`	`"license": "MIT",`
`6`	`6`	`"author": {`