Merge pull request #37 from marshmallow-packages/36-the-process-mysql--h-127001--u-root--p-example_database-downloadsnew_datasql-exceeded-the-timeout-of-60-seconds

(feature) Add configurable process timeout for database sync

Author: stefvanesch

README.md

@@ -13,6 +13,7 @@ A powerful Laravel package that enables seamless synchronization of data from a
     -   [Advanced Options](#advanced-options)
     -   [Per-Table Sync Tracking](#per-table-sync-tracking)
     -   [Table Configuration](#table-configuration)
+    -   [Timeout Configuration](#timeout-configuration)
     -   [Synchronization Suites](#synchronization-suites)
     -   [Multi-Tenant Support](#multi-tenant-support)
 -   [Testing](#testing)
@@ -168,6 +169,30 @@ You can exclude specific tables from synchronization in the `config/database-syn
 ],
 ```
 
+### Timeout Configuration
+
+For large databases, you may need to adjust the process timeout to prevent operations from timing out:
+
+```php
+// Set timeout in seconds (default: 300 seconds / 5 minutes)
+'process_timeout' => 600, // 10 minutes
+
+// Or set to null to disable timeout entirely for very large databases
+'process_timeout' => null,
+```
+
+You can also set this via environment variable:
+
+```env
+DATABASE_SYNC_PROCESS_TIMEOUT=600
+```
+
+This timeout applies to:
+
+-   MySQL dump operations (`mysqldump`)
+-   MySQL import operations (`mysql`)
+-   File transfer operations (`scp`)
+
 ### Synchronization Suites
 
 Define custom synchronization suites in the configuration file to group tables for specific sync tasks:

config/database-sync.php

@@ -119,4 +119,17 @@
     'mysql' => [
         'dump_action_flags' => '--skip-lock-tables --no-create-info --complete-insert --skip-triggers --replace',
     ],
+
+    /*
+    |--------------------------------------------------------------------------
+    | Process Timeout Configuration
+    |--------------------------------------------------------------------------
+    |
+    | Set the timeout (in seconds) for long-running database operations.
+    | Set to null to disable timeout entirely for very large databases.
+    | Default: 300 seconds (5 minutes)
+    |
+    */
+
+    'process_timeout' => env('DATABASE_SYNC_PROCESS_TIMEOUT', 300),
 ];

docs/troubleshooting-large-databases.md

@@ -0,0 +1,81 @@
+# Troubleshooting Large Database Sync Issues
+
+## Timeout Errors
+
+If you encounter timeout errors like:
+
+```
+The process "mysql -h 127.0.0.1 -u root -p'***' example_database < ~/Downloads/new_data.sql" exceeded the timeout of 60 seconds.
+```
+
+### Solution
+
+Configure the `process_timeout` setting in your `config/database-sync.php` file:
+
+```php
+// Set timeout in seconds (default: 300 seconds / 5 minutes)
+'process_timeout' => 600, // 10 minutes
+
+// Or set to null to disable timeout entirely for very large databases
+'process_timeout' => null,
+```
+
+Or set via environment variable:
+
+```env
+DATABASE_SYNC_PROCESS_TIMEOUT=600
+```
+
+### Affected Operations
+
+The timeout setting applies to:
+
+-   MySQL dump operations (`mysqldump`)
+-   MySQL import operations (`mysql`)
+-   File transfer operations (`scp`)
+-   Remote file operations (`ssh rm`)
+
+### Recommendations by Database Size
+
+-   **Small databases (< 100MB)**: Default timeout (300 seconds) should be sufficient
+-   **Medium databases (100MB - 1GB)**: Set timeout to 600-1800 seconds (10-30 minutes)
+-   **Large databases (> 1GB)**: Consider setting timeout to `null` to disable it entirely
+
+### Alternative Solutions
+
+1. **Use batch transfer mode**: This is enabled by default and combines all table dumps into a single file transfer, which is more efficient for large databases.
+
+2. **Sync specific tables**: Use the `--table` option to sync only specific tables:
+
+    ```bash
+    php artisan db-sync --table=users
+    ```
+
+3. **Use suites**: Group related tables and sync them separately:
+
+    ```bash
+    php artisan db-sync --suite=orders
+    ```
+
+4. **Increase MySQL timeouts**: You may also need to adjust MySQL server timeouts:
+    ```sql
+    SET global net_read_timeout=600;
+    SET global net_write_timeout=600;
+    ```
+
+## Memory Issues
+
+For very large datasets, you may need to adjust PHP memory limits:
+
+```bash
+php -d memory_limit=512M artisan db-sync
+```
+
+## Network Issues
+
+If you experience network-related timeouts during file transfers:
+
+1. Ensure stable SSH connection to remote server
+2. Consider using compression in SSH config
+3. Check available disk space on both local and remote systems
+4. Verify network bandwidth between servers

src/Actions/CopyRemoteFileToLocalAction.php

@@ -19,7 +19,9 @@ public static function handle(
             $command->info(__('Copying file to local machine...'));
         }
         $copyCommand = "scp {$config->remote_user_and_host}:{$config->remote_temporary_file} {$config->local_temporary_file}";
-        $result = Process::run($copyCommand);
+
+        $process = Process::timeout($config->process_timeout);
+        $result = $process->run($copyCommand);
 
         if ($result->failed()) {
             throw new \Exception(__('Failed to copy remote file to local: :error', ['error' => $result->errorOutput()]));

src/Actions/Mysql/DumpCreatedOrUpdatedDataAction.php

@@ -26,6 +26,7 @@ public static function handle(
             ]));
         }
 
-        Process::run($exportCommand)->output();
+        $process = Process::timeout($config->process_timeout);
+        $process->run($exportCommand)->output();
     }
 }

src/Actions/Mysql/DumpDeletedDataAction.php

@@ -31,6 +31,7 @@ public static function handle(
             ]));
         }
 
-        Process::run($exportCommand)->output();
+        $process = Process::timeout($config->process_timeout);
+        $process->run($exportCommand)->output();
     }
 }

src/Actions/Mysql/DumpFullTableDataAction.php

@@ -26,6 +26,7 @@ public static function handle(
             ]));
         }
 
-        Process::run($exportCommand)->output();
+        $process = Process::timeout($config->process_timeout);
+        $process->run($exportCommand)->output();
     }
 }

src/Actions/Mysql/ImportDataAction.php

@@ -18,7 +18,9 @@ public static function handle(
         }
 
         $importCommand = "mysql -h {$config->local_host} -u {$config->local_database_username} -p'{$config->local_database_password}' {$config->local_database} < {$config->local_temporary_file}";
-        $result = Process::run($importCommand);
+
+        $process = Process::timeout($config->process_timeout);
+        $result = $process->run($importCommand);
 
         if ($result->failed()) {
             throw new \Exception(__('Failed to import data to local database: :error', ['error' => $result->errorOutput()]));

src/Actions/RemoveLocalFileAction.php

@@ -13,6 +13,7 @@ public static function handle(
         /**
          * Delete the local SQL dump file
          */
-        Process::run("rm -f {$config->local_temporary_file}");
+        $process = Process::timeout($config->process_timeout);
+        $process->run("rm -f {$config->local_temporary_file}");
     }
 }

src/Actions/RemoveRemoteFileAction.php

@@ -13,6 +13,7 @@ public static function handle(
         /**
          * Delete the remote SQL dump file
          */
-        Process::run("ssh {$config->remote_user_and_host} 'rm -f {$config->remote_temporary_file}'");
+        $process = Process::timeout($config->process_timeout);
+        $process->run("ssh {$config->remote_user_and_host} 'rm -f {$config->remote_temporary_file}'");
     }
 }