Skip to content

Commit d02bd33

Browse files
author
Developer
committed
fix: add mysqlbinlog_path DSN param and LOAD_FILE fallback for remote binlogs
- Allow users to specify mysqlbinlog path via DSN parameter (e.g. ?mysqlbinlog_path=C:\mysql\bin\mysqlbinlog.exe) - Add DownloadBinlogFiles method to MySQLConnector that downloads binlog files via LOAD_FILE when mysqlbinlog is not available - Cascade: explicit path → PATH → basedir → LOAD_FILE download - All remote binlog reading works without mysqlbinlog installed
1 parent 680a96b commit d02bd33

4 files changed

Lines changed: 177 additions & 28 deletions

File tree

internal/config/config.go

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -182,12 +182,17 @@ func ParseDSNToConnConfig(dsn string) (connector.ConnConfig, error) {
182182
params["tls"] = cfg.TLSConfig
183183
}
184184

185+
// Extract custom mysqlbinlog_path param before passing to the connector.
186+
mysqlbinlogPath := params["mysqlbinlog_path"]
187+
delete(params, "mysqlbinlog_path")
188+
185189
return connector.ConnConfig{
186-
Host: host,
187-
Port: port,
188-
User: cfg.User,
189-
Password: cfg.Passwd,
190-
Database: cfg.DBName,
191-
Params: params,
190+
Host: host,
191+
Port: port,
192+
User: cfg.User,
193+
Password: cfg.Passwd,
194+
Database: cfg.DBName,
195+
Params: params,
196+
MySQLBinlogPath: mysqlbinlogPath,
192197
}, nil
193198
}

internal/connector/mysql.go

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"database/sql"
66
"fmt"
7+
"os"
78
"path/filepath"
89
"strings"
910
"time"
@@ -187,6 +188,8 @@ func (m *MySQLConnector) GetBinlogDir(ctx context.Context) (string, error) {
187188
// GetBasedir
188189
// ---------------------------------------------------------------------------
189190

191+
// GetBasedir returns the MySQL installation base directory by querying
192+
// SHOW VARIABLES LIKE 'basedir'.
190193
func (m *MySQLConnector) GetBasedir(ctx context.Context) (string, error) {
191194
if m.db == nil {
192195
return "", fmt.Errorf("connector: not connected")
@@ -200,6 +203,113 @@ func (m *MySQLConnector) GetBasedir(ctx context.Context) (string, error) {
200203
}
201204
return basedir, nil
202205
}
206+
207+
// ---------------------------------------------------------------------------
208+
// DownloadBinlogFiles
209+
// ---------------------------------------------------------------------------
210+
211+
// DownloadBinlogFiles reads binlog files from the MySQL server via LOAD_FILE()
212+
// and writes them to a local temporary directory. Returns the local file paths.
213+
// This is used as a fallback when mysqlbinlog is not available and binlog files
214+
// are not accessible on the local filesystem.
215+
//
216+
// Conditions for success:
217+
// - The MySQL user must have the FILE privilege.
218+
// - The MySQL server variable secure_file_priv must be empty ("") or include
219+
// the binlog directory so LOAD_FILE() can read the files.
220+
// - Binlog files must be world-readable on the MySQL server.
221+
func (m *MySQLConnector) DownloadBinlogFiles(ctx context.Context, binlogDir string, binlogNames []string) ([]string, error) {
222+
if m.db == nil {
223+
return nil, fmt.Errorf("connector: not connected")
224+
}
225+
if len(binlogNames) == 0 {
226+
return nil, fmt.Errorf("connector: no binlog files specified")
227+
}
228+
229+
// Check secure_file_priv.
230+
var sfpName, secureFilePriv string
231+
if err := m.db.QueryRowContext(ctx, "SHOW VARIABLES LIKE 'secure_file_priv'").Scan(&sfpName, &secureFilePriv); err != nil {
232+
return nil, fmt.Errorf("connector: query secure_file_priv: %w", err)
233+
}
234+
if secureFilePriv != "" && !strings.HasPrefix(binlogDir, secureFilePriv) {
235+
return nil, fmt.Errorf(
236+
"MySQL secure_file_priv (%s) restricts LOAD_FILE() to that directory; "+
237+
"binlog files are in %s. To fix, either: (1) add --secure-file-priv= \"\" "+
238+
"to the MySQL startup command, or (2) install mysql-client and specify "+
239+
"mysqlbinlog_path in the DSN (e.g. ?mysqlbinlog_path=C:\\mysql\\bin\\mysqlbinlog.exe)",
240+
secureFilePriv, binlogDir)
241+
}
242+
243+
tmpDir, err := os.MkdirTemp("", "pitr-binlog-*")
244+
if err != nil {
245+
return nil, fmt.Errorf("connector: create temp dir: %w", err)
246+
}
247+
248+
var localPaths []string
249+
for _, name := range binlogNames {
250+
fullPath := filepath.Join(binlogDir, name)
251+
252+
// Load the file into a session variable. This avoids re-reading the file
253+
// from disk for each chunk.
254+
_, err := m.db.ExecContext(ctx, "SET @_pitr_binlog = LOAD_FILE(?)", fullPath)
255+
if err != nil {
256+
// Cleanup on failure.
257+
os.RemoveAll(tmpDir)
258+
return nil, fmt.Errorf("connector: LOAD_FILE(%s) failed: %w (check FILE privilege)", fullPath, err)
259+
}
260+
261+
// Check whether LOAD_FILE returned NULL (file not found, no privilege, or
262+
// secure_file_priv restriction).
263+
var isNull bool
264+
if err := m.db.QueryRowContext(ctx, "SELECT @_pitr_binlog IS NULL").Scan(&isNull); err != nil {
265+
os.RemoveAll(tmpDir)
266+
return nil, fmt.Errorf("connector: verify LOAD_FILE(%s): %w", fullPath, err)
267+
}
268+
if isNull {
269+
os.RemoveAll(tmpDir)
270+
return nil, fmt.Errorf(
271+
"connector: LOAD_FILE(%s) returned NULL; the file may not exist, "+
272+
"is not world-readable, or the MySQL user lacks the FILE privilege", fullPath)
273+
}
274+
275+
// Get file size.
276+
var size int64
277+
if err := m.db.QueryRowContext(ctx, "SELECT LENGTH(@_pitr_binlog)").Scan(&size); err != nil {
278+
os.RemoveAll(tmpDir)
279+
return nil, fmt.Errorf("connector: get size of %s: %w", fullPath, err)
280+
}
281+
282+
// Read chunks and write to local file.
283+
localPath := filepath.Join(tmpDir, name)
284+
f, err := os.Create(localPath)
285+
if err != nil {
286+
os.RemoveAll(tmpDir)
287+
return nil, fmt.Errorf("connector: create local file %s: %w", localPath, err)
288+
}
289+
290+
const chunkSize = 64 * 1024 * 1024 // 64 MB per chunk
291+
for pos := int64(1); pos <= size; pos += chunkSize {
292+
var chunk []byte
293+
err := m.db.QueryRowContext(ctx, "SELECT SUBSTRING(@_pitr_binlog, ?, ?)", pos, chunkSize).Scan(&chunk)
294+
if err != nil {
295+
f.Close()
296+
os.RemoveAll(tmpDir)
297+
return nil, fmt.Errorf("connector: read chunk at pos %d of %s: %w", pos, fullPath, err)
298+
}
299+
if _, err := f.Write(chunk); err != nil {
300+
f.Close()
301+
os.RemoveAll(tmpDir)
302+
return nil, fmt.Errorf("connector: write chunk of %s: %w", localPath, err)
303+
}
304+
}
305+
f.Close()
306+
307+
localPaths = append(localPaths, localPath)
308+
}
309+
310+
return localPaths, nil
311+
}
312+
203313
// ---------------------------------------------------------------------------
204314
// ParseBinlog verifies that the requested binlog files exist and returns any
205315
// parse errors. Actual event-level parsing is delegated to the parser package

internal/connector/types.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ type ConnConfig struct {
1111
Database string
1212
// Params holds additional DSN parameters (e.g., TLS config, timeouts).
1313
Params map[string]string
14+
// MySQLBinlogPath is an optional explicit path to the mysqlbinlog binary.
15+
// If empty, the system auto-discovers it from PATH, MySQL basedir, and
16+
// common installation paths.
17+
MySQLBinlogPath string
1418
}
1519

1620
// DSN returns the MySQL Data Source Name string derived from the config.

internal/server/pitr/handler.go

Lines changed: 52 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -497,22 +497,35 @@ func (h *Handler) Progress(w http.ResponseWriter, r *http.Request) {
497497
})
498498
}
499499

500-
// findMySQLBinlog locates the mysqlbinlog binary by checking PATH first, then
501-
// querying MySQL's basedir, and finally checking common installation paths.
502-
func findMySQLBinlog(ctx context.Context, conn *connector.MySQLConnector) (string, error) {
500+
// findMySQLBinlog locates the mysqlbinlog binary. If mysqlbinlogPath is
501+
// non-empty it is validated and returned directly. Otherwise the function
502+
// checks PATH, queries MySQL basedir (if conn is non-nil), and finally
503+
// common installation paths.
504+
func findMySQLBinlog(mysqlbinlogPath string, ctx context.Context, conn *connector.MySQLConnector) (string, error) {
505+
// 0. Explicit path from config.
506+
if mysqlbinlogPath != "" {
507+
if fi, err := os.Stat(mysqlbinlogPath); err == nil && fi.Mode().IsRegular() {
508+
abs, _ := filepath.Abs(mysqlbinlogPath)
509+
return abs, nil
510+
}
511+
return "", fmt.Errorf("specified mysqlbinlog path %q does not exist", mysqlbinlogPath)
512+
}
513+
503514
// 1. Try PATH first.
504515
if path, err := exec.LookPath("mysqlbinlog"); err == nil {
505516
return path, nil
506517
}
507518

508519
// 2. Try MySQL basedir (SHOW VARIABLES LIKE 'basedir').
509-
basedir, err := conn.GetBasedir(ctx)
510-
if err == nil && basedir != "" {
511-
for _, name := range []string{"mysqlbinlog", "mysqlbinlog.exe"} {
512-
candidate := filepath.Join(basedir, "bin", name)
513-
if fi, statErr := os.Stat(candidate); statErr == nil && fi.Mode().IsRegular() {
514-
abs, _ := filepath.Abs(candidate)
515-
return abs, nil
520+
if conn != nil {
521+
basedir, err := conn.GetBasedir(ctx)
522+
if err == nil && basedir != "" {
523+
for _, name := range []string{"mysqlbinlog", "mysqlbinlog.exe"} {
524+
candidate := filepath.Join(basedir, "bin", name)
525+
if fi, statErr := os.Stat(candidate); statErr == nil && fi.Mode().IsRegular() {
526+
abs, _ := filepath.Abs(candidate)
527+
return abs, nil
528+
}
516529
}
517530
}
518531
}
@@ -533,7 +546,7 @@ func findMySQLBinlog(ctx context.Context, conn *connector.MySQLConnector) (strin
533546
}
534547
}
535548

536-
return "", fmt.Errorf("mysqlbinlog not found in $PATH, MySQL basedir, or common installation paths; install mysql-client or add mysqlbinlog to your PATH")
549+
return "", fmt.Errorf("mysqlbinlog not found; install mysql-client or specify the path via DSN parameter mysqlbinlog_path (e.g. ?mysqlbinlog_path=C:\\mysql\\bin\\mysqlbinlog.exe)")
537550
}
538551

539552
// parseBinlogRemote uses mysqlbinlog --read-from-remote-server to parse binlog
@@ -792,20 +805,32 @@ func (h *Handler) runOperation(op *Operation, operator string) {
792805
}
793806

794807
var parseRes *connector.ParseResult
808+
var downloadCleanup string // temp dir to clean up if LOAD_FILE was used
795809

796-
// Check if binlog files are accessible locally; if not, try mysqlbinlog remote.
810+
// Check if binlog files are accessible locally; if not, try alternatives.
797811
if len(paths) > 0 {
798812
if _, statErr := os.Stat(paths[0]); os.IsNotExist(statErr) {
799-
log.Printf("pitr: binlog files not accessible locally, trying mysqlbinlog remote for op %s", op.ID)
800-
mysqlbinlogPath, findErr := findMySQLBinlog(ctx, conn)
801-
if findErr != nil {
802-
h.failOperation(op, "parse binlogs (remote): %v", findErr)
803-
return
804-
}
805-
parseRes, err = h.parseBinlogRemote(mysqlbinlogPath, connCfg, binlogNames, op.TargetTable, op.RecoveryTime)
806-
if err != nil {
807-
h.failOperation(op, "parse binlogs (remote): %v", err)
808-
return
813+
// Binlog files are not on the local filesystem.
814+
// Try mysqlbinlog --read-from-remote-server first.
815+
mysqlbinlogPath, findErr := findMySQLBinlog(connCfg.MySQLBinlogPath, ctx, conn)
816+
if findErr == nil {
817+
log.Printf("pitr: trying mysqlbinlog remote for op %s", op.ID)
818+
parseRes, err = h.parseBinlogRemote(mysqlbinlogPath, connCfg, binlogNames, op.TargetTable, op.RecoveryTime)
819+
if err != nil {
820+
h.failOperation(op, "parse binlogs (remote): %v", err)
821+
return
822+
}
823+
} else {
824+
// mysqlbinlog not available. Try downloading binlog files via
825+
// MySQL LOAD_FILE() and parsing them with the Go-native parser.
826+
log.Printf("pitr: mysqlbinlog not found (%v), trying LOAD_FILE download for op %s", findErr, op.ID)
827+
downloaded, dlErr := conn.DownloadBinlogFiles(ctx, binlogDir, binlogNames)
828+
if dlErr != nil {
829+
h.failOperation(op, "cannot parse binlogs: mysqlbinlog not found and LOAD_FILE download failed: %v", dlErr)
830+
return
831+
}
832+
downloadCleanup = filepath.Dir(downloaded[0])
833+
paths = downloaded
809834
}
810835
}
811836
}
@@ -824,6 +849,11 @@ func (h *Handler) runOperation(op *Operation, operator string) {
824849
}
825850
}
826851

852+
// Clean up downloaded binlog files.
853+
if downloadCleanup != "" {
854+
os.RemoveAll(downloadCleanup)
855+
}
856+
827857
reverseSqls, err := parser.ReverseSQLBatch(parseRes.Events, nil)
828858
if err != nil {
829859
h.failOperation(op, "generate reverse SQL: %v", err)

0 commit comments

Comments
 (0)