Enhance bufferedWriter with bufio.Writer, scratch space, and configurable thresholds

- Add bufio.Writer layer after temp file threshold is crossed, reducing disk
  write syscalls from per-Write to per-buffer-full (default 128 KiB)
- Add scratch [24]byte field for strconv.Append* to avoid heap allocations
  in WriteInt, WriteUint, and WriteFloat helper methods
- Track total bytes written for offset-based operations (WriteAt)
- Add WriteAt method for updating data at specific offsets in both in-memory
  and temp file modes
- Add CopyTo method with large read buffer to minimize Pread syscalls when
  copying data to ZIP writers
- Add Bytes method for direct access to in-memory buffer contents
- Add Reset method for clearing state without closing the temp file
- Add StreamingChunkSize option to control when streaming spills to disk
  (0 = default 16 MiB, -1 = never spill, keeps all data in memory)
- Add StreamingBufSize option to control bufio.Writer size aft- Add StreamingBufSize option to control bufio.Writer size aft- Add StreamingBufSize option to control bufio.Writer size aft- Add Streaminak memory usage during streaming writes

Author: AdamDrewsTR

excelize.go

@@ -122,6 +122,20 @@ type Options struct {
 	LongDatePattern   string
 	LongTimePattern   string
 	CultureInfo       CultureName
+	// StreamingChunkSize is the number of bytes of XML data accumulated in
+	// memory before a streaming worksheet spills to a temp file. A smaller
+	// value reduces peak memory usage at the cost of more disk I/O. Zero
+	// means use the default (StreamChunkSize = 16 MiB). Set to -1 to
+	// disable temp files entirely (all data stays in memory); this
+	// eliminates disk I/O overhead and can be significantly faster when
+	// sufficient memory is available.
+	StreamingChunkSize int
+	// StreamingBufSize is the size of the bufio.Writer used for all disk
+	// writes after the StreamingChunkSize threshold is crossed. Larger values
+	// reduce write syscall counts at the cost of slightly more memory. The
+	// measured inflection point on NVMe and HDD alike is 128 KiB. Zero means
+	// use the default (StreamingBufSizeDefault = 128 KiB).
+	StreamingBufSize int
 }
 
 // OpenFile take the name of a spreadsheet file and returns a populated

stream.go

@@ -12,10 +12,12 @@
 package excelize
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/xml"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"reflect"
 	"strconv"
@@ -119,11 +121,26 @@ func (f *File) NewStreamWriter(sheet string) (*StreamWriter, error) {
 	if sheetID == -1 {
 		return nil, ErrSheetNotExist{sheet}
 	}
+	chunkSize := f.options.StreamingChunkSize
+	switch {
+	case chunkSize < 0:
+		chunkSize = math.MaxInt // never spill to disk
+	case chunkSize == 0:
+		chunkSize = StreamChunkSize
+	}
+	bufSize := f.options.StreamingBufSize
+	if bufSize <= 0 {
+		bufSize = StreamingBufSizeDefault
+	}
 	sw := &StreamWriter{
 		file:    f,
 		Sheet:   sheet,
 		SheetID: sheetID,
-		rawData: bufferedWriter{tmpDir: f.options.TmpDir},
+		rawData: bufferedWriter{
+			tmpDir:    f.options.TmpDir,
+			flushSize: chunkSize,
+			bioSize:   bufSize,
+		},
 	}
 	var err error
 	sw.worksheet, err = f.workSheetReader(sheet)
@@ -791,19 +808,126 @@ func bulkAppendFields(w io.Writer, ws *xlsxWorksheet, from, to int) {
 // is written to the temp file with Sync, which may return an error.
 // Therefore, Sync should be periodically called and the error checked.
 type bufferedWriter struct {
-	tmpDir string
-	tmp    *os.File
-	buf    bytes.Buffer
+	tmpDir    string
+	tmp       *os.File
+	buf       bytes.Buffer  // used before temp file is created
+	bio       *bufio.Writer // used after temp file is created
+	scratch   [24]byte      // scratch space for strconv.Append* to avoid heap allocs
+	flushSize int           // if >0, flush to temp file at this threshold instead of StreamChunkSize
+	bioSize   int           // bufio.Writer buffer size after threshold; 0 = use StreamingBufSizeDefault
+	written   int64         // total bytes written (for tracking offsets)
 }
 
-// Write to the in-memory buffer. The error is always nil.
+// Write to the active writer (bufio if streaming, otherwise in-memory buffer).
 func (bw *bufferedWriter) Write(p []byte) (n int, err error) {
-	return bw.buf.Write(p)
+	if bw.bio != nil {
+		n, err = bw.bio.Write(p)
+	} else {
+		n, err = bw.buf.Write(p)
+	}
+	bw.written += int64(n)
+	return
 }
 
-// WriteString write to the in-memory buffer. The error is always nil.
+// WriteString writes to the active writer.
 func (bw *bufferedWriter) WriteString(p string) (n int, err error) {
-	return bw.buf.WriteString(p)
+	if bw.bio != nil {
+		n, err = bw.bio.WriteString(p)
+	} else {
+		n, err = bw.buf.WriteString(p)
+	}
+	bw.written += int64(n)
+	return
+}
+
+// WriteInt formats and writes an int64 directly using the scratch space,
+// avoiding a heap-allocated string.
+func (bw *bufferedWriter) WriteInt(v int64) {
+	b := strconv.AppendInt(bw.scratch[:0], v, 10)
+	if bw.bio != nil {
+		bw.bio.Write(b)
+	} else {
+		bw.buf.Write(b)
+	}
+	bw.written += int64(len(b))
+}
+
+// WriteUint formats and writes a uint64 directly to the buffer.
+func (bw *bufferedWriter) WriteUint(v uint64) {
+	b := strconv.AppendUint(bw.scratch[:0], v, 10)
+	if bw.bio != nil {
+		bw.bio.Write(b)
+	} else {
+		bw.buf.Write(b)
+	}
+	bw.written += int64(len(b))
+}
+
+// WriteFloat formats and writes a float64 directly to the buffer.
+func (bw *bufferedWriter) WriteFloat(v float64, fmt byte, prec, bitSize int) {
+	b := strconv.AppendFloat(bw.scratch[:0], v, fmt, prec, bitSize)
+	if bw.bio != nil {
+		bw.bio.Write(b)
+	} else {
+		bw.buf.Write(b)
+	}
+	bw.written += int64(len(b))
+}
+
+// Bytes returns the in-memory buffer contents. This is only valid when no
+// temp file has been created (i.e. for small worksheets). Once streaming to
+// disk, this returns nil.
+func (bw *bufferedWriter) Bytes() []byte {
+	if bw.tmp != nil {
+		return nil
+	}
+	return bw.buf.Bytes()
+}
+
+// WriteAt writes data at a specific offset. For in-memory buffers, it
+// modifies the buffer directly. For temp files, it flushes first then
+// uses pwrite. The data must fit within previously written bytes.
+func (bw *bufferedWriter) WriteAt(p []byte, offset int64) error {
+	if bw.tmp == nil {
+		// In-memory: directly modify buffer bytes
+		buf := bw.buf.Bytes()
+		if offset+int64(len(p)) > int64(len(buf)) {
+			return fmt.Errorf("WriteAt: offset %d + len %d exceeds buffer size %d", offset, len(p), len(buf))
+		}
+		copy(buf[offset:], p)
+		return nil
+	}
+	// Temp file: flush bufio first, then use pwrite
+	if err := bw.Flush(); err != nil {
+		return err
+	}
+	_, err := bw.tmp.WriteAt(p, offset)
+	return err
+}
+
+// CopyTo efficiently copies all buffered data to w. For in-memory buffers
+// this is a simple WriteTo. For temp files this uses a large read buffer to
+// minimize syscalls.
+func (bw *bufferedWriter) CopyTo(w io.Writer) (int64, error) {
+	if bw.tmp == nil {
+		return io.Copy(w, bytes.NewReader(bw.buf.Bytes()))
+	}
+	if err := bw.Flush(); err != nil {
+		return 0, err
+	}
+	if _, err := bw.tmp.Seek(0, 0); err != nil {
+		return 0, err
+	}
+	// Use a large read buffer to batch Pread syscalls. Without this,
+	// io.Copy uses 32 KB reads, generating thousands of syscalls for
+	// large worksheets (e.g. 100 MB XML → 3000+ syscalls). A 256 KB
+	// buffer reduces that to ~400.
+	readBufSize := 256 * 1024
+	if bw.bioSize > readBufSize {
+		readBufSize = bw.bioSize
+	}
+	br := bufio.NewReaderSize(bw.tmp, readBufSize)
+	return io.Copy(w, br)
 }
 
 // Reader provides read-access to the underlying buffer/file.
@@ -823,10 +947,16 @@ func (bw *bufferedWriter) Reader() (io.Reader, error) {
 }
 
 // Sync will write the in-memory buffer to a temp file, if the in-memory
-// buffer has grown large enough. Any error will be returned.
+// buffer has grown large enough. Once the temp file is created, all
+// subsequent writes go through the bufio.Writer and the bytes.Buffer is
+// released.
 func (bw *bufferedWriter) Sync() (err error) {
-	// Try to use local storage
-	if bw.buf.Len() < StreamChunkSize {
+	// Already streaming to disk via bufio.Writer — nothing to do here;
+	// the final Flush() call will drain any remaining bytes.
+	if bw.bio != nil {
+		return nil
+	}
+	if bw.buf.Len() < bw.flushSize {
 		return nil
 	}
 	if bw.tmp == nil {
@@ -836,26 +966,47 @@ func (bw *bufferedWriter) Sync() (err error) {
 			return nil
 		}
 	}
-	return bw.Flush()
+	// Drain the in-memory buffer to the temp file
+	if _, err = bw.buf.WriteTo(bw.tmp); err != nil {
+		return err
+	}
+	// Release the bytes.Buffer backing array entirely
+	bw.buf = bytes.Buffer{}
+	// Switch to bufio.Writer for all future writes
+	bw.bio = bufio.NewWriterSize(bw.tmp, bw.bioSize)
+	return nil
 }
 
-// Flush the entire in-memory buffer to the temp file, if a temp file is being
-// used.
+// Flush ensures all buffered data is written to the temp file.
 func (bw *bufferedWriter) Flush() error {
 	if bw.tmp == nil {
 		return nil
 	}
+	if bw.bio != nil {
+		return bw.bio.Flush()
+	}
 	_, err := bw.buf.WriteTo(bw.tmp)
 	if err != nil {
 		return err
 	}
-	bw.buf.Reset()
+	bw.buf = bytes.Buffer{}
 	return nil
 }
 
+// Reset clears all buffered data (in-memory and bufio) without closing the
+// temp file.
+func (bw *bufferedWriter) Reset() {
+	bw.buf.Reset()
+	if bw.bio != nil {
+		bw.bio.Reset(&bw.buf) // detach from temp file
+		bw.bio = nil
+	}
+}
+
 // Close the underlying temp file and reset the in-memory buffer.
 func (bw *bufferedWriter) Close() error {
 	bw.buf.Reset()
+	bw.bio = nil
 	if bw.tmp == nil {
 		return nil
 	}

templates.go

@@ -186,6 +186,7 @@ const (
 	MinColumns              = 1
 	MinFontSize             = 1
 	StreamChunkSize         = 1 << 24
+	StreamingBufSizeDefault = 128 << 10
 	TotalCellChars          = 32767
 	TotalRows               = 1048576
 	TotalSheetHyperlinks    = 65529