batchWriter creates buffers as required, without any constraint

[xiam]

Consider the following program that creates 500 concurrent requests to slightly different URLs:

package main

import (
        "crypto/rand"
        "errors"
        "fmt"
        //"github.com/goware/httpcoala"
        "github.com/pressly/chi"
        "github.com/pressly/chi/middleware"
        "log"
        "net/http"
        "sync"
        "time"
)

var mockData []byte

func init() {
        mockData = make([]byte, 1024*1024*20)
        rand.Read(mockData)
}

func doRequest(i int) error {
        res, err := http.Get(fmt.Sprintf("http://127.0.0.1:1111/?_=%d", i))
        if err != nil {
                return err
        }
        defer res.Body.Close()

        if res.StatusCode != 200 {
                return errors.New("Expecting 200 OK.")
        }

        return err
}

func main() {

        r := chi.NewRouter()

        r.Use(middleware.Logger)
        //r.Use(httpcoala.Route("HEAD", "GET")) // or, Route("*")

        r.Get("/", func(w http.ResponseWriter, r *http.Request) {
                time.Sleep(time.Second * 2) // expensive op
                w.WriteHeader(200)
                w.Write(mockData)
        })

        go http.ListenAndServe(":1111", r)

        time.Sleep(time.Second * 1)

        var wg sync.WaitGroup

        for i := 0; i < 500; i++ {
                wg.Add(1)
                go func(i int) {
                        if err := doRequest(i); err != nil {
                                log.Fatal("doRequest: ", err)
                        }
                        wg.Done()
                }(i)
        }

        wg.Wait()
}

If you run this program on a 2G vm nothing interesting happens, but if you remove the commented out lines to use httpcoala and run the program again you'll see something like this:

You'll probably have to ^C it to prevent the vm from crashing.

I think the problem is that httpcoala tries to allocate as many buffers as possible, without imposing any hard limit.