Merge branch 'master' into release-1.0.1

All changes on master are bugfixes/doc changes.

Author: codysoyland

Makefile

@@ -25,7 +25,7 @@ clean:
 # Set up vendor directory using `dep`
 vendor: Gopkg.toml
 	$(MAKE) require-dep
-	dep ensure
+	dep ensure -vendor-only
 	touch vendor
 
 # Run test suite

docs/webui.md docs/console.md

@@ -1,42 +1,44 @@
 +++
-title = "WebUI"
+title = "Console"
 weight = 9
 nav = [
-    "Console",
+    "Installation",
+    "Query",
     "Cluster Admin",
 ]
 +++
 
-## WebUI
+## Console
 
-A web-based app called Pilosa WebUI is available in a separate package. This can be used for constructing queries and viewing the cluster status.
+A web-based app called Pilosa Console is available in a separate package. This can be used for constructing queries and viewing the cluster status.
 
 ### Installation
 
-Releases are [available on Github](https://github.com/pilosa/webui/releases) as well as on [Homebrew](https://brew.sh/) for Mac.
+Releases are [available on Github](https://github.com/pilosa/console/releases) as well as on [Homebrew](https://brew.sh/) for Mac.
 
 Installing on a Mac with Homebrew is simple; just run:
 
 ```
-brew install pilosa-webui
+brew tap pilosa/homebrew-pilosa
+brew install pilosa-console
 ```
 
-You may also build from source by checking out the [repo on Github](https://github.com/pilosa/webui) and running:
+You may also build from source by checking out the [repo on Github](https://github.com/pilosa/console) and running:
 
 ```
 make install
 ```
 
-### Console
+### Query
 
-The Console view allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
+The Query tab allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
 
 Each query's result will be displayed in the Output section along with the query time. 
 
 The Console will keep a record of each query and its result with the latest query on top.
 
-![webUI console screenshot](/img/docs/webui-console.png)
-*WebUI console screenshot*
+![Console screenshot](/img/docs/webui-console.png)
+*Console query screenshot*
 
 In addition to standard PQL, the console supports a few special commands, prefixed with `:`.
 

encoding/proto/proto.go

@@ -961,6 +961,7 @@ func encodeColumnAttrSets(a []*pilosa.ColumnAttrSet) []*internal.ColumnAttrSet {
 func encodeColumnAttrSet(set *pilosa.ColumnAttrSet) *internal.ColumnAttrSet {
 	return &internal.ColumnAttrSet{
 		ID:    set.ID,
+		Key:   set.Key,
 		Attrs: encodeAttrs(set.Attrs),
 	}
 }
@@ -972,6 +973,7 @@ func encodeRow(r *pilosa.Row) *internal.Row {
 
 	return &internal.Row{
 		Columns: r.Columns(),
+		Keys:    r.Keys,
 		Attrs:   encodeAttrs(r.Attrs),
 	}
 }

executor_test.go

@@ -1279,7 +1279,7 @@ func TestExecutor_Time_Clear_Quantums(t *testing.T) {
 		expected []uint64
 	}{
 		{quantum: "Y", expected: []uint64{3, 4, 5, 6}},
-		{quantum: "M", expected: []uint64{3, 4, 6}},
+		{quantum: "M", expected: []uint64{3, 4, 5, 6}},
 		{quantum: "D", expected: []uint64{3, 4, 5, 6}},
 		{quantum: "H", expected: []uint64{3, 4, 5, 6, 7}},
 		{quantum: "YM", expected: []uint64{3, 4, 5, 6}},

gossip/gossip.go

@@ -93,6 +93,17 @@ func (g *memberSet) Open() (err error) {
 	return nil
 }
 
+// Close attempts to gracefully leave the cluster, and finally calls shutdown
+// after (at most) a timeout period.
+func (g *memberSet) Close() error {
+	leaveErr := g.memberlist.Leave(5 * time.Second)
+	shutdownErr := g.memberlist.Shutdown()
+	if leaveErr != nil || shutdownErr != nil {
+		return fmt.Errorf("leaving: '%v', shutting down: '%v'", leaveErr, shutdownErr)
+	}
+	return nil
+}
+
 // joinWithRetry wraps the standard memberlist Join function in a retry.
 func (g *memberSet) joinWithRetry(hosts []string) error {
 	err := retry(60, 2*time.Second, func() error {

http/handler.go

@@ -55,6 +55,8 @@ type Handler struct {
 
 	ln net.Listener
 
+	closeTimeout time.Duration
+
 	server *http.Server
 }
 
@@ -109,10 +111,20 @@ func OptHandlerListener(ln net.Listener) handlerOption {
 	}
 }
 
+// OptHandlerCloseTimeout controls how long to wait for the http Server to
+// shutdown cleanly before forcibly destroying it. Default is 30 seconds.
+func OptHandlerCloseTimeout(d time.Duration) handlerOption {
+	return func(h *Handler) error {
+		h.closeTimeout = d
+		return nil
+	}
+}
+
 // NewHandler returns a new instance of Handler with a default logger.
 func NewHandler(opts ...handlerOption) (*Handler, error) {
 	handler := &Handler{
-		logger: pilosa.NopLogger,
+		logger:       pilosa.NopLogger,
+		closeTimeout: time.Second * 30,
 	}
 	handler.Handler = newRouter(handler)
 	handler.populateValidators()
@@ -146,10 +158,16 @@ func (h *Handler) Serve() error {
 	return nil
 }
 
+// Close tries to cleanly shutdown the HTTP server, and failing that, after a
+// timeout, calls Server.Close.
 func (h *Handler) Close() error {
-	// TODO: timeout?
-	err := h.server.Shutdown(context.Background())
-	return errors.Wrap(err, "shutdown http server")
+	deadlineCtx, cancelFunc := context.WithDeadline(context.Background(), time.Now().Add(h.closeTimeout))
+	defer cancelFunc()
+	err := h.server.Shutdown(deadlineCtx)
+	if err != nil {
+		err = h.server.Close()
+	}
+	return errors.Wrap(err, "shutdown/close http server")
 }
 
 func (h *Handler) populateValidators() {

server.go

@@ -369,17 +369,28 @@ func (s *Server) Close() error {
 	close(s.closing)
 	s.wg.Wait()
 
+	var errh error
+	var errt error
+	var errc error
 	if s.cluster != nil {
-		s.cluster.close()
+		errc = s.cluster.close()
 	}
 	if s.holder != nil {
-		s.holder.Close()
+		errh = s.holder.Close()
 	}
 	if s.translateFile != nil {
-		s.translateFile.Close()
-	}
-
-	return nil
+		errt = s.translateFile.Close()
+	}
+	// prefer to return holder error over translateFile error over cluster
+	// error. This order is somewhat arbitrary. It would be better if we had
+	// some way to combine all the errors, but probably not important enough to
+	// warrant the extra complexity.
+	if errh != nil {
+		return errors.Wrap(errh, "closing holder")
+	} else if errt != nil {
+		return errors.Wrap(errt, "closing translateFile")
+	}
+	return errors.Wrap(errc, "closing cluster")
 }
 
 // loadNodeID gets NodeID from disk, or creates a new value.

server/server.go

@@ -20,7 +20,7 @@
 package server
 
 import (
-	"fmt"
+	"crypto/tls"
 	"io"
 	"log"
 	"math/rand"
@@ -31,7 +31,7 @@ import (
 	"syscall"
 	"time"
 
-	"crypto/tls"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/boltdb"
@@ -62,6 +62,7 @@ type Command struct {
 
 	// Gossip transport
 	gossipTransport *gossip.Transport
+	gossipMemberSet io.Closer
 
 	// Standard input/output
 	*pilosa.CmdIO
@@ -75,9 +76,10 @@ type Command struct {
 	logOutput io.Writer
 	logger    loggerLogger
 
-	Handler pilosa.Handler
-	API     *pilosa.API
-	ln      net.Listener
+	Handler      pilosa.Handler
+	API          *pilosa.API
+	ln           net.Listener
+	closeTimeout time.Duration
 
 	serverOptions []pilosa.ServerOption
 }
@@ -91,6 +93,13 @@ func OptCommandServerOptions(opts ...pilosa.ServerOption) CommandOption {
 	}
 }
 
+func OptCommandCloseTimeout(d time.Duration) CommandOption {
+	return func(c *Command) error {
+		c.closeTimeout = d
+		return nil
+	}
+}
+
 // NewCommand returns a new instance of Main.
 func NewCommand(stdin io.Reader, stdout, stderr io.Writer, opts ...CommandOption) *Command {
 	c := &Command{
@@ -294,6 +303,7 @@ func (m *Command) SetupServer() error {
 		http.OptHandlerAPI(m.API),
 		http.OptHandlerLogger(m.logger),
 		http.OptHandlerListener(m.ln),
+		http.OptHandlerCloseTimeout(m.closeTimeout),
 	)
 	return errors.Wrap(err, "new handler")
 
@@ -326,6 +336,8 @@ func (m *Command) setupNetworking() error {
 	if err != nil {
 		return errors.Wrap(err, "getting memberset")
 	}
+	m.gossipMemberSet = gossipMemberSet
+
 	return errors.Wrap(gossipMemberSet.Open(), "opening gossip memberset")
 }
 
@@ -338,17 +350,18 @@ func (m *Command) GossipTransport() *gossip.Transport {
 
 // Close shuts down the server.
 func (m *Command) Close() error {
-	var logErr error
-	handlerErr := m.Handler.Close()
-	serveErr := m.Server.Close()
-	if closer, ok := m.logOutput.(io.Closer); ok {
-		logErr = closer.Close()
+	defer close(m.done)
+	eg := errgroup.Group{}
+	eg.Go(m.Handler.Close)
+	eg.Go(m.Server.Close)
+	if m.gossipMemberSet != nil {
+		eg.Go(m.gossipMemberSet.Close)
 	}
-	close(m.done)
-	if serveErr != nil || logErr != nil || handlerErr != nil {
-		return fmt.Errorf("closing server: '%v', closing logs: '%v', closing handler: '%v'", serveErr, logErr, handlerErr)
+	if closer, ok := m.logOutput.(io.Closer); ok {
+		eg.Go(closer.Close)
 	}
-	return nil
+	err := eg.Wait()
+	return errors.Wrap(err, "closing everything")
 }
 
 // newStatsClient creates a stats client from the config

test/pilosa.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/pilosa/pilosa/http"
 	"github.com/pilosa/pilosa/server"
@@ -55,6 +56,11 @@ func newCommand(opts ...server.CommandOption) *Command {
 		panic(err)
 	}
 
+	// set aggressive close timeout by default to avoid hanging tests. This was
+	// a problem with PDK tests which used go-pilosa as well. We put it at the
+	// beginning of the option slice so that it can be overridden by user-passed
+	// options.
+	opts = append([]server.CommandOption{server.OptCommandCloseTimeout(time.Millisecond * 2)}, opts...)
 	m := &Command{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr, opts...), commandOptions: opts}
 	m.Config.DataDir = path
 	m.Config.Bind = "http://localhost:0"

time.go

@@ -140,7 +140,7 @@ func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string
 					break
 				} else if t.Month() != 1 {
 					results = append(results, viewByTimeUnit(name, t, 'M'))
-					t = t.AddDate(0, 1, 0)
+					t = addMonth(t)
 					continue
 				}
 			}
@@ -159,7 +159,7 @@ func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string
 			t = t.AddDate(1, 0, 0)
 		} else if hasMonth && nextMonthGTE(t, end) {
 			results = append(results, viewByTimeUnit(name, t, 'M'))
-			t = t.AddDate(0, 1, 0)
+			t = addMonth(t)
 		} else if hasDay && nextDayGTE(t, end) {
 			results = append(results, viewByTimeUnit(name, t, 'D'))
 			t = t.AddDate(0, 0, 1)
@@ -174,6 +174,19 @@ func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string
 	return results
 }
 
+// addMonth adds a month similar to time.AddDate(0, 1, 0), but
+// in certain edge cases it doesn't normalize for days late in the month.
+// In the "YM" case where t.Day is greater than 28, there are
+// edge cases where using time.AddDate() to add a month will result
+// in two "months" being added (Jan 31 + 1mo = March 2).
+func addMonth(t time.Time) time.Time {
+	if t.Day() > 28 {
+		t = time.Date(t.Year(), t.Month(), 1, t.Hour(), 0, 0, 0, t.Location())
+	}
+	t = t.AddDate(0, 1, 0)
+	return t
+}
+
 func nextYearGTE(t time.Time, end time.Time) bool {
 	next := t.AddDate(1, 0, 0)
 	if next.Year() == end.Year() {

time_internal_test.go

@@ -97,6 +97,24 @@ func TestViewsByTimeRange(t *testing.T) {
 			t.Fatalf("unexpected fields: %#v", a)
 		}
 	})
+	t.Run("YM31up", func(t *testing.T) {
+		a := viewsByTimeRange("F", mustParseTime("2001-10-31 00:00"), mustParseTime("2003-04-01 00:00"), mustParseTimeQuantum("YM"))
+		if !reflect.DeepEqual(a, []string{"F_200110", "F_200111", "F_200112", "F_2002", "F_200301", "F_200302", "F_200303"}) {
+			t.Fatalf("unexpected fields: %#v", a)
+		}
+	})
+	t.Run("YM31mid", func(t *testing.T) {
+		a := viewsByTimeRange("F", mustParseTime("1999-12-31 00:00"), mustParseTime("2000-04-01 00:00"), mustParseTimeQuantum("YM"))
+		if !reflect.DeepEqual(a, []string{"F_199912", "F_200001", "F_200002", "F_200003"}) {
+			t.Fatalf("unexpected fields: %#v", a)
+		}
+	})
+	t.Run("YM31down", func(t *testing.T) {
+		a := viewsByTimeRange("F", mustParseTime("2000-01-31 00:00"), mustParseTime("2001-04-01 00:00"), mustParseTimeQuantum("YM"))
+		if !reflect.DeepEqual(a, []string{"F_2000", "F_200101", "F_200102", "F_200103"}) {
+			t.Fatalf("unexpected fields: %#v", a)
+		}
+	})
 	t.Run("YMD", func(t *testing.T) {
 		a := viewsByTimeRange("F", mustParseTime("2000-11-28 00:00"), mustParseTime("2003-03-02 00:00"), mustParseTimeQuantum("YMD"))
 		if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) {