Skip to content

Commit b88a386

Browse files
tschaubtschaub
committed
Optionally use stdin/stdout for validate, describe, and convert commands
1 parent 859b789 commit b88a386

14 files changed

Lines changed: 486 additions & 117 deletions

File tree

‎cmd/gpq/command/command.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package command
2+
3+
import "io"
4+
5+
var CLI struct {
6+
Convert ConvertCmd `cmd:"" help:"Convert data from one format to another."`
7+
Validate ValidateCmd `cmd:"" help:"Validate a GeoParquet file."`
8+
Describe DescribeCmd `cmd:"" help:"Describe a GeoParquet file."`
9+
Version VersionCmd `cmd:"" help:"Print the version of this program."`
10+
}
11+
12+
type ReaderAtSeeker interface {
13+
io.Reader
14+
io.ReaderAt
15+
io.Seeker
16+
}

‎cmd/gpq/command/command_test.go‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package command_test
2+
3+
import (
4+
"io"
5+
"os"
6+
"testing"
7+
8+
"github.com/stretchr/testify/suite"
9+
)
10+
11+
type Suite struct {
12+
suite.Suite
13+
originalStdin *os.File
14+
mockStdin *os.File
15+
originalStdout *os.File
16+
mockStdout *os.File
17+
}
18+
19+
func (s *Suite) SetupTest() {
20+
stdin, err := os.CreateTemp("", "stdin")
21+
s.Require().NoError(err)
22+
s.originalStdin = os.Stdin
23+
s.mockStdin = stdin
24+
os.Stdin = stdin
25+
26+
stdout, err := os.CreateTemp("", "stdout")
27+
s.Require().NoError(err)
28+
s.originalStdout = os.Stdout
29+
s.mockStdout = stdout
30+
os.Stdout = stdout
31+
}
32+
33+
func (s *Suite) writeStdin(data []byte) {
34+
_, writeErr := s.mockStdin.Write(data)
35+
s.Require().NoError(writeErr)
36+
_, seekErr := s.mockStdin.Seek(0, 0)
37+
s.Require().NoError(seekErr)
38+
}
39+
40+
func (s *Suite) readStdout() []byte {
41+
if _, seekErr := s.mockStdout.Seek(0, 0); seekErr != nil {
42+
// assume the file is closed
43+
stdout, err := os.Open(s.mockStdout.Name())
44+
s.Require().NoError(err)
45+
s.mockStdout = stdout
46+
}
47+
data, err := io.ReadAll(s.mockStdout)
48+
s.Require().NoError(err)
49+
return data
50+
}
51+
52+
func (s *Suite) TearDownTest() {
53+
os.Stdout = s.originalStdout
54+
os.Stdin = s.originalStdin
55+
56+
_ = s.mockStdin.Close()
57+
s.NoError(os.Remove(s.mockStdin.Name()))
58+
59+
_ = s.mockStdout.Close()
60+
s.NoError(os.Remove(s.mockStdout.Name()))
61+
}
62+
63+
func TestSuite(t *testing.T) {
64+
suite.Run(t, &Suite{})
65+
}

cmd/gpq/convert.go renamed to cmd/gpq/command/convert.go

Lines changed: 58 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,13 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
package main
15+
package command
1616

1717
import (
18+
"bytes"
1819
"errors"
1920
"fmt"
21+
"io"
2022
"os"
2123
"strings"
2224

@@ -25,9 +27,9 @@ import (
2527
)
2628

2729
type ConvertCmd struct {
28-
Input string `arg:"" name:"input" help:"Input file." type:"existingfile"`
30+
Input string `arg:"" optional:"" name:"input" help:"Input file. If not provided, input is read from stdin." type:"path"`
2931
From string `help:"Input file format. Possible values: ${enum}." enum:"auto, geojson, geoparquet, parquet" default:"auto"`
30-
Output string `arg:"" name:"output" help:"Output file." type:"path"`
32+
Output string `arg:"" optional:"" name:"output" help:"Output file. If not provided, output is written to stdout." type:"path"`
3133
To string `help:"Output file format. Possible values: ${enum}." enum:"auto, geojson, geoparquet" default:"auto"`
3234
Min int `help:"Minimum number of features to consider when building a schema." default:"10"`
3335
Max int `help:"Maximum number of features to consider when building a schema." default:"100"`
@@ -53,6 +55,9 @@ var validTypes = map[FormatType]bool{
5355
}
5456

5557
func parseFormatType(format string) FormatType {
58+
if format == "" {
59+
return AutoType
60+
}
5661
ft := FormatType(strings.ToLower(format))
5762
if !validTypes[ft] {
5863
return UnknownType
@@ -73,34 +78,72 @@ func getFormatType(filename string) FormatType {
7378
return UnknownType
7479
}
7580

81+
func hasStdin() bool {
82+
stats, err := os.Stdin.Stat()
83+
if err != nil {
84+
return false
85+
}
86+
return stats.Size() > 0
87+
}
88+
7689
func (c *ConvertCmd) Run() error {
90+
inputSource := c.Input
91+
outputSource := c.Output
92+
93+
if outputSource == "" && hasStdin() {
94+
outputSource = inputSource
95+
inputSource = ""
96+
}
97+
7798
outputFormat := parseFormatType(c.To)
7899
if outputFormat == AutoType {
79-
outputFormat = getFormatType(c.Output)
100+
if outputSource == "" {
101+
return fmt.Errorf("when writing to stdout, the --to option must be provided to determine the output format")
102+
}
103+
outputFormat = getFormatType(outputSource)
80104
}
81105
if outputFormat == UnknownType {
82-
return fmt.Errorf("could not determine output format for %s", c.Output)
106+
return fmt.Errorf("could not determine output format for %s", outputSource)
83107
}
84108

85109
inputFormat := parseFormatType(c.From)
86110
if inputFormat == AutoType {
87-
inputFormat = getFormatType(c.Input)
111+
if inputSource == "" {
112+
return fmt.Errorf("when reading from stdin, the --from option must be provided to determine the input format")
113+
}
114+
inputFormat = getFormatType(inputSource)
88115
}
89116
if inputFormat == UnknownType {
90-
return fmt.Errorf("could not determine input format for %s", c.Input)
117+
return fmt.Errorf("could not determine input format for %s", inputSource)
91118
}
92119

93-
input, readErr := os.Open(c.Input)
94-
if readErr != nil {
95-
return fmt.Errorf("failed to read from %q: %w", c.Input, readErr)
120+
var input ReaderAtSeeker
121+
if inputSource == "" {
122+
data, err := io.ReadAll(os.Stdin)
123+
if err != nil {
124+
return fmt.Errorf("trouble reading from stdin: %w", err)
125+
}
126+
input = bytes.NewReader(data)
127+
} else {
128+
i, readErr := os.Open(inputSource)
129+
if readErr != nil {
130+
return fmt.Errorf("failed to read from %q: %w", inputSource, readErr)
131+
}
132+
defer i.Close()
133+
input = i
96134
}
97-
defer input.Close()
98135

99-
output, createErr := os.Create(c.Output)
100-
if createErr != nil {
101-
return fmt.Errorf("failed to open %q for writing: %w", c.Output, createErr)
136+
var output *os.File
137+
if outputSource == "" {
138+
output = os.Stdout
139+
} else {
140+
o, createErr := os.Create(outputSource)
141+
if createErr != nil {
142+
return fmt.Errorf("failed to open %q for writing: %w", outputSource, createErr)
143+
}
144+
defer o.Close()
145+
output = o
102146
}
103-
defer output.Close()
104147

105148
if inputFormat == GeoJSONType {
106149
if outputFormat != ParquetType && outputFormat != GeoParquetType {

‎cmd/gpq/command/convert_test.go‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
package command_test
2+
3+
import (
4+
"bytes"
5+
"encoding/json"
6+
7+
"github.com/apache/arrow/go/v14/parquet/file"
8+
"github.com/planetlabs/gpq/cmd/gpq/command"
9+
"github.com/planetlabs/gpq/internal/geo"
10+
"github.com/planetlabs/gpq/internal/test"
11+
)
12+
13+
func (s *Suite) TestConvertGeoParquetToGeoJSONStdout() {
14+
cmd := &command.ConvertCmd{
15+
From: "auto",
16+
Input: "../../../internal/testdata/cases/example-v1.0.0.parquet",
17+
To: "geojson",
18+
}
19+
20+
s.Require().NoError(cmd.Run())
21+
data := s.readStdout()
22+
23+
collection := &geo.FeatureCollection{}
24+
s.Require().NoError(json.Unmarshal(data, collection))
25+
s.Len(collection.Features, 5)
26+
}
27+
28+
func (s *Suite) TestConvertGeoJSONToGeoParquetStdout() {
29+
cmd := &command.ConvertCmd{
30+
From: "auto",
31+
Input: "../../../internal/geojson/testdata/example.geojson",
32+
To: "parquet",
33+
}
34+
35+
s.Require().NoError(cmd.Run())
36+
data := s.readStdout()
37+
38+
fileReader, err := file.NewParquetReader(bytes.NewReader(data))
39+
s.Require().NoError(err)
40+
defer fileReader.Close()
41+
42+
s.Equal(int64(5), fileReader.NumRows())
43+
}
44+
45+
func (s *Suite) TestConvertGeoParquetToUnknownStdout() {
46+
cmd := &command.ConvertCmd{
47+
From: "auto",
48+
Input: "../../../internal/testdata/cases/example-v1.0.0.parquet",
49+
}
50+
51+
s.ErrorContains(cmd.Run(), "when writing to stdout, the --to option must be provided")
52+
}
53+
54+
func (s *Suite) TestConvertGeoJSONStdinToGeoParquetStdout() {
55+
s.writeStdin([]byte(`{
56+
"type": "FeatureCollection",
57+
"features": [
58+
{
59+
"type": "Feature",
60+
"properties": {
61+
"name": "Null Island"
62+
},
63+
"geometry": {
64+
"type": "Point",
65+
"coordinates": [0, 0]
66+
}
67+
}
68+
]
69+
}`))
70+
71+
cmd := &command.ConvertCmd{
72+
From: "geojson",
73+
To: "geoparquet",
74+
}
75+
76+
s.Require().NoError(cmd.Run())
77+
data := s.readStdout()
78+
79+
fileReader, err := file.NewParquetReader(bytes.NewReader(data))
80+
s.Require().NoError(err)
81+
defer fileReader.Close()
82+
83+
s.Equal(int64(1), fileReader.NumRows())
84+
}
85+
86+
func (s *Suite) TestConvertGeoParquetStdinToGeoJSONStdout() {
87+
s.writeStdin(test.GeoParquetFromJSON(s.T(), `{
88+
"type": "FeatureCollection",
89+
"features": [
90+
{
91+
"type": "Feature",
92+
"properties": {
93+
"name": "Null Island"
94+
},
95+
"geometry": {
96+
"type": "Point",
97+
"coordinates": [0, 0]
98+
}
99+
}
100+
]
101+
}`))
102+
103+
cmd := &command.ConvertCmd{
104+
From: "geoparquet",
105+
To: "geojson",
106+
}
107+
108+
s.Require().NoError(cmd.Run())
109+
data := s.readStdout()
110+
111+
collection := &geo.FeatureCollection{}
112+
s.Require().NoError(json.Unmarshal(data, collection))
113+
s.Len(collection.Features, 1)
114+
}
115+
116+
func (s *Suite) TestConvertUnknownStdinToGeoParquetStdout() {
117+
cmd := &command.ConvertCmd{
118+
To: "geoparquet",
119+
}
120+
121+
s.ErrorContains(cmd.Run(), "when reading from stdin, the --from option must be provided")
122+
}

0 commit comments

Comments
 (0)