Skip to content

Commit e099eb7

Browse files
pjdufourpjdufour
committed
improvements to tags package
1 parent e198565 commit e099eb7

33 files changed

Lines changed: 695 additions & 438 deletions

‎pkg/gss/Convert.go‎

Lines changed: 0 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -125,111 +125,3 @@ func Convert(input *ConvertInput) ([]byte, error) {
125125

126126
return b, nil
127127
}
128-
129-
/*
130-
131-
// Convert converts an input_string from the inputFormat to the outputFormat.
132-
// Returns the output string and error, if any.
133-
//func Convert(inputBytes []byte, inputFormat string, inputHeader []string, inputComment string, inputLazyQuotes bool, inputSkipLines int, inputLimit int, outputFormat string, outputHeader []string, outputLimit int, async bool, verbose bool) (string, error) {
134-
func OldConvert(input *ConvertInput) ([]byte, error) {
135-
136-
inputType, err := GetType(input.InputBytes, input.InputFormat)
137-
if err != nil {
138-
return make([]byte, 0), errors.Wrap(err, "error creating new object for format "+input.InputFormat)
139-
}
140-
141-
if input.Verbose {
142-
fmt.Println("Input Format: " + input.InputFormat)
143-
fmt.Println("Output Format: " + input.OutputFormat)
144-
fmt.Println("Input Type: " + fmt.Sprint(inputType))
145-
}
146-
147-
switch input.InputFormat {
148-
case "bson", "json", "hcl", "hcl2", "properties", "tags", "toml", "yaml":
149-
switch input.OutputFormat {
150-
case "bson", "json", "jsonl", "hcl", "hcl2", "properties", "tags", "toml", "yaml":
151-
object, err := DeserializeBytes(&DeserializeBytesInput{
152-
Bytes: input.InputBytes,
153-
Format: input.InputFormat,
154-
Header: input.InputHeader,
155-
Comment: input.InputComment,
156-
LazyQuotes: input.InputLazyQuotes,
157-
SkipLines: input.InputSkipLines,
158-
Limit: input.InputLimit,
159-
LineSeparator: input.InputLineSeparator,
160-
DropCR: input.InputDropCR,
161-
Type: inputType,
162-
Async: input.Async,
163-
Verbose: input.Verbose,
164-
})
165-
if err != nil {
166-
return make([]byte, 0), errors.Wrap(err, "Error deserializing input")
167-
}
168-
outputBytes, err := SerializeBytes(&SerializeBytesInput{
169-
Object: object,
170-
Format: input.OutputFormat,
171-
Header: input.OutputHeader,
172-
Limit: input.OutputLimit,
173-
Pretty: input.OutputPretty,
174-
Sorted: input.OutputSorted,
175-
LineSeparator: input.OutputLineSeparator,
176-
KeyValueSeparator: input.OutputKeyValueSeparator,
177-
ValueSerializer: input.OutputValueSerializer,
178-
EscapePrefix: input.OutputEscapePrefix,
179-
EscapeSpace: input.OutputEscapeSpace,
180-
EscapeNewLine: input.OutputEscapeNewLine,
181-
EscapeEqual: input.OutputEscapeEqual,
182-
})
183-
if err != nil {
184-
return make([]byte, 0), errors.Wrap(err, "Error serializing output")
185-
}
186-
return outputBytes, nil
187-
case "csv", "tsv":
188-
return make([]byte, 0), &ErrIncompatibleFormats{Input: input.InputFormat, Output: input.OutputFormat}
189-
}
190-
return make([]byte, 0), errors.Wrap(&ErrUnknownFormat{Name: input.OutputFormat}, "unknown output format")
191-
case "jsonl", "csv", "tsv":
192-
switch input.OutputFormat {
193-
case "bson", "json", "hcl", "hcl2", "toml", "yaml", "jsonl", "csv", "tags", "tsv":
194-
object, err := DeserializeBytes(&DeserializeBytesInput{
195-
Bytes: input.InputBytes,
196-
Format: input.InputFormat,
197-
Header: input.InputHeader,
198-
Comment: input.InputComment,
199-
LazyQuotes: input.InputLazyQuotes,
200-
SkipLines: input.InputSkipLines,
201-
Limit: input.InputLimit,
202-
LineSeparator: input.InputLineSeparator,
203-
DropCR: input.InputDropCR,
204-
Type: inputType,
205-
Async: input.Async,
206-
Verbose: input.Verbose,
207-
})
208-
if err != nil {
209-
return make([]byte, 0), errors.Wrap(err, "Error deserializing input")
210-
}
211-
outputBytes, err := SerializeBytes(&SerializeBytesInput{
212-
Object: object,
213-
Format: input.OutputFormat,
214-
Header: input.OutputHeader,
215-
Limit: input.OutputLimit,
216-
Pretty: input.OutputPretty,
217-
Sorted: input.OutputSorted,
218-
LineSeparator: input.OutputLineSeparator,
219-
KeyValueSeparator: input.OutputKeyValueSeparator,
220-
ValueSerializer: input.OutputValueSerializer,
221-
EscapePrefix: input.OutputEscapePrefix,
222-
EscapeSpace: input.OutputEscapeSpace,
223-
EscapeNewLine: input.OutputEscapeNewLine,
224-
EscapeEqual: input.OutputEscapeEqual,
225-
})
226-
if err != nil {
227-
return make([]byte, 0), errors.Wrap(err, "Error serializing output")
228-
}
229-
return outputBytes, nil
230-
}
231-
return make([]byte, 0), errors.Wrap(&ErrUnknownFormat{Name: input.OutputFormat}, "unknown output format")
232-
}
233-
return make([]byte, 0), errors.Wrap(&ErrUnknownFormat{Name: input.InputFormat}, "unknown output format")
234-
}
235-
*/

‎pkg/gss/SerializeBytes.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,9 @@ func SerializeBytes(input *SerializeBytesInput) ([]byte, error) {
5454
if f == serializer.FormatJSONL || f == serializer.FormatProperties || f == serializer.FormatTags {
5555
s = s.LineSeparator(input.LineSeparator)
5656
}
57+
if f == serializer.FormatProperties || f == serializer.FormatTags {
58+
s = s.KeyValueSeparator(input.KeyValueSeparator)
59+
}
5760
if f == serializer.FormatCSV || f == serializer.FormatProperties || f == serializer.FormatTags || f == serializer.FormatTSV {
5861
// Sort the order of the keys/properties
5962
// Does not sort the order of the records (if serializing multiples objects as tags)
@@ -64,12 +67,11 @@ func SerializeBytes(input *SerializeBytesInput) ([]byte, error) {
6467
Sorted(input.Sorted).
6568
Reversed(input.Reversed)
6669
}
67-
if f == serializer.FormatCSV || f == serializer.FormatTSV {
70+
if f == serializer.FormatCSV || f == serializer.FormatTSV || f == serializer.FormatTags {
6871
s = s.Header(input.Header).ExpandHeader(input.ExpandHeader)
6972
}
7073
if f == "properties" {
7174
s = s.
72-
KeyValueSeparator(input.KeyValueSeparator).
7375
EscapePrefix(input.EscapePrefix).
7476
EscapeSpace(input.EscapeSpace).
7577
EscapeColon(input.EscapeColon).

‎pkg/gss/SerializeBytes_test.go‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -162,9 +162,10 @@ func TestSerializeBytesTsvSlice(t *testing.T) {
162162

163163
func TestSerializeBytesTags(t *testing.T) {
164164
b, err := SerializeBytes(&SerializeBytesInput{
165-
Object: map[string]interface{}{"a": 1.0, "b": 2.0, "c": 3.0},
166-
Format: "tags",
167-
Sorted: true,
165+
Object: map[string]interface{}{"a": 1.0, "b": 2.0, "c": 3.0},
166+
KeyValueSeparator: "=",
167+
Format: "tags",
168+
Sorted: true,
168169
})
169170
assert.NoError(t, err)
170171
assert.Equal(t, "a=1 b=2 c=3", string(b))
@@ -176,10 +177,11 @@ func TestSerializeBytesTagsMultiple(t *testing.T) {
176177
map[string]interface{}{"a": 1.0, "b": 2.0, "c": 3.0},
177178
map[string]interface{}{"a": "x", "b": "y", "c": "z"},
178179
},
179-
Format: "tags",
180-
LineSeparator: "\n",
181-
Sorted: true,
182-
Limit: -1,
180+
KeyValueSeparator: "=",
181+
Format: "tags",
182+
LineSeparator: "\n",
183+
Sorted: true,
184+
Limit: -1,
183185
})
184186
assert.NoError(t, err)
185187
assert.Equal(t, "a=1 b=2 c=3\na=x b=y c=z\n", string(b))

‎pkg/gssjs/gssjs.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,16 @@ var (
2626

2727
var (
2828
DeserializeDefaults = map[string]interface{}{
29-
"limit": NoLimit,
30-
"lineSeparator": "\n",
31-
"escapePrefix": "\\",
32-
"expandHeader": true,
29+
"limit": NoLimit,
30+
"keyValueSeparator": "=",
31+
"lineSeparator": "\n",
32+
"escapePrefix": "\\",
33+
"expandHeader": true,
3334
}
3435
SerializeDefaults = map[string]interface{}{
3536
"limit": NoLimit,
36-
"lineSeparator": "\n",
3737
"keyValueSeparator": "=",
38+
"lineSeparator": "\n",
3839
"escapePrefix": "\\",
3940
}
4041
)

‎pkg/inspector/GetUnknownFieldNamesFromValue.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// =================================================================
2+
//
3+
// Copyright (C) 2019 Spatial Current, Inc. - All Rights Reserved
4+
// Released as open source under the MIT License. See LICENSE file.
5+
//
6+
// =================================================================
7+
8+
package inspector
9+
10+
import (
11+
"reflect"
12+
"sort"
13+
)
14+
15+
// GetUnknownFieldNamesFromValue returns the unknown field names for a struct as a []string{} given a set of known field names.
16+
// If you want the field names to be sorted in alphabetical order, pass sorted equal to true.
17+
// If sorted and reversed, then sorts in reverse alphabetical order.
18+
func GetUnknownFieldNamesFromValue(value reflect.Value, knownKeys map[string]struct{}, sorted bool, reversed bool) []string {
19+
unknownFieldNames := make([]string, 0)
20+
t := value.Type()
21+
for i := 0; i < value.NumField(); i++ {
22+
fieldName := t.Field(i).Name
23+
if _, exists := knownKeys[fieldName]; !exists {
24+
unknownFieldNames = append(unknownFieldNames, fieldName)
25+
}
26+
}
27+
if sorted {
28+
sort.Slice(unknownFieldNames, func(i, j int) bool {
29+
if reversed {
30+
return unknownFieldNames[i] > unknownFieldNames[j]
31+
}
32+
return unknownFieldNames[i] < unknownFieldNames[j]
33+
})
34+
}
35+
return unknownFieldNames
36+
}

‎pkg/iterator/Iterator.go‎

Lines changed: 25 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -30,19 +30,20 @@ type Iterator interface {
3030

3131
// Input for NewIterator function.
3232
type NewIteratorInput struct {
33-
Reader io.Reader // the underlying reader
34-
Format string // the format
35-
Header []interface{} // for csv and tsv, the header. If not given, then reads first line of stream as header.
36-
SkipLines int // Skip a given number of lines at the beginning of the stream.
37-
SkipBlanks bool // Skip blank lines. If false, Next() returns a blank line as (nil, nil). If true, Next() simply skips forward until it finds a non-blank line.
38-
SkipComments bool // Skip commented lines. If false, Next() returns a commented line as (nil, nil). If true, Next() simply skips forward until it finds a non-commented line.
39-
Comment string // The comment line prefix. CSV and TSV only support single characters. JSON Lines support any string.
40-
Trim bool // Trim each input line before parsing into an object.
41-
LazyQuotes bool // for csv and tsv, parse with lazy quotes
42-
Limit int // Limit the number of objects to read and return from the underlying stream.
43-
LineSeparator byte // For JSON Lines, the new line byte.
44-
DropCR bool // For JSON Lines, drop carriage returns at the end of lines.
45-
Type reflect.Type //
33+
Reader io.Reader // the underlying reader
34+
Format string // the format
35+
Header []interface{} // for csv and tsv, the header. If not given, then reads first line of stream as header.
36+
SkipLines int // Skip a given number of lines at the beginning of the stream.
37+
SkipBlanks bool // Skip blank lines. If false, Next() returns a blank line as (nil, nil). If true, Next() simply skips forward until it finds a non-blank line.
38+
SkipComments bool // Skip commented lines. If false, Next() returns a commented line as (nil, nil). If true, Next() simply skips forward until it finds a non-commented line.
39+
Comment string // The comment line prefix. CSV and TSV only support single characters. JSON Lines support any string.
40+
Trim bool // Trim each input line before parsing into an object.
41+
LazyQuotes bool // for csv and tsv, parse with lazy quotes
42+
Limit int // Limit the number of objects to read and return from the underlying stream.
43+
KeyValueSeparator string // For tags, the key-value separator.
44+
LineSeparator byte // For JSON Lines, the new line byte.
45+
DropCR bool // For JSON Lines, drop carriage returns at the end of lines.
46+
Type reflect.Type //
4647
}
4748

4849
// NewIterator returns an Iterator for the given input source, format, and other options.
@@ -84,18 +85,18 @@ func NewIterator(input *NewIteratorInput) (Iterator, error) {
8485
Limit: input.Limit,
8586
})
8687
case "tags":
87-
it := tags.NewIterator(&tags.NewIteratorInput{
88-
Reader: input.Reader,
89-
Type: inputType,
90-
SkipLines: input.SkipLines,
91-
SkipBlanks: input.SkipBlanks,
92-
SkipComments: input.SkipComments,
93-
Comment: input.Comment,
94-
LineSeparator: input.LineSeparator,
95-
DropCR: input.DropCR,
96-
Limit: input.Limit,
88+
return tags.NewIterator(&tags.NewIteratorInput{
89+
Reader: input.Reader,
90+
Type: inputType,
91+
SkipLines: input.SkipLines,
92+
SkipBlanks: input.SkipBlanks,
93+
SkipComments: input.SkipComments,
94+
Comment: input.Comment,
95+
KeyValueSeparator: input.KeyValueSeparator,
96+
LineSeparator: input.LineSeparator,
97+
DropCR: input.DropCR,
98+
Limit: input.Limit,
9799
})
98-
return it, nil
99100
case "tsv":
100101
return sv.NewIterator(&sv.NewIteratorInput{
101102
Reader: input.Reader,

‎pkg/iterator/Iterator_examples_test.go‎

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,16 @@ func ExampleIterator_jsonl() {
2626
`
2727

2828
it, err := NewIterator(&NewIteratorInput{
29-
Reader: strings.NewReader(text),
30-
Format: "jsonl",
31-
SkipLines: 0,
32-
Comment: "",
33-
Trim: true,
34-
SkipBlanks: false,
35-
SkipComments: false,
36-
LineSeparator: []byte("\n")[0],
37-
DropCR: true,
29+
Reader: strings.NewReader(text),
30+
Format: "jsonl",
31+
SkipLines: 0,
32+
Comment: "",
33+
Trim: true,
34+
SkipBlanks: false,
35+
SkipComments: false,
36+
KeyValueSeparator: "=",
37+
LineSeparator: []byte("\n")[0],
38+
DropCR: true,
3839
})
3940
if err != nil {
4041
panic(err)
@@ -70,15 +71,16 @@ func ExampleIterator_tags() {
7071
`
7172

7273
it, err := NewIterator(&NewIteratorInput{
73-
Reader: strings.NewReader(text),
74-
Format: "tags",
75-
SkipLines: 0,
76-
Comment: "",
77-
Trim: true,
78-
SkipBlanks: false,
79-
SkipComments: false,
80-
LineSeparator: []byte("\n")[0],
81-
DropCR: true,
74+
Reader: strings.NewReader(text),
75+
Format: "tags",
76+
SkipLines: 0,
77+
Comment: "",
78+
Trim: true,
79+
SkipBlanks: false,
80+
SkipComments: false,
81+
KeyValueSeparator: "=",
82+
LineSeparator: []byte("\n")[0],
83+
DropCR: true,
8284
})
8385
if err != nil {
8486
panic(err)

‎pkg/iterator/Iterator_test.go‎

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616

1717
import (
1818
"github.com/stretchr/testify/assert"
19+
"github.com/stretchr/testify/require"
1920
)
2021

2122
func TestIteratorJsonl(t *testing.T) {
@@ -40,8 +41,8 @@ func TestIteratorJsonl(t *testing.T) {
4041
LineSeparator: []byte("\n")[0],
4142
DropCR: true,
4243
})
43-
assert.NoError(t, err)
44-
assert.NotNil(t, it)
44+
require.NoError(t, err)
45+
require.NotNil(t, it)
4546

4647
// Empty Line
4748
obj, err := it.Next()
@@ -107,19 +108,20 @@ func TestIteratorTags(t *testing.T) {
107108
`
108109

109110
it, err := NewIterator(&NewIteratorInput{
110-
Reader: strings.NewReader(text),
111-
Type: reflect.TypeOf([]map[string]interface{}{}),
112-
Format: "tags",
113-
SkipLines: 0,
114-
Comment: "",
115-
Trim: true,
116-
SkipBlanks: false,
117-
SkipComments: false,
118-
LineSeparator: []byte("\n")[0],
119-
DropCR: true,
111+
Reader: strings.NewReader(text),
112+
Type: reflect.TypeOf([]map[string]interface{}{}),
113+
Format: "tags",
114+
SkipLines: 0,
115+
Comment: "",
116+
Trim: true,
117+
SkipBlanks: false,
118+
SkipComments: false,
119+
KeyValueSeparator: "=",
120+
LineSeparator: []byte("\n")[0],
121+
DropCR: true,
120122
})
121-
assert.NoError(t, err)
122-
assert.NotNil(t, it)
123+
require.NoError(t, err)
124+
require.NotNil(t, it)
123125

124126
// Empty Line
125127
obj, err := it.Next()
@@ -155,4 +157,5 @@ func TestIteratorTags(t *testing.T) {
155157
obj, err = it.Next()
156158
assert.Equal(t, io.EOF, err)
157159
assert.Nil(t, obj)
160+
158161
}

0 commit comments

Comments
 (0)