Skip to content

Commit bb52c58

Browse files
committed
Find subcommand
This commit adds support for the 'find' subcommand which allows you to easily extract elements from yaml files. The find subcommand supports a '--format' parameter which allows you to specify in which format the found element should be printed. Additionally: - Added a "labels" list to the sample yaml files. This for testing simple yaml lists. - Omitting directories containing "travis" from coverage results in an attempt to cleanup travisCI output - Cleaned up README and added samples on the find subcommand
1 parent 128dac4 commit bb52c58

17 files changed

+215
-29
lines changed

README.md

+24-20
Original file line numberDiff line numberDiff line change
@@ -2,37 +2,41 @@
22
[![Build Status](https://travis-ci.org/jorisroovers/yamlpal.svg?branch=master)]
33
(https://travis-ci.org/jorisroovers/yamlpal)
44

5-
Simple tool for inserting new entries in yaml files while keeping the original structure and formatting.
5+
Simple tool for modifying and searching yaml files while keeping the original file formatting.
6+
7+
Yamlpal uses its own version of 'yamlpath', a syntax similar to xpath, to identify elements in a yaml file.
8+
69

710
Basic usage:
811
```bash
9-
$ yamlpal insert -f examples/sample1.yml "invoice" "newkey: newval"
10-
12+
# Inserting new content into files (output is printed to stdout by default)
13+
$ yamlpal insert -f examples/sample1.yml "bill-to/address/city" "newkey: value"
1114
$ yamlpal insert -f examples/sample1.yml "invoice" @examples/insert-multiline.txt
1215

13-
$ yamlpal insert -f examples/sample1.yml "tax" "newkey: newval"
16+
# Specify files via stdin and modify the files directly inline instead of printing to stdout
17+
$ find examples -name \*.yml | yamlpal insert --inline "invoice" "newkey: value"
1418

15-
$ yamlpal insert -f examples/sample1.yml"bill-to/given" "rhel-7-server"
19+
# Finding content in files
20+
$ yamlpal find -f examples/sample1.yml "bill-to/address/city"
21+
city: Royal Oak
1622

17-
$ yamlpal insert -f examples/sample1.yml "product[1]/sku" "newkey: newvalue"
23+
# Specify a custom output format (run "yamlpal find --help" for details on format strings)
24+
$ yamlpal find -f examples/sample1.yml --format "%{linenr} %{key} %{value}" "bill-to/address/city"
25+
11 city Royal Oak
1826

19-
$ yamlpal insert -f examples/sample1.yml "bill-to/address/city" "newkey: value"
20-
21-
# Specify files via stdin:
22-
$ find examples -name \*.yml | yamlpal insert "invoice" "newkey: value"
27+
# Run yamlpal <command> --help for command specific help.
28+
$ yamlpal insert --help
29+
Usage: yamlpal insert [OPTIONS] NEEDLE NEWCONTENT
2330

24-
# More options:
25-
$ yamlpal --help
26-
Usage: yamlpal [OPTIONS] COMMAND [ARGS]...
27-
28-
Modify yaml files while keeping the original structure and formatting.
31+
Insert new content into a yaml file.
2932

3033
Options:
31-
--version Show the version and exit.
32-
--help Show this message and exit.
33-
34-
Commands:
35-
insert Insert new content into a yaml file.
34+
-f, --file PATH File to insert new content in. Can by specified multiple
35+
times to modify multiple files. Files are not modified
36+
inline by default. You can also provide (additional) file
37+
paths via stdin.
38+
-i, --inline Edit file inline instead of dumping it to std out.
39+
--help Show this message and exit.
3640
```
3741

3842

examples/sample1.yml

+5-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
---
2-
# Example from: http://www.yaml.org/start.html
2+
# Modified example from: http://www.yaml.org/start.html
33
invoice: 34843
44
date : 2001-01-23
55
bill-to: &id001
@@ -22,6 +22,10 @@ product:
2222
quantity : 1
2323
description : Super Hoop
2424
price : 2392.00
25+
labels:
26+
- premium customer
27+
- online order
28+
- fast delivery
2529
tax : 251.42
2630
total: 4443.52
2731
comments: >

run_tests.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ run_pep8_check(){
2929
}
3030

3131
run_unit_tests(){
32-
OMIT="/usr/*"
32+
OMIT="/usr/*,*travis*"
3333
if [ -n "$testargs" ]; then
3434
# pytest -s => show std output (i.e print statements)
3535
coverage run --omit=$OMIT -m pytest -s "$testargs"

yamlpal/cli.py

+85-6
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,30 @@
66
import re
77
import os
88

9-
10-
@click.group()
9+
FORMATTING_EPILOG = """
10+
\b
11+
Format Strings:
12+
Format strings determine how yamlpal will output values.
13+
Available keys:
14+
%{key} : Key of the match (or index if matching an item in a list)
15+
%{value} : Value of the match
16+
%{linenr} : Line number where the match occured
17+
%{file} : Name of the file in which the match occured
18+
%{literal} : Literal match in the file (original formatting)
19+
Examples:
20+
$ yamlpal find "bill-to/address/city" --format "%{file} %{linenr}: %{value}"
21+
/abs/path/to/examples/examples/sample1.yml 11: Royal Oak
22+
23+
$ yamlpal find "bill-to/address/city" --format "%{linenr} %{literal}"
24+
11: city : Royal Oak
25+
"""
26+
27+
28+
@click.group(epilog="Run 'yamlpal <command> --help' for command specific help.")
1129
@click.version_option(version=yamlpal.__version__)
1230
def cli():
13-
""" Modify yaml files while keeping the original structure and formatting. """
31+
""" Modify and search yaml files while keeping the original formatting.
32+
"""
1433

1534

1635
def get_files(passed_files):
@@ -55,9 +74,11 @@ def get_str_content(str_value):
5574
@cli.command("insert")
5675
@click.argument('needle')
5776
@click.argument('newcontent')
58-
@click.option('-f', '--file',
59-
type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True), multiple=True)
60-
@click.option('-i', '--inline', help="Edit file inline instead of dumping it to std out", is_flag=True)
77+
@click.option('-f', '--file', type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True),
78+
multiple=True, help="File to insert new content in. Can by specified multiple times to modify " +
79+
"multiple files. Files are not modified inline by default. " +
80+
"You can also provide (additional) file paths via stdin.")
81+
@click.option('-i', '--inline', help="Edit file inline instead of dumping it to std out.", is_flag=True)
6182
def insert(needle, newcontent, file, inline):
6283
""" Insert new content into a yaml file. """
6384
newcontent = get_str_content(newcontent)
@@ -67,6 +88,63 @@ def insert(needle, newcontent, file, inline):
6788
insert_in_file(needle, newcontent, file, inline)
6889

6990

91+
@cli.command("find", epilog=FORMATTING_EPILOG)
92+
@click.argument('needle')
93+
@click.option('-f', '--file', type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True),
94+
help="File to find content in.")
95+
@click.option('-F', '--format', help="Format string in which matched content should be returned. " +
96+
"See the section 'Format Strings' below for details on format strings. " +
97+
"(default: \"%{key}: %{value}\")",
98+
default="%{key}: %{value}")
99+
def find(needle, file, format):
100+
""" Find a line in a yaml file. """
101+
found = find_in_file(needle, file, format)
102+
click.echo(found)
103+
104+
105+
def find_in_file(needle, file, format):
106+
# read yaml file
107+
fp = open(file)
108+
filecontents = fp.read()
109+
fp.close()
110+
111+
# parse the file
112+
data = YamlParser.load_yaml(filecontents)
113+
try:
114+
element = find_element(data, needle)
115+
except exceptions.InvalidSearchStringException:
116+
# TODO (jroovers): we should deduplicate this code. Best done by moving the core business logic
117+
# (like find_element) out of this module into it's own module and then creating a wrapper function
118+
# here that deals with exception handling
119+
click.echo("ERROR: Invalid search string '%s' for file '%s'" % (needle, file), err=True)
120+
exit(1)
121+
122+
return apply_format(file, filecontents, element, format)
123+
124+
125+
def apply_format(file, filecontents, element, format):
126+
""" Given a yaml element and yamlpal format string, return the interpolated string.
127+
We currently support the following placeholders:
128+
- %{key} -> key of the yaml element (index if you are accessing a list)
129+
- %{value} -> value of the yaml element
130+
- %{literal} -> the string corresponding to the yaml element as it literally occurs in the file
131+
- %{linenr} -> line number on which the yaml element is found
132+
- %{file} -> name of the file in which the yaml element is found
133+
"""
134+
135+
result = format.replace("%{key}", str(element.key))
136+
result = result.replace("%{value}", str(element))
137+
result = result.replace("%{linenr}", str(element.line))
138+
result = result.replace("%{file}", file)
139+
140+
# check whether literal occurs before splitting the file, since it's a more expensive operation
141+
if "%{literal}" in format:
142+
lines = filecontents.split("\n", element.line + 1) # don't split more than required
143+
result = result.replace("%{literal}", lines[element.line])
144+
145+
return result
146+
147+
70148
def insert_in_file(needle, newcontent, file, inline):
71149
# read yaml file
72150
fp = open(file)
@@ -123,6 +201,7 @@ def find_element(yaml_dict, search_str):
123201
# dictionary or list.
124202
try:
125203
node.line
204+
node.key = parsed_parts[-1] # add the last parsed key as the node's key
126205
except AttributeError:
127206
click.echo("ERROR: Path exists but not specific enough (%s)." % search_str, err=True)
128207
exit(1)

yamlpal/tests/expected/sample1-after-date.yml

+4
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ product:
2323
quantity : 1
2424
description : Super Hoop
2525
price : 2392.00
26+
labels:
27+
- premium customer
28+
- online order
29+
- fast delivery
2630
tax : 251.42
2731
title: Invoice for purchases
2832
total: 4443.52

yamlpal/tests/expected/sample1-after-float.yml

+4
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ product:
2222
quantity : 1
2323
description : Super Hoop
2424
price : 2392.00
25+
labels:
26+
- premium customer
27+
- online order
28+
- fast delivery
2529
tax : 251.42
2630
newkey: value
2731
title: Invoice for purchases

yamlpal/tests/expected/sample1-after-int.yml

+4
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ product:
2323
quantity : 1
2424
description : Super Hoop
2525
price : 2392.00
26+
labels:
27+
- premium customer
28+
- online order
29+
- fast delivery
2630
tax : 251.42
2731
title: Invoice for purchases
2832
total: 4443.52

yamlpal/tests/expected/sample1-after-string.yml

+4
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ product:
2222
quantity : 1
2323
description : Super Hoop
2424
price : 2392.00
25+
labels:
26+
- premium customer
27+
- online order
28+
- fast delivery
2529
tax : 251.42
2630
title: Invoice for purchases
2731
newkey: value

yamlpal/tests/expected/sample1-multiline-insert.yml

+4
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ product:
2222
quantity : 1
2323
description : Super Hoop
2424
price : 2392.00
25+
labels:
26+
- premium customer
27+
- online order
28+
- fast delivery
2529
tax : 251.42
2630
title: Invoice for purchases
2731
key: value

yamlpal/tests/expected/sample2-after-string.yml

+4
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ product:
2222
quantity : 1
2323
description : Super Hoop
2424
price : 2392.00
25+
labels:
26+
- premium customer
27+
- online order
28+
- fast delivery
2529
tax : 251.42
2630
title: Invoice for purchases
2731
newkey: value

yamlpal/tests/expected/sample3-after-string.yml

+4
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ product:
2121
quantity : 1
2222
description : Super Hoop
2323
price : 2392.00
24+
labels:
25+
- premium customer
26+
- online order
27+
- fast delivery
2428
tax : 251.42
2529
title: Invoice for purchases
2630
newkey: value

yamlpal/tests/samples/sample1.yml

+4
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ product:
2222
quantity : 1
2323
description : Super Hoop
2424
price : 2392.00
25+
labels:
26+
- premium customer
27+
- online order
28+
- fast delivery
2529
tax : 251.42
2630
title: Invoice for purchases
2731
total: 4443.52

yamlpal/tests/samples/sample2.yml

+4
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ product:
2222
quantity : 1
2323
description : Super Hoop
2424
price : 2392.00
25+
labels:
26+
- premium customer
27+
- online order
28+
- fast delivery
2529
tax : 251.42
2630
title: Invoice for purchases
2731
total: 4443.52

yamlpal/tests/samples/sample3.yml

+4
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ product:
2121
quantity : 1
2222
description : Super Hoop
2323
price : 2392.00
24+
labels:
25+
- premium customer
26+
- online order
27+
- fast delivery
2428
tax : 251.42
2529
title: Invoice for purchases
2630
total: 4443.52

yamlpal/tests/test_find.py

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
from yamlpal.tests.base import BaseTestCase
2+
from yamlpal import cli
3+
from click.testing import CliRunner
4+
5+
6+
class FindTests(BaseTestCase):
7+
def setUp(self):
8+
self.cli = CliRunner()
9+
10+
def test_find_simple(self):
11+
result = self.cli.invoke(cli.cli, ["find", "title", "-f", self.get_sample_path("sample1")])
12+
self.assertEqual(result.output, "title: Invoice for purchases\n")
13+
14+
def test_find_number(self):
15+
result = self.cli.invoke(cli.cli, ["find", "tax", "-f", self.get_sample_path("sample1")])
16+
self.assertEqual(result.output, "tax: 251.42\n")
17+
18+
def test_find_complex(self):
19+
result = self.cli.invoke(cli.cli, ["find", "bill-to/address/city", "-f", self.get_sample_path("sample1")])
20+
self.assertEqual(result.output, "city: Royal Oak\n")
21+
22+
def test_find_custom_format(self):
23+
sample_path = self.get_sample_path("sample1")
24+
result = self.cli.invoke(cli.cli, ["find", "bill-to/address/city",
25+
"-F", "%{file} %{linenr} %{key} %{value}",
26+
"-f", sample_path])
27+
self.assertEqual(result.output, "%s 11 city Royal Oak\n" % sample_path)
28+
29+
def test_find_custom_format_literal(self):
30+
result = self.cli.invoke(cli.cli, ["find", "bill-to/address/city",
31+
"-F", "%{literal}",
32+
"-f", self.get_sample_path("sample1")])
33+
self.assertEqual(result.output, " city : Royal Oak\n")
34+
35+
def test_find_key_of_list_item(self):
36+
result = self.cli.invoke(cli.cli, ["find", "labels[0]",
37+
"-f", self.get_sample_path("sample1")])
38+
self.assertEqual(result.output, "0: premium customer\n")
39+
40+
result = self.cli.invoke(cli.cli, ["find", "labels[1]",
41+
"-f", self.get_sample_path("sample1")])
42+
43+
self.assertEqual(result.output, "1: online order\n")

yamlpal/tests/test_find_negative.py

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
from yamlpal.tests.base import BaseTestCase
2+
from yamlpal import cli
3+
from click.testing import CliRunner
4+
5+
6+
class FindNegativeTests(BaseTestCase):
7+
def setUp(self):
8+
self.cli = CliRunner()
9+
10+
def test_inaccurate_path_dictionary(self):
11+
result = self.cli.invoke(cli.cli, ["find", "bill-to/address", "-f", self.get_sample_path("sample1")])
12+
self.assertEqual(result.output, "ERROR: Path exists but not specific enough (bill-to/address).\n")
13+
14+
def test_inaccurate_path_list(self):
15+
result = self.cli.invoke(cli.cli, ["find", "product", "-f", self.get_sample_path("sample1")])
16+
self.assertEqual(result.output, "ERROR: Path exists but not specific enough (product).\n")

yamlpal/tests/test_insert_negative.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from click.testing import CliRunner
44

55

6-
class InsertionTests(BaseTestCase):
6+
class InsertionNegativeTests(BaseTestCase):
77
def setUp(self):
88
self.cli = CliRunner()
99

0 commit comments

Comments
 (0)