Skip to content

Commit 44be918

Browse files
anto6715anto6715
authored
Introduce ruff (#2)
1 parent c554d52 commit 44be918

File tree

13 files changed

+593
-307
lines changed

13 files changed

+593
-307
lines changed

‎.github/workflows/ruff.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
name: Ruff
2+
on: [ pull_request ]
3+
jobs:
4+
ruff:
5+
runs-on: ubuntu-latest
6+
steps:
7+
- uses: actions/checkout@v4
8+
- uses: astral-sh/ruff-action@v3
9+
- name: Ruff format
10+
run: ruff format --diff

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ or [botos3](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html)
66

77
[![boto3](https://img.shields.io/badge/boto3->1.34-blue.svg)](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html)
88
[![copernicusmarine](https://img.shields.io/badge/copernicusmarine->1.06-blue.svg)](https://help.marine.copernicus.eu/en/collections/4060068-copernicus-marine-toolbox)
9-
9+
[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
1010

1111
<!-- TOC -->
1212
* [Marine Data Store ToolBox](#marine-data-store-toolbox)

‎mds.py‎

Lines changed: 232 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -19,119 +19,272 @@ def main():
1919

2020

2121
@main.command()
22-
@click.option('-o', '--output-directory', required=True, type=str, help='Output directory')
23-
@click.option('-f', '--output-filename', required=True, type=str, help='Output filename')
24-
@click.option('-i', '--dataset-id', required=True, type=str, help='Dataset Id')
25-
@click.option('-v', '--variables', multiple=True, type=str, help='Variables to download')
26-
@click.option('-x', '--minimum-longitude', type=float, help='Minimum longitude for the subset.')
27-
@click.option('-X', '--maximum-longitude', type=float, help='Maximum longitude for the subset. ')
28-
@click.option('-y', '--minimum-latitude', type=float,
29-
help='Minimum latitude for the subset. Requires a float within this range: [-90<=x<=90]')
30-
@click.option('-Y', '--maximum-latitude', type=float,
31-
help='Maximum latitude for the subset. Requires a float within this range: [-90<=x<=90]')
32-
@click.option('-z', '--minimum-depth', type=float,
33-
help='Minimum depth for the subset. Requires a float within this range: [x>=0]')
34-
@click.option('-Z', '--maximum-depth', type=float,
35-
help='Maximum depth for the subset. Requires a float within this range: [x>=0]')
36-
@click.option('-t', '--start-datetime', type=str, default=False,
37-
help='Start datetime as: %Y|%Y-%m-%d|%Y-%m-%dT%H:%M:%S|%Y-%m-%d %H:%M:%S|%Y-%m-%dT%H:%M:%S.%fZ')
38-
@click.option('-T', '--end-datetime', type=str, default=False,
39-
help='End datetime as: %Y|%Y-%m-%d|%Y-%m-%dT%H:%M:%S|%Y-%m-%d %H:%M:%S|%Y-%m-%dT%H:%M:%S.%fZ')
40-
@click.option('-r', '--dry-run', is_flag=True, default=False, help='Dry run')
41-
@click.option('-g', '--dataset-version', type=str, default=None, help='Dataset version or tag')
42-
@click.option('-n', '--username', type=str, default=None, help='Username')
43-
@click.option('-w', '--password', type=str, default=None, help='Password')
22+
@click.option(
23+
"-o", "--output-directory", required=True, type=str, help="Output directory"
24+
)
25+
@click.option(
26+
"-f", "--output-filename", required=True, type=str, help="Output filename"
27+
)
28+
@click.option("-i", "--dataset-id", required=True, type=str, help="Dataset Id")
29+
@click.option(
30+
"-v", "--variables", multiple=True, type=str, help="Variables to download"
31+
)
32+
@click.option(
33+
"-x", "--minimum-longitude", type=float, help="Minimum longitude for the subset."
34+
)
35+
@click.option(
36+
"-X", "--maximum-longitude", type=float, help="Maximum longitude for the subset. "
37+
)
38+
@click.option(
39+
"-y",
40+
"--minimum-latitude",
41+
type=float,
42+
help="Minimum latitude for the subset. Requires a float within this range: [-90<=x<=90]",
43+
)
44+
@click.option(
45+
"-Y",
46+
"--maximum-latitude",
47+
type=float,
48+
help="Maximum latitude for the subset. Requires a float within this range: [-90<=x<=90]",
49+
)
50+
@click.option(
51+
"-z",
52+
"--minimum-depth",
53+
type=float,
54+
help="Minimum depth for the subset. Requires a float within this range: [x>=0]",
55+
)
56+
@click.option(
57+
"-Z",
58+
"--maximum-depth",
59+
type=float,
60+
help="Maximum depth for the subset. Requires a float within this range: [x>=0]",
61+
)
62+
@click.option(
63+
"-t",
64+
"--start-datetime",
65+
type=str,
66+
default=False,
67+
help="Start datetime as: %Y|%Y-%m-%d|%Y-%m-%dT%H:%M:%S|%Y-%m-%d %H:%M:%S|%Y-%m-%dT%H:%M:%S.%fZ",
68+
)
69+
@click.option(
70+
"-T",
71+
"--end-datetime",
72+
type=str,
73+
default=False,
74+
help="End datetime as: %Y|%Y-%m-%d|%Y-%m-%dT%H:%M:%S|%Y-%m-%d %H:%M:%S|%Y-%m-%dT%H:%M:%S.%fZ",
75+
)
76+
@click.option("-r", "--dry-run", is_flag=True, default=False, help="Dry run")
77+
@click.option(
78+
"-g", "--dataset-version", type=str, default=None, help="Dataset version or tag"
79+
)
80+
@click.option("-n", "--username", type=str, default=None, help="Username")
81+
@click.option("-w", "--password", type=str, default=None, help="Password")
4482
def subset(**kwargs):
45-
wrapper.mds_download('subset', **kwargs)
83+
wrapper.mds_download("subset", **kwargs)
4684

4785

4886
@main.command()
49-
@click.option('-f', '--filter', required=False, type=str, help='Filter on the online files')
50-
@click.option('-o', '--output-directory', required=True, type=str, help='Output directory')
51-
@click.option('-i', '--dataset-id', required=True, type=str, help='Dataset Id')
52-
@click.option('-g', '--dataset-version', type=str, default=None, help='Dataset version or tag')
87+
@click.option(
88+
"-f", "--filter", required=False, type=str, help="Filter on the online files"
89+
)
90+
@click.option(
91+
"-o", "--output-directory", required=True, type=str, help="Output directory"
92+
)
93+
@click.option("-i", "--dataset-id", required=True, type=str, help="Dataset Id")
94+
@click.option(
95+
"-g", "--dataset-version", type=str, default=None, help="Dataset version or tag"
96+
)
5397
# @click.option('-s', '--service', type=str, default='files',
5498
# help="Force download through one of the available services using the service name among "
5599
# "['original-files', 'ftp'] or its short name among ['files', 'ftp'].")
56-
@click.option('-d', '--dry-run', is_flag=True, default=False, help='Dry run')
57-
@click.option('-u', '--update', is_flag=True, default=False,
58-
help='If the file not exists, download it, otherwise update it it changed on mds')
59-
@click.option('-v', '--dataset-version', type=str, default=None, help='Dry run')
60-
@click.option('-nd', '--no-directories', type=str, default=True,
61-
help='Option to not recreate folder hierarchy in output directory')
62-
@click.option('--force-download', type=str, default=True,
63-
help='Flag to skip confirmation before download')
64-
@click.option('--disable-progress-bar', type=str, default=True, help='Flag to hide progress bar')
65-
@click.option('-n', '--username', type=str, default=None, help='Username')
66-
@click.option('-w', '--password', type=str, default=None, help='Password')
100+
@click.option("-d", "--dry-run", is_flag=True, default=False, help="Dry run")
101+
@click.option(
102+
"-u",
103+
"--update",
104+
is_flag=True,
105+
default=False,
106+
help="If the file not exists, download it, otherwise update it it changed on mds",
107+
)
108+
@click.option("-v", "--dataset-version", type=str, default=None, help="Dry run")
109+
@click.option(
110+
"-nd",
111+
"--no-directories",
112+
type=str,
113+
default=True,
114+
help="Option to not recreate folder hierarchy in output directory",
115+
)
116+
@click.option(
117+
"--force-download",
118+
type=str,
119+
default=True,
120+
help="Flag to skip confirmation before download",
121+
)
122+
@click.option(
123+
"--disable-progress-bar", type=str, default=True, help="Flag to hide progress bar"
124+
)
125+
@click.option("-n", "--username", type=str, default=None, help="Username")
126+
@click.option("-w", "--password", type=str, default=None, help="Password")
67127
def get(**kwargs):
68-
update = kwargs.pop('update')
128+
update = kwargs.pop("update")
69129
if update:
70130
wrapper.mds_update_download(**kwargs)
71131
else:
72-
wrapper.mds_download('get', **kwargs)
132+
wrapper.mds_download("get", **kwargs)
73133

74134

75135
@main.command()
76-
@click.argument('dataset_id', type=str)
77-
@click.argument('mds_filter', type=str)
78-
@click.option('-g', '--dataset-version', type=str, default=None, help='Dataset version or tag')
136+
@click.argument("dataset_id", type=str)
137+
@click.argument("mds_filter", type=str)
138+
@click.option(
139+
"-g", "--dataset-version", type=str, default=None, help="Dataset version or tag"
140+
)
79141
def file_list(*args, **kwargs):
80142
mds_file_list = wrapper.mds_list(*args, **kwargs)
81143
print(f"{' '.join(mds_file_list)}")
82144

83145

84146
@main.command()
85-
@click.option('-e', '--s3_file', type=str, default=None,
86-
help='Path to a specific s3 file - if present, other parameters are ignored.')
87-
@click.option('-p', '--product', type=str, default=None, help='The product name')
88-
@click.option('-i', '--dataset_id', type=str, default=None, help='The datasetID')
89-
@click.option('-g', '--version', type=str, default=None,
90-
help='Force the selection of a specific dataset version')
91-
@click.option('-s', '--subdir', type=str, default=None, help='Subdir structure on mds (i.e. {year}/{month})')
92-
@click.option('-f', '--mds_filter', type=str, default=None, help='Pattern to filter data (no regex)')
147+
@click.option(
148+
"-e",
149+
"--s3_file",
150+
type=str,
151+
default=None,
152+
help="Path to a specific s3 file - if present, other parameters are ignored.",
153+
)
154+
@click.option("-p", "--product", type=str, default=None, help="The product name")
155+
@click.option("-i", "--dataset_id", type=str, default=None, help="The datasetID")
156+
@click.option(
157+
"-g",
158+
"--version",
159+
type=str,
160+
default=None,
161+
help="Force the selection of a specific dataset version",
162+
)
163+
@click.option(
164+
"-s",
165+
"--subdir",
166+
type=str,
167+
default=None,
168+
help="Subdir structure on mds (i.e. {year}/{month})",
169+
)
170+
@click.option(
171+
"-f",
172+
"--mds_filter",
173+
type=str,
174+
default=None,
175+
help="Pattern to filter data (no regex)",
176+
)
93177
def etag(**kwargs):
94178
s3_files = wrapper.mds_etag(**kwargs)
95179
for s3_file in s3_files:
96-
print(f'{s3_file.name} {s3_file.etag}')
180+
print(f"{s3_file.name} {s3_file.etag}")
97181

98182

99183
@main.command()
100-
@click.option('-b', '--bucket', 's3_bucket', required=True, type=str, help='Bucket name')
101-
@click.option('-f', '--filter', 'file_filter', required=True, type=str, help='Filter on the online files')
102-
@click.option('-o', '--output-directory', required=True, type=str, help='Output directory')
103-
@click.option('-p', '--product', required=True, type=str, help='The product name')
104-
@click.option('-i', '--dataset-id', required=True, type=str, help='Dataset Id')
105-
@click.option('-g', '--dataset-version', type=str, default=None, help='Dataset version or tag')
106-
@click.option('-r', '--recursive', is_flag=True, default=False, help='List recursive all s3 files')
107-
@click.option('--threads', 'n_threads', type=int, default=None, help='Downloading file using threads')
108-
@click.option('-s', '--subdir', type=str, default=None,
109-
help='Dataset directory on mds (i.e. {year}/{month}) - If present boost the connection')
110-
@click.option('--overwrite', required=False, is_flag=True, default=False,
111-
help='Force overwrite of the file')
112-
@click.option('--keep-timestamps', required=False, is_flag=True, default=False,
113-
help='After the download, set the correct timestamp to the file')
114-
@click.option('--sync-time', required=False, is_flag=True, default=False,
115-
help='Update the file if it changes on the server using last update information')
116-
@click.option('--sync-etag', required=False, is_flag=True, default=False,
117-
help='Update the file if it changes on the server using etag information')
184+
@click.option(
185+
"-b", "--bucket", "s3_bucket", required=True, type=str, help="Bucket name"
186+
)
187+
@click.option(
188+
"-f",
189+
"--filter",
190+
"file_filter",
191+
required=True,
192+
type=str,
193+
help="Filter on the online files",
194+
)
195+
@click.option(
196+
"-o", "--output-directory", required=True, type=str, help="Output directory"
197+
)
198+
@click.option("-p", "--product", required=True, type=str, help="The product name")
199+
@click.option("-i", "--dataset-id", required=True, type=str, help="Dataset Id")
200+
@click.option(
201+
"-g", "--dataset-version", type=str, default=None, help="Dataset version or tag"
202+
)
203+
@click.option(
204+
"-r", "--recursive", is_flag=True, default=False, help="List recursive all s3 files"
205+
)
206+
@click.option(
207+
"--threads",
208+
"n_threads",
209+
type=int,
210+
default=None,
211+
help="Downloading file using threads",
212+
)
213+
@click.option(
214+
"-s",
215+
"--subdir",
216+
type=str,
217+
default=None,
218+
help="Dataset directory on mds (i.e. {year}/{month}) - If present boost the connection",
219+
)
220+
@click.option(
221+
"--overwrite",
222+
required=False,
223+
is_flag=True,
224+
default=False,
225+
help="Force overwrite of the file",
226+
)
227+
@click.option(
228+
"--keep-timestamps",
229+
required=False,
230+
is_flag=True,
231+
default=False,
232+
help="After the download, set the correct timestamp to the file",
233+
)
234+
@click.option(
235+
"--sync-time",
236+
required=False,
237+
is_flag=True,
238+
default=False,
239+
help="Update the file if it changes on the server using last update information",
240+
)
241+
@click.option(
242+
"--sync-etag",
243+
required=False,
244+
is_flag=True,
245+
default=False,
246+
help="Update the file if it changes on the server using etag information",
247+
)
118248
def s3_get(**kwargs):
119249
mds_s3.download_files(**kwargs)
120250

121251

122252
@main.command()
123-
@click.option('-b', '--bucket', 's3_bucket', required=True, type=str, help='Filter on the online files')
124-
@click.option('-f', '--filter', 'file_filter', required=True, type=str, help='Filter on the online files')
125-
@click.option('-p', '--product', required=True, type=str, help='The product name')
126-
@click.option('-i', '--dataset-id', required=False, type=str, help='Dataset Id')
127-
@click.option('-g', '--dataset-version', type=str, default=None, help='Dataset version or tag')
128-
@click.option('-s', '--subdir', type=str, default=None,
129-
help='Dataset directory on mds (i.e. {year}/{month}) - If present boost the connection')
130-
@click.option('-r', '--recursive', is_flag=True, default=False, help='List recursive all s3 files')
253+
@click.option(
254+
"-b",
255+
"--bucket",
256+
"s3_bucket",
257+
required=True,
258+
type=str,
259+
help="Filter on the online files",
260+
)
261+
@click.option(
262+
"-f",
263+
"--filter",
264+
"file_filter",
265+
required=True,
266+
type=str,
267+
help="Filter on the online files",
268+
)
269+
@click.option("-p", "--product", required=True, type=str, help="The product name")
270+
@click.option("-i", "--dataset-id", required=False, type=str, help="Dataset Id")
271+
@click.option(
272+
"-g", "--dataset-version", type=str, default=None, help="Dataset version or tag"
273+
)
274+
@click.option(
275+
"-s",
276+
"--subdir",
277+
type=str,
278+
default=None,
279+
help="Dataset directory on mds (i.e. {year}/{month}) - If present boost the connection",
280+
)
281+
@click.option(
282+
"-r", "--recursive", is_flag=True, default=False, help="List recursive all s3 files"
283+
)
131284
def s3_list(**kwargs):
132285
s3_files = mds_s3.get_file_list(**kwargs)
133286
print(f"{' '.join([f.file for f in s3_files])}")
134287

135288

136-
if __name__ == '__main__':
289+
if __name__ == "__main__":
137290
main()

0 commit comments

Comments
 (0)