Skip to content

Commit 27577fa

Browse files
redjaxactions-user
andauthored
Dev (#340)
* Fix red_utils.ext imports for time_utils, detect either arrow or pend… (#327) Fix red_utils.ext imports for time_utils, detect either arrow or pendulum * Repo/updates (#328) * Update pre-commit, specify the ruff.ci.toml file * docs: Update repo README. Add docs/ dir for extended README files * Auto-export requirements files * Update developer docs with local testing notes * Update pre-commit * another pre-commit fix --------- Co-authored-by: GitHub Action <[email protected]> * Feat/dataframe utils (#330) * Add/modify dataframe utils, specifically Pandas * Update tests. Rename any references to file_utils -> path_utils * Feat/dataframe utils (#331) * Add/modify dataframe utils, specifically Pandas * Update tests. Rename any references to file_utils -> path_utils * Auto-export requirements files --------- Co-authored-by: GitHub Action <[email protected]> * Fix tests, update .gitignore (#332) * Fix tests, update .gitignore * Auto-export requirements files --------- Co-authored-by: GitHub Action <[email protected]> * Feat/pytests (#333) * Add tests dir to vscode workspace * Fix diskcache utils * Add dict tests * Add hash util tests * Add uuid_util tests * release(v0.2.14): Add dataframe utils. Update tests. * Auto-export requirements files * Fix/path utils (#338) * Dev (#337) * Fix red_utils.ext imports for time_utils, detect either arrow or pend… (#327) Fix red_utils.ext imports for time_utils, detect either arrow or pendulum * Repo/updates (#328) * Update pre-commit, specify the ruff.ci.toml file * docs: Update repo README. Add docs/ dir for extended README files * Auto-export requirements files * Update developer docs with local testing notes * Update pre-commit * another pre-commit fix --------- Co-authored-by: GitHub Action <[email protected]> * Feat/dataframe utils (#330) * Add/modify dataframe utils, specifically Pandas * Update tests. Rename any references to file_utils -> path_utils * Feat/dataframe utils (#331) * Add/modify dataframe utils, specifically Pandas * Update tests. Rename any references to file_utils -> path_utils * Auto-export requirements files --------- Co-authored-by: GitHub Action <[email protected]> * Fix tests, update .gitignore (#332) * Fix tests, update .gitignore * Auto-export requirements files --------- Co-authored-by: GitHub Action <[email protected]> * Feat/pytests (#333) * Add tests dir to vscode workspace * Fix diskcache utils * Add dict tests * Add hash util tests * Add uuid_util tests * release(v0.2.14): Add dataframe utils. Update tests. --------- Co-authored-by: GitHub Action <[email protected]> * Update crawl_dir function * Fix issue with search_str var in crawl_dir._crawl() * Auto-export requirements files --------- Co-authored-by: GitHub Action <[email protected]> * Add time_utils tests * Add more time_utils tests * Start writing test TODOs * Add pytest to ci requirements * Auto-export requirements files * Work on pytest tests workflows * fix workflow duplication with push/pull settings * Update tests workflow * Fix pytest workflow --------- Co-authored-by: GitHub Action <[email protected]>
1 parent f84f957 commit 27577fa

16 files changed

+302
-30
lines changed

.github/workflows/export-requirements.yml

+4
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22
name: Export project requirements.txt file(s)
33

44
on:
5+
# push:
6+
# branches:
7+
# - dev
8+
# - ci
59
pull_request:
610
branches:
711
- main

.github/workflows/pdm-lint-format.yml

+5-2
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,14 @@
22
name: Run lint & format PDM Scripts
33

44
on:
5-
push:
5+
# push:
6+
# branches:
7+
# - dev
8+
# - ci
9+
pull_request:
610
branches:
711
- main
812
- dev
9-
- ci
1013

1114
jobs:
1215
run-pdm-scripts:

.github/workflows/tests.yml

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
---
2+
name: Run Pytest tests
3+
4+
on:
5+
# push:
6+
# branches:
7+
# - dev
8+
# - ci
9+
pull_request:
10+
branches:
11+
- main
12+
# - dev
13+
- ci
14+
15+
jobs:
16+
run-pytest-tests:
17+
runs-on: ubuntu-latest
18+
19+
steps:
20+
- name: Checkout code
21+
uses: actions/checkout@v2
22+
with:
23+
ref: ${{ github.event.pull_request.head.ref }}
24+
25+
- name: Set up Python
26+
uses: actions/setup-python@v2
27+
with:
28+
python-version: 3.11
29+
30+
- name: Install PDM & dependencies
31+
run: |
32+
python -m pip install pdm
33+
pdm install --no-lock
34+
pdm --version
35+
36+
- name: Run PDM Scripts
37+
run: |
38+
pdm run pytest

pdm.lock

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pyproject.toml

+3-11
Original file line numberDiff line numberDiff line change
@@ -63,23 +63,15 @@ http = [
6363
"msgpack>=1.0.5",
6464
"pendulum>=2.1.2",
6565
]
66-
"ci.lint" = [
67-
"ruff>=0.1.7",
68-
"black>=23.12.0",
69-
]
70-
arrow = [
71-
"arrow>=1.3.0",
72-
]
66+
"ci.lint" = ["ruff>=0.1.7", "black>=23.12.0"]
67+
arrow = ["arrow>=1.3.0"]
7368
data = [
7469
"jupyter>=1.0.0",
7570
"ipykernel>=6.27.1",
7671
"pandas>=2.1.4",
7772
"fastparquet>=2023.10.1",
7873
]
79-
dataframes = [
80-
"pandas>=2.1.4",
81-
"fastparquet>=2023.10.1",
82-
]
74+
dataframes = ["pandas>=2.1.4", "fastparquet>=2023.10.1"]
8375

8476
[tool.pdm.dev-dependencies]
8577
dev = [

red_utils/std/path_utils/operations.py

-16
Original file line numberDiff line numberDiff line change
@@ -145,22 +145,6 @@ def _crawl(
145145

146146
return return_obj
147147

148-
# return_obj: dict[str, list[Path]] = {"files": [], "dirs": []}
149-
150-
# for i in target.glob(search_str):
151-
# if i.is_file():
152-
# return_obj["files"].append(i)
153-
# else:
154-
# return_obj["dirs"].append(i)
155-
156-
# match return_type:
157-
# case "all":
158-
# return return_obj
159-
# case "files":
160-
# return return_obj["files"]
161-
# case "dirs":
162-
# return return_obj["dirs"]
163-
164148

165149
def crawl_dir_old(
166150
in_dir: Union[str, Path] = None,

tests/REAMDE.md

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# Pytest tests for red_utils
2+
3+
I am still learning the `pytest` framework, and make no promises or guarantees that there are tests for all of the functionality in this module.
4+
5+
## Tests TODO
6+
7+
- `std` tests
8+
- `database_managers` tests
9+
- [ ] create fixture for `SQLiteConnManager`
10+
- [ ] test initializing database file
11+
- [ ] test getting a connection
12+
- [ ] create default test data
13+
- [ ] test inserting records
14+
- [ ] test updating records
15+
- [ ] test deleting records
16+
- [ ] test `.get_tables()`
17+
- [ ] test `.run_sqlite_stmt()`
18+
- cleanup:
19+
- [ ] delete `sqlite` database file
20+
- `ext` tests
21+
- `dataframe_utils` tests
22+
- Fixtures:
23+
- [ ] dict that can be converted to a Pandas dataframe
24+
- [ ] Pandas dataframe
25+
- [ ] list of Pandas dataframes
26+
- [ ] `dict` for a `.parquet` file
27+
- [ ] CSV data for a `.csv` file
28+
- [ ] test writing `.parquet` file
29+
- [ ] test reading `.parquet` file
30+
- [ ] test reading `.parquet` file to dataframe
31+
- [ ] test saving dataframe to `.parquet` file
32+
- [ ] test writing `.csv` file
33+
- [ ] test reading `.csv` file
34+
- [ ] test reading `.csv` file to dataframe
35+
- [ ] test saving dataframe to `.csv` file
36+
- [ ] test converting `.csv` file to `.parquet` file
37+
- [ ] test converting `.parquet` file to `.csv` file
38+
- [ ] test concatenating dataframes
39+
- [ ] test deduplicating Pandas dataframe
40+
- cleanup:
41+
- [ ] delete all `.csv` files
42+
- [ ] delete all `.parquet` files
43+
- `diskcache_utils` tests
44+
- Fixtures:
45+
- [ ] a `diskcache.Cache` object
46+
- [ ] an in-memory cache
47+
- [ ] test setting value
48+
- [ ] test getting value
49+
- [ ] test expiring value
50+
- [ ] test updating value
51+
- [ ] test updating expire value
52+
- [ ] test checking if key exists in cache
53+
- cleanup:
54+
- [ ] delete diskcache.cache dir/db

tests/conftest.py

+1
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,5 @@
77
"tests.fixtures.std.dict_fixtures",
88
"tests.fixtures.std.hash_fixtures",
99
"tests.fixtures.std.uuid_fixtures",
10+
"tests.fixtures.ext.time_fixtures",
1011
]

tests/ext_tests/__init__.py

Whitespace-only changes.

tests/ext_tests/time_util_tests/__init__.py

Whitespace-only changes.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
from __future__ import annotations
2+
3+
from red_utils.ext import time_utils
4+
from red_utils.ext.time_utils import arrow_utils
5+
6+
import arrow
7+
import pendulum
8+
9+
from pytest import mark, xfail
10+
11+
@mark.xfail
12+
def test_fail_arrow_ts(bad_ts_str: str):
13+
assert bad_ts_str is not None, "bad_ts_str must not be None"
14+
assert isinstance(
15+
bad_ts_str, str
16+
), f"bad_ts_str must be of type str, not {type(bad_ts_str)}"
17+
18+
bad_ts = arrow.get(bad_ts_str)
19+
20+
assert isinstance(
21+
bad_ts, arrow.Arrow
22+
), f"bad_ts must be of type arrow.Arrow, not ({type(bad_ts)})"
23+
24+
25+
@mark.xfail
26+
def test_fail_arrow_shift():
27+
ts = arrow.now()
28+
assert isinstance(ts, arrow.Arrow), "ts must be of type arrow.Arrow"
29+
30+
shifted = ts.shift(weekday=1)
31+
32+
assert ts > shifted, f"ts ({ts}) must be later than date ({shifted})"
33+
34+
35+
@mark.xfail
36+
def test_fail_pendulum_ts(bad_ts_str: str):
37+
assert bad_ts_str is not None, "bad_ts_str must not be None"
38+
assert isinstance(
39+
bad_ts_str, str
40+
), f"bad_ts_str must be of type str, not {type(bad_ts_str)}"
41+
42+
bad_ts = pendulum.parse(bad_ts_str)
43+
44+
assert isinstance(
45+
bad_ts, pendulum.DateTime
46+
), f"bad_ts must be of type pendulum.DateTime, not ({type(bad_ts)})"
47+
48+
49+
@mark.xfail
50+
def test_fail_pendulum_24h_ts():
51+
time_utils.get_ts(as_str=True, str_fmt=1)
52+
53+
54+
@mark.xfail
55+
def test_fail_pendulum_char_replace():
56+
ts = time_utils.get_ts(
57+
as_str=True, safe_str=True, char_replace_map=[{"search": "-", "replace": "^"}]
58+
)
59+
assert ts is not None, "ts must not be None"
60+
assert isinstance(ts, str), f"ts must be of type str, not ({type(ts)})"
61+
assert (
62+
"^" not in ts
63+
), f"Timestamp missing expected '^' character. Timestamp string: {ts}"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
from __future__ import annotations
2+
3+
from red_utils.ext import time_utils
4+
from red_utils.ext.time_utils import arrow_utils
5+
6+
import arrow
7+
import pendulum
8+
9+
from pytest import mark, xfail
10+
11+
@mark.time_utils
12+
def test_arrow_ts_str(ts_str: str):
13+
assert ts_str is not None, "ts_str must not be None"
14+
assert isinstance(ts_str, str), f"ts_str must be of type str, not ({type(ts_str)})"
15+
16+
_ts = arrow.get(ts_str)
17+
assert isinstance(_ts, arrow.Arrow), "_ts must be of type Arrow"
18+
19+
20+
@mark.time_utils
21+
def test_arrow_shift(arrow_now: arrow.Arrow):
22+
assert isinstance(arrow_now, arrow.Arrow), "arrow_now must be of type arrow.Arrow"
23+
24+
shifted = arrow_now.shift(weekday=1)
25+
26+
assert (
27+
arrow_now < shifted
28+
), f"arrow_now ({arrow_now}) must be earlier than date ({shifted})"
29+
30+
31+
@mark.time_utils
32+
def test_pendulum_ts(pendulum_now):
33+
assert pendulum_now is not None, "ts_str must not be None"
34+
assert isinstance(
35+
pendulum_now, pendulum.DateTime
36+
), f"pendulum_now must be of type pendulum.DateTime, not {type(pendulum_now)}"
37+
38+
39+
@mark.time_utils
40+
def test_pendulum_24h_ts():
41+
ts = time_utils.get_ts(as_str=True, str_fmt=time_utils.TIME_FMT_24H)
42+
assert ts is not None, "ts must not be None"
43+
assert isinstance(ts, str), f"ts must be of type str, not ({type(ts)})"
44+
45+
46+
@mark.time_utils
47+
def test_pendulum_get_ts():
48+
ts = time_utils.get_ts()
49+
assert ts is not None, "ts must not be None"
50+
assert isinstance(
51+
ts, pendulum.DateTime
52+
), f"ts must be of type pendulum.DateTime, not ({type(ts)})"
53+
54+
55+
@mark.time_utils
56+
def test_pendulum_safestr():
57+
ts = time_utils.get_ts(as_str=True, safe_str=True)
58+
assert ts is not None, "ts must not be None"
59+
assert isinstance(ts, str), f"ts must be of type str, not ({type(ts)})"
60+
61+
62+
@mark.time_utils
63+
def test_pendulum_char_replace():
64+
ts = time_utils.get_ts(
65+
as_str=True, safe_str=True, char_replace_map=[{"search": "-", "replace": "^"}]
66+
)
67+
assert ts is not None, "ts must not be None"
68+
assert isinstance(ts, str), f"ts must be of type str, not ({type(ts)})"
69+
assert (
70+
"^" in ts
71+
), f"Timestamp missing expected '^' character. Timestamp string: {ts}"

tests/fixtures/__init__.py

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
from __future__ import annotations
2+
3+
from . import ext, std

tests/fixtures/ext/__init__.py

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
from __future__ import annotations
2+
3+
from . import time_fixtures

tests/fixtures/ext/time_fixtures.py

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
from __future__ import annotations
2+
3+
from red_utils.ext import time_utils
4+
from red_utils.ext.time_utils import arrow_utils
5+
6+
import arrow
7+
import pendulum
8+
9+
from pytest import fixture
10+
11+
@fixture
12+
def bad_ts_str() -> str:
13+
return "20023-12-17"
14+
15+
16+
@fixture
17+
def ts_str() -> str:
18+
return "2023-12-17 00:00:00"
19+
20+
21+
@fixture
22+
def arrow_epoch_date() -> arrow.Arrow:
23+
return arrow.get("1970-01-01T00:00:00Z")
24+
25+
26+
@fixture
27+
def pendulum_epoch_date() -> pendulum.DateTime:
28+
return pendulum.parse("1970-01-01T:00:00:00Z")
29+
30+
31+
@fixture
32+
def arrow_now() -> arrow.Arrow:
33+
return arrow.now()
34+
35+
36+
@fixture
37+
def pendulum_now() -> pendulum.DateTime:
38+
return pendulum.now()

tests/test_time_utils.py

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
from __future__ import annotations
2+
3+
from .ext_tests.time_util_tests.expect_fail_tests import (
4+
test_fail_arrow_shift,
5+
test_fail_arrow_ts,
6+
test_fail_pendulum_24h_ts,
7+
test_fail_pendulum_char_replace,
8+
test_fail_pendulum_ts,
9+
)
10+
from .ext_tests.time_util_tests.expect_pass_tests import (
11+
test_arrow_shift,
12+
test_arrow_ts_str,
13+
test_pendulum_24h_ts,
14+
test_pendulum_char_replace,
15+
test_pendulum_get_ts,
16+
test_pendulum_safestr,
17+
test_pendulum_ts,
18+
)

0 commit comments

Comments
 (0)