Skip to content

Commit 348d4be

Browse files
janekmijanekmi
committed
test: introduce DAOS dtx tests
Signed-off-by: Jan Michalski <[email protected]>
1 parent d9fa506 commit 348d4be

21 files changed

+1484
-0
lines changed

‎.github/workflows/daos.yml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# SPDX-License-Identifier: BSD-3-Clause
2+
# Copyright 2024, Intel Corporation
3+
#
4+
5+
name: DAOS
6+
7+
on:
8+
push:
9+
10+
permissions: {}
11+
12+
jobs:
13+
clear_caches:
14+
name: Clear caches
15+
runs-on: ubuntu-latest
16+
strategy:
17+
matrix:
18+
ref: [master, release/2.6]
19+
permissions:
20+
actions: write
21+
steps:
22+
- run: |
23+
curl -L \
24+
-X DELETE \
25+
-H "Accept: application/vnd.github+json" \
26+
-H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
27+
-H "X-GitHub-Api-Version: 2022-11-28" \
28+
"https://api.github.com/repos/pmem/pmdk/actions/caches?key=opt_daos-${{ matrix.ref }}"
29+
daos_tests:
30+
name: DAOS
31+
needs: clear_caches
32+
strategy:
33+
matrix:
34+
ref: [master, release/2.6]
35+
uses: pmem/pmdk/.github/workflows/daos_tests.yml@DAOS-tests
36+
with:
37+
DAOS_REF: ${{ matrix.ref }}

‎.github/workflows/daos_tests.yml

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
# SPDX-License-Identifier: BSD-3-Clause
2+
# Copyright 2024, Intel Corporation
3+
#
4+
5+
name: DAOS tests
6+
7+
on:
8+
workflow_call:
9+
inputs:
10+
DAOS_REF:
11+
required: true
12+
type: string
13+
workflow_dispatch:
14+
inputs:
15+
DAOS_REF:
16+
description: DAOS ref
17+
required: true
18+
type: string
19+
20+
permissions: {}
21+
22+
env:
23+
DAOS_PATH: /opt/daos
24+
25+
jobs:
26+
daos_test:
27+
name: DAOS tests
28+
runs-on: ubuntu-latest
29+
steps:
30+
- name: Generate a safe ref
31+
id: safe_ref
32+
run: |
33+
ref=${{ inputs.DAOS_REF }}
34+
safe_ref=${ref//\//_}
35+
echo "value=$safe_ref" >> $GITHUB_OUTPUT
36+
37+
- name: Clone the DAOS repo
38+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
39+
with:
40+
repository: daos-stack/daos
41+
ref: ${{ inputs.DAOS_REF }}
42+
path: daos
43+
44+
- name: Clone the PMDK repo
45+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
46+
with:
47+
repository: pmem/pmdk
48+
path: pmdk
49+
50+
- name: Restore ${{ env.DAOS_PATH }} from cache
51+
id: cache
52+
uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
53+
with:
54+
path: ${{ env.DAOS_PATH }}
55+
key: opt_daos-${{ inputs.DAOS_REF }}
56+
fail-on-cache-miss: false
57+
58+
- name: Apply DAOS patches
59+
if: steps.cache.outputs.cache-hit != 'true'
60+
working-directory: daos
61+
run: git apply ../pmdk/src/test/daos_dtx/*.patch
62+
63+
# It seems necessary for both building and running.
64+
- name: Install DAOS' dependencies
65+
working-directory: daos
66+
run: |
67+
sudo pip install -r requirements-build.txt
68+
sudo utils/scripts/install-ubuntu.sh
69+
70+
# https://go.dev/doc/install
71+
- name: Update the golang package
72+
if: steps.cache.outputs.cache-hit != 'true'
73+
env:
74+
GOLANG_VER: go1.23.1 # >= 1.21 because of https://go.dev/doc/toolchain
75+
run: |
76+
sudo apt-get remove golang-go
77+
wget https://go.dev/dl/${{ env.GOLANG_VER }}.linux-amd64.tar.gz
78+
sudo tar -C /usr/local -xzf ${{ env.GOLANG_VER }}.linux-amd64.tar.gz
79+
80+
- name: Install Valgrind
81+
working-directory: pmdk/utils/docker/images/
82+
run: sudo ./install-valgrind.sh
83+
84+
- name: Generate ${{ env.OPTS_FILE }}
85+
if: steps.cache.outputs.cache-hit != 'true'
86+
working-directory: daos
87+
env:
88+
OPTS_FILE: daos.conf
89+
run: |
90+
cat <<-EOF >> ${{ env.OPTS_FILE }}
91+
BUILD_TYPE = 'debug'
92+
TARGET_TYPE = 'debug'
93+
PREFIX = '${{ env.DAOS_PATH }}'
94+
GO_BIN = '/usr/local/go/bin/go'
95+
EOF
96+
cat ${{ env.OPTS_FILE }}
97+
98+
- name: Build DAOS
99+
if: steps.cache.outputs.cache-hit != 'true'
100+
working-directory: daos
101+
run: |
102+
git submodule init
103+
git submodule update
104+
scons install -j16 --build-deps=yes
105+
106+
- name: Save ${{ env.DAOS_PATH }} as cache
107+
if: steps.cache.outputs.cache-hit != 'true'
108+
uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
109+
with:
110+
path: ${{ env.DAOS_PATH }}
111+
key: opt_daos-${{ inputs.DAOS_REF }}
112+
113+
- name: Install PMDK's dependencies
114+
run: sudo apt-get install libndctl-dev libdaxctl-dev pandoc
115+
116+
- name: Build PMDK
117+
working-directory: pmdk
118+
env:
119+
PMDK_BUILD_DAOS_TESTS: y
120+
DAOS_INSTALL_PATH: ${{ env.DAOS_PATH }}
121+
run: |
122+
export DAOS_SRC_PATH=$(realpath ../daos)
123+
utils/gha-runners/build-pmdk.sh
124+
125+
- name: Create testconfig files
126+
working-directory: pmdk
127+
env:
128+
PMEM_FS_DIR: /dev/shm
129+
PMEM_FS_DIR_FORCE_PMEM: 1
130+
run: |
131+
utils/create-testconfig.sh
132+
for testconfig in src/test/testconfig.{sh,py}; do
133+
echo "$testconfig"
134+
cat $testconfig
135+
echo
136+
done
137+
138+
- name: Run daos_dtx tests (1/2)
139+
working-directory: pmdk/src/test
140+
env:
141+
LD_LIBRARY_PATH: ${{ env.DAOS_PATH }}/prereq/debug/ofi/lib
142+
run: |
143+
export PATH=${{ env.DAOS_PATH }}/bin:$PATH
144+
./RUNTESTS.py daos_dtx
145+
146+
- name: Run daos_dtx tests (2/2)
147+
working-directory: pmdk/src/test
148+
env:
149+
PMEMOBJ_CONF: sds.at_create=0
150+
run: |
151+
export PATH=${{ env.DAOS_PATH }}/bin:$PATH
152+
./RUNTESTS.sh daos_dtx 2> daos_dtx/err.log
153+
154+
- name: Upload test results as artifact
155+
if: success() || failure()
156+
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
157+
with:
158+
name: daos_dtx-${{ steps.safe_ref.outputs.value }}
159+
path: pmdk/src/test/daos_dtx

‎src/test/Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ OTHER_TESTS = \
122122
compat_incompat_features\
123123
ctl_prefault\
124124
ctl_cow\
125+
daos_dtx\
125126
magic\
126127
out_err\
127128
out_err_mt\

‎src/test/daos_dtx/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
daos_dtx

‎src/test/daos_dtx/00_harmonize_VOS_pool_path.patch

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
diff --git a/src/vos/tests/vts_common.c b/src/vos/tests/vts_common.c
2+
index 618f9feddd..7452ba2a34 100644
3+
--- a/src/vos/tests/vts_common.c
4+
+++ b/src/vos/tests/vts_common.c
5+
@@ -52,11 +52,22 @@ vts_file_exists(const char *filename)
6+
}
7+
8+
int
9+
-vts_alloc_gen_fname(char **fname)
10+
+vts_alloc_gen_fname(char *po_uuid_str, char **fname)
11+
{
12+
int rc;
13+
14+
- rc = asprintf(fname, "%s/vpool.%d", vos_path, gc++);
15+
+ assert(*fname == NULL);
16+
+ rc = asprintf(fname, "%s/%s/", vos_path, po_uuid_str);
17+
+ if (rc < 0) {
18+
+ *fname = NULL;
19+
+ print_error("Failed to allocate memory for fname: rc = %d\n", rc);
20+
+ return rc;
21+
+ }
22+
+ rc = mkdir(*fname, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
23+
+ assert(rc == 0 || errno == EEXIST);
24+
+ free(*fname);
25+
+
26+
+ rc = asprintf(fname, "%s/%s/vpool.%d", vos_path, po_uuid_str, gc++);
27+
if (rc < 0) {
28+
*fname = NULL;
29+
print_error("Failed to allocate memory for fname: rc = %d\n", rc);
30+
@@ -71,9 +82,9 @@ vts_pool_fallocate(char **fname)
31+
{
32+
int ret = 0, fd;
33+
34+
- ret = vts_alloc_gen_fname(fname);
35+
- if (ret)
36+
- return ret;
37+
+ // ret = vts_alloc_gen_fname(fname);
38+
+ // if (ret)
39+
+ // return ret;
40+
41+
fd = open(*fname, O_CREAT | O_TRUNC | O_RDWR, 0666);
42+
if (fd < 0) {
43+
@@ -92,10 +103,17 @@ int
44+
vts_ctx_init(struct vos_test_ctx *tcx, size_t psize)
45+
{
46+
int rc;
47+
+ char uuid_str[] = "dd6728be-696a-11ef-a059-a4bf0165c389";
48+
+ char uuid_str2[] = "591d19e4-69fe-11ef-b13d-a4bf0165c389";
49+
50+
memset(tcx, 0, sizeof(*tcx));
51+
+
52+
+ uuid_parse(uuid_str, tcx->tc_po_uuid);
53+
+ // uuid_generate_time_safe(tcx->tc_po_uuid);
54+
+ // uuid_unparse(tcx->tc_po_uuid, uuid_str);
55+
+
56+
oid_cnt = 0;
57+
- rc = vts_alloc_gen_fname(&tcx->tc_po_name);
58+
+ rc = vts_alloc_gen_fname(uuid_str, &tcx->tc_po_name);
59+
assert_int_equal(rc, 0);
60+
61+
if (vts_file_exists(tcx->tc_po_name)) {
62+
@@ -103,8 +121,8 @@ vts_ctx_init(struct vos_test_ctx *tcx, size_t psize)
63+
assert_int_equal(rc, 0);
64+
}
65+
66+
- uuid_generate_time_safe(tcx->tc_po_uuid);
67+
- uuid_generate_time_safe(tcx->tc_co_uuid);
68+
+ // uuid_generate_time_safe(tcx->tc_co_uuid);
69+
+ uuid_parse(uuid_str2, tcx->tc_co_uuid);
70+
71+
/* specify @psize as both NVMe size and SCM size */
72+
rc = vos_pool_create(tcx->tc_po_name, tcx->tc_po_uuid, psize, psize, 0, 0 /* version */,
73+
@@ -155,14 +173,14 @@ vts_ctx_fini(struct vos_test_ctx *tcx)
74+
assert_rc_equal(rc, 0);
75+
/* fallthrough */
76+
case TCX_CO_CREATE:
77+
- rc = vos_cont_destroy(tcx->tc_po_hdl, tcx->tc_co_uuid);
78+
- assert_rc_equal(rc, 0);
79+
+ // rc = vos_cont_destroy(tcx->tc_po_hdl, tcx->tc_co_uuid);
80+
+ // assert_rc_equal(rc, 0);
81+
/* fallthrough */
82+
case TCX_PO_CREATE_OPEN:
83+
rc = vos_pool_close(tcx->tc_po_hdl);
84+
assert_rc_equal(rc, 0);
85+
- rc = vos_pool_destroy(tcx->tc_po_name, tcx->tc_po_uuid);
86+
- assert_rc_equal(rc, 0);
87+
+ // rc = vos_pool_destroy(tcx->tc_po_name, tcx->tc_po_uuid);
88+
+ // assert_rc_equal(rc, 0);
89+
free(tcx->tc_po_name);
90+
/* fallthrough */
91+
}
92+
diff --git a/src/vos/tests/vts_common.h b/src/vos/tests/vts_common.h
93+
index 2a08cbd8ff..b7dc09dcfe 100644
94+
--- a/src/vos/tests/vts_common.h
95+
+++ b/src/vos/tests/vts_common.h
96+
@@ -83,7 +83,7 @@ bool
97+
vts_file_exists(const char *filename);
98+
99+
int
100+
-vts_alloc_gen_fname(char **fname);
101+
+vts_alloc_gen_fname(char *po_uuid, char **fname);
102+
103+
int
104+
vts_pool_fallocate(char **fname);
105+
diff --git a/src/vos/tests/vts_io.c b/src/vos/tests/vts_io.c
106+
index 5ec1bb08f7..abc269b1f6 100644
107+
--- a/src/vos/tests/vts_io.c
108+
+++ b/src/vos/tests/vts_io.c
109+
@@ -956,7 +956,7 @@ io_obj_cache_test(void **state)
110+
struct vos_object *obj1, *obj2;
111+
daos_epoch_range_t epr = {0, 1};
112+
daos_unit_oid_t oids[2];
113+
- char *po_name;
114+
+ char *po_name = "";
115+
uuid_t pool_uuid;
116+
daos_handle_t l_poh, l_coh;
117+
struct daos_lru_cache *old_cache;
118+
@@ -970,8 +970,8 @@ io_obj_cache_test(void **state)
119+
old_cache = tls->vtl_ocache;
120+
tls->vtl_ocache = occ;
121+
122+
- rc = vts_alloc_gen_fname(&po_name);
123+
- assert_int_equal(rc, 0);
124+
+ // rc = vts_alloc_gen_fname(&po_name);
125+
+ // assert_int_equal(rc, 0);
126+
127+
uuid_generate_time_safe(pool_uuid);
128+
rc = vos_pool_create(po_name, pool_uuid, VPOOL_256M, 0, 0, 0 /* version */, &l_poh);
129+
@@ -1105,7 +1105,7 @@ io_obj_cache_test(void **state)
130+
assert_rc_equal(rc, 0);
131+
vos_obj_cache_destroy(occ);
132+
tls->vtl_ocache = old_cache;
133+
- free(po_name);
134+
+ // free(po_name);
135+
}
136+
137+
static void
138+
diff --git a/src/vos/tests/vts_pool.c b/src/vos/tests/vts_pool.c
139+
index acfd4e46a8..8e2fe3742c 100644
140+
--- a/src/vos/tests/vts_pool.c
141+
+++ b/src/vos/tests/vts_pool.c
142+
@@ -49,7 +49,7 @@ static int
143+
pool_file_setup(void **state)
144+
{
145+
struct vp_test_args *arg = *state;
146+
- int ret = 0;
147+
+ // int ret = 0;
148+
149+
D_ALLOC(arg->fname, sizeof(char *));
150+
assert_ptr_not_equal(arg->fname, NULL);
151+
@@ -57,8 +57,8 @@ pool_file_setup(void **state)
152+
D_ALLOC_ARRAY(arg->poh, 10);
153+
assert_ptr_not_equal(arg->poh, NULL);
154+
155+
- ret = vts_alloc_gen_fname(&arg->fname[0]);
156+
- assert_int_equal(ret, 0);
157+
+ // ret = vts_alloc_gen_fname(&arg->fname[0]);
158+
+ // assert_int_equal(ret, 0);
159+
return 0;
160+
}
161+
162+
@@ -153,8 +153,8 @@ pool_ops_run(void **state)
163+
VPOOL_TEST_WAL_SZ, 0,
164+
0 /* version */, poh);
165+
} else {
166+
- ret = vts_alloc_gen_fname(&arg->fname[j]);
167+
- assert_int_equal(ret, 0);
168+
+ // ret = vts_alloc_gen_fname(&arg->fname[j]);
169+
+ // assert_int_equal(ret, 0);
170+
ret = vos_pool_create_ex(arg->fname[j], arg->uuid[j],
171+
VPOOL_256M, 0, VPOOL_TEST_WAL_SZ,
172+
0, 0 /* version */, poh);

0 commit comments

Comments
 (0)