Skip to content

Commit c2ceca8

Browse files
calccryptocalccrypto
committed
allow clients to call gufi_query
gufi_query.py is not meant to be called directly by users tests are just the first few gufi_query tests changed GitHub Actions builds to build with CLIENT=On
1 parent 06b8cf3 commit c2ceca8

14 files changed

+304
-6
lines changed

‎.github/workflows/check.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ env:
6868
CC: clang
6969
CXX: clang++
7070
DEP_INSTALL_PREFIX: ~/.local
71-
COMMON_CONFIG: -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -pedantic" -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -pedantic" -DDEP_INSTALL_PREFIX="${DEP_INSTALL_PREFIX}" -DDEP_BUILD_THREADS=2 -DENABLE_SUDO_TESTS=On
71+
COMMON_CONFIG: -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -pedantic" -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -pedantic" -DDEP_INSTALL_PREFIX="${DEP_INSTALL_PREFIX}" -DDEP_BUILD_THREADS=2 -DENABLE_SUDO_TESTS=On -DCLIENT=On
7272
PIP_BREAK_SYSTEM_PACKAGES: 1
7373

7474
jobs:

‎.github/workflows/codecov.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ env:
6868
CC: gcc
6969
CXX: g++
7070
DEP_INSTALL_PREFIX: ~/.local
71-
COMMON_CONFIG: -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -pedantic" -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -pedantic -std=c++11" -DDEP_INSTALL_PREFIX="${DEP_INSTALL_PREFIX}" -DDEP_BUILD_THREADS=2 -DENABLE_SUDO_TESTS=On -DGCOV=On -DCMAKE_BUILD_TYPE=Debug
71+
COMMON_CONFIG: -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -pedantic" -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -pedantic -std=c++11" -DDEP_INSTALL_PREFIX="${DEP_INSTALL_PREFIX}" -DDEP_BUILD_THREADS=2 -DENABLE_SUDO_TESTS=On -DGCOV=On -DCMAKE_BUILD_TYPE=Debug -DCLIENT=On
7272
PIP_BREAK_SYSTEM_PACKAGES: 1
7373

7474
jobs:

‎.github/workflows/codeql.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ env:
6868
CC: clang
6969
CXX: clang++
7070
DEP_INSTALL_PREFIX: ~/.local
71-
COMMON_CONFIG: -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -pedantic" -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -pedantic" -DDEP_INSTALL_PREFIX="${DEP_INSTALL_PREFIX}" -DDEP_BUILD_THREADS=2 -DENABLE_SUDO_TESTS=On
71+
COMMON_CONFIG: -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -pedantic" -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -pedantic" -DDEP_INSTALL_PREFIX="${DEP_INSTALL_PREFIX}" -DDEP_BUILD_THREADS=2 -DENABLE_SUDO_TESTS=On -DCLIENT=On
7272
PIP_BREAK_SYSTEM_PACKAGES: 1
7373

7474
jobs:

‎.github/workflows/test.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ env:
6868
CC: clang
6969
CXX: clang++
7070
DEP_INSTALL_PREFIX: ~/.local
71-
COMMON_CONFIG: -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -pedantic" -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -pedantic" -DDEP_INSTALL_PREFIX="${DEP_INSTALL_PREFIX}" -DDEP_BUILD_THREADS=2 -DENABLE_SUDO_TESTS=On
71+
COMMON_CONFIG: -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -pedantic" -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -pedantic" -DDEP_INSTALL_PREFIX="${DEP_INSTALL_PREFIX}" -DDEP_BUILD_THREADS=2 -DENABLE_SUDO_TESTS=On -DCLIENT=On
7272
PIP_BREAK_SYSTEM_PACKAGES: 1
7373

7474
jobs:

‎docs/latex/gufi_user.tex

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,4 +96,10 @@
9696
\include{sections/gufi_stat}
9797
\include{sections/gufi_stats}
9898

99+
\section{\gufiquery}
100+
\gufiquery is available on the client side. However, as it is intended
101+
for use by advanced users, it will not be described here. Please see
102+
the administrator or developer documentation for details about
103+
\gufiquery.
104+
99105
\end{document}

‎docs/man/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363
set(SERVER_CLIENT_MAN_PAGES
6464
gufi_find.1
6565
gufi_ls.1
66+
gufi_query.1
6667
gufi_stat.1
6768
gufi_stats.1
6869
)
@@ -71,7 +72,6 @@ set(SERVER_MAN_PAGES
7172
bfwreaddirplus2db.1
7273
gufi_dir2index.1
7374
gufi_dir2trace.1
74-
gufi_query.1
7575
gufi_trace2index.1
7676
gufi_treesummary.1
7777
querydbs.1

‎scripts/CMakeLists.txt

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,21 @@ foreach(TOOL ${TOOLS})
9494
endif()
9595
endforeach()
9696

97+
# allow for clients to call gufi_query via a script
98+
if (CLIENT)
99+
set(TOOL "query.py")
100+
set(USER_TOOL "gufi_${TOOL}")
101+
configure_file("${USER_TOOL}" "${USER_TOOL}" @ONLY)
102+
install(PROGRAMS "${CMAKE_CURRENT_BINARY_DIR}/${USER_TOOL}" DESTINATION "${BIN}" COMPONENT Server)
103+
104+
set(CLIENT_TOOL "${CMAKE_CURRENT_BINARY_DIR}/gufi_client_query")
105+
configure_file("gufi_client" "${CLIENT_TOOL}" @ONLY)
106+
install(PROGRAMS "${CLIENT_TOOL}"
107+
RENAME "gufi_query"
108+
DESTINATION "${BIN}"
109+
COMPONENT Client) # this is always needed for make package
110+
endif()
111+
97112
# gufi_jail is used by sshd_config to prevent commands other than gufi_* from running
98113
configure_file(gufi_jail gufi_jail @ONLY)
99114
install(PROGRAMS "${CMAKE_CURRENT_BINARY_DIR}/gufi_jail" DESTINATION "${BIN}" COMPONENT Server)

‎scripts/gufi_jail

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ set -f
1313

1414
set -- ${SSH_ORIGINAL_COMMAND}
1515
case "$1" in
16-
gufi_find|gufi_ls|gufi_stat|gufi_stats)
16+
gufi_find|gufi_ls|gufi_query|gufi_stat|gufi_stats)
1717
;;
1818
*)
1919
echo "Error: Command \"$1\" is not allowed" >&2

‎scripts/gufi_query.py

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#!/usr/bin/env python3
2+
# This file is part of GUFI, which is part of MarFS, which is released
3+
# under the BSD license.
4+
#
5+
#
6+
# Copyright (c) 2017, Los Alamos National Security (LANS), LLC
7+
# All rights reserved.
8+
#
9+
# Redistribution and use in source and binary forms, with or without modification,
10+
# are permitted provided that the following conditions are met:
11+
#
12+
# 1. Redistributions of source code must retain the above copyright notice, this
13+
# list of conditions and the following disclaimer.
14+
#
15+
# 2. Redistributions in binary form must reproduce the above copyright notice,
16+
# this list of conditions and the following disclaimer in the documentation and/or
17+
# other materials provided with the distribution.
18+
#
19+
# 3. Neither the name of the copyright holder nor the names of its contributors
20+
# may be used to endorse or promote products derived from this software without
21+
# specific prior written permission.
22+
#
23+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
24+
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
25+
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26+
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
27+
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
28+
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29+
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
30+
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
31+
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
32+
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33+
#
34+
#
35+
# From Los Alamos National Security, LLC:
36+
# LA-CC-15-039
37+
#
38+
# Copyright (c) 2017, Los Alamos National Security, LLC All rights reserved.
39+
# Copyright 2017. Los Alamos National Security, LLC. This software was produced
40+
# under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National
41+
# Laboratory (LANL), which is operated by Los Alamos National Security, LLC for
42+
# the U.S. Department of Energy. The U.S. Government has rights to use,
43+
# reproduce, and distribute this software. NEITHER THE GOVERNMENT NOR LOS
44+
# ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR
45+
# ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software is
46+
# modified to produce derivative works, such modified software should be
47+
# clearly marked, so as not to confuse it with the version available from
48+
# LANL.
49+
#
50+
# THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS
51+
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
52+
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53+
# ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR
54+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
55+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
56+
# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
57+
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
58+
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
59+
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
60+
# OF SUCH DAMAGE.
61+
62+
63+
64+
# This file should generally not be called directly. This script is
65+
# meant for advanced users running on clients to call gufi_query
66+
# without knowing the indexroot.
67+
import subprocess
68+
import sys
69+
70+
import gufi_config # pylint: disable=wrong-import-position
71+
72+
def run(args, config_path):
73+
'''
74+
Send all arguments to gufi_query
75+
'''
76+
77+
config = gufi_config.Server(config_path)
78+
79+
cmd = [config.query] + args[1:] + [config.indexroot]
80+
81+
# run the command
82+
query = subprocess.Popen(cmd) # pylint: disable=consider-using-with
83+
query.communicate() # block until query finishes
84+
85+
return query.returncode
86+
87+
if __name__ == '__main__':
88+
sys.exit(run(sys.argv, gufi_config.PATH))

‎test/regression/CMakeLists.txt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,12 @@ set(PYTHON
9797
gufi_stats
9898
)
9999

100+
if (CLIENT)
101+
list(APPEND PYTHON
102+
gufi_query.py
103+
)
104+
endif()
105+
100106
set(EXAMPLES
101107
deluidgidsummaryrecs
102108
generategidsummary

‎test/regression/gufi_query.py.expected

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
# gufi_query.py help
2+
$ gufi_query.py -h
3+
usage: gufi_query [options] GUFI_index ...
4+
options:
5+
-h help
6+
-H show assigned input values (debugging)
7+
-v version
8+
-T <SQL_tsum> SQL for tree-summary table
9+
-S <SQL_sum> SQL for summary table
10+
-E <SQL_ent> SQL for entries table
11+
-a AND/OR (SQL query combination)
12+
-n <threads> number of threads
13+
-j print the information in terse form
14+
-o <out_fname> output file (one-per-thread, with thread-id suffix)
15+
-d <delim> delimiter (one char) [use 'x' for 0x1E]
16+
-O <out_DB> output DB
17+
-u prefix row with 1 int column count and each column with 1 octet type and 1 size_t length
18+
-I <SQL_init> SQL init
19+
-F <SQL_fin> SQL cleanup
20+
-y <min level> minimum level to go down
21+
-z <max level> maximum level to go down
22+
-J <SQL_interm> SQL for intermediate results
23+
-K <create aggregate> SQL to create the final aggregation table
24+
-G <SQL_aggregate> SQL for aggregated results
25+
-m Keep mtime and atime same on the database files
26+
-B <buffer size> size of each thread's output buffer in bytes
27+
-w open the database files in read-write mode instead of read only mode
28+
-x index/query xattrs
29+
-k <filename> file containing directory names to skip
30+
-M <bytes> target memory footprint
31+
-s <path> File name prefix for swap files
32+
-p <path> Source path prefix for %s in SQL
33+
-e compress work items
34+
-Q <basename>
35+
<table>
36+
<template>.<table>
37+
<view> External database file basename, per-attach table name, template + table name, and the resultant view
38+
39+
GUFI_index find GUFI index here
40+
41+
42+
# Walk without returning results
43+
$ gufi_query.py
44+
45+
# Get only directories
46+
$ gufi_query.py -d " " -n 2 -S "SELECT name FROM vrsummary;"
47+
directory
48+
empty_directory
49+
leaf_directory
50+
prefix
51+
subdirectory
52+
unusual#? directory ,
53+
54+
# Get only non-directories
55+
$ gufi_query.py -d " " -n 2 -E "SELECT name FROM vrpentries;"
56+
.hidden
57+
1KB
58+
1MB
59+
directory_symlink
60+
executable
61+
file_symlink
62+
leaf_file1
63+
leaf_file2
64+
old_file
65+
readonly
66+
repeat_name
67+
repeat_name
68+
unusual, name?#
69+
writable
70+
71+
# Get all directory and non-directory names
72+
$ gufi_query.py -d " " -n 2 -S "SELECT name FROM vrsummary;" -E "SELECT name FROM vrpentries;"
73+
.hidden
74+
1KB
75+
1MB
76+
directory
77+
directory_symlink
78+
empty_directory
79+
executable
80+
file_symlink
81+
leaf_directory
82+
leaf_file1
83+
leaf_file2
84+
old_file
85+
prefix
86+
readonly
87+
repeat_name
88+
repeat_name
89+
subdirectory
90+
unusual#? directory ,
91+
unusual, name?#
92+
writable
93+

‎test/regression/gufi_query.py.sh.in

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#!/usr/bin/env bash
2+
# This file is part of GUFI, which is part of MarFS, which is released
3+
# under the BSD license.
4+
#
5+
#
6+
# Copyright (c) 2017, Los Alamos National Security (LANS), LLC
7+
# All rights reserved.
8+
#
9+
# Redistribution and use in source and binary forms, with or without modification,
10+
# are permitted provided that the following conditions are met:
11+
#
12+
# 1. Redistributions of source code must retain the above copyright notice, this
13+
# list of conditions and the following disclaimer.
14+
#
15+
# 2. Redistributions in binary form must reproduce the above copyright notice,
16+
# this list of conditions and the following disclaimer in the documentation and/or
17+
# other materials provided with the distribution.
18+
#
19+
# 3. Neither the name of the copyright holder nor the names of its contributors
20+
# may be used to endorse or promote products derived from this software without
21+
# specific prior written permission.
22+
#
23+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
24+
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
25+
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26+
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
27+
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
28+
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29+
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
30+
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
31+
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
32+
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33+
#
34+
#
35+
# From Los Alamos National Security, LLC:
36+
# LA-CC-15-039
37+
#
38+
# Copyright (c) 2017, Los Alamos National Security, LLC All rights reserved.
39+
# Copyright 2017. Los Alamos National Security, LLC. This software was produced
40+
# under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National
41+
# Laboratory (LANL), which is operated by Los Alamos National Security, LLC for
42+
# the U.S. Department of Energy. The U.S. Government has rights to use,
43+
# reproduce, and distribute this software. NEITHER THE GOVERNMENT NOR LOS
44+
# ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR
45+
# ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software is
46+
# modified to produce derivative works, such modified software should be
47+
# clearly marked, so as not to confuse it with the version available from
48+
# LANL.
49+
#
50+
# THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS
51+
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
52+
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53+
# ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR
54+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
55+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
56+
# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
57+
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
58+
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
59+
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
60+
# OF SUCH DAMAGE.
61+
62+
63+
64+
set -e
65+
source @CMAKE_CURRENT_BINARY_DIR@/setup.sh 1
66+
67+
OUTPUT="gufi_query.py.out"
68+
69+
(
70+
echo "# ${GUFI_QUERY_PY} help" | replace
71+
run_no_sort "${GUFI_QUERY_PY} -h"
72+
73+
echo "# Walk without returning results"
74+
run_sort "${GUFI_QUERY_PY}"
75+
76+
echo "# Get only directories"
77+
run_sort "${GUFI_QUERY_PY} -d \" \" -n ${THREADS} -S \"SELECT name FROM vrsummary;\""
78+
79+
echo "# Get only non-directories"
80+
run_sort "${GUFI_QUERY_PY} -d \" \" -n ${THREADS} -E \"SELECT name FROM vrpentries;\""
81+
82+
echo "# Get all directory and non-directory names"
83+
run_sort "${GUFI_QUERY_PY} -d \" \" -n ${THREADS} -S \"SELECT name FROM vrsummary;\" -E \"SELECT name FROM vrpentries;\""
84+
) 2>&1 | tee "${OUTPUT}"
85+
86+
@DIFF@ @CMAKE_CURRENT_BINARY_DIR@/gufi_query.py.expected "${OUTPUT}"
87+
rm "${OUTPUT}"

‎test/regression/gufi_tool.py.in

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ TOOLS = {
7777
'getfattr' : os.path.join(SCRIPTS, 'gufi_getfattr'),
7878
'ls' : os.path.join(SCRIPTS, 'gufi_ls'),
7979
'pugsql' : os.path.join(RTESTS, 'gufi_vt_pugsql.py'),
80+
'query.py' : os.path.join(SCRIPTS, 'gufi_query.py'),
8081
'querydbs' : os.path.join(SCRIPTS, 'querydbs'),
8182
'stat' : os.path.join(SCRIPTS, 'gufi_stat'),
8283
'stats' : os.path.join(SCRIPTS, 'gufi_stats'),

0 commit comments

Comments
 (0)