Skip to content

Commit 0699826

Browse files
Optimize linters (dashpay#4637)
* optimize: somehow optimize circular-dependencies.py Signed-off-by: pasta <[email protected]> * optimize: use parallel if available to lint in parallel Signed-off-by: pasta <[email protected]> * suggestions * more suggestions Co-authored-by: UdjinM6 <[email protected]>
1 parent 58b2605 commit 0699826

File tree

3 files changed

+101
-63
lines changed

3 files changed

+101
-63
lines changed

contrib/containers/ci/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ RUN apt-get update && apt-get install $APT_ARGS g++-arm-linux-gnueabihf && rm -r
4343
RUN apt-get update && apt-get install $APT_ARGS g++-mingw-w64-x86-64 && rm -rf /var/lib/apt/lists/*
4444
RUN apt-get update && apt-get install $APT_ARGS wine-stable wine32 wine64 bc nsis xorriso libncurses5 && rm -rf /var/lib/apt/lists/*
4545
RUN apt-get update && apt-get install $APT_ARGS python3-zmq && rm -rf /var/lib/apt/lists/*
46-
RUN apt-get update && apt-get install $APT_ARGS gawk jq && rm -rf /var/lib/apt/lists/*
46+
RUN apt-get update && apt-get install $APT_ARGS gawk jq parallel && rm -rf /var/lib/apt/lists/*
4747
RUN apt-get update && apt-get install $APT_ARGS imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools && rm -rf /var/lib/apt/lists/*
4848

4949
ARG CPPCHECK_VERSION=2.4

contrib/devtools/circular-dependencies.py

Lines changed: 78 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -28,61 +28,83 @@ def module_name(path):
2828
return path[:-4]
2929
return None
3030

31-
files = dict()
32-
deps = dict()
33-
34-
RE = re.compile("^#include <(.*)>")
35-
36-
# Iterate over files, and create list of modules
37-
for arg in sys.argv[1:]:
38-
module = module_name(arg)
39-
if module is None:
40-
print("Ignoring file %s (does not constitute module)\n" % arg)
41-
else:
42-
files[arg] = module
43-
deps[module] = set()
44-
45-
# Iterate again, and build list of direct dependencies for each module
46-
# TODO: implement support for multiple include directories
47-
for arg in sorted(files.keys()):
48-
module = files[arg]
49-
with open(arg, 'r', encoding="utf8") as f:
50-
for line in f:
51-
match = RE.match(line)
52-
if match:
53-
include = match.group(1)
54-
included_module = module_name(include)
55-
if included_module is not None and included_module in deps and included_module != module:
56-
deps[module].add(included_module)
57-
58-
# Loop to find the shortest (remaining) circular dependency
59-
have_cycle = False
60-
while True:
61-
shortest_cycle = None
62-
for module in sorted(deps.keys()):
63-
# Build the transitive closure of dependencies of module
64-
closure = dict()
65-
for dep in deps[module]:
66-
closure[dep] = []
31+
if __name__=="__main__":
32+
files = dict()
33+
deps = dict()
34+
35+
RE = re.compile("^#include <(.*)>")
36+
37+
def handle_module(module):
38+
module = module_name(arg)
39+
if module is None:
40+
print("Ignoring file %s (does not constitute module)\n" % arg)
41+
else:
42+
files[arg] = module
43+
deps[module] = set()
44+
45+
46+
# Iterate over files, and create list of modules
47+
for arg in sys.argv[1:]:
48+
handle_module(arg)
49+
50+
def build_list_direct(arg):
51+
module = files[arg]
52+
with open(arg, 'r', encoding="utf8") as f:
53+
for line in f:
54+
match = RE.match(line)
55+
if match:
56+
include = match.group(1)
57+
included_module = module_name(include)
58+
if included_module is not None and included_module in deps and included_module != module:
59+
deps[module].add(included_module)
60+
61+
62+
# Iterate again, and build list of direct dependencies for each module
63+
# TODO: implement support for multiple include directories
64+
for arg in sorted(files.keys()):
65+
build_list_direct(arg)
66+
# Loop to find the shortest (remaining) circular dependency
67+
68+
def shortest_c_dep():
69+
have_cycle = False
70+
71+
def handle_module(module, shortest_cycle):
72+
73+
# Build the transitive closure of dependencies of module
74+
closure = dict()
75+
for dep in deps[module]:
76+
closure[dep] = []
77+
while True:
78+
old_size = len(closure)
79+
old_closure_keys = sorted(closure.keys())
80+
for src in old_closure_keys:
81+
for dep in deps[src]:
82+
if dep not in closure:
83+
closure[dep] = closure[src] + [src]
84+
if len(closure) == old_size:
85+
break
86+
# If module is in its own transitive closure, it's a circular dependency; check if it is the shortest
87+
if module in closure and (shortest_cycle is None or len(closure[module]) + 1 < len(shortest_cycle)):
88+
shortest_cycle = [module] + closure[module]
89+
90+
return shortest_cycle
91+
6792
while True:
68-
old_size = len(closure)
69-
old_closure_keys = sorted(closure.keys())
70-
for src in old_closure_keys:
71-
for dep in deps[src]:
72-
if dep not in closure:
73-
closure[dep] = closure[src] + [src]
74-
if len(closure) == old_size:
93+
94+
shortest_cycles = None
95+
for module in sorted(deps.keys()):
96+
shortest_cycles = handle_module(module, shortest_cycles)
97+
98+
if shortest_cycles is None:
7599
break
76-
# If module is in its own transitive closure, it's a circular dependency; check if it is the shortest
77-
if module in closure and (shortest_cycle is None or len(closure[module]) + 1 < len(shortest_cycle)):
78-
shortest_cycle = [module] + closure[module]
79-
if shortest_cycle is None:
80-
break
81-
# We have the shortest circular dependency; report it
82-
module = shortest_cycle[0]
83-
print("Circular dependency: %s" % (" -> ".join(shortest_cycle + [module])))
84-
# And then break the dependency to avoid repeating in other cycles
85-
deps[shortest_cycle[-1]] = deps[shortest_cycle[-1]] - set([module])
86-
have_cycle = True
87-
88-
sys.exit(1 if have_cycle else 0)
100+
# We have the shortest circular dependency; report it
101+
module = shortest_cycles[0]
102+
print("Circular dependency: %s" % (" -> ".join(shortest_cycles + [module])))
103+
# And then break the dependency to avoid repeating in other cycles
104+
deps[shortest_cycles[-1]] = deps[shortest_cycles[-1]] - set([module])
105+
have_cycle = True
106+
107+
if have_cycle:
108+
return True
109+
110+
sys.exit(1 if shortest_c_dep() else 0)

test/lint/lint-all.sh

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,29 @@ LINTALL=$(basename "${BASH_SOURCE[0]}")
1818

1919
EXIT_CODE=0
2020

21-
for f in "${SCRIPTDIR}"/lint-*.sh; do
22-
if [ "$(basename "$f")" != "$LINTALL" ]; then
23-
if ! "$f"; then
24-
echo "^---- failure generated from $f"
25-
EXIT_CODE=1
21+
if ! command -v parallel > /dev/null; then
22+
for f in "${SCRIPTDIR}"/lint-*.sh; do
23+
if [ "$(basename "$f")" != "$LINTALL" ]; then
24+
if ! "$f"; then
25+
echo "^---- failure generated from $f"
26+
EXIT_CODE=1
27+
fi
2628
fi
29+
done
30+
else
31+
SCRIPTS=()
32+
33+
for f in "${SCRIPTDIR}"/lint-*.sh; do
34+
if [ "$(basename "$f")" != "$LINTALL" ]; then
35+
SCRIPTS+=("$f")
36+
fi
37+
done
38+
39+
if ! parallel --jobs 100% --will-cite --joblog parallel_out.log bash ::: "${SCRIPTS[@]}"; then
40+
echo "^---- failure generated"
41+
EXIT_CODE=1
2742
fi
28-
done
43+
column -t parallel_out.log && rm parallel_out.log
44+
fi
2945

3046
exit ${EXIT_CODE}

0 commit comments

Comments
 (0)