Skip to content

Commit 4cc9022

Browse files
committed
-frandom-seed: reduce value length
The current default `-frandom-seed="$*"` can run into "Argument list too long" errors (E2BIG). This happened in pytorch with 1300 object files on the command line. Fix that by filtering out flags `-*`, object files and libraries. Signed-off-by: Harmen Stoppels <harmenstoppels@gmail.com>
1 parent e47dd22 commit 4cc9022

2 files changed

Lines changed: 60 additions & 8 deletions

File tree

cc.sh

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -366,16 +366,14 @@ if [ -z "$mode" ] || [ "$mode" = ld ]; then
366366
done
367367
fi
368368

369-
# Finish setting up the mode, and check whether -frandom-seed needs to be set.
370-
eval "_set_frandom_seed=\${SPACK_${comp}_HAS_FRANDOM_SEED:-}"
369+
# Finish setting up the mode.
371370
if [ -z "$mode" ]; then
372371
mode=ccld
373372
for arg in "$@"; do
374373
case $arg in
375-
-E) mode=cpp ;;
376-
-S) mode=as ;;
377-
-c) mode=cc ;;
378-
-frandom-seed=*) _set_frandom_seed= ;;
374+
-E) mode=cpp; break ;;
375+
-S) mode=as; break ;;
376+
-c) mode=cc; break ;;
379377
esac
380378
done
381379
fi
@@ -465,6 +463,18 @@ fi
465463
# Save original command for debug logging
466464
input_command="$*"
467465

466+
eval "_frandom_seed_input=\${SPACK_${comp}_HAS_FRANDOM_SEED:-}"
467+
if [ -n "$_frandom_seed_input" ]; then
468+
_frandom_seed_input=""
469+
for arg in "$@"; do
470+
case "$arg" in
471+
-frandom-seed=*) _frandom_seed_input=; break ;;
472+
-*|*.o|*.so|*.dylib|*.a) ;;
473+
*) _frandom_seed_input="${_frandom_seed_input}${arg}" ;;
474+
esac
475+
done
476+
fi
477+
468478
#
469479
# Parse the command line arguments.
470480
#
@@ -953,11 +963,11 @@ extend args_list libs_list "-l"
953963
full_command_list="$command"
954964
extend full_command_list args_list
955965

956-
if [ -n "$_set_frandom_seed" ]; then
966+
if [ -n "$_frandom_seed_input" ]; then
957967
case "$mode" in
958968
cc|ccld)
959969
# Make GCC deterministic by setting the random seed to command line arguments
960-
append full_command_list "-frandom-seed=$input_command"
970+
append full_command_list "-frandom-seed=$_frandom_seed_input"
961971
;;
962972
esac
963973
fi

test/run.sh

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1011,6 +1011,46 @@ expected: $_expected_LR"
10111011
fi
10121012
}
10131013

1014+
# ---------------------------------------------------------------------------
1015+
# -frandom-seed
1016+
# ---------------------------------------------------------------------------
1017+
1018+
test_frandom_seed_not_added_without_env() {
1019+
wrapper_environment
1020+
_out=$(dump_args cc '-c
1021+
hello.c
1022+
-O2')
1023+
if printf '%s\n' "$_out" | grep -F -- '-frandom-seed=' >/dev/null; then
1024+
fail "frandom_seed_absent: -frandom-seed should not appear without SPACK_CC_HAS_FRANDOM_SEED"
1025+
fi
1026+
}
1027+
1028+
test_frandom_seed_filters_args() {
1029+
wrapper_environment
1030+
SPACK_CC_HAS_FRANDOM_SEED=1; export SPACK_CC_HAS_FRANDOM_SEED
1031+
1032+
# cc mode: -frandom-seed should contain only source files, concatenated
1033+
_out=$(dump_args cc '-c
1034+
-O2
1035+
-I/some/include
1036+
hello.c
1037+
world.c
1038+
foo.o
1039+
bar.a
1040+
baz.so
1041+
quux.dylib')
1042+
expect_contains frandom_seed_value "$_out" '-frandom-seed=hello.cworld.c'
1043+
1044+
# User-supplied -frandom-seed suppresses auto-generated one
1045+
_out=$(dump_args cc '-c
1046+
-frandom-seed=custom
1047+
hello.c')
1048+
if printf '%s\n' "$_out" | grep -cF -- '-frandom-seed=' | grep -qv '^1$'; then
1049+
fail "frandom_seed_user_override: expected exactly one -frandom-seed"
1050+
fi
1051+
expect_contains frandom_seed_user_passthrough "$_out" '-frandom-seed=custom'
1052+
}
1053+
10141054
# ---------------------------------------------------------------------------
10151055
# Runner
10161056
# ---------------------------------------------------------------------------
@@ -1029,6 +1069,8 @@ test_disable_new_dtags
10291069
test_filter_enable_new_dtags
10301070
test_linker_strips_loopopt
10311071
test_spack_managed_dirs_are_prioritized
1072+
test_frandom_seed_not_added_without_env
1073+
test_frandom_seed_filters_args
10321074
'
10331075

10341076
if [ $# -gt 0 ]; then

0 commit comments

Comments
 (0)