Merge pull request LMFDB#7103 from rvisser7/random_seed

Set random seed before generating the code snippet log files

Author: roed314

lmfdb/tests/generate_snippet_tests.py

@@ -39,6 +39,16 @@
 comment_dict = {'magma': '//', 'sage': '#', 'sage_gap': '#',
                          'gp': '\\\\', 'pari': '\\\\', 'oscar': '#', 'gap': '#'}
 
+# To ensure output is fully deterministic and the log files don't change between runs,
+# we set the random seed to 1 before each test run
+SEED = 1
+seed_dict = {'sage': f'set_random_seed({SEED}); gap.set_seed({SEED}); libgap.set_seed({SEED})',
+             'sage_gap': f'set_random_seed({SEED}); gap.set_seed({SEED}); libgap.set_seed({SEED})',
+             'magma': f'SetSeed({SEED});',
+             'oscar': f'import Random; Random.seed!({SEED}); Oscar.set_seed!({SEED}); Oscar.randseed!({SEED});',
+             'gap': f'Reset(GlobalMersenneTwister, {SEED}); Reset(GlobalRandomSource, {SEED});',
+             'gp': f'setrand({SEED});',
+             }
 
 def _setup_test_dir(yaml_file_path=None):
     """ Return dictionary with pair(s) 'yaml-file-path': 'test-file-path'.
@@ -55,7 +65,8 @@ def _setup_test_dir(yaml_file_path=None):
         raise Exception("Please run in same directory as test.sh")
 
     if yaml_file_path is None:
-        code_paths = lmfdb_dir.rglob("code*.yaml")
+        # Ensure snippet files are evaluated in the same order every time (for deterministic output)
+        code_paths = sorted(lmfdb_dir.rglob("code*.yaml"))
     else:
         code_paths = [Path(yaml_file_path)]
         assert code_paths[0].exists(), f"Specified path {yaml_file_path} does not exist"
@@ -150,6 +161,16 @@ def _eval_code_file(data, lang, proc, logfile):
     """
     cmt = comment_dict[lang]
     lines = [l for l in data.splitlines() if l != '' and cmt not in l[:len(cmt)+1]]
+
+    # Reset the random state before every snippet file (to ensure log files are generated deterministically)
+    seed_cmd = seed_dict.get(lang)
+    if seed_cmd is not None:
+        try:
+            proc.run_command(seed_cmd, timeout=60)
+        except Exception as exc:
+            # Raise error if unable to set the random seed
+            raise RuntimeError(f"Error: could not reset random state in {lang} with {seed_cmd!r}") from exc
+
     with logfile.open('w') as f:
         proc.child.logfile = f
         proc.run_command(cmt + " snippet evaluation file generated by generate_snippet_tests.py")
@@ -160,6 +181,8 @@ def _eval_code_file(data, lang, proc, logfile):
                 print("Timeout while running line:")
                 print(line)
 
+    proc.child.logfile = None
+
     # Matches ANSI escape sequences (colour codes etc.), see e.g. https://en.wikipedia.org/wiki/ANSI_escape_code
     # E.g. this sometimes occurs in the Gap snippet log files
     ANSI_ESCAPE_RE = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')

lmfdb/tests/snippet_tests/ecnf/code-81.1-CMa1-oscar.log

@@ -12,7 +12,8 @@ with equation
 julia> conductor(E)
 Ideal of maximal order of number field of degree 2 over QQ
   of norm 81
-with 9-normal generators [81, 9*_a - 9]
+  of minimum 9
+with 3-normal generators [9, 9*_a - 9]
 
 julia> norm(conductor(E))
 81

lmfdb/tests/snippet_tests/elliptic_curves/code-37.a1-magma.log

@@ -9,13 +9,13 @@ Mapping from: Abelian Group isomorphic to Z
 Defined on 1 generator (free) to Set of points of E with coordinates in Rational Field given by a rule [no inverse]
 true true
 magma> Generators(E);
-[ (0 : -1 : 1) ]
+[ (0 : 0 : 1) ]
 true true
 magma> TorsionSubgroup(E);
 Abelian Group of order 1
 magma> IntegralPoints(E);
-[ (-1 : 0 : 1), (0 : -1 : 1), (1 : -1 : 1), (2 : 2 : 1), (6 : -15 : 1) ]
-[ <(-1 : 0 : 1), 1>, <(0 : -1 : 1), 1>, <(1 : -1 : 1), 1>, <(2 : 2 : 1), 1>, <(6 : -15 : 1), 1> ]
+[ (-1 : -1 : 1), (0 : 0 : 1), (1 : 0 : 1), (2 : -3 : 1), (6 : 14 : 1) ]
+[ <(-1 : -1 : 1), 1>, <(0 : 0 : 1), 1>, <(1 : 0 : 1), 1>, <(2 : -3 : 1), 1>, <(6 : 14 : 1), 1> ]
 magma> Conductor(E);
 37
 magma> Discriminant(E);
@@ -42,7 +42,7 @@ magma> Order(TorsionSubgroup(E));
 1
 magma> MordellWeilShaInformation(E);
 [ 1, 1 ]
-[ (0 : -1 : 1) ]
+[ (0 : 0 : 1) ]
 [
 <2, [ 0, 0 ]>
 ]

lmfdb/tests/snippet_tests/groups/abstract/code-A5-magma.log

@@ -8,14 +8,14 @@ magma> AutomorphismGroup(G);
 A group of automorphisms of GrpPerm: G, Degree 5, Order 2^2 * 3 * 5
 Generators:
 Automorphism of GrpPerm: G, Degree 5, Order 2^2 * 3 * 5 which maps:
-(3, 4, 5) |--> (1, 2, 4)
-(1, 2, 3) |--> (2, 5, 3)
-Automorphism of GrpPerm: G, Degree 5, Order 2^2 * 3 * 5 which maps:
-(3, 4, 5) |--> (2, 3, 5)
-(1, 2, 3) |--> (1, 4, 3)
+(3, 4, 5) |--> (3, 5, 4)
+(1, 2, 3) |--> (1, 4, 2)
 Automorphism of GrpPerm: G, Degree 5, Order 2^2 * 3 * 5 which maps:
 (3, 4, 5) |--> (2, 4, 3)
-(1, 2, 3) |--> (1, 3, 5)
+(1, 2, 3) |--> (1, 5, 2)
+Automorphism of GrpPerm: G, Degree 5, Order 2^2 * 3 * 5 which maps:
+(3, 4, 5) |--> (2, 4, 5)
+(1, 2, 3) |--> (1, 3, 2)
 magma> CompositionFactors(G);
 G
 |  Alternating(5)
@@ -310,20 +310,20 @@ Id($)
 [2]     Order 2            Length 15
 Permutation group acting on a set of cardinality 5
 Order = 2
-(1, 2)(4, 5)
+(1, 2)(3, 5)
 [3]     Order 3            Length 10
 Permutation group acting on a set of cardinality 5
 Order = 3
-(2, 5, 4)
+(2, 5, 3)
 [4]     Order 4            Length 5
 Permutation group acting on a set of cardinality 5
 Order = 4 = 2^2
-(1, 2)(4, 5)
-(1, 4)(2, 5)
+(1, 2)(3, 5)
+(1, 3)(2, 5)
 [5]     Order 5            Length 6
 Permutation group acting on a set of cardinality 5
 Order = 5
-(1, 4, 3, 2, 5)
+(1, 3, 4, 2, 5)
 [6]     Order 6            Length 10
 Permutation group acting on a set of cardinality 5
 Order = 6 = 2 * 3
@@ -332,19 +332,19 @@ Order = 6 = 2 * 3
 [7]     Order 10           Length 6
 Permutation group acting on a set of cardinality 5
 Order = 10 = 2 * 5
-(2, 3)(4, 5)
-(1, 4, 3, 2, 5)
+(2, 4)(3, 5)
+(1, 3, 4, 2, 5)
 [8]     Order 12           Length 5
 Permutation group acting on a set of cardinality 5
 Order = 12 = 2^2 * 3
-(2, 5, 4)
-(1, 2)(4, 5)
-(1, 4)(2, 5)
+(2, 5, 3)
+(1, 2)(3, 5)
+(1, 3)(2, 5)
 [9]     Order 60           Length 1
 Permutation group acting on a set of cardinality 5
 Order = 60 = 2^2 * 3 * 5
 (1, 2)(3, 4)
-(1, 3, 5)
+(1, 4, 5)
 magma> Center(G);
 Permutation group acting on a set of cardinality 5
 Order = 1

lmfdb/tests/snippet_tests/groups/abstract/code-D4-magma.log

@@ -145,27 +145,27 @@ Conjugacy classes of subgroups
 [1]     Order 1            Length 1
 Permutation group acting on a set of cardinality 4
 Order = 1
-[2]     Order 2            Length 1
+[2]     Order 2            Length 2
 Permutation group acting on a set of cardinality 4
 Order = 2
-(1, 3)(2, 4)
-[3]     Order 2            Length 2
+(2, 4)
+[3]     Order 2            Length 1
 Permutation group acting on a set of cardinality 4
 Order = 2
-(2, 4)
+(1, 3)(2, 4)
 [4]     Order 2            Length 2
 Permutation group acting on a set of cardinality 4
 Order = 2
 (1, 2)(3, 4)
 [5]     Order 4            Length 1
 Permutation group acting on a set of cardinality 4
 Order = 4 = 2^2
-(1, 2)(3, 4)
+(2, 4)
 (1, 3)(2, 4)
 [6]     Order 4            Length 1
 Permutation group acting on a set of cardinality 4
 Order = 4 = 2^2
-(2, 4)
+(1, 2)(3, 4)
 (1, 3)(2, 4)
 [7]     Order 4            Length 1
 Permutation group acting on a set of cardinality 4