Merge pull request #555 from petrelharp/canonicalize_json

reordered metadata schema and added tests for ordering; closes #543

Author: bhaller

core/slim_globals.cpp

@@ -1734,7 +1734,7 @@ R"V0G0N({"$schema":"http://json-schema.org/schema#","additionalProperties":false
 // (The string replaced is "%d" *including* the quotes, because the format string needs to
 // itself be a legal JSON string in order to pass SLiM's own internal checks, so beware.)
 const std::string gSLiM_tsk_node_metadata_schema_FORMAT =
-R"V0G0N({"$schema":"http://json-schema.org/schema#","additionalProperties":false,"codec":"struct","description":"SLiM schema for node metadata.","examples":[{"slim_id":123,"is_vacant":0}],"properties":{"slim_id":{"binaryFormat":"q","description":"The 'pedigree ID' of the haplosomes associated with this node in SLiM.","index":0,"type":"integer"},"is_vacant":{"description":"A vector of byte (uint8_t) values, with each bit representing whether the node represents a vacant position, either unused or a null haplosome (1), or a non-null haplosome (0), in the corresponding chromosome. This field encodes vacancy for all of the chromosomes in the model, not just the chromosome represented in this file (so that the node table is identical across all chromosomes for a multi-chromosome model). Each chromosome receives one bit here; there are two node table entries per individual, used for the two haplosomes of every chromosome, so only one bit is needed in each entry (making two bits total per chromosome, across the two node table entries). The least significant bit of the first byte is used first (for one haplosome of the first chromosome); the most significant bit of the last byte is used last. The number of bytes present in this field is indicated by this schema's 'binaryFormat' field, which is variable (!), and can also be deduced from the number of chromosomes in the model as given in the top-level 'chromosomes' metadata key, which should always be present if this metadata is present.","index":1,"type":"array","length":"%d","items":{"type":"number","binaryFormat":"B"}}},"required":["slim_id","is_vacant"],"type":["object","null"]})V0G0N";
+R"V0G0N({"$schema":"http://json-schema.org/schema#","additionalProperties":false,"codec":"struct","description":"SLiM schema for node metadata.","examples":[{"is_vacant":0,"slim_id":123}],"properties":{"is_vacant":{"description":"A vector of byte (uint8_t) values, with each bit representing whether the node represents a vacant position, either unused or a null haplosome (1), or a non-null haplosome (0), in the corresponding chromosome. This field encodes vacancy for all of the chromosomes in the model, not just the chromosome represented in this file (so that the node table is identical across all chromosomes for a multi-chromosome model). Each chromosome receives one bit here; there are two node table entries per individual, used for the two haplosomes of every chromosome, so only one bit is needed in each entry (making two bits total per chromosome, across the two node table entries). The least significant bit of the first byte is used first (for one haplosome of the first chromosome); the most significant bit of the last byte is used last. The number of bytes present in this field is indicated by this schema's 'binaryFormat' field, which is variable (!), and can also be deduced from the number of chromosomes in the model as given in the top-level 'chromosomes' metadata key, which should always be present if this metadata is present.","index":1,"items":{"binaryFormat":"B","type":"number"},"length":"%d","type":"array"},"slim_id":{"binaryFormat":"q","description":"The 'pedigree ID' of the haplosomes associated with this node in SLiM.","index":0,"type":"integer"}},"required":["slim_id","is_vacant"],"type":["object","null"]})V0G0N";
 
 const std::string gSLiM_tsk_individual_metadata_schema =
 R"V0G0N({"$schema":"http://json-schema.org/schema#","additionalProperties":false,"codec":"struct","description":"SLiM schema for individual metadata.","examples":[{"age":-1,"flags":0,"pedigree_id":123,"pedigree_p1":12,"pedigree_p2":23,"sex":0,"subpopulation":0}],"flags":{"SLIM_INDIVIDUAL_METADATA_MIGRATED":{"description":"Whether this individual was a migrant, either in the tick when the tree sequence was written out (if the individual was alive then), or in the tick of the last time they were Remembered (if not).","value":1}},"properties":{"age":{"binaryFormat":"i","description":"The age of this individual, either when the tree sequence was written out (if the individual was alive then), or the last time they were Remembered (if not).","index":4,"type":"integer"},"flags":{"binaryFormat":"I","description":"Other information about the individual: see 'flags'.","index":7,"type":"integer"},"pedigree_id":{"binaryFormat":"q","description":"The 'pedigree ID' of this individual in SLiM.","index":1,"type":"integer"},"pedigree_p1":{"binaryFormat":"q","description":"The 'pedigree ID' of this individual's first parent in SLiM.","index":2,"type":"integer"},"pedigree_p2":{"binaryFormat":"q","description":"The 'pedigree ID' of this individual's second parent in SLiM.","index":3,"type":"integer"},"sex":{"binaryFormat":"i","description":"The sex of the individual (0 for female, 1 for male, -1 for hermaphrodite).","index":6,"type":"integer"},"subpopulation":{"binaryFormat":"i","description":"The ID of the subpopulation the individual was part of, either when the tree sequence was written out (if the individual was alive then), or the last time they were Remembered (if not).","index":5,"type":"integer"}},"required":["pedigree_id","pedigree_p1","pedigree_p2","age","subpopulation","sex","flags"],"type":"object"})V0G0N";
@@ -1749,7 +1749,7 @@ R"V0G0N({"$schema":"http://json-schema.org/schema#","additionalProperties":false
 // schema in SLiM since we don't check/validate schemas anyway in SLiM; we will just write out this new schema in
 // SLiM 4, and on read we won't care whether the schema is the 3.7 or the 4.0 schema.
 const std::string gSLiM_tsk_population_metadata_schema = 
-R"V0G0N({"$schema":"http://json-schema.org/schema#","additionalProperties":true,"codec":"json","description":"SLiM schema for population metadata.","examples":[{"bounds_x0":0.0,"bounds_x1":100.0,"bounds_y0":0.0,"bounds_y1":100.0,"female_cloning_fraction":0.25,"male_cloning_fraction":0.0,"migration_records":[{"migration_rate":0.9,"source_subpop":1},{"migration_rate":0.1,"source_subpop":2}],"selfing_fraction":0.5,"sex_ratio":0.5,"slim_id":2,"name":"p2"}],"properties":{"bounds_x0":{"description":"The minimum x-coordinate in this subpopulation.","type":"number"},"bounds_x1":{"description":"The maximum x-coordinate in this subpopulation.","type":"number"},"bounds_y0":{"description":"The minimum y-coordinate in this subpopulation.","type":"number"},"bounds_y1":{"description":"The maximum y-coordinate in this subpopulation.","type":"number"},"bounds_z0":{"description":"The minimum z-coordinate in this subpopulation.","type":"number"},"bounds_z1":{"description":"The maximum z-coordinate in this subpopulation.","type":"number"},"description":{"description":"A description of this subpopulation.","type":"string"},"female_cloning_fraction":{"description":"The frequency with which females in this subpopulation reproduce clonally (for WF models).","type":"number"},"male_cloning_fraction":{"description":"The frequency with which males in this subpopulation reproduce clonally (for WF models).","type":"number"},"migration_records":{"items":{"properties":{"migration_rate":{"description":"The fraction of children in this subpopulation that are composed of 'migrants' from the source subpopulation (in WF models).","type":"number"},"source_subpop":{"description":"The ID of the subpopulation migrants come from (in WF models).","type":"integer"}},"required":["source_subpop","migration_rate"],"type":"object"},"type":"array"},"name":{"description":"A human-readable name for this subpopulation.","type":"string"},"selfing_fraction":{"description":"The frequency with which individuals in this subpopulation self (for WF models).","type":"number"},"sex_ratio":{"description":"This subpopulation's sex ratio (for WF models).","type":"number"},"slim_id":{"description":"The ID of this population in SLiM. Note that this is called a 'subpopulation' in SLiM.","type":"integer"}},"required":[],"type":["object","null"]})V0G0N";
+R"V0G0N({"$schema":"http://json-schema.org/schema#","additionalProperties":true,"codec":"json","description":"SLiM schema for population metadata.","examples":[{"bounds_x0":0.0,"bounds_x1":100.0,"bounds_y0":0.0,"bounds_y1":100.0,"female_cloning_fraction":0.25,"male_cloning_fraction":0.0,"migration_records":[{"migration_rate":0.9,"source_subpop":1},{"migration_rate":0.1,"source_subpop":2}],"name":"p2","selfing_fraction":0.5,"sex_ratio":0.5,"slim_id":2}],"properties":{"bounds_x0":{"description":"The minimum x-coordinate in this subpopulation.","type":"number"},"bounds_x1":{"description":"The maximum x-coordinate in this subpopulation.","type":"number"},"bounds_y0":{"description":"The minimum y-coordinate in this subpopulation.","type":"number"},"bounds_y1":{"description":"The maximum y-coordinate in this subpopulation.","type":"number"},"bounds_z0":{"description":"The minimum z-coordinate in this subpopulation.","type":"number"},"bounds_z1":{"description":"The maximum z-coordinate in this subpopulation.","type":"number"},"description":{"description":"A description of this subpopulation.","type":"string"},"female_cloning_fraction":{"description":"The frequency with which females in this subpopulation reproduce clonally (for WF models).","type":"number"},"male_cloning_fraction":{"description":"The frequency with which males in this subpopulation reproduce clonally (for WF models).","type":"number"},"migration_records":{"items":{"properties":{"migration_rate":{"description":"The fraction of children in this subpopulation that are composed of 'migrants' from the source subpopulation (in WF models).","type":"number"},"source_subpop":{"description":"The ID of the subpopulation migrants come from (in WF models).","type":"integer"}},"required":["source_subpop","migration_rate"],"type":"object"},"type":"array"},"name":{"description":"A human-readable name for this subpopulation.","type":"string"},"selfing_fraction":{"description":"The frequency with which individuals in this subpopulation self (for WF models).","type":"number"},"sex_ratio":{"description":"This subpopulation's sex ratio (for WF models).","type":"number"},"slim_id":{"description":"The ID of this population in SLiM. Note that this is called a 'subpopulation' in SLiM.","type":"integer"}},"required":[],"type":["object","null"]})V0G0N";
 
 
 // *************************************

treerec/tests/test_metadata_schemas.py

@@ -0,0 +1,108 @@
+"""
+Tests for metadata schema stuff.
+Individual recipes can be specified by decorating with
+@pytest.mark.parametrize('recipe', ['test_____my_specific_recipe.slim'], indirect=True)
+"""
+import json
+import tskit
+import msprime
+import pyslim
+import pytest
+
+from recipe_specs import recipe_eq
+
+table_names = [
+    'edges', 'individuals', 'migrations', 'mutations',
+    'nodes', 'populations', 'sites'
+]
+
+# this is for indexing pyslim.slim_metadata_schemas
+table_namex = [
+    'edge', 'individual', 'mutation',
+    'node', 'population', 'site'
+]
+
+class TestMetadataSchemas:
+
+    def round_trip_schema(self, table):
+        a = table.metadata_schema
+        new_table = table.copy()
+        new_table.assert_equals(table)
+        b = new_table.metadata_schema
+        assert a == b
+        assert a.schema == b.schema
+        new_table.clear()
+        new_table.set_columns(**table.asdict())
+        new_table.assert_equals(table)
+        b = new_table.metadata_schema
+        assert a == b
+        assert a.schema == b.schema
+
+    def copy_schema(self, table):
+        new_table = table.__class__()
+        new_table.schema = table.schema
+        assert new_table.schema == table.schema
+        assert new_table.schema.schema == table.schema.schema
+
+    @pytest.mark.parametrize('recipe, table_name', indirect=['recipe'], argvalues=[
+        ('test_____pop_names_pX.slim', x) for x in table_names
+    ])
+    def test_slim_canonical_json(self, recipe, table_name):
+        # Here we want to test that whatever tskit does to the schema
+        # when copying tables doesn't change the underlying .schema;
+        # I can't figure out a good way to actually do that besides
+        # making a copy of the tables.
+        assert len(recipe["results"]) == 1
+        ts = recipe["results"][0].get_normal_ts()
+        t = ts.tables.table_name_map[table_name]
+        tc = t.copy()
+        # this should be true since it's been canonicalized
+        # to produce the repr() which gets compared:
+        assert t.metadata_schema == tc.metadata_schema
+        if t.metadata_schema.schema != tc.metadata_schema.schema:
+            # print out JSON that should give us a correct schema
+            # so we can put it in the SLiM code
+            j = tskit.canonical_json(tc.metadata_schema.schema)
+            # this should succeed because json.loads returns an unordered
+            # dictionary, and the two differ only on ordering
+            assert json.loads(j) == t.metadata_schema.schema
+            # put back in the placeholder
+            if table_name == "nodes":
+                u = '"length":1'
+                ix = j.find(u)
+                j = j[:ix] + '"length":"%d"' + j[(ix+len(u)):]
+            print(f"Schema for {table_name} does not match. "
+                  "Use this (equivalent) JSON.")
+            print(".................... begin")
+            print(j)
+            print("...................... end")
+        assert t.metadata_schema.schema == tc.metadata_schema.schema
+
+    @pytest.mark.parametrize('recipe, table_name', indirect=['recipe'], argvalues=[
+        ('test_____pop_names_pX.slim', x) for x in table_names
+    ])
+    def test_slim_round_trip(self, recipe, table_name):
+        assert len(recipe["results"]) == 1
+        ts = recipe["results"][0].get_normal_ts()
+        self.round_trip_schema(ts.tables.table_name_map[table_name])
+
+    @pytest.mark.parametrize('table_name', argvalues=table_names)
+    def test_annotate_round_trip(self, table_name):
+        ts = msprime.sim_ancestry(1, sequence_length=10, random_seed=123)
+        ts = msprime.sim_mutations(ts, rate=1, random_seed=123)
+        assert ts.num_mutations > 0
+        t = ts.dump_tables()
+        pyslim.annotate_tables(t, model_type='WF', tick=1)
+        self.round_trip_schema(t.table_name_map[table_name])
+
+    @pytest.mark.parametrize('table_name', argvalues=table_namex)
+    def test_json_round_trip(self, table_name):
+        # pyslim schema -> json -> schema
+        orig = pyslim.slim_metadata_schemas[table_name]
+        j = tskit.canonical_json(orig.schema)
+        new = tskit.MetadataSchema(json.loads(j))
+        assert new == orig
+        assert new.schema == orig.schema
+
+
+