cleaning up data

Author: nithinmuthukumar

data/data_normalized/bird_probabilities.json

@@ -0,0 +1,24 @@
+import json
+import pprint
+from collections import defaultdict
+
+
+def get_field_types(data: dict):
+    fields = defaultdict(set)
+    for bird in data:
+        for field in bird:
+            val = bird[field]
+            if isinstance(val, dict):
+                for key in val:
+                    fields[key].add(type(val[key]))
+
+            else:
+                if field == "Wetland" and isinstance(val, int):
+                    print(bird)
+                fields[field].add(type(val))
+    pprint.pprint(fields)
+
+
+if __name__ == "__main__":
+    with open("data/master.json", "r") as file:
+        get_field_types(json.load(file))

data/data_normalized/master.json

@@ -18,21 +18,21 @@
 
 def to_snake_case(s: str) -> str:
     """Convert string to snake_case.
-    
+
     Handles special cases:
     - "/ (food cost)" → "slash_food_cost"
     - "* (food cost)" → "star_food_cost"
     - Spaces and special chars → underscores
     """
-    
+
     # Replace special characters with underscores
-    s = re.sub(r'[^\w\s-]', '', s)  # Remove special chars except spaces and hyphens
+    s = re.sub(r"[^\w\s-]", "", s)  # Remove special chars except spaces and hyphens
     # Convert spaces and hyphens to underscores
-    s = re.sub(r'[\s-]+', '_', s)
+    s = re.sub(r"[\s-]+", "_", s)
     # Convert to lowercase
     s = s.lower()
     # Remove leading/trailing underscores
-    s = s.strip('_')
+    s = s.strip("_")
     return s
 
 
@@ -44,64 +44,114 @@ def normalize_boolean_value(value: Any) -> bool:
         return True
     if value == "X":
         return True
+    # BLUE DUCK??
+    if value == 1:
+        return True
     if value is None:
         return False
     return bool(value)
 
 
+def normalize_float_value(value) -> float:
+    if value is None:
+        return 0.0
+    return value
+
+
 def normalize_master_json(data: List[Dict]) -> List[Dict]:
     """Normalize bird card data from master.json."""
     normalized = []
-    
+
     # Define boolean fields (fields that can have "X" values)
     boolean_fields = {
-        "Forest", "Grassland", "Wetland",
+        "Forest",
+        "Grassland",
+        "Wetland",
         "Bonus card",
-        "Predator", "Flocking",
-        "North America", "Central America", "South America",
-        "Europe", "Asia", "Africa", "Oceania",
+        "Predator",
+        "Flocking",
+        "North America",
+        "Central America",
+        "South America",
+        "Europe",
+        "Asia",
+        "Africa",
+        "Oceania",
         "Fan Art Pack?",
-        "Anatomist", "Cartographer", "Historian", "Photographer",
-        "Backyard Birder", "Bird Bander", "Bird Counter", "Bird Feeder",
-        "Diet Specialist", "Enclosure Builder", "Endangered Species Protector",
-        "Falconer", "Fishery Manager", "Food Web Expert", "Forester",
-        "Large Bird Specialist", "Nest Box Builder", "Omnivore Expert",
-        "Passerine Specialist", "Platform Builder", "Prairie Manager",
-        "Rodentologist", "Small Clutch Specialist", "Viticulturalist",
-        "Wetland Scientist", "Wildlife Gardener","/ (food cost)", "* (food cost)"
+        "Anatomist",
+        "Cartographer",
+        "Historian",
+        "Photographer",
+        "Backyard Birder",
+        "Bird Bander",
+        "Bird Counter",
+        "Bird Feeder",
+        "Diet Specialist",
+        "Enclosure Builder",
+        "Endangered Species Protector",
+        "Falconer",
+        "Fishery Manager",
+        "Food Web Expert",
+        "Forester",
+        "Large Bird Specialist",
+        "Nest Box Builder",
+        "Omnivore Expert",
+        "Passerine Specialist",
+        "Platform Builder",
+        "Prairie Manager",
+        "Rodentologist",
+        "Small Clutch Specialist",
+        "Viticulturalist",
+        "Wetland Scientist",
+        "Wildlife Gardener",
+        "/ (food cost)",
+        "* (food cost)",
+        "Swift Start",
+        "Automa ban",
     }
-    
+    float_fields = {
+        "Invertebrate",
+        "Seed",
+        "Fish",
+        "Fruit",
+        "Rodent",
+        "Nectar",
+        "Wild (food)",
+        "Total food cost",
+    }
+
     for item in data:
         normalized_item = {}
-        
+
         for key, value in item.items():
             new_key = to_snake_case(key)
-            
-            
+
             # Handle boolean fields
             if key in boolean_fields:
                 normalized_item[new_key] = normalize_boolean_value(value)
-            elif new_key =="wingspan":
-                value = str(value)
+            elif key in float_fields:
+                normalized_item[new_key] = normalize_float_value(value)
+            elif key == "Color" and value is None:
+                normalized_item[new_key] = "white"
             # Handle nested structures
             else:
                 normalized_item[new_key] = value
-        
+
         normalized.append(normalized_item)
-    
+
     return normalized
 
 
 def normalize_bonus_json(data: List[Dict]) -> List[Dict]:
     """Normalize bonus card data from bonus.json."""
     normalized = []
-    
+
     for item in data:
         normalized_item = {}
-        
+
         for key, value in item.items():
             new_key = to_snake_case(key)
-            
+
             # Handle special field name mappings
             if key == "Bonus card":
                 new_key = "bonus_card"
@@ -110,32 +160,32 @@ def normalize_bonus_json(data: List[Dict]) -> List[Dict]:
             elif key == "%":
                 new_key = "percentage"
                 # Convert "-" to null
-                if value == "-" or value== "variable":
+                if value == "-" or value == "variable":
                     value = None
 
             elif key == "VP Average":
                 new_key = "vp_average"
             elif key == "Explanatory text":
                 new_key = "explanatory_text"
-            
+
             # Handle boolean field
             if key == "Automa":
                 normalized_item[new_key] = normalize_boolean_value(value)
             else:
                 normalized_item[new_key] = value
-        
+
         normalized.append(normalized_item)
-    
+
     return normalized
 
 
 def normalize_goals_json(data: List[Dict]) -> List[Dict]:
     """Normalize goals data from goals.json."""
     normalized = []
-    
+
     for item in data:
         normalized_item = {}
-        
+
         for key, value in item.items():
             # Handle numeric score keys
 
@@ -151,15 +201,13 @@ def normalize_goals_json(data: List[Dict]) -> List[Dict]:
             else:
                 new_key = key  # Keep id as-is
             normalized_item[new_key] = value
-            
-        
+
         normalized.append(normalized_item)
-    
+
     return normalized
 
 
 def normalize_general_json(data: Dict) -> Dict:
-    
     return data
 
 
@@ -169,45 +217,44 @@ def main():
     script_dir = Path(__file__).parent
     data_dir = script_dir
     output_dir = script_dir / "data_normalized"
-    
+
     # Create output directory if it doesn't exist
     output_dir.mkdir(exist_ok=True)
-    
+
     # Define file processors
     processors = {
         "master.json": normalize_master_json,
         "bonus.json": normalize_bonus_json,
         "goals.json": normalize_goals_json,
         "general.json": normalize_general_json,
     }
-    
+
     # Process each file
     for filename, processor in processors.items():
         input_path = data_dir / filename
-        
+
         if not input_path.exists():
             print(f"Warning: {filename} not found, skipping...")
             continue
-        
+
         print(f"Processing {filename}...")
-        
+
         # Read input file
-        with open(input_path, 'r', encoding='utf-8') as f:
+        with open(input_path, "r", encoding="utf-8") as f:
             data = json.load(f)
-        
+
         # Normalize data
         normalized_data = processor(data)
-        
+
         # Write output file
         output_path = output_dir / filename
-        with open(output_path, 'w', encoding='utf-8') as f:
+        with open(output_path, "w", encoding="utf-8") as f:
             json.dump(normalized_data, f, indent=2, ensure_ascii=False)
-        
+
         print(f"  → Written to {output_path}")
-    
+
     print(f"\nNormalization complete! Output files written to {output_dir}")
 
 
 if __name__ == "__main__":
     main()
-