hugging_face.py

import json
from datasets import Dataset
from huggingface_hub import login
jsonl_file = "crime_training_data.jsonl"
# Login to Hugging Face (you'll need an access token from huggingface.co)
login(token="os.environ.get('HF_TOKEN')")

# Load the JSONL file into a Hugging Face dataset
def create_hf_dataset(jsonl_file):
    """Create a Hugging Face dataset from a JSONL file"""
    # Read the JSONL file
    with open(jsonl_file, 'r', encoding='utf-8') as f:
        data = [json.loads(line) for line in f]
    
    # Create dataset
    dataset = Dataset.from_list(data)
    
    # Push to hub
    dataset.push_to_hub(
        "your-org/crime-entity-dataset",  # Replace with your desired repo name
        private=False  # Set to False if you want a public dataset
    )
    
    return dataset

# Create and upload the dataset after processing
dataset = create_hf_dataset(jsonl_file)
print(f"Dataset uploaded with {len(dataset)} examples")