add_columns.py

import pandas as pd
import json
from tqdm import tqdm

def extract_additional_data(json_string):
    try:
        data = json.loads(json_string)
        person = data.get('person', {})
        organization = person.get('organization', {})
        employment_history = person.get('employment_history', [])

        past_employment_titles = [emp.get('title') for emp in employment_history]
        past_employment_descriptions = [emp.get('description') for emp in employment_history]

        return {
            'summary': person.get('summary'),
            'skills': ';'.join(person.get('skills', [])),
            'organization_industry': organization.get('industry'),
            'past_employment_titles': ';'.join(filter(None, past_employment_titles)),
            'past_employment_descriptions': ';'.join(filter(None, past_employment_descriptions)),
        }
    except (json.JSONDecodeError, AttributeError):
        return {
            'summary': None,
            'skills': None,
            'organization_industry': None,
            'past_employment_titles': None,
            'past_employment_descriptions': None,
        }

tqdm.pandas()

df = pd.read_csv('apollo_people_data.csv')

additional_data = df['full_json_response'].progress_apply(extract_additional_data)

new_df = pd.concat([df, pd.json_normalize(additional_data)], axis=1)

new_df.to_csv('apollo_people_data_enriched.csv', index=False)

print("Successfully created apollo_people_data_enriched.csv with additional columns.")