-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathmain.py
More file actions
166 lines (139 loc) · 9.89 KB
/
Copy pathmain.py
File metadata and controls
166 lines (139 loc) · 9.89 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
from fastapi import FastAPI, UploadFile, File, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
import pandas as pd
import io, json, os
import google.generativeai as genai
from analysis import run_full_analysis, normalize_target, compute_dir
app = FastAPI(title="FairHire India API")
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY", "")
if GEMINI_API_KEY:
genai.configure(api_key=GEMINI_API_KEY)
# Store last uploaded df in memory for what-if (simple single-user demo)
_last_df = {}
@app.get("/")
def root():
return {"status": "FairHire India API running", "version": "3.0", "gemini": bool(GEMINI_API_KEY)}
@app.post("/analyze")
async def analyze(file: UploadFile = File(...)):
if not file.filename.endswith(".csv"):
raise HTTPException(status_code=400, detail="Only CSV files supported.")
contents = await file.read()
try:
df = pd.read_csv(io.StringIO(contents.decode("utf-8")))
except Exception as e:
raise HTTPException(status_code=400, detail=f"Could not parse CSV: {str(e)}")
if df.shape[0] < 5:
raise HTTPException(status_code=400, detail="Dataset too small. Need at least 5 rows.")
result = run_full_analysis(df)
result["gemini_insights"] = generate_gemini_insights(result)
# Store for what-if
_last_df["df"] = df
_last_df["col_types"] = {c: t for c, t in result["columns_analyzed"]}
_last_df["target"] = result["target_column"]
return result
class WhatIfRequest(BaseModel):
columns_to_remove: list[str]
@app.post("/whatif")
async def whatif(req: WhatIfRequest):
if "df" not in _last_df:
raise HTTPException(status_code=400, detail="No dataset uploaded yet. Upload first.")
df = _last_df["df"].copy()
target = _last_df["target"]
# Remove requested columns
cols_removed = [c for c in req.columns_to_remove if c in df.columns]
df_clean = df.drop(columns=cols_removed)
result = run_full_analysis(df_clean)
return {
"new_bias_score": result["bias_score"],
"new_risk_level": result["risk_level"],
"new_risk_color": result["risk_color"],
"new_findings_count": result["findings_count"],
"new_summary": result["summary"],
"columns_removed": cols_removed,
"findings": result["findings"],
}
def generate_gemini_insights(result: dict) -> list:
findings = result.get("findings", [])
if not findings:
return []
if not GEMINI_API_KEY:
return build_fallback_insights(findings[:3])
top = findings[:3]
findings_text = "\n".join([
f"- {f['attribute']} (col: '{f['column']}'): {f['description']} Severity: {f['severity'].upper()}. Recommendation: {f['recommendation']}"
for f in top
])
prompt = f"""You are a senior AI fairness consultant advising an HR director at an Indian company.
Bias issues detected in their hiring dataset:
{findings_text}
Dataset: {result['total_rows']} candidates, {result['overall_selection_rate']}% selection rate, Fairness Score: {result['bias_score']}/100.
For each finding return JSON with keys:
- "finding": attribute name
- "explanation": 2-3 sentences in plain English, specific to India's context, explaining what happened and why it's harmful
- "impact": 1-2 sentences on real impact on actual people (mention career damage, economic harm, constitutional rights where relevant)
- "fixes": array of exactly 2 specific actionable steps HR can implement this quarter
Return ONLY valid JSON array. No markdown, no preamble."""
try:
model = genai.GenerativeModel("gemini-1.5-flash")
response = model.generate_content(prompt)
text = response.text.strip()
if "```" in text:
text = text.split("```")[1]
if text.startswith("json"): text = text[4:]
text = text.strip()
parsed = json.loads(text)
if isinstance(parsed, list):
return parsed
except Exception as e:
print(f"Gemini error: {e}")
return build_fallback_insights(top)
def build_fallback_insights(findings: list) -> list:
fallbacks = {
"Gender Bias": {
"explanation": "Your hiring data shows men and women are selected at significantly different rates. The model learned from historical decisions that already contained human bias — and is now replicating it at scale, automatically rejecting qualified women.",
"impact": "Female candidates with equal or better qualifications are being rejected systematically. This directly reduces women's economic opportunities and violates equal opportunity employment principles.",
"fixes": ["Implement blind resume screening — remove name, gender, and photo fields before any algorithmic scoring.", "Retrain or recalibrate your model with a gender-balanced training dataset and enforce an 80% minimum selection rate parity."]
},
"Caste Proxy Bias (Surname Analysis)": {
"explanation": "Your model has learned to associate candidate surnames with hiring outcomes, effectively using names as a caste proxy. Without any explicit caste field, the algorithm is discriminating based on caste identity inferred from names alone.",
"impact": "SC/ST/OBC candidates are being rejected regardless of qualifications. This perpetuates generational inequality and is a constitutional violation under Article 15 of India.",
"fixes": ["Remove candidate names entirely from shortlisting — use anonymized ID numbers only.", "Commission a retrospective audit of the last 2 years of decisions, filtered by surname caste classification, to measure the cumulative damage."]
},
"College Prestige Bias (India)": {
"explanation": "The model heavily rewards IIT/IIM/NIT graduates and penalizes everyone else, even when skills and experience are equal. Over 95% of Indian graduates are from non-elite institutions — this bias excludes the vast majority of the talent pool.",
"impact": "Talented graduates from state universities and tier-2 colleges are systematically blocked regardless of demonstrated skills, concentrating opportunity among already-privileged candidates who had access to elite college preparation.",
"fixes": ["Replace institution name with standardized skill assessment scores in your screening pipeline.", "Add a rule preventing rejection based on college tier alone without an administered skills test."]
},
"Career Gap Bias": {
"explanation": "Candidates with employment gaps are being selected at much lower rates. The model penalizes any gap as a negative signal, ignoring context completely — maternity leave, health issues, family responsibilities all look the same to it.",
"impact": "This disproportionately harms women returning from maternity or childcare leave, and candidates from economically disadvantaged backgrounds who faced hardship. It punishes people for life circumstances beyond their control.",
"fixes": ["Remove raw gap duration as a model feature. Replace with a contextual flag that is set positive if the candidate provided a reason.", "Add mandatory human review for any candidate flagged only due to career gaps before algorithmic rejection."]
},
"City/Location Bias (India)": {
"explanation": "Candidates from Tier-2 and Tier-3 cities are selected at significantly lower rates than metro candidates. The model learned a spurious correlation — that city of origin predicts job performance — when it actually predicts socioeconomic access.",
"impact": "This perpetuates geographic inequality in India, concentrating white-collar employment in metro areas and excluding talented candidates from smaller cities who have equal or greater potential.",
"fixes": ["Remove city of origin from all model features — location is irrelevant to performance unless the role requires local presence.", "Standardize evaluation using only skill scores, work samples, and structured interview ratings."]
},
"Age Bias": {
"explanation": "Older candidates (35+) are selected at significantly lower rates. The model learned from historical bias in past hiring decisions and is now automatically disadvantaging experienced candidates.",
"impact": "Experienced professionals are excluded from opportunities, wasting accumulated expertise. This also disproportionately affects women re-entering the workforce after family responsibilities.",
"fixes": ["Remove age and date-of-birth from all model inputs.", "Evaluate candidates strictly on demonstrated skills, relevant experience quality, and structured competency assessments."]
},
"Caste-Based Discrimination": {
"explanation": "Explicit caste data in your dataset is directly correlated with hiring outcomes. This means caste category is being used — directly or indirectly — as a selection criterion.",
"impact": "This is an unconstitutional practice under Article 15 of the Indian Constitution and exposes the organization to serious legal liability while causing severe harm to affected candidates.",
"fixes": ["Remove the caste/category column from all model inputs immediately.", "Engage legal counsel to review past decisions and conduct a full remediation audit."]
},
}
results = []
for f in findings:
attr = f["attribute"]
fb = fallbacks.get(attr, {
"explanation": f["description"],
"impact": "Candidates from disadvantaged groups face systematic rejection regardless of their actual capabilities.",
"fixes": [f["recommendation"], "Conduct a full fairness audit with an independent AI ethics consultant."]
})
results.append({"finding": attr, "explanation": fb["explanation"], "impact": fb["impact"], "fixes": fb["fixes"]})
return results