Skip to content

Commit 322b3ba

Browse files
rebecamurillorebecamurillo
committed
feat: separation for mcda quantitative data, prepare for qualitative setup
1 parent 006d18f commit 322b3ba

8 files changed

Lines changed: 326 additions & 55 deletions

File tree

‎src/sum_impact_assessment/api/routes/jobs.py‎

Lines changed: 8 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ def trigger_job(
6161
request: Optional[TriggerJobRequest] = Body(
6262
None,
6363
openapi_examples={
64-
"kpi_group_filter": {
64+
"impact_analysis_sief": {
6565
"summary": "Impact analysis for SIEF KPIs",
6666
"description": "Run impact analysis filtering only SIEF KPIs",
6767
"value": {
@@ -70,39 +70,29 @@ def trigger_job(
7070
}
7171
}
7272
},
73-
"mcda_regulatory_perspective": {
73+
"mcda_quantitative_regulatory_perspective": {
7474
"summary": "MCDA Analysis for regulatory perspective",
75-
"description": "Run MCDA analysis with regulatory stakeholder weights",
75+
"description": "Run MCDA analysis with regulatory stakeholder weights, from quantitative data form KPI/measures impact analysis",
7676
"value": {
7777
"params": {
7878
"kpi_group_type": "MCDA_GOALS",
7979
"perspective": "regulatory"
8080
}
8181
}
8282
},
83-
"mcda_pto_perspective": {
83+
"mcda_quantitative_pto_perspective": {
8484
"summary": "MCDA Analysis for PTO perspective",
85-
"description": "Run MCDA analysis with PTO stakeholder weights",
85+
"description": "Run MCDA analysis with PTO stakeholder weights, from quantitative data form KPI/measures impact analysis",
8686
"value": {
8787
"params": {
8888
"kpi_group_type": "MCDA_GOALS",
8989
"perspective": "pto"
9090
}
9191
}
9292
},
93-
"mcda_citizens_users_perspective": {
94-
"summary": "MCDA Analysis for citizens/users perspective",
95-
"description": "Run MCDA analysis with citizens/users stakeholder weights",
96-
"value": {
97-
"params": {
98-
"kpi_group_type": "MCDA_GOALS",
99-
"perspective": "citizens_users"
100-
}
101-
}
102-
},
103-
"mcda_nsm_providers_perspective": {
93+
"mcda_quantitative_nsm_providers_perspective": {
10494
"summary": "MCDA Analysis for NSM providers perspective",
105-
"description": "Run MCDA analysis with NSM providers stakeholder weights",
95+
"description": "Run MCDA analysis with NSM providers stakeholder weights, from quantitative data form KPI/measures impact analysis",
10696
"value": {
10797
"params": {
10898
"kpi_group_type": "MCDA_GOALS",
@@ -153,7 +143,7 @@ def trigger_job(
153143
job_repo = JobRepository(db)
154144
actual_job_name = job_name.value
155145

156-
if job_name == JobNameEnum.MCDA_ANALYSIS and params and "perspective" in params:
146+
if job_name == JobNameEnum.MCDA_ANALYSIS_QUANTITATIVE and params and "perspective" in params:
157147
perspective = params["perspective"]
158148
actual_job_name = f"{job_name.value}_{perspective}"
159149
logger.info(f"MCDA job with perspective: {perspective}")

‎src/sum_impact_assessment/data/README.md‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
The information in the mcda_goals_ba_configuration.json file is the final result of expert surveys done by VEDECOM.
2+
The methodology for the calculations is described bellow.
3+
Authors : "Axel Le Dreau", VEDECOM
4+
5+
Note:
6+
We start from stakeholder interviews conducted across several Living Labs. During
7+
these interviews, participants assessed the Business Activities (BA) implemented in
8+
their own Living Lab. BA are groups of push and pull measures. For each BA, they
9+
provided a qualitative score for each policy goal, using a 1–5 scale. Each score is
10+
therefore linked to a respondent (actor), a Living Lab, a stakeholder category (for
11+
example PTO, Regulatory, NSM provider), a BA, and a goal. A key feature of the
12+
dataset is that not all Living Labs implemented the same BA. As a result, actors only
13+
score the BAs that exist in their site, which creates an unbalanced and incomplete
14+
evaluation grid.
15+
For decision support, we do not simply average the interview scores. A raw mean (or
16+
median) per BA and goal would mix several effects that are not the BA performance.
17+
First, Living Labs differ in local context, and implementation conditions can influence
18+
how impacts are perceived, so some sites may systematically rate higher or lower.
19+
Second, stakeholder categories often have different perspectives, which can lead to
20+
systematic differences in ratings. Third, individuals use the 1–5 scale differently:
21+
some respondents are generally stricter, others more generous. Finally,
22+
representation is uneven: some categories or sites are more represented in
23+
interviews and would dominate a simple average. In addition, some BAs are rated by
24+
only a few actors. In those cases, a mean can be unstable. Because of these issues,
25+
we aim to produce a scoring table that is fair across stakeholder groups, comparable
26+
across sites, and robust when data are sparse.
27+
We use a Bayesian ordinal hierarchical model (also called a multilevel cumulative
28+
logit model) to transform the interview ratings into a robust BA-Goal scoring matrix.
29+
“Ordinal” means the method is designed for ordered scores such as 1–5, where
30+
higher is better but the exact distance between levels is not assumed to be perfectly
31+
linear. “-Hierarchical (multilevel) means it explicitly accounts for the structure of the
32+
data: respondents belong to a Living Lab (site) and to a stakeholder category, and
33+
each respondent may have a consistent rating style (more strict or more generous).
34+
The model estimates a typical score for each BA and goal while separating BArelated differences from systematic differences linked to sites, stakeholder
35+
categories, and individual rating habits. Because it is Bayesian, it naturally produces
36+
both a central estimate and an uncertainty range for each BA-oal score, which helps
37+
communicate where evidence is strong or limited.
38+
Process :
39+
The first methodological choice is to treat the 1–5 interview score as an ordered
40+
judgement rather than a precise measurement. A score of 4 means “better than 3”,
41+
but we do not assume that the numerical distance between 2 and 3 is exactly the
42+
same as between 4 and 5. We therefore treat scores as ordered categories. This
43+
avoids over-interpreting the exact numeric gap between rating levels and better
44+
reflects the qualitative nature of interview-based scoring.
45+
We then estimate a typical BA score for each goal while controlling for systematic
46+
differences in the data. Conceptually, for each goal we separate four components.
47+
The BA effect captures whether a given BA tends to receive higher or lower scores
48+
on that goal; this is the performance signal we are interested in for comparison
49+
across BA. The site effect captures whether a Living Lab context tends to shift ratings
50+
up or down for that goal, reflecting differences in local conditions and implementation
51+
environments. The stakeholder category effect captures whether certain actor groups
52+
tend to rate differently on that goal due to perspective or institutional role. The actor
53+
effect captures individual rating style, meaning that some respondents are
54+
consistently strict or consistently generous across all their ratings. Estimating these
55+
components together allows us to distinguish “this BA is rated higher” from “this site
56+
rates higher” or “this actor rates higher”, which improves comparability across BAs
57+
despite the incomplete coverage of the evaluation grid.
58+
To ensure fairness between stakeholder categories, we apply a balancing rule during
59+
model estimation. Because categories are not equally represented, we prevent any
60+
single group from dominating the estimated BA performance simply due to sample
61+
size. In practice, each individual rating is given an estimation weight so that each
62+
stakeholder category has equal total influence, each actor has equal influence within
63+
their category, and actors who rated more BAs do not automatically count more than
64+
actors who rated fewer BAs. A diagnostic file is produced to verify that this balancing
65+
behaves as intended.
66+
The model does not directly output a single number per BA and goal. Instead, for
67+
each BA–goal pair it produces a probability distribution across possible score levels.
68+
This means the result is not only a point estimate but also expresses how likely each
69+
rating level is, given the data and the adjustments described above. To build the
70+
input matrix required by PROMETHEE, we convert this distribution into a score using
71+
an expected value approach: we multiply each score level by its probability and sum
72+
the results. The resulting value remains on the familiar 1–5 scale, but it is grounded
73+
in the full distribution rather than in a single observed rating.
74+
When producing the final BA×goal matrix, we deliberately compute a score that is not
75+
tied to a specific site, stakeholder category, or individual respondent. This is done by
76+
predicting BA performance as if the site context were average, the stakeholder
77+
category were average, and the actor rating style were average, while retaining only
78+
the BA-specific effect for each goal. The purpose is to obtain one comparable score
79+
per BA and goal that can be used consistently in PROMETHEE, even though the
80+
underlying interview data were collected in different sites with different respondent
81+
mixes.
82+
Finally, we report uncertainty. Some BA–goal scores are supported by more interview
83+
ratings than others. For each BA and goal we therefore compute a central estimate
84+
and a 95% uncertainty range. This is important for interpretation: if two BAs are close
85+
in score but their uncertainty ranges overlap substantially, the evidence for a
86+
difference is weaker and rankings should be interpreted with caution.
87+
The process produces two main outputs. The first is a BA-goal performance matrix
88+
containing the typical score for each BA on each goal; this is the direct input for
89+
PROMETHEE. The second is the same information in a long format with the
90+
associated uncertainty bounds, which supports transparency and allows users to
91+
identify where results are robust versus uncertain. The resulting scores represent
92+
stakeholder-perceived performance, adjusted to be fair and comparable across sites
93+
and stakeholder groups. They do not directly measure real-world KPI changes, which
94+
can be assessed separately using KPI-based methods.
95+
Ref
96+
Ohnishi, Y., & Sugaya, S. (2022). Applying Bayesian hierarchical probit model to
97+
interview grade evaluation. arXiv preprint arXiv:2003.11591.
98+
Johnson, V. E. (1994). On Bayesian analysis of multi-rater ordinal data: An
99+
application to automated essay grading. Institute of Statistics and Decision Sciences,
100+
Duke University, Discussion Paper 94-03

‎src/sum_impact_assessment/data/mcda_goal_weights.json‎

Lines changed: 0 additions & 22 deletions
This file was deleted.

‎src/sum_impact_assessment/data/mcda_goals_ba_configuration.json‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
{
2+
"perspectives": {
3+
"labels": {
4+
"nsm_providers": "NSM Providers",
5+
"pto": "Public Transport Operators",
6+
"regulatory": "Regulatory Authorities"
7+
},
8+
"weights": {
9+
"nsm_providers": {
10+
"Improve Accessibility": 0.12,
11+
"Improve Mobility Service": 0.16,
12+
"Improve Multimodality": 0.13,
13+
"Noise Hinderance": 0.06,
14+
"Improve Public Transport": 0.12,
15+
"Reduction of Congestion": 0.15,
16+
"Reduction of Emission": 0.14,
17+
"Improve Safety": 0.11
18+
},
19+
"pto": {
20+
"Improve Accessibility": 0.11,
21+
"Improve Mobility Service": 0.13,
22+
"Improve Multimodality": 0.14,
23+
"Noise Hinderance": 0.08,
24+
"Improve Public Transport": 0.16,
25+
"Reduction of Congestion": 0.13,
26+
"Reduction of Emission": 0.13,
27+
"Improve Safety": 0.12
28+
},
29+
"regulatory": {
30+
"Improve Accessibility": 0.15,
31+
"Improve Mobility Service": 0.14,
32+
"Improve Multimodality": 0.12,
33+
"Noise Hinderance": 0.07,
34+
"Improve Public Transport": 0.16,
35+
"Reduction of Congestion": 0.12,
36+
"Reduction of Emission": 0.12,
37+
"Improve Safety": 0.12
38+
}
39+
}
40+
},
41+
"business_activities": {
42+
"labels": {
43+
"BA1": "Integrated Mobility Service Platform (MaaS)",
44+
"BA2": "Demand-Responsive and On-Demand Mobility",
45+
"BA3": "Mobility Hub Development",
46+
"BA4": "Active Mobility Promotion",
47+
"BA5": "Incentive-Based Programs",
48+
"BA6": "NSM Integration into Mobility Ecosystem",
49+
"BA7": "Public Engagement and Awareness Initiatives",
50+
"BA8": "Enhanced Data Collection and Analysis",
51+
"BA9": "Electric and Low-Emission Infrastructure Expansion",
52+
"BA10": "PT Scheduling and Frequency Optimization"
53+
},
54+
"goals_score": {
55+
"BA1": {
56+
"Improve Accessibility": 3.83,
57+
"Improve Mobility Service": 4.16,
58+
"Improve Multimodality": 4.22,
59+
"Noise Hinderance": 3.14,
60+
"Improve Public Transport": 4.23,
61+
"Reduction of Congestion": 3.6,
62+
"Reduction of Emission": 3.53,
63+
"Improve Safety": 2.98
64+
},
65+
"BA2": {
66+
"Improve Accessibility": 3.94,
67+
"Improve Mobility Service": 4.05,
68+
"Improve Multimodality": 3.99,
69+
"Noise Hinderance": 3.1,
70+
"Improve Public Transport": 4.17,
71+
"Reduction of Congestion": 3.53,
72+
"Reduction of Emission": 3.1,
73+
"Improve Safety": 3.42
74+
},
75+
"BA3": {
76+
"Improve Accessibility": 3.94,
77+
"Improve Mobility Service": 4.12,
78+
"Improve Multimodality": 4.18,
79+
"Noise Hinderance": 2.98,
80+
"Improve Public Transport": 3.96,
81+
"Reduction of Congestion": 3.47,
82+
"Reduction of Emission": 2.71,
83+
"Improve Safety": 3.44
84+
},
85+
"BA4": {
86+
"Improve Accessibility": 3.92,
87+
"Improve Mobility Service": 4.06,
88+
"Improve Multimodality": 3.96,
89+
"Noise Hinderance": 3.31,
90+
"Improve Public Transport": 4.14,
91+
"Reduction of Congestion": 3.7,
92+
"Reduction of Emission": 3.97,
93+
"Improve Safety": 3.59
94+
},
95+
"BA5": {
96+
"Improve Accessibility": 3.85,
97+
"Improve Mobility Service": 4.09,
98+
"Improve Multimodality": 3.97,
99+
"Noise Hinderance": 3.18,
100+
"Improve Public Transport": 4.1,
101+
"Reduction of Congestion": 3.6,
102+
"Reduction of Emission": 3.78,
103+
"Improve Safety": 3.34
104+
},
105+
"BA6": {
106+
"Improve Accessibility": 3.86,
107+
"Improve Mobility Service": 4.15,
108+
"Improve Multimodality": 4.06,
109+
"Noise Hinderance": 3.19,
110+
"Improve Public Transport": 4.23,
111+
"Reduction of Congestion": 3.59,
112+
"Reduction of Emission": 3.54,
113+
"Improve Safety": 3.42
114+
},
115+
"BA7": {
116+
"Improve Accessibility": 3.91,
117+
"Improve Mobility Service": 4.13,
118+
"Improve Multimodality": 4.01,
119+
"Noise Hinderance": 3.11,
120+
"Improve Public Transport": 4.28,
121+
"Reduction of Congestion": 3.74,
122+
"Reduction of Emission": 3.77,
123+
"Improve Safety": 3.36
124+
},
125+
"BA8": {
126+
"Improve Accessibility": 3.83,
127+
"Improve Mobility Service": 4.03,
128+
"Improve Multimodality": 3.95,
129+
"Noise Hinderance": 3.06,
130+
"Improve Public Transport": 3.99,
131+
"Reduction of Congestion": 3.57,
132+
"Reduction of Emission": 3.38,
133+
"Improve Safety": 3.23
134+
},
135+
"BA9": {
136+
"Improve Accessibility": 3.98,
137+
"Improve Mobility Service": 4.16,
138+
"Improve Multimodality": 4.0,
139+
"Noise Hinderance": 3.45,
140+
"Improve Public Transport": 4.23,
141+
"Reduction of Congestion": 3.9,
142+
"Reduction of Emission": 4.61,
143+
"Improve Safety": 3.68
144+
},
145+
"BA10": {
146+
"Improve Accessibility": 3.92,
147+
"Improve Mobility Service": 4.17,
148+
"Improve Multimodality": 4.07,
149+
"Noise Hinderance": 3.39,
150+
"Improve Public Transport": 4.31,
151+
"Reduction of Congestion": 3.84,
152+
"Reduction of Emission": 4.29,
153+
"Improve Safety": 3.31
154+
}
155+
}
156+
}
157+
}

‎src/sum_impact_assessment/jobs/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
# Job registry mapping job names to job classes
1010
JOB_REGISTRY: Dict[JobNameEnum, Type] = {
1111
JobNameEnum.KPI_MEASURES_ANALYSIS: KpiMeasuresAnalysisJob,
12-
JobNameEnum.MCDA_ANALYSIS: MCDAAnalysisJob
12+
JobNameEnum.MCDA_ANALYSIS_QUANTITATIVE: MCDAAnalysisJob
1313
}
1414

1515

‎src/sum_impact_assessment/schemas/job.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ class JobNameEnum(str, Enum):
1212
Enumeration of valid job names.
1313
"""
1414
KPI_MEASURES_ANALYSIS = "kpi_measures_analysis"
15-
MCDA_ANALYSIS = "mcda_analysis"
15+
MCDA_ANALYSIS_QUANTITATIVE = "mcda_analysis_quantitative"
1616

1717

1818
class JobStatusEnum(str, Enum):
@@ -31,7 +31,7 @@ class TriggerJobRequest(BaseModel):
3131
"""
3232
params: Optional[Dict] = Field(
3333
None,
34-
description="Optional job-specific parameters. For mcda_analysis: {'kpi_group_type': 'MCDA_GOALS'}"
34+
description="Optional job-specific parameters. For mcda_analysis_quantitative: {'kpi_group_type': 'MCDA_GOALS'}"
3535
)
3636

3737
class Config:

0 commit comments

Comments
 (0)