Skip to content

Commit 9de9b75

Browse files
jtfairbankjtfairbank
committed
feat: pull in and process the 2023-Q3 needs assessment data from generated surveys.
1 parent b802d3f commit 9de9b75

File tree

5 files changed

+254
-6
lines changed

5 files changed

+254
-6
lines changed

‎gatsby/needs-assessment/generated-needs-assessment-mappers.ts

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import { parse } from 'csv-parse/sync'
2+
import { readFileSync } from 'fs'
3+
import { Product } from '../../src/types/product.d'
4+
5+
type ItemKeyMap = {
6+
Category: string
7+
Item: string
8+
'Age / Gender': string
9+
'Size / Style': string
10+
Unit: string
11+
'Category Id': string
12+
'Item Id': string
13+
'Unit Id': string
14+
}
15+
16+
/*
17+
Import Item to Key Map
18+
------------------------------------------------------------
19+
TODO: This should be encapsulated in the `needs-assessment-schemas` repo.
20+
*/
21+
// TODO: make this dynamic
22+
const base = `./gatsby/needs-assessment/generated/2023-Q3`
23+
24+
const itemsMapCsv = readFileSync(
25+
`${base}/generated-item-to-key-map.csv`,
26+
'utf8',
27+
)
28+
const itemsMap: ItemKeyMap[] = parse(itemsMapCsv, {
29+
columns: true,
30+
skip_empty_lines: true,
31+
skip_records_with_empty_values: true,
32+
trim: true,
33+
})
34+
const productsByComboKey = itemsMap.reduce((byComboKey, item) => {
35+
const comboKey = `${item['Category Id']}Needs${item['Item Id']}${item['Unit Id']}`
36+
byComboKey[comboKey] = {
37+
category: item.Category.endsWith('Clothing') ? 'Clothing' : item.Category,
38+
item: item.Item,
39+
ageGender: item['Age / Gender'],
40+
sizeStyle: item['Size / Style'],
41+
unit: item.Unit === 'Individual Item' ? 'Item' : item.Unit,
42+
}
43+
return byComboKey
44+
}, {} as Record<string, Product>)
45+
46+
export const isProductSurveyPage = (categoryKey: string): boolean => {
47+
return categoryKey.endsWith('Needs')
48+
}
49+
50+
export const productMapper = (
51+
categoryKey: string,
52+
itemKey: string,
53+
unitKey: string,
54+
): Product | undefined => {
55+
const comboKey = `${categoryKey}${itemKey}${unitKey}`
56+
return productsByComboKey[comboKey]
57+
}

‎gatsby/needs-assessment/generated/2023-Q3/generated-item-to-key-map.csv

+146
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
Category,Item,Age / Gender,Size / Style,Unit,Category Id,Item Id,Unit Id
2+
Hygiene,Mask,,Cloth,Individual Item,hygiene,clothMask,itemIndividual
3+
Hygiene,Mask,,Surgical,Individual Item,hygiene,surgicalMask,itemIndividual
4+
Hygiene,Mask,,KN95,Individual Item,hygiene,kN95Mask,itemIndividual
5+
Hygiene,Mask,,N95 / FFP2,Individual Item,hygiene,n95FFP2Mask,itemIndividual
6+
Hygiene,Soap,,Bar,100g Bar,hygiene,barSoap,bar100g
7+
Hygiene,Soap,,Liquid,250ml Bottle,hygiene,liquidSoap,bottle250ml
8+
Hygiene,Shampoo,,,250ml Bottle,hygiene,shampoo,bottle250ml
9+
Hygiene,Toothbrush,,,Individual Item,hygiene,toothbrush,itemIndividual
10+
Hygiene,Toothpaste,,,100ml Tube,hygiene,toothpaste,tube100ml
11+
Hygiene,Nail Clipper,,,Individual Item,hygiene,nailClipper,itemIndividual
12+
Hygiene,Deodorant,,,100ml Roller,hygiene,deodorant,roller100ml
13+
Hygiene,Razor,,Disposable,Individual Item,hygiene,disposableRazor,itemIndividual
14+
Hygiene,Shaving Foam,,,200ml Can,hygiene,shavingFoam,can200ml
15+
Hygiene,Sanitary Pad,,Reusable,Individual Item,hygiene,reusableSanitaryPad,itemIndividual
16+
Hygiene,Sanitary Pad,,Disposable,Individual Item,hygiene,disposableSanitaryPad,itemIndividual
17+
Hygiene,Condom,,,Individual Item,hygiene,condom,itemIndividual
18+
Hygiene,Diaper,,Size 0,Individual Item,hygiene,size0Diaper,itemIndividual
19+
Hygiene,Diaper,,Size 1,Individual Item,hygiene,size1Diaper,itemIndividual
20+
Hygiene,Diaper,,Size 2,Individual Item,hygiene,size2Diaper,itemIndividual
21+
Hygiene,Diaper,,Size 3,Individual Item,hygiene,size3Diaper,itemIndividual
22+
Hygiene,Diaper,,Size 4,Individual Item,hygiene,size4Diaper,itemIndividual
23+
Hygiene,Diaper,,Size 5,Individual Item,hygiene,size5Diaper,itemIndividual
24+
Hygiene,Diaper,,Size 6,Individual Item,hygiene,size6Diaper,itemIndividual
25+
Hygiene,Diaper,,Adult,Individual Item,hygiene,adultDiaper,itemIndividual
26+
Hygiene,Wet Wipes,,,100 per Pack,hygiene,wetWipes,packper100
27+
Hygiene,Washing Detergent,,,1L Bottle (Liquid),hygiene,washingDetergent,bottle1L
28+
Hygiene,Washing Detergent,,,5kg Bag (Powedered),hygiene,washingDetergent,bag5kg
29+
Hygiene,Bleach,,,1L Bottle,hygiene,bleach,bottle1L
30+
Hygiene,Sunblock,,,150ml Tube,hygiene,sunblock,tube150ml
31+
Hygiene,Mosquito Repellent,,,100ml Bottle,hygiene,mosquitoRepellent,bottle100ml
32+
Shelter,Tent,,,Individual Item,shelter,tent,itemIndividual
33+
Shelter,Tarp,,,Individual Item,shelter,tarp,itemIndividual
34+
Shelter,Sleeping Bag,,Adult,Individual Item,shelter,adultSleepingBag,itemIndividual
35+
Shelter,Sleeping Bag,,Child,Individual Item,shelter,childSleepingBag,itemIndividual
36+
Shelter,Sleeping Mat,,,Individual Item,shelter,sleepingMat,itemIndividual
37+
Shelter,Blanket,,,Individual Item,shelter,blanket,itemIndividual
38+
Shelter,Bed Sheets,,,Individual Item,shelter,bedSheets,itemIndividual
39+
Shelter,Duvet,,,Individual Item,shelter,duvet,itemIndividual
40+
Shelter,Pillow,,,Individual Item,shelter,pillow,itemIndividual
41+
Shelter,Pillow Case,,,Individual Item,shelter,pillowCase,itemIndividual
42+
Shelter,Bed Rails,,Child,Individual Item,shelter,childBedRails,itemIndividual
43+
Shelter,Backpack,,,Individual Item,shelter,backpack,itemIndividual
44+
Shelter,Suitcase,,,Individual Item,shelter,suitcase,itemIndividual
45+
Education,Notepad,,,Individual Item,education,notepad,itemIndividual
46+
Education,Pen,,,Individual Item,education,pen,itemIndividual
47+
Education,Pencil,,,Individual Item,education,pencil,itemIndividual
48+
Education,Pencil Sharpener,,,Individual Item,education,pencilSharpener,itemIndividual
49+
Education,Eraser,,,Individual Item,education,eraser,itemIndividual
50+
Education,Printer Paper,,,500 Sheets per Pack,education,printerPaper,packperSheets500
51+
Electronics,Laptop,,,Individual Item,electronics,laptop,itemIndividual
52+
Electronics,Tablet,,,Individual Item,electronics,tablet,itemIndividual
53+
Electronics,Phone,,,Individual Item,electronics,phone,itemIndividual
54+
Electronics,Phone Charger,,,Individual Item,electronics,phoneCharger,itemIndividual
55+
Electronics,Powerbank,,,Individual Item,electronics,powerbank,itemIndividual
56+
Medical,Wheelchair,,,Individual Item,medical,wheelchair,itemIndividual
57+
Medical,Crutches,,,Individual Item,medical,crutches,itemIndividual
58+
Medical,Bandaid / Plaster,,,40 per Box,medical,bandaidPlaster,boxper40
59+
Medical,Anti-Lice Shampoo,,,100ml Bottle,medical,antiLiceShampoo,bottle100ml
60+
Medical,Surgical Needles,,,10 per Pack,medical,surgicalNeedles,packper10
61+
Swag,DA Hoodies,,,Individual Item,swag,dAHoodies,itemIndividual
62+
Food,Rice,,,Euro Pallet,food,rice,palletEuro
63+
Food,Rice,,,Kilogram,food,rice,kilogram
64+
Food,Potatoe,,,Euro Pallet,food,potatoe,palletEuro
65+
Food,Potatoe,,,Kilogram,food,potatoe,kilogram
66+
Food,Onion,,,Euro Pallet,food,onion,palletEuro
67+
Food,Onion,,,Kilogram,food,onion,kilogram
68+
Food,Garlic,,,Euro Pallet,food,garlic,palletEuro
69+
Food,Garlic,,,Kilogram,food,garlic,kilogram
70+
Food,Flour,,,Euro Pallet,food,flour,palletEuro
71+
Food,Flour,,,Kilogram,food,flour,kilogram
72+
Food,Salt,,,Euro Pallet,food,salt,palletEuro
73+
Food,Salt,,,Kilogram,food,salt,kilogram
74+
Food,Sugar,,,Euro Pallet,food,sugar,palletEuro
75+
Food,Sugar,,,Kilogram,food,sugar,kilogram
76+
Food,Cooking Oil,,,Euro Pallet,food,cookingOil,palletEuro
77+
Food,Cooking Oil,,,Liter,food,cookingOil,liter
78+
Food,Milk,,,Euro Pallet,food,milk,palletEuro
79+
Food,Milk,,,Liter,food,milk,liter
80+
Food,Canned Tomatoes,,,Euro Pallet,food,cannedTomatoes,palletEuro
81+
Food,Canned Tomatoes,,,Cans (#10 Kitchen Size),food,cannedTomatoes,cans
82+
Food,Canned Beans,,,Euro Pallet,food,cannedBeans,palletEuro
83+
Food,Canned Beans,,,Cans (#10 Kitchen Size),food,cannedBeans,cans
84+
Food,Canned Fish,,,Euro Pallet,food,cannedFish,palletEuro
85+
Food,Canned Fish,,,Cans (#10 Kitchen Size),food,cannedFish,cans
86+
Food,Sweetcorn,,,Euro Pallet,food,sweetcorn,palletEuro
87+
Food,Sweetcorn,,,Cans (#10 Kitchen Size),food,sweetcorn,cans
88+
Food,Tea,,,Euro Pallet,food,tea,palletEuro
89+
Food,Tea,,,Kilogram,food,tea,kilogram
90+
Food,Tea,,,Servings,food,tea,servings
91+
Food,Coffee,,,Euro Pallet,food,coffee,palletEuro
92+
Food,Coffee,,,Kilogram,food,coffee,kilogram
93+
Food,Coffee,,,Servings,food,coffee,servings
94+
Women's Clothing,Jacket,Woman,,Individual Item,womensClothing,jacket,itemIndividual
95+
Women's Clothing,Jumper,Woman,,Individual Item,womensClothing,jumper,itemIndividual
96+
Women's Clothing,T-Shirt,Woman,,Individual Item,womensClothing,tShirt,itemIndividual
97+
Women's Clothing,Long-Sleeved Top,Woman,,Individual Item,womensClothing,longSleevedTop,itemIndividual
98+
Women's Clothing,Shorts,Woman,,Individual Item,womensClothing,shorts,itemIndividual
99+
Women's Clothing,Trousers,Woman,,Individual Item,womensClothing,trousers,itemIndividual
100+
Women's Clothing,Leggings,Woman,,Individual Item,womensClothing,leggings,itemIndividual
101+
Women's Clothing,Pants (Underwear),Woman,,Individual Item,womensClothing,pantsUnderwear,itemIndividual
102+
Women's Clothing,Bras,Woman,,Individual Item,womensClothing,bras,itemIndividual
103+
Women's Clothing,Socks,Woman,,Individual Item,womensClothing,socks,itemIndividual
104+
Women's Clothing,Shoes,Woman,,Individual Item,womensClothing,shoes,itemIndividual
105+
Women's Clothing,Dresses,Woman,,Individual Item,womensClothing,dresses,itemIndividual
106+
Women's Clothing,Hijabs,Woman,,Individual Item,womensClothing,hijabs,itemIndividual
107+
Women's Clothing,Abayas,Woman,,Individual Item,womensClothing,abayas,itemIndividual
108+
Men's Clothing,Jacket,Man,,Individual Item,mensClothing,jacket,itemIndividual
109+
Men's Clothing,Jumper,Man,,Individual Item,mensClothing,jumper,itemIndividual
110+
Men's Clothing,T-Shirt,Man,,Individual Item,mensClothing,tShirt,itemIndividual
111+
Men's Clothing,Long-Sleeved Top,Man,,Individual Item,mensClothing,longSleevedTop,itemIndividual
112+
Men's Clothing,Shorts,Man,,Individual Item,mensClothing,shorts,itemIndividual
113+
Men's Clothing,Trousers,Man,,Individual Item,mensClothing,trousers,itemIndividual
114+
Men's Clothing,Pants (Underwear),Man,,Individual Item,mensClothing,pantsUnderwear,itemIndividual
115+
Men's Clothing,Socks,Man,,Individual Item,mensClothing,socks,itemIndividual
116+
Men's Clothing,Shoes,Man,,Individual Item,mensClothing,shoes,itemIndividual
117+
Girl's Clothing,Jacket,Girl,,Individual Item,girlsClothing,jacket,itemIndividual
118+
Girl's Clothing,Jumper,Girl,,Individual Item,girlsClothing,jumper,itemIndividual
119+
Girl's Clothing,T-Shirt,Girl,,Individual Item,girlsClothing,tShirt,itemIndividual
120+
Girl's Clothing,Long-Sleeved Top,Girl,,Individual Item,girlsClothing,longSleevedTop,itemIndividual
121+
Girl's Clothing,Shorts,Girl,,Individual Item,girlsClothing,shorts,itemIndividual
122+
Girl's Clothing,Trousers,Girl,,Individual Item,girlsClothing,trousers,itemIndividual
123+
Girl's Clothing,Leggings,Girl,,Individual Item,girlsClothing,leggings,itemIndividual
124+
Girl's Clothing,Pants (Underwear),Girl,,Individual Item,girlsClothing,pantsUnderwear,itemIndividual
125+
Girl's Clothing,Bras,Girl,,Individual Item,girlsClothing,bras,itemIndividual
126+
Girl's Clothing,Socks,Girl,,Individual Item,girlsClothing,socks,itemIndividual
127+
Girl's Clothing,Shoes,Girl,,Individual Item,girlsClothing,shoes,itemIndividual
128+
Girl's Clothing,Dresses,Girl,,Individual Item,girlsClothing,dresses,itemIndividual
129+
Boy's Clothing,Jacket,Boy,,Individual Item,boysClothing,jacket,itemIndividual
130+
Boy's Clothing,Jumper,Boy,,Individual Item,boysClothing,jumper,itemIndividual
131+
Boy's Clothing,T-Shirt,Boy,,Individual Item,boysClothing,tShirt,itemIndividual
132+
Boy's Clothing,Long-Sleeved Top,Boy,,Individual Item,boysClothing,longSleevedTop,itemIndividual
133+
Boy's Clothing,Shorts,Boy,,Individual Item,boysClothing,shorts,itemIndividual
134+
Boy's Clothing,Trousers,Boy,,Individual Item,boysClothing,trousers,itemIndividual
135+
Boy's Clothing,Pants (Underwear),Boy,,Individual Item,boysClothing,pantsUnderwear,itemIndividual
136+
Boy's Clothing,Socks,Boy,,Individual Item,boysClothing,socks,itemIndividual
137+
Boy's Clothing,Shoes,Boy,,Individual Item,boysClothing,shoes,itemIndividual
138+
Baby Clothing,Jacket / All-In-Ones,Baby,,Individual Item,babyClothing,jacketAllInOnes,itemIndividual
139+
Baby Clothing,Jumper,Baby,,Individual Item,babyClothing,jumper,itemIndividual
140+
Baby Clothing,T-Shirt,Baby,,Individual Item,babyClothing,tShirt,itemIndividual
141+
Baby Clothing,Shorts,Baby,,Individual Item,babyClothing,shorts,itemIndividual
142+
Baby Clothing,Trousers,Baby,,Individual Item,babyClothing,trousers,itemIndividual
143+
Baby Clothing,Socks,Baby,,Individual Item,babyClothing,socks,itemIndividual
144+
Baby Clothing,Shoes,Baby,,Individual Item,babyClothing,shoes,itemIndividual
145+
Baby Clothing,Bodies / Rompers,Baby,,Individual Item,babyClothing,bodiesRompers,itemIndividual
146+
Baby Clothing,Onsies / Pyjamas,Baby,,Individual Item,babyClothing,onsiesPyjamas,itemIndividual

‎gatsby/needs-assessment/sourceNeedsAssessmentData.ts

+45-6
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import fetch from 'cross-fetch'
22
import { SourceNodesArgs } from 'gatsby'
33
import { Product } from '../../src/types/product.d'
4+
import {
5+
isProductSurveyPage as generatedIsProductSurveyPage,
6+
productMapper as generatedProductMapper,
7+
} from './generated-needs-assessment-mappers'
48
import {
59
isProductSurveyPage,
610
placeMapper,
@@ -11,6 +15,7 @@ import {
1115
enum SURVEY_FORMATS {
1216
ORIGINAL = 'ORIGINAL',
1317
NO_QUARTER = 'NO_QUARTER',
18+
GENERATED_V1 = 'GENERATED_V1',
1419
}
1520

1621
export type SurveyId = {
@@ -21,7 +26,10 @@ export type SurveyId = {
2126
format: SURVEY_FORMATS
2227
}
2328

24-
type NASummary = NASummary_ORIGINAL | NASummary_NO_QUARTER
29+
type NASummary =
30+
| NASummary_ORIGINAL
31+
| NASummary_NO_QUARTER
32+
| NASummary_GENERATED_V1
2533
type NASummary_ORIGINAL = Record<
2634
string,
2735
Record<string, Record<string, Record<string, Record<string, number>>>>
@@ -30,6 +38,7 @@ type NASummary_NO_QUARTER = Record<
3038
string,
3139
Record<string, Record<string, Record<string, number>>>
3240
>
41+
type NASummary_GENERATED_V1 = NASummary_NO_QUARTER
3342

3443
type NAResponse = {
3544
summary: NASummary
@@ -109,6 +118,14 @@ const surveyIds: SurveyId[] = [
109118
url: 'https://storage.needs-assessment.distributeaid.dev/form/01GVZ3SXBA8HCYTPQ1JZ53GZAX/summary?groupBy=basicInfo.region',
110119
format: SURVEY_FORMATS.NO_QUARTER,
111120
},
121+
{
122+
// 2023 Q3
123+
id: '01H3KENGVF837HMQ2S4VXYCC39',
124+
year: '2023',
125+
quarter: 'Q3',
126+
url: 'https://storage.needs-assessment.distributeaid.dev/form/01H3KENGVF837HMQ2S4VXYCC39/summary?groupBy=basicInfo.region',
127+
format: SURVEY_FORMATS.GENERATED_V1,
128+
},
112129
]
113130

114131
export const sourceNeedsAssessments = async ({
@@ -174,15 +191,27 @@ export const preprocess = (
174191
switch (surveyId.format) {
175192
case SURVEY_FORMATS.ORIGINAL:
176193
return summary as NASummary_ORIGINAL
194+
177195
case SURVEY_FORMATS.NO_QUARTER:
178196
if (surveyId.quarter === undefined) {
179197
throw new Error(
180198
`Survey has NO_QUARTER format and no hardcoded quarter: ${surveyId.id}`,
181199
)
182200
}
183-
const structuredSummary: Record<string, NASummary_NO_QUARTER> = {}
184-
structuredSummary[surveyId.quarter] = summary as NASummary_NO_QUARTER
185-
return structuredSummary
201+
const noQuarterSummary: Record<string, NASummary_NO_QUARTER> = {}
202+
noQuarterSummary[surveyId.quarter] = summary as NASummary_NO_QUARTER
203+
return noQuarterSummary
204+
205+
case SURVEY_FORMATS.GENERATED_V1:
206+
if (surveyId.quarter === undefined) {
207+
throw new Error(
208+
`Survey has NO_QUARTER format and no hardcoded quarter: ${surveyId.id}`,
209+
)
210+
}
211+
const generatedV1Summary: Record<string, NASummary_GENERATED_V1> = {}
212+
generatedV1Summary[surveyId.quarter] = summary as NASummary_GENERATED_V1
213+
return generatedV1Summary
214+
186215
default:
187216
throw new Error(`Survey had an unkown format: ${surveyId.id}`)
188217
}
@@ -198,10 +227,20 @@ export const processNeedsAssessment = (
198227
const needsDatas: NeedsData[] = []
199228
const lookupMissLog: string[] = []
200229

230+
const testProductSurveyPage =
231+
surveyId.format === SURVEY_FORMATS.GENERATED_V1
232+
? generatedIsProductSurveyPage
233+
: isProductSurveyPage
234+
235+
const mapProduct =
236+
surveyId.format === SURVEY_FORMATS.GENERATED_V1
237+
? generatedProductMapper
238+
: productMapper
239+
201240
for (let [quarter, places] of Object.entries(summary)) {
202241
for (const [placeKey, pages] of Object.entries(places)) {
203242
for (const [page, questions] of Object.entries(pages)) {
204-
if (!isProductSurveyPage(page)) continue
243+
if (!testProductSurveyPage(page)) continue
205244
for (const [question, units] of Object.entries(questions)) {
206245
const unitKey = Object.keys(units)[0]
207246
if (unitKey === undefined) {
@@ -217,7 +256,7 @@ export const processNeedsAssessment = (
217256
continue
218257
}
219258

220-
const product = productMapper(page, question, unitKey)
259+
const product = mapProduct(page, question, unitKey)
221260
if (product === undefined) {
222261
lookupMissLog.push(
223262
`No product defined for ${page}.${question}.${unitKey}`,

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
"@types/tailwindcss": "3.0.11",
2323
"classnames": "2.3.2",
2424
"cross-fetch": "3.1.5",
25+
"csv-parse": "^5.4.0",
2526
"gatsby": "4.25.6",
2627
"gatsby-alias-imports": "1.0.6",
2728
"gatsby-plugin-image": "2.25.0",

‎yarn.lock

+5
Original file line numberDiff line numberDiff line change
@@ -7631,6 +7631,11 @@ csstype@^3.0.2:
76317631
resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.1.tgz#841b532c45c758ee546a11d5bd7b7b473c8c30b9"
76327632
integrity sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw==
76337633

7634+
csv-parse@^5.4.0:
7635+
version "5.4.0"
7636+
resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-5.4.0.tgz#6793210a4a49a9a74b3fde3f9d00f3f52044fd89"
7637+
integrity sha512-JiQosUWiOFgp4hQn0an+SBoV9IKdqzhROM0iiN4LB7UpfJBlsSJlWl9nq4zGgxgMAzHJ6V4t29VAVD+3+2NJAg==
7638+
76347639
currently-unhandled@^0.4.1:
76357640
version "0.4.1"
76367641
resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea"

0 commit comments

Comments
 (0)