Skip to content

Commit 5de5169

Browse files
dr5hnclaude
dr5hn
and
claude
authored
feat(exports): wire postcodes table into all export formats (#1039) (#1403)
Completes the deferred export-pipeline work from #1398. With the postcodes table now landed, every export command and the workflow itself learns to emit postcode data alongside the existing 5 tables. PHP commands (Symfony Console) - ExportJson: SELECT from postcodes (graceful skip if table missing), emit /json/postcodes.json - ExportCsv: add 'postcodes' to FILES; guard empty arrays so empty source files no longer crash on $csc[0] access - ExportXml / ExportYaml: add 'postcodes' to FILES; replace fragile ?: throw on empty arrays with explicit is_array() check - ExportSqlServer: add 'postcodes' to TABLES with full CREATE TABLE schema (FKs to countries/states/cities, nullable state/city) - ExportMongoDB: add 'postcodes' to COLLECTIONS plus processPostcodes() with country/state DBRef references and GeoJSON Point location Python helpers - export_plist.py: include postcodes.csv with missing-file guard so the script no-ops cleanly until first country PR lands - sync_mysql_to_json.py: new sync_postcodes() per-country file writer mirroring sync_cities; export_schema includes postcodes when present Workflow - postcode_count env var (graceful 0 if table absent) - mysqldump postcodes -> sql/postcodes.sql - pg_dump postcodes -> psql/postcodes.sql - mysql2sqlite postcodes -> sqlite/postcodes.sqlite3 - mongoimport gated on non-empty postcodes.json - gzip postcodes.sql in sql/ and psql/ when present - POSTCODE_COUNT exposed to Release body and PR body Behaviour with empty postcodes table - Importer/JSON/CSV/XML/YAML produce empty postcodes.json (or skip in CSV's case) without erroring - mongoimport skipped via jq length check - mysqldump still emits the (empty) DDL, so consumers can rely on the table existing in every export format Refs: #1039 Builds on: #1398 Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent c5559a4 commit 5de5169

9 files changed

Lines changed: 207 additions & 12 deletions

File tree

‎.github/workflows/export.yml‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,7 @@ jobs:
150150
echo "country_count=$(mysql -uroot -proot -e 'SELECT COUNT(*) FROM world.countries;' -s)" >> $GITHUB_ENV
151151
echo "state_count=$(mysql -uroot -proot -e 'SELECT COUNT(*) FROM world.states;' -s)" >> $GITHUB_ENV
152152
echo "city_count=$(mysql -uroot -proot -e 'SELECT COUNT(*) FROM world.cities;' -s)" >> $GITHUB_ENV
153+
echo "postcode_count=$(mysql -uroot -proot -sN -e 'SELECT COUNT(*) FROM world.postcodes;' 2>/dev/null || echo 0)" >> $GITHUB_ENV
153154
echo "current_date=$(date +'%B %d, %Y')" >> $GITHUB_ENV
154155
155156
- name: Composer Dependencies
@@ -192,6 +193,7 @@ jobs:
192193
mysqldump -uroot -proot --single-transaction --add-drop-table --disable-keys --set-charset --skip-add-locks world countries > sql/countries.sql
193194
mysqldump -uroot -proot --single-transaction --add-drop-table --disable-keys --set-charset --skip-add-locks world states > sql/states.sql
194195
mysqldump -uroot -proot --single-transaction --add-drop-table --disable-keys --set-charset --skip-add-locks world cities > sql/cities.sql
196+
mysqldump -uroot -proot --single-transaction --add-drop-table --disable-keys --set-charset --skip-add-locks world postcodes > sql/postcodes.sql
195197
# Export complete world database
196198
mysqldump -uroot -proot --single-transaction --add-drop-table --disable-keys --set-charset --skip-add-locks world > sql/world.sql
197199
@@ -200,13 +202,15 @@ jobs:
200202
echo "🧹 Adding DROP TABLE commands in correct order..."
201203
# Remove any existing DROP TABLE commands
202204
grep -v "DROP TABLE" sql/world.sql > sql/world_temp.sql
203-
# Create new file with DROP TABLE commands in correct order (reverse of foreign key dependencies)
205+
# Create new file with DROP TABLE commands in correct order (reverse of foreign key dependencies).
206+
# postcodes drops first because it FKs to countries/states/cities.
204207
cat > sql/world.sql << 'EOF'
205208
-- -------------------------------------------------------------
206209
-- Generated from MySQL database
207210
-- Combined world database
208211
-- -------------------------------------------------------------
209212
213+
DROP TABLE IF EXISTS `postcodes`;
210214
DROP TABLE IF EXISTS `cities`;
211215
DROP TABLE IF EXISTS `states`;
212216
DROP TABLE IF EXISTS `countries`;
@@ -239,6 +243,7 @@ jobs:
239243
pg_dump --dbname=postgresql://postgres:postgres@localhost/world -Fp --inserts --clean --if-exists --no-owner --no-acl -t countries > psql/countries.sql
240244
pg_dump --dbname=postgresql://postgres:postgres@localhost/world -Fp --inserts --clean --if-exists --no-owner --no-acl -t states > psql/states.sql
241245
pg_dump --dbname=postgresql://postgres:postgres@localhost/world -Fp --inserts --clean --if-exists --no-owner --no-acl -t cities > psql/cities.sql
246+
pg_dump --dbname=postgresql://postgres:postgres@localhost/world -Fp --inserts --clean --if-exists --no-owner --no-acl -t postcodes > psql/postcodes.sql
242247
pg_dump --dbname=postgresql://postgres:postgres@localhost/world -Fp --inserts --clean --if-exists --no-owner --no-acl > psql/world.sql
243248
244249
# Remove \restrict and \unrestrict commands from all psql files
@@ -257,6 +262,7 @@ jobs:
257262
mysql2sqlite -d world -t countries --mysql-password root -u root -f sqlite/countries.sqlite3
258263
mysql2sqlite -d world -t states --mysql-password root -u root -f sqlite/states.sqlite3
259264
mysql2sqlite -d world -t cities --mysql-password root -u root -f sqlite/cities.sqlite3
265+
mysql2sqlite -d world -t postcodes --mysql-password root -u root -f sqlite/postcodes.sqlite3
260266
mysql2sqlite -d world --mysql-password root -u root -f sqlite/world.sqlite3
261267
262268
- name: Export SQL Server
@@ -281,6 +287,11 @@ jobs:
281287
mongoimport --host localhost:27017 --db world --collection countries --file countries.json --jsonArray
282288
mongoimport --host localhost:27017 --db world --collection states --file states.json --jsonArray
283289
mongoimport --host localhost:27017 --db world --collection cities --file cities.json --jsonArray
290+
if [ -s postcodes.json ] && [ "$(jq 'length' postcodes.json)" -gt 0 ]; then
291+
mongoimport --host localhost:27017 --db world --collection postcodes --file postcodes.json --jsonArray
292+
else
293+
echo "Skipping postcodes import — empty or missing"
294+
fi
284295
echo "Import completed"
285296
286297
# Create a MongoDB dump
@@ -290,7 +301,7 @@ jobs:
290301
tar -czvf world-mongodb-dump.tar.gz mongodb-dump
291302
echo "MongoDB dump created at mongodb/world-mongodb-dump.tar.gz"
292303
293-
rm -rf mongodb-dump regions.json subregions.json countries.json states.json cities.json
304+
rm -rf mongodb-dump regions.json subregions.json countries.json states.json cities.json postcodes.json
294305
295306
- name: Compress All Large Files
296307
run: |
@@ -353,13 +364,21 @@ jobs:
353364
gzip -9 -k -f sql/cities.sql
354365
echo " ✓ sql/world.sql.gz"
355366
echo " ✓ sql/cities.sql.gz"
367+
if [ -f sql/postcodes.sql ]; then
368+
gzip -9 -k -f sql/postcodes.sql
369+
echo " ✓ sql/postcodes.sql.gz"
370+
fi
356371
357372
# PostgreSQL SQL Files
358373
echo "📄 Compressing PostgreSQL SQL files..."
359374
gzip -9 -k -f psql/world.sql
360375
gzip -9 -k -f psql/cities.sql
361376
echo " ✓ psql/world.sql.gz"
362377
echo " ✓ psql/cities.sql.gz"
378+
if [ -f psql/postcodes.sql ]; then
379+
gzip -9 -k -f psql/postcodes.sql
380+
echo " ✓ psql/postcodes.sql.gz"
381+
fi
363382
364383
# SQLite Files
365384
echo "📄 Compressing SQLite files..."
@@ -403,6 +422,7 @@ jobs:
403422
COUNTRY_COUNT: ${{ env.country_count }}
404423
STATE_COUNT: ${{ env.state_count }}
405424
CITY_COUNT: ${{ env.city_count }}
425+
POSTCODE_COUNT: ${{ env.postcode_count }}
406426
with:
407427
script: |
408428
const fs = require('fs');
@@ -468,6 +488,7 @@ jobs:
468488
`- **Countries**: ${process.env.COUNTRY_COUNT}`,
469489
`- **States**: ${process.env.STATE_COUNT}`,
470490
`- **Cities**: ${process.env.CITY_COUNT}`,
491+
`- **Postcodes**: ${process.env.POSTCODE_COUNT}`,
471492
'',
472493
'### Formats',
473494
'JSON, MySQL, PostgreSQL, SQLite, SQL Server, XML, YAML, CSV, GeoJSON, Toon, MongoDB',
@@ -526,7 +547,7 @@ jobs:
526547
commit-message: |
527548
Export database formats - ${{ env.current_date }}
528549
529-
Total records: ${{ env.country_count }} countries, ${{ env.state_count }} states, ${{ env.city_count }} cities
550+
Total records: ${{ env.country_count }} countries, ${{ env.state_count }} states, ${{ env.city_count }} cities, ${{ env.postcode_count }} postcodes
530551
Large files (.gz) uploaded to GitHub Releases instead of git.
531552
committer: Darshan Gada <gadadarshan@gmail.com>
532553
signoff: true
@@ -542,6 +563,7 @@ jobs:
542563
- **Countries**: ${{ env.country_count }}
543564
- **States**: ${{ env.state_count }}
544565
- **Cities**: ${{ env.city_count }}
566+
- **Postcodes**: ${{ env.postcode_count }}
545567
546568
### What's in this PR
547569
Small export files (individual table JSON, CSV, SQL schema, etc.) that are useful in the repo.

‎bin/Commands/ExportCsv.php‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ class ExportCsv extends Command
1919
'cities' => ['from' => '/json/cities.json', 'to' => '/csv/cities.csv'],
2020
'regions' => ['from' => '/json/regions.json', 'to' => '/csv/regions.csv'],
2121
'subregions' => ['from' => '/json/subregions.json', 'to' => '/csv/subregions.csv'],
22+
'postcodes' => ['from' => '/json/postcodes.json', 'to' => '/csv/postcodes.csv'],
2223
];
2324

2425
private const TRANSLATION_FILES = [
@@ -57,11 +58,20 @@ protected function execute(InputInterface $input, OutputInterface $output): int
5758
? file_get_contents($rootDir . $v['from'])
5859
: throw new \RuntimeException("JSON file not found: {$v['from']}");
5960

60-
$csc = json_decode($jsonData, true)
61-
?: throw new \RuntimeException("Invalid JSON in {$v['from']}");
61+
$csc = json_decode($jsonData, true);
62+
if (!is_array($csc)) {
63+
throw new \RuntimeException("Invalid JSON in {$v['from']}");
64+
}
6265

6366
$fp = fopen($rootDir . $v['to'], 'w');
6467

68+
// Skip header writing for empty datasets — write empty CSV
69+
if (empty($csc)) {
70+
fclose($fp);
71+
$io->note("Skipping $root (no records)");
72+
continue;
73+
}
74+
6575
// Set headings
6676
$headings = $csc[0];
6777
unset($headings['translations']);

‎bin/Commands/ExportJson.php‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,11 +281,39 @@ protected function execute(InputInterface $input, OutputInterface $output): int
281281
}
282282
}
283283

284+
// Fetching All Postcodes (issue #1039) — graceful skip if table missing
285+
$postcodesArray = array();
286+
$p = 0;
287+
$hasPostcodes = $db->query("SHOW TABLES LIKE 'postcodes'");
288+
if ($hasPostcodes && $hasPostcodes->num_rows > 0) {
289+
$sql = "SELECT * FROM postcodes ORDER BY country_code, code";
290+
$result = $db->query($sql);
291+
if ($result && $result->num_rows > 0) {
292+
while ($row = $result->fetch_assoc()) {
293+
$postcodesArray[$p]['id'] = (int)$row['id'];
294+
$postcodesArray[$p]['code'] = $row['code'];
295+
$postcodesArray[$p]['country_id'] = (int)$row['country_id'];
296+
$postcodesArray[$p]['country_code'] = $row['country_code'];
297+
$postcodesArray[$p]['state_id'] = $row['state_id'] !== null ? (int)$row['state_id'] : null;
298+
$postcodesArray[$p]['state_code'] = $row['state_code'];
299+
$postcodesArray[$p]['city_id'] = $row['city_id'] !== null ? (int)$row['city_id'] : null;
300+
$postcodesArray[$p]['locality_name'] = $row['locality_name'];
301+
$postcodesArray[$p]['type'] = $row['type'];
302+
$postcodesArray[$p]['latitude'] = $row['latitude'];
303+
$postcodesArray[$p]['longitude'] = $row['longitude'];
304+
$postcodesArray[$p]['source'] = $row['source'];
305+
$postcodesArray[$p]['wikiDataId'] = $row['wikiDataId'];
306+
$p++;
307+
}
308+
}
309+
}
310+
284311
$io->writeln('Total Regions Count : ' . count($regionsArray));
285312
$io->writeln('Total Subregions Count : ' . count($subregionsArray));
286313
$io->writeln('Total Countries Count : ' . count($countriesArray));
287314
$io->writeln('Total States Count : ' . count($statesArray));
288315
$io->writeln('Total Cities Count : ' . count($citiesArray));
316+
$io->writeln('Total Postcodes Count : ' . count($postcodesArray));
289317

290318
// Add a Space
291319
$io->newLine();
@@ -296,6 +324,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
296324
'/json/countries.json' => $countriesArray,
297325
'/json/states.json' => $statesArray,
298326
'/json/cities.json' => $citiesArray,
327+
'/json/postcodes.json' => $postcodesArray,
299328
'/json/countries+states.json' => $countryStateArray,
300329
'/json/countries+cities.json' => $countryCityArray,
301330
'/json/countries+states+cities.json' => $countryStateCityArray

‎bin/Commands/ExportMongoDB.php‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ class ExportMongoDB extends Command
1515
protected static $defaultName = 'export:mongodb';
1616
protected static $defaultDescription = 'Export data to MongoDB format';
1717

18-
private const COLLECTIONS = ['regions', 'subregions', 'countries', 'states', 'cities'];
18+
private const COLLECTIONS = ['regions', 'subregions', 'countries', 'states', 'cities', 'postcodes'];
1919
private Filesystem $filesystem;
2020
private array $dataCache = [];
2121

@@ -67,6 +67,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
6767
$this->processCountries($io, $rootDir);
6868
$this->processStates($io, $rootDir);
6969
$this->processCities($io, $rootDir);
70+
$this->processPostcodes($io, $rootDir);
7071

7172
$io->success('MongoDB export completed successfully');
7273
return Command::SUCCESS;
@@ -287,6 +288,56 @@ private function processCities(SymfonyStyle $io, string $rootDir): void
287288
$io->info('Cities exported to MongoDB format');
288289
}
289290

291+
private function processPostcodes(SymfonyStyle $io, string $rootDir): void
292+
{
293+
$io->section('Processing postcodes');
294+
295+
$postcodes = $this->dataCache['postcodes'] ?? [];
296+
$countries = $this->dataCache['countries'] ?? [];
297+
$states = $this->dataCache['states'] ?? [];
298+
299+
$countryById = [];
300+
foreach ($countries as $c) {
301+
$countryById[(int) $c['id']] = $c;
302+
}
303+
$stateById = [];
304+
foreach ($states as $s) {
305+
$stateById[(int) $s['id']] = $s;
306+
}
307+
308+
$processed = [];
309+
foreach ($postcodes as $p) {
310+
$row = $p;
311+
$row['_id'] = (int) $p['id'];
312+
unset($row['id']);
313+
314+
if (!empty($p['country_id']) && isset($countryById[(int) $p['country_id']])) {
315+
$row['country'] = [
316+
'$ref' => 'countries',
317+
'$id' => (int) $p['country_id'],
318+
];
319+
}
320+
if (!empty($p['state_id']) && isset($stateById[(int) $p['state_id']])) {
321+
$row['state'] = [
322+
'$ref' => 'states',
323+
'$id' => (int) $p['state_id'],
324+
];
325+
}
326+
327+
if (isset($p['latitude'], $p['longitude']) && $p['latitude'] !== null && $p['longitude'] !== null) {
328+
$row['location'] = [
329+
'type' => 'Point',
330+
'coordinates' => [(float) $p['longitude'], (float) $p['latitude']],
331+
];
332+
}
333+
334+
$processed[] = $row;
335+
}
336+
337+
$this->saveCollection($rootDir, 'postcodes', $processed);
338+
$io->info('Postcodes exported to MongoDB format');
339+
}
340+
290341
private function saveCollection(string $rootDir, string $collection, array $data): void
291342
{
292343
$outputFile = "$rootDir/mongodb/$collection.json";

‎bin/Commands/ExportSqlServer.php‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ class ExportSqlServer extends Command
1515
protected static $defaultName = 'export:sql-server';
1616
protected static $defaultDescription = 'Export data to SQL Server format';
1717

18-
private const TABLES = ['regions', 'subregions', 'countries', 'states', 'cities'];
18+
private const TABLES = ['regions', 'subregions', 'countries', 'states', 'cities', 'postcodes'];
1919
private Filesystem $filesystem;
2020

2121
public function __construct()
@@ -143,6 +143,29 @@ private function generateTableSchema(string $table): string
143143
wikiDataId NVARCHAR(255) NULL,
144144
CONSTRAINT FK_cities_states FOREIGN KEY (state_id) REFERENCES world.states(id),
145145
CONSTRAINT FK_cities_countries FOREIGN KEY (country_id) REFERENCES world.countries(id)
146+
);",
147+
'postcodes' => "
148+
IF OBJECT_ID('world.postcodes', 'U') IS NOT NULL DROP TABLE world.postcodes;
149+
CREATE TABLE world.postcodes (
150+
id INT IDENTITY(1,1) PRIMARY KEY,
151+
code NVARCHAR(20) NOT NULL,
152+
country_id INT NOT NULL,
153+
country_code NCHAR(2) NOT NULL,
154+
state_id INT NULL,
155+
state_code NVARCHAR(255) NULL,
156+
city_id INT NULL,
157+
locality_name NVARCHAR(255) NULL,
158+
type NVARCHAR(32) NULL,
159+
latitude DECIMAL(10,8) NULL,
160+
longitude DECIMAL(11,8) NULL,
161+
source NVARCHAR(64) NULL,
162+
wikiDataId NVARCHAR(255) NULL,
163+
created_at DATETIME2 NOT NULL DEFAULT '2014-01-01 12:01:01',
164+
updated_at DATETIME2 NOT NULL DEFAULT GETDATE(),
165+
flag BIT NOT NULL DEFAULT 1,
166+
CONSTRAINT FK_postcodes_countries FOREIGN KEY (country_id) REFERENCES world.countries(id),
167+
CONSTRAINT FK_postcodes_states FOREIGN KEY (state_id) REFERENCES world.states(id),
168+
CONSTRAINT FK_postcodes_cities FOREIGN KEY (city_id) REFERENCES world.cities(id)
146169
);"
147170
];
148171

‎bin/Commands/ExportXml.php‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ class ExportXml extends Command
2020
'countries' => ['from' => '/json/countries.json', 'to' => '/xml/countries.xml', 'singular' => 'country'],
2121
'states' => ['from' => '/json/states.json', 'to' => '/xml/states.xml', 'singular' => 'state'],
2222
'cities' => ['from' => '/json/cities.json', 'to' => '/xml/cities.xml', 'singular' => 'city'],
23+
'postcodes' => ['from' => '/json/postcodes.json', 'to' => '/xml/postcodes.xml', 'singular' => 'postcode'],
2324
];
2425

2526
private Filesystem $filesystem;
@@ -50,8 +51,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int
5051
? file_get_contents($rootDir . $config['from'])
5152
: throw new \RuntimeException("JSON file not found: {$config['from']}");
5253

53-
$data = json_decode($jsonData, true)
54-
?: throw new \RuntimeException("Invalid JSON in {$config['from']}");
54+
$data = json_decode($jsonData, true);
55+
if (!is_array($data)) {
56+
throw new \RuntimeException("Invalid JSON in {$config['from']}");
57+
}
5558

5659
$xml = ArrayToXml::convert(
5760
[$config['singular'] => $data],

‎bin/Commands/ExportYaml.php‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ class ExportYaml extends Command
2020
'countries' => ['from' => '/json/countries.json', 'to' => '/yml/countries.yml', 'singular' => 'country'],
2121
'states' => ['from' => '/json/states.json', 'to' => '/yml/states.yml', 'singular' => 'state'],
2222
'cities' => ['from' => '/json/cities.json', 'to' => '/yml/cities.yml', 'singular' => 'city'],
23+
'postcodes' => ['from' => '/json/postcodes.json', 'to' => '/yml/postcodes.yml', 'singular' => 'postcode'],
2324
];
2425

2526
private Filesystem $filesystem;
@@ -50,8 +51,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int
5051
? file_get_contents($rootDir . $config['from'])
5152
: throw new \RuntimeException("JSON file not found: {$config['from']}");
5253

53-
$data = json_decode($jsonData, true)
54-
?: throw new \RuntimeException("Invalid JSON in {$config['from']}");
54+
$data = json_decode($jsonData, true);
55+
if (!is_array($data)) {
56+
throw new \RuntimeException("Invalid JSON in {$config['from']}");
57+
}
5558

5659
$yaml = Yaml::dump(
5760
[$config['singular'] => $data],

‎bin/scripts/export/export_plist.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,13 @@
1111
'./csv/countries.csv',
1212
'./csv/states.csv',
1313
'./csv/cities.csv',
14+
'./csv/postcodes.csv',
1415
]
1516

1617
for csv_file in files:
18+
if not os.path.exists(csv_file):
19+
print('PLIST: skipping missing source {}'.format(csv_file))
20+
continue
1721
with open(csv_file, 'r', encoding='utf-8') as f:
1822
result = list(csv.DictReader(f))
1923

0 commit comments

Comments
 (0)