Skip to content

Commit ab331b9

Browse files
Rub21Rub21
committed
Remove generalized_tables for water and udpate mviews
1 parent 8e70c6d commit ab331b9

File tree

8 files changed

+417
-238
lines changed

8 files changed

+417
-238
lines changed

‎images/tiler-imposm/Dockerfile‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM osgeo/gdal:ubuntu-small-3.2.3
1+
FROM ghcr.io/osgeo/gdal:ubuntu-small-3.9.3
22

33
RUN apt-get -y update && apt-get install -y \
44
g++ \
@@ -18,21 +18,22 @@ RUN apt-get -y update && apt-get install -y \
1818
curl \
1919
wget \
2020
unzip \
21-
software-properties-common && \
21+
ca-certificates \
22+
software-properties-common \
23+
python3-pip \
24+
postgresql-client && \
2225
rm -rf /var/lib/apt/lists/*
2326

24-
RUN add-apt-repository ppa:deadsnakes/ppa && \
25-
apt-get update && \
26-
apt-get install -y python3.9 python3.9-dev python3-pip && \
27-
rm -rf /var/lib/apt/lists/* && \
28-
python3 -m pip install pip --upgrade && \
29-
python3 -m pip install wheel
27+
RUN python3 -m pip install --no-cache-dir \
28+
--break-system-packages \
29+
awscli psycopg2-binary setuptools && \
30+
rm -rf /root/.cache /var/cache/apk/*
3031

3132
RUN apt-get update && apt-get install -y postgresql-client && \
3233
rm -rf /var/lib/apt/lists/*
3334

34-
RUN wget -c https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz -O - | tar -xz -C /usr/local
35-
ENV PATH $PATH:/usr/local/go/bin
35+
RUN wget -c https://dl.google.com/go/go1.21.9.linux-amd64.tar.gz -O - | tar -xz -C /usr/local
36+
ENV PATH="${PATH}:/usr/local/go/bin"
3637

3738
ENV DATEFUNCTIONS_GITSHA=90d8d0f0daea4c8c5aa62edf440f26e9eb0ab950
3839
RUN git clone https://github.com/OpenHistoricalMap/DateFunctions-plpgsql.git /usr/local/datefunctions && cd /usr/local/datefunctions && git checkout $DATEFUNCTIONS_GITSHA
@@ -45,14 +46,11 @@ RUN git clone https://github.com/omniscale/imposm3.git $GOPATH/src/github.com/om
4546
WORKDIR $GOPATH/src/github.com/omniscale/imposm3
4647
RUN git checkout v0.14.2
4748

48-
4949
# Apply fixes or patches (if necessary)
5050
RUN sed -i '/setMaxFileSize/d' cache/ldb_pre_121.go
5151
RUN go install github.com/omniscale/imposm3/cmd/imposm
5252
ENV PATH $PATH:$GOPATH/bin
5353

54-
# Install AWS CLI for downloading files
55-
RUN pip3 install -U setuptools awscli psycopg2
5654
WORKDIR /osm
5755
COPY . .
5856
CMD ["./start.sh"]

‎images/tiler-imposm/config/layers/water_areas.json‎

Lines changed: 1 addition & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -7,38 +7,7 @@
77
"source:datetime"
88
]
99
},
10-
"generalized_tables": {
11-
"water_areas_z0_2": {
12-
"source": "water_areas_z3_5",
13-
"sql_filter": "type IN ('water', 'riverbank') AND area > 100000000",
14-
"tolerance": 5000
15-
},
16-
"water_areas_z3_5": {
17-
"source": "water_areas_z6_7",
18-
"sql_filter": "area > 50000000",
19-
"tolerance": 1000
20-
},
21-
"water_areas_z6_7": {
22-
"source": "water_areas_z8_9",
23-
"sql_filter": "area > 1000000",
24-
"tolerance": 200
25-
},
26-
"water_areas_z8_9": {
27-
"source": "water_areas_z10_12",
28-
"sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank') AND area > 10000",
29-
"tolerance": 100
30-
},
31-
"water_areas_z10_12": {
32-
"source": "water_areas_z13_15",
33-
"sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank')",
34-
"tolerance": 20
35-
},
36-
"water_areas_z13_15": {
37-
"source": "water_areas",
38-
"sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank', 'dock')",
39-
"tolerance": 5
40-
}
41-
},
10+
"generalized_tables": {},
4211
"tables": {
4312
"water_areas": {
4413
"fields": [

‎images/tiler-imposm/config/layers/water_lines.json‎

Lines changed: 1 addition & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -7,27 +7,7 @@
77
"source:datetime"
88
]
99
},
10-
"generalized_tables": {
11-
"water_lines_z8_9": {
12-
"source": "water_lines",
13-
"sql_filter": "type IN ('river', 'canal')",
14-
"tolerance": 100
15-
},
16-
"water_lines_z10_12": {
17-
"source": "water_lines",
18-
"sql_filter": "type IN ('river', 'canal', 'cliff', 'dam')",
19-
"tolerance": 20
20-
},
21-
"water_lines_z13_15": {
22-
"source": "water_lines",
23-
"sql_filter": "type IN ('river', 'canal', 'cliff', 'dam', 'stream')",
24-
"tolerance": 5
25-
},
26-
"water_lines_z16_20": {
27-
"source": "water_lines",
28-
"sql_filter": "type IN ('river', 'canal', 'cliff', 'dam', 'stream', 'ditch', 'drain')"
29-
}
30-
},
10+
"generalized_tables": {},
3111
"tables": {
3212
"water_lines": {
3313
"fields": [

‎images/tiler-imposm/queries/ohm_mviews/water.sql‎

Lines changed: 63 additions & 157 deletions
Original file line numberDiff line numberDiff line change
@@ -1,168 +1,74 @@
11
-- ============================================================================
2-
-- Function: create_water_areas_subdivided_mview
3-
-- Description:
4-
-- Creates a materialized view for water areas using ST_Subdivide to simplify
5-
-- complex geometries. The input geometries are validated with ST_MakeValid and
6-
-- dumped using ST_Dump to extract components.
7-
--
8-
-- Multilingual name columns are added dynamically from the `languages` table.
9-
--
10-
-- Parameters:
11-
-- input_table TEXT - The source table containing raw geometries.
12-
-- mview_name TEXT - The name of the materialized view to be created.
13-
--
14-
-- Behavior:
15-
-- - Uses a temporary view during creation to avoid downtime.
16-
-- - Only valid POLYGON and MULTIPOLYGON geometries are retained.
17-
-- - Adds GiST spatial index on geometry and unique index on (id).
2+
-- Water Areas Materialized Views for Multiple Zoom Levels
3+
-- Creates a pyramid of materialized views for water areas, optimized for
184
-- ============================================================================
19-
DROP FUNCTION IF EXISTS create_water_areas_subdivided_mview;
20-
21-
CREATE OR REPLACE FUNCTION create_water_areas_subdivided_mview(
22-
input_table TEXT,
23-
mview_name TEXT
24-
)
25-
RETURNS void AS $$
26-
DECLARE
27-
lang_columns TEXT := get_language_columns();
28-
tmp_view_name TEXT := mview_name || '_tmp';
29-
sql_create TEXT;
30-
unique_columns TEXT := 'id';
31-
BEGIN
32-
sql_create := format($sql$
33-
CREATE MATERIALIZED VIEW %I AS
34-
SELECT
35-
row_number() OVER () AS id,
36-
geometry,
37-
osm_id,
38-
NULLIF(name, '') AS name,
39-
NULLIF(type, '') AS type,
40-
NULLIF(start_date, '') AS start_date,
41-
NULLIF(end_date, '') AS end_date,
42-
isodatetodecimaldate(pad_date(start_date, 'start'), FALSE) AS start_decdate,
43-
isodatetodecimaldate(pad_date(end_date, 'end'), FALSE) AS end_decdate,
44-
area,
45-
%s
46-
FROM (
47-
SELECT
48-
ST_Subdivide((g).geom, 512) AS geometry,
49-
osm_id,
50-
name,
51-
type,
52-
start_date,
53-
end_date,
54-
area,
55-
tags,
56-
%s
57-
FROM (
58-
SELECT
59-
osm_id,
60-
name,
61-
type,
62-
start_date,
63-
end_date,
64-
area,
65-
ST_Dump(ST_MakeValid(geometry)) AS g,
66-
tags,
67-
%s
68-
FROM %I
69-
WHERE geometry IS NOT NULL
70-
) AS fixed_geoms
71-
WHERE GeometryType((g).geom) IN ('POLYGON', 'MULTIPOLYGON')
72-
) AS final_data;
73-
$sql$, tmp_view_name, lang_columns, lang_columns, lang_columns, input_table);
74-
75-
PERFORM finalize_materialized_view(
76-
tmp_view_name,
77-
mview_name,
78-
unique_columns,
79-
sql_create
80-
);
81-
END;
82-
$$ LANGUAGE plpgsql;
835

84-
-- ============================================================================
85-
-- Function: create_water_areas_centroids_mview
86-
-- Description:
87-
-- This function creates a materialized view with centroids for named water areas.
88-
-- It uses ST_MaximumInscribedCircle to compute a representative centroid from
89-
-- each polygonal feature. The function can be called per zoom level using different source tables.
90-
--
91-
-- Parameters:
92-
-- source_table TEXT - Source table containing water area polygons.
93-
-- view_name TEXT - Name of the resulting materialized view.
94-
--
95-
-- Notes:
96-
-- - Only features with non-empty names are included.
97-
-- - Geometry is computed as the center of the maximum inscribed circle.
98-
-- - A GiST index is created on geometry, and uniqueness is enforced on osm_id.
99-
-- - Uses a temporary view to avoid downtime during refresh.
100-
-- ============================================================================
6+
-- Delete existing views, in cascade
7+
DROP MATERIALIZED VIEW IF EXISTS mv_water_areas_z16_20 CASCADE;
1018

102-
DROP FUNCTION IF EXISTS create_water_areas_centroids_mview;
103-
104-
CREATE OR REPLACE FUNCTION create_water_areas_centroids_mview(
105-
source_table TEXT,
106-
view_name TEXT
107-
)
108-
RETURNS void AS $$
109-
DECLARE
110-
lang_columns TEXT := get_language_columns();
111-
tmp_view_name TEXT := view_name || '_tmp';
112-
sql_create TEXT;
113-
unique_columns TEXT := 'osm_id, type';
114-
BEGIN
115-
sql_create := format($sql$
116-
CREATE MATERIALIZED VIEW %I AS
117-
SELECT
118-
osm_id,
119-
NULLIF(name, '') AS name,
120-
NULLIF(type, '') AS type,
121-
NULLIF(start_date, '') AS start_date,
122-
NULLIF(end_date, '') AS end_date,
123-
isodatetodecimaldate(pad_date(start_date, 'start'), FALSE) AS start_decdate,
124-
isodatetodecimaldate(pad_date(end_date, 'end'), FALSE) AS end_decdate,
125-
area,
126-
%s,
127-
(ST_MaximumInscribedCircle(geometry)).center AS geometry
128-
FROM %I
129-
WHERE name IS NOT NULL AND name <> '';
130-
$sql$, tmp_view_name, lang_columns, source_table);
131-
132-
PERFORM finalize_materialized_view(
133-
tmp_view_name,
134-
view_name,
135-
unique_columns,
136-
sql_create
137-
);
138-
END;
139-
$$ LANGUAGE plpgsql;
9+
-- Zoom levels 16-20: Base view with full detail, no simplification
10+
SELECT create_areas_mview('osm_water_areas','mv_water_areas_z16_20',0,0,'id, osm_id, type');
14011

141-
-- ============================================================================
142-
-- Create materialized views for water ceontroids
143-
-- ============================================================================
144-
SELECT create_water_areas_centroids_mview('osm_water_areas_z0_2', 'mv_water_areas_centroids_z0_2');
145-
SELECT create_water_areas_centroids_mview('osm_water_areas_z3_5', 'mv_water_areas_centroids_z3_5');
146-
SELECT create_water_areas_centroids_mview('osm_water_areas_z6_7', 'mv_water_areas_centroids_z6_7');
147-
SELECT create_water_areas_centroids_mview('osm_water_areas_z8_9', 'mv_water_areas_centroids_z8_9');
148-
SELECT create_water_areas_centroids_mview('osm_water_areas_z10_12', 'mv_water_areas_centroids_z10_12');
149-
SELECT create_water_areas_centroids_mview('osm_water_areas_z13_15', 'mv_water_areas_centroids_z13_20');
12+
-- Zoom levels 13-15: Light simplification (5m tolerance), no area filter
13+
SELECT create_area_mview_from_mview('mv_water_areas_z16_20','mv_water_areas_z13_15',5,0.0,NULL);
14+
15+
-- Zoom levels 10-12: Moderate simplification (20m tolerance), min area 100
16+
SELECT create_area_mview_from_mview('mv_water_areas_z13_15','mv_water_areas_z10_12',20,100, 'type IN (''water'',''pond'',''basin'',''canal'',''mill_pond'',''riverbank'')');
17+
18+
-- Zoom levels 8-9: Higher simplification (100m tolerance), min area 10,000
19+
SELECT create_area_mview_from_mview('mv_water_areas_z10_12','mv_water_areas_z8_9',100,10000, NULL);
20+
21+
-- Zoom levels 6-7: Very high simplification (200m tolerance), min area 1,000,000
22+
SELECT create_area_mview_from_mview('mv_water_areas_z8_9','mv_water_areas_z6_7',200,1000000, NULL);
23+
24+
-- Zoom levels 3-5: Extreme simplification (1000m tolerance), min area 50,000,000
25+
SELECT create_area_mview_from_mview('mv_water_areas_z6_7','mv_water_areas_z3_5',1000,50000000, NULL);
26+
27+
-- Zoom levels 0-2: Maximum simplification (5000m tolerance), min area 100,000,000
28+
SELECT create_area_mview_from_mview('mv_water_areas_z3_5','mv_water_areas_z0_2',5000,100000000, 'type IN (''water'',''riverbank'')');
15029

15130
-- ============================================================================
152-
-- Create materialized views for water areas using subdivided geometries and generic function
31+
-- Water Areas Centroids Materialized Views for Multiple Zoom Levels
15332
-- ============================================================================
154-
SELECT create_water_areas_subdivided_mview('osm_water_areas_z0_2', 'mv_water_areas_z0_2_subdivided');
155-
SELECT create_water_areas_subdivided_mview('osm_water_areas_z3_5', 'mv_water_areas_z3_5_subdivided');
156-
SELECT create_water_areas_subdivided_mview('osm_water_areas_z6_7', 'mv_water_areas_z6_7_subdivided');
157-
SELECT create_water_areas_subdivided_mview('osm_water_areas_z8_9', 'mv_water_areas_z8_9_subdivided');
158-
SELECT create_generic_mview('osm_water_areas_z10_12', 'mv_water_areas_z10_12', ARRAY['osm_id', 'type']);
159-
SELECT create_generic_mview('osm_water_areas_z13_15', 'mv_water_areas_z13_15', ARRAY['osm_id', 'type']);
160-
SELECT create_generic_mview('osm_water_areas', 'mv_water_areas_z16_20', ARRAY['osm_id', 'type']);
33+
select create_mview_centroid_from_mview('mv_water_areas_z16_20','mv_water_areas_centroids_z16_20', 'name IS NOT NULL AND name <> ''''');
34+
select create_mview_centroid_from_mview('mv_water_areas_z13_15','mv_water_areas_centroids_z13_15', 'name IS NOT NULL AND name <> ''''');
35+
select create_mview_centroid_from_mview('mv_water_areas_z10_12','mv_water_areas_centroids_z10_12', 'name IS NOT NULL AND name <> ''''');
36+
select create_mview_centroid_from_mview('mv_water_areas_z8_9','mv_water_areas_centroids_z8_9', 'name IS NOT NULL AND name <> ''''');
37+
select create_mview_centroid_from_mview('mv_water_areas_z6_7','mv_water_areas_centroids_z6_7', 'name IS NOT NULL AND name <> ''''');
38+
select create_mview_centroid_from_mview('mv_water_areas_z3_5','mv_water_areas_centroids_z3_5', 'name IS NOT NULL AND name <> ''''');
39+
select create_mview_centroid_from_mview('mv_water_areas_z0_2','mv_water_areas_centroids_z0_2', 'name IS NOT NULL AND name <> ''''');
40+
16141

16242
-- ============================================================================
163-
-- Create materialized views for water lines
43+
-- Water lines Materialized Views for Multiple Zoom Levels
16444
-- ============================================================================
165-
SELECT create_generic_mview('osm_water_lines_z8_9', 'mv_water_lines_z8_9');
166-
SELECT create_generic_mview('osm_water_lines_z10_12', 'mv_water_lines_z10_12');
167-
SELECT create_generic_mview('osm_water_lines_z13_15', 'mv_water_lines_z13_15');
168-
SELECT create_generic_mview('osm_water_lines_z16_20', 'mv_water_lines_z16_20');
45+
46+
SELECT create_lines_mview('osm_water_lines', 'mv_water_lines_z16_20', 0, 0, 'id, osm_id, type', 'type IN (''river'', ''canal'', ''cliff'', ''dam'', ''stream'', ''ditch'', ''drain'')');
47+
SELECT create_mview_line_from_mview('mv_water_lines_z16_20', 'mv_water_lines_z13_15', 5, 'type IN (''river'', ''canal'', ''cliff'', ''dam'', ''stream'')');
48+
SELECT create_mview_line_from_mview('mv_water_lines_z13_15', 'mv_water_lines_z10_12', 20, 'type IN (''river'', ''canal'', ''cliff'', ''dam'')');
49+
SELECT create_mview_line_from_mview('mv_water_lines_z10_12', 'mv_water_lines_z8_9', 100, 'type IN (''river'', ''canal'')');
50+
51+
52+
-- Refresh areas views
53+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_areas_z16_20;
54+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_areas_z13_15;
55+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_areas_z10_12;
56+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_areas_z8_9;
57+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_areas_z6_7;
58+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_areas_z3_5;
59+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_areas_z0_2;
60+
61+
-- Refresh centroids views
62+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_areas_centroids_z16_20;
63+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_areas_centroids_z13_15;
64+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_areas_centroids_z10_12;
65+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_areas_centroids_z8_9;
66+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_areas_centroids_z6_7;
67+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_areas_centroids_z3_5;
68+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_areas_centroids_z0_2;
69+
70+
-- Refresh lines views
71+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_lines_z16_20
72+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_lines_z13_15
73+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_lines_z10_12
74+
-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_water_lines_z8_9

0 commit comments

Comments
 (0)