Skip to content

Commit 6fad57a

Browse files
eserteeserte
committed
mapillary-mvt-fetch and helper scripts
Helper scripts are: - parse_mvt_sequences.py: convert vectortile protobuf data into a friendly json format using mapbox_vector_tile - parse_mvt_sequences.pl: the same in Perl, using either Google::ProtocolBuffers or Google::ProtocolBuffers::Dynamic - a copy of vector_tile.proto (needed by the Perl protobuf modules) Aims at replacing mapillary-v4-fetch due to new bounding box size restrictions.
1 parent 4ef9a41 commit 6fad57a

File tree

4 files changed

+719
-0
lines changed

4 files changed

+719
-0
lines changed

‎misc/vector_tile.proto‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
package vector_tile;
2+
3+
option optimize_for = LITE_RUNTIME;
4+
5+
message Tile {
6+
7+
// GeomType is described in section 4.3.4 of the specification
8+
enum GeomType {
9+
UNKNOWN = 0;
10+
POINT = 1;
11+
LINESTRING = 2;
12+
POLYGON = 3;
13+
}
14+
15+
// Variant type encoding
16+
// The use of values is described in section 4.1 of the specification
17+
message Value {
18+
// Exactly one of these values must be present in a valid message
19+
optional string string_value = 1;
20+
optional float float_value = 2;
21+
optional double double_value = 3;
22+
optional int64 int_value = 4;
23+
optional uint64 uint_value = 5;
24+
optional sint64 sint_value = 6;
25+
optional bool bool_value = 7;
26+
27+
extensions 8 to max;
28+
}
29+
30+
// Features are described in section 4.2 of the specification
31+
message Feature {
32+
optional uint64 id = 1 [ default = 0 ];
33+
34+
// Tags of this feature are encoded as repeated pairs of
35+
// integers.
36+
// A detailed description of tags is located in sections
37+
// 4.2 and 4.4 of the specification
38+
repeated uint32 tags = 2 [ packed = true ];
39+
40+
// The type of geometry stored in this feature.
41+
optional GeomType type = 3 [ default = UNKNOWN ];
42+
43+
// Contains a stream of commands and parameters (vertices).
44+
// A detailed description on geometry encoding is located in
45+
// section 4.3 of the specification.
46+
repeated uint32 geometry = 4 [ packed = true ];
47+
}
48+
49+
// Layers are described in section 4.1 of the specification
50+
message Layer {
51+
// Any compliant implementation must first read the version
52+
// number encoded in this message and choose the correct
53+
// implementation for this version number before proceeding to
54+
// decode other parts of this message.
55+
required uint32 version = 15 [ default = 1 ];
56+
57+
required string name = 1;
58+
59+
// The actual features in this tile.
60+
repeated Feature features = 2;
61+
62+
// Dictionary encoding for keys
63+
repeated string keys = 3;
64+
65+
// Dictionary encoding for values
66+
repeated Value values = 4;
67+
68+
// Although this is an "optional" field it is required by the specification.
69+
// See https://github.com/mapbox/vector-tile-spec/issues/47
70+
optional uint32 extent = 5 [ default = 4096 ];
71+
72+
extensions 16 to max;
73+
}
74+
75+
repeated Layer layers = 3;
76+
77+
extensions 16 to 8191;
78+
}

‎miscsrc/mapillary-mvt-fetch‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
#!/usr/bin/env perl
2+
# -*- perl -*-
3+
4+
#
5+
# Author: Slaven Rezic
6+
#
7+
# Copyright (C) 2025 Slaven Rezic. All rights reserved.
8+
# This program is free software; you can redistribute it and/or
9+
# modify it under the same terms as Perl itself.
10+
#
11+
# WWW: https://github.com/eserte/bbbike
12+
#
13+
14+
use strict;
15+
use warnings;
16+
use FindBin;
17+
18+
use LWP::UserAgent;
19+
use JSON;
20+
use lib "$FindBin::RealBin/..";
21+
use BBBikeYAML qw(LoadFile);
22+
23+
use POSIX qw(floor strftime);
24+
use Math::Trig;
25+
26+
sub latlon_to_tile {
27+
my ($lat, $lon, $zoom) = @_;
28+
my $lat_rad = deg2rad($lat);
29+
my $n = 2 ** $zoom;
30+
my $x_tile = floor(( $lon + 180.0 ) / 360.0 * $n);
31+
my $y_tile = floor((1 - (log(tan($lat_rad) + 1 / cos($lat_rad)) / pi())) / 2 * $n);
32+
return ($x_tile, $y_tile);
33+
}
34+
35+
# like in mapillary-v4-fetch
36+
my $do_cache = 1;
37+
my $cache_time = 8 * 3600; # seconds
38+
39+
my $conf_file = "$ENV{HOME}/.mapillary";
40+
my $conf = LoadFile $conf_file;
41+
my $client_token = $conf->{client_token} || die "Can't get client_token from $conf_file";
42+
43+
my @tiles = ();
44+
my $z = 12;
45+
46+
# XXX get from Geography::Berlin_DE or provide by option
47+
my $min_lon = 13.051179;
48+
my $max_lon = 13.764158;
49+
my $min_lat = 52.337621;
50+
my $max_lat = 52.689878;
51+
52+
my $date_range = shift;
53+
my $date_from = "";
54+
my $date_to = "";
55+
if ($date_range) {
56+
if ($date_range =~ /^(\d{8})(?:-|\.\.)(\d{8})?$/) {
57+
$date_from = $1;
58+
$date_to = $2 if defined $2;
59+
} else {
60+
die "Wrong date range syntax, must be: YYYYMMDD- or YYYYMMDD-YYYYMMDD.\n";
61+
}
62+
}
63+
if (!$date_from) {
64+
$date_from = strftime "%Y%m%d", localtime(time-86400);
65+
warn "INFO: date range not given, default to $date_from-$date_to\n";
66+
}
67+
die "Too many arguments" if @ARGV;
68+
69+
my ($min_x, $min_y) = latlon_to_tile($min_lat, $min_lon, $z);
70+
my ($max_x, $max_y) = latlon_to_tile($max_lat, $max_lon, $z);
71+
72+
($min_x,$max_x) = ($max_x,$min_x) if $max_x < $min_x;
73+
($min_y,$max_y) = ($max_y,$min_y) if $max_y < $min_y;
74+
75+
for my $x ($min_x .. $max_x) {
76+
for my $y ($min_y .. $max_y) {
77+
push @tiles, [$z,$x,$y];
78+
}
79+
}
80+
81+
warn "INFO: we need to fetch and process " . scalar(@tiles) . " tile(s)...\n";
82+
83+
my $ua;
84+
if ($do_cache) {
85+
if (!eval { require LWP::UserAgent::WithCache; require HTTP::Date; 1 }) {
86+
die "Module missing, please install. Error: $@";
87+
}
88+
# need to patch set_cache method
89+
my $orig_set_cache = \&LWP::UserAgent::WithCache::set_cache;
90+
{
91+
no warnings 'redefine';
92+
*LWP::UserAgent::WithCache::set_cache = sub {
93+
my($self, $uri, $res) = @_;
94+
95+
if ($res->header('X-Died')) {
96+
warn "X-Died header encountered, do not write to cache...\n";
97+
return;
98+
}
99+
if ($res->header('Client-Aborted')) {
100+
warn "Client-Aborted header encountered, do not write to cache...\n";
101+
return;
102+
}
103+
104+
my $expires = time + $cache_time;
105+
my $expires_formatted = HTTP::Date::time2str($expires);
106+
$res->header('Expires', $expires_formatted);
107+
108+
$orig_set_cache->($self, $uri, $res);
109+
};
110+
}
111+
my %cache_opt = (
112+
'namespace' => 'lwp-cache',
113+
'cache_root' => "$ENV{HOME}/.cache",
114+
'default_expires_in' => $cache_time,
115+
);
116+
$ua = LWP::UserAgent::WithCache->new(\%cache_opt);
117+
} else {
118+
$ua = LWP::UserAgent->new(keep_alive => 1);
119+
}
120+
121+
$ua->agent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0");
122+
123+
print <<'EOF';
124+
#: map: polar
125+
#: line_arrow: last
126+
#:
127+
EOF
128+
129+
for my $tile (@tiles) {
130+
my ($z, $x, $y) = @$tile;
131+
my $file = "/tmp/tile_${z}_${x}_${y}.mvt";
132+
warn "INFO: fetch and process $file...\n";
133+
#if (!-s $file) { # always use cached version
134+
{
135+
my $url = "https://tiles.mapillary.com/maps/vtp/mly1_public/2/$z/$x/$y?access_token=$client_token";
136+
my $response = $ua->get($url);
137+
if ($response->is_success) {
138+
my $content = $response->content;
139+
# XXX workaround for HTTP::Response bug https://github.com/libwww-perl/HTTP-Message/issues/48
140+
if ($do_cache && $HTTP::Response::VERSION <= 7.00) {
141+
$content =~ s{\n$}{};
142+
}
143+
open my $fh, '>', "$file~" or die $!;
144+
binmode $fh;
145+
print $fh $content;
146+
close $fh;
147+
rename "$file~", $file or die $!;
148+
} else {
149+
warn "ERROR: Failed for $url:\n" . $response->dump . "\n";
150+
next;
151+
}
152+
}
153+
154+
# Call Python script
155+
# my $json = `python3 $FindBin::RealBin/parse_mvt_sequences.py $file "$date_from" "$date_to"`;
156+
my $json = `$^X $FindBin::RealBin/parse_mvt_sequences.pl $file "$date_from" "$date_to"`;
157+
my $results = eval { decode_json($json) };
158+
if ($results && ref $results eq 'ARRAY') {
159+
for my $seq (@$results) {
160+
print "#: url: $seq->{url}\n";
161+
print "start_captured_at=$seq->{start_captured_at} creator=$seq->{creator} make=$seq->{make} end_captured_at=$seq->{end_captured_at} start_id=$seq->{start_id} sequence=$seq->{sequence}\tX " . join(" ", map { "$_->[1],$_->[0]" } @{ $seq->{coordinates} }) . "\n";
162+
}
163+
}
164+
# keep cached file! unlink $file; # Optional: cleanup
165+
}
166+
167+
__END__

0 commit comments

Comments
 (0)