Skip to content

New Features #48

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pinkpenguinn wants to merge 1 commit into
base: master
Choose a base branch
from
Open

New Features #48

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
190 changes: 95 additions & 95 deletions geo_heatmap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,17 @@
import ijson
import json
import os
from geopy import Nominatim

@luka1199 luka1199 Feb 10, 2020

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Make sure to add this to the requirements.txt

from progressbar import ProgressBar, Bar, ETA, Percentage
from utils import *
import webbrowser
from xml.etree import ElementTree
from xml.dom import minidom
import zipfile
import datetime
from folium.plugins import HeatMapWithTime
import pandas as pd
from folium.plugins import MarkerCluster


class Generator:
Expand All @@ -37,11 +42,15 @@ def loadJSONData(self, json_file, date_range):
for i, loc in enumerate(data["locations"]):
if "latitudeE7" not in loc or "longitudeE7" not in loc:
continue

timestamp = int(loc['timestampMs'])
year = datetime.datetime.fromtimestamp(timestamp / 1000).year
coords = (round(loc["latitudeE7"] / 1e7, 6),
round(loc["longitudeE7"] / 1e7, 6))
round(loc["longitudeE7"] / 1e7, 6), year)

if timestampInRange(loc["timestampMs"], date_range):
if timestampInRange(loc['timestampMs'], date_range):
self.updateCoord(coords)

pb.update(i)

def streamJSONData(self, json_file, date_range):
Expand All @@ -56,19 +65,19 @@ def streamJSONData(self, json_file, date_range):
# Estimate location amount
max_value_est = sum(1 for line in json_file) / 13
json_file.seek(0)

locations = ijson.items(json_file, "locations.item")
w = [Bar(), Percentage(), " ", ETA()]
with ProgressBar(max_value=max_value_est, widgets=w) as pb:
for i, loc in enumerate(locations):
if "latitudeE7" not in loc or "longitudeE7" not in loc:
continue
coords = (round(loc["latitudeE7"] / 1e7, 6),
round(loc["longitudeE7"] / 1e7, 6))
round(loc["longitudeE7"] / 1e7, 6))

if timestampInRange(loc["timestampMs"], date_range):
if timestampInRange(loc['timestampMs'], date_range):
self.updateCoord(coords)

if i > max_value_est:
max_value_est = i
pb.max_value = i
Expand Down Expand Up @@ -97,29 +106,6 @@ def loadKMLData(self, file_name, date_range):
self.updateCoord(coords)
pb.update(i)

def loadGPXData(self, file_name, date_range):
"""Loads location data from the given GPX file.

Arguments:
file_name {string or file} -- The name of the GPX file
(or an open file-like object) with the GPX data.
date_range {tuple} -- A tuple containing the min-date and max-date.
e.g.: (None, None), (None, '2019-01-01'), ('2017-02-11'), ('2019-01-01')
"""
xmldoc = minidom.parse(file_name)
gxtrack = xmldoc.getElementsByTagName("trkpt")
w = [Bar(), Percentage(), " ", ETA()]

with ProgressBar(max_value=len(gxtrack), widgets=w) as pb:
for i, trkpt in enumerate(gxtrack):
lat = trkpt.getAttribute("lat")
lon = trkpt.getAttribute("lon")
coords = (round(float(lat), 6), round(float(lon), 6))
date = trkpt.getElementsByTagName("time")[0].firstChild.data
if dateInRange(date[:10], date_range):
self.updateCoord(coords)
pb.update(i)

Comment on lines -100 to -122

@luka1199 luka1199 Feb 10, 2020

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any reason you are removing this?

def loadZIPData(self, file_name, date_range):
"""
Load Google location data from a "takeout-*.zip" file.
Expand All @@ -138,7 +124,8 @@ def loadZIPData(self, file_name, date_range):
with zip_file.open(html_path) as read_file:
soup = BeautifulSoup(read_file, "html.parser")
(elem,) = soup.select(
"#service-tile-LOCATION_HISTORY > button > div.service_summary > div > h1[data-english-name=LOCATION_HISTORY]")
"#service-tile-LOCATION_HISTORY > button > div.service_summary > div > h1["
"data-english-name=LOCATION_HISTORY]")
name = elem["data-folder-name"]
(data_path,) = fnmatch.filter(
namelist,
Expand All @@ -154,66 +141,99 @@ def loadZIPData(self, file_name, date_range):
self.loadKMLData(read_file, date_range)
else:
raise ValueError("unsupported extension for {!r}: only .json and .kml supported"
.format(file_name))
.format(file_name))

def updateCoord(self, coords):
self.coordinates[coords] += 1
if self.coordinates[coords] > self.max_magnitude:
self.max_coordinates = coords
self.max_coordinates = coords[0], coords[1]
self.max_magnitude = self.coordinates[coords]

def generateMap(self, settings):
"""Generates the heatmap.

Arguments:
settings {dict} -- The settings for the heatmap.

Returns:
Map -- The Heatmap.
"""
tiles = settings["tiles"]
zoom_start = settings["zoom_start"]
radius = settings["radius"]
blur = settings["blur"]
min_opacity = settings["min_opacity"]
max_zoom = settings["max_zoom"]

def generateMap(self, tiles, map_zoom_start=6, heatmap_radius=7,
heatmap_blur=4, heatmap_min_opacity=0.2,
heatmap_max_zoom=4):

Comment on lines -165 to +155

@luka1199 luka1199 Feb 10, 2020

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are you removing the documentation and changing it back here? Make sure you are using the latest version

map_data = [(coords[0], coords[1], magnitude)
for coords, magnitude in self.coordinates.items()]

map_data_with_year = [(coords[0], coords[1], magnitude, coords[2])
for coords, magnitude in self.coordinates.items()]

df_data = pd.DataFrame(map_data_with_year, columns=['Lat', 'Long', 'Weight', 'Year'])

year_list = df_data.Year.sort_values().unique()

map_data_by_year = []
for year in year_list:
map_data_by_year.append(df_data.loc[df_data.Year == year,
['Lat', 'Long', 'Weight']].groupby(
['Lat', 'Long']).sum().reset_index().values.tolist())

# Generate map
m = folium.Map(location=self.max_coordinates,
zoom_start=zoom_start,
zoom_start=map_zoom_start,
tiles=tiles)

world_imagery = "https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}"

folium.TileLayer(tiles=world_imagery, name="World Imagery",
attr='Tiles © Esri — Source: Esri, i-cubed, USDA, USGS, AEX, GeoEye, Getmapping, Aerogrid, IGN, IGP, UPR-EGP, and the GIS User Community').add_to(
m)
folium.TileLayer(tiles='Stamen Terrain', name='Stamen Terrain').add_to(m)
folium.TileLayer('cartodbdark_matter', name='Dark Matter').add_to(m)

# Generate heat map

heatmap = HeatMap(map_data,
max_val=self.max_magnitude,
min_opacity=min_opacity,
radius=radius,
blur=blur,
max_zoom=max_zoom)
min_opacity=heatmap_min_opacity,
radius=heatmap_radius,
blur=heatmap_blur,
max_zoom=heatmap_max_zoom).add_to(folium.FeatureGroup(name="Combined HeatMap").add_to(m))

locator = Nominatim(user_agent="geocoder", timeout=None)

timesVisited = [x[2] for x in map_data]
timesVisited.sort(reverse=True)
top20Visited = timesVisited[19]

marker_group = folium.FeatureGroup(name="Markers")
mc = MarkerCluster()

for lat, lon, freq in map_data:
if freq >= top20Visited:
coordinates = lat, lon
location = locator.reverse(coordinates)

times_visited = location.address + "\nVisited " + str(freq) + " times"

mc.add_child(folium.Marker(location=[lat, lon], popup=times_visited))


folium.plugins.HeatMapWithTime(map_data_by_year, name = "HeatMap by Year", index= year_list.tolist(),
min_opacity=heatmap_min_opacity,
radius=heatmap_radius,
gradient={0.2: 'blue', 0.4: 'lime', 0.6: 'orange', 1: 'red'}).add_to(m)

mc.add_to(marker_group)
marker_group.add_to(m)

folium.LayerControl().add_to(m)

m.add_child(heatmap)
return m

def run(self, data_files, output_file, date_range, stream_data, settings):
def run(self, data_files, output_file, date_range, stream_data, tiles):
"""Load the data, generate the heatmap and save it.

Arguments:
data_files {list} -- List of names of the data files with the Google
location data or the Google takeout ZIP archive.
output_file {string} -- The name of the output file.
date_range {tuple} -- A tuple containing the min-date and max-date.
e.g.: (None, None), (None, '2019-01-01'), ('2017-02-11'), ('2019-01-01')
stream_data {bool} -- Stream option.
settings {dict} -- The settings for the heatmap.
"""

for i, data_file in enumerate(data_files):
print("\n({}/{}) Loading data from {}".format(
i + 1,
len(data_files) + 2,
i + 1,
len(data_files) + 2,
data_file))
if data_file.endswith(".zip"):
self.loadZIPData(data_file, date_range)
Expand All @@ -225,16 +245,14 @@ def run(self, data_files, output_file, date_range, stream_data, settings):
self.loadJSONData(json_file, date_range)
elif data_file.endswith(".kml"):
self.loadKMLData(data_file, date_range)
elif data_file.endswith(".gpx"):
self.loadGPXData(data_file, date_range)
else:
raise NotImplementedError(
"Unsupported file extension for {!r}".format(data_file))
print("\n({}/{}) generateMapGenerating heatmap".format(
len(data_files) + 1,

print("\n({}/{}) Generating heatmap".format(
len(data_files) + 1,
len(data_files) + 2))
m = self.generateMap(settings)
m = self.generateMap(tiles)
print("\n({}/{}) Saving map to {}\n".format(
len(data_files) + 2,
len(data_files) + 2,
Expand All @@ -245,12 +263,12 @@ def run(self, data_files, output_file, date_range, stream_data, settings):
if __name__ == "__main__":
parser = ArgumentParser(formatter_class=RawTextHelpFormatter)
parser.add_argument(
"files", metavar="file", type=str, nargs="+", help="Any of the following files:\n"
"- Your location history JSON file from Google Takeout\n"
"- Your location history KML file from Google Takeout\n"
"- The takeout-*.zip raw download from Google Takeout \nthat contains either of the above files\n"
"- A GPX file containing GPS tracks")
parser.add_argument("-o", "--output", dest="output", type=str, required=False,
"files", metavar="file", type=str, nargs='+', help="Any of the following files:\n"
"1. Your location history JSON file from Google Takeout\n"
"2. Your location history KML file from Google Takeout\n"
"3. The takeout-*.zip raw download from Google Takeout "
"\nthat contains either of the above files")
parser.add_argument("-o", "--output", dest="output", metavar="", type=str, required=False,
help="Path of heatmap HTML output file.", default="heatmap.html")
parser.add_argument("--min-date", dest="min_date", metavar="YYYY-MM-DD", type=str, required=False,
help="The earliest date from which you want to see data in the heatmap.")
Expand All @@ -259,35 +277,17 @@ def run(self, data_files, output_file, date_range, stream_data, settings):
parser.add_argument("-s", "--stream", dest="stream", action="store_true", help="Option to iteratively load data.")
parser.add_argument("--map", "-m", dest="map", metavar="MAP", type=str, required=False, default="OpenStreetMap",
help="The name of the map tiles you want to use.\n" \
"(e.g. 'OpenStreetMap', 'StamenTerrain', 'StamenToner', 'StamenWatercolor')")
parser.add_argument("-z", "--zoom-start", dest="zoom_start", type=int, required=False,
help="The initial zoom level for the map. (default: %(default)s)", default=6)
parser.add_argument("-r", "--radius", type=int, required=False,
help="The radius of each location point. (default: %(default)s)", default=7)
parser.add_argument("-b", "--blur", type=int, required=False,
help="The amount of blur. (default: %(default)s)", default=4)
parser.add_argument("-mo", "--min-opacity", dest="min_opacity", type=float, required=False,
help="The minimum opacity of the heatmap. (default: %(default)s)", default=0.2)
parser.add_argument("-mz", "--max-zoom", dest="max_zoom", type=int, required=False,
help="The maximum zoom of the heatmap. (default: %(default)s)", default=4)

"(e.g. 'OpenStreetMap', 'StamenTerrain', 'StamenToner', 'StamenWatercolor')")

args = parser.parse_args()
data_file = args.files
output_file = args.output
date_range = args.min_date, args.max_date
tiles = args.map
stream_data = args.stream
settings = {
"tiles": args.map,
"zoom_start": args.zoom_start,
"radius": args.radius,
"blur": args.blur,
"min_opacity": args.min_opacity,
"max_zoom": args.max_zoom
}

generator = Generator()
generator.run(data_file, output_file, date_range, stream_data, settings)
generator.run(data_file, output_file, date_range, stream_data, tiles)
# Check if browser is text-based
if not isTextBasedBrowser(webbrowser.get()):
try:
Expand Down