From ca02120a8b326c7d4a3a556abc80badb0ec4de06 Mon Sep 17 00:00:00 2001 From: Kayra Date: Tue, 7 Feb 2023 18:09:01 -0800 Subject: [PATCH 1/4] skeleton for ingesting depremyardim --- applications/feeds/models.py | 1 + applications/feeds/tasks.py | 49 ++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/applications/feeds/models.py b/applications/feeds/models.py index f627de0..ce7640c 100644 --- a/applications/feeds/models.py +++ b/applications/feeds/models.py @@ -7,6 +7,7 @@ class Entry(models.Model): ("telegram", "telegram"), ("twitch", "twitch"), ("discord", "discord"), + ("depremyardim", "depremyardim") ) full_text = models.TextField() diff --git a/applications/feeds/tasks.py b/applications/feeds/tasks.py index 95a6e43..2ac3ba4 100644 --- a/applications/feeds/tasks.py +++ b/applications/feeds/tasks.py @@ -1,5 +1,6 @@ from trquake.celery import app from typing import List, Dict, Union +import requests, os, json from core.address_api import AddressAPI from feeds.models import Entry, Location from feeds.serializers import BulkEntrySerializer @@ -40,3 +41,51 @@ def write_bulk_entries(entries: List[Dict[str, Union[str, bool]]]): if serializer.is_valid(): entry: Entry = serializer.save() process_entry(entry_id=entry.id) + + +@app.task +def get_all_depremyardim(): + + # 1. Tum datayi depremyardimdan get TODO: Multithread? + data, new_data, i = [], {}, 1 + while len(new_data.get("data", [""])) > 0: + req = requests.get( + url = "https://depremyardim.com/api/list", + params= { + "X-AUTH-KEY": os.environ['DEPREM_YARDIM_AUTH_KEY'], + "page": i, + "per_page": "1000" + } + ) + new_data = json.loads(req.text) + data.extend(new_data["data"]['data']) + i+=1 + + # TODO: Son kaldigimiz page'i kaydet ve her cron run da ordan devam et + # 2. Databaseteki duplicatelari bul + # TODO: extra_parameters le merge_conflict'leri bul ve yeni datadan sil + + # 3. Geri kalan datalar icin yeni entryler olustur + for row in data: + new_entry = Entry( + full_text = json.dumps(row), + is_resolved = False, + channel = "depremyardim", + extra_parameters = json.dumps(row) + ) + new_entry.save() + # TODO: Bu islem nasil calisiyor tam olarak + new_location = AddressAPI.regex_api_request() #??? + if new_location.get("is_resolved", False): + new_entry.is_resolved = True + new_entry.save() + Location.objects.create( + entry=new_entry, + latitude=new_location["latitude"], + longitude=new_location["longitude"], + northeast_lat=new_location["northeast_lat"], + northeast_lng=new_location["northeast_lng"], + southwest_lat=new_location["southwest_lat"], + southwest_lng=new_location["southwest_lng"], + formatted_address=new_location["formatted_address"], + ) \ No newline at end of file From bd3537d531fbd3d87db1ebcab0fbdc64b5b62703 Mon Sep 17 00:00:00 2001 From: Kayra Date: Wed, 8 Feb 2023 08:09:46 -0800 Subject: [PATCH 2/4] model degistirme --- applications/feeds/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/applications/feeds/models.py b/applications/feeds/models.py index ce7640c..6f13293 100644 --- a/applications/feeds/models.py +++ b/applications/feeds/models.py @@ -6,8 +6,7 @@ class Entry(models.Model): ("twitter", "twitter"), ("telegram", "telegram"), ("twitch", "twitch"), - ("discord", "discord"), - ("depremyardim", "depremyardim") + ("discord", "discord") ) full_text = models.TextField() From 5a0e949dcc69a97df41588ce16aa1409f053e51b Mon Sep 17 00:00:00 2001 From: Kayra Date: Wed, 8 Feb 2023 08:10:16 -0800 Subject: [PATCH 3/4] degisikliksiz --- applications/feeds/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/feeds/models.py b/applications/feeds/models.py index 6f13293..f627de0 100644 --- a/applications/feeds/models.py +++ b/applications/feeds/models.py @@ -6,7 +6,7 @@ class Entry(models.Model): ("twitter", "twitter"), ("telegram", "telegram"), ("twitch", "twitch"), - ("discord", "discord") + ("discord", "discord"), ) full_text = models.TextField() From 3dba75cece5a64eeb282772c88e319ad8c9b51b4 Mon Sep 17 00:00:00 2001 From: Kayra Date: Wed, 8 Feb 2023 08:26:12 -0800 Subject: [PATCH 4/4] inputlari format et --- applications/feeds/tasks.py | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/applications/feeds/tasks.py b/applications/feeds/tasks.py index 3625eb6..5598f67 100644 --- a/applications/feeds/tasks.py +++ b/applications/feeds/tasks.py @@ -1,8 +1,8 @@ # Standard Library from typing import Dict, List, Union +import requests, os, json # Applications -import requests, os, json from core.address_api import AddressAPI from feeds.models import Entry, Location from feeds.serializers import BulkEntrySerializer @@ -59,7 +59,7 @@ def get_all_depremyardim(): params= { "X-AUTH-KEY": os.environ['DEPREM_YARDIM_AUTH_KEY'], "page": i, - "per_page": "1000" + "per_page": "10000" } ) new_data = json.loads(req.text) @@ -72,25 +72,14 @@ def get_all_depremyardim(): # 3. Geri kalan datalar icin yeni entryler olustur for row in data: + full_address = f"{row['street']} {row['address']} {row['district']} {row['city']}" new_entry = Entry( - full_text = json.dumps(row), - is_resolved = False, + full_text = full_address, + is_resolved = True, channel = "depremyardim", extra_parameters = json.dumps(row) ) new_entry.save() - # TODO: Bu islem nasil calisiyor tam olarak - new_location = AddressAPI.regex_api_request() #??? - if new_location.get("is_resolved", False): - new_entry.is_resolved = True - new_entry.save() - Location.objects.create( - entry=new_entry, - latitude=new_location["latitude"], - longitude=new_location["longitude"], - northeast_lat=new_location["northeast_lat"], - northeast_lng=new_location["northeast_lng"], - southwest_lat=new_location["southwest_lat"], - southwest_lng=new_location["southwest_lng"], - formatted_address=new_location["formatted_address"], - ) \ No newline at end of file + process_entry(entry_id=new_entry.id) + +get_all_depremyardim() \ No newline at end of file