Skip to content

Commit c03d30e

Browse files
1kaiser1kaiser
committed
Created using Colaboratory
1 parent 4844d98 commit c03d30e

1 file changed

Lines changed: 217 additions & 0 deletions

File tree

‎ECOSTRESS_LST_dataDownloaderParallel.ipynb‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
{
2+
"nbformat": 4,
3+
"nbformat_minor": 0,
4+
"metadata": {
5+
"colab": {
6+
"provenance": [],
7+
"authorship_tag": "ABX9TyPh0CJZpeGYASIKV8RQMGWN",
8+
"include_colab_link": true
9+
},
10+
"kernelspec": {
11+
"name": "python3",
12+
"display_name": "Python 3"
13+
},
14+
"language_info": {
15+
"name": "python"
16+
}
17+
},
18+
"cells": [
19+
{
20+
"cell_type": "markdown",
21+
"metadata": {
22+
"id": "view-in-github",
23+
"colab_type": "text"
24+
},
25+
"source": [
26+
"<a href=\"https://colab.research.google.com/github/1kaiser/Jodhpur_ecostress_LST/blob/main/ECOSTRESS_LST_dataDownloaderParallel.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
27+
]
28+
},
29+
{
30+
"cell_type": "markdown",
31+
"source": [
32+
"# **🛰️ ECOSTRESS DATA DOWNLOAD📡**"
33+
],
34+
"metadata": {
35+
"id": "36FWle_q2vZb"
36+
}
37+
},
38+
{
39+
"cell_type": "markdown",
40+
"source": [
41+
"pre requisite folder creation to keep the data downloaded into a folder"
42+
],
43+
"metadata": {
44+
"id": "n-4a71GX78ZS"
45+
}
46+
},
47+
{
48+
"cell_type": "code",
49+
"source": [
50+
"dir = r'/content/testfiles'\n",
51+
"!mkdir -p {dir}\n",
52+
"%cd {dir}"
53+
],
54+
"metadata": {
55+
"id": "cLh93IAEYs04",
56+
"outputId": "c42f0145-1211-4760-caf8-27f949d330eb",
57+
"colab": {
58+
"base_uri": "https://localhost:8080/"
59+
}
60+
},
61+
"execution_count": 1,
62+
"outputs": [
63+
{
64+
"output_type": "stream",
65+
"name": "stdout",
66+
"text": [
67+
"/content/testfiles\n"
68+
]
69+
}
70+
]
71+
},
72+
{
73+
"cell_type": "markdown",
74+
"source": [
75+
"### APPEEAR LDDAC DATA DOWNLOAD"
76+
],
77+
"metadata": {
78+
"id": "XsU-Aivn7ihA"
79+
}
80+
},
81+
{
82+
"cell_type": "markdown",
83+
"source": [
84+
"### aquiring the token from LPDAAC by using authentication userid and password"
85+
],
86+
"metadata": {
87+
"id": "0OyuN-2i7vQ_"
88+
}
89+
},
90+
{
91+
"cell_type": "code",
92+
"source": [
93+
"!curl --request POST --user kroy0001:/#j%kWrPA,8.HRe --header \"Content-Length: 0\" \"https://appeears.earthdatacloud.nasa.gov/api/login\""
94+
],
95+
"metadata": {
96+
"id": "N7PNs-Mu9ibu",
97+
"outputId": "c5143835-a4cd-43a6-f13a-d1cbf36fdb35",
98+
"colab": {
99+
"base_uri": "https://localhost:8080/"
100+
}
101+
},
102+
"execution_count": 2,
103+
"outputs": [
104+
{
105+
"output_type": "stream",
106+
"name": "stdout",
107+
"text": [
108+
"{\"token_type\": \"Bearer\", \"token\": \"E_OWymNhyv8V7af9aFpjLP0aNhpsz2cFneU0zcuUgRqjYymc61B9cLnQK_p9q3ccOq-5ji8LrIyHhpAdZAo72g\", \"expiration\": \"2023-06-15T04:02:02Z\"}\n"
109+
]
110+
}
111+
]
112+
},
113+
{
114+
"cell_type": "code",
115+
"source": [
116+
"# !curl -L -O --remote-header-name \\\n",
117+
"# --header \"Authorization: Bearer bVVLVOIv29Lds-zADthteUE_1QlykgndjN5T6BaKMzMS-A11Z8UWtVsNbAJ85LWcGGerQH1KpM7eb-1KZS_Nig\" \\\n",
118+
"# --location https://appeears.earthdatacloud.nasa.gov/api/bundle/908b9b61-5acf-48ca-933e-1fcd3b2704fc/c4d1addc-4e43-43e6-aac4-04cdcf04faca/MOD09A1.061_sur_refl_b01_doy2000129_aid0001.tif"
119+
],
120+
"metadata": {
121+
"id": "IDFusRPU9up5"
122+
},
123+
"execution_count": null,
124+
"outputs": []
125+
},
126+
{
127+
"cell_type": "markdown",
128+
"source": [
129+
"### starting the download of the list data with 10 threads parallelly"
130+
],
131+
"metadata": {
132+
"id": "Iia9EHMD72b9"
133+
}
134+
},
135+
{
136+
"cell_type": "code",
137+
"source": [
138+
"import requests\n",
139+
"import time\n",
140+
"from multiprocessing import cpu_count\n",
141+
"from multiprocessing.pool import ThreadPool\n",
142+
"from google.colab import output\n",
143+
"\n",
144+
"def download_url(url):\n",
145+
" t0 = time.time()\n",
146+
"###########################################################################################################################\n",
147+
" !curl -L -O --remote-header-name \\\n",
148+
" --header \"Authorization: Bearer E_OWymNhyv8V7af9aFpjLP0aNhpsz2cFneU0zcuUgRqjYymc61B9cLnQK_p9q3ccOq-5ji8LrIyHhpAdZAo72g\" \\\n",
149+
" --location {url}\n",
150+
"###########################################################################################################################\n",
151+
" return( time.time() - t0)\n",
152+
"\n",
153+
"t0 = time.time()\n",
154+
"\n",
155+
"def download_parallel(args):\n",
156+
" cpus = 10\n",
157+
" results = ThreadPool(cpus - 1).imap_unordered(download_url, args)\n",
158+
" for result in results:\n",
159+
" print('time (s):', result)\n",
160+
" output.clear()\n",
161+
"###########################################################################################################################\n",
162+
"file1 = open(\"/content/url.txt\", 'r')\n",
163+
"###########################################################################################################################\n",
164+
"download_parallel(file1)\n"
165+
],
166+
"metadata": {
167+
"id": "rtfs6--L7RMf"
168+
},
169+
"execution_count": 3,
170+
"outputs": []
171+
},
172+
{
173+
"cell_type": "markdown",
174+
"source": [
175+
"### zipping the tiff files"
176+
],
177+
"metadata": {
178+
"id": "i-0IkxIl7c4N"
179+
}
180+
},
181+
{
182+
"cell_type": "code",
183+
"source": [
184+
"total_files = r'/content/testfiles'\n",
185+
"\n",
186+
"%cd {total_files}\n",
187+
"!zip -r folder.zip {total_files}\n",
188+
"!mv /content/testfiles/folder.zip /content/"
189+
],
190+
"metadata": {
191+
"id": "WvDfex2wHsGz"
192+
},
193+
"execution_count": null,
194+
"outputs": []
195+
},
196+
{
197+
"cell_type": "code",
198+
"source": [
199+
"# from google.colab import output\n",
200+
"\n",
201+
"# file1 = open(\"/content/url.txt\", 'r')\n",
202+
"# link_list = [f for f in enumerate(file1)]\n",
203+
"# for i,link in enumerate(link_list):\n",
204+
"# print(\"ok\")\n",
205+
"# !curl -L -O --remote-header-name \\\n",
206+
"# --header \"Authorization: Bearer bVVLVOIv29Lds-zADthteUE_1QlykgndjN5T6BaKMzMS-A11Z8UWtVsNbAJ85LWcGGerQH1KpM7eb-1KZS_Nig\" \\\n",
207+
"# --location {link_list[i][1]}\n",
208+
"# output.clear()\n"
209+
],
210+
"metadata": {
211+
"id": "690P6zBI_OEb"
212+
},
213+
"execution_count": null,
214+
"outputs": []
215+
}
216+
]
217+
}

0 commit comments

Comments
 (0)