OTD_FlashLoc_Quickview_Linux.ipynb

{"cells":[{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["#!/usr/bin/env python3"]},{"cell_type":"markdown","metadata":{},"source":["<br>\n","       OTD Lightning Flash Heat Map and Flash Location <br>\n","       CSV File for Linux<br>\n","<br>\n","       Decription: This code pulls OTD HDF data files from an OTD tarball <br>\n","       from a directory, decompressed the files, extracts the <br>\n","       flash coordinates from the files and generates a flash <br>\n","       heat map plot. This code also compiles all lightning <br>\n","       flash locations into a single CSV file, so they may <br>\n","       be plotted using other software.<br>\n","<br>\n","       Authors: Amanda Markert and Essence Raphael<br>\n","       Information and Technology Systems Center (ITSC)<br>\n","       University of Alabama in Huntsville<br>\n","       <br>\n","       Last Edit Date: 06 January 2020<br>\n","<br>\n","Import Python Packages"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["import shutil\n","import sys\n","import glob\n","import os\n","import tarfile\n","import subprocess\n","import re\n","from pyhdf.HDF import *\n","from pyhdf.VS import *\n","import numpy as np\n","import datetime\n","import csv\n","import matplotlib.pyplot as plt\n","import cartopy.crs as ccrs\n","import cartopy.feature as cfeature\n","import matplotlib.ticker as mticker\n","from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER\n","\n","# Initial file path. It can be changed by passing a different path as an argument\n","# to the main() function\n","\n","file_path = 'test_files/otd/' #!!!Path should point to folder containing otd.tar file\n","def main(file_path):\n","\n","    #Define the data directoy and identify the .tar files inside the data directory\n","    dataDir = os.path.join(file_path, '')\n","    raw_tar_files = glob.glob(dataDir+'otdlip_*_daily.tar')\n","    tfiles = [os.path.normpath(i) for i in raw_tar_files]\n","\n","    #Extract earliest and latest dates from \"tfiles\" to create directory names\n","    #Create empty list to hold dates and loop through each .tar to extract the date from its filename\n","    file_dates = []\n","    for i in tfiles:\n","        file_dates.append(re.findall('(\\d+\\.\\d+)', i))\n","    \n","    #Select the maximum and minimum dates\n","    file_dates_start = min(file_dates)\n","    file_dates_end = max(file_dates)\n","\n","    #Create a new folder to hold the untarred data files\n","    if file_dates_start != file_dates_end:\n","        os.mkdir(os.path.normpath(dataDir+'otd_' + file_dates_start[0] + '_' + file_dates_end[0] + '_untarred_daily'))\n","        untar_dataDir = os.path.join(file_path, 'otd_' + file_dates_start[0] + '_' + file_dates_end[0] + '_untarred_daily/')\n","    else:\n","        os.mkdir(os.path.normpath(dataDir+'otd_' + file_dates_start[0] + '_untarred_daily'))\n","        untar_dataDir = os.path.join(file_path, 'otd_' + file_dates_start[0] + '_untarred_daily/')\n","\n","    #Untar and extract all files contained within each .tar file\n","    for i in tfiles:\n","        with tarfile.TarFile(i, 'r') as tar_files:\n","            tar_files.extractall(untar_dataDir)\n","\n","    #Identify the .Z files inside the untarred files data directory\n","    raw_untarred_files = glob.glob(untar_dataDir+'mlab.otd.1_1.*.Z')\n","    untarred_files = [os.path.normpath(i) for i in raw_untarred_files]\n","     \n","    #Decompress the .Z files \n","    for i in untarred_files:\n","        subprocess.call(['uncompress', i])\n","                           \n","    #Identify the OTD HDF files in the directory\n","    raw_files = glob.glob(untar_dataDir+'mlab.otd.1_1.*')\n","    files = [os.path.normpath(i) for i in raw_files]   \n","       \n","    #Create empty numpy arrays to hold the flash latitudes, longitudes, and occurence times\n","    flash_lats = np.array([])\n","    flash_lons = np.array([])\n","    times = np.array([])\n","\n","    #Loop through, rename and read each OTD HDF file\n","    for i in files:\n","        vs_file = HDF(i, HC.READ).vstart()\n","        \n","        #Define the location of the lightning flash coordinates in the HDF files using reference numbers \n","        flash_flash = vs_file.attach(14)\n","\n","        #Store the number of flash records in a variable to use when reading the data from the\n","        #HDF file\n","        flash_inquire = flash_flash.inquire()\n","        record_count = flash_inquire[0]\n","\n","        #Create a numpy array of the flash records that have been read \n","        flash_records = flash_flash.read(record_count)\n","\n","        #Loop through each flash record. Extract the latitude, longitude, and occurence time for each \n","        #flash. For each flash, add its latitude, longitude, and occurence time to the \"flash_lats\",\n","        #\"flash_lons\", and \"times\" respectively.\n","        for i in flash_records:\n","            flash_lats = np.concatenate((flash_lats, i[8][0]), axis=None)\n","            flash_lons = np.concatenate((flash_lons, i[8][1]), axis=None)\n","            times = np.concatenate((times, i[1]), axis=None)\n","\n","    #This section extracts and formats the flash occurence times\n","    #Identify the earliest and latest flash timestamps in the \"times\" array\n","    start_seconds = min(times)\n","    end_seconds = max(times)\n","\n","    #Define the units for the start and end times then convert these times \n","    #(seconds since 1993-01-01 00:00:00.000) to dates\n","    x = datetime.datetime(1993,1,1)\n","    start_date = x + datetime.timedelta(seconds=start_seconds)\n","    end_date = x + datetime.timedelta(seconds=end_seconds)\n","\n","    #Create numerical and text date & time strings to use in filenames and the flash heat map title\n","    start_date_txt = start_date.strftime(\"%B %d, %Y\")\n","    end_date_txt = end_date.strftime(\"%B %d, %Y\")\n","    start_int = start_date.strftime(\"%Y%m%d\")\n","    end_int = end_date.strftime(\"%Y%m%d\")\n","    start_time = start_date.strftime(\"%X\")\n","    end_time = end_date.strftime(\"%X\")\n","\n","    #Create CSV file and destination\n","    if start_int != end_int:\n","        csvfile = os.path.join(dataDir, 'otd_'+ start_int + '_' + end_int + '_flashloc.csv')\n","    else:\n","        csvfile = os.path.join(dataDir, 'otd_'+ start_int + '_flashloc.csv')\n","\n","    #Create csv file\n","    with open(csvfile, 'w', newline='') as myfile:\n","        writer = csv.writer(myfile)\n","        writer.writerows(zip([\"flash_lat\"], [\"flash_lon\"])) #Define headers in row (zip creates columns)\n","        writer.writerows(zip(flash_lats,flash_lons)) #Define data rows (zip creates columns)\n","\n","    #Create plot of lightning flash location heat map\n","    plt.figure(figsize=((20,20))) #Set plot dimensions\n","    map = plt.axes(projection=ccrs.PlateCarree(central_longitude=0.0))\n","    gl = map.gridlines(crs=ccrs.PlateCarree(central_longitude=0.0), draw_labels=True, linewidth=0.8, alpha=0.5, color='white', linestyle='--')\n","    lightning = map.hexbin(flash_lons, flash_lats, gridsize=300, bins='log',cmap='jet', mincnt=1 ,zorder=10) #Bin flash counts into hexbins using a gridsize of your choice\n","\n","    #Draw geographic boundaries and meridians/parallels\n","    map.set_extent([-180, 180,-90, 90])\n","    map.coastlines(color='white')\n","    map.add_feature(cfeature.LAND, facecolor='gray')\n","    map.add_feature(cfeature.BORDERS, edgecolor='white')\n","    map.add_feature(cfeature.OCEAN, facecolor='black')\n","    gl.ylocator = mticker.FixedLocator([-90, -60, -30, 0 ,30, 60, 90])\n","    gl.xlocator = mticker.FixedLocator([-180, -150, -120, -90, -60, -30, 0 ,30, 60, 90, 120, 150, 180])\n","    gl.xformatter = LONGITUDE_FORMATTER\n","    gl.yformatter = LATITUDE_FORMATTER\n","    gl.top_labels=False\n","    gl.right_labels=False\n","    \n","    #Create colorbar\n","    cbar = plt.colorbar(lightning, orientation='horizontal', pad=0.02, aspect=50) \n","    cbar.set_label('Flash Count', fontsize=12) #Remember to change label\n","        \n","    #Create plot title based on file dates and times\n","    if start_date_txt != end_date_txt:\n","        plot_title = 'OTD Detected Lightning Flash Locations ' + start_date_txt + ' ' + start_time + ' - ' + end_date_txt + ' ' + end_time\n","        plt.title(plot_title, fontsize = 18)\n","        \n","        #Save the plot as an image\n","        plt.savefig(os.path.join(dataDir, 'otd_' + start_int + '_' + end_int +'_flashloc_plot.png'), bbox_inches='tight') \n","                \n","    else:\n","        plot_title = 'OTD Detected Lightning Flash Locations ' + end_date_txt + ' ' + start_time + ' - ' + end_time\n","        plt.title(plot_title, fontsize = 18) \n","   \n","        #Save the plot as an image\n","        plt.savefig(os.path.join(dataDir, 'otd_' + start_int +'_flashloc_plot.png'), bbox_inches='tight')  \n","\n","\n","if __name__ == \"__main__\":\n","    main(file_path)\n","        "]}],"metadata":{"kernelspec":{"display_name":"Python 3.9.12 ('otd-linux')","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.12"},"vscode":{"interpreter":{"hash":"5bf04772204b0bcd2a613d47e77a80688b37718d2df2c06e419c6c41f5477c57"}}},"nbformat":4,"nbformat_minor":2}