Skip to content

Commit 727b2d7

Browse files
BcubrichBcubrich
authored
Add files via upload
1 parent d5c06e3 commit 727b2d7

File tree

2 files changed

+200
-0
lines changed

2 files changed

+200
-0
lines changed

‎auto_pull_MLH_from_ceilometer_LN_v1.py‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
# -*- coding: utf-8 -*-
2+
"""
3+
Created on Wed Mar 13 09:53:34 2019
4+
5+
@author: bcubrich
6+
"""
7+
8+
# -*- coding: utf-8 -*-
9+
"""
10+
Created on Tue May 22 14:02:20 2018
11+
12+
@author: kweber
13+
"""
14+
15+
from netCDF4 import Dataset
16+
#import matplotlib
17+
#import matplotlib.pyplot as plt
18+
import pandas as pd
19+
import numpy as np #didn't even use numpy!!! HA!
20+
#import seaborn as sns
21+
#from tkinter import Tk
22+
23+
#from tkinter.filedialog import askdirectory
24+
import os
25+
#import xarray as xr
26+
27+
#def get_dat():
28+
# root = Tk()
29+
# root.withdraw()
30+
# root.focus_force()
31+
# root.attributes("-topmost", True) #makes the dialog appear on top
32+
# filename = askdirectory() # Open single file
33+
# root.destroy()
34+
# root.quit()
35+
# return filename
36+
37+
start=pd.datetime.utcnow()
38+
today=pd.datetime.utcnow().strftime("%Y%m%d")
39+
directory='C:/BLViewData'
40+
out_dir='C:/Processed_netCDF'
41+
42+
#i=0
43+
44+
ceil_files=os.listdir(directory)
45+
text_files=os.listdir(out_dir)
46+
47+
ceil_file_names=[x.strip('.nc') for x in ceil_files if '.nc' in x and today not in x and 'L3' in x and 'DEFAULT' in x]
48+
text_file_names=[x.strip('.csv') for x in text_files if '.csv' in x and today not in x and 'L3' in x and 'DEFAULT' in x]
49+
50+
missing_text = np.setdiff1d(ceil_file_names,text_file_names, assume_unique=True)
51+
52+
missing_text_files=[x+'.nc' for x in missing_text]
53+
54+
#%%
55+
for filename in missing_text_files: #loop through files in user's dir
56+
print(filename)
57+
if filename.endswith(".nc"):
58+
# runstart=pd.datetime.utcnow()
59+
rootgrp3 = Dataset(directory+'/'+filename, "r", format="NETCDF4")
60+
# print (rootgrp3.dimensions.keys())
61+
#print (rootgrp3.data_model)
62+
63+
# this is the variable we want!!
64+
mlh_2 = rootgrp3.variables['Mean_Layer_Height'][:]
65+
# time: days since 1970-01-01 00:00:00.000
66+
ml_time = rootgrp3.variables['time'][:]
67+
68+
# close the file after you've had your way with the data you wanted from it
69+
rootgrp3.close()
70+
# runtime=pd.datetime.utcnow()-runstart
71+
df=pd.DataFrame([mlh_2[:,0],ml_time],index=['mlh2','ml_time']).T
72+
df['dt']=pd.to_datetime('1970-01-01 00:00:00.000') +pd.to_timedelta(df['ml_time'], unit='S')-pd.Timedelta('5 hours')
73+
df['dt_trunc']=df['dt'].apply(str).str[:13]
74+
df=df.drop_duplicates(subset=['dt_trunc','mlh2'])
75+
output_df=df
76+
# print("{}. It's working!!! It's working!!! It took {} for this file".format(i,runtime))
77+
78+
79+
80+
output_df['Site']='LN'
81+
output_df['Parameter']='61301'
82+
output_df['Average Interval']='001h'
83+
output_df['Date']=output_df['dt'].dt.strftime("%m/%d/%Y %H")+':00'
84+
output_df['Value']=np.around(output_df['mlh2'],2)
85+
output_df['Raw Value']=output_df['mlh2']
86+
output_df['AQS Null Code']=np.where(output_df['mlh2']<0,'AN','')
87+
output_df['Flags']=np.where(output_df['mlh2']<0,'<-','')
88+
output_df['Qualifier Codes']=''
89+
output_df['AQS Method Code']='128'
90+
output_df['Data Grade']=''
91+
columns=['Site','Parameter','Average Interval','Date','Value','Raw Value',
92+
'AQS Null Code','Flags','Qualifier Codes','AQS Method Code','Data Grade']
93+
output_df=output_df[columns]
94+
output_df=output_df.sort_values(by=['Date','Value'],ascending=False).drop_duplicates(subset=['Date'],keep='first')
95+
#plt.plot(mlh_2)
96+
output_df.to_csv(out_dir+'/'+filename.split('.')[0]+'.csv',index =False)
97+
98+
#end=pd.datetime.utcnow()
99+
#total=(end-start)
100+
#print('runtime: {}'.format(total))

‎auto_pull_MLH_from_ceilometer_SM_v1.py‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
# -*- coding: utf-8 -*-
2+
"""
3+
Created on Wed Mar 13 09:53:34 2019
4+
5+
@author: bcubrich
6+
"""
7+
8+
# -*- coding: utf-8 -*-
9+
"""
10+
Created on Tue May 22 14:02:20 2018
11+
12+
@author: kweber
13+
"""
14+
15+
from netCDF4 import Dataset
16+
#import matplotlib
17+
#import matplotlib.pyplot as plt
18+
import pandas as pd
19+
import numpy as np #didn't even use numpy!!! HA!
20+
#import seaborn as sns
21+
#from tkinter import Tk
22+
23+
#from tkinter.filedialog import askdirectory
24+
import os
25+
#import xarray as xr
26+
27+
#def get_dat():
28+
# root = Tk()
29+
# root.withdraw()
30+
# root.focus_force()
31+
# root.attributes("-topmost", True) #makes the dialog appear on top
32+
# filename = askdirectory() # Open single file
33+
# root.destroy()
34+
# root.quit()
35+
# return filename
36+
37+
start=pd.datetime.utcnow()
38+
today=pd.datetime.utcnow().strftime("%Y%m%d")
39+
directory='C:/BLViewData'
40+
out_dir='C:/Processed_netCDF'
41+
42+
#i=0
43+
44+
ceil_files=os.listdir(directory)
45+
text_files=os.listdir(out_dir)
46+
47+
ceil_file_names=[x.strip('.nc') for x in ceil_files if '.nc' in x and today not in x and 'L3' in x and 'DEFAULT' in x]
48+
text_file_names=[x.strip('.csv') for x in text_files if '.csv' in x and today not in x and 'L3' in x and 'DEFAULT' in x]
49+
50+
missing_text = np.setdiff1d(ceil_file_names,text_file_names, assume_unique=True)
51+
52+
missing_text_files=[x+'.nc' for x in missing_text]
53+
54+
#%%
55+
for filename in missing_text_files: #loop through files in user's dir
56+
print(filename)
57+
if filename.endswith(".nc"):
58+
# runstart=pd.datetime.utcnow()
59+
rootgrp3 = Dataset(directory+'/'+filename, "r", format="NETCDF4")
60+
# print (rootgrp3.dimensions.keys())
61+
#print (rootgrp3.data_model)
62+
63+
# this is the variable we want!!
64+
mlh_2 = rootgrp3.variables['Mean_Layer_Height'][:]
65+
# time: days since 1970-01-01 00:00:00.000
66+
ml_time = rootgrp3.variables['time'][:]
67+
68+
# close the file after you've had your way with the data you wanted from it
69+
rootgrp3.close()
70+
# runtime=pd.datetime.utcnow()-runstart
71+
df=pd.DataFrame([mlh_2[:,0],ml_time],index=['mlh2','ml_time']).T
72+
df['dt']=pd.to_datetime('1970-01-01 00:00:00.000') +pd.to_timedelta(df['ml_time'], unit='S')-pd.Timedelta('5 hours')
73+
df['dt_trunc']=df['dt'].apply(str).str[:13]
74+
df=df.drop_duplicates(subset=['dt_trunc','mlh2'])
75+
output_df=df
76+
# print("{}. It's working!!! It's working!!! It took {} for this file".format(i,runtime))
77+
78+
79+
80+
output_df['Site']='HW'
81+
output_df['Parameter']='61301'
82+
output_df['Average Interval']='001h'
83+
output_df['Date']=output_df['dt'].dt.strftime("%m/%d/%Y %H")+':00'
84+
output_df['Value']=np.around(output_df['mlh2'],2)
85+
output_df['Raw Value']=output_df['mlh2']
86+
output_df['AQS Null Code']=np.where(output_df['mlh2']<0,'AN','')
87+
output_df['Flags']=np.where(output_df['mlh2']<0,'<-','')
88+
output_df['Qualifier Codes']=''
89+
output_df['AQS Method Code']='128'
90+
output_df['Data Grade']=''
91+
columns=['Site','Parameter','Average Interval','Date','Value','Raw Value',
92+
'AQS Null Code','Flags','Qualifier Codes','AQS Method Code','Data Grade']
93+
output_df=output_df[columns]
94+
output_df=output_df.sort_values(by=['Date','Value'],ascending=False).drop_duplicates(subset=['Date'],keep='first')
95+
#plt.plot(mlh_2)
96+
output_df.to_csv(out_dir+'/'+filename.split('.')[0]+'.csv',index =False)
97+
98+
#end=pd.datetime.utcnow()
99+
#total=(end-start)
100+
#print('runtime: {}'.format(total))

0 commit comments

Comments
 (0)