148 lines
4.6 KiB
Python
148 lines
4.6 KiB
Python
#%% Script to read LSU weather files and save as .bts
|
|
# AJMR, APril 12, 2022
|
|
|
|
# Set up
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
import matplotlib.pyplot as plt
|
|
|
|
|
|
import iem
|
|
|
|
import datetime
|
|
from io import StringIO
|
|
|
|
#%% Import ISM Data using iem
|
|
|
|
# Number of attempts to download data
|
|
# HTTPS here can be problematic for installs that don't have Lets Encrypt CA
|
|
SERVICE = "http://mesonet.agron.iastate.edu/cgi-bin/request/asos.py?"
|
|
|
|
# timestamps in UTC to request data for
|
|
startts = datetime.datetime(2000, 1, 1)
|
|
endts = datetime.datetime(2022, 4, 14)
|
|
|
|
# service = SERVICE + "data=all&tz=Etc/UTC&format=comma&latlon=yes&"
|
|
# service = SERVICE + "data=tmpc&data=p01m&tz=Etc/UTC&format=comma&latlon=yes&"
|
|
service = SERVICE + "data=tmpc&data=p01m&data=drct&data=sped" + "&tz=Etc/UTC&format=comma&latlon=yes&"
|
|
|
|
service += startts.strftime("year1=%Y&month1=%m&day1=%d&")
|
|
service += endts.strftime("year2=%Y&month2=%m&day2=%d&")
|
|
|
|
# Two examples of how to specify a list of stations
|
|
# stations = iem.get_stations_from_networks()
|
|
|
|
iem_pd = pd.DataFrame()
|
|
iem_pd_hourly = pd.DataFrame()
|
|
iem_pd_daily = pd.DataFrame()
|
|
|
|
for station in ['HDC', 'BTR', 'MSY']:
|
|
uri = "%s&station=%s" % (service, station)
|
|
print("Downloading: %s" % (station,))
|
|
data = iem.download_data(uri)
|
|
|
|
# Parse the data as a dataframe
|
|
iem_pd_in = pd.read_csv(StringIO(data), sep=",", skiprows=5)
|
|
|
|
# Convert the data to a pandas dataframe date-time index
|
|
iem_pd_in['DateTime'] = pd.to_datetime(iem_pd_in['valid']).dt.tz_localize('UTC')
|
|
iem_pd_in.set_index('DateTime', inplace=True)
|
|
|
|
# Set datatypes
|
|
iem_pd_in.loc[iem_pd_in.p01m == 'M', 'p01m'] = 'nan'
|
|
iem_pd_in['p01m'] = iem_pd_in['p01m'].astype(float)
|
|
|
|
# Set datatypes
|
|
iem_pd_in.loc[iem_pd_in.tmpc == 'M', 'tmpc'] = 'nan'
|
|
iem_pd_in['tmpc'] = iem_pd_in['tmpc'].astype(float)
|
|
|
|
# Resample to hourly
|
|
iem_pd_hourly_in = iem_pd_in.resample('H').mean()
|
|
iem_pd_hourly_in['station'] = station
|
|
|
|
# Resample to Daily
|
|
iem_pd_daily_in = iem_pd_hourly_in.resample('D').sum()
|
|
iem_pd_daily_in['station'] = station
|
|
|
|
# Merge variables into one dataframe
|
|
iem_pd = pd.concat([iem_pd, iem_pd_in])
|
|
iem_pd_hourly = pd.concat([iem_pd_hourly, iem_pd_hourly_in])
|
|
iem_pd_daily = pd.concat([iem_pd_daily, iem_pd_daily_in])
|
|
|
|
# Clear working data
|
|
iem_pd_in = []
|
|
iem_pd_hourly_in = []
|
|
iem_pd_daily_in = []
|
|
|
|
#%% Read in data from other source
|
|
|
|
# Initalize a list to hold the data
|
|
weatherPath = '//srv-mad3/Projects/13522.101 LSU Lakes/03_Data/02_Physical/02_Weather_Reports/'
|
|
|
|
weatherDat = pd.read_excel(weatherPath + '2002-2018 weather_report.xls.xlsx',
|
|
engine='openpyxl', header=1)
|
|
weatherDat.rename(columns={'DateTimeCollected': 'DATETIME'}, inplace=True)
|
|
weatherDat['DATETIME'] = pd.to_datetime(weatherDat['DATETIME'])
|
|
weatherDat.set_index('DATETIME', inplace=True)
|
|
weatherDat['Rain_mm'] = weatherDat['Rain (in)'] * 25.4
|
|
weatherDat['Rain_mm_hour'] = weatherDat['Rain_mm'] /24
|
|
|
|
#%% Plot IEM
|
|
fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(10, 5))
|
|
|
|
iem_pd_daily.loc[iem_pd_daily.station =='BTR', :].p01m.plot(ax=axes, label='BTR')
|
|
# iem_pd_hourly.loc[iem_pd_hourly.station=='BTR'].p01m.plot(ax=axes, label='BTR')
|
|
|
|
weatherDat['Rain_mm'].plot(ax=axes, label='WeatherData')
|
|
# weatherDat['Rain_mm_hour'].plot(ax=axes, label='WeatherData')
|
|
|
|
axes.set_xlim(datetime.datetime(2015, 1, 1), datetime.datetime(2016, 1, 1))
|
|
|
|
axes.set_xlabel('Date')
|
|
axes.set_ylabel('1 Hour Precipitation (mm)')
|
|
plt.show()
|
|
|
|
|
|
#%% Plot QQ
|
|
fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(10, 5))
|
|
|
|
A = pd.merge_asof(weatherDat, iem_pd_daily.p01m,
|
|
left_index=True, right_index=True)
|
|
|
|
axes.scatter(weatherDat['Rain_mm'],
|
|
pd.merge_asof(weatherDat, iem_pd_daily.p01m,
|
|
left_index=True, right_index=True,
|
|
tolerance=pd.Timedelta("24H")).p01m, 2, 'k')
|
|
|
|
axes.set_xlabel('Spreadsheets')
|
|
axes.set_ylabel('IEM Station')
|
|
plt.show()
|
|
|
|
#%% Same as model time series
|
|
|
|
datesOut = iem_pd_hourly.loc[iem_pd_hourly.station == 'BTR', :].index.strftime('%m/%d/%Y').values
|
|
timesOut = iem_pd_hourly.loc[iem_pd_hourly.station == 'BTR', :].index.strftime('%H:%M').values
|
|
valsOut = np.nan_to_num(iem_pd_hourly.loc[iem_pd_hourly.station == 'BTR', :].p01m.values)
|
|
|
|
modelOut = np.vstack([datesOut, timesOut, valsOut]).T
|
|
|
|
np.savetxt("//srv-mad3/Projects/13522.101 LSU Lakes/03_Data/02_Physical/06_Weather_IEM/BTR_IEM.csv",
|
|
modelOut, delimiter=",", fmt='%s')
|
|
|
|
np.savetxt("//srv-mad3/Projects/13522.101 LSU Lakes/03_Data/02_Physical/06_Weather_IEM/BTR_IEM_tab.txt",
|
|
modelOut, delimiter="\t", fmt='%s')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|