456 lines
12 KiB
Python
456 lines
12 KiB
Python
#%%
|
||
#Project: 11934.201 Newtown Creek TPP - Privileged and Confidential
|
||
#Confidentiality Note: For internal discussion only.
|
||
#Description: This code plots the water levels at Field Facility Gage, and National Grid.
|
||
#To run,
|
||
|
||
|
||
#%% # -*- coding: utf-8 -*-
|
||
"""
|
||
Created on Thu Feb 16 11:54:29 2023
|
||
|
||
@author: mrobinson
|
||
"""
|
||
|
||
#%%Packages
|
||
import os
|
||
import pandas as pd
|
||
import numpy as np
|
||
import geopandas as gp
|
||
import pytz
|
||
import matplotlib as mpl
|
||
import matplotlib.pyplot as plt
|
||
import matplotlib.animation as animation
|
||
import dfm_tools as dfmt
|
||
from dfm_tools.get_nc import get_netdata, get_ncmodeldata, plot_netmapdata
|
||
from dfm_tools.get_nc_helpers import get_ncvarproperties, get_stationid_fromstationlist, get_varnamefromattrs
|
||
from dfm_tools.get_nc_helpers import get_timesfromnc
|
||
import contextily as ctx
|
||
from shapely.geometry import Point, MultiPoint
|
||
from shapely.ops import nearest_points
|
||
import pathlib as pl
|
||
import xarray as xr
|
||
from datetime import timedelta
|
||
import datetime as datetime
|
||
from matplotlib import dates as mpl_dates
|
||
from matplotlib.ticker import (MultipleLocator, AutoMinorLocator)
|
||
from sklearn.metrics import mean_squared_error
|
||
|
||
|
||
#%% Read Model Log
|
||
|
||
runLog = pd.read_excel("C:/Users/mrobinson/OneDrive - W.F. Baird & Associates Coastal Engineers Ltd/Documents/11934.202 NTC/Model Log NTC.xlsx", 'ModelLog')
|
||
|
||
dataPath = "FlowFM_map.nc"
|
||
histPath = "FlowFM_his.nc"
|
||
|
||
#%% Load in Water Level Data
|
||
# Field Gauge-EAST
|
||
FFG_pd = pd.read_csv("//srv-ott3/Projects/11934.201 Newtown Creek TPP – Privileged and Confidential/03_Data/02_Physical/00_ProcessingCode/NetCDF/Gauge/FieldFacility.csv")
|
||
FFG_pd['DateTime'] = pd.to_datetime(FFG_pd.Date_time)
|
||
FFG_pd.set_index(FFG_pd['DateTime'], inplace=True)
|
||
# Put in UTC from Eastern Time (with DST!)
|
||
FFG_pd = FFG_pd.tz_localize(
|
||
pytz.timezone('US/Eastern'), ambiguous='infer').tz_convert(pytz.utc)
|
||
|
||
# National Grid Gauge-WEST
|
||
NGG1_pd = pd.read_csv("//srv-ott3/Projects/11934.201 Newtown Creek TPP – Privileged and Confidential/03_Data/02_Physical/00_ProcessingCode/NetCDF/Gauge/NationalGrid1.csv")
|
||
NGG1_pd['DateTime'] = pd.to_datetime(NGG1_pd.Date_time)
|
||
NGG1_pd.set_index(NGG1_pd['DateTime'], inplace=True)
|
||
# Put in UTC from Eastern Time (with DST!)
|
||
NGG1_pd = NGG1_pd.tz_localize(
|
||
pytz.timezone('US/Eastern'), ambiguous='NaT').tz_convert(pytz.utc)
|
||
# Drop ambiguous times
|
||
NGG1_pd = NGG1_pd.dropna()
|
||
|
||
NGG2_pd = pd.read_csv("//srv-ott3/Projects/11934.201 Newtown Creek TPP – Privileged and Confidential/03_Data/02_Physical/00_ProcessingCode/NetCDF/Gauge/NationalGrid2.csv")
|
||
NGG2_pd['DateTime'] = pd.to_datetime(NGG2_pd.datetime)
|
||
NGG2_pd.set_index(NGG2_pd['DateTime'], inplace=True)
|
||
# Put in UTC from Eastern Time (with DST!)
|
||
NGG2_pd = NGG2_pd.tz_localize(
|
||
pytz.timezone('US/Eastern'), ambiguous='NaT').tz_convert(pytz.utc)
|
||
# Drop ambiguous times
|
||
NGG2_pd = NGG2_pd.dropna()
|
||
|
||
#%% Read in time series
|
||
modelPlot = [105, 106, 107, 108]
|
||
|
||
Delft_WL = [None] * (max(modelPlot)+1)
|
||
|
||
# Note, this uses the standard netcdf lib + xarray
|
||
for i in modelPlot:
|
||
file_nc_hist = pl.Path(runLog['Run Location'][i],
|
||
'FlowFM', 'dflowfm', 'output', 'FlowFM_his.nc')
|
||
|
||
# Hist file path
|
||
file_nc_hist = file_nc_hist.as_posix()
|
||
|
||
# Open hist file dataset
|
||
data_xr = xr.open_mfdataset(file_nc_hist, preprocess=dfmt.preprocess_hisnc)
|
||
|
||
# Get Variables
|
||
vars_pd = get_ncvarproperties(data_xr)
|
||
|
||
# Get stations
|
||
stations_pd = data_xr['stations'].to_dataframe()
|
||
|
||
# Convert to Pandas
|
||
Delft_WL[i] = data_xr.waterlevel.sel(stations=['West_WL', 'East_WL']).to_pandas()
|
||
|
||
# Put in UTC if needed
|
||
if i == 104:
|
||
Delft_WL[i] = Delft_WL[i].tz_localize(
|
||
pytz.timezone('EST')).tz_convert(pytz.utc)
|
||
else:
|
||
Delft_WL[i] = Delft_WL[i].tz_localize(
|
||
pytz.timezone('UTC'))
|
||
|
||
#%%
|
||
modelPlot=[107]
|
||
i=107
|
||
|
||
interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0],
|
||
Delft_WL[modelPlot[0]].index[-1],
|
||
freq="20min")
|
||
|
||
|
||
|
||
df1=Delft_WL[i]
|
||
df1_index=df1.index.shift(periods=1,freq='0H')
|
||
|
||
df2=pd.DataFrame(index=df1_index)
|
||
df2['West_WL']=df1['West_WL']
|
||
df2['East_WL']=df1['East_WL']
|
||
df2['West_WL']=df2['West_WL'].shift(30)
|
||
df2['East_WL']=df2['East_WL'].shift(30)
|
||
|
||
x=(np.interp(interpTimes,FFG_pd.index,FFG_pd['Water Surface Elevation (ft)']*0.3048))
|
||
x[x==-1.15332]=np.nan
|
||
x[0:2584]=np.NAN
|
||
|
||
x2= np.interp(interpTimes,NGG1_pd.index,NGG1_pd['Water Surface Elevation (ft)']*0.3048)
|
||
x2[0:2638]=np.NAN
|
||
|
||
|
||
#RMSE Values
|
||
predNGG=np.interp(interpTimes,df2.index, df2['West_WL'])
|
||
rms = mean_squared_error(predNGG[2638:], x2[2638:], squared=False)
|
||
print(rms)
|
||
|
||
predFG=np.interp(interpTimes,df2.index, df2['East_WL'])
|
||
|
||
rms2 = mean_squared_error(predFG[2584:], x[2584:], squared=False)
|
||
print(rms2)
|
||
|
||
#Plot
|
||
|
||
|
||
fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(20, 8))
|
||
fig.patch.set_facecolor('white')
|
||
#interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0],
|
||
# Delft_WL[modelPlot[0]].index[-1],
|
||
# freq="20min")
|
||
|
||
|
||
axes[0].title.set_text('National Grid Gauge')
|
||
axes[0].scatter(np.interp(interpTimes,df2.index, df2['West_WL']),
|
||
x2,s=1,
|
||
label='RMSE =' + str(round(rms,2)))
|
||
|
||
|
||
axes[1].title.set_text('Field Facility Gauge')
|
||
axes[1].scatter(np.interp(interpTimes,df2.index, df2['East_WL']), x,s=1,
|
||
label='RMSE =' + str(round(rms2,2)))
|
||
|
||
|
||
axes[1].set_xlim(-2, 2)
|
||
axes[1].set_ylim(-2, 2)
|
||
axes[0].set_xlim(-2, 2)
|
||
axes[0].set_ylim(-2, 2)
|
||
|
||
|
||
linepts = np.array([-3, 3])
|
||
axes[0].plot(linepts, linepts, color='k')
|
||
axes[1].plot(linepts, linepts, color='k')
|
||
|
||
axes[0].set_xlabel('Modelled water level (m)')
|
||
axes[0].set_ylabel('Measured water level (m)')
|
||
|
||
axes[1].set_xlabel('Modelled water level (m)')
|
||
axes[1].set_ylabel('Measured water level (m)')
|
||
|
||
axes[1].grid()
|
||
axes[0].grid()
|
||
|
||
fig.suptitle(runLog['Plotting Legend Entry'][i])
|
||
|
||
axes[1].legend(loc='lower left')
|
||
axes[0].legend(loc='lower left')
|
||
|
||
#%%
|
||
modelPlot=[106]
|
||
i=106
|
||
|
||
#interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0],
|
||
# Delft_WL[modelPlot[0]].index[-1],
|
||
# freq="20min")
|
||
|
||
#Note For 106 Run, period is from 2012-03-01 to 2012-10-30
|
||
|
||
interpTimes = pd.date_range("2012-03-01", "2012-10-30", freq="20min")
|
||
|
||
|
||
df1=Delft_WL[i]
|
||
df1_index=df1.index.shift(periods=1,freq='0H')
|
||
|
||
df2=pd.DataFrame(index=df1_index)
|
||
df2['West_WL']=df1['West_WL']
|
||
df2['East_WL']=df1['East_WL']
|
||
df2['West_WL']=df2['West_WL'].shift(30)
|
||
df2['East_WL']=df2['East_WL'].shift(30)
|
||
|
||
x=(np.interp(interpTimes,FFG_pd.index,FFG_pd['Water Surface Elevation (ft)']*0.3048))
|
||
x[x==-1.15332]=np.nan
|
||
x[0:2584]=np.NAN
|
||
|
||
x2= np.interp(interpTimes,NGG1_pd.index,NGG1_pd['Water Surface Elevation (ft)']*0.3048)
|
||
x2[0:2638]=np.NAN
|
||
|
||
|
||
#RMSE Values
|
||
predNGG=np.interp(interpTimes,df2.index, df2['West_WL'])
|
||
rms = mean_squared_error(predNGG[2638:], x2[2638:], squared=False)
|
||
print(rms.round(3))
|
||
|
||
predFG=np.interp(interpTimes,df2.index, df2['East_WL'])
|
||
|
||
rms2 = mean_squared_error(predFG[2584:], x[2584:], squared=False)
|
||
print(rms2.round(3))
|
||
|
||
#Plot
|
||
|
||
|
||
fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(18, 8))
|
||
fig.patch.set_facecolor('white')
|
||
plt.rcParams['font.size'] = 16
|
||
#interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0],
|
||
# Delft_WL[modelPlot[0]].index[-1],
|
||
# freq="20min")
|
||
|
||
|
||
axes[0].title.set_text('National Grid Gauge')
|
||
axes[0].scatter(np.interp(interpTimes,df2.index, df2['West_WL']),
|
||
x2,s=1,
|
||
label='RMSE =' + str(round(rms,3)))
|
||
|
||
|
||
axes[1].title.set_text('Field Facility Gauge')
|
||
axes[1].scatter(np.interp(interpTimes,df2.index, df2['East_WL']), x,s=1,
|
||
label='RMSE =' + str(round(rms2,3)))
|
||
|
||
|
||
axes[1].set_xlim(-2, 2)
|
||
axes[1].set_ylim(-2, 2)
|
||
axes[0].set_xlim(-2, 2)
|
||
axes[0].set_ylim(-2, 2)
|
||
|
||
|
||
linepts = np.array([-3, 3])
|
||
axes[0].plot(linepts, linepts, color='k')
|
||
axes[1].plot(linepts, linepts, color='k')
|
||
|
||
axes[0].set_xlabel('Modelled water level (m)')
|
||
axes[0].set_ylabel('Measured water level (m)')
|
||
|
||
axes[1].set_xlabel('Modelled water level (m)')
|
||
axes[1].set_ylabel('Measured water level (m)')
|
||
|
||
axes[1].grid()
|
||
axes[0].grid()
|
||
|
||
axes[0].set_aspect('equal')
|
||
axes[1].set_aspect('equal')
|
||
|
||
fig.suptitle(runLog['Plotting Legend Entry'][i])
|
||
|
||
axes[1].legend(loc='lower left')
|
||
axes[0].legend(loc='lower left')
|
||
|
||
|
||
#%%
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
#%%
|
||
|
||
fig, axes = plt.subplots(nrows=1,ncols=2,figsize=(20,10))
|
||
|
||
axes[0].plot(FFG_pd.index,FFG_pd['Water Surface Elevation (ft)']*0.3048,color='red')
|
||
axes[0].plot(df2.index,df2['East_WL'])
|
||
|
||
|
||
axes[1].plot(NGG1_pd['DateTime'],NGG1_pd['Water Surface Elevation (ft)']*0.3048,color='red')
|
||
axes[1].plot(df2.index,df2['West_WL'])
|
||
|
||
axes[0].set_xlim([np.datetime64('2012-01-01'), np.datetime64('2013-01-01')])
|
||
|
||
|
||
#%%
|
||
modelPlot=[105]
|
||
i=105
|
||
|
||
|
||
|
||
interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0],
|
||
Delft_WL[modelPlot[0]].index[-1],
|
||
freq="20min")
|
||
|
||
df1=Delft_WL[i]
|
||
df1_index=df1.index.shift(periods=1,freq='0H')
|
||
|
||
df2=pd.DataFrame(index=df1_index)
|
||
df2['West_WL']=df1['West_WL']
|
||
df2['East_WL']=df1['East_WL']
|
||
df2['West_WL']=df2['West_WL'].shift(30)
|
||
df2['East_WL']=df2['East_WL'].shift(30)
|
||
|
||
x=(np.interp(interpTimes,FFG_pd.index,FFG_pd['Water Surface Elevation (ft)']*0.3048))
|
||
x2=np.interp(interpTimes,df2.index, df2['West_WL'])
|
||
|
||
pred=np.interp(interpTimes,df2.index, df2['West_WL'])
|
||
meas=np.interp(interpTimes,NGG2_pd.index,NGG2_pd['water_surface_elevation']*0.3048)
|
||
|
||
#RMSE Values
|
||
rms3=mean_squared_error(meas[15:],pred[15:], squared=False)
|
||
print(rms3)
|
||
"""
|
||
predNGG=np.interp(interpTimes,df2.index, df2['West_WL'])
|
||
rms = mean_squared_error(predNGG[2638:], x2[2638:], squared=False)
|
||
print(rms)
|
||
|
||
predFG=np.interp(interpTimes,df2.index, df2['East_WL'])
|
||
|
||
rms2 = mean_squared_error(predFG[2584:], x[2584:], squared=False)
|
||
print(rms2)
|
||
"""
|
||
|
||
# Plot
|
||
|
||
fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(10, 10))
|
||
fig.patch.set_facecolor('white')
|
||
interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0],
|
||
Delft_WL[modelPlot[0]].index[-1],
|
||
freq="20min")
|
||
|
||
|
||
axes.title.set_text('National Grid Gauge')
|
||
axes.scatter(np.interp(interpTimes,df2.index, df2['West_WL']),
|
||
np.interp(interpTimes,NGG2_pd.index,NGG2_pd['water_surface_elevation']*0.3048),s=1,
|
||
label='RMSE =' + str(round(rms3,2)))
|
||
|
||
|
||
|
||
axes.set_xlim(-2, 2)
|
||
axes.set_ylim(-2, 2)
|
||
linepts = np.array([-3, 3])
|
||
axes.plot(linepts, linepts, color='k')
|
||
axes.set_xlabel('Modelled water level (m)')
|
||
axes.set_ylabel('Measured water level (m)')
|
||
axes.grid()
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
fig.suptitle(runLog['Plotting Legend Entry'][i])
|
||
|
||
|
||
#%%
|
||
modelPlot=[105]
|
||
i=105
|
||
|
||
interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0],
|
||
Delft_WL[modelPlot[0]].index[-1],
|
||
freq="20min")
|
||
df1=Delft_WL[i]
|
||
df1_index=df1.index.shift(periods=1,freq='0H')
|
||
|
||
df2=pd.DataFrame(index=df1_index)
|
||
df2['West_WL']=df1['West_WL']
|
||
df2['East_WL']=df1['East_WL']
|
||
df2['West_WL']=df2['West_WL'].shift(30)
|
||
df2['East_WL']=df2['East_WL'].shift(30)
|
||
|
||
x=(np.interp(interpTimes,FFG_pd.index,FFG_pd['Water Surface Elevation (ft)']*0.3048))
|
||
x2=np.interp(interpTimes,df2.index, df2['West_WL'])
|
||
|
||
pred=np.interp(interpTimes,df2.index, df2['West_WL'])
|
||
meas=np.interp(interpTimes,NGG2_pd.index,NGG2_pd['water_surface_elevation']*0.3048)
|
||
|
||
#RMSE Values
|
||
rms3=mean_squared_error(meas[15:],pred[15:], squared=False)
|
||
print(rms3)
|
||
|
||
# Plot
|
||
fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(10, 10))
|
||
fig.patch.set_facecolor('white')
|
||
interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0],
|
||
Delft_WL[modelPlot[0]].index[-1],
|
||
freq="20min")
|
||
|
||
axes.title.set_text('National Grid Gauge')
|
||
axes.scatter(np.interp(interpTimes,df2.index, df2['West_WL']),
|
||
np.interp(interpTimes,NGG2_pd.index,NGG2_pd['water_surface_elevation']*0.3048),s=1,
|
||
label='RMSE =' + str(round(rms3,3)))
|
||
|
||
axes.set_xlim(-2, 2)
|
||
axes.set_ylim(-2, 2)
|
||
linepts = np.array([-3, 3])
|
||
axes.plot(linepts, linepts, color='k')
|
||
axes.set_xlabel('Modelled water level (m)')
|
||
axes.set_ylabel('Measured water level (m)')
|
||
axes.grid()
|
||
axes.legend(loc='lower left')
|
||
|
||
axes.set_aspect('equal')
|
||
fig.suptitle(runLog['Plotting Legend Entry'][i])
|
||
|
||
|
||
|
||
|
||
|