#%% #Project: 11934.201 Newtown Creek TPP - Privileged and Confidential #Confidentiality Note: For internal discussion only. #Description: This code plots the water levels at Field Facility Gage, and National Grid. #To run, #%% # -*- coding: utf-8 -*- """ Created on Thu Feb 16 11:54:29 2023 @author: mrobinson """ #%%Packages import os import pandas as pd import numpy as np import geopandas as gp import pytz import matplotlib as mpl import matplotlib.pyplot as plt import matplotlib.animation as animation import dfm_tools as dfmt from dfm_tools.get_nc import get_netdata, get_ncmodeldata, plot_netmapdata from dfm_tools.get_nc_helpers import get_ncvarproperties, get_stationid_fromstationlist, get_varnamefromattrs from dfm_tools.get_nc_helpers import get_timesfromnc import contextily as ctx from shapely.geometry import Point, MultiPoint from shapely.ops import nearest_points import pathlib as pl import xarray as xr from datetime import timedelta import datetime as datetime from matplotlib import dates as mpl_dates from matplotlib.ticker import (MultipleLocator, AutoMinorLocator) from sklearn.metrics import mean_squared_error #%% Read Model Log runLog = pd.read_excel("C:/Users/mrobinson/OneDrive - W.F. Baird & Associates Coastal Engineers Ltd/Documents/11934.202 NTC/Model Log NTC.xlsx", 'ModelLog') dataPath = "FlowFM_map.nc" histPath = "FlowFM_his.nc" #%% Load in Water Level Data # Field Gauge-EAST FFG_pd = pd.read_csv("//srv-ott3/Projects/11934.201 Newtown Creek TPP – Privileged and Confidential/03_Data/02_Physical/00_ProcessingCode/NetCDF/Gauge/FieldFacility.csv") FFG_pd['DateTime'] = pd.to_datetime(FFG_pd.Date_time) FFG_pd.set_index(FFG_pd['DateTime'], inplace=True) # Put in UTC from Eastern Time (with DST!) FFG_pd = FFG_pd.tz_localize( pytz.timezone('US/Eastern'), ambiguous='infer').tz_convert(pytz.utc) # National Grid Gauge-WEST NGG1_pd = pd.read_csv("//srv-ott3/Projects/11934.201 Newtown Creek TPP – Privileged and Confidential/03_Data/02_Physical/00_ProcessingCode/NetCDF/Gauge/NationalGrid1.csv") NGG1_pd['DateTime'] = pd.to_datetime(NGG1_pd.Date_time) NGG1_pd.set_index(NGG1_pd['DateTime'], inplace=True) # Put in UTC from Eastern Time (with DST!) NGG1_pd = NGG1_pd.tz_localize( pytz.timezone('US/Eastern'), ambiguous='NaT').tz_convert(pytz.utc) # Drop ambiguous times NGG1_pd = NGG1_pd.dropna() NGG2_pd = pd.read_csv("//srv-ott3/Projects/11934.201 Newtown Creek TPP – Privileged and Confidential/03_Data/02_Physical/00_ProcessingCode/NetCDF/Gauge/NationalGrid2.csv") NGG2_pd['DateTime'] = pd.to_datetime(NGG2_pd.datetime) NGG2_pd.set_index(NGG2_pd['DateTime'], inplace=True) # Put in UTC from Eastern Time (with DST!) NGG2_pd = NGG2_pd.tz_localize( pytz.timezone('US/Eastern'), ambiguous='NaT').tz_convert(pytz.utc) # Drop ambiguous times NGG2_pd = NGG2_pd.dropna() #%% Read in time series modelPlot = [105, 106, 107, 108] Delft_WL = [None] * (max(modelPlot)+1) # Note, this uses the standard netcdf lib + xarray for i in modelPlot: file_nc_hist = pl.Path(runLog['Run Location'][i], 'FlowFM', 'dflowfm', 'output', 'FlowFM_his.nc') # Hist file path file_nc_hist = file_nc_hist.as_posix() # Open hist file dataset data_xr = xr.open_mfdataset(file_nc_hist, preprocess=dfmt.preprocess_hisnc) # Get Variables vars_pd = get_ncvarproperties(data_xr) # Get stations stations_pd = data_xr['stations'].to_dataframe() # Convert to Pandas Delft_WL[i] = data_xr.waterlevel.sel(stations=['West_WL', 'East_WL']).to_pandas() # Put in UTC if needed if i == 104: Delft_WL[i] = Delft_WL[i].tz_localize( pytz.timezone('EST')).tz_convert(pytz.utc) else: Delft_WL[i] = Delft_WL[i].tz_localize( pytz.timezone('UTC')) #%% modelPlot=[107] i=107 interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0], Delft_WL[modelPlot[0]].index[-1], freq="20min") df1=Delft_WL[i] df1_index=df1.index.shift(periods=1,freq='0H') df2=pd.DataFrame(index=df1_index) df2['West_WL']=df1['West_WL'] df2['East_WL']=df1['East_WL'] df2['West_WL']=df2['West_WL'].shift(30) df2['East_WL']=df2['East_WL'].shift(30) x=(np.interp(interpTimes,FFG_pd.index,FFG_pd['Water Surface Elevation (ft)']*0.3048)) x[x==-1.15332]=np.nan x[0:2584]=np.NAN x2= np.interp(interpTimes,NGG1_pd.index,NGG1_pd['Water Surface Elevation (ft)']*0.3048) x2[0:2638]=np.NAN #RMSE Values predNGG=np.interp(interpTimes,df2.index, df2['West_WL']) rms = mean_squared_error(predNGG[2638:], x2[2638:], squared=False) print(rms) predFG=np.interp(interpTimes,df2.index, df2['East_WL']) rms2 = mean_squared_error(predFG[2584:], x[2584:], squared=False) print(rms2) #Plot fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(20, 8)) fig.patch.set_facecolor('white') #interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0], # Delft_WL[modelPlot[0]].index[-1], # freq="20min") axes[0].title.set_text('National Grid Gauge') axes[0].scatter(np.interp(interpTimes,df2.index, df2['West_WL']), x2,s=1, label='RMSE =' + str(round(rms,2))) axes[1].title.set_text('Field Facility Gauge') axes[1].scatter(np.interp(interpTimes,df2.index, df2['East_WL']), x,s=1, label='RMSE =' + str(round(rms2,2))) axes[1].set_xlim(-2, 2) axes[1].set_ylim(-2, 2) axes[0].set_xlim(-2, 2) axes[0].set_ylim(-2, 2) linepts = np.array([-3, 3]) axes[0].plot(linepts, linepts, color='k') axes[1].plot(linepts, linepts, color='k') axes[0].set_xlabel('Modelled water level (m)') axes[0].set_ylabel('Measured water level (m)') axes[1].set_xlabel('Modelled water level (m)') axes[1].set_ylabel('Measured water level (m)') axes[1].grid() axes[0].grid() fig.suptitle(runLog['Plotting Legend Entry'][i]) axes[1].legend(loc='lower left') axes[0].legend(loc='lower left') #%% modelPlot=[106] i=106 #interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0], # Delft_WL[modelPlot[0]].index[-1], # freq="20min") #Note For 106 Run, period is from 2012-03-01 to 2012-10-30 interpTimes = pd.date_range("2012-03-01", "2012-10-30", freq="20min") df1=Delft_WL[i] df1_index=df1.index.shift(periods=1,freq='0H') df2=pd.DataFrame(index=df1_index) df2['West_WL']=df1['West_WL'] df2['East_WL']=df1['East_WL'] df2['West_WL']=df2['West_WL'].shift(30) df2['East_WL']=df2['East_WL'].shift(30) x=(np.interp(interpTimes,FFG_pd.index,FFG_pd['Water Surface Elevation (ft)']*0.3048)) x[x==-1.15332]=np.nan x[0:2584]=np.NAN x2= np.interp(interpTimes,NGG1_pd.index,NGG1_pd['Water Surface Elevation (ft)']*0.3048) x2[0:2638]=np.NAN #RMSE Values predNGG=np.interp(interpTimes,df2.index, df2['West_WL']) rms = mean_squared_error(predNGG[2638:], x2[2638:], squared=False) print(rms.round(3)) predFG=np.interp(interpTimes,df2.index, df2['East_WL']) rms2 = mean_squared_error(predFG[2584:], x[2584:], squared=False) print(rms2.round(3)) #Plot fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(18, 8)) fig.patch.set_facecolor('white') plt.rcParams['font.size'] = 16 #interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0], # Delft_WL[modelPlot[0]].index[-1], # freq="20min") axes[0].title.set_text('National Grid Gauge') axes[0].scatter(np.interp(interpTimes,df2.index, df2['West_WL']), x2,s=1, label='RMSE =' + str(round(rms,3))) axes[1].title.set_text('Field Facility Gauge') axes[1].scatter(np.interp(interpTimes,df2.index, df2['East_WL']), x,s=1, label='RMSE =' + str(round(rms2,3))) axes[1].set_xlim(-2, 2) axes[1].set_ylim(-2, 2) axes[0].set_xlim(-2, 2) axes[0].set_ylim(-2, 2) linepts = np.array([-3, 3]) axes[0].plot(linepts, linepts, color='k') axes[1].plot(linepts, linepts, color='k') axes[0].set_xlabel('Modelled water level (m)') axes[0].set_ylabel('Measured water level (m)') axes[1].set_xlabel('Modelled water level (m)') axes[1].set_ylabel('Measured water level (m)') axes[1].grid() axes[0].grid() axes[0].set_aspect('equal') axes[1].set_aspect('equal') fig.suptitle(runLog['Plotting Legend Entry'][i]) axes[1].legend(loc='lower left') axes[0].legend(loc='lower left') #%% #%% fig, axes = plt.subplots(nrows=1,ncols=2,figsize=(20,10)) axes[0].plot(FFG_pd.index,FFG_pd['Water Surface Elevation (ft)']*0.3048,color='red') axes[0].plot(df2.index,df2['East_WL']) axes[1].plot(NGG1_pd['DateTime'],NGG1_pd['Water Surface Elevation (ft)']*0.3048,color='red') axes[1].plot(df2.index,df2['West_WL']) axes[0].set_xlim([np.datetime64('2012-01-01'), np.datetime64('2013-01-01')]) #%% modelPlot=[105] i=105 interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0], Delft_WL[modelPlot[0]].index[-1], freq="20min") df1=Delft_WL[i] df1_index=df1.index.shift(periods=1,freq='0H') df2=pd.DataFrame(index=df1_index) df2['West_WL']=df1['West_WL'] df2['East_WL']=df1['East_WL'] df2['West_WL']=df2['West_WL'].shift(30) df2['East_WL']=df2['East_WL'].shift(30) x=(np.interp(interpTimes,FFG_pd.index,FFG_pd['Water Surface Elevation (ft)']*0.3048)) x2=np.interp(interpTimes,df2.index, df2['West_WL']) pred=np.interp(interpTimes,df2.index, df2['West_WL']) meas=np.interp(interpTimes,NGG2_pd.index,NGG2_pd['water_surface_elevation']*0.3048) #RMSE Values rms3=mean_squared_error(meas[15:],pred[15:], squared=False) print(rms3) """ predNGG=np.interp(interpTimes,df2.index, df2['West_WL']) rms = mean_squared_error(predNGG[2638:], x2[2638:], squared=False) print(rms) predFG=np.interp(interpTimes,df2.index, df2['East_WL']) rms2 = mean_squared_error(predFG[2584:], x[2584:], squared=False) print(rms2) """ # Plot fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(10, 10)) fig.patch.set_facecolor('white') interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0], Delft_WL[modelPlot[0]].index[-1], freq="20min") axes.title.set_text('National Grid Gauge') axes.scatter(np.interp(interpTimes,df2.index, df2['West_WL']), np.interp(interpTimes,NGG2_pd.index,NGG2_pd['water_surface_elevation']*0.3048),s=1, label='RMSE =' + str(round(rms3,2))) axes.set_xlim(-2, 2) axes.set_ylim(-2, 2) linepts = np.array([-3, 3]) axes.plot(linepts, linepts, color='k') axes.set_xlabel('Modelled water level (m)') axes.set_ylabel('Measured water level (m)') axes.grid() fig.suptitle(runLog['Plotting Legend Entry'][i]) #%% modelPlot=[105] i=105 interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0], Delft_WL[modelPlot[0]].index[-1], freq="20min") df1=Delft_WL[i] df1_index=df1.index.shift(periods=1,freq='0H') df2=pd.DataFrame(index=df1_index) df2['West_WL']=df1['West_WL'] df2['East_WL']=df1['East_WL'] df2['West_WL']=df2['West_WL'].shift(30) df2['East_WL']=df2['East_WL'].shift(30) x=(np.interp(interpTimes,FFG_pd.index,FFG_pd['Water Surface Elevation (ft)']*0.3048)) x2=np.interp(interpTimes,df2.index, df2['West_WL']) pred=np.interp(interpTimes,df2.index, df2['West_WL']) meas=np.interp(interpTimes,NGG2_pd.index,NGG2_pd['water_surface_elevation']*0.3048) #RMSE Values rms3=mean_squared_error(meas[15:],pred[15:], squared=False) print(rms3) # Plot fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(10, 10)) fig.patch.set_facecolor('white') interpTimes = pd.date_range(Delft_WL[modelPlot[0]].index[0], Delft_WL[modelPlot[0]].index[-1], freq="20min") axes.title.set_text('National Grid Gauge') axes.scatter(np.interp(interpTimes,df2.index, df2['West_WL']), np.interp(interpTimes,NGG2_pd.index,NGG2_pd['water_surface_elevation']*0.3048),s=1, label='RMSE =' + str(round(rms3,3))) axes.set_xlim(-2, 2) axes.set_ylim(-2, 2) linepts = np.array([-3, 3]) axes.plot(linepts, linepts, color='k') axes.set_xlabel('Modelled water level (m)') axes.set_ylabel('Measured water level (m)') axes.grid() axes.legend(loc='lower left') axes.set_aspect('equal') fig.suptitle(runLog['Plotting Legend Entry'][i])