diff --git a/.gitignore b/.gitignore index 5eb3b10..c48b6c2 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ */launch.json build .idea/ +/sample_data/nwm_output/ +/sample_data/outputs_0629/ +/sample_data/sample_netcdf/ \ No newline at end of file diff --git a/data_assimilation_engine/Streamflow_Scripts/README.md b/data_assimilation_engine/Streamflow_Scripts/README.md new file mode 100644 index 0000000..c4ba784 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/README.md @@ -0,0 +1,57 @@ +# Overview +This directory contains Python scripts for downloading and time-slicing real-time streamflow data from the USGS, US Army Corps of Engineers and Environment Canada. The time-slicing scripts process the native files into NetCDF files that can be directly used by T-Route for streamflow data assimilation. These scripts originated from the NWM v3 codebase. + +USGS data download: usgs_download/stream_flow_download/usgs_current.py +USGS data time-slice: usgs_download/analysis/make_time_slice_from_usgs_waterml.py +USACE data download: ace_download/stream_flow_download/CWMS_download_current.py +USACE data time-slice: ace_download/analysis/make_time_slice_from_ace_xml.py +Env Canada data download: canada_download/parallel_dm_can.py +Env Canada data time-slice: canada_download/make_time_slice_from_canada.py +NCO Env Canada data Download: nco_canada/streamflow_download/get_station_list.sh,get_canadian_streamgauge.sh +NCO Env Canada data time-slice: nco_canada/timeslices_scripts/make_time_slice_from_canada.py + +# Setting Up Required Python Environment +python -m venv venv-streamflow +source venv-streamflow/bin/activate +pip install --upgrade pip +pip install -r requirements.txt + +# Script Usage +Each script has a help option (-h) for printing usage information. + +The USGS streamflow download script (usgs_current.py) runs continuously and downloads the most recent files as they become available. The other streamflow download scripts exit after downloading the latest files available when the script was run. See usgs_download/stream_flow_download/README.TXT + +CWMS_download_current.py [-h] [-f FILE_FORMAT] site_file output_dir +make_time_slice_from_ace_xml.py -i -o -s + +canadian_flow_retrieval.py -o +make_time_slice_from_canada.py -i -o + +See nco_canada/streamflow_download/README.TXT for downloading Canadian gages using NCO's script. +When NCO download script is used, the timeslices should be created using the script in the nco_canada/timeslices_scripts directory. + +#### Examples #### +export USGS_API_KEY=[API KEY] +export DCOMROOT=/path/to/dcomroot +export DBNROOT=/path/to/dbnroot +mkdir $DBNROOT/log +python usgs_current.py +python make_time_slice_from_usgs_waterml.py -i ~/usgs_download -o ~/usgs/timeslice + +python parallel_dm_can.py -o ~/canada_download +python make_time_slice_from_canada.py -i ~/canada_download -o ~/canada/timeslice + +python CWMS_download_current.py -f json site-file.csv ~/usace_download +python make_time_slice_from_ace_xml.py -i ~/usace_download -o ~/usace_timeslice -s site-file.csv + +export DCOMROOT=/path/to/dcomroot +export DBNROOT=/path/to/dbnroot +mkdir -p $DCOMROOT/ +mkdir -p $DBNROOT/user/can_streamgauge +mkdir -p $DBNROOT/log +mkdir -p $DBNROOT/tmp +cd $DBNROOT/user/can_streamgauge +source /path/to/get_station_list.sh +source /path/to/get_canadian_streamgauge.sh +python make_time_slice_from_canada.py -i $DCOMROOT/YYYYMMDD/can_streamgauge -o ~/canada/timeslice + diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/ACE_Observation.py b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/ACE_Observation.py new file mode 100644 index 0000000..75ac018 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/ACE_Observation.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python +############################################################################### +# Module name: ACE_Observation # +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: # +# # +# Last modification date: 05/28/2019 # +# # +# Description: manage data in a ACE CWMS json file # +# # +# 12/11/2024 OWP Add loadCwmsjson() to parse json data format # +# # +############################################################################### + +import os, sys, time, csv, re +import logging +from string import * +from collections import OrderedDict +from datetime import datetime, timedelta, timezone +import dateutil.parser +import pytz +import json +import xml.etree.ElementTree as etree +from TimeSlice import TimeSlice +from Observation import Observation +from CWMS_Sites import CWMS_Sites + +def parseDuration( period ): + regex = re.compile('(?P-?)P(?:(?P\d+)Y)?(?:(?P\d+)M)?(?:(?P\d+)D)?(?:T(?:(?P\d+)H)?(?:(?P\d+)M)?(?:(?P\d+)S)?)?') + + # Fetch the match groups with default value of 0 (not None) + duration = regex.match(period).groupdict(0) + + # Create the timedelta object from extracted groups + delta = timedelta(days=int(duration['days']) + \ + (int(duration['months']) * 30) + \ + (int(duration['years']) * 365), \ + hours=int(duration['hours']), \ + minutes=int(duration['minutes']), \ + seconds=int(duration['seconds'])) + + if duration['sign'] == "-": + delta *= -1 + + return delta + +class ACE_Observation(Observation): + """ + Store one USACE data + """ + def __init__(self, cwmsxmljsonfilename, cwmssites ): + """ + Initialize the ACE_Observation object with a given + filename + """ + self.source = cwmsxmljsonfilename + self.timeValueQuality = OrderedDict() + self._sites = cwmssites + if cwmsxmljsonfilename.endswith( '.json' ): + self.loadCwmsjson( cwmsxmljsonfilename ) + elif cwmsxmljsonfilename.endswith( '.xml' ): + self.loadCWMSxml( cwmsxmljsonfilename ) + else: + raise RuntimeError( "FATAL ERROR: Unknow file type: " + \ + cwmsxmljsonfilename ) + + def loadCwmsjson(self, jsonfilename ): + """ + Read real-time discharge data from a given CWMS JSON file + + Input: jsonfilename - the CWMS json filename + """ + try: + json_file = open(jsonfilename) + + # Reads .json file to Python dictionary + json_data = json.load(json_file) + + name = json_data["name"] + officeId = json_data["office-id"] + self.unit = json_data["units"] + + self.stationName = officeId + "." + name + self.stationID = self._sites.getIndex(officeId, name ) + + # Get Discharge timeseries values + totalFcts = len(json_data["values"]) + + #print(f"total FCST size: {totalFcts}") + + for idx in range(totalFcts): + row = json_data['values'][idx] + if len(row) > 0: + #print(f"ROW={row} time={row[0]}") + + ### Convert date-time to datetime from ms + t1 = self.convert_milliseconds_to_iso_format(row[0]) + t = dateutil.parser.parse( t1 ).astimezone(pytz.utc).replace(tzinfo=None) + + self.timeValueQuality[ t ] = ( float(row[1]), \ + self.calculateDataQuality(float(row[2] ) ) ) + + #print( idx, t, self.timeValueQuality[ t ] ) + + self.obvPeriod = list( self.timeValueQuality.keys() )[0], \ + list( self.timeValueQuality.keys() )[-1] + + unitConvertToM3perSec = self.getUnitConvertToM3perSec() + + self.timeValueQuality = dict(map( \ + lambda kv: (kv[0], (kv[1][0] * unitConvertToM3perSec, \ + kv[1][1])), \ + iter( self.timeValueQuality.items()) )) + self.unit = 'm3/s' + + except Exception as e: + raise RuntimeError( "WARNING: parsing JSON error: " + str( e )\ + + ": " + jsonfilename + '. Or Maybe "values" field are empty; ==> ' \ + + jsonfilename + " Skipping ..." ) + + #print(f"{jsonfilename} stationName: {self.stationName} unit={self.unit}") + #for k, v in self.timeValueQuality.items(): + # print(k, v) + + + def loadCWMSxml(self, xmlfilename ): + """ + Read real-time stream flow data from a given CWMS XML file + + Input: xmlfilename - the CWMS xml filename + """ + try: + obvwml = etree.parse( xmlfilename ) + root= obvwml.getroot() + name_1 = root.find('query-info').find('requested-item')\ + .find('name').text + timeseries = root.find('time-series') + office = timeseries.find('office').text + self.stationName = office + "." + name_1 + self.stationID = self._sites.getIndex(office, name_1 ) + + regularIntervalValues = timeseries.find('regular-interval-values') + if regularIntervalValues is not None: + self.parseRegularIntervalValues( regularIntervalValues ) + else: + irregularIntervalValues = timeseries.find('irregular-interval-values') + self.parseIrregularIntervalValues( irregularIntervalValues) + + except Exception as e: + raise RuntimeError( "WARNING: parsing XML error: " + str( e )\ + + ": " + xmlfilename + " skipping ..." ) + + self.stationName = office + '.' + name_1 + #print(f"stationName={self.stationName}") + self.obvPeriod = list( self.timeValueQuality.keys() )[0], \ + list( self.timeValueQuality.keys() )[-1] + + unitConvertToM3perSec = self.getUnitConvertToM3perSec() + + #print(f"unitConvertToM3perSec={unitConvertToM3perSec}") + + self.timeValueQuality = dict(map( \ + lambda kv: (kv[0], (kv[1][0] * unitConvertToM3perSec, \ + kv[1][1])),\ + iter( self.timeValueQuality.items()) )) + self.unit = 'm3/s' + + #for k, v in self.timeValueQuality.items(): + # print(k, v) + + def parseRegularIntervalValues(self, regularInterval): + self.unit = regularInterval.get('unit') + + interval = parseDuration( regularInterval.get('interval') ) + + for seg in regularInterval.findall('segment'): + beginTime = \ + dateutil.parser.parse( seg.get('first-time')) \ + .astimezone(pytz.utc).replace(tzinfo=None) + for s in seg.text.strip().split('\n'): + self.timeValueQuality[ beginTime ] = \ + ( float(s.split(' ')[0]), \ + self.calculateDataQuality( float(s.split(' ')[1] ) ) ) + + beginTime += interval + + + def parseIrregularIntervalValues(self, irregularInterval): + self.unit = irregularInterval.get('unit') + #print(f"irregularInterval Unit={self.unit}") + for s in irregularInterval.text.strip().split('\n'): + words = s.split(' ') + t = dateutil.parser.parse( words[0] ) \ + .astimezone(pytz.utc).replace(tzinfo=None) + self.timeValueQuality[ t ] = \ + ( float(words[1]), \ + self.calculateDataQuality(float(words[2] ) ) ) + #print( t, self.timeValueQuality[ t ] ) + + + def getUnitConvertToM3perSec(self): + #print(f"self.unit={self.unit}") + if self.unit == 'cfs': + unitConvertToM3perSec = 0.028317 + elif self.unit == 'CMS': + unitConvertToM3perSec = 1.0 + else: + raise RuntimeError( "FATAL ERROR: Unit " + self.unit + \ + " is not known. ") + return unitConvertToM3perSec + + def calculateDataQuality(self, value ): + return 100.0 + + def convert_milliseconds_to_iso_format(self, milliseconds): + """ + Converts milliseconds to ISO 8601 format (e.g., 2016-01-01 06:00:00) + """ + + dt = datetime.fromtimestamp(milliseconds / 1000, tz=timezone.utc) + return dt.strftime("%Y-%m-%d %H:%M:%S") diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/CWMS_Sites.py b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/CWMS_Sites.py new file mode 100644 index 0000000..b416c5b --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/CWMS_Sites.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +############################################################################### +# Module name: CWMS_Sites # +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: # +# # +# Last modification date: 09/05/2019 # +# # +# Description: manage the ACE CWMS sites in CSV format # +# # +# Updated by: Donald W Johnson (donald.w.johnson@noaa.gov) # +# # +# Update Description: Change column names to match new site file # +# # +############################################################################### + +import csv + +class CWMS_Sites: + """ + Store CWMS site information + """ + def __init__(self, csvSitefile ): + """ + Initialize the CWMS_Sites object with a given + filename + """ + self.source = csvSitefile + + self._office_name1_to_index = dict() + with open( csvSitefile, mode='r') as csvsite_file: + csvsite_reader = csv.DictReader( csvsite_file ) + line_count = 0 + for row in csvsite_reader: + if line_count == 0: + print('Column names are ' + ", ".join(row)) + line_count += 1 +# print('\t' + row["office"] + " " + row["name_1"] ) + if row["office"] in self._office_name1_to_index: + self._office_name1_to_index[ \ + row["office"] ][row["gage"] ] = \ + row[ "usace_gage_id" ] + else: + self._office_name1_to_index[ row["office"] ] = \ + dict( { row["gage"] : \ + row[ "usace_gage_id" ] } ) + + line_count += 1 + + print('Processed ' + str( line_count ) + ' lines.') + self.office_name1_to_index = self._office_name1_to_index + + + @property + def source(self): + return self._source + + @source.setter + def source(self, s): + self._source = s + + + @property + def office_name1_to_index(self): + return self._office_name1_to_index + + @office_name1_to_index.setter + def office_name1_to_index(self, o): + self._office_name1_to_index=o + + def getIndex(self, office, name1 ): + + if ( not office in self.office_name1_to_index.keys() ) or \ + ( not name1 in self.office_name1_to_index[office].keys() ): + + raise KeyError( "Cannot find station for " + office + ' ' + \ + name1 + " in the site file: " + \ + self.source + '.' ) + + print( "Found station: " + office + ' ' + name1 ) + return self.office_name1_to_index[ office][name1 ] diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/CWMS_outflow_sites_263_index.csv b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/CWMS_outflow_sites_263_index.csv new file mode 100755 index 0000000..174e07a --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/CWMS_outflow_sites_263_index.csv @@ -0,0 +1,255 @@ +USACE_site_index,office,CWMS_ID,Data_type,latitude,longitude,data_Freq,name3,public_nam,long_name,descriptio,horizontal,estimate,vertical_d,timezone,county,state,nation,nearest_ci,bounding_o,location_k,location_t,name_1,Alternate,Mins,Obs_type,Elev_ft +0,SAJ,S351-Structure,Flow,26.69722,-80.71389,Inst, , , , ,WGS84,Unknown County or County N/A,FL,UNITED STATES, ,SAJ,SITE, , , , ,S351-Structure.Flow.Inst.0.0.SFWMD-db-raw, ,0,SFWMD-db-raw,0 +1,SAJ,S4-Pump,Flow,26.78944,-80.96194,Inst, , , , ,WGS84,Unknown County or County N/A,FL,UNITED STATES, ,SAJ,SITE, , , , ,S4-Pump.Flow.Inst.0.0.SFWMD-db-raw, ,0,SFWMD-db-raw,0 +2,SAJ,S135-Pump,Flow,27.08611,-80.66139,Inst, , , , ,WGS84,Unknown County or County N/A,FL,UNITED STATES, ,SAJ,SITE, , , , ,S135-Pump.Flow.Inst.1Hour.0.SFWMD-WM, ,0,SFWMD-WM,0 +3,SAJ,S133,Flow,27.20583,-80.80083,Inst, ,S-133, , ,WGS84,Unknown County or County N/A,FL,UNITED STATES, ,SAJ,SITE, , , , ,S133.Flow.Inst.1Hour.0.SFWMD-WM, ,0,SFWMD-WM,0 +4,SWF,TX00004-Gated_Total,Flow-Out,29.86861,-98.19861,Ave,CANYON LAKE-Gated_Total, , , ,WGS124,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Fischer, TX",SWF,SITE, ,TX00004-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,CANYON LAKE-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,0,Rev-SWF-REGI,0 +5,SWF,TX00013-Gated_Total,Flow-Out,30.32222,-96.52556,Ave,SOMERVILLE LAKE-Gated_Total, , , ,WGS126,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Somerville, TX",SWF,SITE, ,TX00013-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,SOMERVILLE LAKE-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,0,Rev-SWF-REGI,0 +6,SWF,GGLT2-Gated_Total,Flow-Out,30.6675,-97.72722,Ave,NORTH SAN GABRIEL DAM-Gated_Total, , , ,WGS100,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Georgetown, TX",SWF,SITE, ,GGLT2-Gated_Total.Flow-Out.Ave.~1Day.1Day.Reporting,GGLT2-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,0,Reporting,0 +7,SWF,TX08005-Gated_Total,Flow-Out,30.69278,-97.32611,Ave,GNGT2-Gated_Total, , , ,WGS102,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Davilla, TX",SWF,SITE, ,TX08005-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,GNGT2-Gated_Total.Flow-Out.Ave.~1Day.1Day.Reporting,0,Rev-SWF-REGI,0 +8,SWF,TX00015-Gated_Total,Flow-Out,30.79528,-94.18,Ave,TBLT2-Gated_Total, , , , ,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Spurger, TX",SWG,SITE, ,TX00015-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,TBLT2-Gated_Total.Flow-Out.Ave.~1Day.1Day.Reporting,0,Rev-SWF-REGI,0 +9,SWF,TX00014-Gated_Total,Flow-Out,31.02222,-97.5325,Ave,STILLHOUSE-HOLLOW DAM-Gated_Total, , , ,WGS128,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Belton, TX",SWF,SITE, ,TX00014-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,STILLHOUSE-HOLLOW DAM-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,0,Rev-SWF-REGI,0 +10,SWF,TX00011-Gated_Total,Flow-Out,31.06056,-94.10583,Ave,JSPT2-Gated_Total, , , , ,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Brookeland, TX",SWF,SITE, ,TX00011-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,JSPT2-Gated_Total.Flow-Out.Ave.~1Day.1Day.Reporting,0,Rev-SWF-REGI,0 +11,SWF,TX00002-Gated_Total,Flow-Out,31.10611,-97.47444,Ave,BELTON LAKE-Gated_Total, , , , ,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Belton, TX",SWF,SITE, ,TX00002-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,BELTON LAKE-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,0,Rev-SWF-REGI,0 +12,SAM,AL01433-Spillway,Flow,31.25916,-85.11116,Inst,Andrews-Spillway,Andrews Gated Spillway, , ,NAD83, , ,US/Eastern,Early,GA,UNITED STATES,"Coumbia, AL",SAM,OUTLET, ,AL01433-Spillway.Flow.Inst.1Hour.0.Raw-SCADA_SAM,Andrews-Spillway.Flow.Inst.1Hour.0.Raw-SCADA_SAM,0,Raw-SCADA_SAM,0 +13,SWF,TX00012-Gated_Total,Flow-Out,31.48444,-100.4814,Ave,O C FISHER DAM AND LAKE-Gated_Total, , , ,WGS120,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"San Angelo, TX",SWF,SITE, ,TX00012-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,O C FISHER DAM AND LAKE-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,0,Rev-SWF-REGI,0 +14,SWF,TX00016-Gated_Total,Flow-Out,31.584,-97.202,Ave,ACTT2-Gated_Total, , , ,WGS84,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Waco, TX",SWF,SITE, ,TX00016-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,ACTT2-Gated_Total.Flow-Out.Ave.~1Day.1Day.Reporting,0,Rev-SWF-REGI,0 +15,SWF,TX00006-Gated_Total,Flow-Out,31.83278,-99.56056,Ave,HORDS CREEK LAKE-Gated_Total, , , ,WGS106,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Valera, TX",SWF,SITE, ,TX00006-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,HORDS CREEK LAKE-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,0,Rev-SWF-REGI,0 +16,SWF,TX00017-Gated_Total,Flow-Out,31.86528,-97.37167,Ave,WHITNEY LAKE-Gated_Total, , , ,WGS132,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Laguna Park, TX",SWF,SITE, ,TX00017-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,WHITNEY LAKE-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,0,Rev-SWF-REGI,0 +17,SWF,TX00009-Gated_Total,Flow-Out,31.95,-96.7,Ave,DAWT2-Gated_Total, , , , ,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Dawson, TX",SWF,SITE, ,TX00009-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,DAWT2-Gated_Total.Flow-Out.Ave.~1Day.1Day.Reporting,0,Rev-SWF-REGI,0 +18,SWF,TX00010-Gated_Total,Flow-Out,31.9717,-98.4767,Ave,PCTT2-Gated_Total, , , ,WGS116,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Hasse, TX",SWF,SITE, ,TX00010-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,PCTT2-Gated_Total.Flow-Out.Ave.~1Day.1Day.Reporting,0,Rev-SWF-REGI,0 +19,SWF,TX00001-Gated_Total,Flow-Out,32.25,-96.64,Ave,BARDWELL LAKE-Gated_Total, , , , ,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Bardwell, TX",SWF,SITE, ,TX00001-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,BARDWELL LAKE-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,0,Rev-SWF-REGI,0 +20,SWF,GBYT2-Pump,Flow-Out,32.37417,-97.68889,Ave, , , , , ,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Granbury, TX",SWF,SITE, ,GBYT2-Pump.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI, ,0,Rev-SWF-REGI,0 +21,SAM,MS01491-Spillway,Flow-Discharge,32.4749,-88.79725,Inst,Okatibbee-Spillway,Okatibbee Spillway, , , , , ,US/Central,Lauderdale,MS,UNITED STATES,Meridian,SAM,OUTLET, ,MS01491-Spillway.Flow-Discharge.Inst.15Minutes.0.Raw-CCP,Okatibbee-Spillway.Flow-Discharge.Inst.15Minutes.0.Raw-CCP,0,Raw-CCP,0 +22,SAM,AL01422-Spillway,Flow,32.53531,-85.88785,Inst,Thurlow-Spillway,Thurlow Spillway, , , , , ,US/Central,Unknown County or County N/A,AL,UNITED STATES,Tuskegee,SAM,OUTLET, ,AL01422-Spillway.Flow.Inst.1Hour.0.Raw-APCO,Thurlow-Spillway.Flow.Inst.1Hour.0.Raw-APCO,0,Raw-APCO,0 +23,SAM,Bouldin,Flow-Out,32.58333,-86.28258,Inst, ,Bouldin (APC),Bouldin (APC), ,NAD83, , ,US/Central,Elmore,AL,UNITED STATES, ,SAM,PROJECT, ,Bouldin.Flow-Out.Inst.1Hour.0.Raw-APCO, ,0,Raw-APCO,0 +24,SWF,TX08007-Gated_Total,Flow-Out,32.645,-96.9933,Ave,JOE POOL LAKE-Gated_Total, , , , ,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Florence Hill, TX",SWF,SITE, ,TX08007-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,JOE POOL LAKE-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,0,Rev-SWF-REGI,0 +25,SWF,TX00003-Gated_Total,Flow-Out,32.65056,-97.44833,Ave,BENBROOK LAKE-Gated_Total, , , , ,FALSE,NATIVE,US/Central,Tarrant,TX,UNITED STATES,"Benbrook, TX",SWF,SITE, ,TX00003-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,BENBROOK LAKE-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,0,Rev-SWF-REGI,0 +26,SAM,BartlettsFerry-Powerhouse,Flow,32.66302,-85.09085,Inst, ,BartlettsFerry Powerhouse, , , , , ,US/Eastern,Harris,GA,UNITED STATES,Columbus,SAM,TURBINE, ,BartlettsFerry-Powerhouse.Flow.Inst.1Hour.0.Raw-GPC, ,0,Raw-GPC,0 +27,SWF,CLDL1-Spillway,Flow-Out,32.70333,-93.92,Ave, ,Spillway,Spillway, , ,FALSE,NATIVE,US/Central,Unknown County or County N/A,LA,UNITED STATES,"Mooringsport, LA",MVK,OUTLET, ,CLDL1-Spillway.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI, ,0,Rev-SWF-REGI,0 +28,MVK,D109C1B8,Flow,32.74944,-94.49861,Inst,Jferson_BigC_Bay,Big Cypress at Jefferson,"Big Cypress Bayou @ Jefferson, TX","Big Cypress Bayou @ Jefferson, TX",NAD27,FALSE,NATIVE,America/Chicago,Marion,TX,UNITED STATES, , ,SITE,Stream Gauge,D109C1B8.Flow.Inst.15Minutes.0.DCP-rev,Jferson_BigC_Bay.Flow.Inst.15Minutes.0.DCP-rev,0,DCP-rev,180 +29,SWF,FLWT2-Pump,Flow-Out,32.78917,-97.41611,Ave, , , , , ,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Sansom Park, TX",SWF,SITE, ,FLWT2-Pump.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI, ,0,Rev-SWF-REGI,0 +30,SWF,FRHT2-Pump,Flow-Out,32.8,-96.5,Ave, , , , , ,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Sunnyvale, TX",SWF,SITE, ,FRHT2-Pump.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI, ,0,Rev-SWF-REGI,0 +31,MVK,Renfroe,Flow,32.86222,-89.44333,Inst, ,"Lobutcha Creek @ Renfroe, MS","Lobutcha Creek @ Renfroe, MS","Lobutcha Creek @ Renfroe, MS",NAD27,FALSE,NATIVE,US/Central,Leake,MS, , , ,SITE, ,Renfroe.Flow.Inst.15Minutes.0.DCP-rev, ,0,DCP-rev,354 +32,SWF,TX00005-Gated_Total,Flow-Out,32.9725,-97.05611,Ave,GPVT2-Gated_Total, , , , ,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Coppell, TX",SWF,SITE, ,TX00005-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,GPVT2-Gated_Total.Flow-Out.Ave.~1Day.1Day.Reporting,0,Rev-SWF-REGI,0 +33,SWF,TX00007-Gated_Total,Flow-Out,33.03179,-96.48249,Ave,LAVON LAKE-Gated_Total, , , , ,FALSE,NATIVE,Etc/GMT+6,Unknown County or County N/A,TX,UNITED STATES,"Wylie, TX",SWF,SITE, ,TX00007-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,LAVON LAKE-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,0,Rev-SWF-REGI,0 +34,SWF,TX00008-Gated_Total,Flow-Out,33.06917,-96.96417,Ave,LEWISVILLE LAKE-Gated_Total, , , , ,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Lewisville, TX",SWF,SITE, ,TX00008-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,LEWISVILLE LAKE-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,0,Rev-SWF-REGI,0 +35,MVK,Spring Bank,Flow,33.08944,-93.85944,Inst, ,"Red River @ Spring Bank, AR","Red River @ Spring Bank, AR","Red River @ Spring Bank, AR",NAD27,FALSE,NATIVE,CST6CDT,Miller,AR, , , ,SITE,Stream Gauge,Spring Bank.Flow.Inst.30Minutes.0.DCP-rev, ,0,DCP-rev,160 +36,SAM,AL01426-Spillway,Flow,33.25306,-87.44917,Inst,Holt-Spillway,Holt Spillway, , , , , ,US/Central,Tuscaloosa,AL,UNITED STATES,Northport,SAM,OUTLET, ,AL01426-Spillway.Flow.Inst.1Hour.0.Raw-APCO,Holt-Spillway.Flow.Inst.1Hour.0.Raw-APCO,0,Raw-APCO,0 +37,SWF,TX08012-Gated_Total,Flow-Out,33.3356,-95.6361,Ave,JIM CHAPMAN LAKE-Gated_Total, , , , ,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Cooper, TX",SWF,SITE, ,TX08012-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,JIM CHAPMAN LAKE-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,0,Rev-SWF-REGI,0 +38,SWF,TX08008-Gated_Total,Flow-Out,33.3567,-97.0367,Ave,RAY ROBERTS DAM-Gated_Total, , , , ,FALSE,NATIVE,US/Central,Unknown County or County N/A,TX,UNITED STATES,"Aubrey, TX",SWF,SITE, ,TX08008-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,RAY ROBERTS DAM-Gated_Total.Flow-Out.Ave.~1Day.1Day.Rev-SWF-REGI,0,Rev-SWF-REGI,0 +39,SAM,AL01427-Spillway,Flow,33.45833,-87.35417,Inst,Bankhead-Spillway,Bankhead Spillway, , ,NAD27,FALSE,NATIVE,Etc/GMT+6,Tuscaloosa,AL,UNITED STATES, ,SAM,OUTLET, ,AL01427-Spillway.Flow.Inst.1Hour.0.Raw-APCO,Bankhead-Spillway.Flow.Inst.1Hour.0.Raw-APCO,0,Raw-APCO,0 +40,SAM,MS03056-Spillway,Flow,33.51816,-88.48799,Inst,Stennis-Spillway,Stennis Spillway, , , , , ,US/Central,Lowndes,MS,UNITED STATES,Columbus,SAM,OUTLET, ,MS03056-Spillway.Flow.Inst.15Minutes.0.Raw-LRIT_TTWW,Stennis-Spillway.Flow.Inst.15Minutes.0.Raw-LRIT_TTWW,0,Raw-LRIT_TTWW,0 +41,SWL,AR00536-Tailwater,Flow,33.68717,-93.96391,Inst,Millwood_Dam-Tailwater,Millwood Dam Tailwater,"Little at Millwood Dam (TW), AR","Little at Millwood Dam (TW), AR - Site collects tailwater elevation and precip data (HT,PC)",NAD83,FALSE,NATIVE,America/Chicago,Unknown County or County N/A,0,UNITED STATES,McNab,SWL,SITE, ,AR00536-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,Millwood_Dam-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,0,CCP-Comp,0 +42,SAM,HNHenry-Spillway,Flow,33.78394,-86.05315,Inst, ,HN Henry Spillway, , , , , ,US/Central,Calhoun,AL,UNITED STATES,Gadsden,SAM,OUTLET, ,HNHenry-Spillway.Flow.Inst.1Hour.0.Raw-APCO, ,0,Raw-APCO,0 +43,SPL,Coyote Ck-at Lower SGR,Flow,33.81,-118.0769,Inst, ,Coyote Creek, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Hawaiian Gardens, ,SITE, ,Coyote Ck-at Lower SGR.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,7 +44,SPL,Spring St-SGR in Long Beach,Flow,33.81167,-118.0911,Inst, ,Spring Street, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Hawaiian Gardens, ,SITE, ,Spring St-SGR in Long Beach.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,12 +45,SPL,Villa Park DS-Santiago Ck,Flow,33.81472,-117.765,Inst, ,Villa Park Downstream, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Orange, ,SITE, ,Villa Park DS-Santiago Ck.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,490 +46,SPL,Wardlow-LAR at Long Beach,Flow,33.8175,-118.2064,Inst, ,Wardlow, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Carson, ,SITE, ,Wardlow-LAR at Long Beach.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,12 +47,SPL,Richman Ave-Fullerton Ck blw Fullerton,Flow,33.86306,-117.9328,Inst, ,Richman Ave, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Fullerton, ,SITE, ,Richman Ave-Fullerton Ck blw Fullerton.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,125 +48,SPL,Compton Ck-at Lower LAR,Flow,33.88139,-118.2244,Inst, ,Compton Creek, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Willowbrook, ,SITE, ,Compton Ck-at Lower LAR.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,50 +49,SPL,Carbon Canyon DS-Carbon Ck,Flow,33.91333,-117.8425,Inst, ,Carbon Canyon DS, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Brea, ,SITE, ,Carbon Canyon DS-Carbon Ck.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,403 +50,SPL,Florence Ave-SGR in Downey,Flow,33.93056,-118.1067,Inst, ,Florence Avenue, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Downey, ,SITE, ,Florence Ave-SGR in Downey.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,103 +51,SPL,Stewart and Gray-Rio Hondo blw WN,Flow,33.94583,-118.1631,Inst, ,Stewart and Gray, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Bell Gardens, ,SITE, ,Stewart and Gray-Rio Hondo blw WN.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,90 +52,SPL,Firestone Blvd-LAR abv Rio Hondo,Flow,33.94889,-118.1739,Inst, ,Firestone Blvd, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Cudahy, ,SITE, ,Firestone Blvd-LAR abv Rio Hondo.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,96 +53,SPL,Ballona Ck-Marina Del Rey,Flow,33.99833,-118.4022,Inst, ,Ballona Creek, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Culver City, ,SITE, ,Ballona Ck-Marina Del Rey.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,12 +54,SPL,Parkway-SGR blw WN,Flow,34.01278,-118.0639,Inst, ,Parkway, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Pico Rivera, ,SITE, ,Parkway-SGR blw WN.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,180 +55,SPL,Alhambra W-abv WN Rio Hondo,Flow,34.05611,-118.0875,Inst, ,Alhambra Wash, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Rosemead, ,SITE, ,Alhambra W-abv WN Rio Hondo.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,240 +56,SPL,Walnut Ck-abv WN San Gabriel,Flow,34.06556,-117.9633,Inst, ,Walnut Creek, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Baldwin Park, ,SITE, ,Walnut Ck-abv WN San Gabriel.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,325 +57,SPL,Big Dalton W-abv WN San Gabriel,Flow,34.07472,-117.9636,Inst, ,Big Dalton Wash, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Baldwin Park, ,SITE, ,Big Dalton W-abv WN San Gabriel.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,335 +58,SWL,AR01201-Tailwater,Flow,34.09341,-94.38103,Inst,DeQueen_Dam-Tailwater,DeQueen Dam Tailwater,"Rolling Fork below DeQueen Dam, AR","Rolling Fork below DeQueen Dam, AR - platform collects tailwater elevation and water temp data (HT,TW)",WGS84,FALSE,NATIVE,America/Chicago,Sevier,AR,UNITED STATES,DeQueen, ,SITE, ,AR01201-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,DeQueen_Dam-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,0,CCP-Comp,0 +59,SPL,Tujunga Ave-LAR blw Tujunga W,Flow,34.14111,-118.3797,Inst, ,Tujunga Avenue, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,West Hollywood, ,SITE, ,Tujunga Ave-LAR blw Tujunga W.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,550 +60,SWL,AR01202-Tailwater,Flow,34.14306,-94.095,Inst,Dierks_Dam-Tailwater,Dierks Dam Tailwater,"Saline River at Dierks Dam (TW), AR","Saline River at Dierks Dam (TW), AR - platform collects tailwater elevation and water temp data (HT,TW)",WGS84,FALSE,NATIVE,America/Chicago,Unknown County or County N/A,0,UNITED STATES,Dierks,SWL,SITE, ,AR01202-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,Dierks_Dam-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,0,CCP-Comp,0 +61,SWL,OK10307,Flow-Res Out,34.14306,-94.68333,Ave,BKDO2,Broken Bow Dam,"Mountain Fork at Broken Bow Dam, OK - platform collects pool","Mountain Fork at Broken Bow Dam, OK - platform collects pool elevation, tailwater elevation, and precip (HP,HT,PC)",NAD83, , ,America/Chicago,Unknown County or County N/A,OK,UNITED STATES,Broken Bow,SWT,PROJECT,Corps Reservoir,OK10307.Flow-Res Out.Ave.1Hour.1Hour.Rev-Regi-Flowgroup,07338900.Flow-Res Out.Ave.1Hour.1Hour.Rev-Regi-Flowgroup,0,Rev-Regi-Flowgroup,0 +62,SPL,Verdugo W-at LAR abv Arroyo Seco,Flow,34.15611,-118.2739,Inst, ,Verdugo Wash, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Burbank, ,SITE, ,Verdugo W-at LAR abv Arroyo Seco.Flow.Inst.~15Minutes.0.GOES-rev, ,0,GOES-rev,470 +63,SWL,AR01200-Tailwater,Flow,34.20149,-94.22798,Inst,Gillham_Dam-Tailwater,Gillham Dam Tailwater,"Cossatot River at Gillham Dam (TW), AR","Cossatot River at Gillham Dam, AR - platform collects tailwater elevation and water temp data (HT,TW)", ,FALSE,NATIVE,America/Chicago,Howard,AR,UNITED STATES,Gillham,SWL,SITE, ,AR01200-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,Gillham_Dam-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,0,CCP-Comp,0 +64,SPL,CA10004-abv Hansen,Flow,34.26056,-118.2775,Inst,Haines Cnyn-abv Hansen,Haines Canyon Debris Basin, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,La Crescenta-Montrose, ,PROJECT, ,CA10004-abv Hansen.Flow.Inst.~15Minutes.0.GOES-rev,Haines Cnyn-abv Hansen.Flow.Inst.~15Minutes.0.GOES-rev,0,GOES-rev,2183 +65,SPL,CA10019,Flow,34.26056,-118.3861,Inst,Hansen,Hansen Dam, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,San Fernando, ,PROJECT, ,CA10019.Flow.Inst.~15Minutes.0.GOES-rev,Hansen.Flow.Inst.~15Minutes.0.GOES-rev,0,GOES-rev,990 +66,SPL,Hansen,Flow,34.26056,-118.3861,Inst,CA10019,Hansen Dam, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,San Fernando, ,PROJECT, ,Hansen.Flow.Inst.~15Minutes.0.GOES-rev,CA10019.Flow.Inst.~15Minutes.0.GOES-rev,0,GOES-rev,990 +67,MVK,Langley,Flow,34.31184,-93.89976,Inst, ,Langley,"Little Missouri River @ Langley, AR","Little Missouri River @ Langley, AR",NAD83,FALSE,NATIVE,Etc/GMT+6,Pike,AR,UNITED STATES, , ,SITE,Stream Gauge,Langley.Flow.Inst.15Minutes.0.DCP-rev, ,0,DCP-rev,727 +68,SAM,MS03604-Spillway,Flow,34.4634,-88.36497,Inst,Montgomery-Spillway,Montgomery Spillway, , ,NAD83,FALSE,NATIVE,US/Central,Itawamba,MS,UNITED STATES,Corinth,SAM,OUTLET, ,MS03604-Spillway.Flow.Inst.1Hour.0.Raw-LRIT_TTWW,Montgomery-Spillway.Flow.Inst.1Hour.0.Raw-LRIT_TTWW,0,Raw-LRIT_TTWW,340 +69,SAM,GA00821-Spillway,Flow,34.61397,-84.67105,Inst,Carters-Spillway,Carters Emergency Spillway, , ,NAD27,FALSE,NATIVE,US/Central,Murray,GA,UNITED STATES,"Carters, GA",SAM,OUTLET, ,GA00821-Spillway.Flow.Inst.1Hour.0.Raw-SCADA_SAM,Carters-Spillway.Flow.Inst.1Hour.0.Raw-SCADA_SAM,0,Raw-SCADA_SAM,0 +70,MVK,MS01496-Spillway,Flow,34.75722,-90.12444,Inst,Arkabutla Lake-Spillway,Spillway,Spillway,"300 foot concrete emergency Spillway, crest elevation 238.3 ft",NAD83,FALSE,NATIVE,CST6CDT,Tate,MS,UNITED STATES, , ,OUTLET, ,MS01496-Spillway.Flow.Inst.1Hour.0.Spillway Flow,Arkabutla Lake-Spillway.Flow.Inst.1Hour.0.Spillway Flow,0,Spillway Flow,0 +71,SWL,AR00158-Tailwater,Flow,34.95183,-93.15253,Inst,Nimrod_Dam-Tailwater,Nimrod Dam Tailwater,"Fourche LaFave River at Nimrod Dam (TW), AR","Fourche LaFave River at Nimrod Dam, AR - platform collects tailwater elevation data (HT)",NAD83,FALSE,NATIVE,America/Chicago,Perry,AR,UNITED STATES,Hollis,SWL,SITE, ,AR00158-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,Nimrod_Dam-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,0,CCP-Comp,0 +72,SPL,CA10197,Flow,34.98417,-120.3228,Inst,Twitchell,Twitchell Dam, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,CA,UNITED STATES,Arroyo Grande, ,PROJECT, ,CA10197.Flow.Inst.1Hour.0.GOES-rev,Twitchell.Flow.Inst.1Hour.0.GOES-rev,0,GOES-rev,692 +73,LRN,PICT1-PICKWICK,Flow,35.07083,-88.25111,Ave, ,Pickwick Lock & Dam,Pickwick, ,NAD83, , ,US/Central,Hardin,TN,UNITED STATES,Counce, ,SITE,Reservoir,PICT1-PICKWICK.Flow.Ave.1Hour.1Hour.tva-raw, ,0,tva-raw,0 +74,SWL,AR00157-Tailwater,Flow,35.1047,-93.6314,Inst,Blue_Mtn_Dam-Tailwater,Blue Mountain Dam Tailwater,"Petit Jean River at Blue Mountain Dam Tailwater, AR","Petit Jean River at Blue Mountain Dam Tailwater Sensor, AR - Represents sensor that Collects Tailwater Elevation data (HT)",NAD83,FALSE,NATIVE,US/Central,Yell,AR,UNITED STATES,Waveland,SWL,SITE, ,AR00157-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,Blue_Mtn_Dam-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,0,CCP-Comp,0 +75,SWT,CORD,Flow,35.29111,-98.83639,Inst,AE0050C0,"Washita River, nr Cordell 9E","Washita River, nr Cordell 9E","Washita River, nr Cordell 9E",NAD83, , ,US/Central,Unknown County or County N/A,OK,UNITED STATES,"Cordell, OK",SWT,SITE, ,CORD.Flow.Inst.15Minutes.0.Ccp-Rev,AE0050C0.Flow.Inst.15Minutes.0.Ccp-Rev,0,Ccp-Rev,0 +76,SWL,OZGA4-Tailwater,Flow,35.46977,-93.81382,Inst,AR00164-Tailwater,Lock & Dam 12 - Tailwater,"Arkansas River at Lock & Dam 12 (Ozark) Tailwater, AR","Arkansas River at Lock & Dam 12 (Ozark) Tailwater Sensor, AR - Represents sensor that Collects Tailwater Elevation data (HT)",NAD83,FALSE,NATIVE,America/Chicago,Franklin,AR,UNITED STATES,"Ozark, AR",SWL,SITE, ,OZGA4-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,AR00164-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,0,CCP-Comp,0 +77,SWL,LD12_Ozark,Flow-RelTotal,35.47333,-93.81,Inst,AR00164,Ozark Lock & Dam 12,"Arkansas River at Lock & Dam 12 (Ozark), AR","Arkansas River at Lock & Dam 12 (Ozark), AR - platform collects upstream and downstream elevation, nonpower release, power release, generation, and gate opening data.",NAD83,FALSE,NATIVE,America/Chicago,Unknown County or County N/A,AR,UNITED STATES,"Ozark, AR",SWL,PROJECT,RoR Project,LD12_Ozark.Flow-RelTotal.Inst.1Hour.0.CCP-Comp,AR00164.Flow-RelTotal.Inst.1Hour.0.CCP-Comp,0,CCP-Comp,0 +78,LRN,DBLT1-BigRockCr-nrDoubleBridgesTN,Flow,35.50444,-86.76861,Ave, ,Big Rock Cr @ Double Bridges TN,Big Rock Creek at Double Bridges TN, ,NAD83, , ,US/Central,Marshall,TN,UNITED STATES,Double Springs, ,SITE,Stream Gauge,DBLT1-BigRockCr-nrDoubleBridgesTN.Flow.Ave.~1Day.1Day.dcp-rev, ,0,dcp-rev,0 +79,SWL,GRRA4-Tailwater,Flow,35.51941,-91.99532,Inst,AR00173-Tailwater,Greers Ferry Dam - Tailwater,"Little Red River at Greers Ferry Dam Tailwater, AR","Little Red River at Greers Ferry Dam Tailwater Sensor, AR - Represents sensor that Collects Tailwater Elevation data (HT)",WGS84,FALSE,NATIVE,America/Chicago,Cleburne,AR,UNITED STATES,Heber Springs,SWL,SITE, ,GRRA4-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,AR00173-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,0,CCP-Comp,0 +80,LRN,MLTT1-DuckR-abvMilltownTN,Flow,35.57639,-86.77889,Ave, ,Duck River abv Milltown TN,Duck River above Milltown TN near Lewisburg TN, ,NAD83, , ,US/Central,Marshall,TN,UNITED STATES,Milltown, ,SITE,Stream Gauge,MLTT1-DuckR-abvMilltownTN.Flow.Ave.~1Day.1Day.dcp-rev, ,0,dcp-rev,0 +81,SPK,ISB ISBQ-Lake Isabella Outflow-Kern,Flow,35.63972,-118.4994,Ave, ,Lk Isabella-Kern R, ,ISB - Lk Isabella Main Dam Outflow to the Kern River, ,FALSE,NATIVE,US/Pacific,Kern,CA,UNITED STATES,"Lake Isabella, CA",SPK,SITE, ,ISB ISBQ-Lake Isabella Outflow-Kern.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,2446 +82,SPK,CA10106-Lake Isabella Pool-Kern,Flow-Spill,35.64667,-118.4781,Inst,ISB ISBP-Lake Isabella Pool-Kern,Lk Isabella-Pool,Isabella Pool,ISB - Lake Isabella Pool Elevation Station @ Main Dam,WGS84,FALSE,NATIVE,US/Pacific,Kern,CA,UNITED STATES, ,SPK,SITE,Corps Reservoir,CA10106-Lake Isabella Pool-Kern.Flow-Spill.Inst.1Hour.0.Calc-val,ISB ISBP-Lake Isabella Pool-Kern.Flow-Spill.Inst.1Hour.0.Calc-val,0,Calc-val,2835 +83,LRN,GRTT1-GREAT_FALLS,Flow,35.80806,-85.63361,Ave, ,Rock Island TN,Great Falls Dam Near Rock Island, ,NAD83, , ,US/Central,Warren,TN,UNITED STATES,Rock Island, ,SITE,Reservoir,GRTT1-GREAT_FALLS.Flow.Ave.1Hour.1Hour.tva-raw, ,0,tva-raw,0 +84,LRN,VERT1-PineyR-VernonTN,Flow,35.87111,-87.50139,Ave, ,Piney River @ Vernon TN,Piney River At Vernon, ,NAD83, , ,US/Central,Hickman,TN,UNITED STATES,Vernon, ,SITE,Stream Gauge,VERT1-PineyR-VernonTN.Flow.Ave.~1Day.1Day.dcp-rev, ,0,dcp-rev,0 +85,LRN,MUGT1-WFkStonesR-MurfreesboroTN,Flow,35.9025,-86.42861,Ave, ,WF Stones R nr Murfreesboro TN,West Fork Stones River Near Murfreesboro TN, ,NAD83,FALSE,NATIVE,US/Central,Rutherford,TN,UNITED STATES,Murfreesboro, ,SITE,Stream Gauge,MUGT1-WFkStonesR-MurfreesboroTN.Flow.Ave.~1Day.1Day.dcp-rev, ,0,dcp-rev,515 +86,LRN,HBFT1-HarpethR-blwFranklinTN,Flow,35.94806,-86.88167,Ave, ,Harpeth River blw Franklin TN,Harpeth River Below Franklin TN, ,NAD83, , ,US/Central,Williamson,TN,UNITED STATES,Franklin, ,SITE,Stream Gauge,HBFT1-HarpethR-blwFranklinTN.Flow.Ave.~1Day.1Day.dcp-rev, ,0,dcp-rev,0 +87,SPK,SCC SCCQ-Success Lake Outflow-Tule,Flow,36.05639,-118.9228,Ave, ,Success Lk-Tule R, ,SCC - Success Lk Outflow to the Tule River, ,FALSE,NATIVE,US/Pacific,Tulare,CA,UNITED STATES,"Worth, CA",SPK,SITE, ,SCC SCCQ-Success Lake Outflow-Tule.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,536 +88,SPK,CA10113,Flow-Spill,36.06111,-118.9217,Inst,SCC SCCP-Success Lake Pool-Tule,Success Lk-Pool,Success Lake / Dam,SCC - Success Lk Pool Elevation Station, ,FALSE,NATIVE,US/Pacific,Tulare,CA,UNITED STATES,"Worth, CA",SPK,SITE, ,CA10113.Flow-Spill.Inst.1Hour.0.Calc-val,SCC SCCP-Success Lake Pool-Tule.Flow-Spill.Inst.1Hour.0.Calc-val,0,Calc-val,692 +89,LRN,ANTT1-MillCr-AntiochTN,Flow,36.08167,-86.68083,Ave,3431000-MillCr-AntiochTN,Mill Creek @ Antioch TN,Mill Creek At Antioch TN, ,NAD83,FALSE,NATIVE,US/Central,Davidson,TN,UNITED STATES,Antioch, ,SITE,Stream Gauge,ANTT1-MillCr-AntiochTN.Flow.Ave.~1Day.1Day.dcp-rev,3431000-MillCr-AntiochTN.Flow.Ave.~1Day.1Day.dcp-rev,0,dcp-rev,473 +90,LRN,CETT1-CENTER_HILL,Flow-Orifice,36.0976,-85.82612,Ave, ,Center Hill Dam Tailwater,Center Hill Dam Tailwater, ,NAD83, , ,US/Central,De Kalb,TN,UNITED STATES,Cookeville, ,SITE,Tailwater,CETT1-CENTER_HILL.Flow-Orifice.Ave.1Hour.1Hour.man-raw, ,0,man-raw,0 +91,LRN,CEN,Flow,36.09763,-85.82672,Inst,CEHT1,Center Hill Dam,Center Hill Dam, ,NAD83, , ,US/Central,De Kalb,TN,UNITED STATES,Gordonsville, ,PROJECT,Corps Reservoir,CEN.Flow.Inst.15Minutes.0.DCP-rev,01114500.Flow.Inst.15Minutes.0.DCP-rev,0,DCP-rev,0 +92,SWF,KEYS,Flow-Res Out,36.15139,-96.25139,Ave, ,KEYS,KEYS,ARKANSAS RIVER AT KEYSTONE DAM, , , ,US/Central,Unknown County or County N/A,OK,UNITED STATES,"Sand Springs, OK",SWT,SITE,stream,KEYS.Flow-Res Out.Ave.1Hour.1Hour.Rev-Regi-Flowgroup,07164200.Flow-Res Out.Ave.1Hour.1Hour.Rev-Regi-Flowgroup,0,Rev-Regi-Flowgroup,0 +93,SWT,BISO,Flow,36.18861,-97.96194,Inst,BTCO2,"Turkey Creek nr Bison, OK","Turkey Creek nr Bison, OK","Turkey Creek nr Bison, OK",NAD83, , ,US/Central,Unknown County or County N/A,OK,UNITED STATES,"Bison, OK",SWT,SITE, ,BISO.Flow.Inst.15Minutes.0.Ccp-Rev,BTCO2.Flow.Inst.15Minutes.0.Ccp-Rev,0,Ccp-Rev,0 +94,SWT,AMES,Flow,36.21861,-98.25445,Inst,AE011130,Cimarron River nr Ames 4SW,Cimarron River nr Ames 4SW,Cimarron River nr Ames 4SW,NAD83, , ,US/Central,Unknown County or County N/A,OK,UNITED STATES,"Ames, OK",SWT,SITE, ,AMES.Flow.Inst.1Hour.0.Ccp-Rev,AE011130.Flow.Inst.1Hour.0.Ccp-Rev,0,Ccp-Rev,0 +95,LRN,LBST1-SpringCr-LebanonTN,Flow,36.22167,-86.23694,Ave, ,Spring Creek nr Lebanon TN,Spring Creek nr Lebanon TN, ,NAD83,FALSE,NATIVE,US/Central,Wilson,TN,UNITED STATES,Lebanon, ,SITE,Stream Gauge,LBST1-SpringCr-LebanonTN.Flow.Ave.~1Day.1Day.dcp-rev, ,0,dcp-rev,500 +96,SWL,NFDA4-Tailwater,Flow,36.2468,-92.24106,Inst,AR00159-Tailwater,Norfork Dam - Tailwater,"North Fork River at Norfork Dam Tailwater, AR","North Fork River at Norfork Dam Tailwater Sensor, AR - Represents sensor that Collects Tailwater Elevation data (HP)",WGS84,FALSE,NATIVE,America/Chicago,Baxter,AR,UNITED STATES,Norfork,SWL,SITE, ,NFDA4-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,AR00159-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,0,CCP-Comp,0 +97,LRN,CTHT1-CumberlandR-CarthageTN,Flow,36.2475,-85.95639,Ave, ,Cumberland R @ Carthage TN,Cumberland River At Carthage TN, ,NAD83,FALSE,NATIVE,US/Central,Smith,TN,UNITED STATES,Carthage, ,SITE,Stream Gauge,CTHT1-CumberlandR-CarthageTN.Flow.Ave.~1Day.1Day.dcp-rev, ,0,dcp-rev,438 +98,LRN,CORDELL_HULL-Spillway,Flow,36.29034,-85.94354,Total, ,Cordell Hull Spillway Gates,Cordell Hull Lock and Dam Spillway Gates, ,NAD83,FALSE,NATIVE,US/Central,Smith,TN,UNITED STATES,Carthage, ,OUTLET,Spillway,CORDELL_HULL-Spillway.Flow.Total.15Minutes.15Minutes.dcp-rev, ,0,dcp-rev,424 +99,LRN,OHIT1-OLD_HICKORY,Flow-Spillway,36.29722,-86.65889,Ave, ,Old Hickory Dam Tailwater,Old Hickory Dam Tailwater, ,NAD83,FALSE,NATIVE,US/Central,Davidson,TN,UNITED STATES,Hendersonville, ,SITE,Tailwater,OHIT1-OLD_HICKORY.Flow-Spillway.Ave.1Hour.1Hour.man-raw, ,0,man-raw,0 +100,SPK,TRM CRS-Cross Cr at Houston-Kaweah,Flow,36.29861,-119.5483,Ave, ,Lk Kaweah-Cross Cr, ,TRM - Cross Cr @ Houston, ,FALSE,NATIVE,US/Pacific,Kings,CA,UNITED STATES,"Hanford, CA",SPK,SITE, ,TRM CRS-Cross Cr at Houston-Kaweah.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,250 +101,LRN,ASHT1-CHEATHAM,Flow-Spillway,36.32042,-87.22542,Ave,3435000-CHEATHAM,Cheatham Dam Tailwater,Cheatham Dam Tailwater, ,NAD83,FALSE,NATIVE,US/Central,Cheatham,TN,UNITED STATES,Ashland City, ,SITE,Reservoir,ASHT1-CHEATHAM.Flow-Spillway.Ave.1Hour.1Hour.man-raw,3435000-CHEATHAM.Flow-Spillway.Ave.1Hour.1Hour.man-raw,0,man-raw,350 +102,SPK,TRM YKL-Yokohl Cr at Garcia Br-Kaweah,Flow,36.32694,-119.0808,Ave, ,Lk Kaweah--Yokohl Cr, ,TRM - Yokohl Cr nr Lemoncove, ,FALSE,NATIVE,US/Pacific,Tulare,CA,UNITED STATES,"Lindcove, CA",SPK,SITE, ,TRM YKL-Yokohl Cr at Garcia Br-Kaweah.Flow.Ave.~1Day.1Day.Calc-val, ,0,Calc-val,430 +103,SWL,AR00160-Tailwater,Flow,36.36482,-92.57854,Inst,BSGA4-Tailwater,Bull Shoals Dam - Tailwater,"White R at Bull Shoals Dam Tailwater, AR","White R at Bull Shoals Dam Tailwater Sensor, AR - Represents sensor that Collects Tailwater Elevation data (HT)",WGS84,FALSE,NATIVE,America/Chicago,Marion,AR,UNITED STATES,Flippin,SWL,SITE, ,AR00160-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,BSGA4-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,0,CCP-Comp,0 +104,SPK,PNF AMW-Army Weir-Kings,Flow,36.38639,-119.7869,Ave, ,Pineflat Lk-Army Weir, ,PNF - Army Weir on the Kings nr Lemoore, ,FALSE,NATIVE,US/Pacific,Kings,CA,UNITED STATES,"Camden, CA",SPK,SITE, ,PNF AMW-Army Weir-Kings.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,230 +105,SPK,TRM TRMQ-Lake Kaweah Outflow-Kaweah,Flow,36.41389,-119.0125,Ave, ,Lk Kaweah-Kaweah R, ,TRM - Terminus Dam Outflow to the Kaweah River, ,FALSE,NATIVE,US/Pacific,Tulare,CA,UNITED STATES,"Lemoncove, CA",SPK,SITE, ,TRM TRMQ-Lake Kaweah Outflow-Kaweah.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,496 +106,SPK,CA10114,Flow-Spill,36.41472,-119.0019,Inst,TRM TRMP-Lake Kaweah Pool-Kaweah,Lk Kaweah-Pool,TERMINUS LAKE / DAM,TRM - Lake Kaweah Pool Elevation Station,WGS84,FALSE,NATIVE,US/Pacific,Tulare,CA,UNITED STATES,"Lemoncove, CA",SPK,SITE, ,CA10114.Flow-Spill.Inst.1Hour.0.Calc-val,TRM TRMP-Lake Kaweah Pool-Kaweah.Flow-Spill.Inst.1Hour.0.Calc-val,0,Calc-val,752 +107,LRN,BLCT1-BledsoeCr-GallatinTN,Flow,36.42278,-86.34944,Ave, ,Bledsoe Creek @ Rogana TN,Bledsoe Creek At Rogana TN, ,NAD83,FALSE,NATIVE,US/Central,Sumner,TN,UNITED STATES,Gallatin, ,SITE,Stream Gauge,BLCT1-BledsoeCr-GallatinTN.Flow.Ave.~1Day.1Day.dcp-rev, ,0,dcp-rev,458 +108,LRN,JNCT1-JenningsCr-WhitleyvilleTN,Flow,36.44028,-85.67472,Ave, ,Jennings Cr nr Whitleyville TN,Jennings Creek Near Whitleyville TN, ,NAD83,FALSE,NATIVE,US/Central,Jackson,TN,UNITED STATES,Whitleyville, ,SITE,Stream Gauge,JNCT1-JenningsCr-WhitleyvilleTN.Flow.Ave.~1Day.1Day.dcp-rev, ,0,dcp-rev,500 +109,LRN,PORT1-RedR-PortRoyalTN,Flow,36.55389,-87.14222,Ave, ,Red River @ Port Royal TN,Red River At Port Royal TN, ,NAD83, , ,US/Central,Montgomery,TN,UNITED STATES,Port Royal, ,SITE,Stream Gauge,PORT1-RedR-PortRoyalTN.Flow.Ave.~1Day.1Day.dcp-rev, ,0,dcp-rev,0 +110,LRN,CLAT1-CumberlandR-CelinaTN,Flow,36.55444,-85.51556,Ave,3417500-CumberlandR-CelinaTN,Cumberland R @ Celina TN,Cumberland River At Celina TN, ,NAD83,FALSE,NATIVE,US/Central,Clay,TN,UNITED STATES,Celina, ,SITE,Stream Gauge,CLAT1-CumberlandR-CelinaTN.Flow.Ave.~1Day.1Day.dcp-rev,3417500-CumberlandR-CelinaTN.Flow.Ave.~1Day.1Day.dcp-rev,0,dcp-rev,489 +111,LRN,WLBK2-CumberlandR-WilliamsburgKY,Flow,36.74361,-84.15639,Ave, ,Cumberland R @ Williamsburg KY,Cumberland River At Williamsburg KY, ,NAD83,FALSE,NATIVE,US/Eastern,Whitley,KY,UNITED STATES,Williamsburg, ,SITE,Stream Gauge,WLBK2-CumberlandR-WilliamsburgKY.Flow.Ave.~1Day.1Day.dcp-rev, ,0,dcp-rev,892 +112,LRN,PVLK2-CumberlandR-PinevilleKY,Flow,36.76444,-83.6925,Ave, ,Cumberland R nr Pineville KY,Cumberland River Near Pineville KY, ,NAD83,FALSE,NATIVE,US/Eastern,Bell,KY,UNITED STATES,Pineville, ,SITE,Stream Gauge,PVLK2-CumberlandR-PinevilleKY.Flow.Ave.~1Day.1Day.dcp-rev, ,0,dcp-rev,970 +113,SWT,CHIB,Flow,36.77139,-95.50389,Inst,CBCO2,"Big Creek near Childers, OK","Big Creek near Childers, OK","Big Creek near Childers, OK",NAD83, , ,US/Central,Unknown County or County N/A,OK,UNITED STATES,"Delaware, OK",SWT,SITE, ,CHIB.Flow.Inst.15Minutes.0.Ccp-Rev,CBCO2.Flow.Inst.15Minutes.0.Ccp-Rev,0,Ccp-Rev,0 +114,LRN,CDZK2-LittleR-CadizKY,Flow,36.77778,-87.72167,Ave,3438000-LittleR-CadizKY,Little River nr Cadiz KY,Little River Near Cadiz KY, ,NAD83,FALSE,NATIVE,US/Central,Trigg,KY,UNITED STATES,Cadiz, ,SITE,Stream Gauge,CDZK2-LittleR-CadizKY.Flow.Ave.~1Day.1Day.dcp-rev,3438000-LittleR-CadizKY.Flow.Ave.~1Day.1Day.dcp-rev,0,dcp-rev,391 +115,MVS,Fisk-St Francis,Flow,36.78058,-90.2021,Inst,SFFI,Fisk,Fisk-St Francis,St. Francis River at Fisk,WGS84,FALSE,NATIVE,US/Central,Unknown County or County N/A,MO,UNITED STATES, ,MVS,STREAM_LOCATION, ,Fisk-St Francis.Flow.Inst.1Hour.0.RatingCOE,SFFI.Flow.Inst.1Hour.0.RatingCOE,0,RatingCOE,307 +116,LRN,BRKK2-CumberlandR-BurkesvilleKY,Flow,36.7875,-85.36528,Ave, ,Cumberland R @ Burkesville KY,Cumberland River At Burkesville KY, ,NAD83, , ,US/Central,Cumberland,KY,UNITED STATES,Burkesville, ,SITE,Stream Gauge,BRKK2-CumberlandR-BurkesvilleKY.Flow.Ave.~1Day.1Day.dcp-rev, ,0,dcp-rev,0 +117,LRN,MTCK2-BeaverCr-MonticelloKY,Flow,36.7975,-84.89611,Ave, ,Beaver Creek nr Monticello KY,Beaver Creek Near Monticello KY, ,NAD83, , ,US/Central,Wayne,KY,UNITED STATES,Monticello, ,SITE,Stream Gauge,MTCK2-BeaverCr-MonticelloKY.Flow.Ave.~1Day.1Day.dcp-rev, ,0,dcp-rev,0 +118,SPK,PNF PNFQ-Pine Flat Lake Outflow-Kings,Flow,36.83056,-119.3353,Ave, ,Pineflat Lk-Kings R, ,PNF - Pineflat Lk Outflow to the Kings River,WGS84,FALSE,NATIVE,US/Pacific,Fresno,CA,UNITED STATES,"Trimmer, CA",SPK,SITE,Stream Gauge,PNF PNFQ-Pine Flat Lake Outflow-Kings.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,560 +119,LRN,CFAK2-CumberlandR-CumberlandFallsKY,Flow,36.83583,-84.34139,Ave,3404500-CumberlandR-CumberlandFallsKY,Cumb R Cumberland Falls KY,Cumberland River at Cumberland Falls KY, ,NAD83,FALSE,NATIVE,US/Eastern,Whitley,KY,UNITED STATES,Greenwood, ,SITE,Stream Gauge,CFAK2-CumberlandR-CumberlandFallsKY.Flow.Ave.~1Day.1Day.dcp-rev,3404500-CumberlandR-CumberlandFallsKY.Flow.Ave.~1Day.1Day.dcp-rev,0,dcp-rev,825 +120,LRN,BARKLEY-Spillway,Flow,37.02085,-88.22351,Total, ,Barkley Lock and Dam Spillway,Barkley Lock and Dam Spillway Gates, ,NAD83,FALSE,NATIVE,US/Central,Lyon,KY,UNITED STATES,Grand Rivers, ,OUTLET,Spillway,BARKLEY-Spillway.Flow.Total.15Minutes.15Minutes.dcp-rev, ,0,dcp-rev,230 +121,LRN,BAR,Flow,37.02168,-88.22105,Inst,BAHK2,Barkley Lock & Dam,Barkley Lock & Dam to Cheatham Lock & Dam Drainage Area,"This basin represents the uncontrolled drainage area between Barkley Lock & Dam, Cheatham Lock & Dam, and includes the Red River drainage area.",NAD83, , ,US/Central,Lyon,KY,UNITED STATES,Grand Rivers, ,PROJECT,Corps Reservoir,BAR.Flow.Inst.15Minutes.0.DCP-rev, ,0,DCP-rev,0 +122,SWT,CAME,Flow,37.07778,-96.85194,Inst,CAMK1,Grouse Creek nr Cameron KS,Grouse Creek nr Cameron KS,Grouse Creek nr Cameron KS,NAD83, , ,US/Central,Unknown County or County N/A,KS,UNITED STATES,"Silverdale, KS",SWT,SITE, ,CAME.Flow.Inst.1Hour.0.Ccp-Rev,CAMK1.Flow.Inst.1Hour.0.Ccp-Rev,0,Ccp-Rev,0 +123,SPK,HID HIDQ-Hensley Lake Outflow-Fresno,Flow,37.10389,-119.8875,Ave, ,Hensly Lk-Fresno R, ,HID - Hensley Lk Outflow to the Fresno River, ,FALSE,NATIVE,US/Pacific,Madera,CA,UNITED STATES,"Raymond, CA",SPK,SITE, ,HID HIDQ-Hensley Lake Outflow-Fresno.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,561 +124,SPK,HID HIDP-Hensley Lake Pool-Fresno,Flow-Spill,37.10944,-119.8847,Inst, ,Hensley Lk-Pool,Hensley Lake,HID - Hensley Lk Pool Elevation Station,WGS84,FALSE,NATIVE,US/Pacific,Madera,CA,UNITED STATES,"Raymond, CA",SPK,SITE, ,HID HIDP-Hensley Lake Pool-Fresno.Flow-Spill.Inst.1Hour.0.Calc-val, ,0,Calc-val,581 +125,SWL,MO30203-Tailwater,Flow,37.12965,-90.7611,Inst,Clearwater_Dam-Tailwater,Clearwater Dam Tailwater,"Black R at Clearwater Dam Tailwater, AR","Black R at Clearwater Dam Tailwater Sensor, AR - Represents sensor that Collects Tailwater Elevation data (HT)",NAD83,FALSE,NATIVE,US/Central,Wayne,MO,UNITED STATES,Piedmont,SWL,SITE, ,MO30203-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,Clearwater_Dam-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,0,CCP-Comp,0 +126,SPK,HID FHL-Fresno R abv Hensley Lake-Fresno,Flow,37.15056,-119.8561,Ave, ,Hensly Lk-Fresno R, ,HID - Fresno River abv Hensly Lk, ,FALSE,NATIVE,US/Pacific,Madera,CA,UNITED STATES,"Raymond, CA",SPK,SITE, ,HID FHL-Fresno R abv Hensley Lake-Fresno.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,540 +127,SPK,BUCQ,Flow,37.21556,-119.9914,Ave,BUC BUCQ-Eastman Lake Outflow-Chowchilla,Eastman Lk-Chowchilla R, ,BUC - Eastman Lk Outflow to the Chowchilla River, ,FALSE,NATIVE,US/Pacific,Madera,CA, , ,SPK,SITE, ,BUCQ.Flow.Ave.1Hour.1Hour.Calc-val,BUC BUCQ-Eastman Lake Outflow-Chowchilla.Flow.Ave.1Hour.1Hour.Calc-val,0,Calc-val,450 +128,SWT,PURS,Flow,37.25889,-94.435,Inst,NSFM7,"N Fork Spring River nr Purcell,","North Fork Spring River near Purcell, MO","North Fork Spring River near Purcell, MO",NAD83,FALSE,NATIVE,US/Central,Unknown County or County N/A,MO,UNITED STATES,"Alba, MO",SWL,SITE, ,PURS.Flow.Inst.1Hour.0.Ccp-Rev,NSFM7.Flow.Inst.1Hour.0.Ccp-Rev,0,Ccp-Rev,850 +129,SPK,BUC BUCP-Eastman Lake Pool-Chowchilla,Flow-Spill,37.26667,-119.9844,Ave,BUCP,Eastman Lk-Pool,HENSLEY LAKE / HIDDEN DAM,BUC - Eastman Lk (Buchanan Dam) Pool Elevation Station,WGS84,FALSE,NATIVE,US/Pacific,Madera,CA,UNITED STATES, ,SPK,SITE, ,BUC BUCP-Eastman Lake Pool-Chowchilla.Flow-Spill.Ave.1Hour.1Hour.Calc-val,BUCP.Flow-Spill.Ave.1Hour.1Hour.Calc-val,0,Calc-val,613 +130,SPK,MER MARQ-Mariposa Res Outflow-Mariposa Cr,Flow,37.28,-120.1628,Ave, ,Mariposa Res-Mariposa Cr, ,MER - Mariposa Dam Outflow to Mariposa Cr - Merced Group, ,FALSE,NATIVE,US/Pacific,Merced,CA,UNITED STATES,"Le Grand, CA",SPK,SITE, ,MER MARQ-Mariposa Res Outflow-Mariposa Cr.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,350 +131,SPK,CA10107-Mariposa Res Pool-Mariposa Cr,Flow-Spill,37.29333,-120.1467,Inst,MER MARP-Mariposa Res Pool-Mariposa Cr,Mariposa Res-Pool,Mariposa Pool,MER - Mariposa Reservoir Pool Elevation Station - Merced Group,WGS84,FALSE,NATIVE,US/Pacific,Merced,CA,UNITED STATES,"Le Grand, CA",SPK,SITE, ,CA10107-Mariposa Res Pool-Mariposa Cr.Flow-Spill.Inst.1Hour.0.Calc-val,MER MARP-Mariposa Res Pool-Mariposa Cr.Flow-Spill.Inst.1Hour.0.Calc-val,0,Calc-val,459 +132,SPK,MER MCK-Bear Cr at Mckee Road-Bear Cr,Flow,37.30917,-120.4456,Ave, ,Bear Res-Bear Cr@ McKee Rd, ,MER - Bear Cr @ McKee Rd - Merced Group, ,FALSE,NATIVE,US/Pacific,Merced,CA,UNITED STATES,"Merced, CA",SPK,SITE, ,MER MCK-Bear Cr at Mckee Road-Bear Cr.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,167 +133,SPK,MER OWNQ-Owens Reservoir Outflow-Owens Cr,Flow,37.31194,-120.1897,Ave, ,Owens Res-Owens Cr, ,MER - Owens Reservoir Outflow to Owens Cr - Merced Group, ,FALSE,NATIVE,US/Pacific,Merced,CA,UNITED STATES,"Le Grand, CA",SPK,SITE, ,MER OWNQ-Owens Reservoir Outflow-Owens Cr.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,357 +134,SPK,MER BURQ-Burns Reservoir Outflow-Burns Cr,Flow,37.37333,-120.2803,Ave, ,Burns Res-Burns Cr, ,MER - Burns Reservoir Outflow to Burns Cr - Merced Group, ,FALSE,NATIVE,US/Pacific,Merced,CA,UNITED STATES,"Planada, CA",SPK,SITE, ,MER BURQ-Burns Reservoir Outflow-Burns Cr.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,261 +135,SPK,CA10103-Burns Reservoir Pool-Burns Cr,Flow-Spill,37.37667,-120.275,Inst,Burns Dam -Burns Reservoir Pool-Burns Cr,Burns Res-Pool,Burns Pool,MER - Burns Reservoir Pool Elevation Station - Merced Group,WGS84,FALSE,NATIVE,US/Pacific,Merced,CA,UNITED STATES,"Planada, CA",SPK,SITE, ,CA10103-Burns Reservoir Pool-Burns Cr.Flow-Spill.Inst.1Hour.0.Calc-val,Burns Dam -Burns Reservoir Pool-Burns Cr.Flow-Spill.Inst.1Hour.0.Calc-val,0,Calc-val,319 +136,SPL,Pine Cnyn DS-Pine Cnyn W,Flow,37.47833,-114.3075,Inst, ,Pine Cnyn Downstream, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,NV,UNITED STATES,Cedar City, ,SITE, ,Pine Cnyn DS-Pine Cnyn W.Flow.Inst.15Minutes.0.GOES-rev, ,0,GOES-rev,5609 +137,SPL,Mathews Cnyn DS-Mathews Cnyn W,Flow,37.49972,-114.2242,Inst, ,Mathews Canyon Downstream, , ,WGS84,FALSE,NATIVE,US/Pacific,Unknown County or County N/A,NV,UNITED STATES,Cedar City, ,SITE, ,Mathews Cnyn DS-Mathews Cnyn W.Flow.Inst.15Minutes.0.GOES-rev, ,0,GOES-rev,5439 +138,LRH,WV08902-Outflow,Flow,37.649,-80.886,Inst,BLN-Outflow,Bluestone Outflow (Calc), , ,WGS84,FALSE,NATIVE,US/Eastern,Summers,WV,UNITED STATES,Hinton,LRH,STREAM_LOCATION,Calculated,WV08902-Outflow.Flow.Inst.1Hour.0.OBS,BLN-Outflow.Flow.Inst.1Hour.0.OBS,0,OBS,1400 +139,SPK,FRM SET-Stockton East Tun-Littlejohn Cr,Flow,37.84833,-120.6831,Inst, ,Farmington Lk-Goodwin Tunnel, ,FRM - Goodwin Tunnel (Stockton East Tunnel) abv Farmington Lk, ,FALSE,NATIVE,US/Pacific,San Joaquin,CA,UNITED STATES,"Oakdale, CA",SPK,SITE, ,FRM SET-Stockton East Tun-Littlejohn Cr.Flow.Inst.15Minutes.0.Combined-val, ,0,Combined-val,600 +140,SPK,FRM SHG-Shirley Gulch Div-Littlejohn Cr,Flow,37.9025,-120.7597,Ave, ,Farmington Lk-Shirley Gulch, ,FRM - Shirley Gulch abv Farmington Lk, ,FALSE,NATIVE,US/Pacific,San Joaquin,CA,UNITED STATES,"Oakdale, CA",SPK,SITE, ,FRM SHG-Shirley Gulch Div-Littlejohn Cr.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,225 +141,SPK,FRM FRMQ-Stockton East Outflow Works,Flow,37.91444,-120.9386,Inst, ,Farminton Lk-SEWD Works, ,"FRM - Farmington Dam Outflow Works, Stockton East Water District.", ,FALSE,NATIVE,US/Pacific,San Joaquin,CA,UNITED STATES,"Farmington, CA",SPK,SITE, ,FRM FRMQ-Stockton East Outflow Works.Flow.Inst.1Hour.0.Calc-val, ,0,Calc-val,19 +142,SPK,FRM FRMP-Farmington Pool-Littlejohn Cr,Flow-Spill,37.915,-120.935,Ave, ,Farmington Lk-Pool,Farmington Pool Farmington Pool,FRM - Farmington Dam Pool Elevation Station,WGS84,FALSE,NATIVE,US/Pacific,San Joaquin,CA,UNITED STATES, ,SPK,SITE, ,FRM FRMP-Farmington Pool-Littlejohn Cr.Flow-Spill.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,180 +143,SPK,FRM RCF-Rock Cr nr FRM-Littlejohn Cr,Flow,37.91833,-120.9597,Ave, ,Farmington Lk-Rock Cr, ,FRM - Rock Cr abv Confluence with Littlejohn Cr, ,FALSE,NATIVE,US/Pacific,San Joaquin,CA,UNITED STATES,"Farmington, CA",SPK,SITE, ,FRM RCF-Rock Cr nr FRM-Littlejohn Cr.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,105 +144,SPK,FRM FRG-Littlejohn at FRM-Littlejohn Cr,Flow,37.92611,-121.0014,Ave, ,Farmington Lk-Littlejohn Cr, ,FRM - Littlejohn Cr nr Farminton CA, ,FALSE,NATIVE,US/Pacific,San Joaquin,CA,UNITED STATES,"Farmington, CA",SPK,SITE, ,FRM FRG-Littlejohn at FRM-Littlejohn Cr.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,115 +145,SPK,FRM DUC-Duck Cr Diversion-Littlejohn Cr,Flow,37.93833,-120.9903,Ave, ,Farmington Lk-Duck Cr Div, ,FRM - Duck Cr Diversion nr Farmington, ,FALSE,NATIVE,US/Pacific,San Joaquin,CA,UNITED STATES,"Farmington, CA",SPK,SITE, ,FRM DUC-Duck Cr Diversion-Littlejohn Cr.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,110 +146,SPK,FRM DCK-Duck Cr nr FRM-Littlejohn Cr,Flow,37.94472,-120.9308,Ave, ,Farmington Lk-Duck Cr, ,FRM - Duck Cr nr Farmington, ,FALSE,NATIVE,US/Pacific,San Joaquin,CA,UNITED STATES,"Farmington, CA",SPK,SITE, ,FRM DCK-Duck Cr nr FRM-Littlejohn Cr.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,150 +147,SPK,NHG CGV-Cosgrove Creek-Calaveras,Flow,38.13611,-120.8472,Ave, ,New Hogan Lk-Cosgrove Cr, ,NHG - Cosgrove Cr blw Dam, ,FALSE,NATIVE,US/Pacific,Calaveras,CA,UNITED STATES,"Valley Springs, CA",SPK,SITE, ,NHG CGV-Cosgrove Creek-Calaveras.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,547 +148,SPK,NHG NHGQ-New Hogan Lake Outflow-Calaveras,Flow,38.14806,-120.8239,Ave, ,New Hogan Lk-Calaveras R, ,NHG - New Hogan Lk Outflow to the Calaveras River, ,FALSE,NATIVE,US/Pacific,Calaveras,CA,UNITED STATES,"Valley Springs, CA",SPK,SITE, ,NHG NHGQ-New Hogan Lake Outflow-Calaveras.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,520 +149,SPK,NHG MRS-Mormon Slough-Calaveras,Flow,38.14889,-121.0125,Ave, ,New Hogan Lk-Mormon Slough, ,NHG - Mormon Slough nr Bellota, ,FALSE,NATIVE,US/Pacific,San Joaquin,CA,UNITED STATES,"Lockeford, CA",SPK,SITE, ,NHG MRS-Mormon Slough-Calaveras.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,130 +150,LRH,LON,Flow,38.1928,-81.3692,Inst,LondonLD,London Locks and Dam,Kanawha Riv at London L&D,Kanawha Riv at London L&D,WGS84, , ,US/Eastern,Kanawha,WV,UNITED STATES,London,LRH,PROJECT,Lock,LON.Flow.Inst.15Minutes.0.DCP-rev, ,0,DCP-rev,0 +151,LRH,WV09903-Lake,Flow,38.3039,-82.4164,Inst,BBF-Lake,Beech Fork Lake,Beech Fk Dam near Huntington 7SSE,Beech Fk Dam near Huntington 7SSE,WGS84,FALSE,NATIVE,US/Eastern,Wayne,WV,UNITED STATES,Lavalette,LRH,SITE,Lake,WV09903-Lake.Flow.Inst.1Hour.0.OBS,BBF-Lake.Flow.Inst.1Hour.0.OBS,0,OBS,500 +152,MVS,CFMV,Flow,38.31001,-88.98833,Inst,Mt Vernon-Casey Fork,Mount Vermon Casey,Mt Vernon-Casey Fork,Casey Fork at Mt. Vernon,WGS84,FALSE,NATIVE,US/Central,Unknown County or County N/A,IL,UNITED STATES, ,MVS,STREAM_LOCATION, ,CFMV.Flow.Inst.15Minutes.0.RatingUSGS,Mt Vernon-Casey Fork.Flow.Inst.15Minutes.0.RatingUSGS,0,RatingUSGS,420 +153,MVS,Fayetteville,Flow,38.3775,-89.79093,Inst, ,Kaskaskia River at Fayetteville,Kaskaskia River at Fayetteville, ,WGS84,FALSE,NATIVE,US/Central,Unknown County or County N/A,IL,UNITED STATES, ,MVS,STREAM_LOCATION,Gage,Fayetteville.Flow.Inst.1Hour.0.CCP-Comp,07048600.Flow.Inst.1Hour.0.CCP-Comp,0,CCP-Comp,300 +154,LRH,WIN,Flow,38.5269,-81.9136,Inst,WV07903,Winfield Locks and Dam,Winfield L-D,Winfield L-D,WGS84, , ,US/Eastern,Putnam,WV,UNITED STATES,Winfield,LRH,PROJECT,Lock,WIN.Flow.Inst.15Minutes.0.DCP-rev,01162500.Flow.Inst.15Minutes.0.DCP-rev,0,DCP-rev,0 +155,MVS,St Louis-River Des Peres,Flow,38.55941,-90.2832,Inst,DPSL,St Louis RDP,St Louis-River Des Peres,River Des Peres at St. Louis,WGS84,FALSE,NATIVE,US/Central,St. Louis City,MO,UNITED STATES, ,MVS,STREAM_LOCATION, ,St Louis-River Des Peres.Flow.Inst.5Minutes.0.RatingUSGS,DPSL.Flow.Inst.5Minutes.0.RatingUSGS,0,RatingUSGS,391 +156,SPK,WRS WRSQ-Lake Sonoma Outflow-Russian,Flow,38.71972,-122.9994,Ave, ,Lk Sonoma-Dry Cr, ,WRS - Warm Springs Dam Outflow to Dry Cr, ,FALSE,NATIVE,US/Pacific,Sonoma,CA,UNITED STATES,"Geyserville, CA",SPN,SITE, ,WRS WRSQ-Lake Sonoma Outflow-Russian.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,188 +157,SPK,CA82212,Flow-Spill,38.7225,-123.01,Ave,WRS WRSP-Lake Sonoma Pool-Russian,Lk Sonoma-Pool,LAKE SONOMA / WARM SPRINGS DAM,WRS - Lake Sonoma Pool Elevation Station,WGS84,FALSE,NATIVE,US/Pacific,Sonoma,CA,UNITED STATES,"Cloverdale, CA",SPN,SITE, ,CA82212.Flow-Spill.Ave.1Hour.1Hour.Calc-val,WRS WRSP-Lake Sonoma Pool-Russian.Flow-Spill.Ave.1Hour.1Hour.Calc-val,0,Calc-val,495 +158,LRH,WV00707-Lake,Flow,38.8428,-80.6203,Inst,BRNW2,Burnsville Lake,Burnsville Dam near Burnsville,Burnsville Dam near Burnsville,WGS84,FALSE,NATIVE,US/Eastern,Braxton,WV,UNITED STATES,Burnsville,LRH,SITE,Lake,WV00707-Lake.Flow.Inst.1Hour.0.OBS,BRNW2.Flow.Inst.1Hour.0.OBS,0,OBS,700 +159,MVS,IL00116-Mississippi,Flow,38.86917,-90.15361,Inst,Mel Price-Mississippi,Mel Price L&D, , , , , , ,Unknown County or County N/A,0,UNITED STATES, , ,STREAM_LOCATION, ,IL00116-Mississippi.Flow.Inst.6Hours.0.NCRFCShef-PZ,Mel Price-Mississippi.Flow.Inst.6Hours.0.NCRFCShef-PZ,0,NCRFCShef-PZ,0 +160,MVS,MO10301-Mississippi,Flow,39.00366,-90.69239,Inst,LD 25-Mississippi,LD 25 WQ, , , , , , ,Unknown County or County N/A,0,UNITED STATES, , ,STREAM_LOCATION, ,MO10301-Mississippi.Flow.Inst.6Hours.0.NCRFCShef-PZ,LD 25-Mississippi.Flow.Inst.6Hours.0.NCRFCShef-PZ,0,NCRFCShef-PZ,0 +161,SPK,COY HOP-Hopland-EFRussian,Flow,39.02667,-123.1294,Ave, ,Lk Mendocino-Russian@Hopland, ,COY - Russian River nr Hopland, ,FALSE,NATIVE,US/Pacific,Mendocino,CA,UNITED STATES,"Hopland, CA",SPN,SITE, ,COY HOP-Hopland-EFRussian.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,498 +162,SPK,COY COYF-Coyote Fish Hatchery-EFRussian,Flow,39.19806,-123.1858,Inst, ,Lk Mendocino-Fish Hatchery, ,COY - Coyote Dam Fish Hatchery, ,FALSE,NATIVE,US/Pacific,Mendocino,CA,UNITED STATES,"Calpella, CA",SPN,SITE, ,COY COYF-Coyote Fish Hatchery-EFRussian.Flow.Inst.15Minutes.0.LOS-raw, ,0,LOS-raw,614 +163,SPK,COY COYQ-Lake Mendocino Outflow-EFRussian,Flow,39.19806,-123.1864,Ave, ,Lk Mendocino-EF Russian R, ,COY - Coyote Dam Outflow to the EF Russian River, ,FALSE,NATIVE,US/Pacific,Mendocino,CA,UNITED STATES,"Calpella, CA",SPN,SITE, ,COY COYQ-Lake Mendocino Outflow-EFRussian.Flow.Ave.~1Day.1Day.Calc-val, ,0,Calc-val,614 +164,SPK,CA10105,Flow-Spill,39.23944,-121.2667,Ave,ENG ENGP-Englebright Lake Pool-Yuba,ENGLEBRIGHT LAKE / DAM,ENGLEBRIGHT LAKE / DAM,ENG - Englebright Lake Pool Elevation,WGS84, , ,US/Pacific,Yuba,CA,UNITED STATES,"Smartsville, CA",SPK,SITE, ,CA10105.Flow-Spill.Ave.1Hour.1Hour.Calc-val,ENG ENGP-Englebright Lake Pool-Yuba.Flow-Spill.Ave.1Hour.1Hour.Calc-val,0,Calc-val,0 +165,SPK,MRT MRT-Martis Creek-Truckee,Flow-Res Out,39.32667,-120.1133,Ave, ,Martis Creek Reservoir, , ,WGS84,FALSE,NATIVE,US/Pacific,Nevada,CA,UNITED STATES, , ,SITE, ,MRT MRT-Martis Creek-Truckee.Flow-Res Out.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,5858 +166,SPK,MRT MRTQ-Martis Res Outflow-Truckee,Flow,39.3275,-120.1147,Ave, ,Martis Lk-Martis Cr, ,MRT - Martis Cr Dam Outflow to Martis Cr, ,FALSE,NATIVE,US/Pacific,Nevada,CA,UNITED STATES,"Truckee, CA",SPK,SITE, ,MRT MRTQ-Martis Res Outflow-Truckee.Flow.Ave.1Hour.1Hour.Calc-val, ,0,Calc-val,5858 +167,MVS,MO10300-Mississippi,Flow,39.375,-90.907,Inst,LD 24-Mississippi,LD 24 WQ, , ,WGS84,FALSE,NATIVE,US/Central,Pike,MO,UNITED STATES,Clarksville,MVS,STREAM_LOCATION, ,MO10300-Mississippi.Flow.Inst.6Hours.0.NCRFCShef-PZ,LD 24-Mississippi.Flow.Inst.6Hours.0.NCRFCShef-PZ,0,NCRFCShef-PZ,422 +168,MVS,Ashburn-Salt,Flow,39.52264,-91.20163,Inst,SAAS,Ashburn,Ashburn-Salt,Salt River at Ashburn,WGS84,FALSE,NATIVE,US/Central,Unknown County or County N/A,MO,UNITED STATES,Ashburn,MVS,STREAM_LOCATION, ,Ashburn-Salt.Flow.Inst.15Minutes.0.RatingCOE,SAAS.Flow.Inst.15Minutes.0.RatingCOE,0,RatingCOE,447 +169,SPK,CA00035-Thermolito-Feather R,Flow,39.53889,-121.4856,Inst,SAC ORO-Thermolito-Feather R,Thermolito, , , , , ,US/Pacific,Unknown County or County N/A,0,UNITED STATES, , ,STREAM_LOCATION, ,CA00035-Thermolito-Feather R.Flow.Inst.1Hour.0.Calc-manual,SAC ORO-Thermolito-Feather R.Flow.Inst.1Hour.0.Calc-manual,0,Calc-manual,0 +170,LRH,OH00018-Outflow,Flow,39.54077,-82.0584,Inst,TJE-Outflow,Tom Jenkins Outflow (Calc),Tom Jenkins Lake Outflow,Tom Jenkins Lake Outflow,WGS84,FALSE,NATIVE,US/Eastern,Athens,OH,UNITED STATES,Burr Oak,LRH,STREAM_LOCATION,Calculated,OH00018-Outflow.Flow.Inst.1Hour.0.OBS,TJE-Outflow.Flow.Inst.1Hour.0.OBS,0,OBS,700 +171,MVS,LD 22-Mississippi,Flow,39.63534,-91.24879,Inst, ,LD 22 WQ, , , , , , ,Unknown County or County N/A,0,UNITED STATES, , ,STREAM_LOCATION, ,LD 22-Mississippi.Flow.Inst.~2Hours.0.lpmsShef-raw, ,0,lpmsShef-raw,0 +172,SPK,BLBQ,Flow,39.81846,-122.325,Ave,BLB BLBQ-Black Butte Outflow-Stony Cr,Black Butte Lk-Stony Cr, ,BLB - Black Butte Lk Outflow to Stony Cr,WGS84,FALSE,NATIVE,US/Pacific,Tehama,CA, , ,SPK,STREAM,Stream Gauge,BLBQ.Flow.Ave.1Hour.1Hour.Calc-val,BLB BLBQ-Black Butte Outflow-Stony Cr.Flow.Ave.1Hour.1Hour.Calc-val,0,Calc-val,426 +173,LRH,Alexandria,Flow,40.0869,-82.6106,Inst,ALXE3,Alexandria,Alexandria near Caldwell,Alexandria near Caldwell,WGS84, , ,US/Eastern,Licking,OH,UNITED STATES,Caldwell,LRH,SITE,Gage,Alexandria.Flow.Inst.1Hour.0.DCP-rev,CE40D9B4.Flow.Inst.1Hour.0.DCP-rev,0,DCP-rev,0 +174,LRH,OH00012-Outflow,Flow,40.2694,-81.2792,Inst,CLB-Outflow,Clendening Outflow, ,Outflow calculated by gate settings.,WGS84,FALSE,NATIVE,US/Eastern,Harrison,OH,UNITED STATES,Tippecanoe,LRH,SITE,Calculated,OH00012-Outflow.Flow.Inst.1Hour.0.OBS,CLB-Outflow.Flow.Inst.1Hour.0.OBS,0,OBS,862 +175,LRP,Steubenville,Flow,40.35833,-80.61111,Inst,SBVO1,"Steubenville, OH","Ohio River at Steubenville, OH","Ohio River at Steubenville, OH",WGS84,FALSE,NATIVE,US/Eastern,Jefferson,OH,UNITED STATES,Weirton, ,SITE, ,Steubenville.Flow.Inst.1Hour.0.OHRFC,SBVO1.Flow.Inst.1Hour.0.OHRFC,0,OHRFC,624 +176,LRH,OH00009-Outflow,Flow,40.5039,-82.5775,Inst,NBK-Outflow,NoBrKokosing Outflow,USGS 03136300 North Branch Kokosing R Lake nr Fredericktown OH,Outflow stream gage downstream from North Branch Kokosing River Dam,WGS84,FALSE,NATIVE,US/Eastern,Knox,OH,UNITED STATES,Fredericktown,LRH,SITE,Gage,OH00009-Outflow.Flow.Inst.1Hour.0.OBS,NBK-Outflow.Flow.Inst.1Hour.0.OBS,0,OBS,1100 +177,NAP,Bethlehem,Flow,40.61538,-75.37879,Inst, ,"Lehigh River at Bethlehem, PA","Lehigh River at Bethlehem, PA","Lehigh River at Bethlehem, PA", , , , , , ,UNITED STATES, , ,SITE, ,Bethlehem.Flow.Inst.1Hour.0.Rev-DCP, ,0,Rev-DCP,0 +178,LRH,OH00005-Lake,Flow,40.6356,-81.5581,Inst,BCS-Lake,Beach City Lake,Beach City Lake Beach City 2S,Beach City Lake Beach City 2S,WGS84,FALSE,NATIVE,US/Eastern,Tuscarawas,OH,UNITED STATES,Beach City,LRH,SITE,Lake,OH00005-Lake.Flow.Inst.1Hour.0.OBS,BCS-Lake.Flow.Inst.1Hour.0.OBS,0,OBS,931 +179,LRH,OH00004-Lake,Flow,40.6486,-81.4328,Inst,BIVO1,Bolivar Lake,Bolivar Lk near Bolivar 1E,Bolivar Lk near Bolivar 1E,WGS84,FALSE,NATIVE,US/Eastern,Tuscarawas,OH,UNITED STATES,Bolivar,LRH,SITE,Lake,OH00004-Lake.Flow.Inst.1Hour.0.OBS,BIVO1.Flow.Inst.1Hour.0.OBS,0,OBS,895 +180,LRH,OH00004-Outflow,Flow,40.64861,-81.4328,Inst,BOS-Outflow,Bolivar Outflow,Bolivar Outflow (calculated value),Bolivar Outflow (calculated value),WGS84,FALSE,NATIVE,US/Eastern,Tuscarawas,OH,UNITED STATES,Bolivar,LRH,STREAM_LOCATION,Calculated,OH00004-Outflow.Flow.Inst.1Hour.0.OBS,BOS-Outflow.Flow.Inst.1Hour.0.OBS,0,OBS,895 +181,LRP,Clinton,Flow,40.71603,-79.57887,Inst,PA00116,"L/D-6, Clinton","Allegheny River at L/D-6, Clinton","Allegheny River at L/D-6, Clinton",WGS84,FALSE,NATIVE,US/Eastern,Armstrong,PA,UNITED STATES,Harrison Township, ,PROJECT, ,Clinton.Flow.Inst.1Hour.0.CCP-Comp,07075300.Flow.Inst.1Hour.0.CCP-Comp,0,CCP-Comp,760 +182,NAP,Walnutport,Flow,40.75704,-75.60296,Inst, ,"Lehigh River at Walnutport, PA","Lehigh River at Walnutport, PA","Lehigh River at Walnutport, PA", , , , , , ,UNITED STATES, , ,SITE, ,Walnutport.Flow.Inst.1Hour.0.Rev-DCP, ,0,Rev-DCP,0 +183,LRE,Youngstown,Flow,41.27,-80.67,Inst, ,Youngstown Weather Station, , , , , ,US/Eastern,Trumbull,OH,UNITED STATES,"Cortland, OH",LRP,SITE, ,Youngstown.Flow.Inst.1Hour.0.OBS,03098600.Flow.Inst.1Hour.0.OBS,0,OBS,0 +184,LRE,Meadville,Flow,41.64,-80.21,Inst, ,Meadville Weather Station, , , , , ,US/Eastern,Crawford,PA,UNITED STATES,"Meadville, PA",LRP,SITE, ,Meadville.Flow.Inst.1Hour.0.OBS,03023100.Flow.Inst.1Hour.0.OBS,0,OBS,0 +185,LRE,Bradford,Flow,41.8,-78.62,Inst, ,Bradford Weather Station, , , , , ,US/Eastern,McKean,PA,UNITED STATES,"Cyclone, PA",LRP,SITE, ,Bradford.Flow.Inst.1Hour.0.OBS,03010955.Flow.Inst.1Hour.0.OBS,0,OBS,0 +186,LRP,CambridgeSprings,Flow,41.8072,-80.0633,Inst,CBSP1,"Cambridge Springs, PA","French Creek at Cambridge Springs, PA","French Creek at Cambridge Springs, PA",WGS84,FALSE,NATIVE,US/Eastern,Crawford,PA,UNITED STATES,Erie, ,SITE, ,CambridgeSprings.Flow.Inst.1Hour.0.OBS,CBSP1.Flow.Inst.1Hour.0.OBS,0,OBS,1124 +187,NAE,EBN,Flow,42.22083,-70.97861,Inst, ,Monatiquot River at Braintree,Monatiquot River at East Braintree,Monatiquot River at East Braintree, ,FALSE,NATIVE,America/New_York,Unknown County or County N/A,0, , , ,SITE,Streamgage,EBN.Flow.Inst.15Minutes.0.DCP-rev, ,0,DCP-rev,20 +188,LRE,Jackson,Flow,42.27,-84.47,Inst, ,Jackson Weather Station, , , , , ,US/Eastern,Jackson,MI,UNITED STATES,"Jackson, MI",LRE,SITE, ,Jackson.Flow.Inst.15Minutes.0.DCP-rev,DD3051BC.Flow.Inst.15Minutes.0.DCP-rev,0,DCP-rev,0 +189,NAE,OTR,Flow,42.58833,-72.04139,Inst, ,Otter River @ Otter River,Otter River at Otter River,Otter River at Otter River, ,FALSE,NATIVE,America/New_York,Unknown County or County N/A,0, , , ,SITE,Streamgage,OTR.Flow.Inst.15Minutes.0.DCP-rev, ,0,DCP-rev,-236 +190,NAE,ATH,Flow,42.59284,-72.23912,Inst, ,"Millers River @ Athol, MA",Millers River at Athol,Millers River at Athol,WGS84,FALSE,NATIVE,America/New_York,Worcester,MA, ,Athol, ,STREAM_LOCATION,Streamgage,ATH.Flow.Inst.15Minutes.0.DCP-rev, ,0,DCP-rev,-237 +191,LRB,NY00468,Flow,42.73329,-77.90708,Inst,Mount Morris,Mount Morris Dam,Mount Morris Dam,NERFC stage gauge for MMD.,NAD27,FALSE,NATIVE,US/Eastern,Livingston,NY,UNITED STATES, ,LRB,PROJECT,Dam,NY00468.Flow.Inst.30Minutes.0.CCP-Computed-Rev,Mount Morris.Flow.Inst.30Minutes.0.CCP-Computed-Rev,0,CCP-Computed-Rev,0 +192,MVP,IA04014,Flow,42.785,-91.095,Inst,LockDam_10,Mississippi R Lock and Dam 10, , ,NAD83, , ,US/Central,Unknown County or County N/A,IA,UNITED STATES, ,MVP,PROJECT,Dam,IA04014.Flow.Inst.1Hour.0.rev,LockDam_10.Flow.Inst.1Hour.0.rev,0,rev,0 +193,MVP,WI00733,Flow,43.2116,-91.095,Inst,LockDam_09,Mississippi R Lock and Dam 09, , ,NAD83, , ,US/Central,Unknown County or County N/A,WI,UNITED STATES, ,MVP,PROJECT,Dam,WI00733.Flow.Inst.15Minutes.0.rev,LockDam_09.Flow.Inst.15Minutes.0.rev,0,rev,0 +194,LRE,Fulton,Flow,43.3,-76.39,Inst, ,Fulton Weather Station, , , , , ,US/Eastern,Oswego,NY,UNITED STATES,"Fulton, NY",LRB,SITE, ,Fulton.Flow.Inst.1Hour.0.DCP-rev,CE41E006.Flow.Inst.1Hour.0.DCP-rev,0,DCP-rev,0 +195,MVP,WI00803,Flow,43.5766,-91.2316,Inst,LockDam_08,Mississippi R Lock and Dam 08, , ,NAD83, , ,US/Central,Unknown County or County N/A,WI,UNITED STATES, , ,PROJECT,Dam,WI00803.Flow.Inst.15Minutes.0.rev,LockDam_08.Flow.Inst.15Minutes.0.rev,0,rev,0 +196,MVP,MN00587,Flow,43.8666,-91.3083,Inst,LockDam_07,Mississippi R Lock and Dam 07, , ,NAD83, , ,US/Central,Unknown County or County N/A,MN,UNITED STATES, ,MVP,PROJECT,Dam,MN00587.Flow.Inst.15Minutes.0.rev,LockDam_07.Flow.Inst.15Minutes.0.rev,0,rev,0 +197,LRE,Berlin,Flow,43.9538,-88.95284,Inst, ,Berlin Multi-Parameter Station, , , , , ,US/Central,Green Lake,WI,UNITED STATES,"Berlin, WI",LRE,SITE, ,Berlin.Flow.Inst.0.0.shef-raw, ,0,shef-raw,0 +198,LRE,Oshkosh,Flow,43.98,-88.55,Inst, ,Oshkosh Multi-Parameter Sta, , , , , ,US/Central,Winnebago,WI,UNITED STATES,"Oshkosh, WI",LRE,SITE, ,Oshkosh.Flow.Inst.0.0.shef-raw, ,0,shef-raw,0 +199,MVP,WI00802,Flow,44,-91.4383,Inst,LockDam_06,Mississippi R Lock and Dam 06, , ,NAD83, , ,US/Central,Unknown County or County N/A,WI,UNITED STATES, ,MVP,PROJECT,Dam,WI00802.Flow.Inst.15Minutes.0.rev,LockDam_06.Flow.Inst.15Minutes.0.rev,0,rev,0 +200,LRE,WinnebagoObs,Flow,44.03333,-88.40556,Inst, ,Winnebago Observed Flow, , , , , ,US/Central,Winnebago,WI,UNITED STATES,"Stockbridge, WI",LRE,SITE, ,WinnebagoObs.Flow.Inst.0.0.shef-raw, ,0,shef-raw,0 +201,MVP,MN00588,Flow,44.0883,-91.6699,Inst,LockDam_05a,Mississippi R Lock and Dam 05a, , ,NAD83, , ,US/Central,Unknown County or County N/A,MN,UNITED STATES, ,MVP,PROJECT,Dam,MN00588.Flow.Inst.15Minutes.0.rev,LockDam_05a.Flow.Inst.15Minutes.0.rev,0,rev,0 +202,MVP,MN00589,Flow,44.1616,-91.8116,Inst,LockDam_05,Mississippi R Lock and Dam 05, , ,NAD83, , ,US/Central,Unknown County or County N/A,MN,UNITED STATES, ,MVP,PROJECT,Dam,MN00589.Flow.Inst.15Minutes.0.rev,LockDam_05.Flow.Inst.15Minutes.0.rev,0,rev,0 +203,LRE,Appleton,Flow,44.25,-88.52,Inst, ,Appleton Multi-Parameter Station, , , , , ,US/Central,Outagamie,WI,UNITED STATES,"Greenville, WI",LRE,SITE, ,Appleton.Flow.Inst.0.0.shef-raw, ,0,shef-raw,0 +204,MVP,WI00727,Flow,44.325,-91.9233,Inst,LockDam_04,Mississippi R Lock and Dam 04, , ,NAD83, , ,US/Central,Unknown County or County N/A,WI,UNITED STATES, ,MVP,PROJECT,Dam,WI00727.Flow.Inst.15Minutes.0.rev,LockDam_04.Flow.Inst.15Minutes.0.rev,0,rev,0 +205,LRE,NewLondonObs,Flow,44.39222,-88.74028,Inst, ,New London Observed Flow, , , , , ,US/Central,Waupaca,WI,UNITED STATES,"New London, WI",LRE,SITE, ,NewLondonObs.Flow.Inst.0.0.shef-raw, ,0,shef-raw,0 +206,MVP,MN00595,Flow,44.61,-92.61,Inst,LockDam_03,Mississippi R Lock and Dam 03,Mississippi River Lock and Dam 03, ,NAD83,FALSE,NATIVE,US/Central,Unknown County or County N/A,MN,UNITED STATES, ,MVP,PROJECT,Dam,MN00595.Flow.Inst.15Minutes.0.rev,LockDam_03.Flow.Inst.15Minutes.0.rev,0,rev,600 +207,MVP,MN00594,Flow,44.7599,-92.8683,Inst,LockDam_02,Mississippi R Lock and Dam 02, , ,NAD83, , ,US/Central,Unknown County or County N/A,MN,UNITED STATES, ,MVP,PROJECT,Dam,MN00594.Flow.Inst.15Minutes.0.comp,LockDam_02.Flow.Inst.15Minutes.0.comp,0,comp,0 +208,MVP,MN00591,Flow,44.9783,-93.2466,Inst,LowerSAFalls,Lower St Anthony Falls, ,Elevations are in MSL 1912,NAD83, , , ,Unknown County or County N/A,MN,UNITED STATES, , ,PROJECT,Dam,MN00591.Flow.Inst.~1Day.0.CEMVP-Legacy,LowerSAFalls.Flow.Inst.~1Day.0.CEMVP-Legacy,0,CEMVP-Legacy,0 +209,MVP,MN00579,Flow,45.17139,-96.09333,Inst,MarshLake_Dam,Marsh Lake Dam, , ,NAD83, , ,US/Central,Unknown County or County N/A,MN,UNITED STATES, ,MVP,PROJECT,Dam,MN00579.Flow.Inst.1Hour.0.Fcst-CEMVP,MarshLake_Dam.Flow.Inst.1Hour.0.Fcst-CEMVP,0,Fcst-CEMVP,0 +210,MVP,MN00581-ServiceSpillway,Flow-Out,45.22944,-96.29028,Inst,Highway75_Dam-ServiceSpillway,Highway 75 Dam Service Spillway, , ,NAD83, , ,US/Central,Unknown County or County N/A,0,UNITED STATES, ,MVP,SITE, ,MN00581-ServiceSpillway.Flow-Out.Inst.~1Day.0.CEMVP-Legacy,Highway75_Dam-ServiceSpillway.Flow-Out.Inst.~1Day.0.CEMVP-Legacy,0,CEMVP-Legacy,0 +211,MVP,Benson,Flow,45.3111,-95.6249,Inst,BENM5,Chippewa River at Benson,"Chippewa River at Benson, US12",Owner is MNDNR/MPCA,NAD27,FALSE,NATIVE,US/Central,Unknown County or County N/A,MN,UNITED STATES,Benson,MVP,SITE,DCP Gage,Benson.Flow.Inst.15Minutes.0.rev,BENM5.Flow.Inst.15Minutes.0.rev,0,rev,1009 +212,NAE,WAS,Flow,46.77722,-68.15722,Inst, ,"Arroostoock River @ Washburn, ME",Aroostoock River at Washburn,Aroostoock River at Washburn, ,FALSE,NATIVE,America/New_York,Unknown County or County N/A,ME, , , ,SITE,Streamgage,WAS.Flow.Inst.15Minutes.0.DCP-rev, ,0,DCP-rev,436 +213,MVP,ND00310-Spillway,Flow,48.40498,-97.79103,Inst,Homme_Dam-Spillway,Homme Dam Overflow Spillway, , , , , ,America/Chicago,Unknown County or County N/A,0,UNITED STATES, ,MVP,SITE, ,ND00310-Spillway.Flow.Inst.15Minutes.0.rev,Homme_Dam-Spillway.Flow.Inst.15Minutes.0.rev,0,rev,0 +214,MVP,AlamedaRsvr,Flow-Out,49.25889,-102.2306,Inst, ,Alameda Reservoir near Alameda,"Alameda Reservoir near Alameda, SK",Owner is Environment Canada, , , ,US/Central, , ,CANADA, , ,SITE,DCP Gage,AlamedaRsvr.Flow-Out.Inst.15Minutes.0.ProjectEntry, ,0,ProjectEntry,0 +215,MVP,Alameda,Flow,49.34694,-102.2556,Inst, ,Shepherd Creek near Alameda,"Shepherd Creek near Alameda, SK",Owner is Environment Canada, , , ,US/Central, , ,CANADA, , ,SITE,DCP Gage,Alameda.Flow.Inst.5Minutes.0.Raw-EnvCan, ,0,Raw-EnvCan,0 +216,MVP,AlamedaRsvr-MooseMntCreek,Flow,49.52278,-102.1719,Inst, ,Moose Mnt Creek a Alameda RSVR,"Moose Mountain Creek above Alameda Reservoir, SK",Owner is Environment Canada, , , ,US/Central, , ,CANADA, , ,SITE,DCP Gage,AlamedaRsvr-MooseMntCreek.Flow.Inst.5Minutes.0.Raw-EnvCan, ,0,Raw-EnvCan,0 +217,SPK,CA10201,Flow-Spill,39.19806,-123.1806,Ave,"CA10201 -Lake Mendocino Pool-EFRussian",Lk Mendocino-Pool,Coyote Valley Dam,COY - Lake Mendocino (Coyote Dam) Pool Elevation Station,WGS84,FALSE,NATIVE,US/Pacific,Mendocino,CA,UNITED STATES,"Calpella, CA",SPN,SITE, ,"CA10201 -Lake Mendocino Pool-EFRussian.Flow-Spill.Ave.1Hour.1Hour.Calc-val",CA10201.Flow-Spill.Ave.1Hour.1Hour.Calc-val,0,Calc-val,789 +218,NWDP,WA00302,Flow-Out,47.38537,-123.6057,Ave,12035380,Wynoochee Dam,WYNOOCHEE DAM, ,NAD83, , ,US/Pacific,Grays Harbor,WA,UNITED STATES,Aberdeen,NWS,PROJECT,Dam,WA00302.Flow-Out.Ave.1Hour.1Hour.CENWS-COMPUTED-R*,12035380.Flow-Out.Ave.1Hour.1Hour.CENWS-COMPUTED-RAW,0,CENWS-COMPUTED-RAW,0 +219,NWDP,D10D3406,Flow,46.50047,-116.3923,Ave,PEKI,Clearwater R near Peck,CLEARWATER RIVER NEAR PECK 2NE,USGS #13341050,WGS84,FALSE,NATIVE,US/Pacific,Nez Perce,ID,UNITED STATES, ,NWW,SITE,Stream Gauge,D10D3406.Flow.Ave.~1Day.1Day.CBT-COMPUTED-REV,PEKI.Flow.Ave.~1Day.1Day.CBT-COMPUTED-REV,0,CBT-COMPUTED-REV,930 +220,NWDP,ETSI,Flow-Canal,43.93333,-116.4333,Inst,34433F4C,Emmett Irrigat Dist S Side Cnl,Emmett Irrigation Dist South Side Canal,USBR Hydromet Station ETSI,WGS84,FALSE,NATIVE,US/Mountain,Gem,ID,UNITED STATES, ,NWW,SITE,Stream Gauge,ETSI.Flow-Canal.Inst.0.0.USBR-RAW,34433F4C.Flow-Canal.Inst.0.0.USBR-RAW,0,USBR-RAW,2500 +221,NWDP,12371550,Flow-Out,47.68,-114.23,Inst,KERM,Kerr Dam,Kerr Dam, ,NAD83,FALSE,NATIVE,US/Mountain,Lake,MT, , ,NWS,SITE,Dam,12371550.Flow-Out.Inst.~1Day.0.CBT-REV,KERM.Flow-Out.Inst.~1Day.0.CBT-REV,0,CBT-REV,2890 +222,NWDP,MT00652,Flow-Out,48.41056,-115.3131,Ave,CE1192D4,Libby Dam Near Libby,LIBBY DAM NEAR LIBBY, , ,FALSE,NATIVE,US/Mountain,Lincoln,MT,UNITED STATES,Libby,NWS,PROJECT,Corps Reservoir,MT00652.Flow-Out.Ave.1Hour.1Hour.CBT-REV,CE1192D4.Flow-Out.Ave.1Hour.1Hour.CBT-REV,0,CBT-REV,2342 +223,NWDP,SMCI,Flow-Canal,42.6625,-113.4889,Inst, ,S Side Minidoka Cnl nr Minidoka,S. Side Minidoka Canal Near Minidoka, ,WGS84,FALSE,NATIVE,US/Mountain,Cassia,ID,UNITED STATES, ,NWW,SITE,Stream Gauge,SMCI.Flow-Canal.Inst.0.0.USBR-RAW, ,0,USBR-RAW,4184 +224,NWDP,34437294,Flow-Canal,42.67167,-113.4839,Inst,13080000,N Side Minidoka Cnl nr Minidoka,N. Side Minidoka Canal Near Minidoka,USBR Station ID NMCI,WGS84,FALSE,NATIVE,US/Mountain,Minidoka,ID,UNITED STATES, ,NWW,SITE,Stream Gauge,34437294.Flow-Canal.Inst.0.0.USBR-RAW,13080000.Flow-Canal.Inst.0.0.USBR-RAW,0,USBR-RAW,4180 +225,NWDP,MT00565,Flow-Out,48.34111,-114.0133,Ave,345EB372,Hungry Horse Dam nr Hungry Horse,HUNGRY HORSE DAM NEAR HUNGRY HORSE, , ,FALSE,NATIVE,US/Mountain,Flathead,MT,UNITED STATES,Missoula,NWS,PROJECT,Dam,MT00565.Flow-Out.Ave.1Hour.1Hour.CBT-REV,345EB372.Flow-Out.Ave.1Hour.1Hour.CBT-REV,0,CBT-REV,3196 +226,NWDP,WFCI,Flow-Canal,43.90556,-111.6278,Inst,18D21S,Wilford Canal,Wilford Canal,USBR Station ID WFCI,WGS84,FALSE,NATIVE,US/Mountain,Fremont,ID,UNITED STATES, ,NWW,SITE,Stream Gauge,WFCI.Flow-Canal.Inst.0.0.USBR-RAW,18D21S.Flow-Canal.Inst.0.0.USBR-RAW,0,USBR-RAW,4965 +227,NWDP,MHPI,Flow-Discharge-Power,42.51667,-114.0333,Ave, ,Milner Hydroelectric Plant,Milner Hydroelectric Plant, ,WGS84,FALSE,NATIVE,US/Mountain,Cassia,ID,UNITED STATES, ,NWW,SITE,Stream Gauge,MHPI.Flow-Discharge-Power.Ave.~1Day.1Day.USBR-COM*, ,0,USBR-COMPUTED-REV,4120 +228,NWDP,ID00319,Flow-Out,48.17925,-116.9997,Ave,ALF,Albeni Falls Dam,Albeni Falls Dam, , ,FALSE,NATIVE,US/Pacific,Bonner,ID,UNITED STATES,Coeur d'Alene,NWS,PROJECT,Corps Reservoir,ID00319.Flow-Out.Ave.1Hour.1Hour.CBT-REV,ALF.Flow-Out.Ave.1Hour.1Hour.CBT-REV,0,CBT-REV,2047 +229,NWDP,WA00256,Flow-Out-Total,47.40927,-121.724,Ave,12115900,Chester Morse Lake @ Cedar Falls,CHESTER MORSE LAKE AT CEDAR FALLS NEAR NORTH BEND 7SSE, ,NAD83, , ,US/Pacific,King,WA, , ,NWS,SITE,Dam,WA00256.Flow-Out-Total.Ave.1Hour.1Hour.SCL-RAW,12115900.Flow-Out-Total.Ave.1Hour.1Hour.SCL-RAW,0,SCL-RAW,0 +230,NWDP,OR00002,Flow-Out,45.61,-121.13,Ave,14103950,The Dalles Lock and Dam,The Dalles Dam and Lake Celilo on the Columbia River, ,WGS84, , ,US/Pacific,Wasco,OR,UNITED STATES, ,NWDP,PROJECT,Dam,OR00002.Flow-Out.Ave.1Hour.1Hour.CBT-REV,14103950.Flow-Out.Ave.1Hour.1Hour.CBT-REV,0,CBT-REV,0 +231,NWDP,WA00168,Flow-Out,48.69824,-121.2091,Ave,GOD,Gorge Reservoir nr Nehalem,GORGE RESERVOIR NR NEHALEM, , , , ,US/Pacific,Whatcom,WA,UNITED STATES,Newhalem,NWS,PROJECT,Dam,WA00168.Flow-Out.Ave.1Hour.1Hour.SCL-RAW,GOD.Flow-Out.Ave.1Hour.1Hour.SCL-RAW,0,SCL-RAW,0 +232,NWDP,OR00616,Flow-Out,45.93563,-119.2977,Ave,MCN,McNary Lock and Dam,McNary Dam and Lake Wallula,McNary_Dam,WGS84,FALSE,NATIVE,US/Pacific,Umatilla,OR,UNITED STATES, ,NWDP,PROJECT,Dam,OR00616.Flow-Out.Ave.~1Day.1Day.CBT-REV,MCN.Flow-Out.Ave.~1Day.1Day.CBT-REV,0,CBT-REV,361 +233,NWDP,ID00287,Flow-Out,46.51537,-116.2962,Ave,DWR,Dworshak Dam,Dworshak_Dam,Dworshak_Dam Datum Correction 1929-1988 (CorpsCon 6) = 3.320 Ft,WGS84,FALSE,NATIVE,US/Pacific,Clearwater,ID,UNITED STATES,Moscow,NWW,PROJECT,Corps Reservoir,ID00287.Flow-Out.Ave.~6Hours.6Hours.CBT-COMPUTED-*,DWR.Flow-Out.Ave.~6Hours.6Hours.CBT-COMPUTED-REV,0,CBT-COMPUTED-REV,1613 +234,NWDP,TLCI,Flow-Canal,43.72028,-111.8272,Inst,13038434,Texas And Liberty Weir,Texas and Liberty Weir,USBR Station ID TLCI,WGS84,FALSE,NATIVE,US/Mountain,Madison,ID,UNITED STATES, ,NWW,SITE,Stream Gauge,TLCI.Flow-Canal.Inst.0.0.USBR-RAW,13038434.Flow-Canal.Inst.0.0.USBR-RAW,0,USBR-RAW,4895 +235,NWDP,WA00169,Flow-Loc,48.73278,-121.0672,Ave,ROS,Ross Reservoir nr Newhalem,ROSS RESERVOIR NEAR NEWHALEM, , , , ,US/Pacific,Whatcom,WA,UNITED STATES,Newhalem,NWS,PROJECT,Dam,WA00169.Flow-Loc.Ave.1Hour.1Hour.SCL-RAW,ROS.Flow-Loc.Ave.1Hour.1Hour.SCL-RAW,0,SCL-RAW,0 +236,NWDP,WA00300,Flow-Out,47.1422,-121.9313,Inst,MMD,Mud Mountain Dam nr Buckley WA,MUD MOUNTAIN DAM NR BUCKLEY WA, , , , ,US/Pacific,King,WA,UNITED STATES,Buckley,NWS,PROJECT,Corps Reservoir,WA00300.Flow-Out.Inst.1Hour.0.NWSRADIO-COMPUTED-R*,MMD.Flow-Out.Inst.1Hour.0.NWSRADIO-COMPUTED-RAW,0,NWSRADIO-COMPUTED-RAW,0 +237,NWDP,ARK,Flow-Out,43.59529,-115.9225,Ave,347D061C,Arrowrock Dam,ARROWROCK DAM AND RESERVOIR NEAR BOISE 14E,Datum Correction 1929-1988 (CorpsCon 6) = 3.406 Ft,WGS84,FALSE,NATIVE,US/Mountain,Boise,ID,UNITED STATES,Boise,NWW,PROJECT,Bureau of Reclamation Dam,ARK.Flow-Out.Ave.~6Hours.6Hours.USBR-COMPUTED-REV,347D061C.Flow-Out.Ave.~6Hours.6Hours.USBR-COMPUTED-REV,0,USBR-COMPUTED-REV,3220 +238,NWDP,DIA,Flow-Out,48.71556,-121.1506,Ave, ,Diablo Reservoir nr Newhalem 6NE,DIABLO RESERVOIR NEAR NEWHALEM 6NE, , , , ,US/Pacific,Whatcom,WA, , ,NWS,SITE,Dam,DIA.Flow-Out.Ave.~1Day.1Day.CBT-REV, ,0,CBT-REV,0 +239,NWDP,SQWI,Flow,43.96667,-116.3314,Inst,3483268C,Squaw Cr near Sweet 1S,Squaw Creek near Sweet 1S,USGS #13297355,WGS84,FALSE,NATIVE,US/Mountain,Gem,ID,UNITED STATES, ,NWW,SITE,Stream Gauge,SQWI.Flow.Inst.0.0.USBR-RAW,3483268C.Flow.Inst.0.0.USBR-RAW,0,USBR-RAW,5710 +240,NWDP,WA00173,Flow-Out,48.64929,-121.6907,Inst,12191600,Baker Lk @ Upr Baker Dm nr Conct,BAKER LAKE AT UPPER BAKER DAM NEAR CONCRETE, ,NAD83, , ,US/Pacific,Whatcom,WA,UNITED STATES,Concrete,NWS,PROJECT,Dam,WA00173.Flow-Out.Inst.1Hour.0.PSE-RAW,12191600.Flow-Out.Inst.1Hour.0.PSE-RAW,0,PSE-RAW,0 +241,NWDP,WA00085,Flow-Out,46.87472,-119.9714,Ave,WAN,Wanapum Dam,Wanapum Dam, , , , ,US/Pacific,Kittitas,WA, , ,NWDP,PROJECT,Dam,WA00085.Flow-Out.Ave.1Hour.1Hour.CBT-REV,WAN.Flow-Out.Ave.1Hour.1Hour.CBT-REV,0,CBT-REV,0 +242,NWDP,WA00086,Flow-Out,47.53371,-120.296,Ave,RRH,Rocky Reach Dam,Rocky Reach Dam, ,NAD83, , ,US/Pacific,Chelan,WA, , ,NWDP,PROJECT,Dam,WA00086.Flow-Out.Ave.1Hour.1Hour.CBT-REV,RRH.Flow-Out.Ave.1Hour.1Hour.CBT-REV,0,CBT-REV,0 +243,NWDP,WA00009,Flow-Out,48.98722,-117.3475,Ave,BDY,Boundary Dam,Boundary Dam, , , , ,US/Pacific,Pend Oreille,WA,UNITED STATES,Spokane,NWS,PROJECT,Dam,WA00009.Flow-Out.Ave.~1Day.1Day.CBT-REV,BDY.Flow-Out.Ave.~1Day.1Day.CBT-REV,0,CBT-REV,0 +244,NWDP,14128860,Flow-Out,45.64583,-121.9389,Ave,BON,Bonneville Lock and Dam,Bonneville Dam and Lake On Columbia River, ,WGS84, , ,US/Pacific,Multnomah,OR,UNITED STATES, ,NWDP,PROJECT,Dam,14128860.Flow-Out.Ave.1Hour.1Hour.CBT-REV,BON.Flow-Out.Ave.1Hour.1Hour.CBT-REV,0,CBT-REV,0 +245,NWDP,WA00098,Flow-Out,47.94722,-119.8651,Ave,WEL,Wells Dam,Wells Dam, ,NAD83, , ,US/Central,Douglas,WA, , ,NWDP,PROJECT,Dam,WA00098.Flow-Out.Ave.1Hour.1Hour.CBT-REV,WEL.Flow-Out.Ave.1Hour.1Hour.CBT-REV,0,CBT-REV,0 +246,NWDP,WA00088,Flow-Out,46.64442,-119.9099,Ave,PRD,Priest Rapids Dam,Priest Rapids Dam, , , , ,US/Pacific,Grant,WA, , ,NWDP,PROJECT,Dam,WA00088.Flow-Out.Ave.1Hour.1Hour.CBT-REV,PRD.Flow-Out.Ave.1Hour.1Hour.CBT-REV,0,CBT-REV,0 +247,NWDP,14048006,Flow-Out,45.71485,-120.6937,Ave,JDA,John Day Lock and Dam,John Day Dam and Lake Umatilla, ,WGS84, , ,US/Pacific,Sherman,OR,UNITED STATES, ,NWDP,PROJECT,Dam,14048006.Flow-Out.Ave.1Hour.1Hour.CBT-REV,JDA.Flow-Out.Ave.1Hour.1Hour.CBT-REV,0,CBT-REV,0 +248,NWDP,WA00299,Flow-Out,47.99511,-119.6411,Ave,12437990,Chief Joseph Dam,Chief Joseph Dam, ,NAD83, , ,US/Pacific,Douglas,WA,UNITED STATES, ,NWDP,PROJECT,Dam,WA00299.Flow-Out.Ave.1Hour.1Hour.CBT-REV,12437990.Flow-Out.Ave.1Hour.1Hour.CBT-REV,0,CBT-REV,0 +249,NWDP,GCL,Flow-Out,47.95667,-118.9805,Ave, ,Grand Coulee Dam,Grand Coulee Dam 1 Southwest, , ,FALSE,NATIVE,US/Pacific,Okanogan,WA, , ,NWDP,PROJECT,Dam,GCL.Flow-Out.Ave.1Hour.1Hour.CBT-REV, ,0,CBT-REV,1929 +250,NWDP,WA00084,Flow-Out,47.33984,-120.0945,Ave,RIS,Rock Island Dam,Rock Island Dam, ,NAD83, , ,US/Pacific,Chelan,WA, , ,NWDP,PROJECT,Dam,WA00084.Flow-Out.Ave.1Hour.1Hour.CBT-REV,RIS.Flow-Out.Ave.1Hour.1Hour.CBT-REV,0,CBT-REV,0 +251,NWDP,CSCI,Flow,44.52461,-116.0397,Ave, ,North Fork Payette R @ Cascade,NORTH FORK PAYETTE RIVER AT CASCADE,USGS #13245000,WGS84,FALSE,NATIVE,US/Mountain,Valley,ID,UNITED STATES, ,NWW,SITE,Stream Gauge,CSCI.Flow.Ave.~1Day.1Day.USBR-COMPUTED-REV, ,0,USBR-COMPUTED-REV,4720 +252,NWDP,THFO,Flow,45,-117.7833,Inst, ,Powder R Blw Thief Vly Res,Powder River Below Thief Valley Reservoir Near North Powder,USGS #13285500,WGS84,FALSE,NATIVE,US/Pacific,Union,OR,UNITED STATES, ,NWW,SITE,Stream Gauge,THFO.Flow.Inst.15Minutes.0.OWRD-RAW, ,0,OWRD-RAW,3080 +253,NWDP,REVB,Flow-Out,51.05,-118.19,Ave, ,Revelstoke Dam,Revelstoke Dam, , ,FALSE,NATIVE,Unknown or Not Applicable,Unknown County or County N/A,0, , ,NWDP,SITE,Dam,REVB.Flow-Out.Ave.1Hour.1Hour.BCHYDRO-RAW, ,0,BCHYDRO-RAW,1660 diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/EmptyDirOrFileException.py b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/EmptyDirOrFileException.py new file mode 100644 index 0000000..82f182b --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/EmptyDirOrFileException.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python +class EmptyDirOrFileException( Exception ): + """Exception raised for empty input files or directories. + Attributes: + expression -- input expression in which the error occurred + message -- explanation of the error + """ + pass + +# def __init__(self, message): +# self.expression = expression +# self.message = message diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/Observation.py b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/Observation.py new file mode 100644 index 0000000..cb29241 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/Observation.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python +############################################################################### +# Module name: Observation +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: 5/24/2019 # +# # +# Last modification date: +# # +# Description: Abstract the observed real-time stream flow +# # +############################################################################### + +import os, logging +from string import * +from datetime import datetime, timedelta +import dateutil.parser +import pytz +from abc import ABCMeta, abstractmethod, abstractproperty +from TimeSlice import TimeSlice + + +class Observation: + """ + Abstract real-time flow time series. + """ + __metaclass__ = ABCMeta + +# @abstractproperty +# def source(self): +# pass +# +# @abstractproperty +# def stationID(self): +# pass +# +# @abstractproperty +# def stationName(self): +# pass +# +# @abstractproperty +# def obvPeriod(self): +# pass +# +# @abstractproperty +# def unit(self): +# pass +# +# @abstractproperty +# def timeValueQuality(self): +# pass + + @property + def source(self): + return self._source + + @source.setter + def source(self, s): + self._source = s + + @property + def stationID(self): + return self._stationID + + @stationID.setter + def stationID(self, s): + self._stationID = s + + @property + def stationName(self): + return self._stationName + + @stationName.setter + def stationName(self, s): + self._stationName = s + + @property + def obvPeriod(self): + return self._obvPeriod + + @obvPeriod.setter + def obvPeriod(self, p): + self._obvPeriod = p + + @property + def unit(self): + return self._unit + + @unit.setter + def unit(self, u): + self._unit = u + + @property + def timeValueQuality(self): + return self._timeValueQuality + + @timeValueQuality.setter + def timeValueQuality(self, tvq): + self._timeValueQuality = tvq + + @abstractmethod + def __init__(self, filename ): + pass + + + def getTimeValueAt(self, at_time, resolution = timedelta() ): + """ + Get the closest time-value pair for a given time + + Input: at_time - the given time + resolution - the tolerance time period around the + given time + + Return: Tuple of a time-value pair + """ + + closestTimes = [] + distances = [] + if at_time in self.timeValueQuality: + return ( at_time, self.timeValueQuality.get( at_time ) ) +# for k in sorted( self.timeValueQuality ): + for k in self.timeValueQuality: + if ( abs( k - at_time ) <= resolution / 2 ): + closestTimes.append( k ) + distances.append( abs( k - at_time ) ) + if not closestTimes: + return None + else: + closest = [ x for y, x in \ + sorted( zip( distances, closestTimes ) ) ][ 0 ] + + return ( closest, self.timeValueQuality.get( closest ) ) + + +class All_Observations: + "Store all obvserved data" + def __init__(self, obvs ): + """ + Initialize the All_Objections object for a given + Observation + + Input: list of Observation object + """ + self.logger = logging.getLogger(__name__) + self.observations = obvs + + if not self.observations: + raise RuntimeError( "FATAL ERROR: has no data") + + self.index = -1 + + self.timePeriod = self.observations[0].obvPeriod + for obv in self.observations: + if self.timePeriod[0] > obv.obvPeriod[ 0 ]: + self.timePeriod = ( obv.obvPeriod[ 0 ], \ + self.timePeriod[ 1 ]) + + if self.timePeriod[1] < obv.obvPeriod[ 1 ]: + self.timePeriod = ( self.timePeriod[ 0 ], \ + obv.obvPeriod[ 1 ] ) + + def __iter__(self ): + """ + The iterator + """ + return self + + def __next__( self ): + """ + The next Observation object + """ + if self.index == len( self.observations ) - 1: + self.index = -1 + raise StopIteration + self.index = self.index + 1 + return self.observations[ self.index ] + + def timePeriodForAll( self ): + """ + Get the earlist and latest time for all observations + + Return: Tuple of start time and end time + """ + return self.timePeriod + + def makeTimeSlice( self, timestamp, timeresolution ): + """ + Create one time slice for a given time and resolution + + Input: timestamp - the given time + timeresolution - the resolution + + Return: A time slice object + """ + station_time_value_list = [] + for obv in self: + closestObv = obv.getTimeValueAt( timestamp, timeresolution ) + if closestObv: + station_time_value_list.append( \ + ( obv.stationID, closestObv[ 0 ], \ + # value quality + closestObv[ 1 ][ 0 ], closestObv[ 1 ][ 1 ] ) ) + +# Tracer.theTracer.run( 'timeSlice = TimeSlice( timestamp, timeresolution, station_time_value_list )' ) + timeSlice = TimeSlice( timestamp, \ + timeresolution, \ + station_time_value_list ) + return timeSlice + + def makeAllTimeSlices( self, timeresolution, outdir, suffix='usaceTimeSlice.ncdf' ): + """ + Create the time slice NetCDF files for all USACE observations + + Input: timeresolution - resolution + outdir - the output directory + + Return: Total number of time slice files created + """ + + # the time resultions must divide 60 minutes with on remainder + if 3600 % timeresolution.seconds != 0: + raise RuntimeError( "FATAL ERROR: Time slice resolution must " + "divide 60 minutes with no remainder." ) + + startTime = datetime( self.timePeriod[ 0 ].year, + self.timePeriod[ 0 ].month, + self.timePeriod[ 0 ].day, + self.timePeriod[ 0 ].hour ) + + while startTime < self.timePeriod[ 0 ]: + startTime += timeresolution + + if startTime > self.timePeriod[ 1 ]: + raise RuntimeError( \ + "FATAL ERROR: observation time period wrong! " ) + + count = 0 + while startTime <= self.timePeriod[ 1 ]: + self.logger.info( "making time slice for " + \ + startTime.isoformat() ) +# Tracer.theTracer.run( \ +# 'oneSlice = makeTimeSlice( startTime, timeresolution )' ) + + oneSlice = self.makeTimeSlice( startTime, timeresolution ) + self.logger.info( "Time slice: " + \ + outdir + '/' +oneSlice.getSliceNCFileName( suffix ) ) + if ( not oneSlice.isEmpty() ): + updatedOrNew = True + slicefilename = outdir + '/' +oneSlice.getSliceNCFileName( suffix ) + if os.path.isfile( slicefilename ): +# Tracer.theTracer.run( \ +# 'oldslice = TimeSlice.fromNetCDF( slicefilename )' ) + oldslice = TimeSlice.fromNetCDF( slicefilename ) + updatedOrNew = oneSlice.mergeOld( oldslice ) + + if updatedOrNew: + oneSlice.toNetCDF( outdir, suffix ) + self.logger.info( oneSlice.getSliceNCFileName( suffix ) + \ + " updated!" ) + else: + self.logger.info( oneSlice.getSliceNCFileName( suffix ) + \ + " not updated!" ) +# oneSlice.print_station_time_value() + count = count + 1 + + startTime += timeresolution + +# for eachSlice in allTimeSlices: +# print "-----------------------------" +# eachSlice.print_station_time_value() +# print eachSlice.getSliceNCFileName() +# +# return allTimeSlices + return count diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/TimeSlice.py b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/TimeSlice.py new file mode 100644 index 0000000..740ee96 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/TimeSlice.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python +############################################################################### +# Module name: TimeSlice # +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: # +# # +# Last modification date: 7/12/2017 # +# # +# Description: manage a time slice file that contains real-time stream # +# flow data for all USACE stations for a given time stamp # +# # +############################################################################### +import os, sys, time, math +from string import * +from datetime import datetime, timedelta +import calendar +#import xml.utils.iso8601 +#from netCDF4 import Dataset +import netCDF4 +import numpy as np + +# +# Check if two variables are considered equal. +# +# Input: a - one of the two variables to be compared +# b - one of the two variables to be compared +# rel_tol - relative tolerance +# abs_tol - absolute tolerance +# +# Return: boolean +# +def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): + ''' + Python 2 implementation of Python 3.5 math.isclose() + https://hg.python.org/cpython/file/tip/Modules/mathmodule.c#l1993 + ''' + # sanity check on the inputs + if rel_tol < 0 or abs_tol < 0: + raise ValueError("tolerances must be non-negative") + + # short circuit exact equality -- needed to catch two infinities of + # the same sign. And perhaps speeds things up a bit sometimes. + if a == b: + return True + + # This catches the case of two infinities of opposite sign, or + # one infinity and one finite number. Two infinities of opposite + # sign would otherwise have an infinite relative tolerance. + # Two infinities of the same sign are caught by the equality check + # above. + if math.isinf(a) or math.isinf(b): + return False + + # now do the regular computation + # this is essentially the "weak" test from the Boost library + diff = math.fabs(b - a) + result = (((diff <= math.fabs(rel_tol * b)) or + (diff <= math.fabs(rel_tol * a))) or + (diff <= abs_tol)) + return result + +# +# Compare two Python dictionary objects +# +# Input: d1 - one of the two dictionary object to be compared +# d2 - one of the two dictionary object to be compared +# +# Return: Tuple of added element set, removed element set, modified element +# set and the same element set, +# +def dict_compare(d1, d2): + d1_keys = set(d1.keys()) + d2_keys = set(d2.keys()) + intersect_keys = d1_keys.intersection(d2_keys) + added = d1_keys - d2_keys + removed = d2_keys - d1_keys + modified = dict() + for o in intersect_keys: + if d1[o][0] != d2[o][0] or \ + not isclose( d1[o][1], d2[o][1],abs_tol=0.01 ) \ + or not isclose( d1[o][2], d2[o][2]): + modified[o] = (d1[o], d2[o] ) + same = set(o for o in intersect_keys if d1[o] == d2[o]) + return added, removed, modified, same + +class TimeSlice: + """ + Description: Store one time slice data + Author: Zhengtao Cui (Zhengtao.Cui@noaa.gov) + Date: Aug. 26, 2015 + Date modified: Oct. 20, 2015, Fixed bugs in mergeOld. + """ + + stationIdStrLen = 15 + stationIdLong_name = "USACE station identifer of length 15" + timeStrLen = 19 + timeUnit = "UTC" + timeLong_name = "YYYY-MM-DD_HH:mm:ss UTC" + dischargeUnit = "m^3/s" + dischargeLong_name = "Discharge.cubic_meters_per_second" + dischargeQualityUnit = "-" + dischargeQualaityLong_name = \ + "Discharge quality 0 to 100 to be scaled by 100." + + def __init__(self, time_stamp, resolution, station_time_value ): + """ + Initialize a TimeSlice object + Input: time_stamp - a time stamp + resolution - time resolution of the time slices + station_time_value - Tuple of (station, time, flow, + quality) + """ + self.centralTimeStamp = time_stamp + self.sliceTimeResolution = resolution + self.obvStationTimeValue = station_time_value + + def isEmpty( self ): + """ + Test if the time slice is empty + Return: boolean + """ + return not self.obvStationTimeValue + + def print_station_time_value( self ): + """ + Print the time slice data + """ + for e in self.obvStationTimeValue: + print( "Slice: central time: ", \ + self.centralTimeStamp.isoformat(), \ + e[ 0 ], e[ 1 ].isoformat(), e[ 2 ] ) + + def getStationIDs( self ): + """ + Get all station ids of the time slices + Return: list of station ids + """ + stationL = [] + for e in self.obvStationTimeValue: + stationL.append( list( e[ 0 ] ) ) + #stationL.append( e[ 0 ][5:] ) + for s in stationL: + if len(s) < self.stationIdStrLen: + for i in range( len(s), self.stationIdStrLen ): + s.insert(0, ' ' ) + elif len(s) > self.stationIdStrLen: + s = s[0:self.stationIdStrLen - 1] + + return stationL + + def getDischargeValues( self ): + """ + Get all stream flow values of the time slice + Return: list of flow values + """ + values = [] + for e in self.obvStationTimeValue: + values.append( e[ 2 ] ) + return values + + def getDischargeTimes( self ): + """ + Get all observation times of the time slice + Return: list of observation times + """ + obvtimes = [] + for e in self.obvStationTimeValue: + obvtimes.append( \ + list( e[ 1 ].strftime( "%Y-%m-%d_%H:%M:00" ) ) ) + return obvtimes + + def getQueryTimes( self ): + """ + Get all query times of the time slice + Return: list of query times + """ + qtimes = [] + for e in self.obvStationTimeValue: + # print 'getQueryTime: ', e[ 1 ].isoformat() + # print 'getQueryTime: ', e[ 1 ].utctimetuple() + # print 'getQueryTime: ', time.mktime( e[ 1 ].utctimetuple() ) + #qtimes.append( time.mktime( e[ 1 ].utctimetuple() ) ) + qtimes.append( calendar.timegm( e[ 1 ].utctimetuple() ) ) + return qtimes + + def getSliceNCFileName( self, suffix='usaceTimeSlice.ncdf' ): + """ + Get NetCDF file for this time slice + Return: A NetCDF filename + """ + filename = \ + self.centralTimeStamp.strftime( "%Y-%m-%d_%H:%M:00." ) + \ + str( int( self.sliceTimeResolution.days * 24 * 60 + \ + self.sliceTimeResolution.seconds // 60 ) ).zfill(2) + \ + "min." + suffix + return filename + + def getDischargeQuality( self ): + """ + Get discharge quality for this time slice + Return: A list of discharge quality + """ + dq = [] + for e in self.obvStationTimeValue: + dq.append( e[ 3 ] ) + return dq + + def toNetCDF( self, outputdir = './', suffix='usaceTimeSlice.ncdf' ): + """ + Write the time slice to a NetCDF file + Input: outputdir - the directory where to write the NetCDF + """ + + nc_fid = netCDF4.Dataset( \ + outputdir + '/' + self.getSliceNCFileName( suffix ), \ + 'w', format='NETCDF4' ) + nc_fid.createDimension( 'stationIdStrLen', self.stationIdStrLen ) + nc_fid.createDimension( 'stationIdInd', None ) + nc_fid.createDimension( 'timeStrLen', self.timeStrLen ) + stationId = nc_fid.createVariable( 'stationId', 'S1',\ + ('stationIdInd', 'stationIdStrLen') ) + stationId.setncatts( {'long_name' : \ + self.stationIdLong_name, \ + 'units' : '-'} ) + + time = nc_fid.createVariable( 'time', 'S1',\ + ('stationIdInd', 'timeStrLen' ) ) + time.setncatts( {'long_name' : self.timeLong_name, \ + 'units' : self.timeUnit} ) + + discharge = nc_fid.createVariable( 'discharge', 'f4',\ + ('stationIdInd', ) ) + discharge.setncatts( {'long_name' : \ + self.dischargeLong_name, \ + 'units' : self.dischargeUnit} ) + discharge_quality = \ + nc_fid.createVariable( 'discharge_quality', 'i2',\ + ('stationIdInd', ) ) + discharge_quality.setncatts( {'long_name' : \ + self.dischargeQualaityLong_name, \ + 'units' : self.dischargeQualityUnit, \ + 'multfactor' : '0.01' } ) + queryTime = nc_fid.createVariable( 'queryTime', 'i4',\ + ('stationIdInd', ) ) + + queryTime.setncatts( { 'units' : \ + 'seconds since 1970-01-01 00:00:00 local TZ' \ + } ) + + nc_fid.setncatts( { 'fileUpdateTimeUTC': \ + datetime.utcnow().strftime( "%Y-%m-%d_%H:%M:00" ), \ + 'sliceCenterTimeUTC' : \ + self.centralTimeStamp.strftime( "%Y-%m-%d_%H:%M:00" ),\ + 'sliceTimeResolutionMinutes' : \ + str( int( self.sliceTimeResolution.days * 24 * 60 + \ + self.sliceTimeResolution.seconds // 60 ) ).zfill(2) } ) + + #print 'discharge: ', self.getDischargeValues() + discharge[ : ] = self.getDischargeValues() + queryTime[ : ] = self.getQueryTimes() + + #stationId[ : ] = netCDF4.stringtochar( \ + # np.array( stations ) ) + stations = self.getStationIDs() + stationId[ : ] = stations + # time[ : ] = \ + # [ list( self.centralTimeStamp.strftime( "%Y-%m-%d_%H:%M:00" ) ) \ + # for i in range( len( stations ) ) ] + + time[ : ] = self.getDischargeTimes() + discharge_quality[:] = self.getDischargeQuality() + + nc_fid.close() + + @classmethod + def fromNetCDF( self, ncfilename ): + """ + Read the time slice from a NetCDF file + Input: ncfilename - the NetCDF filename + """ + nc_fid = netCDF4.Dataset( ncfilename, 'r' ) + timestamp = datetime.strptime( \ + nc_fid.getncattr( 'sliceCenterTimeUTC' ), \ + "%Y-%m-%d_%H:%M:00" ) + #print timestamp.isoformat() + + time_resol = timedelta( minutes = \ + int( nc_fid.getncattr( 'sliceTimeResolutionMinutes' ) ) ) + + stations = netCDF4.chartostring( \ + nc_fid.variables[ 'stationId'][ : ] ) + + discharge = nc_fid.variables[ 'discharge'][ : ] + + queryTime = nc_fid.variables[ 'queryTime'][ : ] + + quality = nc_fid.variables[ 'discharge_quality'][ : ] + +# for s, d, q in zip( stations, discharge, queryTime ): +# print 'USACE.' + s.rstrip(), d, \ +# datetime.utcfromtimestamp( q ).isoformat(), \ +# datetime.utcfromtimestamp( q ).tzname(), \ +# q + + +# self.centralTimeStamp = timestamp +# self.sliceTimeResolution = time_resol +# self.obvStationTimeValue = [] + stationTimeValue = [] + for s, d, q, qual in zip( stations, discharge, queryTime, quality ): + stationTimeValue.append( \ + ( s.strip(), \ + datetime.utcfromtimestamp( q ), d, qual ) ) + + nc_fid.close() + + return self( timestamp, time_resol, stationTimeValue ) + + def mergeOld( self, oldTimeSlice ): + """ + Merge data in an existing NetCDF time slice file with this one + Input: oldTimeSlice - the NetCDF filename of the existing time + slice + """ + + if self.centralTimeStamp != oldTimeSlice.centralTimeStamp or \ + self.sliceTimeResolution != oldTimeSlice.sliceTimeResolution: + raise RuntimeError( "FATAL ERROR: the two time slices "\ + " differ, not merging ..." ) + else: + + site_time_value = dict() + for e in self.obvStationTimeValue: + #print "New : ", e + site_time_value[ e[ 0 ] ] = ( e[1], e[2], e[3] ) + old_site_time_value = dict() + + for e in oldTimeSlice.obvStationTimeValue: + #print "Old : ", e + old_site_time_value[ e[ 0 ] ] = ( e[1], e[2], e[3] ) + + added, removed, modified, same = dict_compare( \ + site_time_value, old_site_time_value ) + + #if not added and not removed and not modified \ + # and len(same) == len( self.obvStationTimeValue ): + if not added and not modified : + return False + + old_site_time_value.update( site_time_value ) + + self.obvStationTimeValue = [] + + for site in old_site_time_value: + self.obvStationTimeValue.append( ( site, \ + old_site_time_value[ site ][ 0 ], \ + old_site_time_value[ site ][ 1 ], \ + old_site_time_value[ site ][ 2 ] ) ) + + return True + #print "TimeSlice: merged old time slice!" + #print self.obvStationTimeValue diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/make_time_slice_from_ace.py b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/make_time_slice_from_ace.py new file mode 100644 index 0000000..03d6cb7 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/make_time_slice_from_ace.py @@ -0,0 +1,156 @@ +#! /usr/bin/env python +############################################################################### +# File name: make_time_slice_from_ace.py # +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: # +# # +# Last modification date: 5/30/2019 # +# # +# Description: The driver to create NetCDF time slice files from Army Crops # +# of Engineers real-time observations # +# # +# 12/11/2024 OWP Update code to use JSON file instead of XML # +# # +############################################################################### + +import os, sys, time, urllib, getopt +import logging +from string import * +import xml.etree.ElementTree as etree +from datetime import datetime, timedelta +from TimeSlice import TimeSlice +from Observation import Observation, All_Observations +from CWMS_Sites import CWMS_Sites +from ACE_Observation import ACE_Observation +from EmptyDirOrFileException import EmptyDirOrFileException + +""" + The driver to parse downloaded ACE JSON observations and + create time slices and write to NetCDF files + Author: Zhengtao Cui (Zhengtao.Cui@noaa.gov) + Date: May 30, 2019 +""" +def main(argv): + """ + function to get input arguments + """ + inputdir = '' + try: + opts, args = getopt.getopt(argv,"hi:o:s:",["idir=", "odir=", "sites="]) + except getopt.GetoptError: + print( 'make_time_slice_from_ace.py -i -o -s ' ) + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print( \ + 'make_time_slice_from_ace.py -i -o -s ' ) + sys.exit() + elif opt in ('-i', "--idir"): + inputdir = arg + if not os.path.exists( inputdir ): + raise RuntimeError( 'FATAL ERROR: inputdir ' + \ + inputdir + ' does not exist!' ) + elif opt in ('-o', "--odir" ): + outputdir = arg + if not os.path.exists( outputdir ): + raise RuntimeError( 'FATAL ERROR: outputdir ' + \ + outputdir + ' does not exist!' ) + elif opt in ('-s', "--sites" ): + sitefile = arg + if not os.path.exists( sitefile ): + raise RuntimeError( 'FATAL ERROR: sitefile ' + \ + sitefile + ' does not exist!' ) + + return (inputdir, outputdir, sitefile) + +t0 = time.time() + +logging.basicConfig(format=\ + '%(asctime)s - %(name)s - %(levelname)s - %(message)s',\ + level=logging.INFO) +logger = logging.getLogger(__name__) +formatter = logging.Formatter(\ + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +#logger.setFormatter(formatter) +logger.info( "System Path: " + str( sys.path ) ) + +if __name__ == "__main__": + try: + odir = main(sys.argv[1:]) + except Exception as e: + logger.error("Failed to get program options.", exc_info=True) + +indir = odir[0] +outdir = odir[1] +sitefile = odir[2] +logger.info( 'Input dir is "' + indir + '"') +logger.info( 'Output dir is "' + outdir + '"') +logger.info( 'Site file is "' + sitefile + '"') + +# +# Load ACE observed JSON discharge data +# + +try: + sites=CWMS_Sites( sitefile ) + + obvs = [] + + if not os.path.isdir( indir ): + raise RuntimeError( "FATAL ERROR: " + indir + \ + " is not a directory or does not exist. ") + + for file in os.listdir( indir ): + # Process json file if file size is greater than 2 bytes (not empty) + if file.endswith( ".json" ) and os.path.getsize(indir+"/"+file) > 2: + logger.info( 'Reading ' + indir + '/' + file + ' ... ' ) + try: + obvs.append( ACE_Observation( \ + indir + '/' + file, sites ) ) + except Exception as e: + logger.warning( repr( e ), exc_info=True ) + continue + else: + logger.warning('Skip ' + indir + '/' + file + ' because file is empty (2 bytes filesize).\n') + + if not obvs: + raise EmptyDirOrFileException( "Input directory " + indir + \ + " has no USACE json files or the json files are empty!") + + allobvs = All_Observations( obvs ) + +except EmptyDirOrFileException as e: + logger.warning( str(e), exc_info=True) + sys.exit(0) + +except Exception as e: + logger.error("Failed to load USACE JSON files: " + str(e), exc_info=True) + sys.exit(3) + + +logger.info( 'Earliest time in USACE JSON: ' + \ + allobvs.timePeriodForAll()[0].isoformat() ) +logger.info( 'Latest time in USACE JSON: ' + \ + allobvs.timePeriodForAll()[1].isoformat() ) + +# +# Create time slices from loaded observations +# +# Set time resolution to 15 minutes +# and +# Write time slices to NetCDF files +# +try: + timeslices = allobvs.makeAllTimeSlices( timedelta( minutes = 15 ), \ + outdir, 'usaceTimeSlice.ncdf' ) +except Exception as e: + logger.error("Failed to make time slices: " + str(e), exc_info=True) + logger.error("Input dir = " + indir, exc_info=True) + sys.exit(4) + +logger.info( "Total number of timeslices: " + str( timeslices ) ) +logger.info( "Program finished in: " + \ + "{0:.1f}".format( (time.time() - t0) / 60.0 ) + \ + " minutes" ) diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/nwmcopy.sh b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/nwmcopy.sh new file mode 100644 index 0000000..90a5072 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/nwmcopy.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash + +############################################################################### +# Program Name: nwmcopy.sh # +# # +# Author(s)/Contact(s): NWC # +# # +# copy files to and from the $COM and $DBN alert directories # +# # +# Input: directory in $COM # +# # +# Output: files # +# # +# For non-fatal errors output is witten to $DATA/LOGS # +# # +# Origination Jun, 2019 # +# OWP Dec, 2021 Port to WCOSS2 # +# # +############################################################################### +# --------------------------------------------------------------------------- # + +####################################### +# nwm_copy and nwm_postcopy utilize +# multiple cores to speed up file copying +####################################### +function nwm_copy () { + msg="Begin copying file at `date`" + postmsg "${pgmout}" "${msg}" + + dir=$1 + fourcyclesago=$($NDATE -4 ${PDY}${cyc}) + echo "#!/usr/bin/env bash" > $DATA/nwmcopyscript + for file in $COMIN/${dir}/* + do + test -f "$file" || continue + filedatetime=$(basename $file | cut -c1-4 )$(basename $file | cut -c6-7 )$(basename $file | cut -c9-10)$(basename $file | cut -c12-13 ) + # + # Copy only files of the past 4 hours + # + if [ $fourcyclesago -le $filedatetime ]; then + echo "cpfs $file $DATA/$(basename $file)" >> $DATA/nwmcopyscript + fi + done + + # + # Previous day + # + if [ ${cyc} -le 4 ]; then + for file in $COMINm1/${dir}/* + do + test -f "$file" || continue + filedatetime=$(basename $file | cut -c1-4 )$(basename $file | cut -c6-7 )$(basename $file | cut -c9-10)$(basename $file | cut -c12-13 ) + # + # Copy only files of the past 4 hours + # + if [ $fourcyclesago -le $filedatetime ]; then + echo "cpfs $file $DATA/$(basename $file)" >> $DATA/nwmcopyscript + fi + done + fi + + chmod 755 $DATA/nwmcopyscript + + #aprun -j1 -n$((NODES*NCORES)) -N${NCORES} cfp $DATA/nwmcopyscript + ${CFPCOMMAND} $DATA/nwmcopyscript + export err=$?; err_chk + + msg="Ending copy file at `date`" + postmsg "$pgmout" "$msg" + +} + +function nwm_postcopy () { + msg="Begin post copying file at `date`" + postmsg "${pgmout}" "${msg}" + + dir=$1 + suffix=$2 + fourcyclesago=$($NDATE -4 ${PDY}${cyc}) + echo "#!/usr/bin/env bash" > $DATA/nwmpostcopyscript + for file in $DATA/*.${suffix} + do + test -f "$file" || continue + filedatetime=$(basename $file | cut -c1-4 )$(basename $file | cut -c6-7 )$(basename $file | cut -c9-10)$(basename $file | cut -c12-13 ) + # + # Copy only files of the past 4 hours + # + if [ $fourcyclesago -le $filedatetime ]; then + # + # NOTE: Here, we assume $COMOUT == $COMIN, otherwise, it doesn't work. + # + #export COMOUT_ROOT=${COMOUT_ROOT:-${COMROOT}/${NET}/${envir}} + export COMOUT_ROOT=${COMOUT_ROOT:-${COMROOT}/${NET}/${nwm_ver}} + Outdir=${COMOUT_ROOT}/${RUN}.$(basename $file | cut -c1-4 )$(basename \ + $file | cut -c6-7 )$(basename $file | cut -c9-10)/${dir} + + if [ ! -e $Outdir ]; then + mkdir -p $Outdir + fi + + copyandalert=true + + # + # if the file exists and was not changed, don't copy and alert + # + if [ -e ${Outdir}/$(basename $file) ]; then + diff ${file} ${Outdir}/$(basename $file) > /dev/null 2>&1 + if [ $? -eq 0 ]; then + copyandalert=false + fi + fi + + if [[ "$copyandalert" == true ]]; then + echo "cpfs ${file} ${Outdir}/$(basename $file); if [ "$SENDDBN" = YES ]; then $DBNROOT/bin/dbn_alert MODEL ${DBN_ALERT_TYPE} $job ${Outdir}/$(basename $file); fi" >> $DATA/nwmpostcopyscript + fi + + fi + done + + chmod 755 $DATA/nwmpostcopyscript + + #aprun -j1 -n$((NODES*NCORES)) -N${NCORES} cfp $DATA/nwmpostcopyscript + ${CFPCOMMAND} $DATA/nwmpostcopyscript + export err=$?; err_chk + + msg="Ending post copy file at `date`" + postmsg "$pgmout" "$msg" + +} diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/runTimeSlice.sh b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/runTimeSlice.sh new file mode 100755 index 0000000..2ad4053 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/runTimeSlice.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +wkDir=$(pwd) +cd $wkDir +if [ ! -d output ]; then mkdir -p outputs; fi +rm -f ace.log +time python make_time_slice_from_ace.py \ + -i rawdatafiles \ + -o outputs \ + -s $wkDir/site-file.csv >> ace.log 2>&1 diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/test_CWMS_Sites.py b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/test_CWMS_Sites.py new file mode 100644 index 0000000..99cf494 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/analysis/test_CWMS_Sites.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +############################################################################### +# File name: make_time_slice_from_ace.py +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: # +# # +# Last modification date: 7/12/2017 # +# # +# Description: The driver to create NetCDF time slice files from USACE # +# real-time observations # +# # +############################################################################### + +import os, sys, time, urllib, getopt +import logging +from string import * +from datetime import datetime, timedelta +from CWMS_Sites import CWMS_Sites +import csv + +""" + The driver to parse downloaded waterML 2.0 observations and + create time slices and write to NetCDF files + Author: Zhengtao Cui (Zhengtao.Cui@noaa.gov) + Date: Aug. 26, 2015 +""" +def main(argv): + """ + function to get input arguments + """ + sites=CWMS_Sites( "./CWMS_outflow_sites_263_index.csv") + index = sites.getIndex( "SAJ", "S135-Pump.Flow.Inst.1Hour.0.SFWMD-WM") + print("index = " + index) + + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + except Exception as e: + print("Failed to get program options." + str(e)) + + diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/CWMS_download_current.py b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/CWMS_download_current.py new file mode 100644 index 0000000..b797ead --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/CWMS_download_current.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +import os +import datetime + +import csv +import json + +from warnings import warn +from gov.noaa.nwc.cwmstools import CWMSDownloader + +import argparse + +""" +This program downloads stream flow data from the Army Corp of Engineers (ACE) CWMS web service. + +usage: + python CWMS_download_current --file_format= site_file output_directory + +format is one of "json", "xml", or "csv" +""" + +def main(): + print( datetime.datetime.now(), end = " --- " ) + print( 'Entering CWMS_download_current.py ... ' ) + #parse the command line + parser = argparse.ArgumentParser() + + parser.add_argument("site_file",help="Csv file that contains the gage ids, office codes, and NEDID codes of the gages to download") + parser.add_argument("output_dir",help="The directory to store downloaded files") + parser.add_argument("-f","--file_format",help="The format to store downloaded data [json,xml,csv]",default=json) + + args = parser.parse_args() + + output_path = args.output_dir + csv_data = read_input_csv(args.site_file) + output_format = args.file_format + + downloader = CWMSDownloader() + + for row in csv_data: + + if output_format == "json": + json_data = get_data(downloader, row["office"], row["gage"], "PT-48h", "json") + with open(output_path+"/"+row["usace_gage_id"]+".json","w") as outfile: + json.dump(json_data,outfile,indent=2) + print( datetime.datetime.now(), end = " --- " ) + print( 'Successfully downloaded ' + \ + output_path+"/"+row["usace_gage_id"]+".json" + "!" ) + elif output_format == "xml": + xml_data = get_data(downloader, row["office"], row["gage"], "PT-48h", "xml") + with open(output_path+"/"+row["usace_gage_id"]+".xml", "w") as outfile: + try: + outfile.write(xml_data) + except Exception as e: + print( datetime.datetime.now(), end = " --- " ) + print( 'WARNING: Failed writting ' + \ + output_path+"/"+row["usace_gage_id"]+".xml" + "!" ) + else: + print( datetime.datetime.now(), end = " --- " ) + print( 'Successfully downloaded ' + \ + output_path+"/"+row["usace_gage_id"]+".xml" + "!" ) + + + + elif output_format == "csv": + csv_data = get_data(downloader, row["office"], row["gage"], "PT-48h", "csv") + with open(output_path+"/"+row["usace_gage_id"]+".csv", "w") as outfile: + outfile.write(csv_data) + print( datetime.datetime.now(), end = " --- " ) + print( 'Successfully downloaded ' + \ + output_path+"/"+row["usace_gage_id"]+".csv" + "!" ) + else: + print( datetime.datetime.now(), end = " --- " ) + warn("Unexpected output format",RuntimeWarning) + +# with open( output_path + '/fetch_last_success', 'a'): +# os.utime( output_path + '/fetch_last_success', None ) + print( datetime.datetime.now(), end = " --- " ) + print( 'Leaving CWMS_download_current.py ... ' ) + +def read_input_csv(name): + csv_data = [] + + with open(name) as csv_file: + reader = csv.DictReader(csv_file, delimiter=",") + for row in reader: + csv_data.append(row) + return csv_data + + +def get_data(downloader, office, name, start, data_format): + + stop = False + count = 0 + data = {} + + while not stop and count < 3: + try: + data = downloader.get_data(office, name, start, data_format) + stop = True + except: + count += 1 + + return data + + +if __name__ == "__main__": + main() diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/CWMS_download_range.py b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/CWMS_download_range.py new file mode 100644 index 0000000..340ddfd --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/CWMS_download_range.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# from datetime import datetime + +import csv +import json + +from warnings import warn +from gov.noaa.nwc.cwmstools import CWMSDownloader + +import argparse + +""" +This program downloads stream flow data from the Army Corp of Engineers (ACE) CWMS web service. + +usage: + python CWMS_download_current --file_format= site_file output_directory start_date stop_date + +format is one of "json", "xml", or "csv" + +start_date and stop_date must be formated YYYYMMDD +""" + + +def main(): + #parse the command line + parser = argparse.ArgumentParser() + + parser.add_argument("site_file",help="Csv file that contains the gage ids, office codes, and NEDID codes of the gages to download") + parser.add_argument("output_dir",help="The directory to store downloaded files") + parser.add_argument("begin", help="The first date in the requested date range") + parser.add_argument("end", help="The last date in the request date range") + parser.add_argument("-f","--file_format",help="The format to store downloaded data [json,xml,csv]",default=json) + + args = parser.parse_args() + + output_path = args.output_dir + csv_data = read_input_csv(args.site_file) + output_format = args.file_format + begin = args.begin + end = args.end + + #output_path = "/home/dwj/PycharmProjects/CWMS_eval_1/output/" + #csv_data = read_input_csv("/media/sf_Projects/Lakes_upstream_of_A2W_sites_323_total_Final.csv") + #output_format = "xml" + + downloader = CWMSDownloader() + + for row in csv_data: + + if output_format == "json": + json_data = get_data(downloader, row["office"], row["gage"], begin, end, "json") + with open(output_path+row["usace_gage_id"]+".json","w") as outfile: + json.dump(json_data,outfile) + elif output_format == "xml": + xml_data = get_data(downloader, row["office"], row["gage"], begin, end, "xml") + with open(output_path+row["usace_gage_id"]+".xml", "w") as outfile: + outfile.write(xml_data) + elif output_format == "csv": + csv_data = get_data(downloader, row["office"], row["gage"], begin, end, "csv") + with open(output_path+row["usace_gage_id"]+".csv", "w") as outfile: + outfile.write(csv_data) + else: + warn("Unexpected output format",RuntimeWarning) + + + + +def read_input_csv(name): + csv_data = [] + + with open(name) as csv_file: + reader = csv.DictReader(csv_file, delimiter=",") + for row in reader: + csv_data.append(row) + return csv_data + + +def get_data(downloader, office, name, begin, end, data_format): + + stop = False + count = 0 + data = {} + + while not stop and count < 3: + try: + data = downloader.get_data_range(office, name, begin, end, data_format) + stop = True + except: + count += 1 + + return data + + +if __name__ == "__main__": + main() diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/CWMS_historic_download.py b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/CWMS_historic_download.py new file mode 100644 index 0000000..e1bb6ed --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/CWMS_historic_download.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +from datetime import datetime +import csv + +import isodate +import os.path + +from gov.noaa.nwc.cwmstools import CWMSDownloader + + +def main(): + csv_data = read_input_csv("/home/dwj/PycharmProjects/CWMS_eval_1/Lakes_upstream_of_A2W_sites_with_NIDID_323_Final.csv") + downloader = CWMSDownloader() + + output_data = [] + + for row in csv_data: + + output_path = "output/" + row["NIDID"]+".csv" + + # dont process gages that allready have stored data + if os.path.exists(output_path) and os.path.isfile(output_path): + continue + + # download data about one gage + (time1, data1) = get_flows_and_times(downloader, row["Office"], row["gage"], "P-70y") + + # calculate daily averages for the current gage's data + output_data = find_daily_values(time1, data1) + + # write an output csv with data on this gage + write_output_csv(output_path, output_data) + + return + +def find_daily_values(times, values): + output_data = [] + + if ( len(times) < 1): + output_data.append({"date" : "N/A", "average-flow" : "N/A", + "quality" : "N/A"}) + elif ( len(times) == 1): + output_data.append({"date": times[0], "average-flow": values[0], + "quality": 100}) + else: + current_date = times[0].date() + current_sum = values[0] + current_count = 1 + for i in range(0, len(times)): + if ( times[i].date() == current_date): + current_sum += values[i] + current_count += 1 + else: + output_data.append({"date" : current_date, + "average-flow" : float(current_sum) / current_count, + "quality" : 100}) + current_date = times[i].date() + current_sum = values[i] + current_count = 1 + + return output_data + + +def write_output_csv(name, csv_data): + with open(name, mode='w') as csv_file: + fieldnames = ["date", "average-flow", "quality"] + + # setup a csv reader with the list of field names + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + writer.writeheader() + + # write the file + writer.writerows(csv_data) + + +def read_input_csv(name): + csv_data = [] + + with open(name) as csv_file: + reader = csv.DictReader(csv_file, delimiter=",") + for row in reader: + csv_data.append(row) + return csv_data + + +def get_flows_and_times(downloader, office, name, start): + + stop = False + count = 0 + data = {} + + while not stop and count < 3: + try: + data = downloader.get_data(office, name, start, "json") + stop = True + except: + count += 1 + + times = [] + flows = [] + try: + for part1 in data['time-series']['time-series']: + if 'irregular-interval-values' in part1.keys(): + for part2 in part1['irregular-interval-values']['values']: + times.append(isodate.parse_datetime(part2[0])) + flows.append(part2[1]) + elif 'regular-interval-values' in part1.keys(): + for segment in part1['regular-interval-values']['segments']: + # print (segment.keys()) + first_time_str = segment['first-time'] + last_time_str = segment['last-time'] + + first_time = isodate.parse_datetime(first_time_str) + last_time = isodate.parse_datetime(last_time_str) + + values = segment['values'] + time_step = (last_time - first_time) / (len(values) - 1) + + for i in range(0, len(values)): + flows.append(values[i][0]) + times.append(first_time + (i * time_step)) + + # print(first_time, last_time, time_step) + + else: + print(part1) + except: + print("FATAL ERROR: No data from office="+office+" name="+name) + + # print(times) + # print(flows) + + return times, flows + + +if __name__ == "__main__": + main() diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/README b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/README new file mode 100644 index 0000000..5bf4fb8 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/README @@ -0,0 +1,7 @@ +- Dec 2024: Changed XML to Json file format +- USACE url: +https://cwms.sec.usace.army.mil/cwms-data/timeseries?name=KEYO.Flow-Out.Ave.1Day.1Day.Raw-USBR&office=NWDM&begin=PT-48h + +wget -O out.json "http://water.usace.army.mil/cwms-data/timeseries?office=NWDM&name=KEYO.Flow-Out.Ave.1Day.1Day.Raw-USBR&begin=PT-48h&format=json" | python -m json.tool out.json > out.json + +curl -X 'GET' 'https://cwms.sec.usace.army.mil/cwms-data/timeseries?name=KEYO.Flow-Out.Ave.1Day.1Day.Raw-USBR&office=NWDM&begin=PT-48h' -H 'accept: application/json;version=2' | python -m json.tool > out.json diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/gov/__init__.py b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/gov/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/gov/noaa/__init__.py b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/gov/noaa/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/gov/noaa/nwc/__init__.py b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/gov/noaa/nwc/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/gov/noaa/nwc/cwmstools/CWMSDownloader.py b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/gov/noaa/nwc/cwmstools/CWMSDownloader.py new file mode 100644 index 0000000..30a3029 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/gov/noaa/nwc/cwmstools/CWMSDownloader.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python + +import ssl +import urllib.request + +import json + +import datetime + + +class CWMSDownloader: + #def __init__(self, url="http://cwms-data.usace.army.mil/cwms-data/timeseries?"): + #def __init__(self, url="https://water.usace.army.mil/cwms-data/timeseries?"): + #def __init__(self, url="http://water.usace.army.mil/cwms-data/timeseries?"): + def __init__(self, url="http://cwms.sec.usace.army.mil/cwms-data/timeseries?"): + """Constructor for CWMS Downloader class""" + self.service_url = url + + def get_data(self, office, name, begin, data_format): + """ Download data for request office and station name with + data range determined by begin and file format controled by data_format. """ + + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + # build the request string out of parameters + request_str = self.service_url\ + + "office="+office \ + + "&name="+name\ + + "&begin="+begin\ + + "&format="+data_format + + request_str = urllib.parse.quote(request_str, ":/?&=") + + print( datetime.datetime.now(), end = " --- " ) + print(request_str) + + # get the response from the web service + resp = urllib.request.urlopen(request_str, context=ctx) + + # print(resp) + + if data_format == "json": + # extract the response data as a byte array + resp_data = resp.read() + + # print(type(resp_data)) + # print(resp_data) + + # convert the byte array to string + json_resp = resp_data.decode('utf8') + + # print(type(json_resp)) + # print(json_resp) + + json_data = json.loads(json_resp) + + # s = json.dumps(json_data, indent=4, sort_keys=True) + # print(s) + + return json_data + else: + # extract the response data as a byte array + resp_data = resp.read() + + # print(type(resp_data)) + # print(resp_data) + + # convert the byte array to string + txt_resp = resp_data.decode('utf8') + + return txt_resp + + + def get_data_range(self, office, name, begin, end, data_format): + """ Download data for request office and station name with + data range determined by begin and file format controled by data_format. """ + + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + # build the request string out of parameters + request_str = self.service_url\ + + "office="+office \ + + "&name="+name\ + + "&begin="+begin\ + + "&end="+end\ + + "&format="+data_format + + request_str = urllib.parse.quote(request_str, ":/?&=") + + print( datetime.datetime.now(), end = " --- " ) + print(request_str) + + # get the response from the web service + resp = urllib.request.urlopen(request_str, context=ctx) + + # print(resp) + + if data_format == "json": + # extract the response data as a byte array + resp_data = resp.read() + + # print(type(resp_data)) + # print(resp_data) + + # convert the byte array to string + json_resp = resp_data.decode('utf8') + + # print(type(json_resp)) + # print(json_resp) + + json_data = json.loads(json_resp) + + # s = json.dumps(json_data, indent=4, sort_keys=True) + # print(s) + + return json_data + else: + # extract the response data as a byte array + resp_data = resp.read() + + # print(type(resp_data)) + # print(resp_data) + + # convert the byte array to string + txt_resp = resp_data.decode('utf8') + + return txt_resp diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/gov/noaa/nwc/cwmstools/__init__.py b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/gov/noaa/nwc/cwmstools/__init__.py new file mode 100644 index 0000000..f11dd9d --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/gov/noaa/nwc/cwmstools/__init__.py @@ -0,0 +1 @@ +from gov.noaa.nwc.cwmstools.CWMSDownloader import CWMSDownloader \ No newline at end of file diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/run_usace.sh b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/run_usace.sh new file mode 100644 index 0000000..fb9d7a5 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/run_usace.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +############################################################################### +# File name: run_usace.sh # +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: # +# # +# Last modification date: 05/2022 # +# # +# # +# Description: Run the US SACE Stream Flow scripts # +# Supported by the NWS Water Predication Center # +# # +# 12/18/2024 OWP Download json format datafiles instead of xml # +# # +############################################################################### + +#module unload python +#module load python +#module list + +#log=$DBNROOT/user/usgs_download/parallel_download_master.log +#log=/gpfs/hps3/ptmp/Zhengtao.Cui/usace_download.log +#OUTDIR=/gpfs/hps3/ptmp/Zhengtao.Cui/ace_json_test + +SITE_FILE=./site-file.csv +OUTDIR=$DCOMROOT/usace_streamflow + +log=$DBNROOT/log/usace_download.log.`date -u +%Y%m%d` +touchfiles=("$OUTDIR") + +cd $DBNROOT/user/usace_download + +if [ ! -e ${OUTDIR} ]; then mkdir -p ${OUTDIR}; fi + +USACE_DOWNLOAD_MASTER=$DBNROOT/user/usace_download/CWMS_download_current.py + +DATE=`/bin/date +%H:%M` + +RESETLOGAT="05:28" + +if [ "$DATE" = "$RESETLOGAT" ] +then + echo "Message: reset ${USACE_DOWNLOAD_MASTER} log file" + rm $log +fi + +# Check if process is already running with this package +if pgrep -f CWMS_download_current.py > /dev/null 2>&1 +then + echo "Message: ${USACE_DOWNLOAD_MASTER} package is running" + for f in ${touchfiles[@]} + do + timediff=$((`date +%s` - `stat -c "%Y" $f`)) + if [ $timediff -gt $(( 3600 * 2 )) ] #three hours + then + echo "Message: touch file $f is older than 3 hours" + echo "Message: restart CWMS_download_current.py" + kill -9 `pgrep -f CWMS_download_current.py` + nohup python3 ${USACE_DOWNLOAD_MASTER} -f json ${SITE_FILE} $OUTDIR >> $log 2>&1 & + echo "Message: done restart USACE CWMS_download_current.py" + break + fi + + done + +else + echo "Message: ${USACE_DOWNLOAD_MASTER} package NOT running" + nohup python3 $USACE_DOWNLOAD_MASTER -f json ${SITE_FILE} ${OUTDIR} >> $log 2>&1 & + echo "Message: ${USACE_DOWNLOAD_MASTER} package started" +fi + +## Monitoring adjusted to check for non-zero byte files 09/07/2021 +f=$(ls -ltr $OUTDIR | tail -1 | awk '{print $9}') +if [ -s $OUTDIR/${f} ]; +then + #file is larger than 0-byte + most_recent=`ls -ltr $OUTDIR | tail -1 | awk '{print $6" "$7" "$8}'` + touch -d "$most_recent" $DBNROOT/tmp/usace_streamflow +else + echo "Files are empty. Check to see if the website is down." >> $log +fi + + +## +exit diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/run_usace_DevTest.sh b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/run_usace_DevTest.sh new file mode 100755 index 0000000..c910b7b --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/run_usace_DevTest.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +############################################################################### +# File name: run_usace.sh # +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: # +# # +# Last modification date: 2/11/2020 # +# # +# # +# Description: Run the US SACE Stream Flow scripts # +# Supported by the NWS Water Predication Center # +# # +# OWP 12/12/2024 Change download xml to json data format. # +# # +############################################################################### + +module unload python +module load python +module list + +#log=$DBNROOT/user/usace_streamflow/parallel_download_master.log +log=/lfs/h1/owp/ptmp/$LOGNAME/usace_streamflow/usace_download.log + +SITE_FILE=./site-file.csv + +#OUTDIR=$DCOMROOT/${envir}/usace_streamflow +OUTDIR=/lfs/h1/owp/ptmp/$LOGNAME/usace_streamflow + +if [ ! -e ${OUTDIR} ]; then mkdir -p ${OUTDIR}; fi + +USACE_DOWNLOAD_MASTER=./CWMS_download_current.py + +DATE=`/bin/date +%H:%M` + +RESETLOGAT="05:28" + +if [ "$DATE" = "$RESETLOGAT" ] +then + echo "Message: reset ${USACE_DOWNLOAD_MASTER} log file" + rm $log +fi + +# Check if process is already running with this package +if pgrep -f ${USACE_DOWNLOAD_MASTER} > /dev/null 2>&1 +then + echo "Message: ${USACE_DOWNLOAD_MASTER} package is running" + +else + echo "Message: ${USACE_DOWNLOAD_MASTER} package NOT running" + nohup python $USACE_DOWNLOAD_MASTER -f json ${SITE_FILE} ${OUTDIR} >> $log 2>&1 & + echo "Message: ${USACE_DOWNLOAD_MASTER} package started" +fi +exit diff --git a/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/site-file.csv b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/site-file.csv new file mode 100644 index 0000000..55959a1 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/ace_download/stream_flow_download/site-file.csv @@ -0,0 +1,72 @@ +usace_gage_id,office,gage,gagedFlowl,WaterbodyID,lakeLink,RFC,NWS_LID +UT10131,SPK,Rockport.Flow-Res Out.Ave.~1Day.1Day.Raw-USBRSLC;MANUAL,10093168,10091800,10093168,CBRFC,WBWU1 +CA10186,SPK,SAC SHA-Shasta Dam-Sacramento R.Flow-Res Out.Ave.~1Day.1Day.Calc-usbr,2782699,2781987,7966205,CNRFC, +TX00020,MVK,Jferson_BigC_Bay.Flow.Inst.15Minutes.0.DCP-rev,1011780,1010832,1011780,LMRFC,JFET2 +MO30202,SWL,Table_Rock_Dam-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,7625336,120053569,8586112,LMRFC,FORM7 +AR00174,SWL,Beaver_Dam-Tailwater.Flow.Inst.1Hour.0.CCP-Comp,8589764,167299813,8589608,LMRFC,BVGA4 +KY03001,LRN,BARK2-BARKLEY.Flow.Ave.1Hour.1Hour.man-rev,11878940,166899385,11878948,LMRFC,BAHK2 +KY05017,LRN,KYDK2-KENTUCKY.Flow.Ave.1Hour.1Hour.tva-raw,10575371,120052349,10575371,LMRFC,KYDK2 +NY01055,NAB,Whitney Point.Flow-Reg.Inst.1Hour.0.computed outflow OLD,9423843,9422503,9424107,MARFC,WITN6 +CO01281,NWDM,CHFI.Flow-Out.Inst.1Hour.0.Best-NWO,188657,188031,188743,MBRFC,CFDC2 +KS00024,NWDM,NORN.Flow-Out.Ave.1Day.1Day.Best-NWK,8367756,167266511,8367756,MBRFC,NTDK1 +KS00019,NWDM,CEBL.Flow-Out.Ave.1Day.1Day.Best-NWK,18951675,18950909,18951675,MBRFC,ELSK1 +KS00025,NWDM,WEBR.Flow-Out.Ave.1Day.1Day.Best-NWK,5954109,5943595,5945295,MBRFC,STCK1 +KS00022,NWDM,KIRN.Flow-Out.Ave.1Day.1Day.Best-NWK,7347909,7344817,7347909,MBRFC,KRWK1 +KS00021,NWDM,GLEL.Flow-Out.Ave.1Day.1Day.Best-NWK,3537261,120052972,3537261,MBRFC,GLNK1 +NE01057,NWDM,SC14.Flow-Out.Ave.~1Day.1Day.Best-NWO,17405923,17404975,17405923,MBRFC,ERLN1 +KS00023,NWDM,LOVL.Flow-Out.Ave.1Day.1Day.Best-NWK,19017185,19016317,19017757,MBRFC,LOVK1 +NE01048,NWDM,LAMC.Flow-Out.Ave.~1Day.1Day.Raw-CNPPID,17462840,167794979,17462840,MBRFC,KNGN1 +NE01063,NWDM,SC18.Flow-Out.Ave.~1Day.1Day.Best-NWO,17405437,17404931,17405909,MBRFC,RAYN1 +WY01295,NWDM,PATR.Flow-Out.Ave.1Day.1Day.Raw-USBR,15976270,15975492,15976270,MBRFC,PTDW4 +WY01297,NWDM,SEMR.Flow-Out.Ave.1Day.1Day.Raw-USBR,15976386,167245825,15976386,MBRFC,KORW4 +WY01290,NWDM,ALCR.Flow-Out.Ave.1Day.1Day.Raw-USBR,22106341,22103649,22107769,MBRFC, +WY01378,NWDM,BULA.Flow-Out.Ave.1Day.1Day.Raw-USBR,12901134,12898716,12901134,MBRFC,BLRW4 +WY01299,NWDM,BOYN.Flow-Out.Ave.1Day.1Day.Raw-USBR,12870023,12869679,12871081,MBRFC,SBDW4 +SD01092,NWDM,BEND.Flow-Out.Ave.~1Day.1Day.Best-MRBWM,11546856,11546100,11546856,MBRFC,BBDS2 +SD01093,NWDM,FTRA.Flow-Out.Ave.~1Day.1Day.Best-MRBWM,11550820,120053557,11550820,MBRFC,FRDS2 +WY01300,NWDM,BUBI.Flow-Out.Ave.1Day.1Day.Raw-USBR,4425887,12793741,12794325,MBRFC,CDYW4 +MT00569,NWDM,CLCA.Flow-Out.Ave.1Day.1Day.Raw-USBR,4165978,4167440,4170578,MBRFC,CLKM8 +MT00576,NWDM,YETL.Flow-Out.Ave.1Day.1Day.Raw-USBR,12801362,167204997,12801364,MBRFC,BHRM8 +SD01141,NWDM,SHHI.Flow-Out.Ave.~1Day.1Day.Best-NWO,16188778,21861936,16188778,MBRFC,SHAS2 +ND00151,NWDM,JATO.Flow-Out.Ave.~1Day.1Day.Best-NWO,12577680,120053643,12577680,MBRFC,JMDN8 +MT00568,NWDM,CAFE.Flow-Out.Ave.1Day.1Day.Raw-USBR,3024978,167204871,3024978,MBRFC,CYNM8 +MT00559,NWDM,HOLM.Flow.Inst.1Hour.0.Best-NWDM,3021998,120051950,3022000,MBRFC,HTRM8 +MT00560,NWDM,HSMT.Flow.Inst.1Hour.0.Best-NWDM,3024846,3022162,3024850,MBRFC,HASM8 +MT00025,NWDM,FTPK.Flow-Out.Ave.~1Day.1Day.Best-MRBWM,12461756,167204901,940040008,MBRFC,FPKM8 +IL00117,MVS,Rend Lk-Big Muddy.Flow-Out.Ave.~1Day.1Day.lakerep-rev,13779254,167156996,13779550,NCRFC,RNDI2 +IL00113,MVS,Carlyle Lk-Kaskaskia.Flow-Out.Ave.~1Day.1Day.lakerep-rev,13871378,167122265,13876230,NCRFC,CAYI2 +MO82201,MVS,Mark Twain Lk-Salt.Flow-Out.Ave.~1Day.1Day.lakerep-rev,4867723,937110111,4867727,NCRFC,CDAM7 +IL00118,MVS,Lk Shelbyville-Kaskaskia.Flow-Out.Ave.~1Day.1Day.lakerep-rev,13772418,167122256,13772400,NCRFC,SBYI2 +MN00594,MVP,LockDam_02.Flow.Inst.15Minutes.0.comp,1101436,120051919,1101436,NCRFC,HSTM5 +MN00581,MVP,Highway75_Dam-ServiceSpillway.Flow-Out.Inst.~1Day.0.CEMVP-Legacy,4086200,4083202,4085966,NCRFC,ODEM5 +MN00582,MVP,MissHW_PineRiver.Flow-Out.Inst.~1Day.0.CEMVP-Legacy,4622350,4620258,4622350,NCRFC,CRLM5 +MN00574,MVP,Orwell_Dam.Flow-Out.Inst.15Minutes.0.rev,909020763,6657709,909020763,NCRFC,ORWM5 +MN00586,MVP,MissHW_Winni.Flow-Out.Inst.~1Day.0.CEMVP-Legacy,22328037,167122141,22328037,NCRFC,WNBM5 +MN00585,MVP,MissHW_Leech.Flow-Out.Inst.~1Day.0.CEMVP-Legacy,4819627,4817675,4819627,NCRFC,FEDM5 +PortHuron,LRE,PortHuron.Flow.Inst.0.0.shef-raw,13196034,4800004,13196034,NCRFC, +CT00506,NAE,CRD.Flow.Ave.15Minutes.6Hours.DCP-rev,6107093,6106841,6107211,NERFC, +NH00003,NAE,FFD.Flow.Inst.15Minutes.0.DCP-rev,166174264,166174267,166174264,NERFC, +OR00015,NWDP,CGR.Flow-Out.Inst.0.0.MIXED-COMPUTED-REV,23773011,23777431,23773011,NWRFC,CGRO3 +ID00287,NWDP,DWR.Flow-Out.Ave.~6Hours.6Hours.CBT-COMPUTED-REV,23630892,23634782,23630892,NWRFC,DWRI1 +TN04102,LRN,CETT1-CENTER_HILL.Flow.Ave.1Hour.1Hour.man-rev,18434275,167122309,18434275,OHRFC, +TN03722,LRN,OHIT1-OLD_HICKORY.Flow.Ave.1Hour.1Hour.man-rev,18415989,18415569,18416017,OHRFC,OHGT1 +TN03701,LRN,JPPT1-J_PERCY_PRIEST.Flow.Ave.1Hour.1Hour.man-rev,18401827,120052688,18401827,OHRFC,JPPT1 +KY03029,LRH,Dewey-Outflow.Flow.Inst.1Hour.0.OBS,886789,885333,886789,OHRFC,DWYK2 +KY03028,LRH,Fishtrap-Outflow.Flow.Inst.1Hour.0.OBS,1089105,1086587,1089105,OHRFC,FTLK2 +WV08902,LRH,Bluestone-Outflow.Flow.Inst.1Hour.0.OBS,6906513,6906421,6906513,OHRFC,BLLW2 +WV06702,LRH,Summersville-Outflow.Flow.Inst.1Hour.0.OBS,4548090,4547794,4548090,OHRFC,SUMW2 +WV09901,LRH,EastLynn-Outflow.Flow.Inst.1Hour.0.OBS,3964916,3964366,3964916,OHRFC,ELSW2 +KY03030,LRH,Grayson-Outflow.Flow.Inst.1Hour.0.OBS,1936384,120052749,1936720,OHRFC,GRAK2 +OH00017,LRH,PaintCr-Outflow.Flow.Inst.1Hour.0.OBS,5232686,5231436,5233352,OHRFC,BBRO1 +WV00707,LRH,Burnsville-Outflow.Flow.Inst.1Hour.0.OBS,19417491,76669453,19417491,OHRFC,BRNW2 +OH00010,LRH,Tappan-Lake.Flow.Inst.1Hour.0.OBS,19392734,19390686,19392734,OHRFC,TAPO1 +OH00002,LRH,WillsCr-Outflow.Flow.Inst.1Hour.0.OBS,15371872,15370770,15371872,OHRFC,WCLO1 +OH00016,LRH,Mohawk-Outflow.Flow.Inst.1Hour.0.OBS,167484595,167484603,167484595,OHRFC,MHWO1 +OH00019,LRH,Mohicanville-Outflow.Flow.Inst.1Hour.0.OBS,167484497,167484552,167484498,OHRFC,MCHO1 +OH00020,LRH,CharlesMill-Outflow.Flow.Inst.1Hour.0.OBS,15410097,15408897,15411519,OHRFC,CHAO1 +LA00181,MVK,Caddo Lake.Flow.Inst.1Hour.0.DCP-rev,1016769,167300381,1016769,LMRFC,LCOL1 +PA01939,LRP,LakeLynn-Outflow.Flow.Inst.1Hour.0.OBS,3773975,3773141,3773977,OHRFC,PMAW2 +ND00146,NWDM,PIST.Flow-Out.Ave.~1Day.1Day.Best-NWO,11505172,120053541,11505172,MBRFC,JAMN8 +WA00170,NWDP,DIA.Flow-Out.Ave.~1Day.1Day.CBT-REV,24255201,24260717,24255205,NWRFC,DIAW1 +WY01380,NWDM,KEYO.Flow-Out.Ave.1Day.1Day.Raw-USBR,10906553,10904095,10906421,MBRFC,KEYW4 +OR00001,NWDP,BON.Flow-Out.Ave.1Hour.1Hour.CBT-REV,???,???,???,NWRFC,BONO3 diff --git a/data_assimilation_engine/Streamflow_Scripts/environment.yml b/data_assimilation_engine/Streamflow_Scripts/environment.yml new file mode 100644 index 0000000..7d73f8f --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/environment.yml @@ -0,0 +1,9 @@ +name: NextGen_Streamflow_Scripts +channels: + - defaults + - anaconda +dependencies: + - python~=3.10.0 + - netCDF4 + - python-dateutil + - pytz \ No newline at end of file diff --git a/data_assimilation_engine/Streamflow_Scripts/nco_canada/streamflow_download/README.TXT b/data_assimilation_engine/Streamflow_Scripts/nco_canada/streamflow_download/README.TXT new file mode 100755 index 0000000..4aa2087 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/nco_canada/streamflow_download/README.TXT @@ -0,0 +1,51 @@ +# Overview +This directory contains shell scripts to download Environment Canada streamflow observations in realtime from https://wateroffice.ec.gc.ca/mainmenu/real_time_data_index_e.html. There are two scripts. They are run in twp steps. First, run the `get_station_list.sh` to get a list of station IDs. Than run the `get_canadian_streamgauge.sh` script to download observed values for each of the stations in the list. + +Three types of variables are downloaded, 5(??), 46(water level), and 47(discharge). We use only discharge here. + +# Prerequisite + +A DBNROOT directory for storing log files, temporary files and the station list file. + +A DCOMROOT directory for storing the raw data files. + +# Script description +`get_station_list.sh`: This script downloads all station IDs that have observed data for variables 5, 46 and 47. The `stations_all.txt` file in the current directory that contains a list of station IDs is created from this script. +`get_canadian_streamgauge.sh`: This script read the `stations_all.txt` and download observed values fro variables 5, 46, and 47. Save the files in `${YYYYMMDD}/can_streamgauge` under the given `$DBNROOT`. + +# Script Usage + +## Setup environmental variables +Set the DBNROOT and DCOMROOT environmental variables +For example, + +``` +export DCOMROOT=/path/to/dcomroot +export DBNROOT=/path/to/dbnroot +``` +## Create directories +``` +mkdir -p $DCOMROOT/ +mkdir -p $DBNROOT/user/can_streamgauge +mkdir -p $DBNROOT/log +mkdir -p $DBNROOT/tmp +``` + +## Fetch station IDs + +``` +cd $DBNROOT/user/can_streamgauge +source /path/to/get_station_list.sh +``` +Output file, stations_all.txt, is saved in the current directory, $DBNROOT/user/can_streamgauge + +The log file is saved to the directory at `$DBNROOT/log`. + +## Download the realtime data + +``` +cd $DBNROOT/user/can_streamgauge +source /path/to/get_canadian_streamgauge.sh +``` +The output files files are saved in `$DCOMROOT/${YYYYMMDD}/can_streamgauge` where `${YYYYMMDD}` is current date. + diff --git a/data_assimilation_engine/Streamflow_Scripts/nco_canada/streamflow_download/get_canadian_streamgauge.sh b/data_assimilation_engine/Streamflow_Scripts/nco_canada/streamflow_download/get_canadian_streamgauge.sh new file mode 100644 index 0000000..046a727 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/nco_canada/streamflow_download/get_canadian_streamgauge.sh @@ -0,0 +1,69 @@ +#!/bin/sh + +# +# Script to pull Canadian streamgauge data using list of stations stored locally +# +##---check if process is already running -------------------- +#cb_proc="get_canadian_streamgauge" +#if ps -ef | grep "get_canadian_streamgauge.sh" | grep -v grep > /dev/null +#then +# echo "[$(date)] : $cb_proc process already running" +# exit 1 +#fi +##------------------------------------------------------------ + +YYYYMMDD=`date -u +%Y%m%d` +#DATADIR="/lfs/h1/ops/prod/dcom/${YYYYMMDD}/can_streamgauge" +DATADIR="$DCOMROOT/${YYYYMMDD}/can_streamgauge" + +#LDIR="/dfprod/dbnet/user/can_streamgauge" +LDIR="$DBNROOT/user/can_streamgauge" +LOG=$DBNROOT/log/can_streamgauge.log.${YYYYMMDD} + +DD1=$(date -u +%Y-%m-%d) +DD2=$(date --date '2 days ago' +%Y-%m-%d) + +url_start="https://wateroffice.ec.gc.ca/services/real_time_data/csv/inline?stations[]=" +url_end="¶meters[]=5¶meters[]=46¶meters[]=47&start_date="${DD2}"%2000:00:00&end_date="${DD1}"%2023:59:59" + +# DECODER="/lfs/h1/ops/prod/decoders/decod_dcwcan/scripts/copy_and_run_dcwcan.sh" + +if [ ! -d ${DATADIR} ]; then + mkdir -p -m 775 ${DATADIR} +fi + +## Data pull +######################################## + +stations_file="stations_all.txt" +nstations=$(cat ${LDIR}/${stations_file} | wc -l) + +for i in `seq $nstations` + do + station=$(cat ${LDIR}/${stations_file} | head -${i} | tail -1) + curl_str=$(echo ${url_start}${station}${url_end}) + local_filename=${station}"_hydrometric.csv" + + output=`curl -w "%{size_download}\n" -s -o ${DATADIR}/${local_filename} ${curl_str}` + # options: w=output file size, s=silent/no output to terminal, o=specify local file name + # removed option: R=use upstream file timestamp. The same as local since URL-generated file + + if [ "${output}" -gt 0 ]; then + echo 'Successful pull, station ' ${station} 'at ' $(date -u +%Y%m%d-%H:%M:%S) >> ${LOG} + touch $DBNROOT/tmp/can_streamgauge + decoder=1 + else + echo 'Not pulled, station ' ${station} 'at ' $(date -u +%Y%m%d-%H:%M:%S) >> ${LOG} + fi + +done + +### Run decoder +######################################### +# +## Run the decoder if a new file was pulled +#if [ "${decoder}" == 1 ]; then +# ${DECODER} "${DATADIR}" +#fi + +echo 'Pull complete' diff --git a/data_assimilation_engine/Streamflow_Scripts/nco_canada/streamflow_download/get_station_list.sh b/data_assimilation_engine/Streamflow_Scripts/nco_canada/streamflow_download/get_station_list.sh new file mode 100644 index 0000000..e64c5d7 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/nco_canada/streamflow_download/get_station_list.sh @@ -0,0 +1,20 @@ +#!/usr/bin/sh + +YYYYMMDD=`date -u +%Y%m%d` +#LDIR="/dfprod/dbnet/user/can_streamgauge" +LDIR="$DBNROOT/user/can_streamgauge" + +for varnum in 5 46 47; do + local_tmpname="station_list_tmp_var${varnum}.txt" + local_filename="stations_all_var${varnum}.txt" + curl -s -o ${LDIR}/${local_tmpname} "https://wateroffice.ec.gc.ca/services/recent_real_time_data/csv/inline?parameters[]=${varnum}" + cat ${LDIR}/${local_tmpname} | grep ${varnum} | awk -F "," '{print $1}' > ${LDIR}/${local_filename} +done +echo 'Pull complete' + +cat ${LDIR}/stations_all_var*.txt | sort -u > stations_all.txt + +echo 'Station extraction complete' + +rm -v ${LDIR}/station_list_tmp_var*.txt +rm -v ${LDIR}/stations_all_var*.txt diff --git a/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/EmptyDirOrFileException.py b/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/EmptyDirOrFileException.py new file mode 100644 index 0000000..82f182b --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/EmptyDirOrFileException.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python +class EmptyDirOrFileException( Exception ): + """Exception raised for empty input files or directories. + Attributes: + expression -- input expression in which the error occurred + message -- explanation of the error + """ + pass + +# def __init__(self, message): +# self.expression = expression +# self.message = message diff --git a/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/NWM_canadian_gage_inputs.txt b/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/NWM_canadian_gage_inputs.txt new file mode 100644 index 0000000..f466ffa --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/NWM_canadian_gage_inputs.txt @@ -0,0 +1,349 @@ +OBJECTID,ID,Name___Nom,Latitude,Longitude,Prov_Terr,Timezone___Fuseau_horaire,HydroID2,NEAR_FID,NEAR_DIST +1,02AB006,KAMINISTIQUIA RIVER AT KAMINISTIQUIA,48.532780000000002,-89.594170000000005,ON,UTC-05:00,41041618,41624,0.000000000464077 +2,02AB008,NEEBING RIVER NEAR THUNDER BAY,48.382219999999997,-89.307779999999994,ON,UTC-05:00,41042491,42497,0.000000000244068 +3,02AB014,NORTH CURRENT RIVER NEAR THUNDER BAY,48.501109999999997,-89.179730000000006,ON,UTC-05:00,41041874,41880,0.000000003319852 +4,02AB017,WHITEFISH RIVER AT NOLALU,48.291670000000003,-89.809169999999995,ON,UTC-05:00,41042987,42993,0.000000000116570 +5,02AB019,MCVICAR CREEK AT THUNDER BAY,48.449440000000003,-89.219170000000005,ON,UTC-05:00,41042193,42199,0.000000000066013 +6,02AB020,MCINTYRE RIVER ABOVE THUNDER BAY,48.482500000000002,-89.325280000000006,ON,UTC-05:00,41041994,42000,0.000000000435397 +7,02AB021,CURRENT RIVER AT STEPSTONE,48.562500000000000,-89.240840000000006,ON,UTC-05:00,41041500,41506,0.000000000161894 +8,02AB022,CORBETT CREEK NEAR MURILLO,48.429160000000003,-89.545000000000002,ON,UTC-05:00,41042212,42218,0.000000000175360 +9,02AB023,SLATE RIVER NEAR THUNDER BAY,48.324750000000002,-89.404859999999999,ON,UTC-05:00,41042800,42806,0.000000000365750 +10,02AB024,NEEBING RIVER NEAR INTOLA,48.443559999999998,-89.358559999999997,ON,UTC-05:00,41042197,42203,0.000000002545407 +11,02AB025,KAMINISTIQUIA RIVER AT WEST FORT WILLIAM,48.347861999999999,-89.353279000000001,ON,UTC-05:00,41042748,42754,0.000000000698628 +12,02AB026,KAMINISTIQUIA RIVER ABOVE WEST FORT WILLIAM,48.345531000000001,-89.386780000000002,ON,UTC-05:00,41042715,42721,0.000000000237559 +13,02AC001,WOLF RIVER AT HIGHWAY NO. 17,48.821950000000001,-88.535280000000000,ON,UTC-05:00,41040015,40021,0.000000000529449 +14,02AC002,BLACK STURGEON RIVER AT HIGHWAY NO. 17,48.903610000000000,-88.377219999999994,ON,UTC-05:00,41039385,39391,0.000000000144285 +15,02AD010,BLACKWATER RIVER AT BEARDMORE,49.602780000000003,-87.963890000000006,ON,UTC-05:00,41034617,34623,0.000000000190115 +16,02AD012,NIPIGON RIVER BELOW ALEXANDER GENERATING STATION,49.123750000000001,-88.358277999999999,ON,UTC-05:00,41037694,37700,0.000000000116139 +17,02AE001,GRAVEL RIVER NEAR CAVERS,48.926000000000002,-87.690190000000001,ON,UTC-05:00,41039416,39422,0.000000002921137 +18,02BA003,LITTLE PIC RIVER NEAR COLDWELL,48.849170000000001,-86.607219999999998,ON,UTC-05:00,41040077,40083,0.000000000221626 +19,02BA005,WHITESAND RIVER ABOVE SCHREIBER AT MINOVA MINE,48.978110000000001,-87.376779999999997,ON,UTC-05:00,41039144,39150,0.000000002339008 +20,02BA006,STEEL RIVER BELOW SANTOY LAKE,48.814450000000001,-86.859440000000006,ON,UTC-05:00,41040589,40595,0.000000000187670 +21,02BB003,PIC RIVER NEAR MARATHON,48.773890000000002,-86.296940000000006,ON,UTC-05:00,41040675,40681,0.000000004859485 +22,02BB004,CEDAR CREEK NEAR HEMLO,48.707500000000003,-85.910839999999993,ON,UTC-05:00,41041144,41150,0.000000000391943 +23,02BC004,WHITE RIVER BELOW WHITE LAKE,48.655279999999998,-85.742230000000006,ON,UTC-05:00,41041470,41476,0.000000004898968 +24,02BC006,PUKASKWA RIVER BELOW FOX RIVER,48.160280000000000,-85.730528000000007,ON,UTC-05:00,41044019,44025,0.000000000463279 +27,02BD006,WAWA CREEK AT WAWA,47.989440000000002,-84.767830000000004,ON,UTC-05:00,41044694,44700,0.000000000301728 +28,02BD007,MAGPIE RIVER NEAR WAWA,48.021999999999998,-84.813170000000000,ON,UTC-05:00,41044540,44546,0.000000000427911 +29,02BF001,BATCHAWANA RIVER NEAR BATCHAWANA,47.003610000000002,-84.515559999999994,ON,UTC-05:00,41047018,47024,0.000000001993411 +30,02BF002,GOULAIS RIVER NEAR SEARCHMONT,46.860830000000000,-83.971940000000004,ON,UTC-05:00,41047141,47147,0.000000000274985 +31,02BF004,BIG CARP RIVER NEAR SAULT STE. MARIE,46.515830000000001,-84.465000000000003,ON,UTC-05:00,41047344,47350,0.000000000134143 +32,02BF014,GOULAIS RIVER NEAR KIRBY'S CORNER,46.717469999999999,-84.284610000000001,ON,UTC-05:00,41047281,47287,0.000000000627557 +33,02CA002,ROOT RIVER AT SAULT STE. MARIE,46.562779999999997,-84.281940000000006,ON,UTC-05:00,41011500,11505,0.000000000257833 +35,02CA007,THESSALON RIVER NEAR POPLAR DALE,46.530389999999997,-83.709580000000003,ON,UTC-05:00,41011995,12000,0.000000000059924 +36,02CB003,AUBINADONG RIVER ABOVE SESABIC CREEK,46.968330000000002,-83.416939999999997,ON,UTC-05:00,41008367,8372,0.000000000079675 +37,02CC005,LITTLE WHITE RIVER NEAR BELLINGHAM,46.391669999999998,-83.281670000000005,ON,UTC-05:00,41012842,12847,0.000000002817825 +38,02CC010,LITTLE WHITE RIVER BELOW BOLAND RIVER,46.581110000000002,-82.967780000000005,ON,UTC-05:00,41011377,11382,0.000000012049102 +39,02CD001,SERPENT RIVER AT HIGHWAY NO. 17,46.210780000000000,-82.512420000000006,ON,UTC-05:00,41014103,14108,0.000000000158543 +40,02CD006,SERPENT RIVER ABOVE QUIRKE LAKE,46.511110000000002,-82.600830000000002,ON,UTC-05:00,41011866,11871,0.000000002032239 +41,02CE002,AUX SABLES RIVER AT MASSEY,46.215000000000003,-82.070830000000001,ON,UTC-05:00,41013989,13994,0.000000000144179 +42,02CE007,MINISTIC CREEK ABOVE AGNEW LAKE,46.440420000000003,-81.683220000000006,ON,UTC-05:00,41012304,12309,0.000000000455458 +43,02CF005,JUNCTION CREEK AT SUDBURY,46.478720000000003,-81.010400000000004,ON,UTC-05:00,41011751,11756,0.000000000543519 +44,02CF007,WHITSON RIVER AT CHELMSFORD,46.583060000000003,-81.199169999999995,ON,UTC-05:00,41010809,10814,0.000000000200703 +45,02CF008,WHITSON RIVER AT VAL CARON,46.610280000000003,-81.033060000000006,ON,UTC-05:00,41010626,10631,0.000000000384343 +46,02CF010,ONAPING RIVER NEAR LEVACK,46.599449999999997,-81.381940000000000,ON,UTC-05:00,41010845,10850,0.000000000229032 +47,02CF011,VERMILLION RIVER NEAR VAL CARON,46.685440000000000,-81.009249999999994,ON,UTC-05:00,41009936,9941,0.000000002344198 +48,02CF012,JUNCTION CREEK BELOW KELLEY LAKE,46.427500000000002,-81.098339999999993,ON,UTC-05:00,41012273,12278,0.000000000245030 +49,02CF013,MOOSE CREEK AT LEVACK,46.635829999999999,-81.391390000000001,ON,UTC-05:00,41010391,10396,0.000000002334351 +50,02CF014,VERMILLION RIVER NEAR MILNET,46.819440000000000,-80.957139999999995,ON,UTC-05:00,41008966,8971,0.000000000612597 +51,02CG003,BLUE JAY CREEK NEAR TEHKUMMAH,45.651670000000003,-81.986599999999996,ON,UTC-05:00,41000446,449,0.000000000373549 +55,02DB005,WANAPITEI RIVER NEAR WANUP,46.345550000000003,-80.839449999999999,ON,UTC-05:00,41012718,12723,0.000000001685540 +56,02DB007,CONISTON CREEK ABOVE WANAPITEI RIVER,46.475279999999998,-80.821659999999994,ON,UTC-05:00,41011616,11621,0.000000000146239 +57,02DC004,STURGEON RIVER NEAR GLEN AFTON,46.637219999999999,-80.263339999999999,ON,UTC-05:00,41010118,10123,0.000000003634348 +58,02DC012,STURGEON RIVER AT UPPER GOOSE FALLS,46.971390000000000,-80.463329999999999,ON,UTC-05:00,41007693,7698,0.000000001677848 +59,02DC013,LITTLE STURGEON RIVER BELOW BOOTH LAKE,46.427610000000001,-79.662030000000001,ON,UTC-05:00,41011688,11693,0.000000000319030 +61,02DD010,FRENCH RIVER AT DRY PINE BAY,46.068610000000000,-80.611660000000001,ON,UTC-05:00,41014635,14640,0.000000000440461 +62,02DD012,VEUVE RIVER at VERNER,46.408250000000002,-80.123300000000000,ON,UTC-05:00,41011936,11941,0.000000000201140 +63,02DD013,LA VASE RIVER AT NORTH BAY,46.263330000000003,-79.394999999999996,ON,UTC-05:00,41012827,12832,0.000000000238242 +64,02DD014,CHIPPEWA CREEK AT NORTH BAY,46.311669999999999,-79.448329999999999,ON,UTC-05:00,41012564,12569,0.000000000473163 +65,02DD015,COMMANDA CREEK NEAR COMMANDA,45.949170000000002,-79.606669999999994,ON,UTC-05:00,41014966,14971,0.000000000798938 +66,02DD016,FRENCH RIVER AT PORTAGE DAM,46.122779999999999,-80.013610000000000,ON,UTC-05:00,41014186,14191,0.000000000068227 +67,02DD017,FRENCH RIVER AT CHAUDIERE DAM,46.125830000000001,-80.019440000000003,ON,UTC-05:00,41014052,14057,0.000000001182732 +68,02DD020,LITTLE FRENCH RIVER AT OKIKENDAWT ISLAND,46.123060000000002,-80.078890000000001,ON,UTC-05:00,41014136,14141,0.000000000662517 +72,02DD024,WASI RIVER NEAR ASTORVILLE,46.178359999999998,-79.309970000000007,ON,UTC-05:00,41013530,13535,0.000000000045821 +74,02DD026,FRENCH RIVER AT WOSELEY BAY,46.105390000000000,-80.265000000000001,ON,UTC-05:00,41014355,14360,0.000000000035305 +75,02EA005,NORTH MAGNETAWAN RIVER NEAR BURK'S FALLS,45.669449999999998,-79.379170000000002,ON,UTC-05:00,41015733,15738,0.000000000276833 +76,02EA010,NORTH MAGNETAWAN RIVER ABOVE PICKEREL LAKE,45.703890000000001,-79.309169999999995,ON,UTC-05:00,41015590,15595,0.000000000294416 +77,02EA011,MAGNETAWAN RIVER NEAR BRITT,45.773060000000001,-80.482219999999998,ON,UTC-05:00,41015533,15538,0.000000002726230 +80,02EA018,MAGNETAWAN RIVER NEAR EMSDALE,45.554720000000003,-79.287779999999998,ON,UTC-05:00,41016100,16105,0.000000001996498 +83,02EA021,SHAWANAGA RIVER BELOW SHAWANAGA LAKE,45.558639999999997,-80.038060000000002,ON,UTC-05:00,41016210,16215,0.000000000883371 +84,02EB004,NORTH BRANCH MUSKOKA RIVER AT PORT SYDNEY,45.212780000000002,-79.275279999999995,ON,UTC-05:00,41017106,17111,0.000000001501072 +85,02EB008,SOUTH BRANCH MUSKOKA RIVER AT BAYSVILLE,45.147970000000001,-79.113500000000002,ON,UTC-05:00,41017301,17306,0.000000000120619 +86,02EB011,MOON RIVER AT HIGHWAY NO. 400,45.064999999999998,-79.790279999999996,ON,UTC-05:00,41017521,17526,0.000000000008053 +87,02EB012,MUSQUASH RIVER AT HIGHWAY NO. 400,45.022500000000001,-79.776949999999999,ON,UTC-05:00,41017656,17661,0.000000002533334 +88,02EB013,EAST RIVER NEAR HUNTSVILLE,45.392780000000002,-79.159999999999997,ON,UTC-05:00,41016647,16652,0.000000001613079 +89,02EB014,OXTONGUE RIVER NEAR DWIGHT,45.311940000000000,-78.989440000000002,ON,UTC-05:00,41016825,16830,0.000000002184490 +94,02EB019,LAKE OF BAYS AT BAYSVILLE,45.148809999999997,-79.113280000000003,ON,UTC-05:00,41017301,17306,0.000000000021059 +99,02EC002,BLACK RIVER NEAR WASHAGO,44.713610000000003,-79.281670000000005,ON,UTC-05:00,41018267,18272,0.000000000993320 +100,02EC008,BLACK RIVER AT BALDWIN,44.260829999999999,-79.343890000000002,ON,UTC-05:00,41019808,19813,0.000000000331910 +101,02EC009,HOLLAND RIVER AT HOLLAND LANDING,44.094999999999999,-79.489440000000002,ON,UTC-05:00,41020566,20571,0.000000000315272 +102,02EC010,SCHOMBERG RIVER NEAR SCHOMBERG,44.012219999999999,-79.685550000000006,ON,UTC-05:00,41020959,20964,0.000000003109046 +103,02EC011,BEAVER RIVER NEAR BEAVERTON,44.397080000000003,-79.070830000000001,ON,UTC-05:00,41019190,19195,0.000000000260863 +104,02EC014,SEVERN RIVER ABOVE WASDELL FALLS,44.775280000000002,-79.297229999999999,ON,UTC-05:00,41018140,18145,0.000000000187255 +105,02EC018,PEFFERLAW BROOK NEAR UDORA,44.267499999999998,-79.194730000000007,ON,UTC-05:00,41019746,19751,0.000000000437222 +106,02EC019,BLACK RIVER NEAR VANKOUGHNET,44.992500000000000,-79.049549999999996,ON,UTC-05:00,41017672,17677,0.000000000644717 +107,02EC020,HAWKESTONE CREEK AT HAWKESTONE,44.496969999999997,-79.467969999999994,ON,UTC-05:00,41018944,18949,0.000000000148490 +108,02EC021,UXBRIDGE BROOK NEAR UXBRIDGE,44.137860000000003,-79.112690000000001,ON,UTC-05:00,41020298,20303,0.000000001601682 +109,02EC022,HEAD RIVER NEAR SEBRIGHT,44.725610000000003,-79.070440000000005,ON,UTC-05:00,41018223,18228,0.000000003215664 +110,02ED003,NOTTAWASAGA RIVER NEAR BAXTER,44.249720000000003,-79.821389999999994,ON,UTC-05:00,41019963,19968,0.000000000110689 +111,02ED007,COLDWATER RIVER AT COLDWATER,44.707220000000000,-79.643889999999999,ON,UTC-05:00,41018377,18382,0.000000001207254 +112,02ED013,WYE RIVER NEAR WYEVALE,44.649720000000002,-79.903890000000004,ON,UTC-05:00,41018522,18527,0.000000002030463 +113,02ED014,PINE RIVER NEAR EVERETT,44.200000000000003,-79.959999999999994,ON,UTC-05:00,41020254,20259,0.000000000993424 +114,02ED015,MAD RIVER BELOW AVENING,44.307499999999997,-80.071950000000001,ON,UTC-05:00,41019785,19790,0.000000000744992 +115,02ED017,HOGG CREEK NEAR VICTORIA HARBOUR,44.726109999999998,-79.778890000000004,ON,UTC-05:00,41018382,18387,0.000000000187431 +116,02ED024,NORTH RIVER AT THE FALLS,44.767780000000002,-79.578329999999994,ON,UTC-05:00,41018164,18169,0.000000000049554 +117,02ED026,NOTTAWASAGA RIVER AT HOCKLEY,44.024720000000002,-79.969999999999999,ON,UTC-05:00,41020964,20969,0.000000000418767 +118,02ED027,NOTTAWASAGA RIVER NEAR EDENVALE,44.484999999999999,-79.966110000000000,ON,UTC-05:00,41019022,19027,0.000000000835079 +119,02ED029,INNISFIL CREEK NEAR ALLISTON,44.131110000000000,-79.781109999999998,ON,UTC-05:00,41020502,20507,0.000000000501380 +120,02ED030,SILVER CREEK AT ORILLIA,44.647170000000003,-79.452330000000003,ON,UTC-05:00,41018524,18529,0.000000000238726 +121,02ED031,PRETTY RIVER AT COLLINGWOOD,44.498750000000001,-80.199920000000006,ON,UTC-05:00,41019048,19053,0.000000000393775 +122,02ED032,WILLOW CREEK NEAR MINESING,44.442500000000003,-79.799499999999995,ON,UTC-05:00,41019265,19270,0.000000000081755 +123,02ED100,BEETON CREEK NEAR TOTTENHAM,44.049169999999997,-79.803340000000006,ON,UTC-05:00,41020860,20865,0.000000000222821 +124,02ED101,NOTTAWASAGA RIVER NEAR ALLISTON,44.110550000000003,-79.890270000000001,ON,UTC-05:00,41020581,20586,0.000000000951324 +125,02ED102,BOYNE RIVER AT EARL ROWE PARK,44.152500000000003,-79.896670000000000,ON,UTC-05:00,41020417,20422,0.000000000010827 +126,02FA001,SAUBLE RIVER AT SAUBLE FALLS,44.677500000000002,-81.256110000000007,ON,UTC-05:00,41018632,18637,0.000000000351479 +127,02FA002,STOKES RIVER NEAR FERNDALE,45.036949999999997,-81.336389999999994,ON,UTC-05:00,41017758,17763,0.000000000177692 +128,02FA004,SAUBLE RIVER AT ALLENFORD,44.535559999999997,-81.177779999999998,ON,UTC-05:00,41019038,19043,0.000000003234945 +129,02FB007,SYDENHAM RIVER NEAR OWEN SOUND,44.522219999999997,-80.930269999999993,ON,UTC-05:00,41019129,19134,0.000000001080139 +130,02FB009,BEAVER RIVER NEAR CLARKSBURG,44.520000000000003,-80.467780000000005,ON,UTC-05:00,41019003,19008,0.000000000926358 +131,02FB010,BIGHEAD RIVER NEAR MEAFORD,44.570279999999997,-80.648610000000005,ON,UTC-05:00,41018824,18829,0.000000000455800 +132,02FB012,MILL CREEK NEAR RED WING,44.464359999999999,-80.485579999999999,ON,UTC-05:00,41019241,19246,0.000000000293727 +133,02FB013,BEAVER RIVER NEAR VANDELEUR,44.346139999999998,-80.539969999999997,ON,UTC-05:00,41019700,19705,0.000000000254117 +134,02FB014,BIGHEAD RIVER NEAR STRATHAVON,44.471780000000003,-80.776529999999994,ON,UTC-05:00,41019206,19211,0.000000000163264 +135,02FC001,SAUGEEN RIVER NEAR PORT ELGIN,44.456389999999999,-81.326390000000004,ON,UTC-05:00,41019417,19422,0.000000000822184 +136,02FC002,SAUGEEN RIVER NEAR WALKERTON,44.120559999999998,-81.115279999999998,ON,UTC-05:00,41020789,20794,0.000000000003378 +137,02FC011,CARRICK CREEK NEAR CARLSRUHE,44.113329999999998,-81.019170000000003,ON,UTC-05:00,41020796,20801,0.000000000525659 +138,02FC012,SOUTH SAUGEEN RIVER NEAR HANOVER,44.098610000000001,-80.984440000000006,ON,UTC-05:00,41020868,20873,0.000000000036577 +139,02FC015,TEESWATER RIVER NEAR PAISLEY,44.268329999999999,-81.269170000000003,ON,UTC-05:00,41020162,20167,0.000000003755682 +140,02FC016,SAUGEEN RIVER ABOVE DURHAM,44.185549999999999,-80.787499999999994,ON,UTC-05:00,41020449,20454,0.000000000819420 +141,02FC017,BEATTY SAUGEEN RIVER NEAR HOLSTEIN,44.090470000000003,-80.741940000000000,ON,UTC-05:00,41020951,20956,0.000000000176416 +142,02FC020,TEESWATER RIVER AT TEESWATER,44.000610000000002,-81.283810000000003,ON,UTC-05:00,41021215,21220,0.000000000409401 +143,02FC021,CAMP CREEK AT ALLAN PARK,44.161140000000003,-80.929310000000001,ON,UTC-05:00,41020573,20578,0.000000001064393 +144,02FD001,PINE RIVER AT LURGAN,44.094720000000002,-81.725830000000002,ON,UTC-05:00,41020978,20983,0.000000000263430 +145,02FD002,LUCKNOW RIVER AT LUCKNOW,43.962220000000002,-81.515280000000004,ON,UTC-05:00,41021310,21315,0.000000000559264 +146,02FD003,NORTH PENETANGORE RIVER AT KINCARDINE,44.172780000000003,-81.630549999999999,ON,UTC-05:00,41020624,20629,0.000000000449207 +147,02FE002,MAITLAND RIVER BELOW WINGHAM,43.886670000000002,-81.326390000000004,ON,UTC-05:00,41021430,21435,0.000000000035748 +148,02FE003,MIDDLE MAITLAND RIVER NEAR LISTOWEL,43.727220000000003,-80.972780000000000,ON,UTC-05:00,41021676,21681,0.000000000170365 +149,02FE005,MAITLAND RIVER ABOVE WINGHAM,43.914999999999999,-81.264439999999993,ON,UTC-05:00,41021419,21424,0.000000000042760 +150,02FE007,LITTLE MAITLAND RIVER AT BLUEVALE,43.854439999999997,-81.250559999999993,ON,UTC-05:00,41021503,21508,0.000000000619450 +151,02FE008,MIDDLE MAITLAND RIVER NEAR BELGRAVE,43.812779999999997,-81.306950000000001,ON,UTC-05:00,41021564,21569,0.000000000291568 +152,02FE009,SOUTH MAITLAND RIVER AT SUMMERHILL,43.684440000000002,-81.541110000000003,ON,UTC-05:00,41021778,21783,0.000000006007463 +153,02FE010,BOYLE DRAIN NEAR ATWOOD,43.676389999999998,-81.075000000000003,ON,UTC-05:00,41021756,21761,0.000000000158712 +154,02FE011,MAITLAND RIVER NEAR HARRISTON,43.903889999999997,-80.892499999999998,ON,UTC-05:00,41021382,21387,0.000000000626905 +155,02FE013,MIDDLE MAITLAND RIVER ABOVE ETHEL,43.718330000000002,-81.124730000000000,ON,UTC-05:00,41021719,21724,0.000000003394665 +156,02FE014,BLYTH BROOK BELOW BLYTH,43.760280000000002,-81.463329999999999,ON,UTC-05:00,41021665,21670,0.000000000224897 +157,02FE015,MAITLAND RIVER AT BENMILLER,43.717500000000001,-81.626109999999997,ON,UTC-05:00,41021744,21749,0.000000000171079 +158,02FE016,SOUTH MAITLAND RIVER AT ROXBORO,43.579250000000002,-81.401309999999995,ON,UTC-05:00,41021931,21936,0.000000004318024 +159,02FE017,LAKELET CREEK NEAR GORRIE,43.892919999999997,-81.063280000000006,ON,UTC-05:00,41021399,21404,0.000000001743068 +160,02FE018,BLIND CREEK NEAR FORDWICH,43.896000000000001,-81.037999999999997,ON,UTC-05:00,41021401,21406,0.000000002034856 +161,02FF002,AUSABLE RIVER NEAR SPRINGBANK,43.071950000000001,-81.659719999999993,ON,UTC-05:00,41022440,22445,0.000000000157897 +162,02FF004,SOUTH PARKHILL CREEK NEAR PARKHILL,43.160829999999997,-81.731939999999994,ON,UTC-05:00,41022335,22340,0.000000000079106 +163,02FF007,BAYFIELD RIVER NEAR VARNA,43.551389999999998,-81.589449999999999,ON,UTC-05:00,41021971,21976,0.000000000757140 +164,02FF008,PARKHILL CREEK ABOVE PARKHILL RESERVOIR,43.164169999999999,-81.631670000000000,ON,UTC-05:00,41022317,22322,0.000000001010805 +165,02FF009,AUSABLE RIVER NEAR EXETER,43.361950000000000,-81.509450000000001,ON,UTC-05:00,41022129,22134,0.000000000353928 +166,02FF010,AUSABLE RIVER NEAR PARKHILL,43.193330000000003,-81.814449999999994,ON,UTC-05:00,41022292,22297,0.000000000217899 +167,02FF011,SILVER CREEK AT SEAFORTH,43.545279999999998,-81.396389999999997,ON,UTC-05:00,41021974,21979,0.000000000444781 +168,02FF012,PERCH CREEK AT SARNIA,42.984169999999999,-82.318889999999996,ON,UTC-05:00,41022544,22549,0.000000002588265 +169,02FF013,LITTLE AUSABLE RIVER NEAR LUCAN CROSSING,43.180560000000000,-81.447670000000002,ON,UTC-05:00,41022290,22295,0.000000003390763 +170,02FF014,BLACK CREEK NEAR HENSALL,43.418920000000000,-81.495469999999997,ON,UTC-05:00,41022089,22094,0.000000000847718 +171,02FF015,TRICKS CREEK NEAR CLINTON,43.590310000000002,-81.584109999999995,ON,UTC-05:00,41021929,21934,0.000000000239244 +172,02FF016,LITTLE AUSABLE RIVER NEAR CENTRALIA,43.286250000000003,-81.415639999999996,ON,UTC-05:00,41022201,22206,0.000000000578311 +173,02GA003,GRAND RIVER AT GALT,43.352780000000003,-80.316950000000006,ON,UTC-05:00,41001759,1764,0.000000000201971 +174,02GA005,IRVINE RIVER NEAR SALEM,43.714170000000003,-80.443889999999996,ON,UTC-05:00,41001111,1116,0.000000000170138 +175,02GA006,CONESTOGO RIVER AT ST. JACOBS,43.541110000000003,-80.553340000000006,ON,UTC-05:00,41001414,1419,0.000000000547965 +176,02GA010,NITH RIVER NEAR CANNING,43.189720000000001,-80.455029999999994,ON,UTC-05:00,41002164,2169,0.000000000039226 +177,02GA014,GRAND RIVER NEAR MARSVILLE,43.861719999999998,-80.272220000000004,ON,UTC-05:00,41000941,946,0.000000000448731 +178,02GA015,SPEED RIVER BELOW GUELPH,43.533610000000003,-80.252219999999994,ON,UTC-05:00,41001386,1391,0.000000000446077 +179,02GA016,GRAND RIVER BELOW SHAND DAM,43.730829999999997,-80.340840000000000,ON,UTC-05:00,41001099,1104,0.000000000153203 +180,02GA018,NITH RIVER AT NEW HAMBURG,43.377499999999998,-80.711389999999994,ON,UTC-05:00,41001747,1752,0.000000000105220 +181,02GA023,CANAGAGIGUE CREEK NEAR ELMIRA,43.579920000000001,-80.509190000000004,ON,UTC-05:00,41001295,1300,0.000000000653659 +182,02GA024,LAUREL CREEK AT WATERLOO,43.468890000000002,-80.518889999999999,ON,UTC-05:00,41001538,1543,0.000000000039836 +183,02GA028,CONESTOGO RIVER AT GLEN ALLAN,43.654829999999997,-80.702169999999995,ON,UTC-05:00,41001237,1242,0.000000000383181 +184,02GA029,ERAMOSA RIVER ABOVE GUELPH,43.547780000000003,-80.182000000000002,ON,UTC-05:00,41001345,1350,0.000000001820438 +185,02GA030,ALDER CREEK NEAR NEW DUNDEE,43.371940000000002,-80.551670000000001,ON,UTC-05:00,41001717,1722,0.000000000372488 +186,02GA031,BLUE SPRINGS CREEK NEAR EDEN MILLS,43.576140000000002,-80.108999999999995,ON,UTC-05:00,41001287,1292,0.000000000268859 +187,02GA034,GRAND RIVER AT WEST MONTROSE,43.585000000000001,-80.481390000000005,ON,UTC-05:00,41001297,1302,0.000000000061285 +188,02GA038,NITH RIVER ABOVE NITHBURG,43.483890000000002,-80.834999999999994,ON,UTC-05:00,41001562,1567,0.000000000215467 +189,02GA039,CONESTOGO RIVER ABOVE DRAYTON,43.783529999999999,-80.637780000000006,ON,UTC-05:00,41001028,1033,0.000000000155700 +190,02GA040,SPEED RIVER NEAR ARMSTRONG MILLS,43.638610000000000,-80.269999999999996,ON,UTC-05:00,41001222,1227,0.000000000314620 +191,02GA041,GRAND RIVER NEAR DUNDALK,44.140079999999998,-80.363110000000006,ON,UTC-05:00,41000757,762,0.000000000477638 +192,02GA042,MOOREFIELD CREEK NEAR ROTHSAY,43.823000000000000,-80.717860000000002,ON,UTC-05:00,41001003,1008,0.000000000488560 +193,02GA043,HUNSBURGER CREEK NEAR WILMOT CENTRE,43.364440000000002,-80.632499999999993,ON,UTC-05:00,41001809,1814,0.000000000101925 +194,02GA047,SPEED RIVER AT CAMBRIDGE,43.421390000000002,-80.332220000000007,ON,UTC-05:00,41001616,1621,0.000000000007843 +195,02GA048,GRAND RIVER NEAR DOON,43.412810000000000,-80.417029999999997,ON,UTC-05:00,41001663,1668,0.000000003024404 +196,02GA049,SMITH CREEK NEAR NEWTON,43.596370000000000,-80.894279999999995,ON,UTC-05:00,41001332,1337,0.000000000045473 +197,02GB001,GRAND RIVER AT BRANTFORD,43.132779999999997,-80.267219999999995,ON,UTC-05:00,41002322,2327,0.000000000423326 +198,02GB006,HORNER CREEK NEAR PRINCETON,43.173890000000000,-80.552779999999998,ON,UTC-05:00,41002214,2219,0.000000000241904 +199,02GB007,FAIRCHILD CREEK NEAR BRANTFORD,43.147399999999998,-80.154600000000002,ON,UTC-05:00,41002253,2258,0.000000000732477 +200,02GB008,WHITEMANS CREEK NEAR MOUNT VERNON,43.126390000000001,-80.383610000000004,ON,UTC-05:00,41002321,2326,0.000000000353761 +201,02GB010,MCKENZIE CREEK NEAR CALEDONIA,43.033890000000000,-79.949719999999999,ON,UTC-05:00,41002504,2509,0.000000000903127 +202,02GC002,KETTLE CREEK AT ST. THOMAS,42.777780000000000,-81.214160000000007,ON,UTC-05:00,41003721,3726,0.000000000417129 +203,02GC007,BIG CREEK NEAR WALSINGHAM,42.685549999999999,-80.538610000000006,ON,UTC-05:00,41003952,3957,0.000000000224762 +204,02GC008,LYNN RIVER AT SIMCOE,42.823329999999999,-80.289439999999999,ON,UTC-05:00,41003393,3398,0.000000000766327 +205,02GC010,BIG OTTER CREEK AT TILLSONBURG,42.857300000000002,-80.723579999999998,ON,UTC-05:00,41003311,3316,0.000000000467201 +206,02GC011,BIG CREEK NEAR KELVIN,42.986809999999998,-80.444890000000001,ON,UTC-05:00,41002721,2726,0.000000000543668 +207,02GC014,YOUNG CREEK NEAR VITTORIA,42.765749999999997,-80.294579999999996,ON,UTC-05:00,41003630,3635,0.000000000155808 +208,02GC017,BIG OTTER CREEK ABOVE OTTERVILLE,42.965829999999997,-80.542779999999993,ON,UTC-05:00,41002809,2814,0.000000000479287 +209,02GC018,CATFISH CREEK NEAR SPARTA,42.746110000000002,-81.056950000000001,ON,UTC-05:00,41003785,3790,0.000000000152478 +210,02GC021,VENISON CREEK NEAR WALSINGHAM,42.653359999999999,-80.548439999999999,ON,UTC-05:00,41004067,4072,0.000000000352475 +211,02GC022,NANTICOKE CREEK AT NANTICOKE,42.810000000000002,-80.076110000000000,ON,UTC-05:00,41003427,3432,0.000000000023610 +212,02GC026,BIG OTTER CREEK NEAR CALTON,42.710560000000001,-80.840840000000000,ON,UTC-05:00,41003940,3945,0.000000001317316 +213,02GC029,KETTLE CREEK ABOVE ST. THOMAS,42.835000000000001,-81.134720000000002,ON,UTC-05:00,41003511,3516,0.000000000047122 +214,02GC030,CATFISH CREEK AT AYLMER,42.773890000000002,-80.982780000000005,ON,UTC-05:00,41003703,3708,0.000000006696006 +215,02GC031,DODD CREEK BELOW PAYNES MILLS,42.787500000000001,-81.267499999999998,ON,UTC-05:00,41003682,3687,0.000000000111287 +216,02GC036,SILVER CREEK NEAR GROVESEND,42.675829999999998,-80.953329999999994,ON,UTC-05:00,41004099,4104,0.000000000336510 +217,02GC037,NANTICOKE CREEK NEAR DUNDURN,42.967059999999996,-80.343400000000003,ON,UTC-05:00,41002791,2796,0.000000000366892 +218,02GC038,VENISON CREEK NEAR LANGTON,42.727649999999997,-80.626530000000002,ON,UTC-05:00,41003923,3928,0.000000000438322 +219,02GD001,THAMES RIVER NEAR EALING,42.972499999999997,-81.209720000000004,ON,UTC-05:00,41002909,2914,0.000000000046657 +220,02GD003,NORTH THAMES RIVER BELOW FANSHAWE DAM,43.040280000000003,-81.182220000000001,ON,UTC-05:00,41002682,2687,0.000000000128171 +221,02GD004,MIDDLE THAMES RIVER AT THAMESFORD,43.059170000000002,-80.994720000000001,ON,UTC-05:00,41002565,2570,0.000000000115606 +222,02GD005,NORTH THAMES RIVER AT ST. MARYS,43.255830000000003,-81.145550000000000,ON,UTC-05:00,41002068,2073,0.000000000276899 +223,02GD008,MEDWAY RIVER AT LONDON,43.013890000000004,-81.280559999999994,ON,UTC-05:00,41002752,2757,0.000000000338348 +224,02GD009,TROUT CREEK NEAR ST. MARYS,43.273249999999997,-81.103250000000003,ON,UTC-05:00,41002031,2036,0.000000000277893 +225,02GD010,FISH CREEK NEAR PROSPECT HILL,43.220469999999999,-81.236890000000002,ON,UTC-05:00,41002145,2150,0.000000000485051 +226,02GD011,CEDAR CREEK AT WOODSTOCK,43.121940000000002,-80.751109999999997,ON,UTC-05:00,41002411,2416,0.000000000244874 +227,02GD014,NORTH THAMES RIVER NEAR MITCHELL,43.450279999999999,-81.206670000000003,ON,UTC-05:00,41001680,1685,0.000000000352690 +228,02GD015,NORTH THAMES RIVER NEAR THORNDALE,43.149439999999998,-81.192220000000006,ON,UTC-05:00,41002480,2485,0.000000000481942 +229,02GD016,THAMES RIVER AT INGERSOLL,43.041390000000000,-80.886390000000006,ON,UTC-05:00,41002609,2614,0.000000000021944 +230,02GD018,AVON RIVER BELOW STRATFORD,43.344720000000002,-81.116420000000005,ON,UTC-05:00,41001841,1846,0.000000000033065 +231,02GD019,TROUT CREEK NEAR FAIRVIEW,43.295830000000002,-80.973050000000001,ON,UTC-05:00,41001998,2003,0.000000000330521 +232,02GD021,THAMES RIVER AT INNERKIP,43.215280000000000,-80.691950000000006,ON,UTC-05:00,41002124,2129,0.000000000052454 +233,02GD022,NISSOURI CREEK NEAR EMBRO,43.130000000000003,-80.963329999999999,ON,UTC-05:00,41002402,2407,0.000000000360342 +234,02GD026,AVON RIVER ABOVE STRATFORD,43.376640000000002,-80.937939999999998,ON,UTC-05:00,41001745,1750,0.000000000916391 +235,02GD027,REYNOLDS CREEK NEAR PUTNAM,42.981670000000001,-80.954719999999995,ON,UTC-05:00,41002814,2819,0.000000000302781 +236,02GD028,STONEY CREEK AT LONDON,43.021949999999997,-81.253889999999998,ON,UTC-05:00,41002712,2717,0.000000000430417 +237,02GE002,THAMES RIVER AT BYRON,42.962499999999999,-81.332220000000007,ON,UTC-05:00,41002984,2989,0.000000000025979 +238,02GE003,THAMES RIVER AT THAMESVILLE,42.544719999999998,-81.967219999999998,ON,UTC-05:00,41004437,4442,0.000000000064887 +239,02GE004,THAMES RIVER AT CHATHAM,42.414439999999999,-82.179440000000000,ON,UTC-05:00,41004631,4636,0.000000000684164 +240,02GE005,DINGMAN CREEK BELOW LAMBETH,42.933889999999998,-81.351389999999995,ON,UTC-05:00,41003178,3183,0.000000000111760 +241,02GE006,THAMES RIVER NEAR DUTTON,42.730559999999997,-81.577500000000001,ON,UTC-05:00,41003925,3930,0.000000000187872 +242,02GE007,MCGREGOR CREEK NEAR CHATHAM,42.383519999999997,-82.094170000000005,ON,UTC-05:00,41004658,4663,0.000000000753869 +243,02GE008,OXBOW CREEK NEAR KILWORTH,42.966389999999997,-81.418049999999994,ON,UTC-05:00,41002994,2999,0.000000000766256 +244,02GE009,BIG CREEK NEAR COMBER,42.205010000000001,-82.520259999999993,ON,UTC-05:00,41004964,4969,0.000000000654360 +245,02GG002,SYDENHAM RIVER NEAR ALVINSTON,42.830829999999999,-81.851939999999999,ON,UTC-05:00,41003726,3731,0.000000000154362 +246,02GG003,SYDENHAM RIVER AT FLORENCE,42.650550000000003,-82.008330000000001,ON,UTC-05:00,41004257,4262,0.000000000189345 +247,02GG005,SYDENHAM RIVER AT STRATHROY,42.958610000000000,-81.627219999999994,ON,UTC-05:00,41003017,3022,0.000000001850530 +248,02GG006,BEAR CREEK NEAR PETROLIA,42.905830000000002,-82.119159999999994,ON,UTC-05:00,41003475,3480,0.000000001938673 +249,02GG008,SYDENHAM RIVER AT WALLACEBURG,42.592779999999998,-82.383330000000001,ON,UTC-05:00,41004405,4410,0.000000000228002 +250,02GG009,BEAR CREEK BELOW BRIGDEN,42.811940000000000,-82.298330000000007,ON,UTC-05:00,41003773,3778,0.000000000328356 +251,02GG013,BLACK CREEK NEAR BRADSHAW,42.762439999999998,-82.259219999999999,ON,UTC-05:00,41003878,3883,0.000000000786137 +252,02GH002,RUSCOM RIVER NEAR RUSCOM STATION,42.211390000000002,-82.629170000000002,ON,UTC-05:00,41004954,4959,0.000000000336512 +253,02GH003,CANARD RIVER NEAR LUKERVILLE,42.158890000000000,-83.018889999999999,ON,UTC-05:00,41005062,5067,0.000000001094471 +254,02GH004,TURKEY CREEK AT WINDSOR,42.260559999999998,-83.039720000000003,ON,UTC-05:00,41004894,4899,0.000000000150271 +255,02GH011,LITTLE RIVER AT WINDSOR,42.309719999999999,-82.928340000000006,ON,UTC-05:00,41004809,4814,0.000000000782326 +256,02HA006,TWENTY MILE CREEK AT BALLS FALLS,43.133609999999997,-79.383330000000001,ON,UTC-05:00,41028412,28418,0.000000000149749 +257,02HA007,WELLAND RIVER BELOW CAISTOR CORNERS,43.021670000000000,-79.617769999999993,ON,UTC-05:00,41028587,28593,0.000000000417828 +259,02HA014,REDHILL CREEK AT HAMILTON,43.232219999999998,-79.784719999999993,ON,UTC-05:00,41028309,28315,0.000000001274090 +260,02HA020,TWENTY MILE CREEK ABOVE SMITHVILLE,43.115549999999999,-79.566109999999995,ON,UTC-05:00,41028492,28498,0.000000000094277 +261,02HA024,OSWEGO CREEK AT CANBORO,42.991390000000003,-79.678340000000006,ON,UTC-05:00,41028670,28676,0.000000000049123 +262,02HA030,FOUR MILE CREEK NEAR VIRGIL,43.195610000000002,-79.113330000000005,ON,UTC-05:00,41028327,28333,0.000000000408063 +263,02HA031,TWELVE MILE CREEK NEAR POWER GLEN,43.114780000000003,-79.274559999999994,ON,UTC-05:00,41028447,28453,0.000000007913102 +264,02HA032,NORTH CREEK NEAR SMITHVILLE,43.074530000000003,-79.525000000000006,ON,UTC-05:00,41028504,28510,0.000000000353529 +265,02HB001,CREDIT RIVER NEAR CATARACT,43.835830000000001,-80.022779999999997,ON,UTC-05:00,41027717,27723,0.000000000027031 +266,02HB004,EAST SIXTEEN MILE CREEK NEAR OMAGH,43.498890000000003,-79.776949999999999,ON,UTC-05:00,41028118,28124,0.000000000072100 +267,02HB005,SIXTEEN MILE CREEK AT MILTON,43.513890000000004,-79.879720000000006,ON,UTC-05:00,41028112,28118,0.000000000261293 +268,02HB007,SPENCER CREEK AT DUNDAS,43.265560000000001,-79.964160000000007,ON,UTC-05:00,41028272,28278,0.000000001160448 +269,02HB008,CREDIT RIVER WEST BRANCH AT NORVAL,43.646670000000000,-79.866389999999996,ON,UTC-05:00,41027997,28003,0.000000000518554 +270,02HB011,BRONTE CREEK NEAR ZIMMERMAN,43.436940000000000,-79.864440000000002,ON,UTC-05:00,41028152,28158,0.000000004027846 +271,02HB012,GRINDSTONE CREEK NEAR ALDERSHOT,43.300559999999997,-79.868889999999993,ON,UTC-05:00,41028239,28245,0.000000000009990 +272,02HB013,CREDIT RIVER NEAR ORANGEVILLE,43.891109999999998,-80.062230000000000,ON,UTC-05:00,41027653,27659,0.000000000221532 +273,02HB015,SPENCER CREEK NEAR WESTOVER,43.353079999999999,-80.077889999999996,ON,UTC-05:00,41028210,28216,0.000000000310481 +274,02HB018,CREDIT RIVER AT BOSTON MILLS,43.773609999999998,-79.926940000000002,ON,UTC-05:00,41027882,27888,0.000000001150922 +275,02HB020,CREDIT RIVER ERIN BRANCH ABOVE ERIN,43.771949999999997,-80.093329999999995,ON,UTC-05:00,41027841,27847,0.000000003786977 +276,02HB021,ANCASTER CREEK AT ANCASTER,43.231169999999999,-79.973860000000002,ON,UTC-05:00,41028340,28346,0.000000000775193 +277,02HB022,BRONTE CREEK AT CARLISLE,43.388890000000004,-79.988050000000001,ON,UTC-05:00,41028191,28197,0.000000000062357 +278,02HB023,SPENCER CREEK AT HIGHWAY NO. 5,43.283050000000003,-80.052779999999998,ON,UTC-05:00,41028257,28263,0.000000000014625 +279,02HB024,BLACK CREEK BELOW ACTON,43.629309999999997,-80.010419999999996,ON,UTC-05:00,41028014,28020,0.000000003462511 +280,02HB025,CREDIT RIVER AT NORVAL,43.647500000000001,-79.856110000000001,ON,UTC-05:00,41027984,27990,0.000000001860671 +281,02HB027,FOURTEEN MILE CREEK AT OAKVILLE,43.421109999999999,-79.703059999999994,ON,UTC-05:00,41028162,28168,0.000000000514525 +282,02HB028,GRINDSTONE CREEK NEAR MILLGROVE,43.334859999999999,-79.949060000000003,ON,UTC-05:00,41028218,28224,0.000000000333435 +283,02HB029,CREDIT RIVER AT STREETSVILLE,43.582360000000001,-79.708110000000005,ON,UTC-05:00,41028032,28038,0.000000002075399 +284,02HB030,COOKSVILLE CREEK NEAR COOKSVILLE,43.592329999999997,-79.623890000000003,ON,UTC-05:00,41028035,28041,0.000000000503783 +285,02HB031,CREDIT RIVER ERIN BRANCH AT HILLSBURGH,43.790250000000000,-80.143439999999998,ON,UTC-05:00,41027818,27824,0.000000000588380 +286,02HB032,MOUNTSBERG CREEK BELOW MOUNTSBERG RESERVOIR,43.454830000000001,-80.045829999999995,ON,UTC-05:00,41028159,28165,0.000000001544630 +287,02HB033,MOUNTSBERG CREEK NEAR CARLISLE,43.401530000000001,-79.993030000000005,ON,UTC-05:00,41028190,28196,0.000000000202384 +288,02HC003,HUMBER RIVER AT WESTON,43.698889999999999,-79.520280000000000,ON,UTC-05:00,41027910,27916,0.000000000364318 +289,02HC005,DON RIVER AT YORK MILLS,43.740279999999998,-79.403049999999993,ON,UTC-05:00,41027856,27862,0.000000000162378 +290,02HC009,EAST HUMBER RIVER NEAR PINE GROVE,43.789999999999999,-79.584440000000001,ON,UTC-05:00,41027767,27773,0.000000005008705 +291,02HC013,HIGHLAND CREEK NEAR WEST HILL,43.778329999999997,-79.191389999999998,ON,UTC-05:00,41027746,27752,0.000000000275946 +292,02HC017,ETOBICOKE CREEK AT BRAMPTON,43.691389999999998,-79.759720000000002,ON,UTC-05:00,41027944,27950,0.000000000292045 +293,02HC018,LYNDE CREEK NEAR WHITBY,43.875560000000000,-78.960279999999997,ON,UTC-05:00,41027563,27569,0.000000000096574 +294,02HC019,DUFFINS CREEK ABOVE PICKERING,43.891390000000001,-79.059169999999995,ON,UTC-05:00,41027565,27571,0.000000000278182 +295,02HC022,ROUGE RIVER NEAR MARKHAM,43.858330000000002,-79.233609999999999,ON,UTC-05:00,41027672,27678,0.000000000156329 +296,02HC023,COLD CREEK NEAR BOLTON (02HC023),43.890278000000002,-79.719999999999999,ON,UTC-05:00,41027585,27591,0.000000000449520 +297,02HC024,DON RIVER AT TODMORDEN,43.685830000000003,-79.361390000000000,ON,UTC-05:00,41027916,27922,0.000000000995214 +298,02HC025,HUMBER RIVER AT ELDER MILLS,43.811390000000003,-79.627780000000001,ON,UTC-05:00,41027742,27748,0.000000000065954 +299,02HC027,BLACK CREEK NEAR WESTON,43.674169999999997,-79.504450000000006,ON,UTC-05:00,41027933,27939,0.000000000348976 +300,02HC028,LITTLE ROUGE CREEK NEAR LOCUST HILL,43.907769999999999,-79.216110000000000,ON,UTC-05:00,41027582,27588,0.000000002412505 +301,02HC030,ETOBICOKE CREEK BELOW QUEEN ELIZABETH HIGHWAY,43.601669999999999,-79.556389999999993,ON,UTC-05:00,41028026,28032,0.000000000166322 +302,02HC031,WEST HUMBER RIVER AT HIGHWAY NO. 7,43.758339999999997,-79.678889999999996,ON,UTC-05:00,41027830,27836,0.000000000127784 +303,02HC032,EAST HUMBER RIVER AT KING CREEK,43.902780000000000,-79.612780000000001,ON,UTC-05:00,41027540,27546,0.000000000950341 +304,02HC033,MIMICO CREEK AT ISLINGTON,43.647500000000001,-79.519720000000007,ON,UTC-05:00,41027985,27991,0.000000001811326 +305,02HC038,WEST DUFFINS CREEK ABOVE GREEN RIVER,43.914999999999999,-79.179720000000003,ON,UTC-05:00,41027436,27442,0.000000000432670 +306,02HC047,HUMBER RIVER NEAR PALGRAVE,43.930830000000000,-79.822220000000002,ON,UTC-05:00,41027496,27502,0.000000000024180 +307,02HC049,DUFFINS CREEK AT AJAX,43.848889999999997,-79.056110000000004,ON,UTC-05:00,41027595,27601,0.000000001231707 +308,02HC051,CENTREVILLE CREEK NEAR ALBION,43.924450000000000,-79.834440000000001,ON,UTC-05:00,41027552,27558,0.000000000608771 +309,02HC053,LITTLE ROUGE RIVER NEAR DICKSONS HILL,43.925690000000003,-79.282190000000000,ON,UTC-05:00,41027418,27424,0.000000000452852 +310,02HC054,LYNDE CREEK AT BROOKLIN,43.959170000000000,-78.959999999999994,ON,UTC-05:00,41027353,27359,0.000000000229516 +311,02HC055,LYNDE CREEK TRIBUTARY NEAR KINSALE,43.931950000000001,-78.988609999999994,ON,UTC-05:00,41027512,27518,0.000000000205023 +312,02HC056,DON RIVER EAST BRANCH NEAR THORNHILL,43.826610000000002,-79.438079999999999,ON,UTC-05:00,41027674,27680,0.000000000176035 +313,02HC057,HUMBER RIVER NEAR BALLYCROY,43.970309999999998,-79.887720000000002,ON,UTC-05:00,41027387,27393,0.000000000644851 +314,02HC058,WEST HIGHLAND CREEK NEAR SCARBOROUGH VILLAGE,43.753219999999999,-79.233670000000004,ON,UTC-05:00,41027799,27805,0.000000000466268 +315,02HC059,Humber River at Highway No. 9,43.968769999999999,-79.873739999999998,ON,UTC-05:00,41027387,27393,0.000000004033661 +316,02HD003,GANARASKA RIVER NEAR OSACA,44.015279999999997,-78.437500000000000,ON,UTC-05:00,41027151,27157,0.000000000163541 +317,02HD004,NORTH WEST GANARASKA RIVER NEAR OSACA,44.017220000000002,-78.438890000000001,ON,UTC-05:00,41027138,27144,0.000000000899741 +318,02HD006,BOWMANVILLE CREEK AT BOWMANVILLE,43.921469999999999,-78.701999999999998,ON,UTC-05:00,41027367,27373,0.000000000163046 +319,02HD008,OSHAWA CREEK AT OSHAWA,43.930280000000003,-78.891670000000005,ON,UTC-05:00,41027461,27467,0.000000000747120 +320,02HD009,WILMOT CREEK NEAR NEWCASTLE,43.930280000000003,-78.618889999999993,ON,UTC-05:00,41027385,27391,0.000000000536587 +321,02HD010,SHELTER VALLEY BROOK NEAR GRAFTON,43.991940000000000,-78.001390000000001,ON,UTC-05:00,41027169,27175,0.000000000264562 +322,02HD012,GANARASKA RIVER ABOVE DALE,43.990830000000003,-78.328329999999994,ON,UTC-05:00,41027182,27188,0.000000001566404 +323,02HD013,HARMONY CREEK AT OSHAWA,43.888890000000004,-78.825000000000003,ON,UTC-05:00,41027459,27465,0.000000000175445 +324,02HD018,PROCTORS CREEK NEAR BRIGHTON,44.059719999999999,-77.744450000000001,ON,UTC-05:00,41026960,26966,0.000000000056146 +325,02HD019,COBOURG BROOK AT COBOURG,43.965000000000003,-78.185000000000002,ON,UTC-05:00,41027232,27238,0.000000000358408 +326,02HD020,BALTIMORE CREEK AT BALTIMORE,44.028829999999999,-78.147059999999996,ON,UTC-05:00,41027081,27087,0.000000000474868 +327,02HD021,WILMOT CREEK NEAR LESKARD,44.005719999999997,-78.642330000000001,ON,UTC-05:00,41027220,27226,0.000000000042349 +328,02HD022,COBOURG BROOK NEAR PRECIOUS CORNERS,43.988470000000000,-78.210750000000004,ON,UTC-05:00,41027187,27193,0.000000000128310 +329,02HD023,MACKIE CREEK NEAR HAMPTON,43.979080000000003,-78.680310000000006,ON,UTC-05:00,41027256,27262,0.000000001077702 +330,02HD024,GAGE CREEK NEAR DALE,43.997030000000002,-78.254329999999996,ON,UTC-05:00,41027186,27192,0.000000000281437 +331,02HE002,CONSECON CREEK AT ALISONVILLE,44.027780000000000,-77.366669999999999,ON,UTC-05:00,41026915,26921,0.000000000131297 +332,02HE004,BLACK CREEK AT MILFORD,43.936830000000000,-77.094970000000004,ON,UTC-05:00,41027092,27098,0.000000008144635 +333,02HF002,GULL RIVER AT NORLAND,44.731949999999998,-78.818049999999999,ON,UTC-05:00,41024440,24446,0.000000000708039 +334,02HF003,BURNT RIVER NEAR BURNT RIVER,44.710000000000001,-78.677499999999995,ON,UTC-05:00,41024610,24616,0.000000000240748 +335,02HG001,MARIPOSA BROOK NEAR LITTLE BRITAIN,44.289720000000003,-78.841669999999993,ON,UTC-05:00,41026334,26340,0.000000000359532 +336,02HG002,NONQUON RIVER NEAR PORT PERRY,44.086689999999997,-79.006079999999997,ON,UTC-05:00,41027034,27040,0.000000001293536 +337,02HG003,BLACKSTOCK CREEK NEAR BLACKSTOCK,44.131920000000001,-78.828969999999998,ON,UTC-05:00,41026874,26880,0.000000005944352 +338,02HH003,PIGEON RIVER NEAR LOTUS,44.124169999999999,-78.694999999999993,ON,UTC-05:00,41026906,26912,0.000000000273023 +339,02HH005,PIGEON RIVER AT OMEMEE,44.296720000000001,-78.549999999999997,ON,UTC-05:00,41026208,26214,0.000000001271685 +340,02HJ001,JACKSON CREEK AT PETERBOROUGH,44.302500000000002,-78.321110000000004,ON,UTC-05:00,41026161,26167,0.000000000153602 +341,02HJ003,OUSE RIVER NEAR WESTWOOD,44.298609999999996,-78.045559999999995,ON,UTC-05:00,41026212,26218,0.000000000180677 +342,02HJ005,SQUIRREL CREEK NEAR BAILIEBORO,44.123330000000003,-78.388610000000000,ON,UTC-05:00,41026816,26822,0.000000000346297 +343,02HJ006,JACKSON CREEK NEAR JACKSON HEIGHTS,44.309829999999998,-78.370720000000006,ON,UTC-05:00,41026133,26139,0.000000005220758 +344,02HJ007,BAXTER CREEK AT MILLBROOK,44.150530000000003,-78.447109999999995,ON,UTC-05:00,41026774,26780,0.000000000345383 +346,02HK003,CROWE RIVER AT MARMORA,44.481670000000001,-77.684719999999999,ON,UTC-05:00,41025354,25360,0.000000000321237 +347,02HK005,CROWE RIVER NEAR GLEN ALDA,44.844439999999999,-77.931110000000004,ON,UTC-05:00,41023838,23844,0.000000000694423 +348,02HK006,BEAVER CREEK NEAR MARMORA,44.535280000000000,-77.696659999999994,ON,UTC-05:00,41025076,25082,0.000000000251893 +349,02HK007,COLD CREEK AT ORLAND,44.134720000000002,-77.786940000000001,ON,UTC-05:00,41026670,26676,0.000000000362775 +350,02HK008,RAWDON CREEK NEAR WEST HUNTINGDON,44.338050000000003,-77.477220000000003,ON,UTC-05:00,41025804,25810,0.000000000450549 +351,02HK009,BURNLEY CREEK ABOVE WARKWORTH,44.195839999999997,-77.911670000000001,ON,UTC-05:00,41026458,26464,0.000000002444356 +352,02HK010,TRENT RIVER AT TRENTON (AFFRA),44.108330000000002,-77.579170000000005,ON,UTC-05:00,41026710,26716,0.000000000140056 +353,02HK011,MAYHEW CREEK NEAR TRENTON,44.109169999999999,-77.612530000000007,ON,UTC-05:00,41026694,26700,0.000000000715322 +354,02HK015,SALT CREEK NEAR CODRINGTON,44.204560000000001,-77.819080000000000,ON,UTC-05:00,41026475,26481,0.000000001294932 +355,02HK016,TROUT CREEK NEAR CAMPBELLFORD,44.307000000000002,-77.830939999999998,ON,UTC-05:00,41026076,26082,0.000000000431487 +356,02HK017,HOARDS CREEK NEAR WELLMAN,44.336939999999998,-77.628670000000000,ON,UTC-05:00,41025839,25845,0.000000000247535 +357,02HL001,MOIRA RIVER NEAR FOXBORO,44.253610000000002,-77.419169999999994,ON,UTC-05:00,41026131,26137,0.000000001126335 +358,02HL003,BLACK RIVER NEAR ACTINOLITE,44.539439999999999,-77.370829999999998,ON,UTC-05:00,41024891,24897,0.000000000952768 +359,02HL004,SKOOTAMATTA RIVER NEAR ACTINOLITE,44.549720000000001,-77.328059999999994,ON,UTC-05:00,41024890,24896,0.000000000216154 +360,02HL005,MOIRA RIVER NEAR DELORO,44.499720000000003,-77.618330000000000,ON,UTC-05:00,41025161,25167,0.000000000066994 +361,02HL007,MOIRA RIVER NEAR TWEED,44.488610000000001,-77.319440000000000,ON,UTC-05:00,41025131,25137,0.000000000392577 +362,02HL008,CLARE RIVER NEAR BOGART,44.483330000000002,-77.227789999999999,ON,UTC-05:00,41025078,25084,0.000000000051152 +363,02HM002,DEPOT CREEK AT BELLROCK,44.471390000000000,-76.762780000000006,ON,UTC-05:00,41024978,24984,0.000000000177700 +364,02HM003,SALMON RIVER NEAR SHANNONVILLE,44.207220000000000,-77.209170000000000,ON,UTC-05:00,41026357,26363,0.000000000541323 +365,02HM004,WILTON CREEK NEAR NAPANEE,44.239440000000002,-76.849440000000001,ON,UTC-05:00,41026069,26075,0.000000000487143 +366,02HM005,COLLINS CREEK NEAR KINGSTON,44.256390000000003,-76.612499999999997,ON,UTC-05:00,41025904,25910,0.000000000408978 +367,02HM006,MILLHAVEN CREEK NEAR MILLHAVEN,44.226390000000002,-76.759159999999994,ON,UTC-05:00,41025997,26003,0.000000000005347 +368,02HM007,NAPANEE RIVER AT CAMDEN EAST,44.334719999999997,-76.838890000000006,ON,UTC-05:00,41025603,25609,0.000000000545432 +369,02HM009,WEST BRANCH LITTLE CATARAQUI CREEK NEAR KINGSTON,44.240000000000002,-76.578890000000001,ON,UTC-05:00,41025921,25927,0.000000000032228 +370,02HM010,SALMON RIVER AT TAMWORTH,44.486109999999996,-76.993060000000000,ON,UTC-05:00,41025081,25087,0.000000000868495 +371,02HM011,MILLHAVEN CREEK NEAR SYDENHAM,44.418889999999998,-76.594999999999999,ON,UTC-05:00,41025186,25192,0.000000000046394 +421,02MB006,LYN CREEK NEAR LYN,44.524999999999999,-75.805279999999996,ON,UTC-05:00,41022861,22867,0.000000000473432 +422,02MB010,BUELLS CREEK AT BROCKVILLE,44.585830000000001,-75.691670000000002,ON,UTC-05:00,41022723,22729,0.000000005487209 \ No newline at end of file diff --git a/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/README.TXT b/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/README.TXT new file mode 100644 index 0000000..f0fc035 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/README.TXT @@ -0,0 +1,21 @@ +1) To start the download processes, at the prompt, + +./run_canflow.sh + +or put this script in a cron job. It will clear the log file every day at 5:28. + +Output is saved in the directory pointed to by the $OUTDIR variable. You can change the output directory by redefining the $OUTDIR variable in run_usgs.sh + +The log file is saved to the file defined by the $log variable. You can change the log file by modifying the $log variable in the run_usgs.sh. + +The log file is reset daily at 5:28. To change the time, redefine the $RESETLOGAT variable. + +2) New to NWM v3.1: + +Since v3.1, a site file is needed to run this download script because the server has been changed to https://wateroffice.ec.gc.ca/mainmenu/real_time_data_index_e.html. + +https://wateroffice.ec.gc.ca/services/real_time_data/csv/inline?stations[]=02AB014¶meters[]=47&start_date=2024-04-22%2000:00:00&end_date=2024-04-24%2023:59:59 + +An example site file can be found in the root directory named NWM_canadian_gage_inputs.txt. When stations are added or removed, this file needs to be uipdated. + +NWM uses only stationId start with 02* based on NWM_canadian_gage_inputs.txt file and only use parameter=47 for discharge. diff --git a/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/TimeSliceC.py b/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/TimeSliceC.py new file mode 100644 index 0000000..477482d --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/TimeSliceC.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python + +import os, sys, time, math +from string import * +from datetime import datetime, timedelta +import calendar +import netCDF4 +import numpy as np +import pytz + +def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): + '''Python 2 implementation of Python 3.5 math.isclose() + https://hg.python.org/cpython/file/tip/Modules/mathmodule.c#l1993''' + # sanity check on the inputs + if rel_tol < 0 or abs_tol < 0: + raise ValueError("tolerances must be non-negative") + + # short circuit exact equality -- needed to catch two infinities of + # the same sign. And perhaps speeds things up a bit sometimes. + if a == b: + return True + + # This catches the case of two infinities of opposite sign, or + # one infinity and one finite number. Two infinities of opposite + # sign would otherwise have an infinite relative tolerance. + # Two infinities of the same sign are caught by the equality check + # above. + if math.isinf(a) or math.isinf(b): + return False + + # now do the regular computation + # this is essentially the "weak" test from the Boost library + diff = math.fabs(b - a) + result = (((diff <= math.fabs(rel_tol * b)) or + (diff <= math.fabs(rel_tol * a))) or + (diff <= abs_tol)) + return result + +def dict_compare(d1, d2): + d1_keys = set(d1.keys()) + d2_keys = set(d2.keys()) + intersect_keys = d1_keys.intersection(d2_keys) + added = d1_keys - d2_keys + removed = d2_keys - d1_keys + modified = dict() + for o in intersect_keys: + f0 = d1[o][0] == d2[o][0] + f1 = isclose( d1[o][1], d2[o][1],abs_tol=0.01 ) + f2 = isclose( d1[o][2], d2[o][2]) + if not f0 or not f1 or not f2: + modified[o] = (d1[o], d2[o]) + same = set(o for o in intersect_keys if d1[o] == d2[o]) + return added, removed, modified, same + +class TimeSliceC: + """ Description: Store one time slice data (from Canadian data) + Author: Tim Hunter (tim.hunter@noaa.gov) drawing HEAVILY on + original code by Zhengtao Cui (Zhengtao.Cui@noaa.gov) + Date: Feb 15, 2018 + """ + stationIdStrLen = 15 + stationIdLong_name = "WSC station id padded to length 15" + timeStrLen = 19 + timeUnit = "UTC" + timeLong_name = "YYYY-MM-DD_HH:mm:ss UTC" + dischargeUnit = "m^3/s" + dischargeLong_name = "Discharge.cubic_meters_per_second" + dischargeQualityUnit = "-" + dischargeQualityLong_name = \ + "Discharge quality 0 to 100 to be scaled by 100." + + def __init__(self, time_stamp, resolution, station_time_value ): + # insure that time stamp is tz-aware. Time is already UTC. + self.centralTimeStamp = time_stamp.replace(tzinfo=pytz.UTC) + self.sliceTimeResolution = resolution + self.obvStationTimeValue = station_time_value + + def isEmpty( self ): + return not self.obvStationTimeValue + + def print_station_time_value( self ): + for e in self.obvStationTimeValue: + print( "Slice: central time: ", \ + self.centralTimeStamp.isoformat(), \ + e[0], e[1].isoformat(), e[2] ) + + def getStationIDs( self ): + stationL = [] + for e in self.obvStationTimeValue: + stationL.append( list( e[ 0 ][4:] ) ) + for s in stationL: + if len(s) < self.stationIdStrLen: + for i in range( len(s), self.stationIdStrLen ): + s.insert(0, ' ' ) + elif len(s) > self.stationIdStrLen: + s = s[0:self.stationIdStrLen - 1] + return stationL + + def getDischargeValues( self ): + values = [] + for e in self.obvStationTimeValue: + values.append( e[ 2 ] ) + return values + + def getDischargeTimes( self ): + obvtimes = [] + for e in self.obvStationTimeValue: + obvtimes.append(list(e[1].strftime("%Y-%m-%d_%H:%M:00"))) + return obvtimes + + def getQueryTimes( self ): + qtimes = [] + for e in self.obvStationTimeValue: + qtimes.append( calendar.timegm( e[ 1 ].utctimetuple() ) ) + return qtimes + + def getSliceNCFileName( self ): + tval = (self.sliceTimeResolution.days * 24 * 60) + (self.sliceTimeResolution.seconds // 60) + filename = (self.centralTimeStamp.strftime("%Y-%m-%d_%H_%M_00.") + + str(int(tval)).zfill(2) + "min.wscTimeSlice.ncdf") + return filename + + def getDischargeQuality( self ): + dq = [] + for e in self.obvStationTimeValue: + dq.append( e[ 3 ] ) + return dq + + def toNetCDF( self, outputdir = './' ): + fname = outputdir + '/' + self.getSliceNCFileName() + nc_fid = netCDF4.Dataset(fname, 'w', format='NETCDF4' ) + nc_fid.createDimension( 'stationIdStrLen', self.stationIdStrLen ) + nc_fid.createDimension( 'stationIdInd', None ) + nc_fid.createDimension( 'timeStrLen', self.timeStrLen ) + stationId = nc_fid.createVariable( 'stationId', 'S1',\ + ('stationIdInd', 'stationIdStrLen') ) + stationId.setncatts( {'long_name' : self.stationIdLong_name, \ + 'units' : '-'} ) + + time = nc_fid.createVariable( 'time', 'S1',\ + ('stationIdInd', 'timeStrLen' ) ) + time.setncatts( {'long_name' : self.timeLong_name, \ + 'units' : self.timeUnit} ) + + discharge = nc_fid.createVariable( 'discharge', 'f4',\ + ('stationIdInd', ) ) + discharge.setncatts( {'long_name' : self.dischargeLong_name, \ + 'units' : self.dischargeUnit} ) + discharge_quality = \ + nc_fid.createVariable( 'discharge_quality', 'i2',\ + ('stationIdInd', ) ) + discharge_quality.setncatts( {'long_name' : \ + self.dischargeQualityLong_name, \ + 'units' : self.dischargeQualityUnit, \ + 'multfactor' : '0.01' } ) + queryTime = nc_fid.createVariable( 'queryTime', 'i4',\ + ('stationIdInd', ) ) + + queryTime.setncatts( { 'units' : \ + 'seconds since 1970-01-01 00:00:00 local TZ' } ) + + tres = self.sliceTimeResolution.days * 24 * 60 + \ + self.sliceTimeResolution.seconds // 60 + nc_fid.setncatts( { 'fileUpdateTimeUTC': \ + datetime.utcnow().strftime( "%Y-%m-%d_%H:%M:00" ), \ + 'sliceCenterTimeUTC' : \ + self.centralTimeStamp.strftime( "%Y-%m-%d_%H:%M:00" ),\ + 'sliceTimeResolutionMinutes' : \ + str(int(tres)).zfill(2) } ) + + discharge[ : ] = self.getDischargeValues() + queryTime[ : ] = self.getQueryTimes() + + stations = self.getStationIDs() + stationId[ : ] = stations + + time[ : ] = self.getDischargeTimes() + discharge_quality[:] = self.getDischargeQuality() + + nc_fid.close() + + @classmethod + def fromNetCDF( self, ncfilename ): + nc_fid = netCDF4.Dataset( ncfilename, 'r' ) + timestamp = datetime.strptime( \ + nc_fid.getncattr( 'sliceCenterTimeUTC' ), \ + "%Y-%m-%d_%H:%M:00" ) + + time_resol = timedelta( minutes = \ + int( nc_fid.getncattr( 'sliceTimeResolutionMinutes' ) ) ) + + stations = netCDF4.chartostring( \ + nc_fid.variables[ 'stationId'][ : ] ) + + discharge = nc_fid.variables[ 'discharge'][ : ] + + queryTime = nc_fid.variables[ 'queryTime'][ : ] + + quality = nc_fid.variables[ 'discharge_quality'][ : ] + + stationTimeValue = [] + for s, d, q, qual in zip( stations, discharge, queryTime, quality ): + stationTimeValue.append( \ + ( 'CAN.' + s.strip(), \ + datetime.utcfromtimestamp( q ).replace(tzinfo=pytz.UTC), \ + d, qual ) ) + + nc_fid.close() + return self( timestamp, time_resol, stationTimeValue ) + + def mergeOld( self, oldTimeSlice ): + if self.centralTimeStamp != oldTimeSlice.centralTimeStamp or \ + self.sliceTimeResolution != oldTimeSlice.sliceTimeResolution: + print( 'new_cts=', self.centralTimeStamp ) + print( 'old_cts=', oldTimeSlice.centralTimeStamp ) + print( 'new_res=', self.sliceTimeResolution ) + print( 'old_res=', oldTimeSlice.sliceTimeResolution ) + raise RuntimeError( "FATAL ERROR: the two time slices " + + " differ, not merging ..." ) + else: + site_time_value = dict() + for e in self.obvStationTimeValue: + site_time_value[ e[ 0 ] ] = ( e[1], e[2], e[3] ) + old_site_time_value = dict() + + for e in oldTimeSlice.obvStationTimeValue: + old_site_time_value[ e[ 0 ] ] = ( e[1], e[2], e[3] ) + + added, removed, modified, same = ( + dict_compare(site_time_value, old_site_time_value )) + + if not added and not modified : + return False + + old_site_time_value.update( site_time_value ) + self.obvStationTimeValue = [] + for site in old_site_time_value: + self.obvStationTimeValue.append( ( site, \ + old_site_time_value[ site ][ 0 ], \ + old_site_time_value[ site ][ 1 ], \ + old_site_time_value[ site ][ 2 ] ) ) + return True diff --git a/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/WSC_Observation.py b/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/WSC_Observation.py new file mode 100644 index 0000000..9e7f930 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/WSC_Observation.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python +import os, sys, time, csv +from string import * +from datetime import datetime, timedelta +import xml.etree.ElementTree as etree +from TimeSliceC import TimeSliceC +import pytz +import unicodedata +from EmptyDirOrFileException import EmptyDirOrFileException + +""" + The class to mange dicharge data from one real-time discharge file + Author: Tim Hunter (tim.hunter@noaa.gov) + + Last modified by Zhengtao Cui (Zhengtao.Cui@noaa.gov) + Description: Updated the code to use real-time data from the + new server at + https://wateroffice.ec.gc.ca/mainmenu/real_time_data_index_e.html + +""" + +class WSC_Observation: + "Store one WSC data set " + + def __init__(self, csvfilename): + self.timevalue = {} + + # + # Define the column names I want to use + # If WSC changes file formats, this may need to be updated. + # + #csvfields = ('id', 'date', 'level', 'lgrade', 'lsymbol', 'lqaqc', + # 'discharge', 'dgrade', 'dsymbol', 'dqaqc') + # + #The header has changed since Feb 22, 2023. + #csvfields = ('ID', 'Date', 'Water Level / Niveau d\'eau (m)', + # 'Grade', 'Symbol / Symbole', 'QA/QC', + # 'Discharge / DÃit (cms)', 'Grade', 'Symbol / Symbole', 'QA/QC') + # + # + # Changed to new server https://wateroffice.ec.gc.ca/mainmenu/real_time_data_index_e.html + # header changed, + # + #csvfields = ('ID', 'Date', 'Parameter/ParamÃre', + # 'Value/Valeur', 'Qualifier/Qualificatif', 'Symbol/Symbole', + # 'Approval/Approbation') + csvfields = ('ID','Date','Parameter/ParamÃe','Value/Valeur', + 'Qualifier/Qualificatif','Symbol/Symbole', + 'Approval/Approbation','Grade/Classification', + 'Qualifiers/Qualificatifs') + + with open(csvfilename, 'r', encoding='utf-8-sig') as csvfile: + reader = csv.DictReader(csvfile) + # Strip spaces + reader.fieldnames = [f.strip() for f in reader.fieldnames] + + # Correct the special charaters (i.e.: French) + reader.fieldnames = [unicodedata.normalize('NFKD', f).encode('ascii', 'ignore').decode('ascii') for f in reader.fieldnames] + rows = list(reader) + if not rows: + raise ValueError(f"{csvfilename} has only a header; Skipping ...\n") + + for row in rows: + id_value = row.get('ID') + param_value = row.get('Parameter/Parametre') + + if not (param_value and param_value.isdigit()): + continue + + if int(param_value) == 47: # parameter 47 is for discharge + try: + dischstr = row.get('Value/Valeur') + discharge = float(dischstr) * 1.0 + #print(f"try: {id_value} ==>DISCHARGE={discharge}") + + qstr = row.get('Qualifier/Qualificatif') + #print(f"try: {id_value} ==>Qualifier={qstr}") + + if qstr.strip() != '' and int(qstr) == 10: + print('==>Skip ICE conditions ' + qstr + ' ...') + continue + + #print(f"try: {id_value} ==>qstr={qstr}") + except Exception as e: + print(f"Exception encountered for {id_value}: {e}") + discharge = -999999.0 + + dkey = None + try: + # + # Split the date string into two parts... Y-M-D and H:M:S + # Then parse them into a timezone-naive object + # Per the documentation from WSC, this timestamp is described as: + # "data timestamp in ISO 8601 format, Local Standard Time (LST)" + # So if the timestamp is + # 2018-08-16T13:30:00-0500 + # I am interpreting that as 1:30 pm local time, and that local time + # is 5 hours behind UTC. If this interpretation is incorrect, the + # following code should be modified appropriately. + # + # Once I have a datetime object representing the local time, I can + # use the timezone info to translate that into UTC, which is what + # will be used as the data key value. + # + # NOTE!!!!! When storing the data, we are rounding/stripping the + # seconds field to 0. So I need to do the same thing here. + # That is accomplished very simply, by replacing whatever was + # specified for seconds with '00'. If this is not done we end up + # with mismatch issues when we are merging timeslices. + # + #s = row['Date'].split("T") + #ymd = s[0] + #hms = s[1][:6] + '00' + #dsnz = ymd + ' ' + hms # format is YYYY-MM-DD HH:MM:SS + #dtnz = datetime.strptime(dsnz, '%Y-%m-%d %H:%M:%S') # timezone-naive object + + date_str = row.get('Date') + ymd, hms = date_str.split("T") + dsnz = f"{ymd} {hms[:6]}00" + dt = datetime.strptime(dsnz, '%Y-%m-%d %H:%M:%S') + dkey = dt.replace(tzinfo=pytz.UTC) + +# Server https://wateroffice.ec.gc.ca/mainmenu/real_time_data_index_e.html +# is already in UTC time. Timezone conversion is not needed +# +# Zhengtao Cui, 12/1/2023 +# +# tz = s[1][8:] +# tzh = int(tz[1:3]) # time zone hours, *absolute value*; typically 4 or 5 for GL region +# tzm = int(tz[4:]) # time zone minutes, typically 0 for GL region +# tzoff = timedelta(hours=tzh, minutes=tzm) +# +# # +# # Adjust the time appropriately so that it represents +# # the UTC time. +# # +# if (tz[0] == '-'): +# dtu = dtnz + tzoff # this is GL case +# else: +# dtu = dtnz - tzoff + + # + # time zone conversion is not needed + # + #dtu = dtnz + # + # Make it timezone-aware, assigning it to UTC timezone and + # then using that datetime value as the dictionary key + # + #dkey = dtu.replace(tzinfo=pytz.UTC) + except Exception as e: + raise Exception( \ + 'Can not parse date... {}'.format(e) ) from e + + if dkey and discharge >= 0: + valueQC = int(qstr) if qstr else 0 + dataquality = self.calculateDataQuality(discharge, \ + valueQC ) + self.timevalue[dkey] = (discharge, dataquality) + #print(f"\n+++Discharge={discharge}") + else: + self.timevalue[dkey] = (-999999.0, 0) + #print(f" {id_value} - Discharge=-999999.0\n") + + self.stationID = f"CAN.{id_value}" + + #print(f"{self.stationID} {dkey} {discharge} {dataquality}") + + if not self.timevalue: + raise ValueError(f"No parameter 47 (No valid observation); Skipping {csvfilename}\n") + + timekeys = sorted( self.timevalue.keys() ) + + self.obvPeriod = timekeys[0], timekeys[-1] + self.stationName = self.stationID + self.generationTime = timekeys[0] + self.unit = 'm3/s' + + + def calculateDataQuality(self, value, qacode): + """ Calculate a quality code for the discharge measurement based on + the value and associated QA/QC code. The defined QA/QC codes are: + 1 = preliminary, 2 = reviewed, 3 = checked, 4 = approved + """ +######################################################################## +#Date: Fri, 10 Jul 2020 12:49:57 -0600 +#From: James McCreight +#To: Zhengtao Cui +#Cc: Tim Hunter - NOAA Federal , +#Brian Cosgrove - NOAA Federal , +#Arezoo RafieeiNasab , David Gochis , +#Aubrey Dugger , Ryan Cabell , +#Laura Read +#Subject: Re: High priority....Canada streamflow issue +#Parts/Attachments: +#1 OK ~342 lines Text (charset: UTF-8) +#2 Shown ~728 lines Text (charset: UTF-8) +#---------------------------------------- +#So, I would say it depends on our expectations of what the codes really +#mean, but the weight penalties should be thought of as "relative to model +#error" with .5 giving something like equal weight. Generally, the +#observation is going to be much better (there are exceptions) even when it's +#not revised (at least for USGS).... unless this is just not true for these +#gages. +#I might suggest the weighting scheme below in code, based on my best guess. +#Here is my basic thought process: +#Within 28 hours (the extended look back period), we dont get to QC levels 3 +#and 4 because that process likely takes longer. +#So, once we reach 2, we are probably doing as well as can be expected. +#It would be informative to see how much the OPERATIONAL data stream has +#quality 1 and 2 and what the ?tis between obs time and time that that flag +#arrives. +#It might be informative to also see if/how much the discharge values change +#when their quality codes change. +#It is possible that the necessary data/flow is beyond what is currently +#setup, but doing an aggressive saving of the canada gage obs files could +#shed some light. +# +#def calculateDataQuality(self, value, qacode): +# """ Calculate a quality code for the discharge measurement based on +# the value and associated QA/QC code. The defined QA/QC codes are: +# 1 = preliminary, 2 = reviewed, 3 = checked, 4 = approved +# """ +# quality = 0 +# if value <= 0 or value > 9000: +# quality = 0 +# else: +# if qacode == 1: +# quality = 75 +# elif qacode == 2: +# quality = 100 +# elif qacode == 3: +# quality = 100 +# elif qacode == 4: +# quality = 100 +# else: +# quality = 0 +# return quality +#------------------------------------------------------- +#James L. McCreight +#NCAR Research Applications Lab +#office: FL2 2065 +#office phone: 303-497-8404 +#cell: 831-261-5149 +######################################################################## +#Date: Thu, 16 Jul 2020 14:15:12 -0400 +#From: Brian Cosgrove - NOAA Federal +#To: Arezoo RafieeiNasab +#Cc: Zhengtao Cui , +# James McCreight , +# Tim Hunter - NOAA Federal , +# David Gochis , Aubrey Dugger , +# Ryan Cabell , Laura Read +# Subject: Re: High priority....Canada streamflow issue +# Parts/Attachments: +# 1.1 OK 509 lines Text (charset: UTF-8) +# 1.2 Shown ~39 KB Text (charset: UTF-8) +# 2 OK 977 KB Image +#---------------------------------------- +#Okay, barring any further input/concerns from folks, let's do this.... +#Zhengtao, can you alter it so that it gives 100% weight (quality) to the +#incoming obs even if the QC code is only '1'? I don't' think we're seeing +#the desired impact of the obs currently....let's see how it looks after that +#change in real-time. +# +#So this would change to quality = 100 +# +# > if qacode == 1: +# > quality = 75 +# +# Thanks, +# Brian +# + +# 12/01/2023 Zhengtao +# On the new server, https://wateroffice.ec.gc.ca/mainmenu/real_time_data_index_e.html +# the qualifier values are only -1 and 0, so set the quality to 100 +# + quality = 100 +# quality = 0 +# if value <= 0 or value > 9000: +# quality = 0 +# else: +# if qacode == 1: +# quality = 100 +# elif qacode == 2: +# quality = 100 +# elif qacode == 3: +# quality = 100 +# elif qacode == 4: +# quality = 100 +# else: +# quality = 0 + return quality + + def getTimeValueAt(self, at_time, resolution = timedelta() ): + closestTimes = [] + distances = [] + for k in sorted(self.timevalue): + td = abs(k - at_time) + if td <= resolution/2: + closestTimes.append(k) + distances.append(td) + if closestTimes: + closest = [x for y,x in sorted(zip(distances,closestTimes))] [0] + return (closest, self.timevalue[closest]) + else: + return None + + +class All_WSC_Observations: + """Store all WSC data in a given directory""" + def __init__(self, wscdatadir ): + self.source = wscdatadir + self.wscobvs = [] + if not os.path.isdir(wscdatadir): + raise RuntimeError("FATAL ERROR: " + wscdatadir + + " is not a directory or does not exist. ") + + twodaysago = datetime.now() - timedelta( days = 2 ) + for file in os.listdir(wscdatadir): + if file.endswith(".csv"): + fname = wscdatadir + '/' + file + st = os.stat( fname ) + # + # Don't process data older than 2 days. + # + if datetime.fromtimestamp( st.st_mtime) > twodaysago: + # Skip empty files + if st.st_size == 0: + print(f"Skipping empty file: {fname}") + continue + + try: + print(f"\nReading {fname} ...") + self.wscobvs.append(WSC_Observation(fname)) + except Exception as e: + print( "WARNING: Can not parsing CSV file: " + \ + file + ", Because " ) + print( e ) + continue + + if not self.wscobvs: + raise EmptyDirOrFileException( "Input directory " + wscdatadir \ + + " has no Water Survey of Canada CSV files or the " + "Water Survey CSV files contain no flow data!" ) + + self.index = -1 + + self.timePeriod = self.wscobvs[0].obvPeriod + for obv in self.wscobvs: + if self.timePeriod[0] > obv.obvPeriod[0]: + self.timePeriod = ( obv.obvPeriod[0], self.timePeriod[1]) + if self.timePeriod[1] < obv.obvPeriod[1]: + self.timePeriod = ( self.timePeriod[0], obv.obvPeriod[1]) + + + def __iter__(self): + return self + + def next(self): + if self.index == len( self.wscobvs ) - 1: + self.index = -1 + raise StopIteration + self.index = self.index + 1 + return self.wscobvs[self.index] + + def timePeriodForAll(self): + return self.timePeriod + + # + # Step through the observations, finding the entry that + # is closest (time-wise) to the desired timestamp. + # Append that observation to the station_time_value_list. + # Then create a TimeSliceC object from all of the observations in that list. + # + def makeTimeSlice(self, timestamp, timeresolution): + station_time_value_list = [] + for obv in self.wscobvs: + closestObv = obv.getTimeValueAt( timestamp, timeresolution ) + if closestObv: + c0 = closestObv[0] + c10 = closestObv[1][0] + c11 = closestObv[1][1] + station_time_value_list.append((obv.stationID, c0, c10, c11)) + timeSlice = TimeSliceC(timestamp, timeresolution, station_time_value_list) + return timeSlice + + def makeAllTimeSlices(self, timeresolution, outdir): + # + # the time resolutions must divide 60 minutes with no remainder + # + if 3600 % timeresolution.seconds != 0: + raise RuntimeError( "FATAL ERROR: Time slice resolution must " + "divide 60 minutes with no remainder." ) + + # + # Always start two days ago because sometimes the downloaded realtime + # files contain old data older than six months, See the email from NCO + # Simon Hsiao on Dec 16, 2022, with subject + # "20221216 12z nwm_canada_timeslices job hung". + # + # +# NWM team, +# +# The 20221216 12z nwm_canada_timeslices job hung failed seeing hung reached walltime 10 min. here are the job logfiles and working dir for your investigations, rerun still hung - +# /lfs/h1/ops/prod/output/20221216/nwm_canada_timeslices_12_1725.o33696851 -- 1st run +# /lfs/h1/ops/prod/output/20221216/nwm_canada_timeslices_12_1739.o33703661 -- 2nd run +# /lfs/f1/ops/prod/tmp/nwm_canada_timeslices_12_1725.33696851.dbqs01 - 1st run working dir +# /lfs/f1/ops/prod/tmp/nwm_canada_timeslices_12_1739.33703661.dbqs01/ -- 2nd run working dir +# In the working dir,there are some 2022-06-01 15 min WscTimeSlice.ncdf data files as below , why back to 202206 ? +# 2022-06-01_05_00_00.15min.wscTimeSlice.ncdf +# ... +# 2022-06-03_23_00_00.15min.wscTimeSlice.ncdf +# +# Thanks, +# +# /Simon +# SPA Office + + # + # need to use UTC time here + startTime = datetime.now().replace(tzinfo=pytz.UTC) - timedelta( days = 2 ) + + #Round the start time to the nearist 0, 15, 30 and 45 minutes + #Otherwise the filename will be wrong. + + #startTime = self.timePeriod[0] + + if startTime.minute >= 8 and startTime.minute <= 22: + startTime = datetime( startTime.year, startTime.month, + startTime.day, startTime.hour, + 15, tzinfo=startTime.tzinfo ) + elif startTime.minute >= 23 and startTime.minute <= 37: + startTime = datetime( startTime.year, startTime.month, + startTime.day, startTime.hour, + 30, tzinfo=startTime.tzinfo ) + elif startTime.minute >= 38 and startTime.minute <= 52: + startTime = datetime( startTime.year, startTime.month, + startTime.day, startTime.hour, + 45, tzinfo=startTime.tzinfo ) + elif startTime.minute < 8: + startTime = datetime( startTime.year, startTime.month, + startTime.day, startTime.hour, + 0, tzinfo=startTime.tzinfo ) + else: # > 52 + startTime = datetime( startTime.year, startTime.month, + startTime.day, startTime.hour, + 0, tzinfo=startTime.tzinfo ) + \ + timedelta( hours = 24 ) + + #print ("startTime < self.timePeriod[0]",startTime,self.timePeriod[0]); + while startTime < self.timePeriod[0]: + startTime += timeresolution + + #print ("startTime < self.timePeriod[0]",startTime,self.timePeriod[0]); + #print ("startTime > self.timePeriod[0]",startTime,self.timePeriod[1]); + + if startTime > self.timePeriod[1]: + raise RuntimeError("FATAL ERROR: observation time period wrong!") + + count = 0 + print ("Timeslice start time: ", startTime ) + while startTime <= self.timePeriod[1]: + print ("making time slice for ", startTime.isoformat()) + oneSlice = self.makeTimeSlice(startTime, timeresolution) + if ( not oneSlice.isEmpty() ): + updatedOrNew = True + slicefilename = outdir + '/' +oneSlice.getSliceNCFileName() + if os.path.isfile( slicefilename ): + oldslice = TimeSliceC.fromNetCDF( slicefilename ) + updatedOrNew = oneSlice.mergeOld( oldslice ) + if updatedOrNew: + oneSlice.toNetCDF(outdir) + print (oneSlice.getSliceNCFileName() + " updated!") + else: + print (oneSlice.getSliceNCFileName() + " not updated!") + count = count + 1 + + startTime += timeresolution + return count diff --git a/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/make_time_slice_from_canada.py b/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/make_time_slice_from_canada.py new file mode 100644 index 0000000..77f02dc --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/nco_canada/timeslices_scripts/make_time_slice_from_canada.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +import os, sys, time, urllib, getopt +import logging +from string import * +import xml.etree.ElementTree as etree +from datetime import datetime, timedelta +from WSC_Observation import WSC_Observation, All_WSC_Observations +from TimeSliceC import TimeSliceC +from EmptyDirOrFileException import EmptyDirOrFileException + +""" + The driver to parse downloaded hydrologic data files from + Environment and Climate Change Canada (Water Survey of Canada), + and then to create time slices and write to NetCDF files. + Author: Tim Hunter (tim.hunter@noaa.gov) drawing heavily on + the prior work done by Zhengtao Cui for the USGS data. + Date: February 2018 +""" +def main(argv): + """ + function to get input arguments + """ + inputdir = '' + try: + opts, args = getopt.getopt(argv,"hi:o:",["idir=", "odir="]) + except getopt.GetoptError: + print( 'make_time_slice_from_canada.py -i -o ' ) + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print( \ + 'make_time_slice_from_canada.py -i -o ' ) + sys.exit() + elif opt in ('-i', "--idir"): + inputdir = arg + if not os.path.exists( inputdir ): + raise RuntimeError( 'FATAL ERROR: inputdir ' + \ + inputdir + ' does not exist!' ) + elif opt in ('-o', "--odir" ): + outputdir = arg + if not os.path.exists( outputdir ): + raise RuntimeError( 'FATAL ERROR: outputdir ' + \ + outputdir + ' does not exist!' ) + + return (inputdir, outputdir) + +logging.basicConfig(format=\ + '%(asctime)s - %(name)s - %(levelname)s - %(message)s',\ + level=logging.INFO) +logger = logging.getLogger(__name__) +formatter = logging.Formatter(\ + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +#logger.setFormatter(formatter) +logger.info( "System Path: " + str( sys.path ) ) + +if __name__ == "__main__": + odir = main(sys.argv[1:]) + +indir = odir[0] +outdir = odir[1] + +# +# Load discharge data as downloaded from ECCC datamart. Current URL +# for a parent directory is http://dd.weather.gc.ca/hydrometric/csv/ +# Files are found further down that tree. +# +try: + allobvs = All_WSC_Observations( indir ) +except EmptyDirOrFileException as e: + logger.warning( str(e), exc_info=True) + sys.exit(0) +except Exception as e: + logger.error("Failed to load Canadian CSV files: " + str(e), exc_info=True) + sys.exit(3) + +print( 'earliest time: ', allobvs.timePeriodForAll()[0] ) +print( 'latest time: ', allobvs.timePeriodForAll()[1] ) + +# +# Create time slices from loaded observations +# +# Set time resolution to 15 minutes +# and +# Write time slices to NetCDF files +# +try: + timeslices = allobvs.makeAllTimeSlices( timedelta( minutes = 15 ), outdir ) +except Exception as e: + logger.error("Failed to make time slices: " + str(e) , exc_info=True) + logger.error("Input dir = " + indir, exc_info=True) + sys.exit(3) + +print( "total number of timeslices: ", timeslices ) diff --git a/data_assimilation_engine/Streamflow_Scripts/requirements.txt b/data_assimilation_engine/Streamflow_Scripts/requirements.txt new file mode 100644 index 0000000..7dbf266 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/requirements.txt @@ -0,0 +1,3 @@ +netcdf4 +python-dateutil +pytz diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/EmptyDirOrFileException.py b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/EmptyDirOrFileException.py new file mode 100755 index 0000000..82f182b --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/EmptyDirOrFileException.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python +class EmptyDirOrFileException( Exception ): + """Exception raised for empty input files or directories. + Attributes: + expression -- input expression in which the error occurred + message -- explanation of the error + """ + pass + +# def __init__(self, message): +# self.expression = expression +# self.message = message diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/Observation.py b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/Observation.py new file mode 100755 index 0000000..100e9f0 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/Observation.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python +############################################################################### +# Module name: Observation +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: 5/24/2019 # +# # +# Last modification date: +# # +# Description: Abstract the observed real-time stream flow +# # +############################################################################### + +import os, logging +from string import * +from datetime import datetime, timedelta +import dateutil.parser +import pytz +from abc import ABCMeta, abstractmethod, abstractproperty +from TimeSlice import TimeSlice + + +class Observation: + """ + Abstract real-time flow time series. + """ + __metaclass__ = ABCMeta + +# @abstractproperty +# def source(self): +# pass +# +# @abstractproperty +# def stationID(self): +# pass +# +# @abstractproperty +# def stationName(self): +# pass +# +# @abstractproperty +# def obvPeriod(self): +# pass +# +# @abstractproperty +# def unit(self): +# pass +# +# @abstractproperty +# def timeValueQuality(self): +# pass + + @property + def source(self): + return self._source + + @source.setter + def source(self, s): + self._source = s + + @property + def stationID(self): + return self._stationID + + @stationID.setter + def stationID(self, s): + self._stationID = s + + @property + def stationName(self): + return self._stationName + + @stationName.setter + def stationName(self, s): + self._stationName = s + + @property + def obvPeriod(self): + return self._obvPeriod + + @obvPeriod.setter + def obvPeriod(self, p): + self._obvPeriod = p + + @property + def unit(self): + return self._unit + + @unit.setter + def unit(self, u): + self._unit = u + + @property + def timeValueQuality(self): + return self._timeValueQuality + + @timeValueQuality.setter + def timeValueQuality(self, tvq): + self._timeValueQuality = tvq + + @abstractmethod + def __init__(self, filename ): + pass + + + def getTimeValueAt(self, at_time, resolution = timedelta() ): + """ + Get the closest time-value pair for a given time + + Input: at_time - the given time + resolution - the tolerance time period around the + given time + + Return: Tuple of a time-value pair + """ + + closestTimes = [] + distances = [] + if at_time in self.timeValueQuality: + return ( at_time, self.timeValueQuality.get( at_time ) ) +# for k in sorted( self.timeValueQuality ): + for k in self.timeValueQuality: + if ( abs( k - at_time ) <= resolution / 2 ): + closestTimes.append( k ) + distances.append( abs( k - at_time ) ) + if not closestTimes: + return None + else: + closest = [ x for y, x in \ + sorted( zip( distances, closestTimes ) ) ][ 0 ] + + return ( closest, self.timeValueQuality.get( closest ) ) + + +class All_Observations: + "Store all obvserved data" + def __init__(self, obvs ): + """ + Initialize the All_Objections object for a given + Observation + + Input: list of Observation object + """ + self.logger = logging.getLogger(__name__) + self.observations = obvs + + if not self.observations: + raise RuntimeError( "FATAL ERROR: has no data") + + self.index = -1 + + self.timePeriod = self.observations[0].obvPeriod + for obv in self.observations: + if self.timePeriod[0] > obv.obvPeriod[ 0 ]: + self.timePeriod = ( obv.obvPeriod[ 0 ], \ + self.timePeriod[ 1 ]) + + if self.timePeriod[1] < obv.obvPeriod[ 1 ]: + self.timePeriod = ( self.timePeriod[ 0 ], \ + obv.obvPeriod[ 1 ] ) + + def __iter__(self ): + """ + The iterator + """ + return self + + def __next__( self ): + """ + The next Observation object + """ + if self.index == len( self.observations ) - 1: + self.index = -1 + raise StopIteration + self.index = self.index + 1 + return self.observations[ self.index ] + + def timePeriodForAll( self ): + """ + Get the earlist and latest time for all observations + + Return: Tuple of start time and end time + """ + return self.timePeriod + + def makeTimeSlice( self, timestamp, timeresolution ): + """ + Create one time slice for a given time and resolution + + Input: timestamp - the given time + timeresolution - the resolution + + Return: A time slice object + """ + station_time_value_list = [] + for obv in self: + closestObv = obv.getTimeValueAt( timestamp, timeresolution ) + if closestObv: + station_time_value_list.append( \ + ( obv.stationID, closestObv[ 0 ], \ + # value quality + closestObv[ 1 ][ 0 ], closestObv[ 1 ][ 1 ] ) ) + +# Tracer.theTracer.run( 'timeSlice = TimeSlice( timestamp, timeresolution, station_time_value_list )' ) + timeSlice = TimeSlice( timestamp, \ + timeresolution, \ + station_time_value_list ) + return timeSlice + + def makeAllTimeSlices( self, timeresolution, outdir, suffix='usgsTimeSlice.ncdf' ): + """ + Create the time slice NetCDF files for all USGS observations + + Input: timeresolution - resolution + outdir - the output directory + + Return: Total number of time slice files created + """ + + # the time resultions must divide 60 minutes with on remainder + if 3600 % timeresolution.seconds != 0: + raise RuntimeError( "FATAL ERROR: Time slice resolution must " + "divide 60 minutes with no remainder." ) + + startTime = datetime( self.timePeriod[ 0 ].year, + self.timePeriod[ 0 ].month, + self.timePeriod[ 0 ].day, + self.timePeriod[ 0 ].hour ) + + while startTime < self.timePeriod[ 0 ]: + startTime += timeresolution + + if startTime > self.timePeriod[ 1 ]: + raise RuntimeError( \ + "FATAL ERROR: observation time period wrong! " ) + + count = 0 + while startTime <= self.timePeriod[ 1 ]: + self.logger.info( "making time slice for " + \ + startTime.isoformat() ) +# Tracer.theTracer.run( \ +# 'oneSlice = makeTimeSlice( startTime, timeresolution )' ) + + oneSlice = self.makeTimeSlice( startTime, timeresolution ) + self.logger.info( "Time slice: " + \ + outdir + '/' +oneSlice.getSliceNCFileName( suffix ) ) + if ( not oneSlice.isEmpty() ): + updatedOrNew = True + slicefilename = outdir + '/' +oneSlice.getSliceNCFileName( suffix ) + if os.path.isfile( slicefilename ): +# Tracer.theTracer.run( \ +# 'oldslice = TimeSlice.fromNetCDF( slicefilename )' ) + oldslice = TimeSlice.fromNetCDF( slicefilename ) + updatedOrNew = oneSlice.mergeOld( oldslice ) + + if updatedOrNew: + oneSlice.toNetCDF( outdir, suffix ) + self.logger.info( oneSlice.getSliceNCFileName( suffix ) + \ + " updated!" ) + else: + self.logger.info( oneSlice.getSliceNCFileName( suffix ) + \ + " not updated!" ) +# oneSlice.print_station_time_value() + count = count + 1 + + startTime += timeresolution + +# for eachSlice in allTimeSlices: +# print "-----------------------------" +# eachSlice.print_station_time_value() +# print eachSlice.getSliceNCFileName() +# +# return allTimeSlices + return count diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/TimeSlice.py b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/TimeSlice.py new file mode 100755 index 0000000..ce186c8 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/TimeSlice.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python +############################################################################### +# Module name: TimeSlice # +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: # +# # +# Last modification date: 7/12/2017 # +# # +# Description: manage a time slice file that contains real-time stream # +# flow data for all USGS stations for a given time stamp # +# # +############################################################################### +import os, sys, time, math +from string import * +from datetime import datetime, timedelta +import calendar +#import xml.utils.iso8601 +#from netCDF4 import Dataset +import netCDF4 +import numpy as np + +# +# Check if two variables are considered equal. +# +# Input: a - one of the two variables to be compared +# b - one of the two variables to be compared +# rel_tol - relative tolerance +# abs_tol - absolute tolerance +# +# Return: boolean +# +def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): + ''' + Python 2 implementation of Python 3.5 math.isclose() + https://hg.python.org/cpython/file/tip/Modules/mathmodule.c#l1993 + ''' + # sanity check on the inputs + if rel_tol < 0 or abs_tol < 0: + raise ValueError("tolerances must be non-negative") + + # short circuit exact equality -- needed to catch two infinities of + # the same sign. And perhaps speeds things up a bit sometimes. + if a == b: + return True + + # This catches the case of two infinities of opposite sign, or + # one infinity and one finite number. Two infinities of opposite + # sign would otherwise have an infinite relative tolerance. + # Two infinities of the same sign are caught by the equality check + # above. + if math.isinf(a) or math.isinf(b): + return False + + # now do the regular computation + # this is essentially the "weak" test from the Boost library + diff = math.fabs(b - a) + result = (((diff <= math.fabs(rel_tol * b)) or + (diff <= math.fabs(rel_tol * a))) or + (diff <= abs_tol)) + return result + +# +# Compare two Python dictionary objects +# +# Input: d1 - one of the two dictionary object to be compared +# d2 - one of the two dictionary object to be compared +# +# Return: Tuple of added element set, removed element set, modified element +# set and the same element set, +# +def dict_compare(d1, d2): + d1_keys = set(d1.keys()) + d2_keys = set(d2.keys()) + intersect_keys = d1_keys.intersection(d2_keys) + added = d1_keys - d2_keys + removed = d2_keys - d1_keys + modified = dict() + for o in intersect_keys: + if d1[o][0] != d2[o][0] or \ + not isclose( d1[o][1], d2[o][1],abs_tol=0.01 ) \ + or not isclose( d1[o][2], d2[o][2]): + modified[o] = (d1[o], d2[o] ) + same = set(o for o in intersect_keys if d1[o] == d2[o]) + return added, removed, modified, same + +class TimeSlice: + """ + Description: Store one time slice data + Author: Zhengtao Cui (Zhengtao.Cui@noaa.gov) + Date: Aug. 26, 2015 + Date modified: Oct. 20, 2015, Fixed bugs in mergeOld. + """ + + stationIdStrLen = 15 + stationIdLong_name = "USGS station identifer of length 15" + timeStrLen = 19 + timeUnit = "UTC" + timeLong_name = "YYYY-MM-DD_HH:mm:ss UTC" + dischargeUnit = "m^3/s" + dischargeLong_name = "Discharge.cubic_meters_per_second" + dischargeQualityUnit = "-" + dischargeQualaityLong_name = \ + "Discharge quality 0 to 100 to be scaled by 100." + + def __init__(self, time_stamp, resolution, station_time_value ): + """ + Initialize a TimeSlice object + Input: time_stamp - a time stamp + resolution - time resolution of the time slices + station_time_value - Tuple of (station, time, flow, + quality) + """ + self.centralTimeStamp = time_stamp + self.sliceTimeResolution = resolution + self.obvStationTimeValue = station_time_value + + def isEmpty( self ): + """ + Test if the time slice is empty + Return: boolean + """ + return not self.obvStationTimeValue + + def print_station_time_value( self ): + """ + Print the time slice data + """ + for e in self.obvStationTimeValue: + print( "Slice: central time: ", \ + self.centralTimeStamp.isoformat(), \ + e[ 0 ], e[ 1 ].isoformat(), e[ 2 ] ) + + def getStationIDs( self ): + """ + Get all station ids of the time slices + Return: list of station ids + """ + stationL = [] + for e in self.obvStationTimeValue: + stationL.append( list( e[ 0 ] ) ) + #stationL.append( e[ 0 ][5:] ) + for s in stationL: + if len(s) < self.stationIdStrLen: + for i in range( len(s), self.stationIdStrLen ): + s.insert(0, ' ' ) + elif len(s) > self.stationIdStrLen: + s = s[0:self.stationIdStrLen - 1] + + return stationL + + def getDischargeValues( self ): + """ + Get all stream flow values of the time slice + Return: list of flow values + """ + values = [] + for e in self.obvStationTimeValue: + values.append( e[ 2 ] ) + return values + + def getDischargeTimes( self ): + """ + Get all observation times of the time slice + Return: list of observation times + """ + obvtimes = [] + for e in self.obvStationTimeValue: + obvtimes.append( \ + list( e[ 1 ].strftime( "%Y-%m-%d_%H:%M:00" ) ) ) + return obvtimes + + def getQueryTimes( self ): + """ + Get all query times of the time slice + Return: list of query times + """ + qtimes = [] + for e in self.obvStationTimeValue: + # print 'getQueryTime: ', e[ 1 ].isoformat() + # print 'getQueryTime: ', e[ 1 ].utctimetuple() + # print 'getQueryTime: ', time.mktime( e[ 1 ].utctimetuple() ) + #qtimes.append( time.mktime( e[ 1 ].utctimetuple() ) ) + qtimes.append( calendar.timegm( e[ 1 ].utctimetuple() ) ) + return qtimes + + def getSliceNCFileName( self, suffix='usgsTimeSlice.ncdf' ): + """ + Get NetCDF file for this time slice + Return: A NetCDF filename + """ + filename = \ + self.centralTimeStamp.strftime( "%Y-%m-%d_%H:%M:00." ) + \ + str( int( self.sliceTimeResolution.days * 24 * 60 + \ + self.sliceTimeResolution.seconds // 60 ) ).zfill(2) + \ + "min." + suffix + return filename + + def getDischargeQuality( self ): + """ + Get discharge quality for this time slice + Return: A list of discharge quality + """ + dq = [] + for e in self.obvStationTimeValue: + dq.append( e[ 3 ] ) + return dq + + def toNetCDF( self, outputdir = './', suffix='usgsTimeSlice.ncdf' ): + """ + Write the time slice to a NetCDF file + Input: outputdir - the directory where to write the NetCDF + """ + + nc_fid = netCDF4.Dataset( \ + outputdir + '/' + self.getSliceNCFileName( suffix ), \ + 'w', format='NETCDF4' ) + nc_fid.createDimension( 'stationIdStrLen', self.stationIdStrLen ) + nc_fid.createDimension( 'stationIdInd', None ) + nc_fid.createDimension( 'timeStrLen', self.timeStrLen ) + stationId = nc_fid.createVariable( 'stationId', 'S1',\ + ('stationIdInd', 'stationIdStrLen') ) + stationId.setncatts( {'long_name' : \ + self.stationIdLong_name, \ + 'units' : '-'} ) + + time = nc_fid.createVariable( 'time', 'S1',\ + ('stationIdInd', 'timeStrLen' ) ) + time.setncatts( {'long_name' : self.timeLong_name, \ + 'units' : self.timeUnit} ) + + discharge = nc_fid.createVariable( 'discharge', 'f4',\ + ('stationIdInd', ) ) + discharge.setncatts( {'long_name' : \ + self.dischargeLong_name, \ + 'units' : self.dischargeUnit} ) + discharge_quality = \ + nc_fid.createVariable( 'discharge_quality', 'i2',\ + ('stationIdInd', ) ) + discharge_quality.setncatts( {'long_name' : \ + self.dischargeQualaityLong_name, \ + 'units' : self.dischargeQualityUnit, \ + 'multfactor' : '0.01' } ) + queryTime = nc_fid.createVariable( 'queryTime', 'i4',\ + ('stationIdInd', ) ) + + queryTime.setncatts( { 'units' : \ + 'seconds since 1970-01-01 00:00:00 local TZ' \ + } ) + + nc_fid.setncatts( { 'fileUpdateTimeUTC': \ + datetime.utcnow().strftime( "%Y-%m-%d_%H:%M:00" ), \ + 'sliceCenterTimeUTC' : \ + self.centralTimeStamp.strftime( "%Y-%m-%d_%H:%M:00" ),\ + 'sliceTimeResolutionMinutes' : \ + str( int( self.sliceTimeResolution.days * 24 * 60 + \ + self.sliceTimeResolution.seconds // 60 ) ).zfill(2) } ) + + #print 'discharge: ', self.getDischargeValues() + discharge[ : ] = self.getDischargeValues() + queryTime[ : ] = self.getQueryTimes() + + #stationId[ : ] = netCDF4.stringtochar( \ + # np.array( stations ) ) + stations = self.getStationIDs() + stationId[ : ] = stations + # time[ : ] = \ + # [ list( self.centralTimeStamp.strftime( "%Y-%m-%d_%H:%M:00" ) ) \ + # for i in range( len( stations ) ) ] + + time[ : ] = self.getDischargeTimes() + discharge_quality[:] = self.getDischargeQuality() + + nc_fid.close() + + @classmethod + def fromNetCDF( self, ncfilename ): + """ + Read the time slice from a NetCDF file + Input: ncfilename - the NetCDF filename + """ + nc_fid = netCDF4.Dataset( ncfilename, 'r' ) + timestamp = datetime.strptime( \ + nc_fid.getncattr( 'sliceCenterTimeUTC' ), \ + "%Y-%m-%d_%H:%M:00" ) + #print timestamp.isoformat() + + time_resol = timedelta( minutes = \ + int( nc_fid.getncattr( 'sliceTimeResolutionMinutes' ) ) ) + + stations = netCDF4.chartostring( \ + nc_fid.variables[ 'stationId'][ : ] ) + + discharge = nc_fid.variables[ 'discharge'][ : ] + + queryTime = nc_fid.variables[ 'queryTime'][ : ] + + quality = nc_fid.variables[ 'discharge_quality'][ : ] + +# for s, d, q in zip( stations, discharge, queryTime ): +# print 'USGS.' + s.rstrip(), d, \ +# datetime.utcfromtimestamp( q ).isoformat(), \ +# datetime.utcfromtimestamp( q ).tzname(), \ +# q + + +# self.centralTimeStamp = timestamp +# self.sliceTimeResolution = time_resol +# self.obvStationTimeValue = [] + stationTimeValue = [] + for s, d, q, qual in zip( stations, discharge, queryTime, quality ): + stationTimeValue.append( \ + ( s.strip(), \ + datetime.utcfromtimestamp( q ), d, qual ) ) + + nc_fid.close() + + return self( timestamp, time_resol, stationTimeValue ) + + def mergeOld( self, oldTimeSlice ): + """ + Merge data in an existing NetCDF time slice file with this one + Input: oldTimeSlice - the NetCDF filename of the existing time + slice + """ + + if self.centralTimeStamp != oldTimeSlice.centralTimeStamp or \ + self.sliceTimeResolution != oldTimeSlice.sliceTimeResolution: + raise RuntimeError( "FATAL ERROR: the two time slices "\ + " differ, not merging ..." ) + else: + + site_time_value = dict() + for e in self.obvStationTimeValue: + #print "New : ", e + site_time_value[ e[ 0 ] ] = ( e[1], e[2], e[3] ) + old_site_time_value = dict() + + for e in oldTimeSlice.obvStationTimeValue: + #print "Old : ", e + old_site_time_value[ e[ 0 ] ] = ( e[1], e[2], e[3] ) + + added, removed, modified, same = dict_compare( \ + site_time_value, old_site_time_value ) + + #if not added and not removed and not modified \ + # and len(same) == len( self.obvStationTimeValue ): + if not added and not modified : + return False + + old_site_time_value.update( site_time_value ) + + self.obvStationTimeValue = [] + + for site in old_site_time_value: + self.obvStationTimeValue.append( ( site, \ + old_site_time_value[ site ][ 0 ], \ + old_site_time_value[ site ][ 1 ], \ + old_site_time_value[ site ][ 2 ] ) ) + + return True + #print "TimeSlice: merged old time slice!" + #print self.obvStationTimeValue diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/USGS_Observation.py b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/USGS_Observation.py new file mode 100755 index 0000000..bf59b79 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/USGS_Observation.py @@ -0,0 +1,632 @@ +#!/usr/bin/env python +############################################################################### +# Module name: USGS_Observation # +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: # +# # +# Last modification date: 12/20/2017 # +# # +# Description: manage data in a USGS WaterXML 2.0 file # +# # +# 06/11/2024 ChamP - Added loadJSON() function to decode the USGS Json file # +# 02/2026 ChamP - Added parseJson() to use new USGS API json structure. # +# # +############################################################################### + +import os, sys, time, csv +import logging +from string import * +from datetime import datetime, timedelta +import dateutil.parser +import pytz +import xml.etree.ElementTree as etree +import json +from TimeSlice import TimeSlice +from Observation import Observation + +logger = logging.getLogger(__name__) + +class USGS_Observation(Observation): + """ + Store one USGS Water data APIs + """ + def __init__(self, waterdatafilename ): + """ + Initialize the USGS_Observation object with a given + filename + """ + self.source = waterdatafilename + self.obvPeriod = None + self.stationID = None + self.timeValueQuality = {} + + try: + if waterdatafilename.endswith( '.json' ): + self.parseJson( waterdatafilename ) + else: + logger.warning("Skipping unknown file type: %s", waterdatafilename) + return None + + + except Exception as e: + logger.warning( + "Unexpected error parsing %s: %s", + waterdatafilename,e,exc_info=True) + return None + + def parseJson(self, jsonfilename): + """ + Read real-time stream flow data from a given Json file + by parsing the JSON file + + Input: jsonfilename - the Json filename + """ + + try: + logger.info("Parsing file: %s",jsonfilename) + with open(jsonfilename, "r") as f: + data = json.load(f) + + # --------------------------------------------------------- + # Get features and handle empty features[] + # --------------------------------------------------------- + features = data.get("features", []) + if not features: + logger.warning("No features found in %s, skipping it...\n\n",jsonfilename) + return None + + # Begin and end times + beginTime = features[0]["properties"]["time"] + endTime = features[-1]["properties"]["time"] + + self._obvPeriod = dateutil.parser.parse( beginTime )\ + .astimezone(pytz.utc).replace(tzinfo=None), \ + dateutil.parser.parse( endTime )\ + .astimezone(pytz.utc).replace(tzinfo=None) + + # --------------------------------------------------------- + # Process each feature + # --------------------------------------------------------- + self._timeValueQuality = {} + MISSING_VALUE = -999999.0 + + for feature in features: + props = feature.get("properties", {}) + #print (f"features={props}") + + monitoring_location_id = props.get("monitoring_location_id") + #print (f"monitoring_location_id={monitoring_location_id}") + + self._stationID = monitoring_location_id.replace("USGS-", "") + #print (f"stationID={self._stationID}") + + valuetime = props.get("time") + #print (f"time={valuetime}") + + self._unit = props.get("unit_of_measure") + #print (f"unit={self._unit}") + + if self._unit == "ft^3/s": + unitConvertToM3perSec = 0.028317 + elif self._unit == "m^3/s": + unitConvertToM3perSec = 1.0 + else: + raise RuntimeError("FATAL ERROR: Unit " + self._unit + " is not known.") + + qualifier1 = props.get("approval_status") + + qualifiers = props.get("qualifier") or [] + qualifier2 = qualifiers[0] if qualifiers else None + #qualifier2 = qualifiers[1] if len(qualifiers) > 1 else None + #print (f"qualifier1={qualifier1}; qualifier2={qualifier2}") + + value = props.get("value") + dt = dateutil.parser.parse( valuetime ).astimezone(pytz.utc).replace(tzinfo=None) + + if (value is None + or (isinstance(value, str) and value.strip().lower() == "null")): + rvalue = MISSING_VALUE + dataquality = 0 + + else: + rvalue = float(value) * unitConvertToM3perSec + dataquality = self.calculateDataQuality(\ + rvalue,\ + qualifier1, qualifier2 ) + + # If no timestep exists yet, store it + if dt not in self._timeValueQuality: + self._timeValueQuality[ dt ] = (rvalue, dataquality) + else: + old_v = self._timeValueQuality[dt][0] + + # Replace only if existing value is missing + if old_v == MISSING_VALUE and rvalue != MISSING_VALUE: + self._timeValueQuality[ dt ] = ( rvalue, dataquality ) + + logger.info ("Parsed JSON: obvPeriod=%s, ID=%s, unit=%s, NumberTimeSteps=%d",self._obvPeriod,self._stationID,self._unit,len(self._timeValueQuality)) + for vt, (discharge, quality) in self._timeValueQuality.items(): + logger.info("%s %.6f %d", vt, float(discharge), int(quality)) + logger.info("================================\n") + + self.stationID = self._stationID + self.obvPeriod = self._obvPeriod + self.timeValueQuality = self._timeValueQuality + + except (OSError, json.JSONDecodeError) as e: + logger.warning("JSON parse error in %s: %s. Or maybe json file is empty; ==> %s Skipping...\n",\ + jsonfilename, e, jsonfilename) + return None + + + def loadJSON(self, jsonfilename ): + """ + Read real-time stream flow data from a given Json file + by parsing the JSON file + + Input: jsonfilename - the Json filename + """ + + try: + json_file = open(jsonfilename) + + # Reads .json file to Python dictionary + json_data = json.load(json_file) + + # gets data information on the USGS time series available + timeseries = json_data['value']['timeSeries'][0] + + self._stationName = timeseries['sourceInfo']['siteName'] + self._stationID = timeseries['sourceInfo']['siteCode'][0]['value'] + self._unit = timeseries['variable']['unit']['unitCode'] + + if self._unit == 'ft3/s': + unitConvertToM3perSec = 0.028317 + elif self._unit == 'm3/s': + unitConvertToM3perSec = 1.0 + else: + raise RuntimeError( "FATAL ERROR: Unit " + self._unit + \ + " is not known. ") + self._unit = 'm3/s' + + # Get Discharge timeseries values + for tsIndex in range(len(timeseries['values'])): + tsVals = timeseries['values'][tsIndex]['value'] + lengthTS = len(tsVals) + + if lengthTS > 0: + self._timeValueQuality = dict() + + qualifier1 = timeseries['values'][tsIndex]['qualifier'][0]['qualifierCode'] + + beginTime = timeseries['values'][tsIndex]['value'][0]['dateTime'] + endTime = timeseries['values'][tsIndex]['value'][lengthTS-1]['dateTime'] + + self._obvPeriod = dateutil.parser.parse( beginTime )\ + .astimezone(pytz.utc).replace(tzinfo=None), \ + dateutil.parser.parse( endTime )\ + .astimezone(pytz.utc).replace(tzinfo=None) + + for point in range(0, lengthTS): + valuetime = timeseries['values'][tsIndex]['value'][point]['dateTime'] + + if valuetime and not valuetime.isspace(): + value = timeseries['values'][tsIndex]['value'][point]['value'] + + qualifiers = timeseries['values'][tsIndex]['value'][point]['qualifiers'] + + qualifier2 = '' + + if len(qualifiers) > 0: + qualifier1 = timeseries['values'][tsIndex]['value'][point]['qualifiers'][0] + if len(qualifiers) > 1: + qualifier2 = timeseries['values'][tsIndex]['value'][point]['qualifiers'][1] + + if value == '-999999': + self._timeValueQuality[ dateutil.parser.parse( valuetime )\ + .astimezone(pytz.utc).replace(tzinfo=None) ] = \ + ( -999999.0, 0) # 0 is the quality + else: + dataquality = self.calculateDataQuality(\ + float( value ) * unitConvertToM3perSec,\ + qualifier1, qualifier2 ) + #logging.info( "qualifier1=%s qualifier2=%s dataquality=%s",qualifier1,qualifier2,dataquality ) + self._timeValueQuality[ dateutil.parser.parse( valuetime )\ + .astimezone(pytz.utc).replace(tzinfo=None) ] = \ + ( float( value ) * unitConvertToM3perSec, dataquality ) + + + self.obvPeriod = self._obvPeriod + self.stationName = self._stationName + self.stationID = self._stationID + self.unit = self._unit + self.timeValueQuality = self._timeValueQuality + + except Exception as e: + raise RuntimeError( "WARNING: parsing JSON error: " + str( e )\ + + ": " + jsonfilename + " skipping ..." ) + + + + def loadWaterML2(self, waterml2filename ): + """ + Read real-time stream flow data from a given WaterML 2.0 file + by parsing the WaterML 2.0 file + + Input: waterml2filename - the WaterML 2.0 filename + """ + try: + obvwml = etree.parse( waterml2filename ) + root= obvwml.getroot() + identifier = root.find(\ + '{http://www.opengis.net/gml/3.2}identifier') + #remove 'USGS.' from stationID + self._stationID = identifier.text[5:] + + self._stationName = \ + root.find('{http://www.opengis.net/gml/3.2}name').text + self._generationTime = dateutil.parser.parse(\ + root.find('{http://www.opengis.net/waterml/2.0}metadata')\ + .find('{http://www.opengis.net/waterml/2.0}DocumentMetadata')\ + .find('{http://www.opengis.net/waterml/2.0}generationDate')\ + .text ).astimezone(pytz.utc).replace(tzinfo=None) + timePeriod = root.find(\ + '{http://www.opengis.net/waterml/2.0}observationMember')\ + .find('{http://www.opengis.net/om/2.0}OM_Observation')\ + .find( '{http://www.opengis.net/om/2.0}phenomenonTime')\ + .find('{http://www.opengis.net/gml/3.2}TimePeriod') + beginTime = timePeriod.find(\ + '{http://www.opengis.net/gml/3.2}beginPosition').text + endTime = timePeriod.find(\ + '{http://www.opengis.net/gml/3.2}endPosition').text + self._obvPeriod = dateutil.parser.parse( beginTime )\ + .astimezone(pytz.utc).replace(tzinfo=None), \ + dateutil.parser.parse( endTime )\ + .astimezone(pytz.utc).replace(tzinfo=None) + + uom = root.find(\ + '{http://www.opengis.net/waterml/2.0}observationMember')\ + .find('{http://www.opengis.net/om/2.0}OM_Observation')\ + .find('{http://www.opengis.net/om/2.0}result')\ + .find('{http://www.opengis.net/waterml/2.0}MeasurementTimeseries')\ + .find('{http://www.opengis.net/waterml/2.0}defaultPointMetadata')\ + .find('{http://www.opengis.net/waterml/2.0}DefaultTVPMeasurementMetadata')\ + .find('{http://www.opengis.net/waterml/2.0}uom') + + self._unit = uom.get( '{http://www.w3.org/1999/xlink}title') + + if self._unit == 'ft3/s': + unitConvertToM3perSec = 0.028317 + elif self._unit == 'm3/s': + unitConvertToM3perSec = 1.0 + else: + raise RuntimeError( "FATAL ERROR: Unit " + self._unit + \ + " is not known. ") + + self._unit = 'm3/s' + + measurementTS = root.find(\ + '{http://www.opengis.net/waterml/2.0}observationMember')\ + .find('{http://www.opengis.net/om/2.0}OM_Observation')\ + .find('{http://www.opengis.net/om/2.0}result')\ + .find('{http://www.opengis.net/waterml/2.0}MeasurementTimeseries') + + qualifier1 = measurementTS.find( \ + '{http://www.opengis.net/waterml/2.0}defaultPointMetadata')\ + .find( \ + '{http://www.opengis.net/waterml/2.0}DefaultTVPMeasurementMetadata')\ + .find( \ + '{http://www.opengis.net/waterml/2.0}qualifier')\ + .find( \ + '{http://www.opengis.net/swe/2.0}Category')\ + .find( \ + '{http://www.opengis.net/swe/2.0}value').text + + self._timeValueQuality = dict() + + for point in measurementTS.findall(\ + '{http://www.opengis.net/waterml/2.0}point'): + valuetime = point.find(\ + '{http://www.opengis.net/waterml/2.0}MeasurementTVP')\ + .find('{http://www.opengis.net/waterml/2.0}time') + if valuetime is not None: + value = point.find(\ + '{http://www.opengis.net/waterml/2.0}MeasurementTVP')\ + .find('{http://www.opengis.net/waterml/2.0}value') + + pointmetadata = point.find(\ + '{http://www.opengis.net/waterml/2.0}MeasurementTVP')\ + .find('{http://www.opengis.net/waterml/2.0}metadata') + + qualifier2 = '' + if pointmetadata is not None: + measurementMetadataQualifiers = pointmetadata.find(\ + '{http://www.opengis.net/waterml/2.0}TVPMeasurementMetadata')\ + .findall('{http://www.opengis.net/waterml/2.0}qualifier') + + if len( measurementMetadataQualifiers ) > 0: + qualifier1 = measurementMetadataQualifiers[ 0 ].find(\ + '{http://www.opengis.net/swe/2.0}Category')\ + .find( \ + '{http://www.opengis.net/swe/2.0}value').text + + if len( measurementMetadataQualifiers ) > 1: + qualifier2 = measurementMetadataQualifiers[ 1 ].find(\ + '{http://www.opengis.net/swe/2.0}Category')\ + .find( \ + '{http://www.opengis.net/swe/2.0}value').text + + isValueNull = value.get(\ + '{http://www.w3.org/2001/XMLSchema-instance}nil') + + if isValueNull: + if isValueNull == "true": + self._timeValueQuality[ dateutil.parser.parse( valuetime.text )\ + .astimezone(pytz.utc).replace(tzinfo=None)] = \ + ( -999999.0, 0) # 0 is the quality + else: + + dataquality = self.calculateDataQuality(\ + float( value.text ) * unitConvertToM3perSec,\ + qualifier1, qualifier2 ) + + #logging.info( "111: qualifier1=%s qualifier2=%s dataquality=%s",qualifier1,qualifier2,dataquality ) + self._timeValueQuality[ dateutil.parser.parse( valuetime.text )\ + .astimezone(pytz.utc).replace(tzinfo=None)] = \ + ( float( value.text ) * unitConvertToM3perSec,\ + dataquality ) + else: + + dataquality = self.calculateDataQuality(\ + float( value.text ) * unitConvertToM3perSec,\ + qualifier1, qualifier2 ) + #logging.info( "222: qualifier1=%s qualifier2=%s dataquality=%s",qualifier1,qualifier2,dataquality ) + self._timeValueQuality[ dateutil.parser.parse( valuetime.text )\ + .astimezone(pytz.utc).replace(tzinfo=None)] = \ + ( float( value.text ) * unitConvertToM3perSec,\ + dataquality ) + + except Exception as e: + raise RuntimeError( "WARNING: parsing water XML error: " + str( e )\ + + ": " + waterml2filename + " skipping ..." ) + + self.obvPeriod = self._obvPeriod + self.stationName = self._stationName + self.stationID = self._stationID + self.unit = self._unit + self.timeValueQuality = self._timeValueQuality + + def loadCSV(self, csvfilename ): + """ + Read real-time stream flow data from a given CSV file + by parsing the CSV file + + Input: csvfilename - the CSV filename + """ + + self._timeValueQuality = dict() + with open( csvfilename ) as csvfile: + reader = csv.DictReader( csvfile ) + for row in reader: + dischstr = row['X_00060_00011'] + discharge = float( dischstr ) * 0.028317 # cfs to cms + if row['X_00060_00011'] != '-999999' : + + qualifiers = row['X_00060_00011_cd'].split() + qualifier1 = qualifiers[ 0 ] + if len( qualifiers ) > 1 : + qualifier2 = qualifiers[ 1 ] + else: + qualifier2 = '' + + dataquality = self.calculateDataQuality( \ + discharge, qualifier1, qualifier2 ) + self._timeValueQuality[ datetime.strptime( \ + row[ 'dateTime' ] + ' ' + row[ 'tz_cd' ], \ + '%Y-%m-%d %H:%M:%S %Z' ) ] = \ + ( discharge, dataquality ) + else: + self._timeValueQuality[ datetime.strptime( \ + row[ 'dateTime' ] + ' ' + row[ 'tz_cd' ], \ + '%Y-%m-%d %H:%M:%S %Z' ) ] = \ + ( -999999.0, 0 ) + + self._stationID = row[ 'site_no' ] + + timekeys = sorted( self._timeValueQuality.keys() ) + + self._obvPeriod = timekeys[ 0 ], timekeys[ -1 ] + self._stationName = self._stationID + self._generationTime = timekeys[ 0 ] + self._unit = 'm3/s' + + self.obvPeriod=self._obvPeriod + self.stationName=self._stationID + self.stationID=self._stationID + self.unit=self._unit + self.timeValueQuality=self._timeValueQuality + + def calculateDataQuality(self, value, qualifier1, qualifier2 ): + """ + Calculate the real-time data quality for given qualifier codes + found in the WaterML 2.0 files. + + Input: value - the stream flow value + qualifier1 - the first qualifier + qualifier2 - the second qualifier + + Return: Data quality value + """ +# +#%STATUS_CODES = +# ( +# '--' => 'Parameter not determined', +# '***' => 'Temporarily unavailable', +# 'Bkw' => 'Flow affected by backwater', +# 'Dis' => 'Data-collection discontinued', +# 'Dry' => 'Dry', +# 'Eqp' => 'Equipment malfunction', +# 'Fld' => 'Flood damage', +# 'Ice' => 'Ice affected', +# 'Mnt' => 'Maintenance in progress', +# 'Pr' => 'Partial-record site', +# 'Rat' => 'Rating being developed or revised', +# 'Ssn' => 'Parameter monitored seasonally', +# 'ZFl' => 'Zero flow', +# ); +# +#Also "approval" codes applicable to both UV and DV data: +# +#%DATA_AGING_CODES = +# ( +# 'P' => ['0', 'P', 'PROVISIONAL data subject to revision.'], +# 'A' => ['0', 'A', 'APPROVED for publication -- Processing and review +#completed.'], +# ); +# +# +#There are also unit value quality codes the vast majority of which you should never see. The ones in red below "may" be the only one you can expect to see in the public view: +# +#%UV_REMARKS = +# ( +# # -- Threshold flags +# 'H' => ['1', 'H', 'Value exceeds "very high" threshold.'], +# 'h' => ['1', 'h', 'Value exceeds "high" threshold.'], +# 'l' => ['1', 'l', 'Value exceeds "low" threshold.'], +# 'L' => ['1', 'L', 'Value exceeds "very low" threshold.'], +# 'I' => ['1', 'I', 'Value exceeds "very rapid increase" threshold.'], +# 'i' => ['1', 'i', 'Value exceeds "rapid increase" threshold.'], +# 'd' => ['1', 'd', 'Value exceeds "rapid decrease" threshold.'], +# 'D' => ['1', 'D', 'Value exceeds "very rapid decrease" threshold.'], +# 'T' => ['1', 'T', 'Value exceeds "standard difference" threshold.'], +# +# # -- Source Flags +# 'o' => ['1', 'o', 'Value was observed in the field.'], +# 'a' => ['1', 'a', 'Value is from paper tape.'], +# 's' => ['1', 's', 'Value is from a DCP.'], +# '~' => ['1', '~', 'Value is a system interpolated value.'], +# 'g' => ['1', 'g', 'Value recorded by data logger.'], +# 'c' => ['1', 'c', 'Value recorded on strip chart.'], +# 'p' => ['1', 'p', 'Value received by telephone transmission.'], +# 'r' => ['1', 'r', 'Value received by radio transmission.'], +# 'f' => ['1', 'f', 'Value received by machine readable file.'], +# 'z' => ['1', 'z', 'Value received from backup recorder.'], +# +# # -- Processing Status Flags +# '*' => ['1', '*', 'Value was EDITED by USGS personnel.'], +# +# # -- Other Flags (historical UVs only) +# 'S' => ['1', 'S', 'Value could be a result of a stuck recording instrument.'], +# 'M' => ['1', 'M', 'Redundant satellite value doesnt match original value.'], +# 'Q' => ['1', 'Q', 'Value was a satellite random transmission.'], +# 'V' => ['1', 'V', 'Value was an alert value.'], +# +# # -- UV remarks +# '&' => ['1', '&', 'Value was computed from affected unit values by UNSPECIFIED reasons.'], +# '<' => ['0', '<', 'Actual value is known to be less than reported value.'], +# '>' => ['0', '>', 'Actual value is known to be greater than reported value.'], +# 'C' => ['1', 'C', 'Value is affected by ice at the measurement site.'], +# 'B' => ['1', 'B', 'Value is affected by backwater at the measurement site.'], +# 'E' => ['0', 'e', 'Value was computed from estimated unit values.'], +# 'e' => ['0', 'e', 'Value has been ESTIMATED.'], +# 'F' => ['1', 'F', 'Value was modified due to automated filtering.'], +# 'K' => ['1', 'K', 'Value is affected by instrument calibration drift.'], +# 'R' => ['1', 'R', 'Rating is undefined for this value.'], +# 'X' => ['1', 'X', 'Value is erroneous and will not be used.'], +# ); +# +#Some daily value specific codes: +# +#%DV_REMARKS = +# ( +# '&' => ['1', '&', 'Value was computed from affected unit values by unspecified reasons.'], +# '1' => ['1', '1', 'Daily value is write protected without any remark code to be printed.'], +# '2' => ['1', '2', 'Remark is suppressed and will not print on a publication daily values table.'], +# '<' => ['0', '<', 'Actual value is known to be less than reported value.'], +# '>' => ['0', '>', 'Actual value is known to be greater than reported value.'], +# 'E' => ['0', 'e', 'Value was computed from estimated unit values.'], +# 'e' => ['0', 'e', 'Value has been estimated.'], +# ); +# +#I think we should set the ?e? (estimated) to 50% since we don?t know how the estimation was done, set the ?&? to 25%, and may be the ?*? to 5% since the data could be incomplete. The remainder set to 0 for the time being. That should wrap up this QC at this stage. We can always revisit them in the future as we progress. +# +#I made contact today with the USGS staff on location here and inform them that I will visit them next week for question about the rating curve gages. May be we can visit them together. +# +# +# +#Cheers, +# +# +# +#Oubeid +# +#Let?s stick with a weight of ?0? then for ?***?. I don?t have a lot of confidence on the data appended with that code. +# +# +# +#Cheers, +# +# +# +#Oubeid +# +# +#Date: Wed, 18 Nov 2015 14:41:13 -0700 +#From: James McCreight +#To: Zhengtao Cui - NOAA Affiliate , +# David Gochis , +# Brian Cosgrove - NOAA Federal +#Subject: preprocessing followup +#Parts/Attachments: +# 1 OK 33 lines Text (charset: UTF-8) +# 2 Shown ~28 lines Text (charset: UTF-8) +#---------------------------------------- +# +#Zhengtao-Great to talk to you about the code today. +#Here's the recap of things to do: +# * unit conversion to cms for the csv code +# * set quality to zero instead of raising errors in the quality code +# * use an upper limit for flow: 90,000 cms (see note below). +# * exclude 0 (?) as a valid flow +#Some notes on the last two bullets: +### JLM limit maximum flows to approx twice what I believe is the largest +#gaged flow +### on MS river *2 +### http://nwis.waterdata.usgs.gov/nwis/peak?site_no=07374000&agency_cd=USGS&format=h +#tml [nwis.waterdata.usgs.gov] +### baton rouge 1945: 1,473,000cfs=41,711cms +### multiply it roughly by 2 +#dataDf$quality[which(dataDf$discharge.cms > 0 & dataDf$discharge.cms < +#90000)] <- 100 +# +#As the above shows, I'm really torn about using zero values from NWIS. I'm +#inclined not to until we develop more advanced QC procedures. +# +#Thanks, +# +#James + + quality = 0 + + if value <= 0 or value > 90000 : # see James' email on Nov. 18 above + quality = 0 + else : + if qualifier2: + if qualifier1 in ("Approved", "Provisional"): + if qualifier2 == 'ESTIMATED': + quality = 50 + elif qualifier2 == 'UNSPECIFIED': + quality = 25 + elif qualifier2 == 'EDITED': + quality = 5 + elif qualifier2 in ("APPROVED","PROVISIONAL"): + quality = 100 + else: + quality = 0 + else: + if qualifier1 in ("Approved", "Provisional"): + quality = 100 + + return quality diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/make_time_slice_from_usgs_waterml.py b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/make_time_slice_from_usgs_waterml.py new file mode 100755 index 0000000..4230203 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/make_time_slice_from_usgs_waterml.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +############################################################################### +# File name: make_time_slice_from_usgs_waterml.py # +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: # +# # +# Last modification date: 7/12/2017 # +# # +# Description: The driver to create NetCDF time slice files from USGS # +# real-time observations # +# # +# 06/11/2024 ChamP - Use USGS .json file instead of .xml file # +############################################################################### + +import os, sys, time, getopt +import logging +from string import * +import xml.etree.ElementTree as etree +from datetime import datetime, timedelta +from USGS_Observation import USGS_Observation +from TimeSlice import TimeSlice +from Observation import Observation, All_Observations +from EmptyDirOrFileException import EmptyDirOrFileException + +""" + The driver to parse downloaded Json observations and + create time slices and write to NetCDF files + Author: Zhengtao Cui (Zhengtao.Cui@noaa.gov) + Date: Aug. 26, 2015 +""" +def main(argv): + """ + function to get input arguments + """ + inputdir = '' + try: + opts, args = getopt.getopt(argv,"hi:o:",["idir=", "odir="]) + except getopt.GetoptError: + print('make_time_slice_from_usgs_waterml.py -i -o ') + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print( \ + 'make_time_slice_from_usgs_waterml.py -i -o ') + sys.exit() + elif opt in ('-i', "--idir"): + inputdir = arg + if not os.path.exists( inputdir ): + raise RuntimeError( 'FATAL ERROR: inputdir ' + \ + inputdir + ' does not exist!' ) + elif opt in ('-o', "--odir" ): + outputdir = arg + if not os.path.exists( outputdir ): + raise RuntimeError( 'FATAL ERROR: outputdir ' + \ + outputdir + ' does not exist!' ) + + return (inputdir, outputdir) + +t0 = time.time() + +logging.basicConfig(format=\ + '%(asctime)s - %(name)s - %(levelname)s - %(message)s',\ + level=logging.INFO) +logger = logging.getLogger(__name__) +formatter = logging.Formatter(\ + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +#logger.setFormatter(formatter) +logger.info( "System Path: " + str( sys.path ) ) + +if __name__ == "__main__": + try: + odir = main(sys.argv[1:]) + except Exception as e: + logger.error("Failed to get program options.", exc_info=True) + +indir = odir[0] +outdir = odir[1] +logger.info( 'Input dir is "' + indir + '"') +logger.info( 'Output dir is "' + outdir + '"\n\n') + +# +# Load USGS observed JSON discharge data +# + +try: + usgsobvs = [] + + if not os.path.isdir( indir ): + raise RuntimeError( "FATAL ERROR: " + indir + \ + " is not a directory or does not exist. ") + for filename in os.listdir( indir ): + file = os.path.join(indir, filename) + if file.endswith( ".json" ): + #logger.info("\n\n") + logger.info(' Reading ' + file + ' ... ' ) + + usgsobv = USGS_Observation( file ) + + if usgsobv.timeValueQuality: + usgsobvs.append( usgsobv ) + + if not usgsobvs: + raise EmptyDirOrFileException( "Input directory " + indir + \ + " has no USGS json files, or the files are empty, or no discharge values!") + + allobvs = All_Observations( usgsobvs ) + +except EmptyDirOrFileException as e: + logger.warning( str(e), exc_info=True) + sys.exit(0) + +except Exception as e: + logger.error("Failed to load WaterJson files: " + str(e), exc_info=True) + sys.exit(3) + +logger.info( 'Earliest time in WaterJson: ' + \ + allobvs.timePeriodForAll()[0].isoformat() ) +logger.info( 'Latest time in WaterJson:: ' + \ + allobvs.timePeriodForAll()[1].isoformat() ) + +# +# Create time slices from loaded observations +# +# Set time resolution to 15 minutes +# and +# Write time slices to NetCDF files +# +try: + timeslices = allobvs.makeAllTimeSlices( timedelta( minutes = 15 ), outdir ) +except Exception as e: + logger.error("Failed to make time slices: " + str(e), exc_info=True) + logger.error("Input dir = " + indir, exc_info=True) + sys.exit(3) + +logger.info( "Total number of timeslices: " + str( timeslices ) ) +logger.info( "Program finished in: " + \ + "{0:.1f}".format( (time.time() - t0) / 60.0 ) + \ + " minutes" ) diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/nwmcopy.sh b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/nwmcopy.sh new file mode 100755 index 0000000..a581bfd --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/nwmcopy.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash + +############################################################################### +# Program Name: nwmcopy.sh # +# # +# Author(s)/Contact(s): NWC # +# # +# copy files to and from the $COM and $DBN alert directories # +# # +# Input: directory in $COM # +# # +# Output: files # +# # +# For non-fatal errors output is witten to $DATA/LOGS # +# # +# Origination Jun, 2019 # +# OWP Dec, 2021 Port to WCOSS2 # +# # +############################################################################### +# --------------------------------------------------------------------------- # + +####################################### +# nwm_copy and nwm_postcopy utilize +# multiple cores to speed up file copying +####################################### +function nwm_copy () { + msg="Begin copying file at `date`" + postmsg "${pgmout}" "${msg}" + + dir=$1 + fourcyclesago=$($NDATE -4 ${PDY}${cyc}) + echo "#!/usr/bin/env bash" > $DATA/nwmcopyscript + for file in $COMIN/${dir}/* + do + test -f "$file" || continue + filedatetime=$(basename $file | cut -c1-4 )$(basename $file | cut -c6-7 )$(basename $file | cut -c9-10)$(basename $file | cut -c12-13 ) + # + # Copy only files of the past 4 hours + # + if [ $fourcyclesago -le $filedatetime ]; then + echo "cpfs $file $DATA/$(basename $file) || true" >> $DATA/nwmcopyscript + fi + done + + # + # Previous day + # + if [ ${cyc} -le 4 ]; then + for file in $COMINm1/${dir}/* + do + test -f "$file" || continue + filedatetime=$(basename $file | cut -c1-4 )$(basename $file | cut -c6-7 )$(basename $file | cut -c9-10)$(basename $file | cut -c12-13 ) + # + # Copy only files of the past 4 hours + # + if [ $fourcyclesago -le $filedatetime ]; then + echo "cpfs $file $DATA/$(basename $file) || true" >> $DATA/nwmcopyscript + fi + done + fi + + chmod 755 $DATA/nwmcopyscript + + #aprun -j1 -n$((NODES*NCORES)) -N${NCORES} cfp $DATA/nwmcopyscript + ${CFPCOMMAND} $DATA/nwmcopyscript + export err=$?; err_chk + + msg="Ending copy file at `date`" + postmsg "$pgmout" "$msg" + +} + +function nwm_postcopy () { + msg="Begin post copying file at `date`" + postmsg "${pgmout}" "${msg}" + + dir=$1 + suffix=$2 + fourcyclesago=$($NDATE -4 ${PDY}${cyc}) + echo "#!/usr/bin/env bash" > $DATA/nwmpostcopyscript + for file in $DATA/*.${suffix} + do + test -f "$file" || continue + filedatetime=$(basename $file | cut -c1-4 )$(basename $file | cut -c6-7 )$(basename $file | cut -c9-10)$(basename $file | cut -c12-13 ) + # + # Copy only files of the past 4 hours + # + if [ $fourcyclesago -le $filedatetime ]; then + # + # NOTE: Here, we assume $COMOUT == $COMIN, otherwise, it doesn't work. + # + #export COMOUT_ROOT=${COMOUT_ROOT:-${COMROOT}/${NET}/${envir}} + export COMOUT_ROOT=${COMOUT_ROOT:-${COMROOT}/${NET}/${nwm_ver}} + Outdir=${COMOUT_ROOT}/${RUN}.$(basename $file | cut -c1-4 )$(basename \ + $file | cut -c6-7 )$(basename $file | cut -c9-10)/${dir} + + if [ ! -e $Outdir ]; then + mkdir -p $Outdir + fi + + copyandalert=true + + # + # if the file exists and was not changed, don't copy and alert + # + if [ -e ${Outdir}/$(basename $file) ]; then + diff ${file} ${Outdir}/$(basename $file) > /dev/null 2>&1 + if [ $? -eq 0 ]; then + copyandalert=false + fi + fi + + if [[ "$copyandalert" == true ]]; then + echo "cpfs ${file} ${Outdir}/$(basename $file); if [ "$SENDDBN" = YES ]; then $DBNROOT/bin/dbn_alert MODEL ${DBN_ALERT_TYPE} $job ${Outdir}/$(basename $file); fi" >> $DATA/nwmpostcopyscript + fi + + fi + done + + chmod 755 $DATA/nwmpostcopyscript + + #aprun -j1 -n$((NODES*NCORES)) -N${NCORES} cfp $DATA/nwmpostcopyscript + ${CFPCOMMAND} $DATA/nwmpostcopyscript + export err=$?; err_chk + + msg="Ending post copy file at `date`" + postmsg "$pgmout" "$msg" + +} diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/runTestUSGSTimeslice.sh b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/runTestUSGSTimeslice.sh new file mode 100755 index 0000000..638cdf6 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/runTestUSGSTimeslice.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +########################################################## +# Test usgs timeslice using xml or json format data file.# +# ./runTestUSGSTimeslice.sh # +# # +# 06/12/2024 CPham # +########################################################## +cd $(pwd) + +if [[ ! -d test_data/usgs_timeslices$1 ]]; then + mkdir -p test_data/usgs_timeslices$1 +else + rm -f test_data/usgs_timeslices$1/* +fi + +python make_time_slice_from_usgs_waterml.py -i test_data/$1 -o test_data/usgs_timeslices$1 diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/01122500.json b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/01122500.json new file mode 100755 index 0000000..37dd6c1 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/01122500.json @@ -0,0 +1,318 @@ +{ + "name": "ns1:timeSeriesResponseType", + "declaredType": "org.cuahsi.waterml.TimeSeriesResponseType", + "scope": "javax.xml.bind.JAXBElement$GlobalScope", + "value": { + "queryInfo": { + "queryURL": "http://waterservices.usgs.gov/nwis/iv/sites=01122500&format=json¶meterCd=00060&period=PT6H", + "criteria": { + "locationParam": "[ALL:01122500]", + "variableParam": "[00060]", + "parameter": [] + }, + "note": [ + { + "value": "[ALL:01122500]", + "title": "filter:sites" + }, + { + "value": "[mode=PERIOD, period=PT6H, modifiedSince=null]", + "title": "filter:timeRange" + }, + { + "value": "methodIds=[ALL]", + "title": "filter:methodId" + }, + { + "value": "2024-06-12T22:59:53.807Z", + "title": "requestDT" + }, + { + "value": "7b15cfd0-290f-11ef-b8c9-005056beda50", + "title": "requestId" + }, + { + "value": "Provisional data are subject to revision. Go to http://waterdata.usgs.gov/nwis/help/?provisional for more information.", + "title": "disclaimer" + }, + { + "value": "caas01", + "title": "server" + } + ] + }, + "timeSeries": [ + { + "sourceInfo": { + "siteName": "SHETUCKET RIVER NEAR WILLIMANTIC, CT", + "siteCode": [ + { + "value": "01122500", + "network": "NWIS", + "agencyCode": "USGS" + } + ], + "timeZoneInfo": { + "defaultTimeZone": { + "zoneOffset": "-05:00", + "zoneAbbreviation": "EST" + }, + "daylightSavingsTimeZone": { + "zoneOffset": "-04:00", + "zoneAbbreviation": "EDT" + }, + "siteUsesDaylightSavingsTime": true + }, + "geoLocation": { + "geogLocation": { + "srs": "EPSG:4326", + "latitude": 41.70037644, + "longitude": -72.1820223 + }, + "localSiteXY": [] + }, + "note": [], + "siteType": [], + "siteProperty": [ + { + "value": "ST", + "name": "siteTypeCd" + }, + { + "value": "01100002", + "name": "hucCd" + }, + { + "value": "09", + "name": "stateCd" + }, + { + "value": "09015", + "name": "countyCd" + } + ] + }, + "variable": { + "variableCode": [ + { + "value": "00060", + "network": "NWIS", + "vocabulary": "NWIS:UnitValues", + "variableID": 45807197, + "default": true + } + ], + "variableName": "Streamflow, ft³/s", + "variableDescription": "Discharge, cubic feet per second", + "valueType": "Derived Value", + "unit": { + "unitCode": "ft3/s" + }, + "options": { + "option": [ + { + "name": "Statistic", + "optionCode": "00000" + } + ] + }, + "note": [], + "noDataValue": -999999.0, + "variableProperty": [], + "oid": "45807197" + }, + "values": [ + { + "value": [ + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T13:00:00.000-04:00" + }, + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T13:15:00.000-04:00" + }, + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T13:30:00.000-04:00" + }, + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T13:45:00.000-04:00" + }, + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T14:00:00.000-04:00" + }, + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T14:15:00.000-04:00" + }, + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T14:30:00.000-04:00" + }, + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T14:45:00.000-04:00" + }, + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T15:00:00.000-04:00" + }, + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T15:15:00.000-04:00" + }, + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T15:30:00.000-04:00" + }, + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T15:45:00.000-04:00" + }, + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T16:00:00.000-04:00" + }, + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T16:15:00.000-04:00" + }, + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T16:30:00.000-04:00" + }, + { + "value": "318", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T16:45:00.000-04:00" + }, + { + "value": "322", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T17:00:00.000-04:00" + }, + { + "value": "322", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T17:15:00.000-04:00" + }, + { + "value": "322", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T17:30:00.000-04:00" + }, + { + "value": "322", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T17:45:00.000-04:00" + }, + { + "value": "322", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T18:00:00.000-04:00" + }, + { + "value": "322", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T18:15:00.000-04:00" + }, + { + "value": "322", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-12T18:30:00.000-04:00" + } + ], + "qualifier": [ + { + "qualifierCode": "P", + "qualifierDescription": "Provisional data subject to revision.", + "qualifierID": 0, + "network": "NWIS", + "vocabulary": "uv_rmk_cd" + } + ], + "qualityControlLevel": [], + "method": [ + { + "methodDescription": "", + "methodID": 66508 + } + ], + "source": [], + "offset": [], + "sample": [], + "censorCode": [] + } + ], + "name": "USGS:01122500:00060:00000" + } + ] + }, + "nil": false, + "globalScope": true, + "typeSubstituted": false +} \ No newline at end of file diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/0208111310.json b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/0208111310.json new file mode 100755 index 0000000..fa8cb02 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/0208111310.json @@ -0,0 +1,325 @@ +{ + "name": "ns1:timeSeriesResponseType", + "declaredType": "org.cuahsi.waterml.TimeSeriesResponseType", + "scope": "javax.xml.bind.JAXBElement$GlobalScope", + "value": { + "queryInfo": { + "queryURL": "http://waterservices.usgs.gov/nwis/iv/sites=0208111310&format=json¶meterCd=00060&period=PT6H", + "criteria": { + "locationParam": "[ALL:0208111310]", + "variableParam": "[00060]", + "parameter": [] + }, + "note": [ + { + "value": "[ALL:0208111310]", + "title": "filter:sites" + }, + { + "value": "[mode=PERIOD, period=PT6H, modifiedSince=null]", + "title": "filter:timeRange" + }, + { + "value": "methodIds=[ALL]", + "title": "filter:methodId" + }, + { + "value": "2024-06-14T18:24:41.728Z", + "title": "requestDT" + }, + { + "value": "5df228f0-2a7b-11ef-8171-2cea7f58f5ca", + "title": "requestId" + }, + { + "value": "Provisional data are subject to revision. Go to http://waterdata.usgs.gov/nwis/help/?provisional for more information.", + "title": "disclaimer" + }, + { + "value": "vaas01", + "title": "server" + } + ] + }, + "timeSeries": [ + { + "sourceInfo": { + "siteName": "CASHIE RIVER AT SR1257 NEAR WINDSOR, NC", + "siteCode": [ + { + "value": "0208111310", + "network": "NWIS", + "agencyCode": "USGS" + } + ], + "timeZoneInfo": { + "defaultTimeZone": { + "zoneOffset": "-05:00", + "zoneAbbreviation": "EST" + }, + "daylightSavingsTimeZone": { + "zoneOffset": "-04:00", + "zoneAbbreviation": "EDT" + }, + "siteUsesDaylightSavingsTime": true + }, + "geoLocation": { + "geogLocation": { + "srs": "EPSG:4326", + "latitude": 36.04777778, + "longitude": -76.9841667 + }, + "localSiteXY": [] + }, + "note": [], + "siteType": [], + "siteProperty": [ + { + "value": "ST", + "name": "siteTypeCd" + }, + { + "value": "03010107", + "name": "hucCd" + }, + { + "value": "37", + "name": "stateCd" + }, + { + "value": "37015", + "name": "countyCd" + } + ] + }, + "variable": { + "variableCode": [ + { + "value": "00060", + "network": "NWIS", + "vocabulary": "NWIS:UnitValues", + "variableID": 45807197, + "default": true + } + ], + "variableName": "Streamflow, ft³/s", + "variableDescription": "Discharge, cubic feet per second", + "valueType": "Derived Value", + "unit": { + "unitCode": "ft3/s" + }, + "options": { + "option": [ + { + "name": "Statistic", + "optionCode": "00000" + } + ] + }, + "note": [], + "noDataValue": -999999.0, + "variableProperty": [], + "oid": "45807197" + }, + "values": [ + { + "value": [ + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T08:30:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T08:45:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T09:00:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T09:15:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T09:30:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T09:45:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T10:00:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T10:15:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T10:30:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T10:45:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T11:00:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T11:15:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T11:30:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T11:45:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T12:00:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T12:15:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T12:30:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T12:45:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T13:00:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T13:15:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T13:30:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T13:45:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T14:00:00.000-04:00" + }, + { + "value": "0.64", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-14T14:15:00.000-04:00" + } + ], + "qualifier": [ + { + "qualifierCode": "P", + "qualifierDescription": "Provisional data subject to revision.", + "qualifierID": 0, + "network": "NWIS", + "vocabulary": "uv_rmk_cd" + } + ], + "qualityControlLevel": [], + "method": [ + { + "methodDescription": "", + "methodID": 88874 + } + ], + "source": [], + "offset": [], + "sample": [], + "censorCode": [] + } + ], + "name": "USGS:0208111310:00060:00000" + } + ] + }, + "nil": false, + "globalScope": true, + "typeSubstituted": false +} \ No newline at end of file diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/02186645.json b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/02186645.json new file mode 100755 index 0000000..7cb5397 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/02186645.json @@ -0,0 +1,326 @@ +{ + "name": "ns1:timeSeriesResponseType", + "declaredType": "org.cuahsi.waterml.TimeSeriesResponseType", + "scope": "javax.xml.bind.JAXBElement$GlobalScope", + "value": { + "queryInfo": { + "queryURL": "http://waterservices.usgs.gov/nwis/iv/sites=02186645&format=json¶meterCd=00060&period=PT6H", + "criteria": { + "locationParam": "[ALL:02186645]", + "variableParam": "[00060]", + "parameter": [] + }, + "note": [ + { + "value": "[ALL:02186645]", + "title": "filter:sites" + }, + { + "value": "[mode=PERIOD, period=PT6H, modifiedSince=null]", + "title": "filter:timeRange" + }, + { + "value": "methodIds=[ALL]", + "title": "filter:methodId" + }, + { + "value": "2024-06-11T22:59:24.174Z", + "title": "requestDT" + }, + { + "value": "3f026cc0-2846-11ef-b8c9-005056beda50", + "title": "requestId" + }, + { + "value": "Provisional data are subject to revision. Go to http://waterdata.usgs.gov/nwis/help/?provisional for more information.", + "title": "disclaimer" + }, + { + "value": "caas01", + "title": "server" + } + ] + }, + "timeSeries": [ + { + "sourceInfo": { + "siteName": "CONEROSS CK NR SENECA, SC", + "siteCode": [ + { + "value": "02186645", + "network": "NWIS", + "agencyCode": "USGS" + } + ], + "timeZoneInfo": { + "defaultTimeZone": { + "zoneOffset": "-05:00", + "zoneAbbreviation": "EST" + }, + "daylightSavingsTimeZone": { + "zoneOffset": "-04:00", + "zoneAbbreviation": "EDT" + }, + "siteUsesDaylightSavingsTime": true + }, + "geoLocation": { + "geogLocation": { + "srs": "EPSG:4326", + "latitude": 34.64926575, + "longitude": -82.9915382 + }, + "localSiteXY": [] + }, + "note": [], + "siteType": [], + "siteProperty": [ + { + "value": "ST", + "name": "siteTypeCd" + }, + { + "value": "03060101", + "name": "hucCd" + }, + { + "value": "45", + "name": "stateCd" + }, + { + "value": "45073", + "name": "countyCd" + } + ] + }, + "variable": { + "variableCode": [ + { + "value": "00060", + "network": "NWIS", + "vocabulary": "NWIS:UnitValues", + "variableID": 45807197, + "default": true + } + ], + "variableName": "Streamflow, ft³/s", + "variableDescription": "Discharge, cubic feet per second", + "valueType": "Derived Value", + "unit": { + "unitCode": "ft3/s" + }, + "options": { + "option": [ + { + "name": "Statistic", + "optionCode": "00000" + } + ] + }, + "note": [], + "noDataValue": -999999.0, + "variableProperty": [], + "oid": "45807197" + }, + "values": [ + { + "value": [], + "qualifier": [], + "qualityControlLevel": [], + "method": [ + { + "methodDescription": "", + "methodID": 177457 + } + ], + "source": [], + "offset": [], + "sample": [], + "censorCode": [] + }, + { + "value": [ + { + "value": "60.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T13:00:00.000-04:00" + }, + { + "value": "61.9", + "qualifiers": [ + "P" + ], + "dateTime": " " + }, + { + "value": "60.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T13:30:00.000-04:00" + }, + { + "value": "61.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T13:45:00.000-04:00" + }, + { + "value": "60.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T14:00:00.000-04:00" + }, + { + "value": "60.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T14:15:00.000-04:00" + }, + { + "value": "60.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T14:30:00.000-04:00" + }, + { + "value": "60.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T14:45:00.000-04:00" + }, + { + "value": "60.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T15:00:00.000-04:00" + }, + { + "value": "59.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T15:15:00.000-04:00" + }, + { + "value": "59.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T15:30:00.000-04:00" + }, + { + "value": "59.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T15:45:00.000-04:00" + }, + { + "value": "59.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T16:00:00.000-04:00" + }, + { + "value": "58.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T16:15:00.000-04:00" + }, + { + "value": "59.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T16:30:00.000-04:00" + }, + { + "value": "59.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T16:45:00.000-04:00" + }, + { + "value": "59.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T17:00:00.000-04:00" + }, + { + "value": "59.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T17:15:00.000-04:00" + }, + { + "value": "58.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T17:30:00.000-04:00" + }, + { + "value": "58.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T17:45:00.000-04:00" + }, + { + "value": "58.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T18:00:00.000-04:00" + }, + { + "value": "58.9", + "qualifiers": [ + "P" + ], + "dateTime": "2024-06-11T18:15:00.000-04:00" + } + ], + "qualifier": [ + { + "qualifierCode": "P", + "qualifierDescription": "Provisional data subject to revision.", + "qualifierID": 0, + "network": "NWIS", + "vocabulary": "uv_rmk_cd" + } + ], + "qualityControlLevel": [], + "method": [ + { + "methodDescription": "[(2)]", + "methodID": 337012 + } + ], + "source": [], + "offset": [], + "sample": [], + "censorCode": [] + } + ], + "name": "USGS:02186645:00060:00000" + } + ] + }, + "nil": false, + "globalScope": true, + "typeSubstituted": false +} diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/02289060.json b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/02289060.json new file mode 100755 index 0000000..7b3b03a --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/02289060.json @@ -0,0 +1,340 @@ +{ + "name": "ns1:timeSeriesResponseType", + "declaredType": "org.cuahsi.waterml.TimeSeriesResponseType", + "scope": "javax.xml.bind.JAXBElement$GlobalScope", + "value": { + "queryInfo": { + "queryURL": "http://waterservices.usgs.gov/nwis/iv/sites=02289060&format=json¶meterCd=00060&period=PT6H", + "criteria": { + "locationParam": "[ALL:02289060]", + "variableParam": "[00060]", + "parameter": [] + }, + "note": [ + { + "value": "[ALL:02289060]", + "title": "filter:sites" + }, + { + "value": "[mode=PERIOD, period=PT6H, modifiedSince=null]", + "title": "filter:timeRange" + }, + { + "value": "methodIds=[ALL]", + "title": "filter:methodId" + }, + { + "value": "2024-06-12T23:26:17.754Z", + "title": "requestDT" + }, + { + "value": "2b311890-2913-11ef-b8c9-005056beda50", + "title": "requestId" + }, + { + "value": "Provisional data are subject to revision. Go to http://waterdata.usgs.gov/nwis/help/?provisional for more information.", + "title": "disclaimer" + }, + { + "value": "caas01", + "title": "server" + } + ] + }, + "timeSeries": [ + { + "sourceInfo": { + "siteName": "TAMIAMI CANAL OUTLETS L-30 TO L-67A NR MIAMI, FL", + "siteCode": [ + { + "value": "02289060", + "network": "NWIS", + "agencyCode": "USGS" + } + ], + "timeZoneInfo": { + "defaultTimeZone": { + "zoneOffset": "-05:00", + "zoneAbbreviation": "EST" + }, + "daylightSavingsTimeZone": { + "zoneOffset": "-04:00", + "zoneAbbreviation": "EDT" + }, + "siteUsesDaylightSavingsTime": true + }, + "geoLocation": { + "geogLocation": { + "srs": "EPSG:4326", + "latitude": 25.76114167, + "longitude": -80.5617056 + }, + "localSiteXY": [] + }, + "note": [], + "siteType": [], + "siteProperty": [ + { + "value": "ST-CA", + "name": "siteTypeCd" + }, + { + "value": "03090202", + "name": "hucCd" + }, + { + "value": "12", + "name": "stateCd" + }, + { + "value": "12086", + "name": "countyCd" + } + ] + }, + "variable": { + "variableCode": [ + { + "value": "00060", + "network": "NWIS", + "vocabulary": "NWIS:UnitValues", + "variableID": 45807197, + "default": true + } + ], + "variableName": "Streamflow, ft³/s", + "variableDescription": "Discharge, cubic feet per second", + "valueType": "Derived Value", + "unit": { + "unitCode": "ft3/s" + }, + "options": { + "option": [ + { + "name": "Statistic", + "optionCode": "00000" + } + ] + }, + "note": [], + "noDataValue": -999999.0, + "variableProperty": [], + "oid": "45807197" + }, + "values": [ + { + "value": [ + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T13:30:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T13:45:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T14:00:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T14:15:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T14:30:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T14:45:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T15:00:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T15:15:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T15:30:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T15:45:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T16:00:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T16:15:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T16:30:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T16:45:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T17:00:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T17:15:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T17:30:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T17:45:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T18:00:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T18:15:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T18:30:00.000-04:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Rat" + ], + "dateTime": "2024-06-12T18:45:00.000-04:00" + } + ], + "qualifier": [ + { + "qualifierCode": "Rat", + "qualifierDescription": "Rating being developed.", + "qualifierID": 0, + "network": "NWIS", + "vocabulary": "uv_rmk_cd" + }, + { + "qualifierCode": "P", + "qualifierDescription": "Provisional data subject to revision.", + "qualifierID": 1, + "network": "NWIS", + "vocabulary": "uv_rmk_cd" + } + ], + "qualityControlLevel": [], + "method": [ + { + "methodDescription": "[NGVD29]", + "methodID": 31249 + } + ], + "source": [], + "offset": [], + "sample": [], + "censorCode": [] + } + ], + "name": "USGS:02289060:00060:00000" + } + ] + }, + "nil": false, + "globalScope": true, + "typeSubstituted": false +} \ No newline at end of file diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/07027005.json b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/07027005.json new file mode 100755 index 0000000..d94cdf7 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/07027005.json @@ -0,0 +1,163 @@ +{ + "name": "ns1:timeSeriesResponseType", + "declaredType": "org.cuahsi.waterml.TimeSeriesResponseType", + "scope": "javax.xml.bind.JAXBElement$GlobalScope", + "value": { + "queryInfo": { + "queryURL": "http://waterservices.usgs.gov/nwis/iv/sites=07027005&format=json¶meterCd=00060&period=PT6H", + "criteria": { + "locationParam": "[ALL:07027005]", + "variableParam": "[00060]", + "parameter": [] + }, + "note": [ + { + "value": "[ALL:07027005]", + "title": "filter:sites" + }, + { + "value": "[mode=PERIOD, period=PT6H, modifiedSince=null]", + "title": "filter:timeRange" + }, + { + "value": "methodIds=[ALL]", + "title": "filter:methodId" + }, + { + "value": "2024-06-13T20:18:28.070Z", + "title": "requestDT" + }, + { + "value": "18596240-29c2-11ef-b8c9-005056beda50", + "title": "requestId" + }, + { + "value": "Provisional data are subject to revision. Go to http://waterdata.usgs.gov/nwis/help/?provisional for more information.", + "title": "disclaimer" + }, + { + "value": "caas01", + "title": "server" + } + ] + }, + "timeSeries": [ + { + "sourceInfo": { + "siteName": "RUNNING REELFOOT BAYOU BELOW REELFOOT SPILLWAY", + "siteCode": [ + { + "value": "07027005", + "network": "NWIS", + "agencyCode": "USGS" + } + ], + "timeZoneInfo": { + "defaultTimeZone": { + "zoneOffset": "-06:00", + "zoneAbbreviation": "CST" + }, + "daylightSavingsTimeZone": { + "zoneOffset": "-05:00", + "zoneAbbreviation": "CDT" + }, + "siteUsesDaylightSavingsTime": true + }, + "geoLocation": { + "geogLocation": { + "srs": "EPSG:4326", + "latitude": 36.3483333, + "longitude": -89.4080556 + }, + "localSiteXY": [] + }, + "note": [], + "siteType": [], + "siteProperty": [ + { + "value": "LK", + "name": "siteTypeCd" + }, + { + "value": "08010202", + "name": "hucCd" + }, + { + "value": "47", + "name": "stateCd" + }, + { + "value": "47095", + "name": "countyCd" + } + ] + }, + "variable": { + "variableCode": [ + { + "value": "00060", + "network": "NWIS", + "vocabulary": "NWIS:UnitValues", + "variableID": 45807197, + "default": true + } + ], + "variableName": "Streamflow, ft³/s", + "variableDescription": "Discharge, cubic feet per second", + "valueType": "Derived Value", + "unit": { + "unitCode": "ft3/s" + }, + "options": { + "option": [ + { + "name": "Statistic", + "optionCode": "00000" + } + ] + }, + "note": [], + "noDataValue": -999999.0, + "variableProperty": [], + "oid": "45807197" + }, + "values": [ + { + "value": [], + "qualifier": [], + "qualityControlLevel": [], + "method": [ + { + "methodDescription": "", + "methodID": 170149 + } + ], + "source": [], + "offset": [], + "sample": [], + "censorCode": [] + }, + { + "value": [], + "qualifier": [], + "qualityControlLevel": [], + "method": [ + { + "methodDescription": "Discharge from Sidelooker", + "methodID": 241470 + } + ], + "source": [], + "offset": [], + "sample": [], + "censorCode": [] + } + ], + "name": "USGS:07027005:00060:00000" + } + ] + }, + "nil": false, + "globalScope": true, + "typeSubstituted": false +} \ No newline at end of file diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/08329918.json b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/08329918.json new file mode 100755 index 0000000..ee02692 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/08329918.json @@ -0,0 +1,348 @@ +{ + "name": "ns1:timeSeriesResponseType", + "declaredType": "org.cuahsi.waterml.TimeSeriesResponseType", + "scope": "javax.xml.bind.JAXBElement$GlobalScope", + "value": { + "queryInfo": { + "queryURL": "http://waterservices.usgs.gov/nwis/iv/sites=08329918&format=json¶meterCd=00060&period=PT6H", + "criteria": { + "locationParam": "[ALL:08329918]", + "variableParam": "[00060]", + "parameter": [] + }, + "note": [ + { + "value": "[ALL:08329918]", + "title": "filter:sites" + }, + { + "value": "[mode=PERIOD, period=PT6H, modifiedSince=null]", + "title": "filter:timeRange" + }, + { + "value": "methodIds=[ALL]", + "title": "filter:methodId" + }, + { + "value": "2024-06-12T21:52:15.579Z", + "title": "requestDT" + }, + { + "value": "083183a0-2906-11ef-bdf8-2cea7f5e5ede", + "title": "requestId" + }, + { + "value": "Provisional data are subject to revision. Go to http://waterdata.usgs.gov/nwis/help/?provisional for more information.", + "title": "disclaimer" + }, + { + "value": "sdas01", + "title": "server" + } + ] + }, + "timeSeries": [ + { + "sourceInfo": { + "siteName": "RIO GRANDE AT ALAMEDA BRIDGE AT ALAMEDA, NM", + "siteCode": [ + { + "value": "08329918", + "network": "NWIS", + "agencyCode": "USGS" + } + ], + "timeZoneInfo": { + "defaultTimeZone": { + "zoneOffset": "-07:00", + "zoneAbbreviation": "MST" + }, + "daylightSavingsTimeZone": { + "zoneOffset": "-06:00", + "zoneAbbreviation": "MDT" + }, + "siteUsesDaylightSavingsTime": true + }, + "geoLocation": { + "geogLocation": { + "srs": "EPSG:4326", + "latitude": 35.1977222, + "longitude": -106.6427778 + }, + "localSiteXY": [] + }, + "note": [], + "siteType": [], + "siteProperty": [ + { + "value": "ST", + "name": "siteTypeCd" + }, + { + "value": "13020203", + "name": "hucCd" + }, + { + "value": "35", + "name": "stateCd" + }, + { + "value": "35001", + "name": "countyCd" + } + ] + }, + "variable": { + "variableCode": [ + { + "value": "00060", + "network": "NWIS", + "vocabulary": "NWIS:UnitValues", + "variableID": 45807197, + "default": true + } + ], + "variableName": "Streamflow, ft³/s", + "variableDescription": "Discharge, cubic feet per second", + "valueType": "Derived Value", + "unit": { + "unitCode": "ft3/s" + }, + "options": { + "option": [ + { + "name": "Statistic", + "optionCode": "00000" + } + ] + }, + "note": [], + "noDataValue": -999999.0, + "variableProperty": [], + "oid": "45807197" + }, + "values": [ + { + "value": [ + { + "value": "1470", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:00:00.000-06:00" + }, + { + "value": "1470", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:15:00.000-06:00" + }, + { + "value": "1470", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:30:00.000-06:00" + }, + { + "value": "1470", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:45:00.000-06:00" + }, + { + "value": "1470", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:00:00.000-06:00" + }, + { + "value": "1470", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:15:00.000-06:00" + }, + { + "value": "1450", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:30:00.000-06:00" + }, + { + "value": "1450", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:45:00.000-06:00" + }, + { + "value": "1450", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T12:00:00.000-06:00" + }, + { + "value": "1470", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T12:15:00.000-06:00" + }, + { + "value": "1450", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T12:30:00.000-06:00" + }, + { + "value": "1470", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T12:45:00.000-06:00" + }, + { + "value": "1470", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T13:00:00.000-06:00" + }, + { + "value": "1470", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T13:15:00.000-06:00" + }, + { + "value": "1450", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T13:30:00.000-06:00" + }, + { + "value": "1450", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T13:45:00.000-06:00" + }, + { + "value": "1430", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T14:00:00.000-06:00" + }, + { + "value": "1450", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T14:15:00.000-06:00" + }, + { + "value": "1430", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T14:30:00.000-06:00" + }, + { + "value": "1450", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T14:45:00.000-06:00" + }, + { + "value": "1450", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T15:00:00.000-06:00" + }, + { + "value": "1450", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T15:15:00.000-06:00" + }, + { + "value": "1450", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T15:30:00.000-06:00" + } + ], + "qualifier": [ + { + "qualifierCode": "e", + "qualifierDescription": "Value has been estimated.", + "qualifierID": 0, + "network": "NWIS", + "vocabulary": "uv_rmk_cd" + }, + { + "qualifierCode": "P", + "qualifierDescription": "Provisional data subject to revision.", + "qualifierID": 1, + "network": "NWIS", + "vocabulary": "uv_rmk_cd" + } + ], + "qualityControlLevel": [], + "method": [ + { + "methodDescription": "", + "methodID": 101239 + } + ], + "source": [], + "offset": [], + "sample": [], + "censorCode": [] + } + ], + "name": "USGS:08329918:00060:00000" + } + ] + }, + "nil": false, + "globalScope": true, + "typeSubstituted": false +} \ No newline at end of file diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/09423000.json b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/09423000.json new file mode 100755 index 0000000..de39d68 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/09423000.json @@ -0,0 +1,332 @@ +{ + "name": "ns1:timeSeriesResponseType", + "declaredType": "org.cuahsi.waterml.TimeSeriesResponseType", + "scope": "javax.xml.bind.JAXBElement$GlobalScope", + "value": { + "queryInfo": { + "queryURL": "http://waterservices.usgs.gov/nwis/iv/sites=09423000&format=json¶meterCd=00060&period=PT6H", + "criteria": { + "locationParam": "[ALL:09423000]", + "variableParam": "[00060]", + "parameter": [] + }, + "note": [ + { + "value": "[ALL:09423000]", + "title": "filter:sites" + }, + { + "value": "[mode=PERIOD, period=PT6H, modifiedSince=null]", + "title": "filter:timeRange" + }, + { + "value": "methodIds=[ALL]", + "title": "filter:methodId" + }, + { + "value": "2024-06-13T00:20:20.981Z", + "title": "requestDT" + }, + { + "value": "b84e6a50-291a-11ef-8171-2cea7f58f5ca", + "title": "requestId" + }, + { + "value": "Provisional data are subject to revision. Go to http://waterdata.usgs.gov/nwis/help/?provisional for more information.", + "title": "disclaimer" + }, + { + "value": "vaas01", + "title": "server" + } + ] + }, + "timeSeries": [ + { + "sourceInfo": { + "siteName": "COLORADO RIVER BELOW DAVIS DAM, AZ-NV", + "siteCode": [ + { + "value": "09423000", + "network": "NWIS", + "agencyCode": "USGS" + } + ], + "timeZoneInfo": { + "defaultTimeZone": { + "zoneOffset": "-07:00", + "zoneAbbreviation": "MST" + }, + "daylightSavingsTimeZone": { + "zoneOffset": "-06:00", + "zoneAbbreviation": "MDT" + }, + "siteUsesDaylightSavingsTime": false + }, + "geoLocation": { + "geogLocation": { + "srs": "EPSG:4326", + "latitude": 35.19166556, + "longitude": -114.5721876 + }, + "localSiteXY": [] + }, + "note": [], + "siteType": [], + "siteProperty": [ + { + "value": "ST", + "name": "siteTypeCd" + }, + { + "value": "15030101", + "name": "hucCd" + }, + { + "value": "32", + "name": "stateCd" + }, + { + "value": "32003", + "name": "countyCd" + } + ] + }, + "variable": { + "variableCode": [ + { + "value": "00060", + "network": "NWIS", + "vocabulary": "NWIS:UnitValues", + "variableID": 45807197, + "default": true + } + ], + "variableName": "Streamflow, ft³/s", + "variableDescription": "Discharge, cubic feet per second", + "valueType": "Derived Value", + "unit": { + "unitCode": "ft3/s" + }, + "options": { + "option": [ + { + "name": "Statistic", + "optionCode": "00000" + } + ] + }, + "note": [], + "noDataValue": -999999.0, + "variableProperty": [], + "oid": "45807197" + }, + "values": [ + { + "value": [ + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T11:30:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T11:45:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T12:00:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T12:15:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T12:30:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T12:45:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T13:00:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T13:15:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T13:30:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T13:45:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T14:00:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T14:15:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T14:30:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T14:45:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T15:00:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T15:15:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T15:30:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T15:45:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T16:00:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T16:15:00.000-07:00" + }, + { + "value": "-999999", + "qualifiers": [ + "P", + "Dis" + ], + "dateTime": "2024-06-12T16:30:00.000-07:00" + } + ], + "qualifier": [ + { + "qualifierCode": "Dis", + "qualifierDescription": "Record has been discontinued at the measurement site.", + "qualifierID": 0, + "network": "NWIS", + "vocabulary": "uv_rmk_cd" + }, + { + "qualifierCode": "P", + "qualifierDescription": "Provisional data subject to revision.", + "qualifierID": 1, + "network": "NWIS", + "vocabulary": "uv_rmk_cd" + } + ], + "qualityControlLevel": [], + "method": [ + { + "methodDescription": "", + "methodID": 6342 + } + ], + "source": [], + "offset": [], + "sample": [], + "censorCode": [] + } + ], + "name": "USGS:09423000:00060:00000" + } + ] + }, + "nil": false, + "globalScope": true, + "typeSubstituted": false +} \ No newline at end of file diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/16296500.json b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/16296500.json new file mode 100755 index 0000000..ff57755 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/analysis/test_data/JSON/16296500.json @@ -0,0 +1,692 @@ +{ + "name": "ns1:timeSeriesResponseType", + "declaredType": "org.cuahsi.waterml.TimeSeriesResponseType", + "scope": "javax.xml.bind.JAXBElement$GlobalScope", + "value": { + "queryInfo": { + "queryURL": "http://waterservices.usgs.gov/nwis/iv/sites=16296500&format=json¶meterCd=00060&period=PT6H", + "criteria": { + "locationParam": "[ALL:16296500]", + "variableParam": "[00060]", + "parameter": [] + }, + "note": [ + { + "value": "[ALL:16296500]", + "title": "filter:sites" + }, + { + "value": "[mode=PERIOD, period=PT6H, modifiedSince=null]", + "title": "filter:timeRange" + }, + { + "value": "methodIds=[ALL]", + "title": "filter:methodId" + }, + { + "value": "2024-06-12T23:17:56.933Z", + "title": "requestDT" + }, + { + "value": "00ae0750-2912-11ef-bdf8-2cea7f5e5ede", + "title": "requestId" + }, + { + "value": "Provisional data are subject to revision. Go to http://waterdata.usgs.gov/nwis/help/?provisional for more information.", + "title": "disclaimer" + }, + { + "value": "sdas01", + "title": "server" + } + ] + }, + "timeSeries": [ + { + "sourceInfo": { + "siteName": "Kahana Str at alt 30 ft nr Kahana, Oahu, HI", + "siteCode": [ + { + "value": "16296500", + "network": "NWIS", + "agencyCode": "USGS" + } + ], + "timeZoneInfo": { + "defaultTimeZone": { + "zoneOffset": "-10:00", + "zoneAbbreviation": "HST" + }, + "daylightSavingsTimeZone": { + "zoneOffset": "-09:00", + "zoneAbbreviation": "HDT" + }, + "siteUsesDaylightSavingsTime": false + }, + "geoLocation": { + "geogLocation": { + "srs": "EPSG:4326", + "latitude": 21.5406111, + "longitude": -157.8825556 + }, + "localSiteXY": [] + }, + "note": [], + "siteType": [], + "siteProperty": [ + { + "value": "ST", + "name": "siteTypeCd" + }, + { + "value": "20060000", + "name": "hucCd" + }, + { + "value": "15", + "name": "stateCd" + }, + { + "value": "15003", + "name": "countyCd" + } + ] + }, + "variable": { + "variableCode": [ + { + "value": "00060", + "network": "NWIS", + "vocabulary": "NWIS:UnitValues", + "variableID": 45807197, + "default": true + } + ], + "variableName": "Streamflow, ft³/s", + "variableDescription": "Discharge, cubic feet per second", + "valueType": "Derived Value", + "unit": { + "unitCode": "ft3/s" + }, + "options": { + "option": [ + { + "name": "Statistic", + "optionCode": "00000" + } + ] + }, + "note": [], + "noDataValue": -999999.0, + "variableProperty": [], + "oid": "45807197" + }, + "values": [ + { + "value": [ + { + "value": "22.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T07:20:00.000-10:00" + }, + { + "value": "22.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T07:25:00.000-10:00" + }, + { + "value": "22.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T07:30:00.000-10:00" + }, + { + "value": "22.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T07:35:00.000-10:00" + }, + { + "value": "22.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T07:40:00.000-10:00" + }, + { + "value": "22.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T07:45:00.000-10:00" + }, + { + "value": "22.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T07:50:00.000-10:00" + }, + { + "value": "22.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T07:55:00.000-10:00" + }, + { + "value": "22.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T08:00:00.000-10:00" + }, + { + "value": "22.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T08:05:00.000-10:00" + }, + { + "value": "22.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T08:10:00.000-10:00" + }, + { + "value": "22.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T08:15:00.000-10:00" + }, + { + "value": "22.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T08:20:00.000-10:00" + }, + { + "value": "21.9", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T08:25:00.000-10:00" + }, + { + "value": "21.9", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T08:30:00.000-10:00" + }, + { + "value": "21.9", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T08:35:00.000-10:00" + }, + { + "value": "21.9", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T08:40:00.000-10:00" + }, + { + "value": "21.9", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T08:45:00.000-10:00" + }, + { + "value": "21.7", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T08:50:00.000-10:00" + }, + { + "value": "21.7", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T08:55:00.000-10:00" + }, + { + "value": "21.7", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T09:00:00.000-10:00" + }, + { + "value": "21.7", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T09:05:00.000-10:00" + }, + { + "value": "21.7", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T09:10:00.000-10:00" + }, + { + "value": "21.5", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T09:15:00.000-10:00" + }, + { + "value": "21.5", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T09:20:00.000-10:00" + }, + { + "value": "21.5", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T09:25:00.000-10:00" + }, + { + "value": "21.5", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T09:30:00.000-10:00" + }, + { + "value": "21.5", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T09:35:00.000-10:00" + }, + { + "value": "21.5", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T09:40:00.000-10:00" + }, + { + "value": "21.5", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T09:45:00.000-10:00" + }, + { + "value": "21.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T09:50:00.000-10:00" + }, + { + "value": "21.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T09:55:00.000-10:00" + }, + { + "value": "21.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:00:00.000-10:00" + }, + { + "value": "21.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:05:00.000-10:00" + }, + { + "value": "21.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:10:00.000-10:00" + }, + { + "value": "21.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:15:00.000-10:00" + }, + { + "value": "21.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:20:00.000-10:00" + }, + { + "value": "21.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:25:00.000-10:00" + }, + { + "value": "21.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:30:00.000-10:00" + }, + { + "value": "21.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:35:00.000-10:00" + }, + { + "value": "21.3", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:40:00.000-10:00" + }, + { + "value": "21.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:45:00.000-10:00" + }, + { + "value": "21.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:50:00.000-10:00" + }, + { + "value": "21.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T10:55:00.000-10:00" + }, + { + "value": "21.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:00:00.000-10:00" + }, + { + "value": "21.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:05:00.000-10:00" + }, + { + "value": "21.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:10:00.000-10:00" + }, + { + "value": "21.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:15:00.000-10:00" + }, + { + "value": "21.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:20:00.000-10:00" + }, + { + "value": "21.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:25:00.000-10:00" + }, + { + "value": "21.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:30:00.000-10:00" + }, + { + "value": "21.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:35:00.000-10:00" + }, + { + "value": "21.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:40:00.000-10:00" + }, + { + "value": "21.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:45:00.000-10:00" + }, + { + "value": "21.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:50:00.000-10:00" + }, + { + "value": "20.9", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T11:55:00.000-10:00" + }, + { + "value": "20.9", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T12:00:00.000-10:00" + }, + { + "value": "21.1", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T12:05:00.000-10:00" + }, + { + "value": "20.9", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T12:10:00.000-10:00" + }, + { + "value": "20.9", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T12:15:00.000-10:00" + }, + { + "value": "20.9", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T12:20:00.000-10:00" + }, + { + "value": "20.9", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T12:25:00.000-10:00" + }, + { + "value": "20.9", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T12:30:00.000-10:00" + }, + { + "value": "20.9", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T12:35:00.000-10:00" + }, + { + "value": "20.9", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T12:40:00.000-10:00" + }, + { + "value": "20.7", + "qualifiers": [ + "P", + "e" + ], + "dateTime": "2024-06-12T12:45:00.000-10:00" + } + ], + "qualifier": [ + { + "qualifierCode": "e", + "qualifierDescription": "Value has been estimated.", + "qualifierID": 0, + "network": "NWIS", + "vocabulary": "uv_rmk_cd" + }, + { + "qualifierCode": "P", + "qualifierDescription": "Provisional data subject to revision.", + "qualifierID": 1, + "network": "NWIS", + "vocabulary": "uv_rmk_cd" + } + ], + "qualityControlLevel": [], + "method": [ + { + "methodDescription": "", + "methodID": 42319 + } + ], + "source": [], + "offset": [], + "sample": [], + "censorCode": [] + } + ], + "name": "USGS:16296500:00060:00000" + } + ] + }, + "nil": false, + "globalScope": true, + "typeSubstituted": false +} \ No newline at end of file diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/exnwm_usgs_timeslices.sh b/data_assimilation_engine/Streamflow_Scripts/usgs_download/exnwm_usgs_timeslices.sh new file mode 100755 index 0000000..df165ae --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/exnwm_usgs_timeslices.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash + +############################################################################### +# Program Name: nwm_usgs_timeslices # +# # +# Author(s)/Contact(s): NWC # +# # +# This is regriding USGS time slices creation # +# # +# Input: /lfs/h1/ops/prod/dcom/usgs_streamflow # +# # +# Output: YYYY-MM-DD_HH:mm:00.15min.usgsTimeSlice.ncdf # +# # +# For non-fatal errors output is witten to $DATA/LOGS # +# # +# Origination Feb, 2016 # +# OWP 12/07/2021 Port to WCOSS2 system # +# OWP 06/18/2024 Change Water ML 2.0 to Json data format # +# # +############################################################################### +# --------------------------------------------------------------------------- # +seton='-xa' +setoff='+xa' + +postmsg "$pgmout" "HAS BEGUN on `hostname`" +msg="Starting USGS real time time slices creation at `date`" +postmsg "$pgmout" "$msg" + +source $USHnwm/usgs_download/analysis/nwmcopy.sh + +set $setoff +echo ' ' +echo '********************************' +echo '** NWM USGS TIME SLICES **' +echo '********************************' +echo -e "\nStarting $0 at : `date`\n" +set $seton + +if [ ! -e $COMOUT/usgs_timeslices ]; then + mkdir -p $COMOUT/usgs_timeslices +fi + +################################# +# copy raw data files to local working dir +################################# +if [ ! -e $DATA/rawdatafiles ]; then + mkdir -p $DATA/rawdatafiles +fi + +for file in $DCOM/*.json +do + test -f "$file" || continue + cpfs $file $DATA/rawdatafiles +done +################################# +# copy existing data files +################################# +nwm_copy usgs_timeslices + +postmsg "$pgmout" "Execute $USHnwm/usgs_download/analysis/make_time_slice_from_usgs_waterml.py" +postmsg "$pgmout" "$USHnwm/usgs_download/analysis/make_time_slice_from_usgs_waterml.py depends on:" +postmsg "$pgmout" "$USHnwm/usgs_download/analysis/TimeSlice.py and $USHnwm/usgs_download/analysis/USGS_Observation.py" + +python $USHnwm/usgs_download/analysis/make_time_slice_from_usgs_waterml.py \ + -i $DATA/rawdatafiles -o $DATA >> $pgmout 2>&1 + +#$USHnwm/usgs_download/analysis/make_time_slice_from_usgs_waterml.py \ +# -i $DCOM -o $DATA >> $pgmout 2>&1 + +export err=$? #; err_chk + +if [ "$err" -ne 0 ]; then + errMsg=$(sed -n 's/^.* - __main__ - ERROR - //p' $pgmout) + + if [ ! -z "$errMsg" ]; then + echo -e "\n${jobid} failed because:\n\t${errMsg}\n" >> emailMsg + fi + + cat emailMsg | mail.py -v -s "ERROR: NWM USGS timeslice job Failed" $maillist2 +else + errMsg=$(sed -n 's/^[0-2][0-9]\{3\}-[01][0-9]-[0-3][0-9] [0-9:,]* - __main__ - WARNING - \(Input directory\)/\1/p' $pgmout) + + if [ "$errMsg" == "Input directory ${DATA}/rawdatafiles has no USGS Json files or the files are empty!" ]; then + hour=$(date +"%H"); min=$(date +"%M") + DISABLE_EMAIL=${DISABLE_USGS_EMAIL_ENV:-NO} + # + # Only send the warning message every other hour between minutes 10 to + # 25 + # + if [[ "$DISABLE_EMAIL" == "NO" && $((10#$hour % 2)) == 0 && \ + "$min" -ge 10 && "$min" -lt 25 ]]; then + errMsg="Input directory ${DCOM} has no USGS Json files or the files are empty!" + echo -e "\n${jobid} WARNING:\n\t${errMsg}\n" >> emailMsg + cat emailMsg | mail.py -v -s "WARNING: NWM USGS timeslice job has empty input" $maillist2 + else + postmsg "$pgmout" "WARNING: $errMsg" + fi + else + err_chk + + # + # send output time slice files to /com + # + if [ "$SENDCOM" == "YES" ]; then + nwm_postcopy usgs_timeslices usgsTimeSlice.ncdf + fi + fi +fi + +export err=$?; err_chk + +postmsg "$pgmout" "Finishing USGS real time time slices creation on `date`." + +#----------------------------------- +# Save log file to $COMOUT/logs +#---------------------------------- +if [ ! -e $COMOUT/logs ]; then + mkdir -p $COMOUT/logs +fi + +if [ -e $DATAlogs ]; then + gzip $pgmout + cp -rf ${pgmout}.gz $COMOUT/logs +fi + +export err=$?; err_chk + +set $setoff +echo ' ' +echo -e "\nEnding at : `date`\n" +echo '***********************************' diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/stream_flow_download/README.TXT b/data_assimilation_engine/Streamflow_Scripts/usgs_download/stream_flow_download/README.TXT new file mode 100755 index 0000000..6868d13 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/stream_flow_download/README.TXT @@ -0,0 +1,39 @@ +# Overview +This directory contains Python scripts to download USGS streamflow observations in realtime. The script runs as a daemon process when it is started. It will continously check the USGS server for any locations that have updated streamflow data since it was last checked, and then download only the updated streamflows. + +# Prerequisite +It requires an USGS API Key. USGS API Keys can be obtained from https://api.waterdata.usgs.gov/signup. + +A DBNROOT directory for storing log files. + +A DCOMROOT directory for storing the raw data files. + +# Script description +usgs_current.py: The main driver and the realtime download script using USGS OGC APIs + +# Script Usage + +## Setup environmental variables +Set the USGS_API_KEY, DBNROOT and DCOMROOT environmental variables +For example, + +``` +export USGS_API_KEY=[API KEY] +export DCOMROOT=/path/to/dcomroot +export DBNROOT=/path/to/dbnroot +mkdir $DBNROOT/log +``` +## Start the daemon script +``` +python ./usgs_current.py +``` +Output is saved in the directory pointed to by the $DCOMROOT variable. + +The log file is saved to the file defined by the $DBNROOT variable. + +# To kill the daemon program + +``` +ps -ef|grep usgs_current.py +kill -9 `pgrep -f parallel_download_master.py` +``` diff --git a/data_assimilation_engine/Streamflow_Scripts/usgs_download/stream_flow_download/usgs_current.py b/data_assimilation_engine/Streamflow_Scripts/usgs_download/stream_flow_download/usgs_current.py new file mode 100644 index 0000000..6a56738 --- /dev/null +++ b/data_assimilation_engine/Streamflow_Scripts/usgs_download/stream_flow_download/usgs_current.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 + +################################################################### +# Script: usgs_monitor.py +# Version: 1.0 +# Purpose: Find updated usgs steamflow gauge data and pull in json +# format. +# Author: Salemi 01/25/2025 +# +################################################################### + +import requests +import datetime +import json +import pytz +import os +import time +import sys +import signal +import concurrent.futures +import logging +from logging.handlers import TimedRotatingFileHandler + +# Import environmental variables +required_env_vars = ['DCOMROOT', 'DBNROOT'] +env_vars = {} + +for var in required_env_vars: + value = os.getenv(var) + if value: + # If the variable is found, store it in the dictionary. + env_vars[var] = value + else: + # If not found, log a critical error and exit. + print(f"Required {var} not found") + exit() + +# --- CONFIGURATION CONSTANTS --- +DBN_ROOT = env_vars['DBNROOT'] +DCOM_ROOT = env_vars['DCOMROOT'] +SLEEP_INTERVAL_MIN = 2 # How long to wait between checks +LOOKBACK_BUFFER_MIN = 3 # Time window to search for updates +MAX_WORKERS = 20 # Maximum number of concurrent threads for downloading + +BASE_URL_CHECK = "https://api.waterdata.usgs.gov/ogcapi/v0/collections/latest-continuous/items" +BASE_URL_PULL = "https://api.waterdata.usgs.gov/ogcapi/v0/collections/continuous/items" +# We only care about streamflow data (00060) +PARAMETER_CODE = '00060' +HEARTBEAT_TIMEOUT_MINS = 15 + +STATE_FILE = os.path.join(DBN_ROOT, 'user/usgs_api/last_pull_time.txt') +PID_FILE = os.path.join(DBN_ROOT, 'user/usgs_api/daemon.pid') + + +# --- HELPER FUNCTIONS --- + +def setup_logging(log_path): + + logger = logging.getLogger('usgs_streamflow_monitor.log') + logger.setLevel(logging.INFO) +# Create a TimedRotatingFileHandler. + handler = TimedRotatingFileHandler( + LOG_FILE_PATH, + when='midnight', + interval=1, + backupCount=7, + delay=True + ) +# Create a formatter and add it to the handler. + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + return logger + + +def load_manual_env(file_path=None): + if file_path is None: + file_path = os.path.join(DBN_ROOT, "user/usgs_api", ".usgs_env") + + if os.path.exists(file_path): + with open(file_path, "r") as f: + for line in f: + # Remove whitespace and ignore comments or empty lines + line = line.strip() + if not line or line.startswith("#"): + continue + + # Split by the first '=' found + if "=" in line: + key, value = line.split("=", 1) + # Set it in the environment so os.environ.get works later + os.environ[key.strip()] = value.strip().strip('"').strip("'") + + +def load_last_pull_time(): + """Loads the last recorded pull time from the state file.""" + if os.path.exists(STATE_FILE): + with open(STATE_FILE, 'r') as f: + # Strip newline and return the ISO 8601 string + return f.read().strip() + + # If file doesn't exist, start 3 minutes in the past + logger.info("INFO: No state file found. Starting 5 minutes ago.") + initial_start = datetime.datetime.now(pytz.utc) - datetime.timedelta(minutes=5) + return initial_start.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def save_last_pull_time(timestamp_str): + """Saves the current time as the new starting time for the next cycle.""" + with open(STATE_FILE, 'w') as f: + f.write(timestamp_str) + logger.info(f"State updated to {timestamp_str}") + + +def get_time_range(last_pull_time_str): + """Calculates the time range (start/end) for the API query.""" + # Ensure current time is UTC + now_utc = datetime.datetime.now(pytz.utc) + + # Calculate the end time (1 minute ago for safety) + end_time = now_utc - datetime.timedelta(minutes=1) + end_time_str = end_time.strftime("%Y-%m-%dT%H:%M:%SZ") + + # Use the previous pull time as the start time + start_time_str = last_pull_time_str + + return start_time_str, end_time_str + + +def fetch_full_site_data(site_id, odir, site_name): + """Fetches the full JSON data for a specific site and saves it.""" + + # The time parameter requests the last 6 hours of instantaneous data (PT6H) + params = { + 'monitoring_location_id': site_id, + 'parameter_code': PARAMETER_CODE, + 'time': 'PT6H', + 'f': 'json', # Request JSON format explicitly + 'limit': 50 + } + + API_KEY = os.environ.get("USGS_API_KEY") + + headers = { + 'X-Api-Key': API_KEY, + 'Accept': 'application/json' + } + + try: + response = requests.get(BASE_URL_PULL, params=params, headers=headers, timeout=15) + response.raise_for_status() + data = response.json() + + data['monitoring_location_name'] = site_name + + # Save the file + site_id = site_id.replace('USGS-', '') + file_name = f"{site_id}.json" + output_path = os.path.join(odir, file_name) + + with open(output_path, 'w') as f: + json.dump(data, f, indent=2) + + os.chmod(output_path, 0o664) + + logger.info(f"SUCCESS: Downloaded and saved data for site {site_id}.") + return True + + except requests.exceptions.RequestException as e: + logger.info(f"WARNING: Failed to pull full data for site {site_id}. Error: {e}") + return False + + +def run_monitor_cycle(last_pull_time_str, odir): + """Runs one complete cycle: discovers updated sites and initiates data pull.""" + + start_time_str, end_time_str = get_time_range(last_pull_time_str) + + logger.info(f"\n--- Starting Check: {start_time_str} to {end_time_str} ---") + + time_range_value = f"{start_time_str}/{end_time_str}" + + try: + # datetime_value_encoded = quote(time_range_value, safe=':/Z-') + + # --- 1. Discover Updated Sites --- + params = { + 'last_modified': time_range_value, + 'parameter_code': PARAMETER_CODE, + 'limit': 700 # Adjust based on expected volume + } + + response = requests.get(BASE_URL_CHECK, params=params, timeout=30) + response.raise_for_status() + data = response.json() + + except requests.exceptions.RequestException as e: + logger.info(f"FATAL: Discovery API request failed. Skipping cycle. Error: {e}") + return last_pull_time_str, 0 # Return old time to retry range later + + features = data.get('features', []) + updated_sites = {} + + for feature in features: + # Extract the site ID from the properties dictionary + props = feature.get('properties', {}) + site_id = props.get('monitoring_location_id') + site_name = ( + props.get('monitoring_location_name') or + props.get('name') or + "Unknown Name" + ) + + if site_id: + # Remove the "USGS-" prefix for consistency with the site URL query + # site_id = site_id.replace('USGS-', '') + updated_sites[site_id] = site_name + + if not updated_sites: + logger.info("INFO: No sites found with updates in the interval.") + # Only update state time if we successfully queried the server + return end_time_str, 0 + + logger.info(f"INFO: Found {len(updated_sites)} unique sites with updates.") + + # --- 2. Pull Full Data for Each Updated Site (Parallelized) --- + successful_downloads = 0 + + # Using ThreadPoolExecutor to run downloads concurrently + with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: + # Submitting all download tasks to the thread pool + future_to_site = { + executor.submit(fetch_full_site_data, sid, odir, sname): sid + for sid, sname in updated_sites.items() + } + + # Waiting for results and counting successes + for future in concurrent.futures.as_completed(future_to_site): + site_id = future_to_site[future] + try: + # future.result() returns the boolean (True/False) result of fetch_full_site_data + if future.result(): + successful_downloads += 1 + except Exception as exc: + logger.info(f'WARNING: Site {site_id} generated an unhandled exception during pull: {exc}') + + # Only update the state time if the discovery was successful + logger.info(f"INFO: {successful_downloads} out of {len(updated_sites)} sites downloaded successfully.") + return end_time_str, successful_downloads + + +def is_process_running(pid): + """Check if there is any running process with given PID.""" + try: + os.kill(pid, 0) # Signal 0 does nothing but checks if process exists + except OSError: + return False + return True + + +def manage_process_state(): + # 1. Check if PID file exists + if os.path.exists(PID_FILE): + with open(PID_FILE, 'r') as f: + try: + old_pid = int(f.read().strip()) + except ValueError: + old_pid = None + + if old_pid and is_process_running(old_pid): + # 2. Check if the state file is "stale" + if os.path.exists(STATE_FILE): + last_mtime = os.path.getmtime(STATE_FILE) + seconds_since_update = time.time() - last_mtime + + if seconds_since_update > (HEARTBEAT_TIMEOUT_MINS * 60): + logger.info(f"Stale process detected (PID {old_pid}). Killing and restarting...") + try: + os.kill(old_pid, signal.SIGTERM) + time.sleep(2) # Give it a moment to release ports/files + except OSError: + pass + else: + logger.info(f"Daemon already running (PID {old_pid}) and active. Exiting.") + sys.exit(0) + + # 3. Create/Overwrite PID file for the current process + with open(PID_FILE, 'w') as f: + f.write(str(os.getpid())) + + +# --- MAIN EXECUTION BLOCK --- +def run_daemon(): + # Load initial state + last_pull_time = load_last_pull_time() + + # Define the main monitoring loop + while True: + try: + current_date = datetime.datetime.now().strftime('%Y%m%d') + output_dir = os.path.join(DCOM_ROOT, current_date, 'obs/raw/water_level/usgs_streamflow') + # Setup output directory + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + new_parts = [current_date, "obs", "raw", "water_level", "usgs_streamflow"] + + current_path = DCOM_ROOT + for part in new_parts: + current_path = os.path.join(current_path, part) + # Apply 775 only to these segments + os.chmod(current_path, 0o775) + logger.info(f"INFO: Created output directory: {output_dir}") + + # Run the data retrieval cycle + new_pull_time, count = run_monitor_cycle(last_pull_time, output_dir) + + # Update the state for the next run + if new_pull_time != last_pull_time: + save_last_pull_time(new_pull_time) + last_pull_time = new_pull_time + + # Wait for the next interval + logger.info(f"\nINFO: Cycle complete. Sleeping for {SLEEP_INTERVAL_MIN} minutes...") + time.sleep(SLEEP_INTERVAL_MIN * 60) + + except KeyboardInterrupt: + logger.info("\nMonitor stopped by user.") + sys.exit(0) + except Exception as e: + logger.info(f"CRITICAL ERROR in main loop: {e}. Sleeping for 5 minutes.") + time.sleep(300) # Sleep longer on critical errors + + +if __name__ == "__main__": + load_manual_env() + + lock_file = manage_process_state() + + LOG_FILE_PATH = os.path.join(DBN_ROOT, "log", "usgs_streamflow_monitor.log") + logger = setup_logging(LOG_FILE_PATH) + + logger.info("Daemon started successfully. Monitoring cycle beginning...") + + try: + run_daemon() + finally: + # Cleanup PID file on clean exit + if os.path.exists(PID_FILE): + os.remove(PID_FILE) diff --git a/data_assimilation_engine/output_variables/DataProcessor.py b/data_assimilation_engine/output_variables/DataProcessor.py index 4eee28c..33deb08 100644 --- a/data_assimilation_engine/output_variables/DataProcessor.py +++ b/data_assimilation_engine/output_variables/DataProcessor.py @@ -1,323 +1,824 @@ +import sys import os +import json import geopandas as gpd import numpy as np +import pandas as pd import xarray as xr import dask.array as da import time -import fiona -from shapely.geometry import Point +import math +import re +import shapely +# from shapely.geometry import Point +# from shapely.ops import unary_union +# from shapely import intersects_xy, intersects +# from shapely import points from pyproj import CRS from typing import List, Optional from .DataReader import DataReader - +from .utils import NetCDFMetadata +from . import consts class DataProcessor(DataReader): """ Handles catchment-time NetCDF and grid generation. """ - def __init__(self, netcdf_file: str, gpkg_file: str, template_grid_file: str, - produce_template: bool, produce_output: bool, chunk_size: int = 100) -> None: - super().__init__(netcdf_file, chunk_size) - - self.input_ds: xr.Dataset = self.dataset - self.output_variables = list(self.input_ds.data_vars) - # Normalize catchment ids in netcdf with a cat- prefix for consistency. - self.nc_catchments = self._normalize_catchment_ids(self.input_ds[self.catchment_coord].values) - + def __init__(self, catchment_netcdf_file: str, gpkg_file: str, chunk_size: int = 100) -> None: + + super().__init__(catchment_netcdf_file, chunk_size) + + self._catchment_ds: xr.Dataset = self.dataset + self._gpkg_file = gpkg_file + self._output_class = None + self._category = None + self._domain = None + filename = os.path.basename(gpkg_file) + self._geo_id = filename.replace(consts.GPKG_FILE_PREFIX, '').replace('.gpkg', '') + + nc_catchments = self._catchment_ds[self.catchment_coord].values + # Read geopackage, determine schema and "divides" layer - self.is_new_NHF_schema: bool = self._is_new_NHF_schema(gpkg_file, "reference_flowpaths") - print (self.is_new_NHF_schema) - gpkg_gdf = gpd.read_file(gpkg_file, layer="divides") + is_new_NHF_schema: bool = self._is_new_NHF_schema(gpkg_file, consts.NHF_REF_OBJECT) + gpkg_gdf = gpd.read_file(gpkg_file, layer=consts.GPKG_DIVIDES_LYR) if gpkg_gdf.empty: raise ValueError("No polygon geometries found in GeoPackage") + gpkg_gdf["geometry"] = gpkg_gdf["geometry"].make_valid() # Check schema and assign catchment ID field. - if self.is_new_NHF_schema: - self.catchment_field = "div_id" + if is_new_NHF_schema: + self._catchment_field = consts.NHF_DIV_ID else: - self.catchment_field = "divide_id" - - # Normalize catchment ids in gpkg with a 'cat-' prefix for consistency. - gpkg_gdf[self.catchment_field] = self._normalize_catchment_ids(gpkg_gdf[self.catchment_field].values) - self.gpkg_catchment_list = gpkg_gdf[self.catchment_field].values.tolist() + self._catchment_field = consts.NONNHF_DIV_ID + gpkg_catchment_list = gpkg_gdf[self._catchment_field].values.tolist() + self._gpkg_gdf = gpkg_gdf + # Check if catchments from netcdf and geopackage match - if set(self.nc_catchments) != set(self.gpkg_catchment_list): - raise ValueError("There is a mistmatch between catchment IDs in geopackage and netcdf") - - if produce_template: - self.create_template_grid_netcdf_from_geopackage(gpkg_gdf, template_grid_file) - self.ds_grid = xr.open_dataset(template_grid_file) + if set(nc_catchments) != set(gpkg_catchment_list): + raise ValueError("There is a mistmatch between catchment IDs in geopackage and netcdf") - # explicitly set CRS to CONUS. - # TO DO: Read an existing NWM grid file for CRS info - self.ds_grid = self.ds_grid.rio.write_crs("EPSG:5070") - - if produce_output: - self.grid_points = self.build_grid_centroids() - self.build_grid_lookup(gpkg_gdf) - output_path = 'sample_data/sample_netcdf/sample_output/final_test' #Change this to command line argument later - start_time = time.perf_counter() - self.create_grids_per_timestep_dask(output_path) - end_time = time.perf_counter() - duration_minutes = (end_time - start_time) / 60 - print(f"Function execution time: {duration_minutes:.4f} minutes") - + # Set log file which will be redirect to stdout. + # Only for testing + self._original_stdout = sys.stdout + self._log_file = None + + @property + def nwm_output_class(self): + return self._output_class + + @property + def nwm_category(self): + return self._category + + @property + def nwm_domain(self): + return self._domain + + @property + def log_file(self): + return self._log_file + + @property + def geo_id(self): + return self._geo_id + + @nwm_output_class.setter + def nwm_output_class(self, value): + self._output_class = value + + @nwm_category.setter + def nwm_category(self, value): + self._category = value + + @nwm_domain.setter + def nwm_domain(self, value): + self._domain = value + + @log_file.setter + def log_file(self, log_file_path: str): + self.close_log() + if log_file_path is None: + return + + log_folder = os.path.dirname(log_file_path) + if log_folder: + os.makedirs(log_folder, exist_ok=True) + + self._log_file = open(log_file_path, "a", encoding="utf-8", buffering=1) + sys.stdout = self._log_file + + def set_template_netcdf(self, template_grid_path: str): + self._template_netcdf_ds = xr.open_dataset(template_grid_path) + + def set_troute_netcdf(self, troute_outpath: str): + self._troute_netcdf_ds = xr.open_dataset(troute_outpath) + + def set_troute_lakeout_netcdf(self, troute_lakeoutpath: str): + self._troute_lakeout_netcdf_ds = xr.open_dataset(troute_lakeoutpath) + def _is_new_NHF_schema(self, geopackage_path: str, layer_name: str) -> bool: """ Check the data schema in the geopackage for the new NHF format """ layers = gpd.list_layers(geopackage_path) - tabular_layers = layers[layers['geometry_type'].isna()] + tabular_layers = layers[layers[consts.GPKG_GEOMETRY_TYPE_IDENTIFIER].isna()] return layer_name.lower() in tabular_layers['name'].tolist() - def _normalize_catchment_ids(self, ids) -> np.ndarray: - """ - Ensure all IDs are prefixed with 'cat-'. This is necessary when catchments are not prefixed - with 'cat-' in netcdf and geopackage for consistency. - """ - ids_str = ids.astype(str) - - normalized = np.array([ - id_ if id_.startswith("cat-") else f"cat-{id_}" - for id_ in ids_str - ]) - return normalized + def _snap_to_grid(self, value: float, origin: float, resolution: int, direction: np.ufunc): + if direction == np.floor: + return origin + np.floor((value - origin) / resolution) * resolution + elif direction == np.ceil: + return origin + np.ceil((value - origin) / resolution) * resolution - def create_template_grid_netcdf_from_geopackage(self, geopackage_gdf: gpd.GeoDataFrame, output_grid_netcdf: str, - resolution: int = 1000, epsg_code: int = 5070) -> None: + def old_create_template_netcdf_using_config(self, mdata: NetCDFMetadata, template_netcdf_folder: str) -> None: """ - Create a NetCDF grid with 1 km x 1 km resolution - covering the extent of catchment polygons. - - Default CRS: EPSG:5070 (NAD83 / Conus Albers, meters) + Create a template grid aligned to a reference grid defined in config.json. + The output template grid covers the extents of the divides in the geopackage. """ - if os.path.exists(output_grid_netcdf) == False: - target_crs = CRS.from_epsg(epsg_code) - gdf = geopackage_gdf.to_crs(target_crs) - - # Get bounding box - minx, miny, maxx, maxy = gdf.total_bounds + os.makedirs(template_netcdf_folder, exist_ok = True) + + self._output_class = mdata.output_class + self._category = mdata.category + self._domain = mdata.domain + origin_x = mdata.origin_x + origin_y = mdata.origin_y + res_x = mdata.resolution_x + res_y = mdata.resolution_y + wkt = mdata.crs_wkt + file_name = mdata.file_path + x_name = mdata.x_name + y_name = mdata.y_name + + # Check if the template file already exists for this request + template_nc_name = self._geo_id + '_' + self._output_class + '_' + self._category + '_' + self._domain + '.nc' + template_nc_file = os.path.join(template_netcdf_folder, template_nc_name) + if os.path.isfile(template_nc_file): + print(f"----Reusing existing template file in local for {self._output_class}, {self._category}, {self._domain}") + elif mdata.category.startswith('channel_rt') or mdata.category.startswith('reservoir'): # indicates that it is non-geospatial. For example, channel_rt + ds = xr.open_dataset(file_name) + + # Delete any variable that is in the ignore list. Zero the valid min and max attribute in the time dimension + ds = ds.drop_vars(consts.NWM_VARS_IGNORE_LIST, errors="ignore") + if consts.DIM_TIME in ds.coords: + attrs_to_reset = ['valid_min', 'valid_max'] + for attr in attrs_to_reset: + if attr in ds[consts.DIM_TIME].attrs: + ds[consts.DIM_TIME].attrs[attr] = 0 + + # Slice all coordinates and variable arrays to zero length. + dims = list(ds.sizes.keys()) + zero_slices = {dim: slice(0, 0) for dim in dims} + ds_template = ds.isel(zero_slices) + + # Save to nc file + ds_template.to_netcdf(template_nc_file) + else: + ds = xr.open_dataset(file_name) + # To do: Have to figure out a workflow when CRS is "Not Available" + target_crs = CRS.from_user_input(wkt) - # Create grid coordinates and default to zero for the data - x_coords = np.arange(minx, maxx + resolution, resolution) - y_coords = np.arange(miny, maxy + resolution, resolution) - grid_data = np.zeros((len(y_coords), len(x_coords))) + gdf = self._gpkg_gdf.to_crs(target_crs) + geom = shapely.ops.unary_union(gdf.geometry) + + # Get bounding box and snap to origin in the national reference grid + minx, miny, maxx, maxy = gdf.total_bounds - ds = xr.Dataset( + snapped_minx = self._snap_to_grid(minx, origin_x, res_x, np.floor) + snapped_maxx = self._snap_to_grid(maxx, origin_x, res_x, np.ceil) + snapped_miny = self._snap_to_grid(miny, origin_y, res_y, np.floor) + snapped_maxy = self._snap_to_grid(maxy, origin_y, res_y, np.ceil) + + # Filter national grid to a sub-grid within the snapped bounding box + # 1. Determine if the national grid Y-coordinate counts upwards or downwards + y_dir = 1 if ds[y_name].values[1] > ds[y_name].values[0] else -1 + + # 2. Slice dynamically based on the direction of the reference grid + if y_dir == 1: + y_slice = slice(snapped_miny, snapped_maxy) # South to North + else: + y_slice = slice(snapped_maxy, snapped_miny) # North to South + + # 3. Perform your selection using the verified slice orientation + ds_subset = ds.sel( + { + x_name: slice(snapped_minx, snapped_maxx), + y_name: y_slice #slice(snapped_miny, snapped_maxy), + } + ) + x_subset = ds_subset[x_name].values + y_subset = ds_subset[y_name].values + xx, yy = np.meshgrid(x_subset, y_subset) + + flat_mask = shapely.intersects_xy(geom, xx.ravel(), yy.ravel()) + grid_mask = flat_mask.reshape(ds_subset[y_name].size, ds_subset[x_name].size) + mask_da = xr.DataArray(grid_mask, + dims=(y_name, x_name), coords={ - "x": ("x", x_coords), - "y": ("y", y_coords) + y_name: ds_subset[y_name].values, + x_name: ds_subset[x_name].values } ) - # Coordinate metadata - ds["x"].attrs["standard_name"] = "projection_x_coordinate" - ds["x"].attrs["units"] = "m" + ds_masked = ds_subset.copy() + + for var in ds_subset.data_vars: + da = ds_subset[var] + # Only mask numeric variables + if np.issubdtype(da.dtype, np.number): + ds_masked[var] = da.where(mask_da) + else: + ds_masked[var] = da # Leave non-numeric untouched + + # Drop empty rows/cols that are outside of the polygon boundary + dataset_mask = ds_masked.to_array().notnull().any(dim="variable") + ds_clipped = ds_masked.dropna(dim=y_name, how="all") + ds_clipped = ds_clipped.dropna(dim=x_name, how="all") + + # Set values of NWM variables to zero in the template grid + nwm_vars = [name for name, var in ds_clipped.data_vars.items() + if var.ndim > 0 and name not in ds_clipped.coords] + for var in nwm_vars: + ds_clipped[var] = ds_clipped[var] * 0 + + # Create crs as a scalar variable. + crs_attrs = ds_clipped["crs"].attrs + ds_clipped = ds_clipped.drop_encoding() + del ds_clipped["crs"] + ds_clipped["crs"] = xr.DataArray("", dims=()) + ds_clipped["crs"].attrs = crs_attrs + + print("--- FINAL DATASET CHECK FOR 3S ---") + print(f"Dimension name for Y is '{y_name}', first 3 values are: {ds_clipped[y_name].values[:3]}") + print(f"Dimension name for X is '{x_name}', first 3 values are: {ds_clipped[x_name].values[:3]}") + + # Save to nc file + ds_clipped.to_netcdf(template_nc_file) + + self._template_netcdf_ds = xr.open_dataset(template_nc_file) + print(f"NetCDF template grid written to {template_nc_file}") + + def create_template_netcdf_using_config(self, mdata: NetCDFMetadata, template_netcdf_folder: str) -> None: + """ + Create a template grid aligned to a reference grid defined in config.json. + The output template grid covers the extents of the divides in the geopackage. + """ - ds["y"].attrs["standard_name"] = "projection_y_coordinate" - ds["y"].attrs["units"] = "m" + os.makedirs(template_netcdf_folder, exist_ok = True) + + self._output_class = mdata.output_class + self._category = mdata.category + self._domain = mdata.domain + origin_x = mdata.origin_x + origin_y = mdata.origin_y + res_x = mdata.resolution_x + res_y = mdata.resolution_y + wkt = mdata.crs_wkt + file_name = mdata.file_path + x_name = mdata.x_name + y_name = mdata.y_name + + # Check if the template file already exists for this request + template_nc_name = self._geo_id + '_' + self._output_class + '_' + self._category + '_' + self._domain + '.nc' + template_nc_file = os.path.join(template_netcdf_folder, template_nc_name) + if os.path.isfile(template_nc_file): + print(f"----Reusing existing template file in local for {self._output_class}, {self._category}, {self._domain}") + elif mdata.category.startswith('channel_rt') or mdata.category.startswith('reservoir'): # indicates that it is non-geospatial. For example, channel_rt + ds = xr.open_dataset(file_name) + + # Delete any variable that is in the ignore list. Zero the valid min and max attribute in the time dimension + ds = ds.drop_vars(consts.NWM_VARS_IGNORE_LIST, errors="ignore") + if consts.DIM_TIME in ds.coords: + attrs_to_reset = ['valid_min', 'valid_max'] + for attr in attrs_to_reset: + if attr in ds[consts.DIM_TIME].attrs: + ds[consts.DIM_TIME].attrs[attr] = 0 + + # Slice all coordinates and variable arrays to zero length. + dims = list(ds.sizes.keys()) + zero_slices = {dim: slice(0, 0) for dim in dims} + ds_template = ds.isel(zero_slices) + + # Save to nc file + ds_template.to_netcdf(template_nc_file) + else: + ds = xr.open_dataset(file_name) + # To do: Have to figure out a workflow when CRS is "Not Available" + target_crs = CRS.from_user_input(wkt) + + gdf = self._gpkg_gdf.to_crs(target_crs) + union_geom = shapely.ops.unary_union(gdf.geometry) + + # Get bounding box and snap to origin in the national reference grid + minx, miny, maxx, maxy = gdf.total_bounds - # Write CRS. - ds.rio.write_crs("EPSG:5070", inplace=True) + # snapped_minx = self._snap_to_grid(minx, origin_x, res_x, np.floor) + # snapped_maxx = self._snap_to_grid(maxx, origin_x, res_x, np.ceil) + # snapped_miny = self._snap_to_grid(miny, origin_y, res_y, np.floor) + # snapped_maxy = self._snap_to_grid(maxy, origin_y, res_y, np.ceil) + + snapped_minx = np.floor(minx / res_x) * res_x + snapped_miny = np.floor(miny / res_y) * res_y + snapped_maxx = np.ceil(maxx / res_x) * res_x + snapped_maxy = np.ceil(maxy / res_y) * res_y + + # Filter national grid to a sub-grid within the snapped bounding box using slice + ds_subset = ds.sortby([x_name, y_name]).sel( + { + x_name: slice(snapped_minx, snapped_maxx), + y_name: slice(snapped_miny, snapped_maxy) + } + ) - # Without the following line the grid was not lining up with the catchments - # Set the X and Y as spatial dims for rioxarray - ds.rio.set_spatial_dims(x_dim="x", y_dim="y", inplace=True) + # create 1D coordinate arrays for the snapped subset + x_subset_1D = ds_subset[x_name].values + y_subset_1D = ds_subset[y_name].values + xx, yy = np.meshgrid(x_subset_1D, y_subset_1D) - # Save - ds.to_netcdf(output_grid_netcdf) - print(f"NetCDF template grid written to {output_grid_netcdf}") - - def build_grid_centroids(self) -> gpd.GeoDataFrame: - """ - Create GeoDataFrame of grid cell centroids. - """ - x = self.ds_grid["x"].values - y = self.ds_grid["y"].values - xx, yy = np.meshgrid(x, y) - points = [Point(px, py) for px, py in zip(xx.ravel(), yy.ravel())] + shapely.prepare(union_geom) # for faster processing, just in case. + flat_mask = shapely.intersects_xy(union_geom, xx.ravel(), yy.ravel()) + grid_mask = flat_mask.reshape(len(y_subset_1D), len(x_subset_1D)) + + mask_da = xr.DataArray(grid_mask, + dims=(y_name, x_name), + coords={ + y_name: y_subset_1D, + x_name: x_subset_1D + } + ) + ds_masked = ds_subset.copy() + + for var in ds_subset.data_vars: + da = ds_subset[var] + # Only mask numeric variables + if np.issubdtype(da.dtype, np.number): + ds_masked[var] = da.where(mask_da) + else: + ds_masked[var] = da # Leave non-numeric untouched + + # Combine variables to find valid outer coordinate indices + combined_mask = ds_masked.to_array().notnull() + dims_to_collapse = [dim for dim in combined_mask.dims if dim not in [x_name, y_name]] + dataset_mask = combined_mask.any(dim=dims_to_collapse) + + # Extract non-null coordinates along each axis to locate the outer envelope borders + y_valid = ds_masked[y_name].where(dataset_mask.any(dim=x_name), drop=True) + x_valid = ds_masked[x_name].where(dataset_mask.any(dim=y_name), drop=True) + + if y_valid.size > 0 and x_valid.size > 0: + ymin, ymax = y_valid.values.min(), y_valid.values.max() + xmin, xmax = x_valid.values.min(), x_valid.values.max() + ds_clipped = ds_masked.sel({ + x_name: slice(xmin, xmax), + y_name: slice(ymin, ymax) + }) + else: + ds_clipped = ds_masked.copy() + + # Set values of NWM variables to zero in the template grid + nwm_vars = [name for name, var in ds_clipped.data_vars.items() + if var.ndim > 0 and name not in ds_clipped.coords] + for var in nwm_vars: + ds_clipped[var] = ds_clipped[var] * 0 + + # Create crs as a scalar variable. + crs_attrs = ds_clipped["crs"].attrs + ds_clipped = ds_clipped.drop_encoding() + del ds_clipped["crs"] + ds_clipped["crs"] = xr.DataArray("", dims=()) + ds_clipped["crs"].attrs = crs_attrs + + # Save to nc file + ds_clipped.to_netcdf(template_nc_file) + + self._template_netcdf_ds = xr.open_dataset(template_nc_file) + print(f"NetCDF template grid written to {template_nc_file}") + + def produce_nwm_output_product(self, mdata: NetCDFMetadata, output_dir: str) -> None: + produce_output = False + is_gridded = True + ds_modified = self._catchment_ds + + # if the output needs to have SOIL_M or SOIL_T, we need to + # stack the ngen output into the layers. + var_prefix_list = [] + if 'SOIL_M' in mdata.nwm_variables: + var_prefix_list.append('SOIL_M_') + if 'SOIL_T' in mdata.nwm_variables: + var_prefix_list.append('SOIL_T_') + + if len(var_prefix_list) > 0: + ds_modified = self.stack_soil_variables(var_prefix_list) - return gpd.GeoDataFrame( - {"cell_index": np.arange(len(points))}, - geometry=points, - crs=self.ds_grid.rio.crs - ) + # if the output needs to have SNLIQ (snow layer liquid water), we need to expand + # dimensions to include a snow layer. It is assumed to be of length=1 + if 'SNLIQ' in mdata.nwm_variables: + expanded_var = ds_modified['SNLIQ'].expand_dims(dim = consts.DIM_SNOW_LYR) + expanded_var = expanded_var.transpose(consts.DIM_TIME, consts.DIM_SNOW_LYR, consts.DIM_CATCHMENTS) + ds_modified['SNLIQ'] = expanded_var + + cat_class_domain = mdata.output_class + '.' + mdata.category + '.' + mdata.domain + if (cat_class_domain in consts.NWM_PRODUCTS_LIST): + produce_output = True + if mdata.category in ['channel_rt', 'reservoir']: + is_gridded = False + + print(f"----Produce output: {produce_output} and Gridded: {is_gridded}") + + if produce_output and is_gridded: + # Remove data variables that should not be part of the product. + # You can remove variables that are in the ignore variables as well. + target_variables = [var.strip() for var in mdata.nwm_variables.split(",")] + removed_items = list(set(target_variables).intersection(set(consts.NWM_VARS_IGNORE_LIST))) # for logging + print(f"----Variables ignored: {removed_items}") + ignore_set = set(consts.NWM_VARS_IGNORE_LIST) + pruned_variables = [item for item in target_variables if item not in ignore_set] + variables_to_drop = [var for var in ds_modified.data_vars if var not in pruned_variables and len(ds_modified[var].dims) > 0] + ds_filtered = ds_modified.drop_vars(variables_to_drop, errors="ignore") + + # Log any variables that are missing in ngen output. + for var_name in pruned_variables: + if var_name in ds_filtered.data_vars: + continue # the variable exists in ngen output. We don't need to do anything + else: + # If not in ngen output + print(f"----'{var_name}' is missing in ngen output") + + catchment_grid = self.build_catchment_id_grid(mdata.x_name, mdata.y_name) + mapped_grid = self.map_catchment_data_to_grid(ds_filtered, catchment_grid, consts.DIM_CATCHMENTS, mdata.x_name, mdata.y_name) - def build_grid_lookup(self, catchment_gdf: gpd.GeoDataFrame) -> None: + start_time = time.perf_counter() + self.write_netcdf_per_timestep(mapped_grid, output_dir, consts.DIM_TIME) + end_time = time.perf_counter() + duration_minutes = (end_time - start_time) / 60 + print(f"----Function execution time: {duration_minutes:.2f} minutes") + elif produce_output and not is_gridded: + self.produce_channel_reservoir_nwm_product(mdata, output_dir) + else: + print(f"----Production skipped for {cat_class_domain}") + + def stack_soil_variables(self, var_prefix_list: List[str]) -> xr.Dataset: + stacked_ds = self._catchment_ds.copy() + for var_prefix in var_prefix_list: + matching_vars = [var for var in stacked_ds.data_vars if var.startswith(var_prefix)] + if not matching_vars: + raise ValueError(f"ngen output netcdf has no variables found matching the prefix '{var_prefix}'") + + var_val_dict = {} + for var in stacked_ds.data_vars: + if var in matching_vars: + soil_depth = var[len(var_prefix) :] + depth_num = re.search(r"\d*\.\d+|\d+", soil_depth) + var_val_dict[var] = float(depth_num.group()) if depth_num else 0.0 + + sorted_vars = sorted(var_val_dict.keys(), key = lambda k: var_val_dict[k]) + arrays_for_stack = [stacked_ds[v].rename(var_prefix.strip("_")) for v in sorted_vars] + stacked_var = xr.concat(arrays_for_stack, dim = consts.DIM_SOIL_LYR) + stacked_var = stacked_var.transpose(consts.DIM_TIME, consts.DIM_SOIL_LYR, consts.DIM_CATCHMENTS) + stacked_ds[var_prefix.strip("_")] = stacked_var + stacked_ds = stacked_ds.drop_vars(matching_vars) + return stacked_ds + + def build_catchment_id_grid(self, x_dim_name: str, y_dim_name: str) -> xr.DataArray: """ - Build the grid-to-catchment mapping (vectorized index array). - Must be called just once before generating grids for every timestep. + Returns a DataArray (y, x) where each cell in the basin-level grid contains a catchment ID. """ - # Spatial join - # TO DO: Have to determine if there are other methods that are less expensive computationally. - joined = gpd.sjoin( - self.grid_points, - catchment_gdf[[self.catchment_field, "geometry"]], - how="left", - predicate="within" - ) - - # Build index map to allow working with integer indices (potentially faster than catchment ID strings) - catchment_index = {c: i for i, c in enumerate(self.nc_catchments)} #use netcdf list to ensure order of gpkg catchments match with netcdf. - mapped = joined[self.catchment_field].map(catchment_index) - grid_to_catchment = mapped.fillna(-1).astype(int).to_numpy() - self.grid_to_catchment = grid_to_catchment - print("Grid-to-catchment mapping built!") + try: + ds = self._template_netcdf_ds + x = ds[x_dim_name].values + y = ds[y_dim_name].values + + # Origin point is assumed to be bottom-left. It may be others too. + # So, sort the values for consistency + if x[0] > x[-1]: + ds = ds.sortby(x_dim_name) + x = ds[x_dim_name].values + if y[0] > y[-1]: + ds = ds.sortby(y_dim_name) + y = ds[y_dim_name].values + + # Build centroid points and create geodataframe + xx, yy = np.meshgrid(x, y) + df_points = pd.DataFrame({ + x_dim_name: xx.ravel(), + y_dim_name: yy.ravel() + }) + crs = CRS.from_wkt(ds["crs"].attrs["spatial_ref"]) + gdf_ncgrid_points = gpd.GeoDataFrame( + df_points, + geometry=gpd.points_from_xy(df_points[x_dim_name], df_points[y_dim_name]), + crs=crs + ) - def _build_crs_variable(self, ds_out: xr.Dataset) -> xr.Dataset: + # Spatial join with catchments for grid point to catchment association + # To do: Using sjoin here. For larger areas, we may need to revisit if computing efficiency drops. + gdf_poly = self._gpkg_gdf.to_crs(crs) + joined = gpd.sjoin( + gdf_ncgrid_points, + gdf_poly[[self._catchment_field, "geometry"]], + how="left", + predicate="within" + ) + catchment_ids = joined[self._catchment_field].to_numpy() + catchment_grid = catchment_ids.reshape(len(y), len(x)) + + catchment_id_da = xr.DataArray( + catchment_grid, + dims=(y_dim_name, x_dim_name), + coords={y_dim_name: y, x_dim_name: x}, + name="catchment_id" + ) + # It is important to pass on the CRS info to the functions downstream + # Attaching the crs wkt as a xr.dataarray attribute + catchment_id_da.attrs["crs"] = ds["crs"].attrs["spatial_ref"] + except Exception as e: + raise RuntimeError(f"Error in building catchment grid: {e}") from e + + return catchment_id_da + + def map_catchment_data_to_grid(self, ds: xr.Dataset, catchment_grid: xr.DataArray, catchment_dim: str, + x_dim_name: str, y_dim_name: str + + ) -> xr.Dataset: """ - Build CF-compliant CRS variable from a template NWM grid. - Currently, this is not using the template NWM grid. It uses catchment grid instead. + Assign catchment-based variables onto basin-level grid using the catchment_id grid. """ - # Get CRS from template - crs_obj = CRS.from_user_input(self.ds_grid.rio.crs) - - # Convert to CF dict - cf_attrs = crs_obj.to_cf() - - # Create CRS variable - ds_out["crs"] = xr.DataArray(0) - - # Assign CF attributes - ds_out["crs"].attrs.update(cf_attrs) - - # Add WKT (for QGIS) - wkt = crs_obj.to_wkt() - ds_out["crs"].attrs["spatial_ref"] = wkt - ds_out["crs"].attrs["esri_pe_string"] = wkt - - # Add GeoTransform - x = ds_out["x"].values - y = ds_out["y"].values + ds_data = ds - dx = float(x[1] - x[0]) - dy = float(y[1] - y[0]) + # Convert catchment IDs to index positions + catchment_ids = ds_data[catchment_dim].values - xmin = float(x[0] - dx / 2) - ymax = float(y[-1] + dy / 2) + # Build mapping: catchment_id -> index + id_to_index = {cid: i for i, cid in enumerate(catchment_ids)} - ds_out["crs"].attrs["GeoTransform"] = f"{xmin} {dx} 0 {ymax} 0 {-dy}" + # Convert grid IDs to indices + grid_index = xr.apply_ufunc( + np.vectorize(lambda x: id_to_index.get(x, -1)), + catchment_grid, + vectorize=True, + dask="parallelized", + output_dtypes=[int] + ) - # Optional (additional metadata) - ds_out["crs"].attrs["_CoordinateAxes"] = "y x" - ds_out["crs"].attrs["_CoordinateTransformType"] = "Projection" - ds_out["crs"].attrs["long_name"] = "CRS definition" + valid_mask = grid_index >= 0 # Mask invalid cells + out_vars = {} + for var in ds_data.data_vars: + if catchment_dim not in ds_data[var].dims: + continue + data = ds_data[var] + mapped = data.isel({catchment_dim: grid_index}) + mapped = mapped.where(valid_mask) + out_vars[var] = mapped + ds_out = xr.Dataset(out_vars) + + # Attach coordinates from template + ds_out = ds_out.assign_coords({ + x_dim_name: self._template_netcdf_ds[x_dim_name], + y_dim_name: self._template_netcdf_ds[y_dim_name] + }) + + # Add CRS info from template grid + crs_attrs = self._template_netcdf_ds["crs"].attrs.copy() + ds_out["crs"] = xr.DataArray(data=0,dims=()) + ds_out["crs"].attrs = crs_attrs + + # Link all variables to CRS + for var in ds_out.data_vars: + ds_out[var].attrs["grid_mapping"] = "crs" return ds_out - def create_grids_per_timestep(self, output_dir: str) -> None: + def write_netcdf_per_timestep(self, mapped_grid: xr.Dataset, output_dir: str, time_dim: str = "time") -> None: """ - Generates netcdf grids per timestep for all the variables in the catchments netcdf. + Writes one NetCDF per timestep. """ os.makedirs(output_dir, exist_ok=True) - n_y = self.ds_grid.dims["y"] - n_x = self.ds_grid.dims["x"] - valid_mask = self.grid_to_catchment != -1 - - # may be a robust approach is not to consider "time". Instead use - # np.issubdtype(coord.dtype, np.datetime64)? - time_dim = next(dim for dim in self.input_ds.dims if "time" in dim.lower()) - reference_time = np.datetime64('1970-01-01T00:00:00') - - for t in self.input_ds[time_dim].values: - print(f"Processing timestep: {t}") - ds_t = self.input_ds.sel({time_dim: t}) - ds_out = self.ds_grid.copy(deep=True) - - time_value_min = (t - reference_time) / np.timedelta64(1, 'm') - ds_out = ds_out.assign_coords({time_dim: [time_value_min]}) - ds_out[time_dim].attrs["units"] = "minutes since 1970-01-01 00:00:00 UTC" - ds_out[time_dim].attrs["standard_name"] = "time" - ds_out = self._build_crs_variable(ds_out) - - # spatial_ref variable gets added somewhere. - # To Do: Need to find this later. a quick fix for now is to delete, if exists. - if "spatial_ref" in ds_out.variables: - ds_out = ds_out.drop_vars("spatial_ref") - - # Loop over all variables dynamically - for var in self.output_variables: - print(f" Processing variable: {var}") - values_1d = ds_t[var].values # Get catchment values (1D) - flat_grid = np.full(self.grid_to_catchment.shape, np.nan) # Prepare flat grid - flat_grid[valid_mask] = values_1d[self.grid_to_catchment[valid_mask]] # Assign values using lookup - grid_2d = flat_grid.reshape(n_y, n_x) # Reshape to 2D - ds_out[var] = (("time", "y", "x"), grid_2d[np.newaxis, :, :]) # Add to output dataset - ds_out[var].attrs["grid_mapping"] = "crs" # Ensure the variable references CRS + reference_epoch = np.datetime64("1970-01-01T00:00:00") + + for t in mapped_grid[time_dim].values: + ds_t = mapped_grid.sel({time_dim: t}).copy() + time_value_mins = (t - reference_epoch) / np.timedelta64(1, "m") # time to CF format + ds_t = ds_t.expand_dims({time_dim: [time_value_mins]}) + ds_t[time_dim].attrs["units"] = "minutes since 1970-01-01 00:00:00 UTC" + ds_t[time_dim].attrs["standard_name"] = "time" + + # Rebuild CRS info and mapping. + if "crs" in ds_t: + ds_t = ds_t.drop_vars("crs") + ds_t["crs"] = xr.DataArray(0, dims=()) + ds_t["crs"].attrs = mapped_grid["crs"].attrs.copy() + for var in ds_t.data_vars: + if not var == "crs": + ds_t[var].attrs["grid_mapping"] = "crs" + + # Add reference_time variable and attributes to netcdf + # To do: Replace this with the actual time when the model simulations were run. + ds_t[consts.DIM_REF_TIME] = xr.DataArray( + data=time_value_mins.astype("int32"), + dims=() + ) + ds_t[consts.DIM_REF_TIME].attrs = { + "long_name": "model initialization time", + "standard_name": "forecast_reference_time", + "units": "minutes since 1970-01-01 00:00:00 UTC" + } # Output filename and save t_str = np.datetime_as_string(t, unit='s') formatted_t = t_str.replace('-', '_').replace(':', '_') - output_file = os.path.join(output_dir, f"grid_{formatted_t}.nc") - ds_out.to_netcdf(output_file) + output_file = os.path.join(output_dir, f"nwm.{self._geo_id}.{self._output_class}.{self._category}.{self._domain}.{formatted_t}.nc") + ds_t.to_netcdf(output_file) - def create_grids_per_timestep_dask(self, output_dir: str, chunk_size: int = 1000) -> None: + def produce_channel_reservoir_nwm_product(self, mdata: NetCDFMetadata, output_dir: str): """ - Generates netcdf grids per timestep for all the variables in the catchments netcdf. + Expands a zeroed NetCDF template and populates it with data from a single + time snapshot, including remapping misnamed variables. """ + if not self._template_netcdf_ds: + raise ValueError("Template netcdf not set") + + if mdata.category == 'channel_rt' and not self._catchment_ds: + raise ValueError("ngen catchment netcdf not set") + + if mdata.category == 'channel_rt' and not self._troute_netcdf_ds: + raise ValueError("troute output netcdf not set") - os.makedirs(output_dir, exist_ok=True) - - # Dimensions - n_y = self.ds_grid.dims["y"] - n_x = self.ds_grid.dims["x"] - n_cells = n_y * n_x - - # Mask for valid cells - valid_mask = self.grid_to_catchment != -1 - - # Convert to Dask arrays - grid_to_catchment_da = da.from_array(self.grid_to_catchment, chunks=chunk_size) - valid_mask_da = da.from_array(valid_mask, chunks=chunk_size) - - # Rechunk by time dimension as we are processing one timestep at a time - time_dim = next(dim for dim in self.input_ds.dims if np.issubdtype(self.input_ds[dim].dtype, np.datetime64)) - reference_time = np.datetime64('1970-01-01T00:00:00') - ds_in = self.input_ds.chunk({time_dim: 1}) - - for t in ds_in[time_dim].values: - print(f"Processing timestep: {t}") - - ds_t = ds_in.sel({time_dim: t}) - ds_out = self.ds_grid.copy(deep=True) - time_value_min = (t - reference_time) / np.timedelta64(1, 'm') - ds_out = ds_out.assign_coords({time_dim: [time_value_min]}) - ds_out = self._build_crs_variable(ds_out) - - # spatial_ref variable gets added somewhere. - # To Do: Need to find this later. a quick fix for now is to delete, if exists. - if "spatial_ref" in ds_out.variables: - ds_out = ds_out.drop_vars("spatial_ref") - - for var in ds_in.data_vars: - print(f" Processing variable: {var}") - - values_1d = ds_t[var].data # (catchment,) → Dask array - - # Convert to Dask array if needed - if not isinstance(values_1d, da.Array): - values_1d = da.from_array(values_1d, chunks=values_1d.shape) + if mdata.category == 'reservoir' and not self._troute_lakeout_netcdf_ds: + raise ValueError("troute lakeout netcdf not set") + + reference_epoch = np.datetime64("1970-01-01T00:00:00") # set reference epoch + # We have to identify min and max times and reference times for the products. + # The approach is different because each product uses different way of reporting time. + # ngen catchment output reports in seconds since reference_epoch. + # troute output reports in offset seconds since the model initialization time. + # troute lakeout (waterbody) reports in minutes since reference epoch. + if mdata.category == 'reservoir': + troute_source_ds = self._troute_lakeout_netcdf_ds + source_ds_times = troute_source_ds[consts.DIM_TIME].values + sorted_times = np.sort(source_ds_times)[::-1] # sort and reverse slice + time_value_min = sorted_times[-1] + time_value_max = sorted_times[0] + time_value_min = np.int32((sorted_times[-1] - reference_epoch) / np.timedelta64(1, "m")) # time to CF format + time_value_max = np.int32((sorted_times[0] - reference_epoch) / np.timedelta64(1, "m")) # time to CF format + elif mdata.category == 'channel_rt': + troute_source_ds = self._troute_netcdf_ds + units_attr_val = troute_source_ds[consts.DIM_TIME].encoding.get('units') + time_str = units_attr_val.split("since ")[1].strip() + base_datetime = np.datetime64(time_str) + source_ds_times = troute_source_ds[consts.DIM_TIME].values + source_ds_minutes = (source_ds_times - reference_epoch).astype("timedelta64[m]").astype(int) + sorted_times = np.sort(source_ds_minutes)[::-1] # sort and reverse slice + time_value_min = sorted_times[-1] + time_value_max = sorted_times[0] + + # Get reference time for output + ref_time_val = time_value_min - 60 # 60 mins less than the smallest time. + + re_index_args = { + consts.DIM_FEATURE_ID: troute_source_ds[consts.DIM_FEATURE_ID].values, + consts.DIM_REF_TIME: [ref_time_val] + } + + # Create one netcdf each for tm00, tm01 and tm02 + for time_step, snapshot_time_val in enumerate(sorted_times): + populated_ds = self._template_netcdf_ds.reindex(**re_index_args, fill_value = np.nan) + time_args = {consts.DIM_TIME: [snapshot_time_val]} + populated_ds = populated_ds.assign_coords(**time_args) + populated_ds.attrs.update(self._template_netcdf_ds.attrs) + + # Populate variables defined in the template using the negen output and troute data + for var_name in self._template_netcdf_ds.variables: + # leave the dimensions out as they (except time) have already been populated. + if var_name in self._template_netcdf_ds.dims: + continue + + # Add reference time value to the output + if var_name == consts.DIM_REF_TIME: + # Assign the scalar or 1D value directly to the pre-allocated template dimension + populated_ds[var_name].values = np.array([ref_time_val]).astype(populated_ds[var_name].dtype) + continue + + # Confirm if the variable is present both in lakeout and final output + in_snapshot = var_name in troute_source_ds.variables + in_populated = var_name in populated_ds.variables + if var_name.lower() == 'qbucket': + in_ngenout = var_name in self._catchment_ds.variables + + if not in_snapshot and not in_populated: + print(f"{var_name} is missing from both troute and output datasets.") + continue + elif not in_snapshot: + if self._template_netcdf_ds[var_name].ndim == 0: # Scalar variables not in the data, but in template + populated_ds[var_name] = xr.DataArray(self._template_netcdf_ds[var_name].values.item()) + print(f"----Found a scalar variable - {var_name} that is not in the data, but present in the template.Template value has been copied over.") + else: + print(f"----Found a data variable - {var_name} that is not in the data, but present in the template. It is filled with NaN.") + continue + elif not in_populated: + print(f"{var_name} is in the template but not found in the output dataset.") + continue + elif var_name.lower() == 'qbucket' and not in_ngenout: + print(f"{var_name} is in the template but not found in the catchment output dataset.") + continue + + # Get the time slice data for this variable, if time is one of the dimensions. + # Otherwise, get the full variable (typically 1D based on feature_id dimension) + template_var = self._template_netcdf_ds[var_name] + if var_name.lower() == 'qbucket': + # get time slice data from ngen catchment output. + # Convert time (mins) to secs to match ngen catchment output. + snapshot_time_val_sec = int(snapshot_time_val * 60) + source_var = self._catchment_ds[var_name].sel(time = snapshot_time_val_sec) #expecting parameter name: 'time' + else: + troute_unsliced_var = troute_source_ds[var_name] + if consts.DIM_TIME in troute_unsliced_var.dims: + # this loop is in descending order of time. but, the source is in ascending order. + # recalculate time index. + total_timesteps = troute_unsliced_var.sizes[consts.DIM_TIME] + inverted_time_index = total_timesteps - 1 - time_step + source_var = troute_unsliced_var.isel(time = inverted_time_index) #expecting parameter name: 'time' + else: + source_var = troute_unsliced_var - flat_grid = da.full((n_cells,), np.nan, chunks=chunk_size) - assigned = da.where( - valid_mask_da, - values_1d[grid_to_catchment_da], - np.nan - ) + tgt_shape = populated_ds[var_name].shape + + if source_var.ndim == len(tgt_shape): + # print(f"----Dimensions match for variable {var_name}") + # print(f"----Source - Shape: {lakeout_var.shape} | Dtype: {lakeout_var.dtype}") + # print(f"----Template - Shape: {tgt_shape} | Dtype: {populated_ds[var_name].dtype}") + + data_values = source_var.values + # Verify the types because the lakeout variables such as inflow and outflow are float + # But, they are int in the template. So, we are casting from float to int. + if np.issubdtype(template_var.dtype, np.integer): + if np.issubdtype(data_values.dtype, np.floating): + data_values = np.nan_to_num(data_values, nan=0) + # Use np.array().astype() to ensure scalars cast works as well. + data_values = np.array(data_values).astype(template_var.dtype) + populated_ds[var_name].values = data_values # populate the values. + else: + print(f"----The dimensions don't match between template ({template_var.dims}) and snapshot ({source_var.dims}) for {var_name}") - grid_2d = assigned.reshape((n_y, n_x)) - grid_3d = grid_2d[None, :, :] # (time=1, y, x) - ds_out[var] = (("time", "y", "x"), grid_3d) - ds_out[var].attrs = ds_in[var].attrs - ds_out[var].attrs["grid_mapping"] = "crs" #"spatial_ref" + # Transfer all other attributes from template + for var_name in self._template_netcdf_ds.variables: + if var_name in populated_ds.variables: + populated_ds[var_name].attrs.update(self._template_netcdf_ds[var_name].attrs) + populated_ds[var_name].encoding.update(self._template_netcdf_ds[var_name].encoding) + + # Prevent overwriting conflicts by clearing 'units' and calendar + populated_ds[var_name].attrs.pop('units', None) + populated_ds[var_name].attrs.pop('calendar', None) + + # Copy Fill and missing values from the template DS + has_missing = 'missing_value' in self._template_netcdf_ds[var_name].encoding + has_fill = '_FillValue' in self._template_netcdf_ds[var_name].encoding + if has_missing and has_fill: + tgt_value = self._template_netcdf_ds[var_name].encoding['missing_value'] + populated_ds[var_name].encoding['missing_value'] = tgt_value + populated_ds[var_name].encoding['_FillValue'] = tgt_value + elif has_missing and not has_fill: + tgt_value = self._template_netcdf_ds[var_name].encoding['missing_value'] + populated_ds[var_name].encoding['missing_value'] = tgt_value + populated_ds[var_name].encoding.pop('_FillValue', None) + populated_ds[var_name].attrs.pop("_FillValue", None) + elif has_fill and not has_missing: + tgt_value = self._template_netcdf_ds[var_name].encoding['_FillValue'] + populated_ds[var_name].encoding['_FillValue'] = tgt_value + populated_ds[var_name].encoding.pop('missing_value', None) + populated_ds[var_name].attrs.pop("missing_value", None) + + # Add valid min and max times to the "time" attributes + populated_ds[consts.DIM_TIME].attrs["valid_min"] = np.int32(time_value_min) + populated_ds[consts.DIM_TIME].attrs["valid_max"] = np.int32(time_value_max) - ds_out[time_dim].attrs["units"] = "minutes since 1970-01-01 00:00:00 UTC" - ds_out[time_dim].attrs["standard_name"] = "time" + # Manually assign units and encoding for reference time. + # Without this it was encoding as nanoseconds since 1970-01-01 + populated_ds[consts.DIM_REF_TIME].attrs['units'] = "minutes since 1970-01-01 00:00:00" - # Output file and save - t_str = np.datetime_as_string(t, unit='s') - formatted_t = t_str.replace('-', '_').replace(':', '_') - output_file = os.path.join(output_dir, f"grid_{formatted_t}.nc") - ds_out.to_netcdf(output_file) \ No newline at end of file + # Output filename and save + os.makedirs(output_dir, exist_ok=True) + formatted_t = f"{time_step:02d}" + output_file = os.path.join(output_dir, f"nwm.{self._geo_id}.{self._output_class}.{self._category}.{self._domain}.tm{formatted_t}.nc") + populated_ds.to_netcdf(output_file) + + def close_log(self): + """Restores the original terminal output and closes the file.""" + original_stream = getattr(self, "_original_stdout", sys.stdout) + if sys.stdout != original_stream: + sys.stdout = original_stream + + log_file_obj = getattr(self, "_log_file", None) + if log_file_obj and not log_file_obj.closed: + log_file_obj.flush() + log_file_obj.close() + + # Reset the private attribute safely + if hasattr(self, "_log_file"): + self._log_file = None + + def __del__(self): + """ + Close the log file if the script ends unexpectedly. + """ + self.close_log() \ No newline at end of file diff --git a/data_assimilation_engine/output_variables/DataReader.py b/data_assimilation_engine/output_variables/DataReader.py index 3f1168c..b5f8ed1 100644 --- a/data_assimilation_engine/output_variables/DataReader.py +++ b/data_assimilation_engine/output_variables/DataReader.py @@ -1,7 +1,7 @@ import fsspec import xarray as xr import numpy as np - +from . import consts class DataReader: def __init__(self, netcdf_file: str, chunk_size: int = 100) -> None: @@ -16,8 +16,8 @@ def _load_dataset(self) -> xr.Dataset: ds: xr.Dataset = xr.open_dataset( f, chunks={ - "time": 1, - "catchment": self.chunk_size + consts.DIM_TIME: 1, + consts.DIM_CATCHMENTS: self.chunk_size } ) return ds.load() @@ -29,7 +29,7 @@ def _infer_catchment(self): # Find coordinate with IDs for coord_name in ds.coords: coord = ds[coord_name] - if coord.dtype.kind in {"U", "S", "O"} and coord.ndim == 1: #assumes catchments are string type. + if coord.dtype.kind in {"i", "u"} and coord.dtype.itemsize == 8 and coord.ndim == 1: #assumes catchments are the only long type. return coord.dims[0], coord_name raise ValueError("Could not find catchment IDs") @@ -39,16 +39,46 @@ def assign_random_values(self, output_file: str) -> None: Assign random values between 1 and 10 to all data variables. This function can be deleted once we address the catchment netcdf writing issue. """ - + var_list = ['sm_frac_0.4m', 'sm_profile_0.1m', 'sm_profile_0.4m', 'sm_profile_1.5m', 'sm_profile_2m', 'SWE_mm'] with fsspec.open(self.netcdf_file, mode="rb") as f: ds = xr.open_dataset(f) + if ds.sizes.get(consts.DIM_TIME, 0) > 48: + ds_48 = ds.isel(time=slice(0, 48)) + else: + ds_48 = ds + + # Check if all the vairables are present in the netcdf. If not add it. Only for testing. + for var in var_list: + if var not in ds_48.data_vars: + ds_48[var] = ds['SWE_mm'].copy(deep = True) - for var_name in ds.data_vars: + for var_name in ds_48.data_vars: print(f"Assigning random values to {var_name}") random_values = np.random.uniform( - 1.0, 10.0, size=ds[var_name].shape + 10.0, 20.0, size=ds_48[var_name].shape ) - ds[var_name].values = random_values + ds_48[var_name].values = random_values + + ds_48.to_netcdf(output_file) + print(f"Saved updated dataset to {output_file}") + + def add_missing_variables(self, output_file) -> None: + """ + Add any missing variables in the variables list to the netcdf file. + This function can be deleted once we address the catchment netcdf writing issue. + """ + var_list = ['sm_frac_0.4m', 'sm_profile_0.1m', 'sm_profile_0.4m', 'sm_profile_1.5m', 'sm_profile_2m', 'SWE_mm'] + with fsspec.open(self.netcdf_file, mode="rb") as f: + ds = xr.open_dataset(f) + if ds.sizes.get(consts.DIM_TIME, 0) > 48: + ds_48 = ds.isel(time=slice(0, 48)) + else: + ds_48 = ds + + # Check if all the vairables are present in the netcdf. If not add it. Only for testing. + for var in var_list: + if var not in ds_48.data_vars: + ds_48[var] = ds['SWE_mm'].copy(deep = True) - ds.to_netcdf(output_file) + ds_48.to_netcdf(output_file) print(f"Saved updated dataset to {output_file}") \ No newline at end of file diff --git a/data_assimilation_engine/output_variables/NetCdfProductionManager.py b/data_assimilation_engine/output_variables/NetCdfProductionManager.py new file mode 100644 index 0000000..6ddf385 --- /dev/null +++ b/data_assimilation_engine/output_variables/NetCdfProductionManager.py @@ -0,0 +1,181 @@ +import sys +import os +from datetime import datetime +import time +from typing import List, Any, Optional +from data_assimilation_engine.output_variables.DataReader import DataReader +from data_assimilation_engine.output_variables.DataProcessor import DataProcessor +import data_assimilation_engine.output_variables.utils as utils +import data_assimilation_engine.output_variables.consts as consts + +_processor: Optional[DataProcessor] = None # Shared instance of DataProcessor + +def create_dataprocessor(root_output_folder: str, netcdf_file: str, gpkg_file: str) -> DataProcessor: + global _processor + if _processor is None: + _processor = DataProcessor(netcdf_file, gpkg_file) + # Create log file + log_folder = os.path.join(root_output_folder, consts.LOG_FOLDER) + os.makedirs(log_folder, exist_ok = True) + log_file = os.path.join(log_folder, 'nwm_postprocessing_' + datetime.now().strftime("%Y%m%d_%H%M%S") + '.log') + _processor.log_file = log_file + return _processor + +def download_netcdf_from_nomads(root_output_folder: str, re_download: bool = False) -> str: + return utils.download_nwm_data_from_server(root_output_folder, re_download) + +def create_template_files_for_gpkg(root_output_folder: str, netcdf_file: str, gpkg_file: str, + config_json: str, output_templates_folder: str | None): + + if not os.path.isfile(netcdf_file): + raise ValueError("Specified ngen output cathments netcdf file does not exist") + + if not os.path.isfile(gpkg_file): + raise ValueError("Specified ngen geopackage file does not exist") + + netcdf_metadata_list = [] + if os.path.isfile(config_json): + netcdf_metadata_list = utils.read_output_variables_info_from_config(config_json) + else: + raise ValueError("Specified config file does not exist") + + # create necessary folders + os.makedirs(root_output_folder, exist_ok = True) + if output_templates_folder is None or output_templates_folder == '': + # If no folder is specified, we will create it as a subfolder within the root. + ngen_template_nc_folder = os.path.join(root_output_folder, consts.NWM_NGEN_TEMPLATE_FOLDER) + else: + ngen_template_nc_folder = output_templates_folder + os.makedirs(ngen_template_nc_folder, exist_ok = True) + + # Create templates. + for mdata in netcdf_metadata_list: + _processor.create_template_netcdf_using_config(mdata, ngen_template_nc_folder) + +def create_nwm_products_for_gpkg(root_output_folder: str, troute_output_netcdf: str, troute_lakeout_netcdf: str, + config_json: str, output_templates_folder: str | None, + output_cycle_hour: int, output_cycle_type: str): + + if output_templates_folder is None or output_templates_folder == '': + # If no folder is specified, we will assume it as a subfolder within the root as defined in consts.py. + ngen_template_nc_folder = os.path.join(root_output_folder, consts.NWM_NGEN_TEMPLATE_FOLDER) + else: + ngen_template_nc_folder = output_templates_folder + + netcdf_metadata_list = [] + if os.path.isfile(config_json): + netcdf_metadata_list = utils.read_output_variables_info_from_config(config_json) + else: + raise ValueError("Specified config file does not exist") + + # Confirm that template files exist for the cycle type. Gather a dictionary as well + # To do: Update this to include only conus for demo? + template_files_dict = {} + for mdata in netcdf_metadata_list: + if mdata.output_class == output_cycle_type: + template_nc_name = _processor.geo_id + '_' + mdata.output_class + '_' + mdata.category + '_' + mdata.domain + template_nc_file = os.path.join(ngen_template_nc_folder, template_nc_name + '.nc') + if os.path.isfile(template_nc_file): + template_files_dict[template_nc_name] = template_nc_file + else: + raise ValueError(f"Template file for {_processor.geo_id}.{mdata.output_class}.{mdata.category}.{mdata.domain} does not exist") + + # set output folder for the nwm products for the geopackage + nwm_output_folder = os.path.join(root_output_folder, consts.NWM_OUTPUT_FOLDER) + os.makedirs(nwm_output_folder, exist_ok = True) + + # set cycle hour + formatted_hr = f"{output_cycle_hour:02d}" + + for mdata in netcdf_metadata_list: + if mdata.output_class == output_cycle_type: + _processor.nwm_output_class = mdata.output_class + _processor.nwm_category = mdata.category + _processor.nwm_domain = mdata.domain + template_nc_name = _processor.geo_id + '_' + mdata.output_class + '_' + mdata.category + '_' + mdata.domain + _processor.set_template_netcdf(template_files_dict[template_nc_name]) + _processor.set_troute_netcdf(troute_output_netcdf) + _processor.set_troute_lakeout_netcdf(troute_lakeout_netcdf) + _processor.produce_nwm_output_product(mdata, nwm_output_folder) + +# Postporcessing Entrypoint +def netcdf_production_workflow(args_list) -> Optional[Any]: + """ + Main entrypoint function. Parses the args list and handles action + based on user preferences. + """ + + global _processor + + if not args_list: + print("The arguments list to post processing is either null or empty.") + raise ValueError("Workflow aborted: The arguments list cannot be None or empty.") + + action_item = str(args_list[-1]).lower() # last item is the action. + + match action_item: + case "download": + if len(args_list) < 2: + raise ValueError("'download' action requires argument for root output folder: [root output folder path, 'download']") + root_output_folder = args_list[0] + config_json_file = download_netcdf_from_nomads(root_output_folder) + return config_json_file + + case "template": + if len(args_list) < 6: + raise ValueError("'template' action requires argument for root output folder, " + + "ngen catchments netcdf, geopackage and config json: [root output folder path, " + + "catchments netcdf, geopackage, config json, optional template output folder, 'template']") + root_output_folder = args_list[0] + ngen_catchments_netcdf = args_list[1] + ngen_geopackage = args_list[2] + config_json_file = args_list[3] + output_templates_folder = args_list[4] + if(_processor is None): + _processor = create_dataprocessor(root_output_folder, ngen_catchments_netcdf, ngen_geopackage) + create_template_files_for_gpkg(root_output_folder, ngen_catchments_netcdf, ngen_geopackage, config_json_file, output_templates_folder) + + case "output": + if len(args_list) < 10: + raise ValueError("'output' action requires argument for root output folder, " + + "ngen catchments netcdf, geopackage troute outputs and config json: [root output folder path, " + + "catchments netcdf, geopackage, troute output, troute lakeout, config json, " + + "optional template output folder, 'output']") + root_output_folder = args_list[0] + ngen_catchments_netcdf = args_list[1] + ngen_geopackage = args_list[2] + troute_output_netcdf = args_list[3] + troute_lakeout_netcdf = args_list[4] + config_json_file = args_list[5] + output_templates_folder = args_list[6] + output_cycle_hour = int(args_list[7]) + output_cycle_type = args_list[8] + if(_processor is None): + _processor = create_dataprocessor(root_output_folder, ngen_catchments_netcdf, ngen_geopackage) + create_nwm_products_for_gpkg(root_output_folder, troute_output_netcdf, troute_lakeout_netcdf, + config_json_file, output_templates_folder, output_cycle_hour, output_cycle_type) + + case "all": + if len(args_list) < 8: + raise ValueError("'output' action requires argument for root output folder, " + + "ngen catchments netcdf, geopackage and config json: [root output folder path, " + + "catchments netcdf, geopackage, troute output, troute lakeout, config json, " + + "optional template output folder, 'all']") + root_output_folder = args_list[0] + ngen_catchments_netcdf = args_list[1] + ngen_geopackage = args_list[2] + troute_output_netcdf = args_list[3] + troute_lakeout_netcdf = args_list[4] + config_json_file = args_list[5] + output_templates_folder = args_list[6] + output_cycle_hour = int(args_list[7]) + output_cycle_type = args_list[8] + + config_json_file = download_netcdf_from_nomads(root_output_folder) + if(_processor is None): + _processor = create_dataprocessor(root_output_folder, ngen_catchments_netcdf, ngen_geopackage) + create_template_files_for_gpkg(root_output_folder, ngen_catchments_netcdf, ngen_geopackage, config_json_file, output_templates_folder) + create_nwm_products_for_gpkg(root_output_folder, troute_output_netcdf, troute_lakeout_netcdf, + config_json_file, output_templates_folder, output_cycle_hour, output_cycle_type) + + print("NetCDF Production workflow completed Successfully") diff --git a/data_assimilation_engine/output_variables/Readme_wip.md b/data_assimilation_engine/output_variables/Readme_wip.md index adcf9d8..05a402b 100644 --- a/data_assimilation_engine/output_variables/Readme_wip.md +++ b/data_assimilation_engine/output_variables/Readme_wip.md @@ -12,6 +12,7 @@ All the required packages are already part of the toml file. All paths specified below can be absolute or relative. #### randomize: +This is a function used for testing to create random values for all variables in the catchment output. This should be deleted when production ready. python netcdf_production_sample.py randomize catchment_netcdf catchment_netcdf_randomized Positional Arguments: @@ -19,20 +20,57 @@ Positional Arguments: - catchment_netcdf_randomized: Path where the output netcdf file is created with random values from 1-10 assigned to the output variables. #### create-template-grid: -python netcdf_production_sample.py create-template-grid catchment_netcdf_randomized basin_geopackage output_netcdf_grid +This function creates the template grid that spans the geopackage catchments extent (not the bounding box). This works only for gridded files. +python netcdf_production_sample.py create-template-grid catchment_netcdf_randomized basin_geopackage output_template_netcdf_grid config_json Positional Arguments: - catchment_netcdf_randomized: Path to the output netcdf file with random values from 1-10 assigned to the output variables. - basin_geopackage: Path to the basin geopackage. It can be in the existing or the new NHF schema. -- output_netcdf_grid: Path where the gridded 1 km x 1km netcdf file is created. +- output_template_netcdf_grid: Path where the gridded netcdf file is created. Specify the name as well. +- config_json: Path to the config json that holds the metadata information for NWM output products. #### create-nwm-grid: -python netcdf_production_sample.py create-nwm-grid catchment_netcdf_randomized basin_geopackage output_netcdf_grid +This function creates a basin level NetCDF grid from the template grid. +python netcdf_production_sample.py create-nwm-grid catchment_netcdf_randomized basin_geopackage template_netcdf_grid config_json Positional Arguments: - catchment_netcdf_randomized: Path to the output netcdf file with random values from 1-10 assigned to the output variables. - basin_geopackage: Path to the basin geopackage. It can be in the existing or the new NHF schema. -- output_netcdf_grid: Path where the gridded 1 km x 1km netcdf file is created. +- template_netcdf_grid: Path for the gridded template netcdf file to use for producing the NWM grid product. +- config_json: Path to the config json that holds the metadata information for NWM output products. + +#### download-nwm-outputs: +This function downloads a unique set of NWM output products from the NOMADS server that can be used for templating. +python netcdf_production_sample.py download-nwm-outputs nomads_root_url output_folder_path + +Positional Arguments: +- nomads_root_url: Root URL for NOMADS. This folder should not include the date +- output_folder_path: Folder to download the NWM data products. + +#### obtain-netcdf-metadata: +This function reads all the metadata information needed from the NWM output files and saves it in a csv and json. +python netcdf_production_sample.py obtain-netcdf-metadata nwm_local_download_path output_file_name + +Positional Arguments: +- nwm_local_download_path: Path to the folder where all netcdf files have been downloaded from NOMADS. +- output_file_name: Output file name for the csv and json. + +#### create-template-nwm-grid: +This function is primarily used for producing intermediate files for testing. This should be deleted when scripts are production-ready. +python netcdf_production_sample.py create-template-nwm-grid nwm_local_download_path nwm_local_template_path + +Positional Arguments: +- nwm_local_download_path: Path to the folder where all netcdf files have been downloaded from NOMADS. +- nwm_local_template_path: Path to the folder where all template grids will be created. + +#### create-combined-basin-grid: +This function is primarily used for producing intermediate files for testing. This should be deleted when scripts are production-ready. +python netcdf_production_sample.py create-combined-basin-grid nwm_reference_grid nwm_timestep_grid_path nwm_multibasin_grid_path + +Positional Arguments: +- nwm_reference_grid: Reference NWM grid for a given category, class and domain. +- nwm_timestep_grid_path: Path to the folder where the necessary timestep netcdf files have been created. +- nwm_multibasin_grid_path: Path to the folder where all the combined/merged multi-basin grids will be saved. # Examples @@ -40,8 +78,19 @@ Positional Arguments: python netcdf_production_sample.py randomize sample_data/sample_netcdf/g01123000.nc sample_data/sample_netcdf/catchment_randomvals.nc #### create-template-grid -python netcdf_production_sample.py create-template-grid sample_data/sample_netcdf/catchment_randomvals.nc sample_data/sample_gpkg/gages-01123000.gpkg sample_data/sample_netcdf/grid_template.nc +python netcdf_production_sample.py create-template-grid sample_data/sample_netcdf/final_test/catchment_randomvals.nc sample_data/sample_gpkg/gages-01123000.gpkg sample_data/sample_netcdf/final_test/new_grid_template.nc sample_data/nwm_output/metadata_config.json #### create-nwm-grid -python netcdf_production_sample.py create-nwm-grid sample_data/sample_netcdf/catchment_randomvals.nc sample_data/sample_gpkg/gages-01123000.gpkg sample_data/sample_netcdf/grid_template.nc +python netcdf_production_sample.py create-nwm-grid sample_data/sample_netcdf/final_test/catchment_randomvals.nc sample_data/sample_gpkg/gages-01123000.gpkg sample_data/sample_netcdf/final_test/new_grid_template.nc sample_data/nwm_output/metadata_config.json + +#### download-nwm-outputs +python netcdf_production_sample.py download-nwm-outputs https://nomads.ncep.noaa.gov/pub/data/nccf/com/nwm/prod sample_data/nwm_output + +#### obtain-netcdf-metadata +python netcdf_production_sample.py obtain-netcdf-metadata sample_data/nwm_output metadata + +#### create-template-nwm-grid +python netcdf_production_sample.py create-template-nwm-grid sample_data/nwm_output sample_data/nwm_templates +#### create-combined-basin-grid +python netcdf_production_sample.py create-combined-basin-grid sample_data/nwm_output/analysis_assim/nwm.t00z.analysis_assim.land.tm00.conus.nc sample_data/sample_netcdf/sample_output/merge_test sample_data/sample_netcdf/sample_output/merge_test \ No newline at end of file diff --git a/data_assimilation_engine/output_variables/consts.py b/data_assimilation_engine/output_variables/consts.py new file mode 100644 index 0000000..ef18394 --- /dev/null +++ b/data_assimilation_engine/output_variables/consts.py @@ -0,0 +1,59 @@ +NWM_VARIABLES_LIST = ['sfcheadsubrt', 'zwattablrt', 'inflow', 'outflow', 'reservoir_assimilated_value', +'water_sfc_elev', 'nudge', 'qBucket', 'streamflow', 'velocity', 'qBtmVertRunoff', 'qSfcLatRunoff', +'ACSNOM', 'ACCET', 'SNOWT_AVG', 'EDIR', 'SOILICE', 'SOILSAT_TOP', 'ISNOW', 'QRAIN', 'FSNO', 'SNOWH', +'SNLIQ', 'SNEQV', 'QSNOW', 'SOIL_T', 'SOIL_M', 'SFCRNOFF', 'ACCECAN', 'ACCEDIR', 'ACCETRAN', 'UGDRNOFF', +'GRDFLX', 'TRAD', 'FSA', 'CANWAT', 'LH', 'FIRA', 'HFX'] + +NWM_VARS_IGNORE_LIST = ['ACCET', 'ACCECAN', 'ACCEDIR', 'ACCETRAN', 'CANWAT', 'SOILSAT', + 'EDIR', 'FSA', 'GRDFLX', 'ISNOW', 'UGDRNOFF', 'zwattablrt', + 'qBtmVertRunoff', 'qSfcLatRunoff'] + +NWM_PRODUCTS_LIST = ['analysis_assim.land.conus', 'analysis_assim.terrain_rt.conus', + 'analysis_assim.channel_rt.conus', 'analysis_assim.reservoir.conus', + 'medium_range_blend.land.conus', 'medium_range_blend.terrain_rt.conus', + 'medium_range.land_1.conus', 'medium_range.terrain_rt_1.conus', + 'medium_range.land_2.conus', 'medium_range.terrain_rt_2.conus', + 'medium_range.land_3.conus', 'medium_range.terrain_rt_3.conus', + 'medium_range.land_4.conus', 'medium_range.terrain_rt_4.conus', + 'long_range.land_1.conus', 'long_range.land_2.conus', 'long_range.land_3.conus', 'long_range.land_4.conus', + 'short_range.land.conus', 'short_range.terrain_rt.conus'] + +NOMADS_BASE_URL = 'https://nomads.ncep.noaa.gov/pub/data/nccf/com/nwm/prod' +NWM_DATA_LOCAL_FOLDER = 'nwm_ref_files' +NWM_NGEN_TEMPLATE_FOLDER = 'ngen_templates' +NWM_NGEN_OUTPUT_FOLDER = 'ngen_outputs' +NWM_OUTPUT_FOLDER = 'nwm_output' +NWM_CONFIG_LOCAL_FOLDER = 'configs' +NWM_CONFIG_FILE_NAME = 'metadata' +NWM_CONFIG_FILE_SUFFIX = '_config' +LOG_FOLDER = 'logs' +DIM_TIME = 'time' +DIM_CATCHMENTS = 'catchments' +DIM_SOIL_LYR = 'soil_layers_stag' +DIM_SNOW_LYR = 'snow_layers' +DIM_FEATURE_ID = 'feature_id' +DIM_REF_TIME = 'reference_time' +NHF_REF_OBJECT = 'reference_flowpaths' +NHF_DIV_ID = 'div_id' +NONNHF_DIV_ID = 'divide_id' +GPKG_DIVIDES_LYR = 'divides' +GPKG_GEOMETRY_TYPE_IDENTIFIER = 'geometry_type' +GPKG_FILE_PREFIX = 'gauge_' +X_LOC = ['x', 'lon', 'longitude'] +Y_LOC = ['y', 'lat', 'latitude'] +CRS_INFO = ['crs', 'CRS', 'spatial_ref'] +JSON_X = 'x' +JSON_Y = 'y' +JSON_FILE_PATH = 'file_path' +JSON_RESOLUTION = 'resolution' +JSON_ORIGIN = 'origin' +JSON_LOC = 'location_name' +JSON_CRS = 'crs_wkt' +JSON_NWM_VAR = 'nwm_variables' +JSON_DIMENSION = 'nwm_dimensions' +JSON_VAR_DIM_MAP = 'nwm_var_dimensions' +JSON_SCALAR_VAR = 'nwm_scalar_variables' +JSON_CLASS = 'class' +JSON_CATEGORY = 'category' +JSON_DOMAIN = 'domain' +CSV_BASE_COLS = ['time step','time'] #use lowercase diff --git a/data_assimilation_engine/output_variables/utils.py b/data_assimilation_engine/output_variables/utils.py new file mode 100644 index 0000000..ec27468 --- /dev/null +++ b/data_assimilation_engine/output_variables/utils.py @@ -0,0 +1,698 @@ +import os +import shutil +import requests +import xarray as xr +import csv +import pandas as pd +import numpy as np +import json +import logging +from . import consts +from urllib.parse import urljoin +from bs4 import BeautifulSoup +from datetime import datetime +from pyproj import CRS +from typing import List, Set, Tuple, Dict, Union, Optional +from functools import reduce + + +# region common +def parse_filename_metadata(filename: str) -> Tuple[str, str, str]: + """ + Extract output_class, category and domain from filename. + """ + components = filename.split(".") + + if len(components) < 7: + raise Exception(f"NWM Output file name doesn't follow expected format: {filename}") + + output_class = components[2] + category = components[3] + domain = components[5] + return output_class, category, domain + +def convert_csvs_to_netcdf(csv_folder: str): + """ + Takes a directory for ngen output CSVs, automatically discovers NWM variables, + analyzes their native data types, and converts them to netcdf. + """ + # Find all CSV files in the target directory + csv_files = [] + try: + with os.scandir(csv_folder) as entries: + for entry in entries: + if entry.is_file() and entry.name.lower().endswith('.csv'): + csv_files.append(entry.path) + except FileNotFoundError: + print(f"The directory '{csv_folder}' does not exist.") + return + + if not csv_files: + print(f"No CSV files found in directory: {csv_folder}") + return + + # 1. Discover variables using the first valid CSV file + template_df = pd.read_csv(csv_files[0]) + base_cols = consts.CSV_BASE_COLS + nwm_variables = [col for col in template_df.columns if col.lower() not in base_cols] + + print(f"NWM variables for netcdf: {nwm_variables}") + + catchment_nc_data = [] + + for file_path in csv_files: + filename = os.path.basename(file_path) + try: + # Use file names to extract the catchment IDs. We are using splitext and isdigit + # instead of the usual replace 'cat-' so that it can handle other filename formats + # Assumes that the file name contains long integer catchment IDs. + catchment_id_name = os.path.splitext(filename)[0] + catchment_id_str = ''.join(filter(str.isdigit, catchment_id_name)) + catchment_id = np.int64(catchment_id_str) # for latest NHF schema + except ValueError: + print(f"Skipping {filename}: Could not parse catchment ID as a long integer.") + continue + + # Load CSV into Pandas + df = pd.read_csv(file_path) + + #Convert Time column to lower case + if 'Time' in df.columns: + df.rename(columns={'Time': consts.DIM_TIME}, inplace=True) + + # Safety check: Ensure this specific CSV contains all discovered data columns + if not set(nwm_variables).issubset(df.columns): + print(f"Skipping {filename}: Headers do not match the expected dataset schema.") + continue + + # Convert time to UNIX Epoch + df['datetime'] = pd.to_datetime(df['time']) + df['epoch'] = (df['datetime'].astype('int64') // 10**9).astype(np.int32) # int32 good until year 2038 + + # 2. Dynamically extract columns based on their native types + nwm_vars_dict = {} + for var in nwm_variables: + col_type = df[var].dtype + + if col_type.kind in {"U", "S", "O"}: # If the column is text/string or generic object data + nwm_vars_dict[var] = (["time"], df[var].astype(str).values) + elif col_type.kind in {"i", "u"}: # If the column is an integer type (like status codes, IDs) + nwm_vars_dict[var] = (["time"], df[var].values) + else: # Default floats/doubles + nwm_vars_dict[var] = (["time"], df[var].values) + + # Set coordinates for xarray translation + df = df.set_index('epoch') + + # Convert individual dataframe to an xarray Dataset + catchment_ds = xr.Dataset( + data_vars = nwm_vars_dict, + coords = { + consts.DIM_TIME: (["time"], df.index.values.astype(np.int32)), + consts.DIM_CATCHMENTS: catchment_id + } + ) + + # Expand the dataset to include 'entity' as a dimension axis + catchment_ds = catchment_ds.expand_dims(consts.DIM_CATCHMENTS) + catchment_nc_data.append(catchment_ds) + + if not catchment_nc_data: + print("No valid data was extracted. NetCDF generation aborted.") + return + + print(" Combining and aligning all catchments") + + # Merge all separate entities together along the 'entity' dimension + combined_ds = xr.combine_by_coords(catchment_nc_data) + + # Explicitly enforce structural array coordinates to be 64-bit long integers + combined_ds[consts.DIM_TIME] = combined_ds[consts.DIM_TIME].astype(np.int32) + combined_ds[consts.DIM_CATCHMENTS] = combined_ds[consts.DIM_CATCHMENTS].astype(np.int64) + + # Add descriptive metadata attributes dynamically + combined_ds[consts.DIM_TIME].attrs = {"units": "seconds since 1970-01-01 00:00:00", "calendar": "gregorian"} + combined_ds[consts.DIM_CATCHMENTS].attrs = {"Catchment_ID": "Catchment identifier in input"} + + for var in nwm_variables: + # Do we need to have a dictionary of variable name, mapping and units for attributes? + type_label = combined_ds[var].dtype.name + combined_ds[var].attrs = {"long_name": f"{var}"} + combined_ds[var].attrs["_FillValue"] = -1.0 + combined_ds[var].attrs["missing_value"] = -1.0 + + + output_netcdf_path = os.path.join(csv_folder, 'catchment_output.nc') + print(f"Saving unified NetCDF to: {output_netcdf_path}") + combined_ds.to_netcdf(output_netcdf_path) + print("Process complete!") + +# endregion + +# region data download +def download_nwm_data_from_server(local_root: str, re_download: bool) -> str: + """ + Main function to download a unique set of output files from the NWM server. + The root URL and the subfolder where the content is downloaded is dictated through + variables in consts.py + + Args: + local_root (str): The root folder for outputs. + re_download (bool): argument indicating that we need to redownload the data. + Defaults to False. + """ + os.makedirs(local_root, exist_ok = True) + nwm_data_folder = os.path.join(local_root, consts.NWM_DATA_LOCAL_FOLDER) + os.makedirs(nwm_data_folder, exist_ok = True) + + # Re-download if requested + # To consider: may be we should do away with the re_download argument altogether + if (re_download): + if len(os.listdir(nwm_data_folder)) > 0: + # Delete all the contents in the local folder and re-create + shutil.rmtree(nwm_data_folder) + os.makedirs(nwm_data_folder, exist_ok = True) + + formatted_date = datetime.now().strftime("%Y%m%d") + formatted_url = f"{consts.NOMADS_BASE_URL}/nwm.{formatted_date}/" + existing_keys = build_existing_keys(nwm_data_folder) # build class, category, domain keys in local folder, if exists. + download_nwm_data_recursive(formatted_url, nwm_data_folder, existing_keys) + + # After successful download of data, build the metadata config and save it to configs subfolder + config_json = obtain_metadata_information(local_root) + return config_json + +def download_nwm_data_recursive(download_url: str, local_path: str, + existing_keys: Set[Tuple[str, str, str]] +) -> None: + """ + Recursively download a unique set of output files from the server + and create a mirrored folder structure locally + """ + response = requests.get(download_url) + if response.status_code != 200: + raise Exception(f"Failed to access {download_url}") + + soup = BeautifulSoup(response.text, 'html.parser') + links = soup.find_all('a', href=True) + file_links = [link['href'] for link in links if not link['href'].startswith('/')] + if len(file_links) == 0: + raise Exception(f"No valid file links available for download from {download_url}") + + for file_link in file_links: + file_url = urljoin(download_url, file_link) + if (file_url.endswith('/')): + local_file_path = os.path.join(local_path, file_link) + download_nwm_data_recursive(file_url, local_file_path, existing_keys) + else: + if not file_link.endswith(".nc"): + continue + + output_class, category, domain = parse_filename_metadata(file_link) + key = (output_class, category, domain) + + # Skip if already exists + if key in existing_keys: + continue + else: + if not os.path.exists(local_path): + os.makedirs(local_path) + save_path = os.path.join(local_path, file_link) + download_file(file_url, save_path) + # Add to existing set after successful download + existing_keys.add(key) + +def download_file(url: str, save_path: str) -> None: + """ + Downloads a file from a URL and saves it locally. + """ + response = requests.get(url) + if response.status_code == 200: + with open(save_path, 'wb') as f: + f.write(response.content) + print(f"Downloaded: {save_path}") + else: + print(f"Failed to download: {url}, HTTP status code {response.status_code}") + +def build_existing_keys(local_root: str) -> Set[Tuple[str, str, str]]: + """ + Builds file keys in local folder. These keys are used when function is re-run + and enables the capability to not download a file again. + """ + keys = set() + for root, _, files in os.walk(local_root): + for file in files: + if file.endswith(".nc"): + output_class, category, domain = parse_filename_metadata(file) + keys.add((output_class, category, domain)) + + return keys + +# endregion + +# region metadata extraction +class NetCDFMetadata: + def __init__( + self, + file_path: str, + resolution_x: float, + resolution_y: float, + origin_x: float, + origin_y: float, + x_loc: str, + y_loc: str, + wkt: Optional[str], + variables: str, + dimensions: str, + scalar_variables: Dict[str, List[str]], + data_variables_dim: Dict[str, Union[int, float, str]], + output_class: str, + category: str, + domain: str, + ) -> None: + self.file_path = file_path + self.resolution_x = resolution_x + self.resolution_y = resolution_y + self.origin_x = origin_x + self.origin_y = origin_y + self.x_name = x_loc + self.y_name = y_loc + self.nwm_variables = variables + self.nwm_dimensions = dimensions + self.scalar_variables = scalar_variables + self.data_variables_dim = data_variables_dim + self.crs_wkt = wkt + self.output_class = output_class + self.category = category + self.domain = domain + + def key(self) -> tuple[str, str, str]: # not used yet. + """ + Unique identifier for duplicate checking + """ + return (self.output_class, self.category, self.domain) + + def __repr__(self) -> str: # for debugging purposes + return ( + f"NetCDFMetadata(file_path={self.file_path}, " + f"class={self.output_class}, category={self.category}, domain={self.domain})" + ) + +def obtain_metadata_information(local_root: str) -> str: + """ + Parse the metadata information from the downloaded NWM output files and write a CSV and a json + """ + if not os.path.exists(local_root): + raise Exception(f"Folder does not exist: {local_root}") + + nwm_local_folder = os.path.join(local_root, consts.NWM_DATA_LOCAL_FOLDER) + if not os.path.exists(nwm_local_folder): + raise Exception(f"Folder does not exist: {nwm_local_folder}") + + nwm_config_folder = os.path.join(local_root, consts.NWM_CONFIG_LOCAL_FOLDER) + os.makedirs(nwm_config_folder, exist_ok = True) + metadata_list = extract_metadata_from_downloaded_files(nwm_local_folder) + output_json = os.path.join(nwm_config_folder, + consts.NWM_CONFIG_FILE_NAME + consts.NWM_CONFIG_FILE_SUFFIX + ".json") + write_metadata_to_config_json(metadata_list, output_json) + return output_json + +def extract_metadata_from_downloaded_files(local_root: str) -> List[NetCDFMetadata]: + """ + Process downloaded files and create a list of metadata objects for config json + """ + metadata_list: List[NetCDFMetadata] = [] + + for root, _, files in os.walk(local_root): + for file in files: + if file.endswith(".nc"): + file_path = os.path.join(root, file) + metadata_list.append(extract_netcdf_metadata_from_netcdf(file_path)) + + return metadata_list + +def extract_netcdf_metadata_from_netcdf(file_path: str) -> NetCDFMetadata: + + filename = os.path.basename(file_path) + output_class, category, domain = parse_filename_metadata(filename) + res_x = -9999 + res_y = -9999 + origin_x = -9999 + origin_y = -9999 + proj_wkt = 'Not Available' + vars_in_netcdf = [] + nwm_variables = '' + dimensions = '' + x_loc_name = '' + y_loc_name = '' + scalar_dict = {} + data_dict = {} + + ds = xr.open_dataset(file_path) + dimensions = ", ".join(ds.sizes.keys()); + for nwm_var in ds.variables: + dims_list = list(ds[nwm_var].dims) + num_dims = len(dims_list) + if num_dims == 0: + scalar_val = ds[nwm_var].values.item() + if isinstance(scalar_val, bytes): #scalar variables need to be converted from bytes to string. + scalar_val = scalar_val.decode('utf-8') + scalar_dict[str(nwm_var)] = scalar_val + elif num_dims > 0: + data_dict[str(nwm_var)] = dims_list + + if nwm_var in consts.X_LOC: #XRes: assume only one of the x_loc will be in the file + x = ds.variables[nwm_var].values + res_x = compute_resolution(x) + origin_x = float(x.min()) + x_loc_name = nwm_var + elif nwm_var in consts.Y_LOC: #YRes: assume only one of the y_loc will be in the file + y = ds.variables[nwm_var].values + res_y = compute_resolution(y) + origin_y = float(y.min()) + y_loc_name = nwm_var + elif nwm_var in consts.CRS_INFO: #CRS wkt + proj_wkt = extract_wkt_from_crs(ds.variables[nwm_var]) + elif nwm_var in consts.NWM_VARIABLES_LIST or len(ds[nwm_var].dims) > 1: #NWM output variables + vars_in_netcdf.append(nwm_var) + nwm_variables = ", ".join(vars_in_netcdf) + return NetCDFMetadata(file_path, res_x, res_y, origin_x, origin_y, x_loc_name, y_loc_name, proj_wkt, + nwm_variables, dimensions, scalar_dict, data_dict, output_class, category, domain) + +def compute_resolution(coords: np.ndarray) -> float: + """ + Compute resolution from coordinate array using median spacing. + """ + if coords.size < 2: + return 0.0 + + diffs = np.diff(coords) + return float(np.median(np.abs(diffs))) + +def extract_wkt_from_crs(crs_var) -> Optional[str]: + """ + Extract wkt attribute from CRS variable. + """ + if "spatial_ref" in crs_var.attrs: + return crs_var.attrs["spatial_ref"] + elif "esri_pe_string" in crs_var.attrs: + return crs_var.attrs["esri_pe_string"] + return None + +def write_metadata_to_csv(metadata_list: List[NetCDFMetadata], output_csv: str) -> None: + """ + Writes a list of NetCDFMetadata objects to a CSV file. + """ + fieldnames: List[str] = [ + "file_path", + "resolution_x", + "resolution_y", + "origin_x", + "origin_y", + "x_name", + "y_name", + "crs_wkt", + "variables", + "class", + "category", + "domain", + ] + + with open(output_csv, mode="w", newline="", encoding="utf-8") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + + writer.writeheader() + + for metadata in metadata_list: + writer.writerow( + { + "file_path": metadata.file_path, + "resolution_x": metadata.resolution_x, + "resolution_y": metadata.resolution_y, + "origin_x": metadata.origin_x, + "origin_y": metadata.origin_y, + "x_name": metadata.x_name, + "y_name": metadata.y_name, + "crs_wkt": metadata.crs_wkt if metadata.crs_wkt is not None else "Not Available", + "variables": metadata.nwm_variables, + "class": metadata.output_class or "", + "category": metadata.category or "", + "domain": metadata.domain or "", + } + ) + +def write_metadata_to_config_json(metadata_list: List[NetCDFMetadata], output_json: str) -> None: + """ + Writes NetCDFMetadata objects to config JSON. + """ + # Delete any existing config file + if os.path.exists(output_json): + os.remove(output_json) + + info_list = [] + for mdata in metadata_list: + metadata_info_dict = { + consts.JSON_FILE_PATH: mdata.file_path, + consts.JSON_RESOLUTION: { + consts.JSON_X: mdata.resolution_x, + consts.JSON_Y: mdata.resolution_y, + }, + consts.JSON_ORIGIN: { + consts.JSON_X: mdata.origin_x, + consts.JSON_Y: mdata.origin_y, + }, + consts.JSON_LOC: { + consts.JSON_X: mdata.x_name, + consts.JSON_Y: mdata.y_name, + }, + consts.JSON_CRS: mdata.crs_wkt, + consts.JSON_NWM_VAR: mdata.nwm_variables, + consts.JSON_DIMENSION: mdata.nwm_dimensions, + consts.JSON_SCALAR_VAR: mdata.scalar_variables, + consts.JSON_VAR_DIM_MAP: mdata.data_variables_dim, + consts.JSON_CLASS: mdata.output_class, + consts.JSON_CATEGORY: mdata.category, + consts.JSON_DOMAIN: mdata.domain + } + info_list.append(metadata_info_dict) + + config = { + "files": info_list + } + with open(output_json, "w", encoding="utf-8") as f: + json.dump(config, f, indent = 2) + + print(f"Config JSON written to: {output_json}") + +def debug_netcdf_structure_in_folder(folder_path: str) -> None: + """ + Recursively scans a folder for NetCDF files and prints their structure. + """ + # This function is used for testing purposes. + # It also uses a local logging to save all information into a file. + nc_files: List[str] = [] + data_variables_list = [] + logging.basicConfig(filename='app.log', level=logging.INFO, format='%(message)s') + for root, _, files in os.walk(folder_path): + for file in files: + if file.endswith(".nc"): + nc_files.append(os.path.join(root, file)) + + if not nc_files: + print(f"No NetCDF files found in: {folder_path}") + return + logging.info(f"Found {len(nc_files)} NetCDF files within the folder") + + for file_path in nc_files: + logging.info("-" * 80) + logging.info(f"FILE: {file_path}") + + try: + with xr.open_dataset(file_path) as ds: + # Variables + logging.info("--Variables:") + for var_name, var in ds.data_vars.items(): + logging.info(f"---- {var_name}: shape={var.shape}, dtype={var.dtype}") + output_class, category, domain = parse_filename_metadata(file_path) + data_variables_list.append(var_name + ";" + output_class + ";" + category + ";" + domain) + + # Dimensions + logging.info("--Dimensions:") + for dim_name, dim_size in ds.sizes.items(): + logging.info(f"---- {dim_name}: size={dim_size}") + + # CRS-specific attributes if present + is_crs_present = False + is_sref_present = False + for crs_name in ["crs", "CRS", "spatial_ref"]: + if crs_name in ds.variables: + logging.info(f"--CRS Variable: {crs_name}") + is_crs_present = True + crs_var = ds.variables[crs_name] + if 'spatial_ref' in crs_var.attrs: + spatial_ref_raw = ds.variables[crs_name].attrs['spatial_ref'] + is_sref_present = True + logging.info(f"----{spatial_ref_raw}") + logging.info(f"-- CRS Available: {is_crs_present}; Spatial_ref Available: {is_sref_present}") + + # The snippet below is use to write variables along with the class, category and domain + # This produces an additional csv with just basic variables information. + with open('variables_ccategories.csv', 'w', newline='') as f: + writer = csv.writer(f) + writer.writerow(["Variable", "Class", "Category", "Domain"]) + for var in data_variables_list: + if not var.startswith('crs'): + row = var.split(";") + writer.writerow(row) + except Exception as e: + print(f"\n Failed to read file: {file_path}") + print(f"Error: {e}") + +def read_output_variables_info_from_config(json_file: str) -> List[NetCDFMetadata]: + """ + Parses a multi-element JSON string into a list of NetCDFMetadata objects. + """ + netcdf_metadata_list: List[NetCDFMetadata] = [] + if os.path.isfile(json_file): + with open(json_file, "r", encoding="utf-8") as file_stream: + config_data = json.load(file_stream) + + for item in config_data.get("files", []): + info_item = NetCDFMetadata( + file_path = item.get(consts.JSON_FILE_PATH, ''), + resolution_x = item.get(consts.JSON_RESOLUTION, {}).get(consts.JSON_X, -9999), + resolution_y = item.get(consts.JSON_RESOLUTION, {}).get(consts.JSON_Y, -9999), + origin_x = item.get(consts.JSON_ORIGIN, {}).get(consts.JSON_X, -9999), + origin_y = item.get(consts.JSON_ORIGIN, {}).get(consts.JSON_Y, -9999), + x_loc = item.get(consts.JSON_LOC, {}).get(consts.JSON_X, ''), + y_loc = item.get(consts.JSON_LOC, {}).get(consts.JSON_Y, ''), + wkt = item.get(consts.JSON_CRS, ''), + variables = item.get(consts.JSON_NWM_VAR, ''), + dimensions = item.get(consts.JSON_DIMENSION, ''), + scalar_variables = item.get(consts.JSON_SCALAR_VAR, {}), + data_variables_dim = item.get(consts.JSON_VAR_DIM_MAP, {}), + output_class = item.get(consts.JSON_CLASS, ''), + category = item.get(consts.JSON_CATEGORY, ''), + domain = item.get(consts.JSON_DOMAIN, '') + ) + netcdf_metadata_list.append(info_item) + else: + raise ValueError("Specified config file does not exist") + + return netcdf_metadata_list +# endregion + +# region basin grid products +def create_combined_basin_netcdf_products (reference_grid: str, timestep_netcdf_folder: str, output_folder: str) -> None: + """ + Main calling function to create a combined basins product + """ + # Randomly pick timestep outputs at 6 hour intervals + for i in range(0, 20, 6): + search_string = f"01T{i:02}" + files_list = find_files_by_timestep(timestep_netcdf_folder, search_string) + variables_of_interest = ["ACCET"] + # merged_ds = merge_basin_netcdfs(reference_grid, files_list) + merged_ds, encoding = create_multi_basin_netcdfs(reference_grid, files_list, variables_of_interest, 1.0, None, True) + output_file = os.path.join(output_folder, f"combined_grid_{i:02}_.nc") + merged_ds.to_netcdf(output_file, encoding = encoding, engine='netcdf4') + +def find_files_by_timestep(root_dir: str, search_timestep: str) -> List[str]: + """ + Function to recursively search a folder for a specific substring + """ + matched_files = [] + for dirpath, _, filenames in os.walk(root_dir): + for fname in filenames: + if search_timestep in fname and fname.endswith('.nc'): + full_path = os.path.join(dirpath, fname) + matched_files.append(full_path) + return matched_files + +def create_multi_basin_netcdfs(reference_grid: str, nc_files: list[str], variables_of_interest: list[str] = None, + tolerance: float = 1.0, fill_value: float | None = None, + check_crs: bool = True +) -> xr.Dataset: + """ + Merge multiple NetCDF subsets. + Assumes same grid resolution, same coordinate system and no overlaps + """ + # Check CRS and confirm that they are the same for all the netcdf files + if check_crs: + crs_list = [] + for nc_file in nc_files: + with xr.open_dataset(nc_file) as ds: + if "crs" in ds: + crs_list.append(ds.variables["crs"].attrs['spatial_ref']) + else: + raise ValueError("One dataset missing CRS") + + if len(set(crs_list)) != 1: + raise ValueError("CRS mismatch between datasets") + + ref_grid = xr.open_dataset(reference_grid) + # the test datasets have different units for variables. + # For testing, just making them all same units. + with xr.open_dataset(nc_files[0]) as ds_sample: + standardized_time = ds_sample.time.values + + # Automatically grab all data variables from the first file + # if variables of interest is not provided. + if variables_of_interest is None: + with xr.open_dataset(nc_files[0]) as temp_ds: + variables_of_interest = list(temp_ds.data_vars) + + combined_variables_dict = {} + for var in variables_of_interest: + reindexed_var_arrays = [] + for nc_file in nc_files: + with xr.open_dataset(nc_file) as ds: + # the test datasets have different units for variables. + # For testing, just making them all same units. + for data_var in ds.data_vars: + ds[data_var].attrs['variable units'] = 'm' + + # Isolate the target variable to protect other variables from blooming into NaNs prematurely + var_of_interest = ds[var] + + # Map to the national spatial coordinates layout + var_reindexed = var_of_interest.reindex(y=ref_grid.y, x=ref_grid.x, method="nearest", tolerance=tolerance) + + # Unify the timestamp - this is only for testing. + # In reality, all grids will have the same simulation timesteps + var_aligned = var_reindexed.assign_coords(time=standardized_time) + reindexed_var_arrays.append(var_aligned) + + # merge the variable for all individual basins + combined_variables_dict[var] = reduce(lambda left, right: left.combine_first(right), reindexed_var_arrays) + + ds_combined = xr.Dataset(data_vars=combined_variables_dict, + coords={"time": standardized_time, "y": ref_grid.y, "x": ref_grid.x}, + attrs=ref_grid.attrs) + + encoding_config = {} + for var_name in variables_of_interest: + encoding_config[str(var_name)] = { + "zlib": True, + "complevel": 4, + "_FillValue": fill_value, + } + + return ds_combined, encoding_config + +def combined_channel_reservoir_netcdfs(nc_files_folder: str, output_folder: str, + output_class: str, category: str, domain: str): + + # Path to your NetCDF files + file_pattern = "path/to/files/*.nc" + + # Combine strictly along the 'entity' dimension + ds = xr.open_mfdataset( + file_pattern, + combine="nested", + concat_dim="entity", + coords="minimal", # Keeps identical coordinates (like time) from duplicating + compat="override" # Skips expensive consistency checks for identical variables + ) + +# endregion \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/EmptyDirOrFileException.py b/data_assimilation_engine/rfc_ingestion/EmptyDirOrFileException.py new file mode 100644 index 0000000..dbc5ea2 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/EmptyDirOrFileException.py @@ -0,0 +1,11 @@ +class EmptyDirOrFileException( Exception ): + """Exception raised for empty input files or directories. + Attributes: + expression -- input expression in which the error occurred + message -- explanation of the error + """ + pass + +# def __init__(self, message): +# self.expression = expression +# self.message = message diff --git a/data_assimilation_engine/rfc_ingestion/PI_XML.py b/data_assimilation_engine/rfc_ingestion/PI_XML.py new file mode 100644 index 0000000..b5ab05d --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/PI_XML.py @@ -0,0 +1,620 @@ +############################################################################### +# Module name: PI_XML +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: # +# # +# Last modification date: 07/02/2019 # +# # +# Description: manage data in a RFC FEWS PI xml file # +# # +############################################################################### + +import os, sys, time, csv, re +import logging +from string import * +from collections import OrderedDict +from datetime import datetime, timedelta +import dateutil.parser +import math +import pytz +#import iso8601 +import xml.etree.ElementTree as etree +import copy +from RFC_Forecast import RFC_Forecast +from RFC_Sites import RFC_Sites +class PI_XML: + """ + Store one RFC forecast data + """ + + @property + def timeZone(self): + return self._timeZone + + @timeZone.setter + def timeZone(self, t): + self._timeZone = t + + @property + def series(self): + return self._series + + @series.setter + def series(self, s): + self._series = s + + @property + def rfcsites(self): + return self._rfcsites + + @rfcsites.setter + def rfcsites(self, r): + self._rfcsites = r + + def __init__(self, pixmlfilename, sites ): + """ + Initialize the RFC_Forecast object with a given + filename + """ + self.source = pixmlfilename + self.series = [] + self.rfcsites = sites + + self.loadPIxml( pixmlfilename ) + + def loadPIxml( self, pixmlfilename ): + try: + fstwml = etree.parse( pixmlfilename ) + root= fstwml.getroot() + self.timeZone = root.find( '{http://www.wldelft.nl/fews/PI}timeZone' ).text + for series in \ + root.iter('{http://www.wldelft.nl/fews/PI}series'): + self.series.append( series ) +# for series in self.series: +# print(series.tag, series.attrib ) +# print ( 'timeZone = ', self.timeZone ) + + except Exception as e: + raise RuntimeError( "WARNING: parsing PI XML error: " + str( e )\ + + ": " + pixmlfilename + " skipping ..." ) + + def toRFCForecast(self): + +# print("to rfcforecast") + rfc_series = [] + for s in self.series: +# print(s.tag, s.attrib ) + rfc_series.append( RFC_Forecast( s, self.rfcsites )) + + return rfc_series + + def getFlowTimeseries( self ): + return list( filter( lambda s: s.parameterId in \ + [ "RQOT", "QINE", "SQIN" ] and \ + s.rfcname is not None and \ + not s.isEmpty(), self.toRFCForecast() ) ) + + def getAllStationIDs( self ): + return set( map( lambda s: s.get5CharStationID(), \ + self.getFlowTimeseries() ) ) + + + def getObservedAndForecastForID( self, id ): + fst = next( ( s for s in self.getFlowTimeseries() \ + if s.isForecast() and \ + s.get5CharStationID() == id ), None ) + obv = next( ( s for s in self.getFlowTimeseries() \ + if s.isObserved() and \ + s.get5CharStationID() == id ), None ) + # + # Some RFCs already combined observed and forecast series + # + if fst is not None and obv is None: + # + # Split the forecast series into observed and forecast + # + obv = copy.deepcopy( fst ) + T0 = fst.getT0() + dt = timedelta( seconds = obv.getTimeStepInSeconds() ) + t = obv.getTimePeriod()[1] + while t >=T0: + if obv.timeValueQuality: + obv.timeValueQuality.popitem() + t = t - dt + + obv.resetTimePeriod() + obv.qualifierId = "observed" + + t = fst.getTimePeriod()[0] + while t < T0: + fst.timeValueQuality.popitem(last=False) + t = t + dt + fst.resetTimePeriod() + + return( fst, obv ) + + #Some RFCs such as NE have observed and forecast in separate + #time series + def getReserviorForecastWithT0(self): + series = self.toRFCForecast() + observed = [] + forecast = [] + parameterset = set( ["RQOT", "QINE", "SQIN" ] ) + for s in series: + if s.parameterId in parameterset and not s.isEmpty(): + if s.isObserved(): + observed.append( s ) + else: + if s.isForecast(): + forecast.append( s ) + else: + # + # CNRFC doesn't have the tag + # + if s.rfcname == "CNRFC": + forecast.append( s ) +# # NCRFC observed doesn't have the qualifierId tag +# if s.rfcname == "NCRFC": +# observed.append( s ) + + + + if observed: + for o in observed: + for s in forecast: + if o.stationID == s.stationID: + if s.startDate == s.getT0(): +# #NERFC +# if s.stationID == "SWRN6" or \ +# s.stationID == "STDM1" or \ +# s.stationID == "RKWM1": +# +# s.timeValueQuality.update( {s.startDate :\ +# o.timeValueQuality[ next(reversed( \ +# o.timeValueQuality.keys() ) ) ] } ) +# +# else: + if s.startDate in o.timeValueQuality.keys(): + if not math.isclose( \ + o.timeValueQuality[ s.startDate ][ 0 ], + float( o.missVal ), abs_tol=0.0001 ): + + s.timeValueQuality.update( {s.startDate :\ + o.timeValueQuality.get( s.startDate, \ + (float( s.missVal ), 0) ) } ) + + elif s.startDate < s.getT0(): + tvq = OrderedDict() + for k in s.timeValueQuality: + if k >= s.getT0(): + tvq[ k ] = s.timeValueQuality[ k ] + s.timeValueQuality=tvq + if s.timeValueQuality.has_key( s.getT0() ): + s.timeValueQuality.update( {s.getT0(): \ + o.timeValueQuality.get( s.getT0()) } ) + s.timeValueQuality.move_to_end( s.getT0(), \ + last=False ) + + s.startDate = s.getT0() + + s.fstPeriod = \ + list( s.timeValueQuality.keys() )[0], \ + list( s.timeValueQuality.keys() )[-1] + else: + t = s.startDate - timedelta( \ + seconds = s.getTimeStepInSeconds()) + while t > s.getT0(): + s.timeValueQuality.update({t : \ + ( float( s.missVal ), 0 ) } ) + s.timeValueQuality.move_to_end( t, \ + last=False ) + t = t - timedelta( \ + seconds = s.getTimeStepInSeconds()) + + s.timeValueQuality.update({t: \ + o.timeValueQuality.get( t, \ + (float( s.missVal) , 0)) } ) + s.timeValueQuality.move_to_end( t, \ + last=False ) + s.startDate = s.getT0() + + s.fstPeriod = \ + list( s.timeValueQuality.keys() )[0], \ + list( s.timeValueQuality.keys() )[-1] + break + forcast_without_observed = [] + for s in forecast: + found = False + for o in observed: + if o.stationID == s.stationID: + found = True + break + if not found: + forcast_without_observed.append( s ) + + for f in forcast_without_observed: + if f.startDate < f.getT0(): + tvq = OrderedDict() + for k in f.timeValueQuality: + if k >= f.getT0(): + tvq[ k ] = f.timeValueQuality[ k ] + f.timeValueQuality=tvq + else: + t = f.startDate - timedelta( \ + seconds = f.getTimeStepInSeconds()) + while t >= f.getT0(): + f.timeValueQuality.update({t : \ + ( float( f.missVal ), 0 ) } ) + f.timeValueQuality.move_to_end( t, \ + last=False ) + t = t - timedelta( \ + seconds = f.getTimeStepInSeconds()) + +# print( "getReserviorForecastWithT0:" + f.stationID) + f.startDate = f.getT0() + f.fstPeriod = list( f.timeValueQuality.keys() )[0], \ + list( f.timeValueQuality.keys() )[-1] + + else: + for f in forecast: + if f.startDate < f.getT0(): + tvq = OrderedDict() + for k in f.timeValueQuality: + if k >= f.getT0(): + tvq[ k ] = f.timeValueQuality[ k ] + f.timeValueQuality=tvq + else: + t = f.startDate - timedelta( \ + seconds = f.getTimeStepInSeconds()) + while t >= f.getT0(): + f.timeValueQuality.update({t : \ + ( float( f.missVal ), 0 ) } ) + f.timeValueQuality.move_to_end( t, \ + last=False ) + t = t - timedelta( \ + seconds = f.getTimeStepInSeconds()) + +# print( "getReserviorForecastWithT0:" + f.stationID) + f.startDate = f.getT0() + f.fstPeriod = list( f.timeValueQuality.keys() )[0], \ + list( f.timeValueQuality.keys() )[-1] + return forecast + + def getRFCName( self ): + YYYYMMDDHH = '[0-9]{4}(0[1-9]|1[0-2])(0[1-9]|[1-2][0-9]|3[0-1])(2[0-3]|[01][0-9])' + YYYYMMDD = '[0-9]{4}(0[1-9]|1[0-2])(0[1-9]|[1-2][0-9]|3[0-1])' + YYYYMMDDHHMMSS = YYYYMMDDHH + '(0[0-9]|[1-5][0-9]){2}' + CBRFCPattern = '^(.*/)?CBRFC_Reservoirs_' + YYYYMMDDHH + '.xml' + NERFCPattern = '^(.*/)?NERFC_Reservoir_Export.xml' + LMRFCPattern = '^(.*/)?LMRFC_RESERVOIR_FORECAST\.' + YYYYMMDD + + ABRFCPattern = '^(.*/)?' + YYYYMMDDHH + '_RES_NWM_pixml_export.' + YYYYMMDDHHMMSS + NWRFCPattern = '^(.*/)?' + YYYYMMDDHH + '_QINE_NWM_Res_export.' + YYYYMMDDHHMMSS + MBRFCPattern = '^(.*/)?' + YYYYMMDDHHMMSS + '_RQOT_Forecast_.+_xml_' + YYYYMMDDHH + RFCName = None + result = re.match(CBRFCPattern, self.source) + if result: + RFCName = 'CBRFC' + else: + result = re.match(NERFCPattern, self.source) + if result: + RFCName = 'NERFC' + else: + result = re.match(ABRFCPattern, self.source) + if result: + RFCName = 'ABRFC' + else: + result = re.match(NWRFCPattern, self.source) + if result: + RFCName = 'NWRFC' + else: + result = re.match(LMRFCPattern, self.source) + if result: + RFCName = 'LMRFC' + else: + result = re.match(MBRFCPattern, self.source) + if result: + RFCName = 'MBRFC' + + return RFCName + + + def allStations( self ): + stations = set() + series = self.toRFCForecast() + for s in series: + stations.add( s.get5CharStationID() ) + return stations + + def getRFCNameBySitelist( self, gages ): + sta = self.allStations() + for rfc in gages.RFC: + rfcgages = gages.getSitesByRFC( rfc ) + if not sta.isdisjoint( set( rfcgages ) ): + return rfc + + # + # Apply the persistence and linear interplation algorithm + # + # For observed, persist both forward and backward up to 24 hours, + # linear interpolate between observed values. However, the maximum gap + # between interplation is 48 hours. That is if the gap between is + # more than 48 hours, fill the gap with -999 (missing) values. + # + # For forecast, persist only in the forward direction up to 24 hours. + # Linear interpolate between values. If the gap is more than 48 hours, + # fill with missing values. + # + # see email thread "rfc obs" Nov 13, 2019 + # + def combineObvFstAndApplyPresistenceLinearInterpolation( self, \ + obvfst): + obv = copy.deepcopy( obvfst[1] ) + fst = copy.deepcopy( obvfst[0] ) + obv.linearInterpolate(timedelta( hours = 1 ) ) + fst.interForwardPersist(timedelta( hours = 1 ) ) + combined = copy.deepcopy( obv ) + obvperiod = obv.getTimePeriod() + fstperiod = fst.getTimePeriod() + if obvperiod[1] >= fstperiod[ 0 ]: + for k, v in fst.timeValueQuality.items(): + if k not in combined.timeValueQuality: + combined.timeValueQuality[ k ] = v + elif obvperiod[1] + timedelta( days = 1 ) < fstperiod[ 0 ]: + combined.persistForward( obvperiod[1] + timedelta( days = 1 ) ) + t = obvperiod[1] + timedelta( days = 1 ) + while t < fstperiod[ 0 ]: + combined.timeValueQuality[ t ] = \ + ( float( combined.missVal ), 0.0, True ) + t = t + \ + timedelta( seconds = combined.getTimeStepInSeconds() ) + for k, v in fst.timeValueQuality.items(): + combined.timeValueQuality[ k ] = v + else: + combined.persistForward( fstperiod[ 0 ] ) + for k, v in fst.timeValueQuality.items(): + combined.timeValueQuality[ k ] = v + + combined.resetTimePeriod() + combined.qualifierId = fst.qualifierId + combined.forecastDate = fst.forecastDate + + combined_period = combined.getTimePeriod() + T0 = combined.getT0() + + if T0 - timedelta( days = 2 ) < combined_period[0]: + combined.persistBackward( T0 - timedelta( days = 2 ),\ + timedelta( days = 1 ) ) + + if T0 + timedelta( days = 10 ) > combined_period[ 1 ]: + combined.persistForward( T0 + timedelta( days = 10, + seconds = combined.getTimeStepInSeconds() ) ) + + return combined + + # + # See the comment below for the presist and interpolate algorithms + # + def linearInterpolateObservedPresisForecastAndCombine( self, o, f): + + f.interForwardPersist( timedelta( hours = 1 ) ) + fstperiod = f.getTimePeriod() + o.linearInterpolate( timedelta( hours = 1 ) ) + obvperiod = o.getTimePeriod() + # + # If the observation period and the forecast period + # overlap. + if obvperiod[1] >= fstperiod[ 0 ]: + t = fstperiod[ 0 ] - \ + timedelta( seconds = f.getTimeStepInSeconds() ) + while t >= obvperiod[ 0 ]: + f.timeValueQuality.update( {t: \ + o.timeValueQuality[ t ] } ) + f.timeValueQuality.move_to_end( t, last=False ) + t = t - timedelta( \ + seconds = f.getTimeStepInSeconds() ) + # + # If the last observation is less than 2 days from the + # T0 value. + elif obvperiod[1] + timedelta( days = 2 ) >= fstperiod[ 0 ]: + t0 = f.getT0() + #interpolate between t0 and obvperiod[1] + if t0 == fstperiod[0]: + t1 = obvperiod[ 1 ] + t = t0 - timedelta( hours = 1 ) + while t > t1: + f.timeValueQuality.update( {t: \ + ( ( f.timeValueQuality[ t0 ][0] - \ + o.timeValueQuality[ t1 ][ 0 ] ) *\ + ( ( t - t1 ).total_seconds() / \ + ( t0 - t1 ).total_seconds() ) + \ + o.timeValueQuality[ t1 ][ 0 ] , + ( f.timeValueQuality[ t0 ][1] - \ + o.timeValueQuality[ t1 ][ 1 ] ) *\ + ( ( t - t1 ).total_seconds() / \ + ( t0 - t1 ).total_seconds() ) + \ + o.timeValueQuality[ t1 ][ 1 ] , + True ) } ) + f.timeValueQuality.move_to_end( t, last=False ) + t = t - timedelta( hours = 1 ) + t = t1 + while t >= obvperiod[0]: + f.timeValueQuality.update( {t: \ + o.timeValueQuality[ t ] } ) + f.timeValueQuality.move_to_end( t, last=False ) + t = t - timedelta( hours = 1 ) + + #persist obvperiod[1] up to 1 day + else: #t0 < fstperiod[0] or t0 > fstperiod[0]: + t1 = obvperiod[ 1 ] + t = fstperiod[0] - timedelta( hours = 1 ) + while t > t1 + timedelta( days = 1 ): + f.timeValueQuality.update( {t: \ + ( float( s.missVal ), 0.0, True ) } ) + f.timeValueQuality.move_to_end( t, \ + last=False ) + t = t - timedelta( hours = 1 ) + while t > t1: + f.timeValueQuality.update( {t: \ + o.timeValueQuality[ t1 ] } ) + f.timeValueQuality.move_to_end( t, last=False ) + t = t - timedelta( hours = 1 ) + while t >= obvperiod[0]: + f.timeValueQuality.update( {t: \ + o.timeValueQuality[ t ] } ) + f.timeValueQuality.move_to_end( t, last=False ) + t = t - timedelta( hours = 1 ) + + # + # If the last observation is more than 2 days from the + # T0 value. + else: #persist obvperiod[1] up to 1 day + t1 = obvperiod[ 1 ] + t = fstperiod[0] - timedelta( hours = 1 ) + while t > t1 + timedelta( days = 1 ): + f.timeValueQuality.update( {t: \ + ( float( s.missVal ), 0.0, True ) } ) + f.timeValueQuality.move_to_end( t, \ + last=False ) + t = t - timedelta( hours = 1 ) + while t > t1: + f.timeValueQuality.update( {t: \ + o.timeValueQuality[ t1 ] } ) + f.timeValueQuality.move_to_end( t, last=False ) + t = t - timedelta( hours = 1 ) + while t >= obvperiod[0]: + f.timeValueQuality.update( {t: \ + o.timeValueQuality[ t ] } ) + f.timeValueQuality.move_to_end( t, last=False ) + t = t - timedelta( hours = 1 ) + + +# 1) No interpolation for forecast values. +# 2) Interpolate observed values if the gap is less or equal to 48 +# hours. Otherwise, fill the gap with missing values such as -999. +# 3) Persist forecast values if there is a gap between the values. +# The maximum gap is 10 days (this is also the total length of our +# forecast values). +# 4) When it is necessary, persist observed values backward and +# forward for up to 1 day at the first and last values respectively. +# 5) Persist the last forecast value to 10 days from T0. +# 4*) I think any observed value could be persisted forward/backward up +# to 1 day. For example, if there are observations at T-6 through T-1 +# and nothing before, then the value at T-6 could be persisted +# backwards to T-30. +# 5) If there is a T0 value and nothing beyond that, then the T0 value is +# persisted through day 10. If the last available forecast value is at +# T36, then that value is persisted through day 10. +# 6) If no observation (including T0 value) is provided, fill the +# observation including T0 with missing value (-999). +# 7) When RFC provides observations longer than 2 days before T0 , +# keep the values. That means the time silce will have observation +# longer than 2 dyas +# 8) If RFC provides forecast longer than 10 days, keep the values even +# the time slices will have forecast values longer than 10 days from +# T0. +# +# NOTE: * Later it is changed to persisted backward up to 2 days, in this +# example, the value at T-6 could persisted backwards to T-54. + + def getReserviorObservedForecastCombinedWithT0(self): + + series = self.toRFCForecast() + obs = [] + fst = [] + # + # get only then non-empty timeseries + # + for s in series: + if s.parameterId in [ "RQOT", "QINE", "SQIN" ] and \ + s.rfcname is not None and not s.isEmpty(): + # if s.parameterId in [ "RQOT", "QINE", "SQIN" ]: + if s.isForecast(): + fst.append( s ) + else: + obs.append( s ) + + + for f in fst: + obv = None + t0 = f.getT0() + + # + # APRFC linear interpolate 6 hourly data to hourly data + # Here is the Google Doc about the APRFC forecast data + # https://docs.google.com/document/d/1vMvBvi-aBbXkq6goYBGjentlUxt4HhxHfygGs01l9xE/edit + # + # Section: + # + # Data Preprocessing on APRFC pixml files for the GDL release + # 1) Check with APRFC and ensure: + # They follow one of the naming conventions provided to them by Zhengtao (WGRFC_02.24.2021.06.00.xml or 202102230102_LMRFC_Reservoir_export.xml) (confirmed) + # They send data in a regular time resolution (6-hourly) (confirmed) + # They are fine with us utilizing linear interpolation for in-between values of a time series to convert 6-hr data to hourly data (confirmed) + # 2) Implement a zero-value filling for expansion of time series to 12-day, instead of persisting on the last value in the time series. + # + # Email from john.mattern@noaa.gov on 4/13/2021 + # Summary: + # 1) Interpolate 6 hourly data to hourly + # 2) Extend forecast data to 10 days from T0 with 0s + # 3) If there is no observed, fill observed period, T0 - 2 days to + # T0, with 0s, + # 4) If there is observed, but shorter than 2 days, then extend + # backward with 0s. + # + # Email thread with subject "Glacial lake damming/outflow for NWM V3.0" + # on June 15, 2021. + # After discussions with Dave Streubel, David Mattern, Brain Cosgrove, + # and Mehdei Rezaeianzadeh, we will interpolate the values between + # zero and the first non-zero values, and between the last non-zero + # and zero. That is if the first value provided is non-zero, we + # interpolate from the preceding zero (valid at an earlier 6 hour time) + # to the first (non-zero) value you provide in the timeseries. Likewise, + # we apply linear interpolation for the timestep after the last value. + # + if f.rfcname == 'APRFC': + f.addLeadingAndTrailingZeros() + f.linearInterpolate( timedelta( hours = 1 ) ) + f.persistBackward( t0 - timedelta( days=2 ), \ + timedelta( days = 2 ), 0.0 ) + f.persistForward( t0 + timedelta( days=10 ), 0.0 ) + + else: + for o in obs: + if o.stationID == f.stationID: + obv = o + break + # + #For RFCs have separate observation and forecast sections + # + if obv and not obv.isEmpty(): + + self.linearInterpolateObservedPresisForecastAndCombine( \ + obv, f ) + # + #For RFCs have combined observation and forecast into one + # section. + # + else: + f.linearInterpolateAndInterForwardPersist(\ + timedelta( hours = 1 ) ) + + f.persistBackward( t0 - timedelta( days=2 ), \ + timedelta( days = 2 ) ) + f.persistForward( t0 + timedelta( days=10 ) ) + + f.resetTimePeriod() + + # + # get the time series even it is empty + # + fst.clear() + for s in series: + if s.parameterId in [ "RQOT", "QINE", "SQIN" ]: + if s.isForecast(): + fst.append( s ) + return fst + + diff --git a/data_assimilation_engine/rfc_ingestion/PI_XML_test.py b/data_assimilation_engine/rfc_ingestion/PI_XML_test.py new file mode 100644 index 0000000..67a79d7 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/PI_XML_test.py @@ -0,0 +1,283 @@ +import unittest +from datetime import datetime, timedelta +from PI_XML import PI_XML +from RFC_Sites import RFC_Sites + +class PI_XML_test(unittest.TestCase): + def test(self): + self.assertTrue(True) + + def testInit(self): + pixml = PI_XML('./NERFC_Reservoir_Export.xml') + self.assertEqual('NERFC', pixml.getRFCName()) + self.assertEqual(pixml.timeZone, '0.0') + + def test_toRFCForecast(self): + pixml = PI_XML('./NERFC_Reservoir_Export.xml') + + print("test_toRFCForecast", pixml.source) + +# for s in pixml.series: +# print(s.tag) + + rfc_series = pixml.toRFCForecast() + + for s in rfc_series: + print( s.stationID ) + print( s.missVal ) + if len(s.timeValueQuality) > 0: + for k, v in s.timeValueQuality.items(): + if abs(float( v[0] ) - float( s.missVal )) > 0.0001: + print(k, v) + + self.assertEqual( len( rfc_series ), 5 ) + self.assertEqual( rfc_series[ 0 ].stationID, 'INRN6' ) + self.assertEqual( rfc_series[ 1 ].stationID, 'SWRN6' ) + self.assertEqual( rfc_series[ 2 ].stationID, 'INRN6' ) + self.assertEqual( rfc_series[ 3 ].stationID, 'SWRN6' ) + self.assertEqual( rfc_series[ 4 ].stationID, 'INDN6HUD' ) + + def test_2nd_RFC(self): + pixml = PI_XML('./2019052412_QINE_NWM_Res_export.20190702202800') + self.assertEqual('NWRFC', pixml.getRFCName()) + self.assertEqual( len( pixml.series ), 2 ) + + def test_2nd_RFC_toRFCForecast(self): + print( 'test_2nd_RFC_toRFCForecast:') + pixml = PI_XML('./2019052812_QINE_NWM_Res_export.20190702202832') + rfc_series = pixml.toRFCForecast() + self.assertEqual( len( rfc_series ), len(pixml.series) ) + self.assertAlmostEqual( rfc_series[0].getTimeValueAt( \ + datetime( 2019, 5, 28, 12, 0, 0) )[1][0], \ + 1.59751 ) + + def test_2nd_RFC_QPF(self): + print( 'test_2nd_RFC_QPF:') + pixml = PI_XML('./2019052812_QPF_chps_pixml_export.20190702202832') + rfc_series = pixml.toRFCForecast() + self.assertEqual( len( rfc_series ), len(pixml.series) ) + self.assertAlmostEqual( rfc_series[3].getTimeValueAt( \ + datetime( 2019, 6, 1, 0, 0, 0) )[1][0], \ + 0.011811 ) + + + def test_ABRFC(self): + pixml = PI_XML('./2019052112_RES_NWM_pixml_export.20190705164143') + self.assertEqual('ABRFC', pixml.getRFCName()) + self.assertEqual( len( pixml.series ), 108 ) + + + def test_ABRFC_toRFCForecast(self): + print( 'test_ABRFC_toRFCForecast:') + pixml = PI_XML('./2019052112_RES_NWM_pixml_export.20190705164143') + rfc_series = pixml.toRFCForecast() + + for s in rfc_series: + print( s.stationID ) + if len(s.timeValueQuality) > 0: + for k, v in s.timeValueQuality.items(): + print(k, v) + + self.assertEqual( len( rfc_series ), len(pixml.series) ) + self.assertEqual( rfc_series[8].stationID, 'CHNK1' ) + self.assertAlmostEqual( rfc_series[8].getTimeValueAt( \ + datetime( 2019, 5, 19, 6, 0, 0) )[1][0], \ + 1534.9979 ) + def test_CBRFC(self): + pixml = PI_XML('./CBRFC_Reservoirs_2019070612.xml') + self.assertEqual('CBRFC', pixml.getRFCName()) + self.assertEqual( len( pixml.series ), 2 ) + + def test_CBRFC_toRFCForecast(self): + print( 'test_CBRFC_toRFCForecast:') + pixml = PI_XML('./CBRFC_Reservoirs_2019070712.xml') + rfc_series = pixml.toRFCForecast() + + for s in rfc_series: + print( s.stationID ) + if len(s.timeValueQuality) > 0: + for k, v in s.timeValueQuality.items(): + print(k, v) + + self.assertEqual( len( rfc_series ), len(pixml.series) ) + self.assertEqual( rfc_series[1].stationID, 'GRZU1OSF' ) + self.assertAlmostEqual( rfc_series[1].getTimeValueAt( \ + datetime( 2019, 7, 16, 17, 0, 0) )[1][0], \ + 1487.9976 ) + + def test_LMRFC(self): + pixml = PI_XML('./LMRFC_RESERVOIR_FORECAST.20190819') + self.assertEqual('LMRFC', pixml.getRFCName()) + self.assertEqual( len( pixml.series ), 180 ) + print( "LMRFC length: ", len( pixml.series ) ) + + def test_LMRFC_toRFCForecast(self): + print( 'test_LMRFC_toRFCForecast:') + pixml = PI_XML('./LMRFC_RESERVOIR_FORECAST.20190819') + rfc_series = pixml.toRFCForecast() + sites = RFC_Sites( './RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv') + lmsites = sites.getSitesByRFC( 'LMRFC' ) + + for lms in lmsites: + print( lms + ':') + for s in rfc_series: + if s.stationID[:5] == lms: + if len(s.timeValueQuality) > 0: + print( ' found: ', s.stationID[:5], "OK", s.getTimePeriod()[0].strftime("%m/%d/%Y %H:%M") + ' - ' + s.getTimePeriod()[1].strftime("%m/%d/%Y %H:%M") ) + else: + print( ' found: ', s.stationID[:5], "missing data" ) +# if len(s.timeValueQuality) > 0: +# for k, v in s.timeValueQuality.items(): +# print(k, v) +# else: +# print( 'not found: ', s.stationID ) + + def test_MBRFC_Hourly(self): + pixml = PI_XML('./20190819182258_RQOT_Forecast_1hr_missings_xml_2019080512') + sites = RFC_Sites( './RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv') + self.assertEqual('MBRFC', pixml.getRFCNameBySitelist(sites)) + self.assertEqual( len( pixml.series ), 1 ) + print( "MBRFC length: ", len( pixml.series ) ) + + def test_MBRFC_Hourly_toRFCForecast(self): + print( 'test_MBRFC_Hourly_toRFCForecast:') + pixml = PI_XML('./20190819182258_RQOT_Forecast_1hr_missings_xml_2019080512') + rfc_series = pixml.toRFCForecast() + + for s in rfc_series: + print( s.stationID ) + if len(s.timeValueQuality) > 0: + for k, v in s.timeValueQuality.items(): + print(' ', k, v) + print( 'missVal = ', s.missVal ) + + def test_MBRFC_6Hourly(self): + pixml = PI_XML('./20190819182258_RQOT_Forecast_6hr_missings_xml_2019080512') + self.assertEqual('MBRFC', pixml.getRFCName()) + self.assertEqual( len( pixml.series ), 1 ) + print( "MBRFC 6 hourly length: ", len( pixml.series ) ) + + def test_MBRFC_6Hourly_toRFCForecast(self): + print( 'test_MBRFC_6Hourly_toRFCForecast:') + pixml = PI_XML('./20190819182258_RQOT_Forecast_6hr_missings_xml_2019080512') + rfc_series = pixml.toRFCForecast() + + for s in rfc_series: + print( s.stationID ) + if len(s.timeValueQuality) > 0: + for k, v in s.timeValueQuality.items(): + print(' 6 hourly: ', k, v) + print( '6 hourly missVal = ', s.missVal ) + + def test_MBRFC(self): + pixml = PI_XML('./MBRFC_RQOT_Forecast.xml') +# self.assertEqual('MBRFC', pixml.getRFCName()) + self.assertEqual( len( pixml.series ), 19 ) + print( "MBRFC length: ", len( pixml.series ) ) + + def test_MBRFC_toRFCForecast(self): + print( 'test_MBRFC_toRFCForecast:') + pixml = PI_XML('./MBRFC_RQOT_Forecast.xml') + rfc_series = pixml.toRFCForecast() + + for s in rfc_series: + print( s.stationID ) + if len(s.timeValueQuality) > 0: + for k, v in s.timeValueQuality.items(): + if abs( v[0] - float( s.missVal ) ) > 0.0001: + print(' : ', k, v) + print( 'missVal = ', s.missVal ) + + def test_MBRFC_Monday_toRFCForecast(self): + print( 'test_MBRFC_Monday_toRFCForecast:') + pixml = PI_XML('./MBRFC_RQOT_Forecast_Monday.xml') + sites = RFC_Sites( './RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv') + lmsites = sites.getSitesByRFC( 'MBRFC' ) + rfc_series = pixml.toRFCForecast() + self.assertEqual( len( pixml.series ), 19 ) + + self.assertAlmostEqual( rfc_series[1].getTimeValueAt( \ + datetime( 2019, 8, 30, 13, 0, 0) )[1][0], \ + 2635.29 ) + for lms in lmsites: + for s in rfc_series: + if s.stationID[:5] == lms: + if len(s.timeValueQuality) > 0: + print( ' found: ', s.stationID[:5], "OK", s.getTimePeriod()[0].strftime("%m/%d/%Y %H:%M") + ' - ' + s.getTimePeriod()[1].strftime("%m/%d/%Y %H:%M") ) + else: + print( ' found: ', s.stationID[:5], "missing data" ) + + for s in rfc_series: + if len(s.timeValueQuality) > 0: + for k, v in s.timeValueQuality.items(): + if abs( v[0] - float( s.missVal ) ) > 0.0001: + print(' : ', k, v) + + def test_NERFC_toRFCForecast(self): + print( 'test_NERFC_toRFCForecast:') + pixml = PI_XML('./20190822_12z_NERFC_Reservoir_Export.xml') + sites = RFC_Sites( './RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv') + lmsites = sites.getSitesByRFC( 'NERFC' ) + rfc_series = pixml.toRFCForecast() + print( len( rfc_series ) ) + self.assertEqual( len( rfc_series ), 18 ) + self.assertAlmostEqual( rfc_series[1].getTimeValueAt( \ + datetime( 2019, 8, 23, 18, 0, 0) )[1][0], \ + 300.0332 ) + for lms in lmsites: + for s in rfc_series: + if s.stationID[:5] == lms: + if len(s.timeValueQuality) > 0: + print( ' found: ', s.stationID[:5], "OK", s.getTimePeriod()[0].strftime("%m/%d/%Y %H:%M") + ' - ' + s.getTimePeriod()[1].strftime("%m/%d/%Y %H:%M") ) + else: + print( ' found: ', s.stationID[:5], "missing data" ) + + for s in rfc_series: + if len(s.timeValueQuality) > 0: + for k, v in s.timeValueQuality.items(): + if abs( v[0] - float( s.missVal ) ) > 0.0001: + print(' : ', k, v) + + def test_NERFC_1_toRFCForecast(self): + print( 'test_NERFC_1_toRFCForecast:') + pixml = PI_XML('./20190820_12z_NERFC_Reservoir_Export.xml') + sites = RFC_Sites( './RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv') + lmsites = sites.getSitesByRFC( 'NERFC' ) + rfc_series = pixml.toRFCForecast() + print( len( rfc_series ) ) +# self.assertEqual( len( rfc_series ), 18 ) +# self.assertAlmostEqual( rfc_series[1].getTimeValueAt( \ +# datetime( 2019, 8, 23, 18, 0, 0) )[1][0], \ +# 300.0332 ) + for lms in lmsites: + for s in rfc_series: + if s.stationID[:5] == lms: + if len(s.timeValueQuality) > 0: + print( ' found: ', s.stationID[:5], "OK", s.getTimePeriod()[0].strftime("%m/%d/%Y %H:%M") + ' - ' + s.getTimePeriod()[1].strftime("%m/%d/%Y %H:%M") ) + else: + print( ' found: ', s.stationID[:5], "missing data" ) + + for s in rfc_series: + if len(s.timeValueQuality) > 0: + for k, v in s.timeValueQuality.items(): + if abs( v[0] - float( s.missVal ) ) > 0.0001: + print(' : ', k, v) + def test_MARFC_toRFCForecast(self): + print( 'test_MARFC_toRFCForecast:') + sites = RFC_Sites( './RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv') + for stn in ["BCHP1BEC", "JCKV2JCK", "DWNN6DEL", "CNNN6DEL" ]: + pixml = PI_XML('./20190824_' + stn + '.xml') + rfc_series = pixml.toRFCForecast() + print( stn + ":" ) + self.assertEqual( len( rfc_series ), 1 ) + self.assertEqual('MARFC', pixml.getRFCNameBySitelist(sites)) + for s in rfc_series: + print(" " + s.stationID[:5], "OK", s.getTimePeriod()[0].strftime("%m/%d/%Y %H:%M") + ' - ' + s.getTimePeriod()[1].strftime("%m/%d/%Y %H:%M") ) + if len(s.timeValueQuality) > 0: + for k, v in s.timeValueQuality.items(): + if abs( v[0] - float( s.missVal ) ) > 0.0001: + print(' : ', k, v) + + +if __name__ == '__main__': + unittest.main() diff --git a/data_assimilation_engine/rfc_ingestion/RFCHelper.py b/data_assimilation_engine/rfc_ingestion/RFCHelper.py new file mode 100644 index 0000000..4bbe26e --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/RFCHelper.py @@ -0,0 +1,150 @@ +############################################################################### +# Module name: RFCHelper +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: 7/31/2019 # +# # +# Last modification date: +# # +# Description: Abstract the RFC forecast to time series +# # +############################################################################### + +import os, logging +from string import * +from datetime import datetime, timedelta +import dateutil.parser +from shutil import copyfile +#import pytz +#import iso8601 +#import Tracer +#from abc import ABCMeta, abstractmethod, abstractproperty +from RFCTimeSeries import RFCTimeSeries + + +class RFCHelper: + "Store all RFC data" + def __init__(self, rfcdata ): + """ + Initialize the RFCHelper object for a given + RFC forecasts + + Input: list of forecast object + """ + self.logger = logging.getLogger(__name__) + self.rfcForecasts = rfcdata + + if not self.rfcForecasts: + raise RuntimeError( "FATAL ERROR: rfcForecasts has no data") + + self.index = -1 + + self.timePeriod = ( self.rfcForecasts[0].fstPeriod[0], \ + self.rfcForecasts[0].fstPeriod[0] ) + for fst in self.rfcForecasts: + #print( "RFCHelper inti:" + fst.stationID, fst.fstPeriod, fst.getT0() ) + if self.timePeriod[0] > fst.fstPeriod[ 0 ]: + self.timePeriod = ( fst.fstPeriod[ 0 ], \ + self.timePeriod[1] ) + + if self.timePeriod[1] < fst.fstPeriod[ 0 ]: + self.timePeriod = ( self.timePeriod[ 0 ],\ + fst.fstPeriod[ 0 ] ) + + def __iter__(self ): + """ + The iterator + """ + return self + + def __next__( self ): + """ + The next RFC_Forecast object + """ + if self.index == len( self.rfcForecasts ) - 1: + self.index = -1 + raise StopIteration + self.index = self.index + 1 + return self.rfcForecasts[ self.index ] + + def timePeriodForAll( self ): + """ + Get the earlist and latest time for all observations + + Return: Tuple of start time and end time + """ + return self.timePeriod + + def makeTimeSeries( self ): + """ + Create time series from forecast data + + Return: list of time series objects + """ + timeseries = [] + for fst in self: + if not fst.isEmpty(): + timestamp = fst.getT0() + synthetics = list(map(int, fst.getSyntheticValues() ) ) + timeSeries = RFCTimeSeries( fst.get5CharStationID(), \ + [ timestamp ], \ + timedelta( seconds = fst.getTimeStepInSeconds()), \ + fst.getStartTime(), \ + [ fst.getValuesInCMS() ], \ + [ fst.getQuality() ], \ + [ synthetics ], \ + [ fst.getTimeStepInSeconds() ],\ + [ fst.getQueryTime() ], \ + fst.missVal) + timeseries.append(timeSeries) + + return timeseries + + def makeAllTimeSeries( self, outdir, suffix='RFCTimeSeries.ncdf' ): + """ + Create the time series NetCDF files for all USGS observations + + Input: outdir - the output directory + suffix - suffix of the filename + + Return: Total number of time series files created + """ + + # the time resultions must divide 60 minutes with no remainder + series = self.makeTimeSeries() + count = 0 + if series: + for s in series: +# s.print_station_offsettime_value() + self.logger.info( "Making time series: " + \ + outdir + '/' + s.getSeriesNCFileName( suffix ) ) + seriesfilename = outdir + '/' +s.getSeriesNCFileName( suffix ) + if os.path.isfile( seriesfilename ): + oldseries = RFCTimeSeries.fromNetCDF( seriesfilename ) + if s.compare( oldseries ): + self.logger.info( s.getSeriesNCFileName( suffix )+ \ + " has no change!" ) + else: + self.logger.info( s.getSeriesNCFileName( suffix ) + \ + " updated!" ) +# copyfile( seriesfilename, seriesfilename + '.bak' ) + ismerged = s.mergeOld( oldseries ) + if not ismerged: + self.logger.info( "RFCHelper :" + \ + s.getSeriesNCFileName( suffix )+ " is not merged" ) + + s.toNetCDF( outdir, suffix ) + count = count + 1 + else: +# s.print_station_offsettime_value() + s.toNetCDF( outdir, suffix ) + self.logger.info( s.getSeriesNCFileName( suffix )+\ + " created!" ) + count = count + 1 + + else: + self.logger.info( "Skip making time series for " + \ + startTime.isoformat() + ' - no data' ) + + return count diff --git a/data_assimilation_engine/rfc_ingestion/RFCTimeSeries.py b/data_assimilation_engine/rfc_ingestion/RFCTimeSeries.py new file mode 100644 index 0000000..7726200 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/RFCTimeSeries.py @@ -0,0 +1,495 @@ +############################################################################### +# Module name: RFCTimeSeries # +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: # +# # +# Last modification date: 7/10/2019 # +# # +# Description: manage a RFC time series file that contains forecast stream # +# flow data a given time stamp # +# # +############################################################################### +import os, sys, time, math +from string import * +from datetime import datetime, timedelta +import calendar +#import xml.utils.iso8601 +#from netCDF4 import Dataset +import netCDF4 +import numpy as np +import random + +class RFCTimeSeries: + """ + Description: Store one RFC time series data + Author: Zhengtao Cui (Zhengtao.Cui@noaa.gov) + Date: July. 10, 2019 + Date modified: + """ + + stationIdStrLen = 5 + stationIdLong_name = "RFC station identifer of length 5" + timeStrLen = 19 + timeUnit = "UTC" + timeLong_name = "YYYY-MM-DD_HH:mm:ss UTC" + qualifierStrLen = 15 + dischargeUnit = "m^3/s" + dischargeLong_name = "Obsverved and forecasted discharges. There are 48 hours of observed values before issue time (T0), that is the start time is T0 - 48 hours. Values after T0 (including T0) are forecasts that genenrally extend to 10 days, that is T0 + 240 hours. The total length of dischargs is 12 days in general except in cases where forecasts or observations that are longer than 10 days or 2 days respectively are available." + syntheticsUnit = "-" + syntheticsLong_name = "Whether the discharge value is synthetic or orginal, 1 - synthetic, 0 - original" + dischargeQualityUnit = "-" + dischargeQualaityLong_name = \ + "Discharge quality 0 to 100 to be scaled by 100." + +# offsetTimeUnit = "seconds since T0" +# offsetTimeLong_name = "Offset time from T0" + + totalCountsUnit = "-" + totalCountsLong_name = "Total count of all observation and forecast values" + + observedCountsUnit = "-" + observedCountsLong_name = "Total observed values before T0." + + forecastCountsUnit = "-" + forecastCountsLong_name = "Total forecasted values including and after T0." + + timeStepsUnit = "seconds" + timeStepsLong_name = "Frequency/temporal resolution of forecast values" + + queryTimeUnit = "seconds since 1970-01-01 00:00:00 local TZ" + queryTimeLong_name = "Query time at RFC" + + @property + def issueTimeStamps(self): + return self._issueTimeStamps + @issueTimeStamps.setter + def issueTimeStamps(self, its): + self._issueTimeStamps = its + + @property + def startTimeStamp(self): + return self._startTimeStamp + @startTimeStamp.setter + def startTimeStamp(self, sts): + self._startTimeStamp = sts + + def __init__(self, staID, time_stamp, resolution, starttime, \ + dis, quality, synthetics, timestep, querytime, missVal ): + """ + Initialize a RFC TimeSeries object + Input: time_stamp - a time stamp + resolution - time resolution of the time series + station_time_value - Tuple of (station, time, flow, + quality) + """ + self.issueTimeStamps = time_stamp + self.seriesTimeResolution = resolution + self.stationID = staID + self.startTimeStamp = starttime + self.discharges = dis + self.discharge_qualities = quality + self.synthetics = synthetics + self.timeSteps = timestep + self.querytime = querytime + self.missingValue = missVal + + def isEmpty( self ): + """ + Test if the time series is empty + Return: boolean + """ + return not self.discharges + + def print_station_offsettime_value( self ): + """ + Print the time series data + """ + print( "Series: start time: " + \ + self.startTimeStamp.isoformat() ) + print( self.issueTimeStamps ) + print( self.discharges ) + print( self.stationID ) + print( self.querytime ) + for issuetime, dis in \ + zip( self.issueTimeStamps,\ + self.discharges): + print( " issue time: " + issuetime.isoformat() + \ + " " + self.stationID ) + for v in dis: + print( " ", v ) + + def get_total_counts(self): + counts = [] + for dis in self.discharges: + counts.append( len( dis ) ) + + return counts + + def get_observed_counts(self): + counts = [] + for t, s in zip( self.issueTimeStamps, self.timeSteps): + counts.append( \ + ( t - self.startTimeStamp ).total_seconds() / s ) + + return counts + + def get_forecast_counts(self): + counts = [] + obvcounts = self.get_observed_counts() + for oc, dis in zip(obvcounts, self.discharges): + counts.append( len( dis ) - oc ) + + return counts + def getStationID( self ): + """ + Get all station ids of the time series + Return: list of station ids + """ + station = list( self.stationID ) + if len(station) < self.stationIdStrLen: + for i in range( len(station), self.stationIdStrLen ): + station.insert(0, ' ' ) + elif len(station) > self.stationIdStrLen: + station = station[0:self.stationIdStrLen - 1] + + return station + + def getDischargeValues( self ): + """ + Get all stream flow values of the time series + Return: list of flow values + """ + return self.discharges + +# def getOffsetTimeInSeconds( self ): +# """ +# Get all observation times of the time series +# Return: list of observation times +# """ +# return int( self.offsettime.total_seconds() ) + + def getSeriesNCFileName( self, suffix='RFCTimeSeries.ncdf' ): + """ + Get NetCDF file for this time series + Return: A NetCDF filename + """ + filename = \ + self.issueTimeStamps[-1].strftime( "%Y-%m-%d_%H." ) + \ + str( int( self.seriesTimeResolution.days * 24 * 60 + \ + self.seriesTimeResolution.seconds // 60 ) ).zfill(2) + \ + "min." + self.stationID + "." + suffix + return filename + + def getDischargeQualities( self ): + """ + Get discharge quality for this time series + Return: A list of discharge quality + """ + return self.discharge_qualities + + + def toNetCDF( self, outputdir = './', suffix='RFCTimeSeries.ncdf' ): + """ + Write the time series to a NetCDF file + Input: outputdir - the directory where to write the NetCDF + """ + nc_fid = netCDF4.Dataset( \ + outputdir + '/' + self.getSeriesNCFileName( suffix ), \ + 'w', format='NETCDF4' ) + nc_fid.createDimension( 'stationIdStrLen', self.stationIdStrLen ) + nc_fid.createDimension( 'timeStrLen', self.timeStrLen ) + nc_fid.createDimension( 'forecastInd', None ) + nc_fid.createDimension( 'nseries', len( self.discharges ) ) + nc_fid.createDimension( 'zero', 0 ) + + + stationId = nc_fid.createVariable( 'stationId', 'S1',\ + ('stationIdStrLen',) ) + + stationId.setncatts( {'long_name' : \ + self.stationIdLong_name, \ + 'units' : '-'} ) + + issuetime = nc_fid.createVariable( 'issueTimeUTC', 'S1', \ + ('nseries', 'timeStrLen', ) ) + + issuetime.setncatts( {'long_name' : self.timeLong_name, \ + 'unts' : self.timeUnit } ) + + discharges = nc_fid.createVariable( 'discharges', 'f4',\ + ('nseries', 'forecastInd', ) ) + + discharges.setncatts( {'long_name' : \ + self.dischargeLong_name, \ + 'units' : self.dischargeUnit} ) + +# offsettime = nc_fid.createVariable( 'offsetTime', 'i4', ('zero')) +# offsettime.setncatts( {'long_name' : self.offsetTimeLong_name, \ +# 'units' : self.offsetTimeUnit} ) + synthetics = nc_fid.createVariable( 'synthetic_values', 'i1',\ + ('nseries', 'forecastInd', ) ) + + synthetics.setncatts( {'long_name' : \ + self.syntheticsLong_name, \ + 'units' : self.syntheticsUnit} ) + + totalcounts = nc_fid.createVariable( 'totalCounts', 'i2',\ + ('nseries',)) + + totalcounts.setncatts( {'long_name' : \ + self.totalCountsLong_name, \ + 'units' : self.totalCountsUnit} ) + + obvcounts = nc_fid.createVariable( 'observedCounts', 'i2',\ + ('nseries',)) + + obvcounts.setncatts( {'long_name' : \ + self.observedCountsLong_name, \ + 'units' : self.observedCountsUnit} ) + + fstcounts = nc_fid.createVariable( 'forecastCounts', 'i2',\ + ('nseries',)) + + fstcounts.setncatts( {'long_name' : \ + self.forecastCountsLong_name, \ + 'units' : self.forecastCountsUnit} ) + + timeSteps = nc_fid.createVariable( 'timeSteps', 'i4',\ + ('nseries',)) + + timeSteps.setncatts( {'long_name' : \ + self.timeStepsLong_name, \ + 'units' : self.timeStepsUnit} ) + + discharge_qualities = \ + nc_fid.createVariable( 'discharge_qualities', 'i2',\ + ('nseries',)) + + discharge_qualities.setncatts( {'long_name' : \ + self.dischargeQualaityLong_name, \ + 'units' : self.dischargeQualityUnit, \ + 'multfactor' : '0.01' } ) + + queryTime = nc_fid.createVariable( 'queryTime', 'i8',\ + ('nseries',)) + + queryTime.setncatts( { 'units' : \ + 'seconds since 1970-01-01 00:00:00 local TZ' \ + } ) + + nc_fid.setncatts( { 'fileUpdateTimeUTC': \ + datetime.utcnow().strftime( "%Y-%m-%d_%H:%M:00" ), \ + 'sliceStartTimeUTC' : \ + self.startTimeStamp.strftime( "%Y-%m-%d_%H:%M:00" ),\ + 'sliceTimeResolutionMinutes' : \ + str( int( self.seriesTimeResolution.days * 24 * 60 + \ + self.seriesTimeResolution.seconds // 60 ) ).zfill(2), \ + 'missingValue' : self.missingValue, \ + 'newest_forecast' : str( len( self.discharges ) - 1 ), \ + 'NWM_version_number' : 'v2.1' } ) + + discharges[:,:] = np.asarray( self.discharges ) + synthetics[:,:] = np.asarray( self.synthetics ) +# issuetime = netCDF4.stringtochar( [ ''.join('2019-08-18_00:00:00' ) ] ) +# chars = '1234567890aabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' + timestrings = [] + timestrings = np.empty((len(self.issueTimeStamps),),\ + 'S'+repr(self.timeStrLen)) + for n in range(len(self.issueTimeStamps)): + timestrings[ n ] = \ + self.issueTimeStamps[n].strftime( "%Y-%m-%d_%H:%M:00" ) + issuetime[:,:] = netCDF4.stringtochar( timestrings ) +# issuetime = netCDF4.stringtochar( \ +# [ ''.join( t.strftime( "%Y-%m-%d_%H:%M:00" ) \ +# for t in self.issueTimeStamps ] ) ] ) + + queryTime[:] = self.querytime + totalcounts[:] = self.get_total_counts() + obvcounts[:] = self.get_observed_counts() + fstcounts[:] = self.get_forecast_counts() + +# offsettime[:] = self.getOffsetTimeInSeconds() + timeSteps[:] = self.timeSteps + + stationId[:] = self.getStationID() + + discharge_qualities[:] = self.discharge_qualities + + nc_fid.close() + + @classmethod + def fromNetCDF( self, ncfilename ): + """ + Read the time series from a NetCDF file + Input: ncfilename - the NetCDF filename + """ + nc_fid = netCDF4.Dataset( ncfilename, 'r' ) + startTimeStamp = datetime.strptime( \ + nc_fid.getncattr( 'sliceStartTimeUTC' ), \ + "%Y-%m-%d_%H:%M:00" ) + + time_resol = timedelta( minutes = \ + int( nc_fid.getncattr( 'sliceTimeResolutionMinutes' ) ) ) + + missVal = nc_fid.getncattr( 'missingValue' ) + + newest_series = nc_fid.getncattr( 'newest_forecast' ) + + station = netCDF4.chartostring( \ + nc_fid.variables[ 'stationId'][ : ] ) + + discharges = nc_fid.variables[ 'discharges'][ :, : ].tolist() + synthetics = nc_fid.variables[ 'synthetic_values'][ :, : ].tolist() + + #queryTime = int( nc_fid.variables[ 'queryTime'][0] ) + queryTime = [ int( x ) for x in \ + nc_fid.variables[ 'queryTime'][:] ] +# queryTime = nc_fid.variables[ 'queryTime'][:].tolist() + + qualities = [ int( x ) for x in \ + nc_fid.variables[ 'discharge_qualities'][:] ] + + timeSteps = [ int( x ) for x in \ + nc_fid.variables[ 'timeSteps' ][:] ] + # offsettime = nc_fid.variables[ 'offsetTime' ][0] + # print( "offsetitme:", int( offsettime)) + + issueTimes = [ datetime.strptime( t, "%Y-%m-%d_%H:%M:00") \ + for t in netCDF4.chartostring ( \ + nc_fid.variables[ 'issueTimeUTC' ][:] ) ] + + nc_fid.close() + +# d = timedelta( seconds = int( offsettime ) ) +# print(d) +# + return self( str( station ), issueTimes, time_resol, \ + startTimeStamp ,\ + discharges,\ + qualities, synthetics, timeSteps, queryTime, missVal ) + + def mergeOld( self, oldTimeSeries ): + """ + Merge data in an existing NetCDF time series file with this one + Input: oldTimeSeries - the NetCDF filename of the existing time + series + """ + + return False + if self.startTimeStamp != oldTimeSeries.startTimeStamp or \ + self.seriesTimeResolution != oldTimeSeries.seriesTimeResolution \ + or self.stationID != oldTimeSeries.stationID or \ + self.missingValue != oldTimeSeries.missingValue: + #raise RuntimeError( "FATAL ERROR: the two time series "\ + # " have different time stamps, temporal resolutions, "\ + # "or station IDs, not merging ..." ) + print ( "The two time series "\ + " have different start time, temporal resolutions, "\ + "or station IDs, not merging ..." ) + return False + else: + extraIssueTimes = [] + for n in range( len(self.issueTimeStamps ) ): + for m in range( len( oldTimeSeries.issueTimeStamps ) ): + if self.issueTimeStamps[n] == \ + oldTimeSeries.issueTimeStamps[ m ]: + self.discharges[ n ] = oldTimeSeries.discharges[ m ] + self.synthetics[ n ] = oldTimeSeries.synthetics[ m ] + break + + for m in range( len( oldTimeSeries.issueTimeStamps ) ): + found = False + for n in range( len(self.issueTimeStamps ) ): + if self.issueTimeStamps[n] == \ + oldTimeSeries.issueTimeStamps[ m ]: + found = True + break + if not found: + extraIssueTimes.append( m ) + + if extraIssueTimes: + for i in extraIssueTimes: + + if len( self.discharges[ -1 ] ) == \ + len( oldTimeSeries.discharges[ i ] ): + pass + elif len( self.discharges[ -1 ] ) < \ + len( oldTimeSeries.discharges[ i ] ): + for d in self.discharges: + for j in range( len(self.discharges[ -1 ] ),\ + len( oldTimeSeries.discharges[ i ] ) ): + d.append( self.missingValue ) + for s in self.synthetics: + for j in range( len(self.synthetics[ -1 ] ),\ + len( oldTimeSeries.synthetics[ i ] ) ): + s.append( 1 ) + + else: + for j in range( \ + len(oldTimeSeries.discharges[ i ] ),\ + len( self.discharges[ -1 ] ) ): + oldTimeSeries.discharges[ i ].append( \ + self.missingValue ) + oldTimeSeries.synthetics[ i ].append( 1 ) + + self.discharges.append( oldTimeSeries.discharges[ i ] ) + self.synthetics.append( oldTimeSeries.synthetics[ i ] ) + self.issueTimeStamps.append( \ + oldTimeSeries.issueTimeStamps[ i ] ) + self.timeSteps.append( oldTimeSeries.timeSteps[ i ] ) + self.discharge_qualities.append( \ + oldTimeSeries.discharge_qualities[ i ] ) + self.querytime.append( oldTimeSeries.querytime[ i ] ) + return True + + + def compare( self, otherTimeSeries ): + + if self.stationID != otherTimeSeries.stationID: + return False + + if self.seriesTimeResolution != \ + otherTimeSeries.seriesTimeResolution: + return False + + if self.startTimeStamp != otherTimeSeries.startTimeStamp: + return False + + if len(self.discharges) != len( otherTimeSeries.discharges): + return False + + if len(self.synthetics) != len( otherTimeSeries.synthetics): + return False + + if self.issueTimeStamps != otherTimeSeries.issueTimeStamps: + return False + + if self.timeSteps != otherTimeSeries.timeSteps: + return False + + if self.missingValue != otherTimeSeries.missingValue: + return False + + if self.discharge_qualities != \ + otherTimeSeries.discharge_qualities: + return False + + for x, y in zip( self.discharges, otherTimeSeries.discharges): + if len(x) != len(y): + return False + for a, b in zip( x, y): + if not math.isclose( a, b, abs_tol=0.001 ): +# print("value diff", a, b) + return False + + for x, y in zip( self.synthetics, otherTimeSeries.synthetics): + if len(x) != len(y): + return False + for a, b in zip( x, y): + if a != b: +# print("synthetics diff", a, b) + return False + + return True diff --git a/data_assimilation_engine/rfc_ingestion/RFCTimeSeries_test.py b/data_assimilation_engine/rfc_ingestion/RFCTimeSeries_test.py new file mode 100644 index 0000000..10ac42d --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/RFCTimeSeries_test.py @@ -0,0 +1,166 @@ +import unittest +import time, os +import numpy as np +from datetime import datetime, timedelta +from RFCTimeSeries import RFCTimeSeries +from PI_XML import PI_XML +from RFC_Forecast import RFC_Forecast + +class RFCTimeSeries_test(unittest.TestCase): + def test(self): + self.assertTrue(True) + + def testInit(self): + timestamp = datetime(2019, 5, 19, 12, 0, 0) + starttime = datetime(2019, 5, 19, 12, 0, 0) + timeresolution = timedelta( minutes = 180 ) + stations = [ "CHNK1", "CNGK1" ] + discharges = [ [ 1218.9941, 1218.9941, 1217.003, 1228.0027, \ + 1221.9971, 1221.9971 ], \ + [ 15936.987, 16065.989, 16035.982, 15959.986, \ + 15950.99, 15937.987, 15858.99, \ + 1218.9941, 1217.003, 1228.0027, \ + 1221.9971, 1221.9971 ] ] + quality = [ 100, 50 ] + + timesteps = [ 21600, 21600 ] + + querytime = [ int( time.mktime( \ + datetime( 2019, 5, 26, 7, 12, 45).timetuple()) ), \ + int( time.mktime( \ + datetime( 2019, 5, 26, 7, 12, 45).timetuple()) ) ] + + for ( sta, dis, qual, dt, queryt) in \ + zip( stations, discharges, quality, timesteps,\ + querytime ): + + print( sta, qual, dt, queryt) + timeseries = RFCTimeSeries( sta, [ timestamp ], timeresolution, \ + starttime, \ + [ dis ], [ qual ], [ dt ], [ queryt ], '-999.0' ) + + timeseries.toNetCDF() + + def test_fromNetCDF(self): + + timeseries = RFCTimeSeries.fromNetCDF( "./2019-05-19_12.180min.CHNK1.RFCTimeSeries.ncdf" ) + print( "from Netcdf:" ) + timeseries.print_station_offsettime_value() + self.assertEqual( timeseries.startTimeStamp, \ + datetime(2019, 5, 19, 12, 0, 0) ) + + self.assertAlmostEqual( \ + timeseries.getDischargeValues()[0][3], \ + 1228.003, 3 ) + + def test_compare_equal(self): + timeseries = RFCTimeSeries.fromNetCDF( "./2019-05-19_12.180min.CHNK1.RFCTimeSeries.ncdf" ) + timeseries1 = RFCTimeSeries.fromNetCDF( "./2019-05-19_12.180min.CHNK1.RFCTimeSeries.ncdf.bak" ) + + self.assertTrue( timeseries.compare( timeseries1 ) ) + + def test_compare_nonequal(self): + timeseries = RFCTimeSeries.fromNetCDF( "./2019-05-19_12.180min.CHNK1.RFCTimeSeries.ncdf" ) + + timestamp = datetime(2019, 5, 19, 12, 0, 0) + starttime = datetime(2019, 5, 19, 12, 0, 0) + timeresolution = timedelta( minutes = 180 ) + station = "CHNK1" + discharges = [ [ 1218.9941, 1218.9941, 1217.003, 1228.0027, \ + 1221.9971, 1221.9981 ] ] + quality = [ 100] + + timesteps = [ 21600 ] + + querytime = [ int( time.mktime( \ + datetime( 2019, 5, 26, 7, 12, 45).timetuple()) ) ] + + timeseries1 = RFCTimeSeries( station, [ timestamp ], timeresolution, \ + starttime, discharges, quality, \ + timesteps, querytime, '-999.0' ) + + + self.assertFalse( timeseries.compare( timeseries1 ) ) + + def test_mergeOld(self): + + timestamp = datetime(2019, 5, 19, 12, 0, 0) + starttime = datetime(2019, 5, 19, 12, 0, 0) + timeresolution = timedelta( minutes = 180 ) + station = "CHNK1" + newdischarges = [ [ 1218.9941, 1218.9941, 1217.003, 1228.0027, \ + 1221.9971, 1221.9981 ] ] + quality = [ 100] + + timesteps = [ 21600 ] + + querytime = [ int( time.mktime( \ + datetime( 2019, 5, 26, 7, 12, 45).timetuple()) ) ] + + tsnew = RFCTimeSeries( station, [ timestamp ], timeresolution, \ + starttime, newdischarges, quality, \ + timesteps, querytime, '-999.0' ) + + olddischarges = [ [ 1218.9941, 1218.9941, 1217.003, 1228.0027, \ + 1223.9971, 1221.9981 ] ] + + tsold = RFCTimeSeries( station, [ timestamp ], timeresolution, \ + starttime, olddischarges, quality, \ + timesteps, querytime, '-999.0' ) + + tsnew.mergeOld( tsold ) + + tsnew.toNetCDF( suffix='RFCTimeSeries.ncdf.merged' ) + + self.assertTrue( os.path.exists(\ + './2019-05-19_12.180min.CHNK1.RFCTimeSeries.ncdf.merged' ) ) + +# def test_NERFC_to_Timeseries(self): +# pixml = PI_XML('./NERFC_Reservoir_Export.xml') +# rfc_series = pixml.toRFCForecast() +# +# timeresolution = timedelta( minutes = 180 ) +# +# for s in rfc_series: +# print( "ParameterID = ", s.parameterId ) +# print( "stationID = ", s.stationID ) +# print( "size = ", len(s.timeValueQuality)) +# if s.parameterId == 'RQOT': +# print( "qualifierID = ", s.qualifierId ) +# print( "timeStep = ", s.timeStep ) +# print( "forecastDate = ", s.forecastDate ) +# print( "unit = ", s.unit ) +# print( "period = ", s.fstPeriod ) +# if s.creationDateTime is not None: +# querytime = time.mktime( s.creationDateTime.timetuple() ) +# elif s.forecastDate is not None: +# querytime = time.mktime( s.forecastDate.timetuple() ) +# else: +# querytime = time.mktime( datetime.now().timetuple() ) +# +# values = [] +# +# for k, v in s.timeValueQuality.items(): +# print("{0} ===> {1}".format( k, v ) ) +# if s.unit == 'CFS': +# values.append( v[0] * 0.028316846592 ) +# else: +# values.append( v[0] ) +# +# timeseries = RFCTimeSeries( s.get5CharStationID(),\ +# s.getT0(), \ +# timeresolution, \ +# timedelta( seconds = 0 ),\ +# values,\ +# 100,\ +# s.getTimeStepInSeconds(),\ +# querytime, '-999.0') +# +# +# timeseries.toNetCDF() + + + +if __name__ == '__main__': + unittest.main() + diff --git a/data_assimilation_engine/rfc_ingestion/RFC_Forecast.py b/data_assimilation_engine/rfc_ingestion/RFC_Forecast.py new file mode 100644 index 0000000..b7798be --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/RFC_Forecast.py @@ -0,0 +1,775 @@ +############################################################################### +# Module name: RFC_Intestion +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: # +# # +# Last modification date: 08/06/2019 # +# # +# Description: manage data in a RFC FEWS PI xml file # +# # +############################################################################### + +import os, sys, time, csv, re +sys.path.append( os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))\ + + '/usgs_download/analysis/') +import logging +import math +from string import * +from collections import OrderedDict +from datetime import datetime, timedelta +import dateutil.parser +import pytz +#import iso8601 +import xml.etree.ElementTree as etree +#import Tracer +from Observation import Observation +from RFC_Sites import RFC_Sites + +class RFC_Forecast(Observation): + """ + Store one RFC forecast data + """ + @property + def type(self): + return self._type + + @type.setter + def type(self, t): + self._type = t + + @property + def parameterId(self): + return self._parameterId + + @parameterId.setter + def parameterId(self, p): + self._parameterId = p + + @property + def qualifierId(self): + return self._qualifierId + + @qualifierId.setter + def qualifierId(self, q): + self._qualifierId = q + + @property + def timeStep(self): + return self._timeStep + + @timeStep.setter + def timeStep(self, t): + self._timeStep = t + + @property + def forecastDate(self): + return self._forecastDate + + @forecastDate.setter + def forecastDate(self,f): + self._forecastDate = f + + @property + def creationDateTime(self): + return self._creationDateTime + + @creationDateTime.setter + def creationDateTime(self, c): + self._creationDateTime = c + + @property + def missVal(self): + return self._missVal + + @missVal.setter + def missVal(self, m): + self._missVal = m + + @property + def lat(self): + return self._lat + + @lat.setter + def lat(self, l): + self._lat = l + + @property + def lon(self): + return self._lon + + @lon.setter + def lon(self, l): + self._lon = l + + @property + def x(self): + return self._x + + @x.setter + def x(self, l): + self._x = l + + @property + def y(self): + return self._y + + @y.setter + def y(self, l): + self._y = l + + @property + def z(self): + return self._z + + @z.setter + def z(self, l): + self._z = l + + @property + def timeValueQuality(self): + return self._timeValueQuality + + @timeValueQuality.setter + def timeValueQuality(self, v): + self._timeValueQuality = v + + @property + def fstPeriod(self): + return self._obvPeriod + + @fstPeriod.setter + def fstPeriod(self, p): + self._obvPeriod = p + + @property + def rfcname(self): + return self._rfcname + + @rfcname.setter + def rfcname(self, r): + self._rfcname = r + + def __init__(self, pixmlseries, stations ): + """ + Initialize the RFC_Forecast object with a given + pixml series + """ + self.source = pixmlseries + header = pixmlseries.find('{http://www.wldelft.nl/fews/PI}header') + self.type = header.find('{http://www.wldelft.nl/fews/PI}type').text + self.stationID = header.find('{http://www.wldelft.nl/fews/PI}locationId').text + self.stationName = header.find('{http://www.wldelft.nl/fews/PI}stationName').text + self.parameterId = header.find('{http://www.wldelft.nl/fews/PI}parameterId').text + if header.find('{http://www.wldelft.nl/fews/PI}qualifierId') \ + is not None: + + self.qualifierId = header.find('{http://www.wldelft.nl/fews/PI}qualifierId').text + else: + self.qualifierId = None + + timeStepUnit = \ + header.find('{http://www.wldelft.nl/fews/PI}timeStep').\ + attrib[ 'unit' ] + timeStepMultiplier = \ + header.find('{http://www.wldelft.nl/fews/PI}timeStep').\ + attrib[ 'multiplier' ] + self.timeStep = (timeStepUnit, timeStepMultiplier) + + if header.find('{http://www.wldelft.nl/fews/PI}startDate') \ + is not None: + self.startDate = datetime.strptime( \ + header.find('{http://www.wldelft.nl/fews/PI}startDate').\ + attrib['date'] + ' ' + \ + header.find('{http://www.wldelft.nl/fews/PI}startDate').\ + attrib['time'], '%Y-%m-%d %H:%M:%S') + else: + self.startDate = None + + if header.find('{http://www.wldelft.nl/fews/PI}forecastDate') \ + is not None: + self.forecastDate = datetime.strptime( \ + header.find('{http://www.wldelft.nl/fews/PI}forecastDate').\ + attrib['date'] + ' ' + \ + header.find('{http://www.wldelft.nl/fews/PI}forecastDate').\ + attrib['time'], '%Y-%m-%d %H:%M:%S') + else: + self.forecastDate = None +# if self.startDate is not None: +# self.forecastDate = self.startDate +# else: +# self.forecastDate = None + + if header.find('{http://www.wldelft.nl/fews/PI}creationDate') \ + is not None: + self.creationDateTime = datetime.strptime( \ + header.find('{http://www.wldelft.nl/fews/PI}creationDate').\ + text + ' ' + \ + header.find('{http://www.wldelft.nl/fews/PI}creationTime').\ + text, '%Y-%m-%d %H:%M:%S') + else: + self.creationDateTime = None + + self.missVal = \ + header.find('{http://www.wldelft.nl/fews/PI}missVal').text + + if math.isnan( float( self.missVal ) ): + self.missVal = '-999' + + if header.find('{http://www.wldelft.nl/fews/PI}lat') is not None: + self.lat = float( \ + header.find('{http://www.wldelft.nl/fews/PI}lat').text ) + if header.find('{http://www.wldelft.nl/fews/PI}lon') is not None: + self.lon = float( \ + header.find('{http://www.wldelft.nl/fews/PI}lon').text ) + + if header.find('{http://www.wldelft.nl/fews/PI}x') is not None: + self.x = float( \ + header.find('{http://www.wldelft.nl/fews/PI}x').text ) + + if header.find('{http://www.wldelft.nl/fews/PI}y') is not None: + self.y = float( \ + header.find('{http://www.wldelft.nl/fews/PI}y').text ) + + if header.find('{http://www.wldelft.nl/fews/PI}z') is not None: + self.z = float( \ + header.find('{http://www.wldelft.nl/fews/PI}z').text ) + + self.unit = \ + header.find('{http://www.wldelft.nl/fews/PI}units').text + + self.timeValueQuality = OrderedDict() + + for event in pixmlseries.iter( \ + '{http://www.wldelft.nl/fews/PI}event'): + value = float( event.attrib[ 'value' ] ) + if math.isnan( value ): + value = -999.0 + + # set negative values to missing values + if not math.isclose( value, float( self.missVal ),\ + abs_tol=0.0001 ) and value < 0: + value = float( self.missVal ) + + #NERFC +# if self.stationID == "RKWM1ME": +# value = -999 + + self.timeValueQuality[ datetime.strptime( \ + event.attrib['date'] + ' ' + event.attrib['time'], \ + '%Y-%m-%d %H:%M:%S' ) ] = ( \ + value, \ + float( event.attrib[ 'flag' ] ), False ) + + if len( self.timeValueQuality ) > 0: + self.fstPeriod = list( self.timeValueQuality.keys() )[0], \ + list( self.timeValueQuality.keys() )[-1] + if self.startDate != self.fstPeriod[0]: + self.startDate = self.fstPeriod[0] + + self.rfcname = stations.getRFCBySite( self.get5CharStationID() ) +# # +# #Set time step for WGRFC +# # +# if self.rfcname == "WGRFC": +# self.timeStep = (timeStepUnit, "3600") + +# #NERFC +# if self.stationID == "SWRN6" or self.stationID == "SACN6": +# if self.isForecast(): +# self.timeStep = (timeStepUnit, "86400") +# if self.stationID == "SWRN6": +# if self.isForecast(): +# self.startDate = self.fstPeriod[0] +# self.forecastDate = self.startDate - timedelta( days = 1 ) + # + # CNRFC assumes T0 = startDate + 48 hours + # + if self.rfcname == "CNRFC": + self.forecastDate = self.startDate + timedelta( hours = 48 ) + + if self.rfcname == "NCRFC" and self.forecastDate is not None: + if self.startDate - self.forecastDate > timedelta( hours = 12 ): + self.forecastDate = self.startDate + + def getTimeStepInSeconds(self): + if self.timeStep[0] == "second": + return int( self.timeStep[1] ) + elif self.timeStep[0] == "minute": + return int( self.timeStep[1] ) * 60 + elif self.timeStep[0] == "hour": + return int( self.timeStep[1] ) * 3600 + else: + return None + + + def setTimeStepInSeconds(self, dtInSec): + self.timeStep = ( "second", f"{dtInSec}") + + def getTimePeriod(self): + return self.fstPeriod + + def resetTimePeriod( self ): + if self.isEmpty(): + self.fstPeriod = None + self.startDate = None + return + + self.fstPeriod = list( self.timeValueQuality.keys() )[0], \ + list( self.timeValueQuality.keys() )[-1] + self.startDate = self.fstPeriod[0] + + + def getStartTime( self ): + return self.fstPeriod[0] + + def getT0(self): + if self.isForecast(): + # + # MBRFC has synoptic time, round down to 0, 6, 12 and 18 + # +# if self.rfcname == "MBRFC": +# hr = self.forecastDate.hour // 6 * 6 +# return self.forecastDate.replace(hour=hr) +# else: + return self.forecastDate + elif self.isObserved(): + return None + elif self.startDate is not None: + return self.startDate + elif not isEmpty(): + return self.fstPeriod[0] + else: + return None + + def getSyntheticValues( self ): + syntheticValues = [] + for k, v in self.timeValueQuality.items(): + syntheticValues.append( v[ 2 ] ) + return syntheticValues + + def getStationTimeValueInCMSQuality(self): + parameterset = set( ["RQOT", "QINE", "SQIN" ] ) + if self.parameterId in parameterset: + values = [] + for k, v in self.timeValueQuality.items(): + if self.unit == 'CFS': + values.append( v[0] * 0.028316846592 if \ + abs( v[0] - float( self.missVal ) ) > 0.0001 else \ + v[0] ) + elif self.unit == 'KCFS': + values.append( v[0] * 28.316846592 if \ + abs( v[0] - float( self.missVal ) ) > 0.0001 else \ + v[0] ) + elif self.unit == 'CMS': + values.append( v[0] ) + else: + raise RuntimeError( 'FATAL Error: RFC_Forecast: ' + \ + ' unknown unit: ' + self.unit ) + + offset = self.getOffsetTime() + return (self.stationID, offset, values, 100) + else: + return None + + def getValuesInCMS(self): + parameterset = set( ["RQOT", "QINE", "SQIN" ] ) + values = [] + if self.parameterId in parameterset: + for k, v in self.timeValueQuality.items(): + if self.unit == 'CFS': + values.append( v[0] * 0.028316846592 if \ + abs( v[0] - float( self.missVal ) ) > 0.0001 else \ + v[0] ) + elif self.unit == 'KCFS': + values.append( v[0] * 28.316846592 if \ + abs( v[0] - float( self.missVal ) ) > 0.0001 else \ + v[0] ) + elif self.unit == 'CMS': + values.append( v[0] ) + else: + raise RuntimeError( 'FATAL Error: RFC_Forecast: ' + \ + ' unknown unit: ' + self.unit ) + return values + + def getQueryTime(self): + querytime = None + if self.creationDateTime is not None: + querytime = int( time.mktime( \ + self.creationDateTime.timetuple() ) ) + elif self.forecastDate is not None: + querytime = int( time.mktime( \ + self.forecastDate.timetuple() ) ) + else: + querytime = int( time.mktime( datetime.now().timetuple() ) ) + + return querytime + + def getOffsetTime(self): + return self.fstPeriod[0] - self.getT0() + + def isEmpty(self): + if len( self.timeValueQuality ) <= 0: + return True + else: + for v in self.getValuesInCMS(): + if not math.isclose( v, float( self.missVal ),\ + abs_tol=0.0001 ): + return False + return True + + + def get5CharStationID(self): + if len( self.stationID ) > 5: + return self.stationID[:5] + else: + return self.stationID + + def getQuality(self): + return 100 + + def isObserved(self): + if self.qualifierId is not None and \ + self.qualifierId == "observed": + return True + elif self.qualifierId is not None and \ + self.qualifierId == "forecast": + return False + else: + return False + + def isForecast(self): + if self.qualifierId is not None and \ + self.qualifierId == "forecast": + return True + elif self.qualifierId is not None and \ + self.qualifierId == "observed": + return False + elif self.forecastDate is not None: + return True + else: + return False + + def print(self): + print( "Station: " + self.get5CharStationID() ) + print( "isForecast: ", "True" if self.isForecast() else "False" ) + print( "isObserved: ", "True" if self.isObserved() else "False" ) + print( "Parameter: " + self.parameterId ) + print( "Period: (" + self.getTimePeriod()[0].strftime( \ + "%m/%d/%Y %H:%M:%S" ) + ", " + \ + self.getTimePeriod()[1].strftime( \ + "%m/%d/%Y %H:%M:%S" ) + ")" ) + if self.isForecast(): + print( "T0: " + self.getT0().strftime( "%m/%d/%Y %H:%M:%S" ) ) + print( "Time step: " + self.timeStep[1] + " " + self.timeStep[0] ) + print( "Missing Value: " + self.missVal ) + print( "Time Value(org.) Quality" ) + for k, v in self.timeValueQuality.items(): + print(k.strftime( "%m/%d/%Y %H:%M"), v[0], v[1], v[2] ) + + + def getPreviousValueTime( self, t ): + previousTime = None + for k, v in self.timeValueQuality.items(): + if k <= t: + if not math.isclose( v[ 0 ], \ + float( self.missVal ),\ + abs_tol=0.0001 ): + previousTime = k + else: + break + + return previousTime + + def getNextValueTime( self, t ): + nextTime = None + for k, v in self.timeValueQuality.items(): + if k >= t: + if not math.isclose( v[ 0 ], \ + float( self.missVal ),\ + abs_tol=0.0001 ): + nextTime = k + break + return nextTime + + def removeMissVals(self): + keyMissVal = [] + for k in self.timeValueQuality.keys(): + v = self.timeValueQuality[ k ] + if math.isclose( v[ 0 ], float( self.missVal ),\ + abs_tol=0.0001 ): + keyMissVal.append( k ) + + if keyMissVal: + for k in keyMissVal: + self.timeValueQuality.pop( k ) + + self.resetTimePeriod() + + def addLeadingAndTrailingZeros(self): + # + # Add trailing and leading zeros when the event raw data + # doesn't begin or end with a zero value. + # This function can only be used for a forecast timeseries + # because T0 is not available for observed timeseries + # + t0 = self.getT0() + dt = timedelta( seconds = self.getTimeStepInSeconds() ) + if self.fstPeriod[0] > t0 - timedelta( hours = 48 ): + flag = self.timeValueQuality[ self.fstPeriod[0] ][1] + self.timeValueQuality.update( {self.fstPeriod[0] - dt: ( \ + 0.0, flag, False ) } ) + + self.timeValueQuality.move_to_end( self.fstPeriod[0] - dt, \ + last=False ) + + if self.fstPeriod[1] < t0 + timedelta( hours = 240 ): + flag = self.timeValueQuality[ self.fstPeriod[1] ][1] + self.timeValueQuality.update( {self.fstPeriod[1] + dt: ( \ + 0.0, flag, False ) } ) + + self.timeValueQuality.move_to_end( self.fstPeriod[1] + dt, \ + last=True ) + + self.resetTimePeriod() + + def linearInterpolate(self, dt): + # + # remove missing values + # + self.removeMissVals() + if self.isEmpty(): + return self.timeValueQuality + + t = list( self.timeValueQuality.keys() )[ 0 ] + last = list( self.timeValueQuality.keys() )[ -1 ] + + tvq = OrderedDict() + while t <= last: + if t in self.timeValueQuality: + tvq[ t ] = self.timeValueQuality[ t ] + else: + t1 = self.getPreviousValueTime( t ) + t2 = self.getNextValueTime( t ) + + if ( t2 - t1 ).total_seconds() > 48 * 3600: + + tvq[ t ] = ( float( self.missVal ), 0.0, False ) + + else: + + tvq[t] = ( ( self.timeValueQuality[t2][0] - \ + self.timeValueQuality[t1][0] ) * \ + ( ( t - t1 ).total_seconds() / \ + ( t2 - t1 ).total_seconds() ) + \ + self.timeValueQuality[t1][0] , + ( self.timeValueQuality[t2][1] - \ + self.timeValueQuality[t1][1] ) * \ + ( ( t - t1 ).total_seconds() / \ + ( t2 - t1 ).total_seconds() ) + \ + self.timeValueQuality[t1][1], \ + True ) + + t = t + dt + + if self.getTimeStepInSeconds() > int( dt.total_seconds() ): + self.setTimeStepInSeconds(int( dt.total_seconds() ) ) + + self.fstPeriod = list( tvq.keys() )[0], list( tvq.keys() )[-1] + self.startDate = self.fstPeriod[0] + self.timeValueQuality = tvq + return tvq + + def interForwardPersist(self, dt): + # + # remove missing values + # + self.removeMissVals() + + t = list( self.timeValueQuality.keys() )[ 0 ] + last = list( self.timeValueQuality.keys() )[ -1 ] + + tvq = OrderedDict() + while t <= last: + if t in self.timeValueQuality: + tvq[ t ] = self.timeValueQuality[ t ] + else: + t1 = self.getPreviousValueTime( t ) + + if ( t - t1 ).total_seconds() > 10 * 24 * 3600: + tvq[ t ] = ( float( self.missVal ), 0.0, False ) + else: + tvq[t] = ( self.timeValueQuality[t1][0], \ + self.timeValueQuality[t1][1], \ + True ) + + t = t + dt + + if self.getTimeStepInSeconds() != int( dt.total_seconds() ): + self.setTimeStepInSeconds(int( dt.total_seconds() ) ) + + self.fstPeriod = list( tvq.keys() )[0], list( tvq.keys() )[-1] + self.startDate = self.fstPeriod[0] + self.timeValueQuality = tvq + return tvq + + def linearInterpolateAndInterForwardPersist(self, dt): + # + # remove missing values + # + self.removeMissVals() + if self.isEmpty(): + return self.timeValueQuality + + t = list( self.timeValueQuality.keys() )[ 0 ] + last = list( self.timeValueQuality.keys() )[ -1 ] + + t0 = self.getT0() + + tvq = OrderedDict() + + while t < t0: + if t in self.timeValueQuality: + tvq[ t ] = self.timeValueQuality[ t ] + else: + t1 = self.getPreviousValueTime( t ) + t2 = self.getNextValueTime( t ) + + # + # For LMRFC files that have missing values starting before + # T0, that is t2 is None, next value for t is missing or + # T0 > last. + # + if not t1 or not t2: + tvq[ t ] = ( float( self.missVal ), 0.0, True ) + + else: + if ( t2 - t1 ).total_seconds() > 48 * 3600: + + tvq[ t ] = ( float( self.missVal ), 0.0, True ) + + else: + + tvq[t] = ( ( self.timeValueQuality[t2][0] - \ + self.timeValueQuality[t1][0] ) * \ + ( ( t - t1 ).total_seconds() / \ + ( t2 - t1 ).total_seconds() ) + \ + self.timeValueQuality[t1][0] , + ( self.timeValueQuality[t2][1] - \ + self.timeValueQuality[t1][1] ) * \ + ( ( t - t1 ).total_seconds() / \ + ( t2 - t1 ).total_seconds() ) + \ + self.timeValueQuality[t1][1], \ + True ) + + t = t + dt + + t = t0 + while t <= last: + if t in self.timeValueQuality: + tvq[ t ] = self.timeValueQuality[ t ] + else: + t1 = self.getPreviousValueTime( t ) + + if t1: + if ( t - t1 ).total_seconds() > 10 * 24 * 3600: + tvq[ t ] = ( float( self.missVal ), 0.0, True ) + else: + tvq[t] = ( self.timeValueQuality[t1][0], \ + self.timeValueQuality[t1][1], \ + True ) + # + # If the previous value doesn't exist or + # observed values (including T0) are not provided + # + else: + tvq[ t ] = ( float( self.missVal ), 0.0, True ) + + t = t + dt + + if self.getTimeStepInSeconds() != int( dt.total_seconds() ): + self.setTimeStepInSeconds(int( dt.total_seconds() ) ) + + self.fstPeriod = list( tvq.keys() )[0], list( tvq.keys() )[-1] + self.startDate = self.fstPeriod[0] + self.timeValueQuality = tvq + return tvq + + def persistBackward(self, t, maxt, v = None ): + + if self.isEmpty(): + return self.timeValueQuality + + start = list( self.timeValueQuality.keys() )[ 0 ] + dt = timedelta( seconds = self.getTimeStepInSeconds() ) + +# while \ +# math.isclose( self.timeValueQuality[ start ][ 0 ], \ +# float( self.missVal ), abs_tol=0.0001 ): +# start = start + dt + + if t >= start: + return None + + firstvalue = self.timeValueQuality[ start ] + + t1 = t + it = start - dt + if start - t > maxt: + t1 = start - maxt + + while t1 <= it: + + self.timeValueQuality.update({it: ( \ + firstvalue[0] if not isinstance(v, float) else v, \ + firstvalue[1], True ) } ) + self.timeValueQuality.move_to_end( it, last=False ) + + it = it - dt + + if start - t > maxt: + it = t1 - dt + + while t <= it: + + self.timeValueQuality.update({it: ( \ + float( self.missVal ) if not isinstance(v, float) else v, \ + 0.0, True ) } ) + self.timeValueQuality.move_to_end( it, last=False ) + + it = it - dt + + self.resetTimePeriod() + + return self.timeValueQuality + + def persistForward(self, t, v = None ): + + if self.isEmpty(): + return self.timeValueQuality + + end = list( self.timeValueQuality.keys() )[ -1 ] + + if t <= end: + return self.timeValueQuality + + dt = timedelta( seconds = self.getTimeStepInSeconds() ) + + while \ + math.isclose( self.timeValueQuality[ end ][ 0 ], \ + float( self.missVal ), abs_tol=0.0001 ): + end = end - dt + + vlast = self.timeValueQuality[ end ] + + end = end + dt + + while end <= t: + self.timeValueQuality[end] = ( \ + vlast[0] if not isinstance(v, float) else v,\ + vlast[1], True ) + end = end + dt + + self.resetTimePeriod() + return self.timeValueQuality diff --git a/data_assimilation_engine/rfc_ingestion/RFC_Forecast_test.py b/data_assimilation_engine/rfc_ingestion/RFC_Forecast_test.py new file mode 100644 index 0000000..8ad2f33 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/RFC_Forecast_test.py @@ -0,0 +1,145 @@ +import unittest +import copy +from datetime import datetime, timedelta +from PI_XML import PI_XML +from RFC_Forecast import RFC_Forecast +from RFC_Sites import RFC_Sites + +class RFC_Forecast_test(unittest.TestCase): + def run_rfc_test(self, pixmlfile, sitefile ): + rfcsites = RFC_Sites( sitefile ) + pixml = PI_XML( pixmlfile, rfcsites ) + + allids = pixml.getAllStationIDs() + print(allids) + for s in pixml.getFlowTimeseries(): + s.print() + print("=========================================") + + + for id in allids: + print( "============== " + id + " ===============" ) + ts = pixml.getObservedAndForecastForID( id ) +# if ts[0] is None or ts[1] is None: +# print( "Missing observed or forecast: " + id + " : " + +# rfcsites.getRFCBySite( id ) ) +# break + ts[0].print() + ts[1].print() + obvhourly = copy.deepcopy( ts[0] ) + fsthourly = copy.deepcopy( ts[1] ) + obvhourly.linearInterpolate( timedelta( hours = 1 )) + fsthourly.linearInterpolate( timedelta( hours = 1 )) + obvhourly.persistForward( obvhourly.getTimePeriod()[1] + \ + timedelta( days = 2 ) ) + obvhourly.persistBackward( obvhourly.getTimePeriod()[0] - \ + timedelta( days = 2 ) ) + fsthourly.persistForward( fsthourly.getTimePeriod()[1] + \ + timedelta( days = 2 ) ) + fsthourly.persistBackward( fsthourly.getTimePeriod()[0] - \ + timedelta( days = 2 ) ) + print( "============== " + "hourly: " + id + " ===============" ) + + print( "============== " + "combined: " + id + " ===============" ) + combined = \ + pixml.combineObvFstAndApplyPresistenceLinearInterpolation( ts ) + combined.print() + allcombined = \ + pixml.getReserviorObservedForecastCombinedWithT0() + + print( "============== all combined: ===============" ) + for c in allcombined: + print( "============== all combined: " + c.get5CharStationID() + " ===============" ) + c.print() + + self.assertTrue(True) + + def test_NCRFC( self ): + sitefile = "./RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv" + pixmlfile = "./testdata/ncrfc/201911181200_NCRFC_Reservoir_Export.xml" + self.run_rfc_test( pixmlfile, sitefile ) + self.assertTrue(True) + + def test_NERFC( self ): + sitefile = "./RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv" + pixmlfile = "./testdata/nerfc/NERFC_Reservoir_Export.xml" + + self.run_rfc_test( pixmlfile, sitefile ) + + self.assertTrue(True) + + def test_WGRFC( self ): + sitefile = "./RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv" + pixmlfile = "./testdata/wgrfc/WGRFC_11.27.2019.16.00.xml" + + self.run_rfc_test( pixmlfile, sitefile ) + + self.assertTrue(True) + + def test_MARFC( self ): + sitefile = "./RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv" + pixmlfile = "./testdata/marfc/2019112712_MARFC_Reservoir_Export.xml" + + self.run_rfc_test( pixmlfile, sitefile ) + self.assertTrue(True) + + def test_OHRFC( self ): + sitefile = "./RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv" + for pixmlfile in [ \ + "./testdata/ohrfc/201911271319_OHRFC_Reservoir_Export_SAGU.xml", + "./testdata/ohrfc/201911271327_OHRFC_Reservoir_Export_SAGL.xml", + "./testdata/ohrfc/201911271331_OHRFC_Reservoir_Export_SMNU.xml", + "./testdata/ohrfc/201911271332_OHRFC_Reservoir_Export_SBVR.xml", + "./testdata/ohrfc/201911271332_OHRFC_Reservoir_Export_SMNL.xml" ]: + self.run_rfc_test( pixmlfile, sitefile ) + + def test_ABRFC( self ): + + sitefile = "./RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv" + pixmlfile = "./testdata/abrfc/2019112712_ABRFC_RES_NWM_pixml_export.xml" + + self.run_rfc_test( pixmlfile, sitefile ) + + def test_LMRFC( self ): + + sitefile = "./RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv" + pixmlfile = "./testdata/lmrfc/201911290111_LMRFC_Reservoir_export.xml" + + self.run_rfc_test( pixmlfile, sitefile ) + + def test_SERFC( self ): + + sitefile = "./RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv" + pixmlfile = "./testdata/serfc/201911290600_SERFC_Reservoir_Export.xml" + + self.run_rfc_test( pixmlfile, sitefile ) + + def test_CBRFC( self ): + + sitefile = "./RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv" + pixmlfile = "./testdata/cbrfc/201911281800_CBRFC_Reservoir_Export.xml" + + self.run_rfc_test( pixmlfile, sitefile ) + + def test_CNRFC( self ): + + sitefile = "./RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv" + pixmlfile = "./testdata/cnrfc/201911290600_CNRFC_Reservoir_Export_for_NWM.xml" + + self.run_rfc_test( pixmlfile, sitefile ) + def test_NWRFC( self ): + + sitefile = "./RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv" + pixmlfile = "./testdata/nwrfc/NWRFC_11.29.2019.2100.xml" + + self.run_rfc_test( pixmlfile, sitefile ) + + def test_MBRFC( self ): + + sitefile = "./RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv" + pixmlfile = "./testdata/mbrfc/201911111541_MBRFC_Reservoir_Export.xml" + + self.run_rfc_test( pixmlfile, sitefile ) + +if __name__ == '__main__': + unittest.main() diff --git a/data_assimilation_engine/rfc_ingestion/RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv b/data_assimilation_engine/rfc_ingestion/RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv new file mode 100644 index 0000000..94b3577 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv @@ -0,0 +1,395 @@ +gage,gagedFlowline,NHDWaterbodyComID,lakeLink,SiteName,RFC +SOMT2,5570785,5569731,0,Somerville Lake,WGRFC +SMCT2,3589578,3588682,3589578, ,WGRFC +GGLT2,5671667,5670445,5671667, ,WGRFC +BLNT2,2572334,2571688,2572334,Belton Lake at North Texas Lakes,WGRFC +STIT2,5587560,120053388,5587560, ,WGRFC +GNGT2,5671657,5670421,5671657, ,WGRFC +TBLT2,1111447,1111211,1111447, ,WGRFC +WDRF1,2293124,10361596,2311949, ,SERFC +FOGG1,2306737,166758723,3443790, ,SERFC +CLBA1,18239649,21638156,18239647,Claiborne Dam near Alabama River,SERFC +BDWT2,1447048,1445978,1447048, ,WGRFC +DAWT2,4133643,4132531,4133643,Navarro Mills Lake at North Texas Lakes,WGRFC +ALAT2,166414852,166414857,166414852, ,WGRFC +PCTT2,2569848,120053201,2569848,Proctor Lake at North Texas Lakes,WGRFC +SAGT2,5712857,120053389,0,O. C. Fisher Lake at West Central Texas Lakes,WGRFC +SVIC1,20334576,20334374,20334576, ,CNRFC +GCDA3,21274156,21318615,21320525,GILA RIVER BELOW COOLIDGE DAM AZ.,CBRFC +BNBT2,1269108,1267968,1269108, ,WGRFC +LEWT2,1278674,120053021,1278674,Lake Lewisville at North Texas Lakes,WGRFC +GPVT2,1287177,120051934,1287177,Lake Grapevine at North Texas Lakes,WGRFC +WETG1,3298918,166758703,3298566, ,SERFC +CHDS1,22720895,167484077,6290331, ,SERFC +CMMG1,2051313,6495106,2051313,Lake Lanier at Chattahoochee River,SERFC +CVLG1,6505972,6495140,6505972, ,SERFC +GRNM6,18019788,18014672,18019788,Grenada Dam at Yalobusha River,LMRFC +MYST2,8348979,8347705,8348979,Pat Mayse Lake at North Texas Lakes,ABRFC +MGCO2,707038,167484743,0,"McGee Creek Reservoir, OK",ABRFC +DSNT2,426416,167300255,426416, ,ABRFC +ARBO2,19959560,167300271,19959986, ,ABRFC +SYOT2,13733430,13732800,13733430,WICHITA RV NR MABELLE TX,ABRFC +LYMA3,20585864,20584396,20585864, ,CBRFC +SMDA3,20478096,20476542,0,SALT RIVER BLW STEWART MOUNTAIN DAM AZ.,CBRFC +VDBA3,20439512,20438850,0,Bartlett Dam,CBRFC +HORA3,20440738,120053952,20440738, ,CBRFC +BWAA3,21384136,21383092,21384184, ,CBRFC +LKPC1,17575779,17573833,17575779,Santa Felicia Dam below Piru Creek,CNRFC +CCHC1,17611491,17609317,17611491, ,CNRFC +PYMC1,17569485,17568947,17572445, ,CNRFC +CBDN2,21436863,120052757,21436857, ,CBRFC +ALTO2,13757101,13756271,13757101, ,ABRFC +FOSO2,3142766,3140356,3142766,Foss Reservoir at Washita River,ABRFC +MTNO2,562357,167300241,562485, ,ABRFC +FTCO2,683617,681831,683617, ,ABRFC +NRMO2,481932,481656,0,Little River At Lake Thunderbird,ABRFC +TENO2,402464,402142,402464, ,ABRFC +CTRG1,6479573,6479321,6479573,Carters Lake at Coosawattee River,SERFC +KERV2,8653306,167496465,8653306,John H. Kerr Dam at Roanoke River,SERFC +NUDN7,166737579,166737717,166737584, ,SERFC +NHPN7,26325148,166755060,8898132, ,SERFC +WLKN7,9251746,9250140,9251746, ,SERFC +WTGT1,19744394,166997499,19745278, ,LMRFC +NFDA4,7653861,120052259,7653895, ,LMRFC +PENO2,21773239,167299912,7586119, ,ABRFC +MFDO2,21773255,21772239,21773125,Hudson Lake at Eastern Oklahoma Lakes,ABRFC +CNLO2,390140,389804,390140,Canton Lake above North Canadian River,ABRFC +NACC1,8212787,8210941,0, other Nacimiento Reservoir,CNRFC +SNRC1,8212925,120053442,8212925,SAN ANTONIO RIVER - LAKE SAN ANTONIO,CNRFC +LEXC1,17695749,17693799,0,LEXINGTON RESERVOIR NEAR LOS GATOS,CNRFC +COYC1,17695825,17693823,17695825,Coyote Reservoir (COYC1),CNRFC +FRAC1,17116483,120053836,0,SAN JOAQUIN RIVER NEAR FRIANT,CNRFC +GUUU1,10025198,10022260,10025198,GUNLOCK RESERVOIR,CBRFC +CHNK1,21160795,21160241,21161233, ,ABRFC +STXM7,7372051,167267899,7373063,Stockton Dam at Sac River,MBRFC +WPPM7,3665570,167166315,0,Wappapello Lake Pool at St. Francis River,LMRFC +PTTV2,8674429,8674019,8675539, ,SERFC +JCKV2,8521381,120053459,8523459,Gathright Dam at Lake Moomaw,MARFC +PTXM7,7388811,7387259,7388811, ,MBRFC +HILK1,2993926,2992128,2993756, ,MBRFC +PLKK1,940290180,10114004,940290180,Pomona Lake at 110 Mile Creek,MBRFC +MLVK1,10132098,167267891,10132098,Melvern Lake at Marais Des Cygnes River,MBRFC +JRLK1,20932366,167297255,20932366,John Redmond Lake at Neosho River,ABRFC +JSPT2,1159739,1158647,1159739,Lake Sam Rayburn at Angelina River,WGRFC +BKDL1,25040602,15078462,25040602, ,LMRFC +TYLA1,21687508,120052975,21687488, ,SERFC +MRFA1,21456722,21454020,21456716,Miller's Ferry near Alabama River,SERFC +LBUL1,19936525,120053896,19936525,Lake Bistineau at Bayou Dorcheat,LMRFC +WAGL1,8342801,8341399,0,WALLACE LAKE NEAR KEITHVILLE,LMRFC +ACTT2,5531594,5531274,5531594,Waco Lake at North Texas Lakes,WGRFC +WTYT2,5513692,120053204,5513692,Whitney Lake at North Texas Lakes,WGRFC +HAWC1,20344395,20342929,20344473,Lake Henshaw,CNRFC +ELPC1,20332816,20332350,20334592,EL CAPITAN DAM,CNRFC +LVNT2,1292396,1291638,1292396, ,WGRFC +CRLL1,1136266,167300374,1136426, ,LMRFC +BCEA4,19927057,19925795,19927057, ,LMRFC +LCAL1,15223102,167182354,15223102, ,LMRFC +HRTG1,6270218,166759840,6270218, ,SERFC +UBRA1,19569560,19568056,0,UPPER BEAR DAM NEAR BEAR CREEK,LMRFC +LBRA1,19568264,19626258,0,LITTLE BEAR RTU NEAR BELGREEN,LMRFC +CCRA1,19566578,19566308,19566880,CEDAR CREEK RTU,LMRFC +SRDM6,15290344,167182317,15290596,Tallahatchie River at Sardis Dam,LMRFC +ENDM6,15276792,167166404,15276792, ,LMRFC +DGDA4,22002032,22000120,22002032, ,LMRFC +NADA4,22700300,22697680,22700420, ,LMRFC +AHDA4,3747834,167300345,0,LITTLE RIVER AT MILLWOOD DAM NEAR ASHDOWN,ABRFC +DIEA4,3746252,3745478,0,SALINE RIVER AT DAM NEAR DIERKS,ABRFC +DQDA4,3745840,3745526,0,ROLLING FORK RIVER AT DAM NEAR DEQUEEN,ABRFC +GLLA4,3746224,3745418,0,COSSATOT RIVER AT GILLHAM DAM NEAR GILLHAM,ABRFC +BKDO2,630139,630011,0,MOUNTAIN FORK RIVER AT BROKEN BOW DAM NEAR BROKEN BOW,ABRFC +PCLO2,617464,617546,0,LITTLE RIVER AT PINE CREEK DAM NEAR WRIGHT CITY,ABRFC +HGLO2,592836,591880,592836,Hugo Lake at Eastern Oklahoma Lakes,ABRFC +WRLO2,8358571,8357843,8358571,Waurika Lake at Beaver Creek (OK),ABRFC +RSVA3,22439652,120052904,22439652, ,CBRFC +EUFO2,512899,167801019,512899, ,ABRFC +WSLO2,6047652,6043008,6047652,Wister Lake at Eastern Oklahoma Lakes,ABRFC +BMTA4,7803227,7801259,7803225, ,ABRFC +BMDA4,15234506,15231972,15234506, ,LMRFC +GRRA4,11774999,167299819,11774971,Greers Ferry Dam at Little Red River,LMRFC +ARKM6,15256122,15252778,15256386, ,LMRFC +NRMT1,19534722,166997625,19534808, ,LMRFC +OCCT1,19668053,19667261,19668053, ,LMRFC +HADT1,19678313,19674973,19678329,Apalachia Dam above Hiwassee River,LMRFC +BRDG1,19669385,19668381,19669385,Toccoa River at Blue Ridge Dam,LMRFC +NOTG1,19680543,166997585,19680543, ,LMRFC +FONN7,19736617,166912876,19736617,FONTANA DAM TVA at Little Tennessee River,LMRFC +CHAN7,19674637,166997584,0,Chatuge Dam,LMRFC +SHDT1,19753533,166997474,0,South Holston Dam,LMRFC +FPHT1,19754649,166997475,19754649, ,LMRFC +CRKT1,22180610,166997501,22180610,Cherokee Dam above Holston River,LMRFC +DUGT1,22144632,166997463,22146716, ,LMRFC +NRST1,22124294,166997561,14643271, ,LMRFC +BSGA4,7629426,167299818,7637444, ,LMRFC +GIBO2,21773859,167299918,21773859,Eastern Oklahoma Lakes at Ft. Gibson Lake,ABRFC +OOLO2,21799057,167299904,21799057, ,ABRFC +KEYO2,20973380,20971700,20973380, ,ABRFC +BIRO2,846662,846266,846662, ,ABRFC +SKLO2,941070050,941070128,941070050, ,ABRFC +HEYO2,373089,371555,373089, ,ABRFC +FSLO2,13811232,13811164,13811232, ,ABRFC +MRIT2,19981020,19980072,19981020,Lake Meredith at Canadian River,ABRFC +LKSA3,25138762,22071698,25138762, ,CBRFC +ISAC1,14972947,22670080,0,KERN RIVER AT ISABELLA DAM NEAR LAKE ISABELLA 2N,CNRFC +SCSC1,14931007,14930103,0,TULE RIVER AT SUCCESS DAM NEAR PORTERVILLE 5E,CNRFC +TMDC1,24758173,120052624,24758173,KAWEAH RIVER AT TERMINUS DAM NEAR WOODLAKE,CNRFC +CADC1,17695921,17693805,0,CALERO RESERVOIR,CNRFC +ANDC1,17695819,17693795,17695819, ,CNRFC +PFTC1,22063199,22050039,0,KINGS RIVER AT PINE FLAT DAM NEAR CENTERVILLE 10NE,CNRFC +PCON2,20603436,20603252,0,PINE CANYON DAM NEAR CALIENTE,CBRFC +MVDN2,20603440,20603248,0,MATHEW VALLEY DAM NEAR CALIENTE,CBRFC +FRVC2,17014767,17014523,17015277,Florida River below Lemon Resv.,CBRFC +LPVC2,17034231,17034145,17034971, ,CBRFC +GSPO2,21001215,21000069,21001215, ,ABRFC +KAWO2,21038211,21038107,21038211, ,ABRFC +FLLK1,21517708,21514748,21517708,Fall River Lake above Fall River (KS),ABRFC +HULO2,21785290,21784832,21785290, ,ABRFC +CPLO2,21784744,21783714,21784744, ,ABRFC +ECLK1,20943984,20943110,20943984,Elk City Lake above Elk River (KS),ABRFC +BIGK1,21797549,21795481,21797721, ,ABRFC +CLRM7,7669160,7666880,7669104,Clearwater Lake Pool at Black River,LMRFC +TRLK1,20955434,20953830,20955434, ,ABRFC +CNGK1,20929492,20927390,20929492, ,ABRFC +EDRK1,20082038,20079044,0,Walnut River At El Dorado Lake,ABRFC +MLBK1,20919169,20917521,20919169,Marion Reservoir at Cottonwood River,ABRFC +KANK1,25068602,167266519,25068602, ,MBRFC +CRFC2,3251439,3251185,3252185, ,CBRFC +MPSC2,3254131,3252989,0,MORROW POINT RESERVOIR NEAR MONTROSE,CBRFC +BPRC1,8915961,8914219,8915961, ,CNRFC +TOPN2,10743558,10742110,0,TOPAZ CANAL BELOW TOPAZ LAKE NEAR TOPAZ,CNRFC +HETC1,17082365,17080669,17082207, ,CNRFC +CHVC1,17082185,17080601,17082185, ,CNRFC +NIMC1,15022633,15021939,15024897, ,CNRFC +LBEC1,15032995,130951632,8016105,Lake Berryessa (LBEC1),CNRFC +WSDC1,8276453,8271433,8276453, ,CNRFC +LAMC1,8269153,8306696,8269145,Lake Mendocino,CNRFC +INVC1,8007863,8005383,0,INDIAN VALLEY RESERVOIR NEAR LOWER LAKE,CNRFC +HLEC1,8062551,8060131,8062551,Englebright Lake (HLEC1),CNRFC +DNRC1,8934506,8932994,0,DONNER LAKE NEAR TRUCKEE,CNRFC +TAHC1,8942039,120053784,8942039, ,CNRFC +BCVC1,8933684,8932970,8934482, ,CNRFC +STPC1,8933230,120053476,8934442, ,CNRFC +PSRC1,8933720,8932974,8934488,Prosser Creek Reservoir at Prosser Creek,CNRFC +SFSU1,3907377,3906255,0,SCOFIELD RESERVOIR NEAR SCOFIELD,CBRFC +MLSU1,4876845,4876263,4877561, ,CBRFC +MDCC2,1337204,1336910,1338026, ,CBRFC +TRBC2,1333490,1332558,1333610, ,CBRFC +BLRC2,1312847,1312051,1313385, ,CBRFC +WLSK1,7331754,7330556,7331754, ,MBRFC +MLFK1,5927296,167800866,5927296,MILFORD LK NR JUNCTION CITY KS,MBRFC +MTTK1,2281171,18880944,2281171, ,MBRFC +PRRK1,748101,746373,748101, ,MBRFC +CLIK1,24668460,3730511,24668460,CLINTON LK NR LAWRENCE KS,MBRFC +SLKM7,2530631,2529647,2530631,Smithville Reservoir other Reservoirs,MBRFC +LBRM7,4461528,4460822,4461528, ,MBRFC +STJW2,3717136,167484418,167484406,Stonewall Jackson Dam,OHRFC +TGOW2,4352134,166899099,4352136, ,OHRFC +YGOP1,3809037,3806779,3809045, ,OHRFC +CMDP1,4754924,167484295,4754908,CONEMAUGH RIVER AT TUNNELTON PA,OHRFC +CCDP1,4741239,4739601,0,CROOKED CREEK AT CROOKED CREEK DAM NEAR FORD CITY,OHRFC +LHDP1,4583044,167484347,0,LOYALHANNA CREEK AT LOYALHANNA DAM PA,OHRFC +RADI4,940280022,120053324,0,Rathbun Dam,MBRFC +CBGC2,1234513,1233515,0,COLORADO RIVER NEAR GRANBY CO,CBRFC +WCKC2,1234173,1233595,0,WILLOW CREEK BELOW WILLOW CREEK RESERVOIR (WILWCRCO),CBRFC +WFRC2,1232803,1230577,1232989,WILLIAMS FORK BELOW WILLIAMS FORK RESERVOIR CO,CBRFC +BGMC2,1313213,1311881,1313287, ,CBRFC +LAAU1,11976201,11975087,0,MOON LAKE RESERVOIR NEAR MOUNTAIN HOME,CBRFC +DCRU1,10376632,10375440,10376632, ,CBRFC +WHSC1,2782777,2781993,2782763, ,CNRFC +LLKC1,8246426,8245384,8246596, ,CNRFC +IRGC1,361671,361273,362919,Klamath River below Iron Gate Dam,CNRFC +HLZU1,664760,664014,0,Hyrum at Little Bear River,CBRFC +OPDU1,10275828,10273702,10275998, ,CBRFC +ECWU1,10093052,10091588,10093132, ,CBRFC +ECCU1,10277268,10276260,10277316,EAST CANYON CREEK NEAR MORGAN UT,CBRFC +CRDU1,10089426,10089124,10090806,LOST CREEK NEAR CROYDEN UTAH,CBRFC +BRWU1,7879744,166248016,7880842, ,CBRFC +MCWW4,3197078,120051967,0,BLACKS FORK NEAR MILLBURNE WY,CBRFC +GRZU1,10040950,10038598,10040912,Greendale near Green River (WY-UT-CO),CBRFC +SAYI4,6597700,6597410,0,Saylorville Dam Pool,NCRFC +PELI4,4995175,167121028,22249819,Red Rock Reservoir at Des Moines River,NCRFC +CRVI4,17540805,17539821,17540805, ,NCRFC +KITO1,13156205,13155459,13156523, ,OHRFC +MLPO1,13154265,13152651,13154569,"Milton Dam, Ohio",OHRFC +BRWO1,13156389,166899115,13156577, ,OHRFC +MSQO1,13153121,13152353,13154427,Mosquito Creek Lake below Mosquito Creek,OHRFC +SHDP1,12999322,12996298,12999322,Shenango Dam Tailwater at Shenango River,OHRFC +TNTP1,10222062,10922417,10222062,Tionesta Dam Pool at Tionesta Creek,OHRFC +KNZP1,8975668,166899002,0,Kinzua Dam on Allegheny River other Central PA Dams,OHRFC +GHDP1,6874115,6873355,6874143, ,OHRFC +BCHP1,8139444,120053440,8139718,Blanchard at Bald Eagle Creek,MARFC +STVC3,7718288,120052268,7718286, ,NERFC +SCIR1,6129639,6127281,6130387,Scituate Reservoir,NERFC +GBRN6,3247596,3247342,3247596,Gilboa Dam at Schoharie Creek,NERFC +DWNN6,1748727,1748473,1748773, ,MARFC +CNNN6,2614136,2613174,2614136, ,MARFC +GBFW4,18312556,18352167,18355239, ,CBRFC +LOSO3,23923436,23927716,23923436,Lost Creek Lake other Rogue River,NWRFC +BNGM1,3319220,3318110,3319188, ,NERFC +DRBI1,24158991,24161747,24158993, ,NWRFC +SCOO3,23805106,23807386,23805110,Reservoir at Henry Hagg Lake,NWRFC +HAHW1,23977690,23979340,23977692, ,NWRFC +BMDC2,3254269,3252975,0,BLUE MESA DAM NEAR GUNNISON,CBRFC +MYSU1,10814786,10814778,10815916,Marysvale near Sevier River,CBRFC +OCEU1,10806037,120051930,0,OTTER CREEK RESERVOIR- ANTIMONY- NR,CBRFC +KNFC1,348545,347987,349887, ,CNRFC +NHGC1,17068300,17065952,0,NEW HOGAN LAKE (NEW HOGAN DAM),CNRFC +CMCC1,3953573,3950979,0,Lake Camanche (CMCC1),CNRFC +EPRC1,7994771,7993809,7996275, ,CNRFC +SGEC1,7993205,7990259,7993205,STONY GORGE RESERVOIR NEAR ELK CREEK,CNRFC +BLBC1,7992981,7989989,0,STONY CREEK AT BLACK BUTTE DAM NEAR ORLAND,CNRFC +ORDC1,2778598,12076080,0,Lake Oroville (ORDC1),CNRFC +CFWC1,15013989,15012277,0,BEAR RIVER AT CAMP FAR WEST DAM NEAR NEAR WHEATLAND 6E,CNRFC +NBBC1,8062451,8060079,8062451, ,CNRFC +JOVU1,4877487,4876219,4877545, ,CBRFC +ELLU1,4876063,4875263,4876063, ,CBRFC +VEGC2,3180780,3180118,3181278,VEGA RESERVOIR NEAR COLLBRAN CO,CBRFC +RRGC2,3175298,3174852,3175958, ,CBRFC +REPN1,19043517,19040443,19043517,Harlan Reservoir at Republican River,MBRFC +STAU1,11966427,11965847,11967775, ,CBRFC +STIU1,11964915,11964599,0,STRAWBERRY - STRAWBERRY RESERVOIR- SOLDIER SPRINGS,CBRFC +UTLU1,10329217,10327875,10329101,Jordan - Utah Lake- Provo,CBRFC +GPDN1,11761568,11758154,11761566,Gavins Point Dam at Missouri River,MBRFC +BIGW4,18301696,18300278,18303140, ,CBRFC +VIVW4,3192982,3192800,0,Hams Fork below Viva Naughton Reservoir,CBRFC +BLZI1,4472263,7897413,4472263,BEAR LAKE OUTLET CANAL NR PARIS,CBRFC +BBOI1,4560894,4558474,0,BEAR R BLO UT P&L @ ONEIDA,CBRFC +BEAI1,4556428,4467951,4469009, ,CBRFC +HIKN6,22743759,22741995,22743759, ,NERFC +STDM1,6724973,6719717,0,SEBAGO LAKE AT STANDISH,NERFC +PRVO3,23713968,24518496,24515510, ,NWRFC +RKWM1,1022672,166195943,1022652, ,NERFC +HHWM8,22965000,22971534,22965004,HUNGRY HORSE RESERVOIR NR HUNGRY HORSE MT,NWRFC +CPNN6,21982301,21980955,21982301,Canandaigua at Canandaigua Lake,NERFC +MTHT2,3170376,3169144,3170376,Mathis 5 SSW - Lake Corpus Christi Mathis 5 SSW - Lake Corpus Christi,WGRFC +CTDT2,10664160,120052363,10664532,Three Rivers 4 W - Choke Canyon Dam,WGRFC +CKDT2,1639279,1637751,1639279,Schroeder - Coleto Creek Reservoir,WGRFC +HSJT2,1440417,120053035,1469612,Sheldon - Lake Houston,WGRFC +TBRT2,5702849,5702167,5702849,San Angelo 8 SW - Twin Buttes Reservoir,WGRFC +BCRT2,5754180,5753298,5754180,Brady 3 W - Brady Creek Reservoir,WGRFC +MSDT2,5781955,120053393,5781955,Lake Travis near Austin,WGRFC +LCTT2,1467932,120053033,1468386,Conroe 7 W - Lake Conroe,WGRFC +BKLT2,9535873,8327942,9535873,Burkeville 16 NNE - Toledo Bend Dam,WGRFC +TRNT2,1300600,120051936,1300600,Trinidad 7 NNW - Cedar Creek Reservoir,WGRFC +GBYT2,5499430,120053199,5499430,Granbury - Lake Granbury,WGRFC +CLRT2,5636271,5635571,5636271,Colorado City 5 SW - Lake Colorado City,WGRFC +EVST2,5722903,5722027,5722903,Robert Lee 2 W - E. V. Spence Reservoir,WGRFC +RLAT2,3210709,3210503,3211187,Orla - Red Bluff Reservoir,WGRFC +HAWT2,5489975,5489185,5489975,Hawley 8 E - Fort Phantom Hill Reservoir,WGRFC +PSMT2,5497152,120053359,5497152,Palo Pinto 11 NW - Possum Kingdom Lake,WGRFC +EAMT2,1306945,120051933,1306945,Eagle Mountain Reservoir above Fort Worth,WGRFC +FLWT2,1268404,1267710,1269054,Fort Worth 6 NW - Lake Worth,WGRFC +LART2,1269102,1267898,1269102,Arlington 10 W - Lake Arlington,WGRFC +GPET2,1269878,1269548,1269878,Venus - Mountain Creek Lake,WGRFC +FRHT2,1294840,120053026,1294840,Forney 4 NW - Lake Ray Hubbard,WGRFC +PNTT2,5256789,120053340,5256787,Point 7 S - Lake Tawakoni Sabine River,WGRFC +LFKT2,5293507,5292739,5293507,Quitman 4 W - Lake Fork Reservoir,WGRFC +LNGC1,2823750,17080041,17080371,DON PEDRO RES NR LA GRANGE CA,CNRFC +WCDP1,9052441,9051313,9053149,Woodcock Creek at Woodcock Dam-PA,OHRFC +HDLN6,22294712,22294608,22294820,Sacandaga River,NERFC +CLEW1,24125829,24136807,24125831,Cle Elum Outflow at Yakima River,NWRFC +SWRN6,15514540,166890766,15514540, ,NERFC +FALT2,326437,120051895,326437,Falcon Reservoir near Falcon Heights,WGRFC +MDLT2,10835508,10834778,10835508,Bandera Falls - Medina Lake,WGRFC +AMIT2,285566,120052275,285566,Amistad Reservoir near Del Rio Del Rio - Amistad Reservoir Dam,WGRFC +BUDT2,5757778,5756886,5757778,Lake Buchanan near Burnet,WGRFC +LLST2,5576175,5575089,5576175,Seale 3 NE - Lake Limestone,WGRFC +LVDT2,1498834,1492358,1492992,Goodrich 5 NW - Lake Livingston,WGRFC +LMTT2,5279734,5278224,5279734,Dirgin - Martin Creek Lake near Tatum,WGRFC +LMVT2,5279800,5278562,5279800,Buncomb 1 SE - Lake Murvaul,WGRFC +LCRT2,5279506,5278062,5279686,Tatum - Lake Cherokee,WGRFC +SKCT2,1149839,1148575,1150543,New Salem 1 W - Striker Lake Reservoir,WGRFC +LPTT2,4451942,4451216,4452342,Frankston - Lake Palestine,WGRFC +LCLT2,5512476,5511586,5512476,Cleburne 4 S - Lake Pat Cleburne,WGRFC +LLET2,2568408,2567012,2568408,Ranger 7 S - Leon Reservoir,WGRFC +LBWT2,5735541,5734861,5735541,Brownwood 8 N - Lake Brownwood,WGRFC +LKCT2,5740776,5740630,5740956,Silver Valley 6 NNE - Lake Coleman,WGRFC +CCRT2,5635913,5635591,5636277,Colorado City 7 S - Champion Creek Reservoir,WGRFC +BLKT2,5721979,5721267,5721979,Blackwell - Oak Creek Reservoir,WGRFC +JBTT2,5633473,5632093,5633473,Knapp 3 SSW - J. B. Thomas Reservoir,WGRFC +LSWT2,5490441,5490089,5491015,Sweetwater 7 ESE - Lake Sweetwater,WGRFC +SMKT2,5543698,5541638,5543698,Millers Creek Reservoir near Bomartin,WGRFC +LSFT2,5508410,5506926,5508410,Haskell 10 SE - Paint Creek at Lake Stamford,WGRFC +HCRT2,5526171,5525777,5526171,Breckenridge - Hubbard Creek Reservoir,WGRFC +LMGT2,5494750,5494328,5494750,Graham 2 NW - Lake Graham,WGRFC +LPPT2,5497354,5495308,5497354,Lake Palo Pinto near Santo,WGRFC +BPRT2,1305617,1304891,1306513,Bridgeport 5 NW - Bridgeport Reservoir,WGRFC +NMBA4,7829970,7827790,7830220, ,ABRFC +MSWC1,21608717,21606713,0,LAKE MCCLURE AT EXCHEQUER DAM NEAR SNELLING 10ENE,CNRFC +AMFI1,23163005,120052939,24557503,Neeley at Snake River,NWRFC +APPO3,23936033,23938563,23936037,,NWRFC +COTO3,23759308,23761342,23759312,,NWRFC +DORO3,23759442,23761332,23759588,Dorena Reservoir at Reservoir,NWRFC +FALO3,23752598,23756162,,Fall Creek at Reservoir,NWRFC +PALI1,24438932,120052927,,SNAKE RIVER NR IRWIN ID,NWRFC +JLKW4,23123437,23127757,23123437,,NWRFC +ISLI1,24460018,24465514,24460018,,NWRFC +CSDI1,24177103,120054056,24177561,,NWRFC +BRNI1,24192804,120054057,24192806,,NWRFC +BLUO3,23773405,23777375,23773407,,NWRFC +FOSO3,23785769,23788981,23785769,Foster nr Sweet Home at Reservoir,NWRFC +MAYW1,24248946,24252328,24248950,,NWRFC +MSRW1,24248994,24252336,24248994,"RIFFE LAKE NEAR MOSSYROCK, MOSSYROCK DAM",NWRFC +COEI1,24379413,23016890,24379413,Coeur d'Alene at Lake Coeur d'Alene,NWRFC +CABI1,22975980,22983738,,CLARK FORK-CABINET GORGE DAM,NWRFC +RODW1,24255291,24260289,24255291,ROSS RESERVOIR NEAR NEWHALEM WA,NWRFC +SHAW1,24255877,24260837,25184464,Concrete at Baker River,NWRFC +UBDW1,24255915,24260671,24255915,,NWRFC +SNOA2,XXXXXX,15243900,xxxxxx,Snow River,APRFC +SKLA2,XXXXXX,15266110,xxxxxx,Skilak River,APRFC +GGRT1,18432917,22134539,18432921,GREAT FALLS DAM NEAR ROCK ISLAND,LMRFC +RDTA4,21999712,15237526,15237620,Ouachita River at Remmel Dam above Jones Mill AR,LMRFC +KYDK2,10575371,120052349,10575371,Tennessee River at Kentucky Lake Pool,LMRFC +FLDT1,22134009,166997493,22134007,Tennessee River at Ft Loudoun Dam Lenoir City 1E,LMRFC +PICT1,19578817,19598582,25314235,Tennessee River at Pickwick Dam Tailwater,LMRFC +MHDT1,14644171,120052841,14644171,Clinch River above Melton Hill Dam,LMRFC +GVDA1,25311913,167182204,25311913,Tennessee River below Guntersville Dam,LMRFC +LDBL1,15226990,15222888,15223068,Lake DArbonne at Bayou DArbonne,LMRFC +CKDT1,25309298,166997575,25309298,Tennesse River at Chickamauga Dam,LMRFC +WLSA1,19633474,166997604,19633480,WILSON LOCK DAM TAILWATER NEAR FLORENCE 2E,LMRFC +TDTT1,22134049,936010061,19720259,L TENN TELLICO DAM,LMRFC +WBOT1,22134501,166997494,22134453,Tennessee River at Watts Bar Dam,LMRFC +NVXN6,4147432,4146742,4148038,NEVERSINK RIVER AT NEVERSINK NY,MARFC +RVRC3,6111421,6106841,6107211,WEST BRANCH FARMINGTON RIVER AT RIVERTON CT,NERFC +MCKO3,23648272,23655210,23648274,MCKAY RESERVOIR NEAR PENDLETON,NWRFC +LGRW1,24282032,24284232,24282042,NISQUALLY RIVER AT LA GRANDE Alder lake,NWRFC +DEXO3,23751830,23756168,23751844,Dexter Dam Lake MIDDLE FORK WILLAMETTE RIVER NEAR DEXTER OR,NWRFC +CDRW1,24537944,24539838,24537962,Cedar River At Cedar Falls,NWRFC +GPRO3,23785925,23788951,23785925,Quartzville Creek at Green Peter Reservoir,NWRFC +MILI1,24491294,24500352,24495672,Snake River near Milner below Milner Dam ,NWRFC +BWMI1,23250977,120053985,947040096,Big Wood River below Magic Dam,NWRFC +BULO3,24150401,24152927,24150401,BULLY CREEK RESERVOIR NEAR VALE,NWRFC +OGCN2,23320100,23326906,23320108,Owyhee River near Wildhorse,NWRFC +CGRO3,23773009,23777431,23773011,Cougar Dam below South Fork McKenzie River,NWRFC +WYNW1,23856921,23859017,23856923,WYNOOCHEE RIVER NEAR GRISDALE WA,NWRFC +STOW1,23970763,23974227,23970765,SOUTH FORK TOLT RIVER NEAR CARNATION WA,NWRFC +WODI1,23266994,23272336,23267042,Little Wood River near Carey Little Wood River Dam,NWRFC +PHLO3,24208367,24215523,24208369,Phillips Lake,NWRFC +BEUO3,23412937,23422703,23412939,NORTH FORK MALHEUR RIVER NEAR BEULAH,NWRFC +KERM8,24338704,120054065,24338704,Polson at Flathead Lake,NWRFC +OCHO3,23713982,23717064,23713984,Ochoco Creek Below Ochoco Dam,NWRFC +KBDM8,22878293,22886855,22878297,Libby Dam near Libby below Kootenai River,NWRFC +PSLI1,22988683,22992347,22988683,Priest Lake at Outlet near Coolin,NWRFC +ANDI1,23386379,23393893,23387319,South Fork Boise River below Anderson Ranch Dam Idaho,NWRFC +BUMW1,24423019,24428511,24423025,Bumping Dam below Bumping River,NWRFC +BCLO3,23780511,23783629,23780525,Detroit Lake at N Santiam River,NWRFC +KACW1,24126097,24136753,24126101,Kachess Outflow at Yakima River,NWRFC +SLKW1,23963709,23966965,23963711,SPADA LAKE,NWRFC +DWRI1,23630892,23634782,23630892,Dworshak Reservoir near Ahsahka,NWRFC +OWYO3,24540979,120052938,24540985,Owyhee Dam at Owyhee River,NWRFC +SKBW1,24285508,24286248,24285520,NF SKOKOMISH R NR LWR CUSHMAN DAM NR POTLATCH WA,NWRFC +MAKI1,23238444,23246628,23239256,BIG LOST RIVER BL MACKAY RES NR MACKAY ID,NWRFC +UNYO3,24200221,24204827,24200223,Burnt River below Unity Reservoir,NWRFC +HCRO3,23751946,23756364,23751946,Hills Creek near Oakridge,NWRFC +HCDI1,24219835,24197070,24192562,Snake River at Hells Canyon,NWRFC +KEEW1,24126167,24136769,24126169,Yakima River at Keechelus Outflow,NWRFC +NOXM8,22976068,22983840,22976094,NOXM8,NWRFC +RIMW1,24423299,24428693,24423307,TIETON AT TIETON DAM,NWRFC +WARO3,23413305,23422747,23413307,Malheur River below Warm Springs Reservoir,NWRFC +FRNO3,23763139,23769069,23763141,Long Tom River below Fern Ridge Reservoir,NWRFC +HMAC2,1321058,1320604,1321114,HOMESTAKE RESERVOIR NEAR RED CLIFF 10SW HOMRESCO HOMOUTCO,CBRFC +CLFA3,20733845,3528295,3528925,Colorado River at Lees Ferry Glen Canyon Dam,CBRFC +ARCN5,17040461,120053831,17040459,San Juan River near Archuleta probably Navajo dam,CBRFC +WBWU1,10093214,10091800,10093168,WEBER RIVER NEAR WANSHIP UT,CBRFC +ELCC2,1352642,1351760,1353072,Elkhead Ck Elkhead Reservoir,CBRFC +RUDC2,1326889,1326659,1327495,FRYINGPAN RIVER NEAR RUEDI CO,CBRFC +LLDP1,3773975,3773141,3773977,CHEAT RIVER AT LAKE LYNN DAM tailwater gage,OHRFC diff --git a/data_assimilation_engine/rfc_ingestion/RFC_Sites.py b/data_assimilation_engine/rfc_ingestion/RFC_Sites.py new file mode 100644 index 0000000..603abe1 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/RFC_Sites.py @@ -0,0 +1,138 @@ +############################################################################### +# Module name: RFC_Sites +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: # +# # +# Last modification date: 08/19/2019 # +# # +# Description: manage the RFC sites in CSV format # +# # +############################################################################### + +import csv + +class RFC_Sites: + """ + Store RFC site information + """ + def __init__(self, csvSitefile ): + """ + Initialize the RFC_Sites object with a given + filename + """ + self.source = csvSitefile + + self.gauge = [] + self.gaugedFlowline = [] + self.NHDWaterbodyComID = [] + self.lakeLink = [] + self.SiteName = [] + self.RFC = [] + self.comments = [] + with open( csvSitefile, mode='r') as csvsite_file: + csvsite_reader = csv.DictReader( csvsite_file ) + line_count = 0 + for row in csvsite_reader: + if line_count == 0: + print('Column names are ' + ", ".join(row)) + line_count += 1 + self.gauge.append( row["gage"] ) + self.gaugedFlowline.append( row["gagedFlowline"] ) + self.NHDWaterbodyComID.append( row["NHDWaterbodyComID"] ) + self.lakeLink.append( row["lakeLink"] ) + self.SiteName.append( row["SiteName"] ) + self.RFC.append( row["RFC"] ) + self.comments.append( "Not present!" ) + line_count += 1 + + print('Processed ' + str( line_count ) + ' lines.') + + + @property + def source(self): + return self._source + + @source.setter + def source(self, s): + self._source = s + + @property + def gauge(self): + return self._gauge + + @gauge.setter + def gauge(self, g): + self._gauge=g + + @property + def gaugedFlowline(self): + return self._gaugedFlowline + + @gaugedFlowline.setter + def gaugedFlowline(self, g): + self._gaugedFlowline=g + + @property + def NHDWaterbodyComID(self): + return self._NHDWaterbodyComID + + @NHDWaterbodyComID.setter + def NHDWaterbodyComID(self, n): + self._NHDWaterbodyComID=n + + @property + def lakeLink(self): + return self._lakeLink + + @lakeLink.setter + def lakeLink(self, l): + self._lakeLink=l + + @property + def SiteName(self): + return self._SiteName + + @SiteName.setter + def SiteName(self, s): + self._SiteName=s + + @property + def RFC(self): + return self._RFC + + @RFC.setter + def RFC(self, r): + self._RFC=r + + @property + def comments(self): + return self._comments + + @comments.setter + def comments(self, c): + self._comments=c + + def siteExist(self, s): + return s in self.gauge + + def addComment( self, sta, message ): + try: + ind = self.gauge.index( sta ) + self._comments[ ind ] = message + except ValueError as e: + raise RuntimeError( e ) + + def getSitesByRFC( self, rfcname ): + sites = [] + for s, rfc in zip(self.gauge, self.RFC): + if rfc == rfcname: + sites.append( s ) + return sites + + def getRFCBySite( self, sta ): + for s, rfc in zip(self.gauge, self.RFC): + if s == sta: + return rfc + return None diff --git a/data_assimilation_engine/rfc_ingestion/RFC_Sites_test.py b/data_assimilation_engine/rfc_ingestion/RFC_Sites_test.py new file mode 100644 index 0000000..6320274 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/RFC_Sites_test.py @@ -0,0 +1,17 @@ +import unittest +from RFC_Sites import RFC_Sites + +class RFC_Sites_test(unittest.TestCase): + def test(self): + self.assertTrue(True) + + def testInit(self): + sites = RFC_Sites('./RFC_Reservoir_Locations_for_Forecast_Ingest_into_NWM_All_RFCs.csv' ) + self.assertEqual( len(sites.gauge), 380 ) + self.assertEqual( len(sites.gaugedFlowline), 380 ) + self.assertEqual( len(sites.NHDWaterbodyComID), 380 ) + self.assertEqual( len(sites.lakeLink), 380 ) + self.assertEqual( len(sites.SiteName), 380 ) + +if __name__ == '__main__': + unittest.main() diff --git a/data_assimilation_engine/rfc_ingestion/make_time_series_from_pi_xml.py b/data_assimilation_engine/rfc_ingestion/make_time_series_from_pi_xml.py new file mode 100644 index 0000000..eac55e3 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/make_time_series_from_pi_xml.py @@ -0,0 +1,220 @@ +#! /usr/bin/env python +############################################################################### +# File name: make_time_slice_from_pi_xml.py # +# # +# Author : Zhengtao Cui (Zhengtao.Cui@noaa.gov) # +# # +# Initial version date: # +# # +# Last modification date: 5/30/2019 # +# # +# Description: The driver to create NetCDF time slice files from Army Crops # +# of Engineers real-time observations # +# # +############################################################################### + +import os, sys, time, urllib, getopt, re +import logging +import glob +from string import * +import xml.etree.ElementTree as etree +from datetime import datetime, timedelta +from PI_XML import PI_XML +from RFC_Forecast import RFC_Forecast +from RFCTimeSeries import RFCTimeSeries +from RFCHelper import RFCHelper +from RFC_Sites import RFC_Sites +from EmptyDirOrFileException import EmptyDirOrFileException +#import Tracer + +""" + The driver to parse downloaded ACE XML observations and + create time slices and write to NetCDF files + Author: Zhengtao Cui (Zhengtao.Cui@noaa.gov) + Date: May 30, 2019 +""" +def main(argv): + """ + function to get input arguments + """ + inputdir = '' + try: + opts, args = getopt.getopt(argv,"hi:o:s:",["idir=", "odir=", \ + "sites="]) + except getopt.GetoptError: + print('make_time_slice_from_pi_xml.py -i -o -s ') + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print( \ + 'make_time_slice_from_pi_xml.py -i -o -s ') + sys.exit() + elif opt in ('-i', "--idir"): + inputdir = arg + if not os.path.exists( inputdir ): + raise RuntimeError( 'FATAL Error: inputdir ' + \ + inputdir + ' does not exist!' ) + elif opt in ('-o', "--odir" ): + outputdir = arg + if not os.path.exists( outputdir ): + raise RuntimeError( 'FATAL Error: outputdir ' + \ + outputdir + ' does not exist!' ) + + elif opt in ('-s', "--rfcsitefile" ): + sitefile = arg + if not os.path.exists( sitefile ): + raise RuntimeError( 'FATAL Error: sitefile ' + \ + sitefile + ' does not exist!' ) + + return (inputdir, outputdir, sitefile) + + + + +t0 = time.time() + +logging.basicConfig(format=\ + '%(asctime)s - %(name)s - %(levelname)s - %(message)s',\ + level=logging.INFO) +logger = logging.getLogger(__name__) +formatter = logging.Formatter(\ + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +#logger.setFormatter(formatter) +logger.info( "System Path: " + str( sys.path ) ) + +if __name__ == "__main__": + try: + odir = main(sys.argv[1:]) + except Exception as e: + logger.error("Failed to get program options.", exc_info=True) + +indir = odir[0] +outdir = odir[1] +rfcsitefile = odir[2] +logger.info( 'Input dir is "' + indir + '"') +logger.info( 'Output dir is "' + outdir + '"') +logger.info( 'RFC site file is "' + rfcsitefile + '"') + +# +# Load ACE observed XML discharge data +# + +try: + fcsts = [] + + stationsNotInListFile = { 'ABRFC': set(), \ + 'SERFC': set(), \ + 'LMRFC': set(), \ + 'MARFC': set(), \ + 'NERFC': set(), \ + 'WGRFC': set(), \ + 'MBRFC': set(), \ + 'CNRFC': set(), \ + 'NWRFC': set(), \ + 'NCRFC': set(), \ + 'CBRFC': set(), \ + 'OHRFC': set(), \ + 'APRFC': set(), \ + 'UNKNOWN': set() } + + if not os.path.isdir( indir ): +# raise SystemExit( "FATAL ERROR: " + indir + \ + raise RuntimeError( "FATAL ERROR: " + indir + \ + " is not a directory or does not exist. ") + + YYYYMMDDHH = '[0-9]{4}(0[1-9]|1[0-2])(0[1-9]|[1-2][0-9]|3[0-1])(2[0-3]|[01][0-9])' + YYYYMMDDHHMMSS = YYYYMMDDHH + '(0[0-9]|[1-5][0-9]){2}' + ABRFCPattern = '^(.*/)?' + YYYYMMDDHH + '_RES_NWM_pixml_export.' + YYYYMMDDHHMMSS + NWRFCPattern = '^(.*/)?' + YYYYMMDDHH + '_QINE_NWM_Res_export.' + YYYYMMDDHHMMSS + rfcsites = RFC_Sites( rfcsitefile ) + + thirtyMinAgo = datetime.now() - timedelta(minutes=30) +# for file in os.listdir( indir ): + for file in sorted( glob.glob( indir + '/*' ), key=os.path.getmtime ): +# mtime=datetime.fromtimestamp(\ +# os.stat(os.path.join( indir, file)).st_mtime) +# m1 = re.match( ABRFCPattern, file ) +# m2 = re.match( NWRFCPattern, file ) +# if file.endswith( ".xml" ) or m1 or m2: +# if mtime > thirtyMinAgo: + logger.info( 'Reading ' + file + ' ... ' ) + try: + pixml = PI_XML( file, rfcsites ) +# rfc_series = pixml.getReserviorForecastWithT0() + t1 = time.time() + rfc_series = pixml.getReserviorObservedForecastCombinedWithT0() + logger.info( "Processing PI XML file done: " + \ + "{0:.1f}".format( (time.time() - t1) ) + \ + " seconds" ) + for s in rfc_series: + if rfcsites.siteExist( s.get5CharStationID() ): + if not s.isEmpty(): + fcsts.append( s ) + rfcsites.addComment( s.get5CharStationID(), \ + "OK") + else: + rfcsites.addComment( s.get5CharStationID(), \ + "Empty") + else: + rfcname = re.search( "[A-Z][A-Z]RFC", file).group() if \ + re.search( "[A-Z][A-Z]RFC", file) else "UNKNOWN" + if not s.isEmpty(): + stationsNotInListFile[ rfcname ].add( \ + s.get5CharStationID() ) + + else: + stationsNotInListFile[ rfcname ].add( \ + s.get5CharStationID() + "*" ) + + except Exception as e: + logger.warn( repr( e ), exc_info=True ) + continue + + if not fcsts: + raise EmptyDirOrFileException( "Input directory " + indir + \ + " has no PI XML files or no forecast data" + " in PI XML files!" ) +# raise SystemExit(0) + for s, r, c in zip( rfcsites.gauge, rfcsites.RFC, rfcsites.comments ): + logger.info( 'Site info: ' + s + " " + r + " " + c ) + logger.info( 'Sits not in the site file:' ) + for k, v in stationsNotInListFile.items(): + if v: + for e in v: + logger.info( ' ' + e + (': ' if len(e) == 6 else ' : ') + k ) + + logger.info( 'Note: \'*\' - flow values are missing.' ) + +except EmptyDirOrFileException as e: + logger.warning( str(e), exc_info=True) + sys.exit(0) +except Exception as e: + logger.error("Failed to load PI XML files:" + str(e), exc_info=True) + sys.exit(3) + +try: + helper = RFCHelper( fcsts ) + + logger.info( 'Earliest time in PI XML: ' + \ + helper.timePeriodForAll()[0].isoformat() ) + logger.info( 'Latest time in PI XML: ' + \ + helper.timePeriodForAll()[1].isoformat() ) + + # + # Create time slices from loaded observations + # + # Set time resolution to 60 minutes + # and + # Write time slices to NetCDF files + # + timeslices = helper.makeAllTimeSeries( outdir, 'RFCTimeSeries.ncdf' ) +except Exception as e: + logger.error("Failed to make time series:" + str(e), exc_info=True) + logger.error("Input dir = " + indir, exc_info=True) + sys.exit(3) + +logger.info( "Total number of timeseries: " + str( timeslices ) ) +logger.info( "Program finished in: " + \ + "{0:.1f}".format( (time.time() - t0) / 60.0 ) + \ + " minutes" ) + diff --git a/data_assimilation_engine/rfc_ingestion/nwmrfccopy.sh b/data_assimilation_engine/rfc_ingestion/nwmrfccopy.sh new file mode 100644 index 0000000..a1e6eba --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/nwmrfccopy.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +############################################################################### +# Program Name: nwmrfccopy.sh # +# # +# Author(s)/Contact(s): NWC # +# # +# copy files to and from the $COM and $DBN alert directories # +# # +# Input: directory in $COM # +# # +# Output: files # +# # +# For non-fatal errors output is witten to $DATA/LOGS # +# # +# Origination Jun, 2019 # +# # +############################################################################### +# --------------------------------------------------------------------------- # + +####################################### +# nwm_rfc_copy and nwm_rfc_postcopy utilize +# multiple cores to speed up file copying +####################################### +function nwm_rfc_copy () { + msg="Begin copying file at `date`" + postmsg "${pgmout}" "${msg}" + + dir=$1 + echo "#!/usr/bin/env bash" > $DATA/nwmcopyscript + # + # All timeslices in previous 3 days + # + for file in $COMIN/${dir}/*.RFCTimeSeries.ncdf \ + $COMINm1/${dir}/*.RFCTimeSeries.ncdf \ + $COMINm2/${dir}/*.RFCTimeSeries.ncdf \ + $COMINm3/${dir}/*.RFCTimeSeries.ncdf + do + test -f "$file" || continue + echo "cpfs $file $DATA/$(basename $file)" >> $DATA/nwmcopyscript + done + + chmod 755 $DATA/nwmcopyscript + + #aprun -j1 -n$((NODES*NCORES)) -N${NCORES} cfp $DATA/nwmcopyscript + ${CFPCOMMAND} $DATA/nwmcopyscript + export err=$?; err_chk + + msg="Ending copy file at `date`" + postmsg "$pgmout" "$msg" + +} + +function nwm_rfc_postcopy () { + msg="Begin post copying file at `date`" + postmsg "${pgmout}" "${msg}" + + dir=$1 + suffix=$2 + #export COMOUT_ROOT=${COMOUT_ROOT:-${COMROOT}/${NET}/${envir}} + export COMOUT_ROOT=${COMOUT_ROOT:-${COMROOT}/${NET}/${nwm_ver}} + echo "#!/usr/bin/env bash" > $DATA/nwmpostcopyscript + for file in $DATA/*.${suffix} + do + test -f "$file" || continue + # + # NOTE: Here, we assume $COMOUT == $COMIN, otherwise, it doesn't work. + # + Outdir=${COMOUT_ROOT}/${RUN}.$(basename $file | cut -c1-4 )$(basename \ + $file | cut -c6-7 )$(basename $file | cut -c9-10)/${dir} + + if [ ! -e $Outdir ]; then + mkdir -p $Outdir + fi + + copyandalert=true + + # + # if the file exists and was not changed, don't copy and alert + # + if [ -e ${Outdir}/$(basename $file) ]; then + diff ${file} ${Outdir}/$(basename $file) > /dev/null 2>&1 + if [ $? -eq 0 ]; then + copyandalert=false + fi + fi + + if [[ "$copyandalert" == true ]]; then + echo "cpfs ${file} ${Outdir}/$(basename $file); if [ "$SENDDBN" = YES ]; then $DBNROOT/bin/dbn_alert MODEL ${DBN_ALERT_TYPE} $job ${Outdir}/$(basename $file); fi" >> $DATA/nwmpostcopyscript + fi + + done + + chmod 755 $DATA/nwmpostcopyscript + + #aprun -j1 -n$((NODES*NCORES)) -N${NCORES} cfp $DATA/nwmpostcopyscript + ${CFPCOMMAND} $DATA/nwmpostcopyscript + export err=$?; err_chk + + msg="Ending post copy file at `date`" + postmsg "$pgmout" "$msg" +} diff --git a/data_assimilation_engine/rfc_ingestion/testdata/abrfc/2019112712_ABRFC_RES_NWM_pixml_export.xml b/data_assimilation_engine/rfc_ingestion/testdata/abrfc/2019112712_ABRFC_RES_NWM_pixml_export.xml new file mode 100644 index 0000000..bcdb033 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/abrfc/2019112712_ABRFC_RES_NWM_pixml_export.xml @@ -0,0 +1,4297 @@ + + + 0.0 + +
+ instantaneous + MRIT2 + QINE + ALLQPF + 1 + + + + + -999 + LAKE MEREDITH-SANFORD 1NW + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PDAC2 + QINE + ALLQPF + 1 + + + + -999 + PDAC2 + CFS +
+ + +
+ instantaneous + TDDC2 + QINE + ALLQPF + 1 + + + + + -999 + TDDC2 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TLRC2 + QINE + ALLQPF + 1 + + + + -999 + TLRC2 + CFS +
+ + +
+ instantaneous + TWLC2 + QINE + ALLQPF + 1 + + + + -999 + TWLC2 + CFS +
+ + +
+ instantaneous + TRQC2 + QINE + ALLQPF + 1 + + + + -999 + TRQC2 + CFS +
+ + +
+ instantaneous + TQLC2 + QINE + ALLQPF + 1 + + + + -999 + TQLC2 + CFS +
+ + +
+ instantaneous + JMCC2 + QINE + ALLQPF + 1 + + + + + -999 + JMCC2 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DWSC2 + QINE + ALLQPF + 1 + + + + -999 + DWSC2 + CFS +
+ + +
+ instantaneous + HTRK1 + QINE + ALLQPF + 1 + + + + + -999 + HTRK1 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CHNK1 + QINE + ALLQPF + 1 + + + + + -999 + CHNK1 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + EDRK1 + QINE + ALLQPF + 1 + + + + + -999 + EL DORADO 4NE - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + KAWO2 + QINE + ALLQPF + 1 + + + + + -999 + KAW CITY - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GSPO2 + QINE + ALLQPF + 1 + + + + + -999 + GSPO2 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TRLK1 + QINE + ALLQPF + 1 + + + + + -999 + TRLK1 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FLLK1 + QINE + ALLQPF + 1 + + + + + -999 + FALL RIVER 4NW - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ECLK1 + QINE + ALLQPF + 1 + + + + + -999 + ELK CITY DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BIGK1 + QINE + ALLQPF + 1 + + + + + -999 + BIG HILL DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MLBK1 + QINE + ALLQPF + 1 + + + + + -999 + MARION 3NW - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CNGK1 + QINE + ALLQPF + 1 + + + + + -999 + COUNCIL GROVE 1NW - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + JRLK1 + QINE + ALLQPF + 1 + + + + + -999 + JOHN REDMOND DAM BURLINGTON 3N + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PENO2 + QINE + ALLQPF + 1 + + + + + -999 + PENSACOLA DAM-DISNEY 1SW + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MFDO2 + QINE + ALLQPF + 1 + + + + + -999 + LAKE HUDSON - MARKHAM FRY W + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SPAO2 + QINE + ALLQPF + 1 + + + + + -999 + SPAVINAW LAKE (LOWER) + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FSLO2 + QINE + ALLQPF + 1 + + + + + -999 + FSLO2 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CNLO2 + QINE + ALLQPF + 1 + + + + + -999 + CANTON 2NW - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ACDO2 + QINE + ALLQPF + 1 + + + + + -999 + ARCADIA + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NRMO2 + QINE + ALLQPF + 1 + + + + + -999 + NORMAN DAM - NORMAN 13E + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CPLO2 + QINE + ALLQPF + 1 + + + + + -999 + COPAN DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HULO2 + QINE + ALLQPF + 1 + + + + + -999 + HULAH 2W - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BIRO2 + QINE + ALLQPF + 1 + + + + + -999 + BIRO2 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SKLO2 + QINE + ALLQPF + 1 + + + + + -999 + SKLO2 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TENO2 + QINE + ALLQPF + 1 + + + + + -999 + GORE 7NE-TENKILLER FERRY DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + KEYO2 + QINE + ALLQPF + 1 + + + + + -999 + KEYSTONE DAM-SAND SPRINGS 8W + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HEYO2 + QINE + ALLQPF + 1 + + + + + -999 + KELLYVILLE 4W (HEYBURN LAKE) + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + OOLO2 + QINE + ALLQPF + 1 + + + + + -999 + OOLOGAH 2SE - DAM (BLO DAM) + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GIBO2 + QINE + ALLQPF + 1 + + + + + -999 + OKAY 5E-FORT GIBSON DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + EUFO2 + QINE + ALLQPF + 1 + + + + + -999 + EUFAULA 12E + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + VBRA4 + QINE + ALLQPF + 1 + + + + + -999 + VBRA4 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WSLO2 + QINE + ALLQPF + 1 + + + + + -999 + WISTER 2S - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + OZGA4 + QINE + ALLQPF + 1 + + + + + -999 + OZGA4 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TWELQ + QINE + ALLQPF + 1 + + + + -999 + TWELQ + CFS +
+ + +
+ instantaneous + DARA4 + QINE + ALLQPF + 1 + + + + -999 + DARA4 + CFS +
+ + +
+ instantaneous + TODA4 + QINE + ALLQPF + 1 + + + + -999 + TODA4 + CFS +
+ + +
+ instantaneous + FOSO2 + QINE + ALLQPF + 1 + + + + + -999 + FOSO2 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FTCO2 + QINE + ALLQPF + 1 + + + + + -999 + FORT COBB 4NW + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CMTO2L + QINE + ALLQPF + 1 + + + + -999 + CMTO2L + CFS +
+ + +
+ instantaneous + CMTO2R + QINE + ALLQPF + 1 + + + + -999 + CMTO2R + CFS +
+ + +
+ instantaneous + NKHO2L + QINE + ALLQPF + 1 + + + + -999 + NKHO2L + CFS +
+ + +
+ instantaneous + NKHO2R + QINE + ALLQPF + 1 + + + + -999 + NKHO2R + CFS +
+ + +
+ instantaneous + ARBO2 + QINE + ALLQPF + 1 + + + + + -999 + ROCK CREEK + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ALTO2 + QINE + ALLQPF + 1 + + + + + -999 + GRANITE 7SE + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MTNO2 + QINE + ALLQPF + 1 + + + + + -999 + MOUNTAIN PARK 4N + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SYOT2 + QINE + ALLQPF + 1 + + + + + -999 + SEYMOUR 16NNE + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WRLO2 + QINE + ALLQPF + 1 + + + + + -999 + WAURIKA 6N - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PCLO2 + QINE + ALLQPF + 1 + + + + + -999 + PINE CREEK DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BKDO2 + QINE + ALLQPF + 1 + + + + + -999 + BROKEN BOW 9NE - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MYST2 + QINE + ALLQPF + 1 + + + + + -999 + PAT MAYES DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MGCO2 + QINE + ALLQPF + 1 + + + + + -999 + FARRIS 3N + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DSNT2 + QINE + ALLQPF + 1 + + + + + -999 + DENISON 4NNW - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CYDO2 + QINE + ALLQPF + 1 + + + + + -999 + SARDIS + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HGLO2 + QINE + ALLQPF + 1 + + + + + -999 + HUGO 7E - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + EGLN5 + QINE + ALLQPF + 1 + + + + + -999 + EGLN5 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + EGCN5 + QINE + ALLQPF + 1 + + + + -999 + EGCN5 + CFS +
+ + +
+ instantaneous + CNCN5 + QINE + ALLQPF + 1 + + + + + -999 + CNCN5 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + UTEN5 + QINE + ALLQPF + 1 + + + + + -999 + UTEN5 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + RTBT2 + QINE + ALLQPF + 1 + + + + + -999 + RTBT2 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CLRC2 + QINE + ALLQPF + 1 + + + + -999 + CLRC2 + CFS +
+ + +
+ instantaneous + CCBC2 + QINE + ALLQPF + 1 + + + + -999 + CCBC2 + CFS +
+ + +
+ instantaneous + PBAC2 + QINE + ALLQPF + 1 + + + + -999 + PUEBLO 6W - DAM + CFS +
+ + +
+ instantaneous + TBTC2 + QINE + ALLQPF + 1 + + + + + -999 + TWO BUTTES RESERVOIR + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + OPLO2 + QINE + ALLQPF + 1 + + + + + -999 + OPTIMA DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SPRT2 + QINE + ALLQPF + 1 + + + + + -999 + SPRT2 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + INLO2 + QINE + ALLQPF + 1 + + + + -999 + INLO2 + CFS +
+ + +
+ instantaneous + WFLO2 + QINE + ALLQPF + 1 + + + + -999 + WEBBERS FALLS 3NW-LD 16 + CFS +
+ + +
+ instantaneous + WAGO2 + QINE + ALLQPF + 1 + + + + -999 + WAGO2 + CFS +
+ + +
+ instantaneous + KERO2 + QINE + ALLQPF + 1 + + + + + -999 + LD 15 - SALLISAW 8S + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BARA4 + QINE + ALLQPF + 1 + + + + -999 + BARA4 + CFS +
+ + +
+ instantaneous + MAYO2 + QINE + ALLQPF + 1 + + + + -999 + MAYO2 + CFS +
+ + +
+ instantaneous + DRDA4 + QINE + ALLQPF + 1 + + + + -999 + DRDA4 + CFS +
+ + +
+ instantaneous + BMTA4 + QINE + ALLQPF + 1 + + + + + -999 + BLUE MOUNTAIN DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BMRA4 + QINE + ALLQPF + 1 + + + + -999 + BMRA4 + CFS +
+ + +
+ instantaneous + NMLA4 + QINE + ALLQPF + 1 + + + + -999 + NIMROD 5W - DAM + CFS +
+ + +
+ instantaneous + NMBA4 + QINE + ALLQPF + 1 + + + + + -999 + NMBA4 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MAUA4 + QINE + ALLQPF + 1 + + + + + -999 + MAUA4 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CONA4 + QINE + ALLQPF + 1 + + + + + -999 + CONA4 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LNDA4 + QINE + ALLQPF + 1 + + + + -999 + LNDA4 + CFS +
+ + +
+ instantaneous + DTLA4 + QINE + ALLQPF + 1 + + + + -999 + DTLA4 + CFS +
+ + +
+ instantaneous + AMCT2 + QINE + ALLQPF + 1 + + + + -999 + AMARILLO CITY LAKE + CFS +
+ + +
+ instantaneous + BLAT2 + QINE + ALLQPF + 1 + + + + -999 + BLAT2 + CFS +
+ + +
+ instantaneous + TGLT2 + QINE + ALLQPF + 1 + + + + -999 + TGLT2 + CFS +
+ + +
+ instantaneous + MKZT2 + QINE + ALLQPF + 1 + + + + + -999 + MKZT2 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GRET2 + QINE + ALLQPF + 1 + + + + + -999 + GRET2 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LTLO2 + QINE + ALLQPF + 1 + + + + + -999 + LTLO2 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ELWO2 + QINE + ALLQPF + 1 + + + + + -999 + ELWO2 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DDET2 + QINE + ALLQPF + 1 + + + + + -999 + DDET2 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LWTT2 + QINE + ALLQPF + 1 + + + + + -999 + LWTT2 + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ARRT2 + QINE + ALLQPF + 1 + + + + + -999 + LAKE ARROWHEAD-HENRIETTA 11SW + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DQDA4 + QINE + ALLQPF + 1 + + + + + -999 + DEQUEEN 4NW - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DQTA4 + QINE + ALLQPF + 1 + + + + -999 + DQTA4 + CFS +
+ + +
+ instantaneous + GLLA4 + QINE + ALLQPF + 1 + + + + + -999 + GILLHAM 6NE - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GLTA4 + QINE + ALLQPF + 1 + + + + -999 + GLTA4 + CFS +
+ + +
+ instantaneous + DIEA4 + QINE + ALLQPF + 1 + + + + + -999 + DIERKS 5NW - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + AHDA4 + QINE + ALLQPF + 1 + + + + + -999 + ASHDOWN 10E + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MWTA4 + QINE + ALLQPF + 1 + + + + -999 + MWTA4 + CFS +
+ + +
+ instantaneous + ATKO2 + QINE + ALLQPF + 1 + + + + + -999 + ATOKA 5NE - DAM + CFS + 2019-11-27 + 13:10:01 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + UCBO2P + QINE + ALLQPF + 1 + + + + -999 + UCBO2P + CFS +
+ + +
+ instantaneous + DLWO2P + QINE + ALLQPF + 1 + + + + -999 + DLWO2P + CFS +
+ + +
+ instantaneous + CNEO2L + QINE + ALLQPF + 1 + + + + -999 + CNEO2L + CFS +
+ + +
+ instantaneous + CNEO2R + QINE + ALLQPF + 1 + + + + -999 + CNEO2R + CFS +
+ + +
+ instantaneous + CAYO2P + QINE + ALLQPF + 1 + + + + -999 + CAYO2P + CFS +
+ + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/cbrfc/201911281800_CBRFC_Reservoir_Export.xml b/data_assimilation_engine/rfc_ingestion/testdata/cbrfc/201911281800_CBRFC_Reservoir_Export.xml new file mode 100644 index 0000000..f857630 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/cbrfc/201911281800_CBRFC_Reservoir_Export.xml @@ -0,0 +1,15559 @@ + + + 0.0 + +
+ instantaneous + STIU1 + QINE + + + + + NaN + STRAWBERRY - STRAWBERRY RES + CFS + 2019-11-28 + 14:38:26 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + STAU1 + QINE + + + + + NaN + STRAWBERRY - STARVATION RES, DUCHESNE, NR + CFS + 2019-11-28 + 14:38:26 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LAAU1 + QINE + + + + + NaN + LAKE FORK- MOON LAKE RES, MTN HOME, NR + CFS + 2019-11-28 + 14:38:26 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SFSU1 + QINE + + + + + NaN + PRICE - SCOFIELD RES, SCOFIELD, NR + CFS + 2019-11-28 + 14:38:26 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GBFW4 + QINE + + + + + NaN + GREEN - FONTENELLE RES, BLO + CFS + 2019-11-28 + 14:42:47 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BIGW4 + QINE + + + + + NaN + BIG SANDY - BIG SANDY RES, FARSON, NR + CFS + 2019-11-28 + 14:42:47 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + VIVW4 + QINE + + + + + NaN + HAMS FORK - VIVA NAUGHTON RES + CFS + 2019-11-28 + 14:42:47 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MCWW4 + QINE + + + + + NaN + MEEKS CABIN DAM + CFS + 2019-11-28 + 14:42:47 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GRZU1 + QINE + + + + + NaN + GREEN - GREENDALE, NR + CFS + 2019-11-28 + 14:42:47 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ELLU1 + QINE + + + + + NaN + HUNTINGTON CK - ELECTRIC LAKE + CFS + 2019-11-28 + 14:39:47 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + JOVU1 + QINE + + + + + NaN + SEELEY CK - JOES VLY RES, ORANGEVILLE, NR + CFS + 2019-11-28 + 14:39:47 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MLSU1 + QINE + + + + + NaN + MILLSITE DAM + CFS + 2019-11-28 + 14:39:47 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WCKC2 + QINE + + + + + NaN + WILLOW CK - GRANBY, NR, WILLOW CK RES, BLO + CFS + 2019-11-28 + 14:40:17 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CBGC2 + QINE + + + + + NaN + COLORADO - GRANBY, NR + CFS + 2019-11-28 + 14:40:17 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WFRC2 + QINE + + + + + NaN + WILLIAMS FORK - WILLIAMS FORK RES, BLO + CFS + 2019-11-28 + 14:40:17 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BLRC2 + QINE + + + + + NaN + BLUE - DILLON, BLO + CFS + 2019-11-28 + 14:40:17 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BGMC2 + QINE + + + + + NaN + BLUE - GREEN MTN RES, BLO + CFS + 2019-11-28 + 14:40:17 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + RRGC2 + QINE + + + + + NaN + RIFLE CK - RIFLE GAP RESERVOIR + CFS + 2019-11-28 + 14:43:27 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + VEGC2 + QINE + + + + + NaN + PLATEAU CK - VEGA RES, COLLBRAN, NR + CFS + 2019-11-28 + 14:43:27 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TRBC2 + QINE + + + + + NaN + TAYLOR - TAYLOR PARK RES, BLO + CFS + 2019-11-28 + 14:40:57 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BMDC2 + QINE + + + + + NaN + GUNNISON - BLUE MESA RES + CFS + 2019-11-28 + 14:40:57 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MPSC2 + QINE + + + + + NaN + GUNNISON - MORROW POINT RES + CFS + 2019-11-28 + 14:40:57 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MDCC2 + QINE + + + + + NaN + MUDDY CK - PAONIA RES, BLO + CFS + 2019-11-28 + 14:40:57 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CRFC2 + QINE + + + + + NaN + CRFC2OAF + CFS + 2019-11-28 + 14:40:57 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LPVC2 + QINE + + + + + NaN + LOS PINOS - BAYFIELD, NR, VALLECITO RES, BLO + CFS + 2019-11-28 + 14:42:17 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FRVC2 + QINE + + + + + NaN + FLORIDA - DURANGO, NR, LEMON RES, BLO + CFS + 2019-11-28 + 14:42:17 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LYMA3 + QINE + + + + + NaN + LITTLE COLORADO - LYMAN LAKE, ST. JOHNS, NR + CFS + 2019-11-28 + 19:56:06 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GUUU1 + QINE + + + + + NaN + GUNLOCK RESERVOIR + CFS + 2019-11-28 + 19:57:26 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PCON2 + QINE + + + + + NaN + PINE CANYON DAM-CALIENTE 16SE + CFS + 2019-11-28 + 19:57:46 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MVDN2 + QINE + + + + + NaN + MATHEW VLY DAM-CALIENTE 20SE + CFS + 2019-11-28 + 19:57:46 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BWAA3 + QINE + + + + + NaN + BILL WILLIAMS - ALAMO DAM, BLO + CFS + 2019-11-28 + 19:58:26 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LKSA3 + QINE + + + + + NaN + LAKE MEAD + CFS + 2019-11-28 + 15:09:43 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CBDN2 + QINE + + + + + NaN + COLORADO - DAVIS DAM, BLO + CFS + 2019-11-28 + 15:09:43 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HORA3 + QINE + + + + + NaN + HORSESHOE RESERVOIR + CFS + 2019-11-28 + 20:01:47 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + VDBA3 + QINE + + + + + NaN + VERDE - BARTLETT DAM, BLO + CFS + 2019-11-28 + 20:01:47 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + RSVA3 + QINE + + + + + NaN + SALT - ROOSEVELT RESERVOIR, AT + CFS + 2019-11-28 + 20:07:38 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SMDA3 + QINE + + + + + NaN + SALT - STEWART MTN DAM, BLO + CFS + 2019-11-28 + 20:07:38 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GCDA3 + QINE + + + + + NaN + GILA - COOLIDGE DAM, BLO + CFS + 2019-11-28 + 20:06:08 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DCRU1 + QINE + + + + + NaN + PROVO - DEER CK RES + CFS + 2019-11-28 + 14:44:28 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + UTLU1 + QINE + + + + + NaN + JORDAN - UTAH LAKE, PROVO, NR + CFS + 2019-11-28 + 14:44:28 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ECWU1 + QINE + + + + + NaN + WEBER - ECHO + CFS + 2019-11-28 + 14:46:08 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CRDU1 + QINE + + + + + NaN + LOST CK - CROYDEN, NR + CFS + 2019-11-28 + 14:46:08 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ECCU1 + QINE + + + + + NaN + E CANYON CR - MORGAN, NR + CFS + 2019-11-28 + 14:46:08 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + OPDU1 + QINE + + + + + NaN + OGDEN - PINEVIEW DAM, BLO, HUNTSVILLE, NR + CFS + 2019-11-28 + 14:46:08 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BRWU1 + QINE + + + + + NaN + BEAR - WOODRUFF, NR, WOODRUFF NARROWS RES, BLO + CFS + 2019-11-28 + 14:48:38 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BLZI1 + QINE + + + + + NaN + BEAR LAKE OUTLET CANAL NR PARIS + CFS + 2019-11-28 + 14:48:38 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BEAI1 + QINE + + + + + NaN + BEAR - ALEXANDER, AT + CFS + 2019-11-28 + 14:48:38 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BBOI1 + QINE + + + + + NaN + BEAR R BLO UT P AND L AT ONEIDA + CFS + 2019-11-28 + 14:48:38 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HLZU1 + QINE + + + + + NaN + LITTLE BEAR - HYRUM RESERVOIR + CFS + 2019-11-28 + 14:48:38 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + OCEU1 + QINE + + + + + NaN + OTTER CK - OTTER CREEK RES, ANTIMONY, NR + CFS + 2019-11-28 + 14:37:56 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MYSU1 + QINE + + + + + NaN + SEVIER - PIUTE DAM, BLO, MARYSVALE, NR + CFS + 2019-11-28 + 14:37:56 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/cnrfc/201911290600_CNRFC_Reservoir_Export_for_NWM.xml b/data_assimilation_engine/rfc_ingestion/testdata/cnrfc/201911290600_CNRFC_Reservoir_Export_for_NWM.xml new file mode 100644 index 0000000..1cbde7e --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/cnrfc/201911290600_CNRFC_Reservoir_Export_for_NWM.xml @@ -0,0 +1,19358 @@ + + + 0.0 + +
+ instantaneous + LLKC1 + RQOT + + + + -999 + TRINITY - LEWISTON DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + IRGC1 + RQOT + + + + -999 + KLAMATH - IRON GATE DAM, + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LAMC1 + RQOT + + + + -999 + EF RUSSIAN - COYOTE DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WSDC1 + RQOT + + + + -999 + DRY CK - WARM SPRINGS DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PORC1 + RQOT + + + + -999 + POTTER VLY DIVERSION FR LK MENDOCINO + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SNRC1 + RQOT + + + + -999 + SAN ANTONIO-SAN ANTONIO DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + COYC1 + RQOT + + + + -999 + COYOTE CK - COYOTE RES + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LEXC1 + RQOT + + + + -999 + LOS GATOS CK - LEXINGTON RES + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NACC1 + RQOT + + + + -999 + NACIMIENTO-NACIMIENTO DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ANDC1 + RQOT + + + + -999 + COYOTE CK - ANDERSON DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ALRC1 + RQOT + + + + -999 + Almaden Res + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GUAC1 + RQOT + + + + -999 + Guadalupe Res + KCFS +
+ + +
+ instantaneous + CADC1 + RQOT + + + + -999 + Calero Res + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ELPC1 + RQOT + + + + -999 + SAN DIEGO - EL CAPITAN D + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TWDC1 + RQOT + + + + -999 + CUYAMA - TWITCHELL DAM + KCFS +
+ + +
+ instantaneous + PYMC1 + RQOT + + + + -999 + PYRAMID LK - GORMAN + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LKPC1 + RQOT + + + + -999 + LAKE PIRU - NR PIRU + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SVIC1 + RQOT + + + + -999 + SAN VICENTE CK - SAN VICE + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CCHC1 + RQOT + + + + -999 + SANTA YNEZ-CACHUMA DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SRWC1 + RQOT + + + + -999 + SANTA ANA - MENTONE, NR + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MVDC1 + RQOT + + + + -999 + MOJAVE DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HAWC1 + RQOT + + + + -999 + SAN LUIS REY-LAKE HENSHAW + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + KWKC1 + RQOT + + + + -999 + SACRAMENTO - KESWICK DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BLBC1 + RQOT + + + + -999 + STONY CK - BLACK BUTTE D + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WHSC1 + RQOT + + + + -999 + CLEAR CK - WHISKEYTOWN + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SGEC1 + RQOT + + + + -999 + STONY CK - STONY GORGE RES + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + EPRC1 + RQOT + + + + -999 + LITTLE STONY CK - EAST PARK RES + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CFWC1 + RQOT + + + + -999 + BEAR - CAMP FAR WEST + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HLEC1 + RQOT + + + + -999 + YUBA - ENGLEBRIGHT RESERVOIR + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LEDC1 + RQOT + + + + -999 + LK ELEANOR DIV TUNNEL - C + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ORDC1 + RQOT + + + + -999 + FEATHER - OROVILLE DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NBBC1 + RQOT + + + + -999 + NF YUBA - NEW BULLARDS BAR + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NIMC1 + RQOT + + + + -999 + AMERICAN - NIMBUS DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FMDC1 + RQOT + + + + -999 + FRENCH MEADOWS RES + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + INVC1 + RQOT + + + + -999 + NF CACHE CK-INDIAN VLY D + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CLKC1 + RQOT + + + + -999 + CACHE CK - CLEAR LAKE + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LBEC1 + RQOT + + + + -999 + PUTAH CK - LAKE BERRYESSA + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SWRC1 + RQOT + + + + -999 + SACRAMENTO-SACTO WEIR + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ISAC1 + RQOT + + + + -999 + KERN - ISABELLA DAM, BLO + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SCSC1 + RQOT + + + + -999 + TULE - SUCCESS DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PFTC1 + RQOT + + + + -999 + KINGS - PINE FLAT DAM, BL + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TMDC1 + RQOT + + + + -999 + KAWEAH - TERMINUS DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + KRCC1 + RQOT + + + + -999 + KINGS - CRESCENT WEIR, BLO + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LNGC1 + RQOT + + + + -999 + TUOLUMNE - LA GRANGE, NR + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HETC1 + RQOT + + + + -999 + TUOLUMNE - HETCH HETCHY, + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HIDC1 + RQOT + + + + -999 + FRESNO - HIDDEN DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FRAC1 + RQOT + + + + -999 + SAN JOAQUIN - FRIANT DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BIFC1 + RQOT + + + + -999 + SAN JOAQ-BIFURCATION, BLO + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + KNFC1 + RQOT + + + + -999 + STANISLAUS-GOODWIN DAM, B + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CHVC1 + RQOT + + + + -999 + CHERRY CK - CHERRY VLY D + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MSWC1 + RQOT + + + + -999 + MERCED - MC SWAIN DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BHNC1 + RQOT + + + + -999 + CHOWCHILLA - BUCHANAN DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FKCC1 + RQOT + + + + -999 + FRIANT-KERN CANAL-FRIANT + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MACC1 + RQOT + + + + -999 + MADERA CANAL - FRIANT + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FRGC1 + RQOT + + + + -999 + LITTLEJOHNS CK - FARMINGTON + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NHGC1 + RQOT + + + + -999 + CALAVERAS - NEW HOGAN DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CMCC1 + RQOT + + + + -999 + MOKELUMNE - CAMANCHE DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BPRC1 + RQOT + + + + -999 + EF WALKER - BRIDGEPORT RES + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + STPC1 + RQOT + + + + -999 + LTL TRUCKEE - STAMPEDE DA + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TAHC1 + RQOT + + + + -999 + TAHOE CITY + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCVC1 + RQOT + + + + -999 + LTL TRUCKEE - BOCA DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DNRC1 + RQOT + + + + -999 + DONNER CK - DONNER LK DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ILAC1 + RQOT + + + + -999 + INDEPENDENCE LK - TRUCKEE, NR + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MTSC1 + RQOT + + + + -999 + MARTIS CK - MARTIS RESERVOIR + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PSRC1 + RQOT + + + + -999 + PROSSER CK - PROSSER CK DAM + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TOPN2 + RQOT + + + + -999 + TOPAZ LAKE + KCFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/lmrfc/201911290111_LMRFC_Reservoir_export.xml b/data_assimilation_engine/rfc_ingestion/testdata/lmrfc/201911290111_LMRFC_Reservoir_export.xml new file mode 100644 index 0000000..ba74af0 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/lmrfc/201911290111_LMRFC_Reservoir_export.xml @@ -0,0 +1,16144 @@ + + + 0.0 + +
+ instantaneous + ARKM6OUT + SQIN + + + + + -9999 + ARKABUTLA DAM + CFS + 2019-11-29 + 13:21:55 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCDA1OUT + SQIN + + + + + -9999 + BIG CK DAM + CFS + 2019-11-29 + 13:16:33 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCDN7OUT + SQIN + + + + -9999 + BEAR CREEK AT BEAR CREEK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCEA4OUT + SQIN + + + + + -9999 + BAYOU BODCAU AT L. ERLING + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCRA1OUT + SQIN + + + + -9999 + BEAR CK/BEAR CK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BKDL1OUT + SQIN + + + + -9999 + BUNDICK LK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BMDA4OUT + SQIN + + + + + -9999 + BLAKLEY MT DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BOOT1OUT + SQIN + + + + + -9999 + BOOT1OUT + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BRDG1OUT + SQIN + + + + -9999 + BLUE RIDGE DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BSGA4OUT + SQIN + + + + + -9999 + BULL SHOALS + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BSLT2OUT + SQIN + + + + + -9999 + BOB SANDLIN LAKE + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BVGA4OUT + SQIN + + + + + -9999 + BEAVER DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CCRA1OUT + SQIN + + + + -9999 + CEDAR CK/CEDAR CK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CHAN7OUT + SQIN + + + + + -9999 + CHATUGE DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CHEN7OUT + SQIN + + + + + -9999 + CHEN7OUT + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CLRM7OUT + SQIN + + + + + -9999 + Clearwater + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CRKT1OUT + SQIN + + + + + -9999 + CRKT1OUT + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CRLL1OUT + SQIN + + + + + -9999 + SHREVEPORT/CROSS LK + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DGDA4OUT + SQIN + + + + + -9999 + DEGRAY DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DUGT1OUT + SQIN + + + + + -9999 + F BROAD/DOUGLAS DAM + CFS + 2019-11-29 + 13:29:06 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ENDM6OUT + SQIN + + + + + -9999 + ENID DAM + CFS + 2019-11-29 + 13:21:55 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FONN7OUT + SQIN + + + + + -9999 + L TENN/FONTANA DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FORM7OUT + SQIN + + + + + -9999 + TABLE ROCK DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FPHT1OUT + SQIN + + + + + -9999 + FPHT1OUT + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GRNM6OUT + SQIN + + + + + -9999 + GRENADA DAM + CFS + 2019-11-29 + 13:28:36 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GRRA4OUT + SQIN + + + + + -9999 + GREERS FRRY DM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HADT1OUT + SQIN + + + + + -9999 + HADT1OUT + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HIWN7OUT + SQIN + + + + + -9999 + HIWN7OUT + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ICCN7OUT + SQIN + + + + -9999 + TUCKASEEGE/CEDAR CLIFF DM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + JFNT2OUT + SQIN + + + + + -9999 + FERRELS BR DM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + JSNM6OUT + SQIN + + + + + -9999 + ROSS BARNETT DAM + CFS + 2019-11-29 + 13:08:51 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LBBL1OUT + SQIN + + + + + -9999 + Bayou Bodcau + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LBRA1OUT + SQIN + + + + -9999 + L BEAR CK/LTL BEAR CK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LBUL1OUT + SQIN + + + + + -9999 + RINGGOLD + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LCAL1OUT + SQIN + + + + -9999 + LAKE CLAIBORNE + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LCOL1OUT + SQIN + + + + -9999 + CADDO LAKE + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LDBL1OUT + SQIN + + + + -9999 + BAYOU D'ARBONNE LAKE + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MRDM6OUT + SQIN + + + + + -9999 + OKATIBBEE RES/MERIDIAN + CFS + 2019-11-29 + 13:16:33 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NADA4OUT + SQIN + + + + + -9999 + NARROWS DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NANN7OUT + SQIN + + + + -9999 + NANTAHALA/NANTAHALA DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NFDA4OUT + SQIN + + + + + -9999 + NORFORK DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NOTG1OUT + SQIN + + + + + -9999 + NOTG1OUT + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NRMT1OUT + SQIN + + + + + -9999 + DUCK/NORMANDY + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NRST1OUT + SQIN + + + + + -9999 + NORRIS DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + OCAT1OUT + SQIN + + + + -9999 + OCAT1OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + OCCT1OUT + SQIN + + + + -9999 + DAM #3 AT DUCKTOWN + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + REDA4OUT + SQIN + + + + + -9999 + REMMEL DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + RODL1OUT + SQIN + + + + + -9999 + BLACK BAYOU AT RODESSA + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SCLT2OUT + SQIN + + + + + -9999 + COOPER DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SHDT1OUT + SQIN + + + + + -9999 + SF HOLSTON/SOUTH HOLSTON DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SNTN7OUT + SQIN + + + + + -9999 + CHEOAH/SANTEELAH DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SRDM6OUT + SQIN + + + + + -9999 + SARDIS DAM + CFS + 2019-11-29 + 13:21:55 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + THPN7OUT + SQIN + + + + -9999 + W.F.TUCKASEGEE/THORPE DM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TMFT1OUT + SQIN + + + + + -9999 + TIMS FORD DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TXKT2OUT + SQIN + + + + + -9999 + TEXARKANA + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + UBRA1OUT + SQIN + + + + -9999 + BEAR CREEK/UPPER BEAR CK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WAGL1OUT + SQIN + + + + + -9999 + WALLACE LAKE/KEITHVILLE + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WPPM7OUT + SQIN + + + + + -9999 + Wappapello + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WTDN7OUT + SQIN + + + + -9999 + PIGEON AT WALTERS DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WTGT1OUT + SQIN + + + + + -9999 + WATAUGA/WATAUGA DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ARKM6OUT + SQIN + + + + -9999 + ARKABUTLA DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCDA1OUT + SQIN + + + + -9999 + BIG CK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCDN7OUT + SQIN + + + + -9999 + BEAR CREEK AT BEAR CREEK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCEA4OUT + SQIN + + + + -9999 + BAYOU BODCAU AT L. ERLING + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCRA1OUT + SQIN + + + + -9999 + BEAR CK/BEAR CK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BKDL1OUT + SQIN + + + + + -9999 + BUNDICK LK DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BMDA4OUT + SQIN + + + + -9999 + BLAKLEY MT DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BOOT1OUT + SQIN + + + + -9999 + BOOT1OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BRDG1OUT + SQIN + + + + -9999 + BLUE RIDGE DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BSGA4OUT + SQIN + + + + -9999 + BULL SHOALS + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BSLT2OUT + SQIN + + + + -9999 + BOB SANDLIN LAKE + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BVGA4OUT + SQIN + + + + -9999 + BEAVER DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CCRA1OUT + SQIN + + + + -9999 + CEDAR CK/CEDAR CK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CHAN7OUT + SQIN + + + + -9999 + CHATUGE DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CHEN7OUT + SQIN + + + + -9999 + CHEN7OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CLRM7OUT + SQIN + + + + -9999 + Clearwater + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CRKT1OUT + SQIN + + + + -9999 + CRKT1OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CRLL1OUT + SQIN + + + + -9999 + SHREVEPORT/CROSS LK + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DGDA4OUT + SQIN + + + + -9999 + DEGRAY DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DUGT1OUT + SQIN + + + + -9999 + F BROAD/DOUGLAS DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ENDM6OUT + SQIN + + + + -9999 + ENID DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FONN7OUT + SQIN + + + + -9999 + L TENN/FONTANA DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FORM7OUT + SQIN + + + + -9999 + TABLE ROCK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FPHT1OUT + SQIN + + + + -9999 + FPHT1OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GRNM6OUT + SQIN + + + + -9999 + GRENADA DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GRRA4OUT + SQIN + + + + -9999 + GREERS FRRY DM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HADT1OUT + SQIN + + + + -9999 + HADT1OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HIWN7OUT + SQIN + + + + -9999 + HIWN7OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ICCN7OUT + SQIN + + + + -9999 + TUCKASEEGE/CEDAR CLIFF DM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + JFNT2OUT + SQIN + + + + -9999 + FERRELS BR DM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + JSNM6OUT + SQIN + + + + -9999 + ROSS BARNETT DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LBBL1OUT + SQIN + + + + -9999 + Bayou Bodcau + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LBRA1OUT + SQIN + + + + -9999 + L BEAR CK/LTL BEAR CK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LBUL1OUT + SQIN + + + + -9999 + RINGGOLD + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LCAL1OUT + SQIN + + + + + -9999 + LAKE CLAIBORNE + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LCOL1OUT + SQIN + + + + + -9999 + CADDO LAKE + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LDBL1OUT + SQIN + + + + + -9999 + BAYOU D'ARBONNE LAKE + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MRDM6OUT + SQIN + + + + -9999 + OKATIBBEE RES/MERIDIAN + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NADA4OUT + SQIN + + + + -9999 + NARROWS DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NANN7OUT + SQIN + + + + -9999 + NANTAHALA/NANTAHALA DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NFDA4OUT + SQIN + + + + -9999 + NORFORK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NOTG1OUT + SQIN + + + + -9999 + NOTG1OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NRMT1OUT + SQIN + + + + -9999 + DUCK/NORMANDY + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NRST1OUT + SQIN + + + + -9999 + NORRIS DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + OCAT1OUT + SQIN + + + + -9999 + OCAT1OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + OCCT1OUT + SQIN + + + + -9999 + DAM #3 AT DUCKTOWN + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + REDA4OUT + SQIN + + + + -9999 + REMMEL DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + RODL1OUT + SQIN + + + + -9999 + BLACK BAYOU AT RODESSA + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SCLT2OUT + SQIN + + + + -9999 + COOPER DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SHDT1OUT + SQIN + + + + -9999 + SF HOLSTON/SOUTH HOLSTON DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SNTN7OUT + SQIN + + + + -9999 + CHEOAH/SANTEELAH DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SRDM6OUT + SQIN + + + + -9999 + SARDIS DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + THPN7OUT + SQIN + + + + -9999 + W.F.TUCKASEGEE/THORPE DM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TMFT1OUT + SQIN + + + + -9999 + TIMS FORD DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TXKT2OUT + SQIN + + + + -9999 + TEXARKANA + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + UBRA1OUT + SQIN + + + + -9999 + BEAR CREEK/UPPER BEAR CK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WAGL1OUT + SQIN + + + + -9999 + WALLACE LAKE/KEITHVILLE + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WPPM7OUT + SQIN + + + + -9999 + Wappapello + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WTDN7OUT + SQIN + + + + -9999 + PIGEON AT WALTERS DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WTGT1OUT + SQIN + + + + -9999 + WATAUGA/WATAUGA DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ARKM6OUT + SQIN + + + + -9999 + ARKABUTLA DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCDA1OUT + SQIN + + + + -9999 + BIG CK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCDN7OUT + SQIN + + + + + -9999 + BEAR CREEK AT BEAR CREEK DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCEA4OUT + SQIN + + + + -9999 + BAYOU BODCAU AT L. ERLING + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCRA1OUT + SQIN + + + + + -9999 + BEAR CK/BEAR CK DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BKDL1OUT + SQIN + + + + -9999 + BUNDICK LK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BMDA4OUT + SQIN + + + + -9999 + BLAKLEY MT DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BOOT1OUT + SQIN + + + + -9999 + BOOT1OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BRDG1OUT + SQIN + + + + + -9999 + BLUE RIDGE DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BSGA4OUT + SQIN + + + + -9999 + BULL SHOALS + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BSLT2OUT + SQIN + + + + -9999 + BOB SANDLIN LAKE + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BVGA4OUT + SQIN + + + + -9999 + BEAVER DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CCRA1OUT + SQIN + + + + + -9999 + CEDAR CK/CEDAR CK DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CHAN7OUT + SQIN + + + + -9999 + CHATUGE DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CHEN7OUT + SQIN + + + + -9999 + CHEN7OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CLRM7OUT + SQIN + + + + -9999 + Clearwater + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CRKT1OUT + SQIN + + + + -9999 + CRKT1OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CRLL1OUT + SQIN + + + + -9999 + SHREVEPORT/CROSS LK + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DGDA4OUT + SQIN + + + + -9999 + DEGRAY DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DUGT1OUT + SQIN + + + + -9999 + F BROAD/DOUGLAS DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ENDM6OUT + SQIN + + + + -9999 + ENID DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FONN7OUT + SQIN + + + + -9999 + L TENN/FONTANA DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FORM7OUT + SQIN + + + + -9999 + TABLE ROCK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FPHT1OUT + SQIN + + + + -9999 + FPHT1OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GRNM6OUT + SQIN + + + + -9999 + GRENADA DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GRRA4OUT + SQIN + + + + -9999 + GREERS FRRY DM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HADT1OUT + SQIN + + + + -9999 + HADT1OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HIWN7OUT + SQIN + + + + -9999 + HIWN7OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ICCN7OUT + SQIN + + + + + -9999 + TUCKASEEGE/CEDAR CLIFF DM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + JFNT2OUT + SQIN + + + + -9999 + FERRELS BR DM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + JSNM6OUT + SQIN + + + + -9999 + ROSS BARNETT DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LBBL1OUT + SQIN + + + + -9999 + Bayou Bodcau + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LBRA1OUT + SQIN + + + + + -9999 + L BEAR CK/LTL BEAR CK DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LBUL1OUT + SQIN + + + + -9999 + RINGGOLD + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LCAL1OUT + SQIN + + + + -9999 + LAKE CLAIBORNE + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LCOL1OUT + SQIN + + + + -9999 + CADDO LAKE + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LDBL1OUT + SQIN + + + + -9999 + BAYOU D'ARBONNE LAKE + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MRDM6OUT + SQIN + + + + -9999 + OKATIBBEE RES/MERIDIAN + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NADA4OUT + SQIN + + + + -9999 + NARROWS DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NANN7OUT + SQIN + + + + + -9999 + NANTAHALA/NANTAHALA DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NFDA4OUT + SQIN + + + + -9999 + NORFORK DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NOTG1OUT + SQIN + + + + -9999 + NOTG1OUT + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NRMT1OUT + SQIN + + + + -9999 + DUCK/NORMANDY + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NRST1OUT + SQIN + + + + -9999 + NORRIS DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + OCAT1OUT + SQIN + + + + + -9999 + OCAT1OUT + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + OCCT1OUT + SQIN + + + + + -9999 + DAM #3 AT DUCKTOWN + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + REDA4OUT + SQIN + + + + -9999 + REMMEL DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + RODL1OUT + SQIN + + + + -9999 + BLACK BAYOU AT RODESSA + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SCLT2OUT + SQIN + + + + -9999 + COOPER DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SHDT1OUT + SQIN + + + + -9999 + SF HOLSTON/SOUTH HOLSTON DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SNTN7OUT + SQIN + + + + -9999 + CHEOAH/SANTEELAH DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SRDM6OUT + SQIN + + + + -9999 + SARDIS DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + THPN7OUT + SQIN + + + + + -9999 + W.F.TUCKASEGEE/THORPE DM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TMFT1OUT + SQIN + + + + -9999 + TIMS FORD DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TXKT2OUT + SQIN + + + + -9999 + TEXARKANA + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + UBRA1OUT + SQIN + + + + + -9999 + BEAR CREEK/UPPER BEAR CK DAM + CFS + 2019-11-29 + 12:05:07 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WAGL1OUT + SQIN + + + + -9999 + WALLACE LAKE/KEITHVILLE + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WPPM7OUT + SQIN + + + + -9999 + Wappapello + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WTDN7OUT + SQIN + + + + + -9999 + PIGEON AT WALTERS DAM + CFS + 2019-11-29 + 13:29:06 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WTGT1OUT + SQIN + + + + -9999 + WATAUGA/WATAUGA DAM + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/marfc/2019112712_MARFC_Reservoir_Export.xml b/data_assimilation_engine/rfc_ingestion/testdata/marfc/2019112712_MARFC_Reservoir_Export.xml new file mode 100644 index 0000000..73eb246 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/marfc/2019112712_MARFC_Reservoir_Export.xml @@ -0,0 +1,456 @@ + + + 0.0 + +
+ instantaneous + CNNN6DEL + QINE + + + + + -9999.0 + Cannonsville Reservoir + CFS + 2019-11-27 + 13:36:02 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DWNN6DEL + QINE + + + + + -9999.0 + Downsville Reservoir + CFS + 2019-11-27 + 13:36:02 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCHP1BEC + QINE + + + + + -9999.0 + Blanchard (d.s. Sayers Dam) + CFS + 2019-11-27 + 13:36:02 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + JCKV2JCK + QINE + + + + + -9999.0 + Gathright Dam + CFS + 2019-11-27 + 13:36:02 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/mbrfc/201911111541_MBRFC_Reservoir_Export.xml b/data_assimilation_engine/rfc_ingestion/testdata/mbrfc/201911111541_MBRFC_Reservoir_Export.xml new file mode 100644 index 0000000..3bb3b5d --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/mbrfc/201911111541_MBRFC_Reservoir_Export.xml @@ -0,0 +1,3519 @@ + + + 0.0 + +
+ instantaneous + GPDN1 + QINE + + + + + -9999 + GPDN1: GAVINS PT RES NE + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + KANK1 + QINE + + + + + -9999 + KANK1: KANOPOLIS RES KS + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WLSK1 + QINE + + + + + -9999 + WLSK1: WILSON RES KS + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + REPN1 + QINE + + + + + -9999 + REPN1: HARLAN COUNTY RES NE + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MLFK1 + QINE + + + + + -9999 + MLFK1: MILFORD RES KS + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MTTK1 + QINE + + + + + -9999 + MTTK1: TUTTLE CR RES KS + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PRRK1 + QINE + + + + + -9999 + PRRK1: PERRY RES KS + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CLIK1 + QINE + + + + + -9999 + CLIK1: CLINTON RES KS + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SLKM7 + QINE + + + + + -9999 + SLKM7: SMITHVILLE RES MO + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LNZM7 + QINE + + + + + -9999 + LNZM7: LONGVIEW RES MO + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BSZM7 + QINE + + + + + -9999 + BSZM7: BLUE SPRINGS RES MO + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MLVK1 + QINE + + + + + -9999 + MLVK1: MELVERN RES KS + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PLKK1 + QINE + + + + + -9999 + PLKK1: POMONA RES KS + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HILK1 + QINE + + + + + -9999 + HILK1: HILLSDALE RES KS + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + STXM7 + QINE + + + + + -9999 + STXM7: STOCKTON RES MO + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PTXM7 + QINE + + + + + -9999 + PTXM7: POMME DE TERRE RES MO + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TKZM7 + QINE + + + + + -9999 + TKZM7: HARRY TRUMAN RES MO + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + RADI4 + QINE + + + + + -9999 + RADI4: RATHBUN RES IA + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LBRM7 + QINE + + + + + -9999 + LBRM7: LONG BRANCH RES MO + CFS + 2019-11-11 + 14:03:44 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/ncrfc/201911081100_NCRFC_Reservoir_Export.xml b/data_assimilation_engine/rfc_ingestion/testdata/ncrfc/201911081100_NCRFC_Reservoir_Export.xml new file mode 100644 index 0000000..d3696f3 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/ncrfc/201911081100_NCRFC_Reservoir_Export.xml @@ -0,0 +1,328 @@ + + + 0.0 + +
+ instantaneous + Process_Res_Fcsts_NWM + CRVI4 + RQOT + observed + + + + -9999 + CRVI4 + 41.7252777777778 + -91.5352777777778 + -91.5352777777778 + 41.7252777777778 + 0.0 + CFS +
+ + + + + + + + + + + + +
+ instantaneous + RRS_PreProcessing_Inst_FRQOT + CRVI4 + QINE + forecast + + + + + -9999 + CRVI4 + 41.7252777777778 + -91.5352777777778 + -91.5352777777778 + 41.7252777777778 + 0.0 + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + Process_Res_Fcsts_NWM + SAYI4 + RQOT + observed + + + + -9999 + SAYI4 + 41.703611111111 + -93.689166666667 + -93.689166666667 + 41.703611111111 + 0.0 + CFS +
+ + + + + + + + + + + + +
+ instantaneous + RRS_PreProcessing_Inst_FRQOT + SAYI4 + QINE + forecast + + + + + -9999 + SAYI4 + 41.703611111111 + -93.689166666667 + -93.689166666667 + 41.703611111111 + 0.0 + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + Process_Res_Fcsts_NWM + PELI4 + RQOT + observed + + + + -9999 + PELI4 + 41.369722222222 + -92.98 + -92.98 + 41.369722222222 + 0.0 + CFS +
+ + + + + + + + + + + + +
+ instantaneous + RRS_PreProcessing_Inst_FRQOT + PELI4 + QINE + forecast + + + + + -9999 + PELI4 + 41.369722222222 + -92.98 + -92.98 + 41.369722222222 + 0.0 + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/ncrfc/201911181200_NCRFC_Reservoir_Export.xml b/data_assimilation_engine/rfc_ingestion/testdata/ncrfc/201911181200_NCRFC_Reservoir_Export.xml new file mode 100644 index 0000000..85b435e --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/ncrfc/201911181200_NCRFC_Reservoir_Export.xml @@ -0,0 +1,322 @@ + + + 0.0 + +
+ instantaneous + Process_Res_Fcsts_NWM + CRVI4 + RQOT + observed + + + + -9999 + CRVI4 + 41.7252777777778 + -91.5352777777778 + -91.5352777777778 + 41.7252777777778 + 0.0 + CFS +
+ + + + + + + + + + + +
+ instantaneous + RRS_PreProcessing_Inst_FRQOT + CRVI4 + QINE + forecast + + + + + -9999 + CRVI4 + 41.7252777777778 + -91.5352777777778 + -91.5352777777778 + 41.7252777777778 + 0.0 + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + Process_Res_Fcsts_NWM + SAYI4 + RQOT + observed + + + + -9999 + SAYI4 + 41.703611111111 + -93.689166666667 + -93.689166666667 + 41.703611111111 + 0.0 + CFS +
+ + + + + + + + + + + +
+ instantaneous + RRS_PreProcessing_Inst_FRQOT + SAYI4 + QINE + forecast + + + + + -9999 + SAYI4 + 41.703611111111 + -93.689166666667 + -93.689166666667 + 41.703611111111 + 0.0 + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + Process_Res_Fcsts_NWM + PELI4 + RQOT + observed + + + + -9999 + PELI4 + 41.369722222222 + -92.98 + -92.98 + 41.369722222222 + 0.0 + CFS +
+ + + + + + + + + + + +
+ instantaneous + RRS_PreProcessing_Inst_FRQOT + PELI4 + QINE + forecast + + + + + -9999 + PELI4 + 41.369722222222 + -92.98 + -92.98 + 41.369722222222 + 0.0 + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/nerfc/NERFC_11.07.2019.16.xml b/data_assimilation_engine/rfc_ingestion/testdata/nerfc/NERFC_11.07.2019.16.xml new file mode 100644 index 0000000..b620858 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/nerfc/NERFC_11.07.2019.16.xml @@ -0,0 +1,684 @@ + + + 0.0 + +
+ accumulative + RKWM1ME + FMAP + + + + + NaN + RKWM1ME + 45.58 + -69.72 + -69.72 + 45.58 + 313.33 + IN +
+ + + + + + + + + + + + + + +
+ instantaneous + SWRN6 + RQOT + forecast + + + + + NaN + SWRN6 + 43.9 + -75.05 + -75.05 + 43.9 + 515.1 + CFS +
+ + + + + + +
+ instantaneous + STDM1 + RQOT + observed + + + + NaN + STDM1 + 43.78 + -70.51 + -70.51 + 43.78 + 270.0 + CFS +
+ + + + + +
+ instantaneous + HIKN6 + RQOT + observed + + + + NaN + HIKN6 + 43.32 + -75.12 + -75.12 + 43.32 + 385.6 + CFS +
+ + + + + +
+ instantaneous + SCIR1 + RQOT + observed + + + + NaN + SCIR1 + 41.75 + -71.59 + -71.59 + 41.75 + 64.0 + CFS +
+ + + + + + + + + + + + +
+ instantaneous + STDM1 + QINE + forecast + + + + + NaN + STDM1ME + 43.78 + -70.51 + -70.51 + 43.78 + 0.0 + CFS + 2019-11-07 + 15:13:17 +
+ + + + + + + + + + + + + + + +
+ instantaneous + RKWM1 + QINE + forecast + + + + + NaN + RKWM1ME + 45.58 + -69.72 + -69.72 + 45.58 + 313.33 + CFS + 2019-11-07 + 15:13:17 +
+ + + + + + + + + + + + + + + +
+ instantaneous + SCIR1 + QINE + forecast + + + + + NaN + SCIR1SNE + 41.75 + -71.59 + -71.59 + 41.75 + 64.01 + CFS + 2019-11-07 + 15:19:28 +
+ + + + + + + + + + + + + + + +
+ instantaneous + HIKN6 + QINE + forecast + + + + + NaN + HIKN6HUD + 43.32 + -75.12 + -75.12 + 43.32 + 385.57 + CFS + 2019-11-07 + 15:57:35 +
+ + + + + + + + + + + + + + + +
+ instantaneous + RKWM1 + RQOT + observed + + + + + NaN + RKWM1ME + 45.58 + -69.72 + -69.72 + 45.58 + 313.33 + CFS + 2019-11-07 + 15:13:17 +
+ + + +
+ instantaneous + BNGM1 + RQOT + observed + + + + + NaN + WYMM1ME + 45.1 + -69.9 + -69.9 + 45.1 + 149.35 + CFS + 2019-11-07 + 15:13:17 +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + STVC3 + RQOT + observed + + + + + NaN + STVC3SNE + 41.38 + -73.17 + -73.17 + 41.38 + 35.05 + CFS + 2019-11-07 + 15:19:28 +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HDLN6 + RQOT + observed + + + + + NaN + HDLN6HUD + 43.31 + -73.87 + -73.87 + 43.31 + 177.39 + CFS + 2019-11-07 + 15:57:35 +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CPNN6 + RQOT + observed + + + + + NaN + CPNN6GRL + 42.92 + -77.23 + -77.23 + 42.92 + 212.45 + CFS + 2019-11-07 + 15:51:13 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GBRN6 + RQOT + observed + + + + + NaN + GBRN6HUD + 42.4 + -74.45 + -74.45 + 42.4 + 0.0 + CFS + 2019-11-07 + 15:57:35 +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SWRN6 + RQOT + observed + + + + NaN + SWRN6 + 43.9 + -75.05 + -75.05 + 43.9 + 515.1 + CFS +
+ + + + + +
+ instantaneous + CPNN6 + QINE + forecast + + + + + NaN + CPNN6 + 42.92 + -77.23 + -77.23 + 42.92 + 212.4 + CFS +
+ + + + + + + + + + +
+ instantaneous + GBRN6 + QINE + forecast + + + + + NaN + GBRN6 + 42.4 + -74.45 + -74.45 + 42.4 + 0.0 + CFS +
+ + + + + + + + + + +
+ instantaneous + HDLN6 + QINE + forecast + + + + + NaN + HDLN6 + 43.31 + -73.87 + -73.87 + 43.31 + 177.4 + CFS +
+ + + + + + + + + + +
+ instantaneous + STVC3 + QINE + forecast + + + + + NaN + STVC3 + 41.38 + -73.17 + -73.17 + 41.38 + 35.1 + CFS +
+ + + + + + + + + + +
+ instantaneous + BNGM1 + QINE + forecast + + + + + NaN + BNGM1 + 45.07 + -69.9 + -69.9 + 45.07 + 121.9 + CFS +
+ + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/nerfc/NERFC_11.07.2019.17.xml b/data_assimilation_engine/rfc_ingestion/testdata/nerfc/NERFC_11.07.2019.17.xml new file mode 100644 index 0000000..ddb4f16 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/nerfc/NERFC_11.07.2019.17.xml @@ -0,0 +1,699 @@ + + + 0.0 + +
+ accumulative + RKWM1ME + FMAP + + + + + NaN + RKWM1ME + 45.58 + -69.72 + -69.72 + 45.58 + 313.33 + IN +
+ + + + + + + + + + + + + + +
+ instantaneous + SWRN6 + RQOT + forecast + + + + + NaN + SWRN6 + 43.9 + -75.05 + -75.05 + 43.9 + 515.1 + CFS +
+ + + + + + +
+ instantaneous + STDM1 + RQOT + observed + + + + NaN + STDM1 + 43.78 + -70.51 + -70.51 + 43.78 + 270.0 + CFS +
+ + + + + +
+ instantaneous + HIKN6 + RQOT + observed + + + + NaN + HIKN6 + 43.32 + -75.12 + -75.12 + 43.32 + 385.6 + CFS +
+ + + + + +
+ instantaneous + SCIR1 + RQOT + observed + + + + NaN + SCIR1 + 41.75 + -71.59 + -71.59 + 41.75 + 64.0 + CFS +
+ + + + + + + + + + + + +
+ instantaneous + STDM1 + QINE + forecast + + + + + NaN + STDM1ME + 43.78 + -70.51 + -70.51 + 43.78 + 0.0 + CFS + 2019-11-07 + 15:13:17 +
+ + + + + + + + + + + + + + + +
+ instantaneous + RKWM1 + QINE + forecast + + + + + NaN + RKWM1ME + 45.58 + -69.72 + -69.72 + 45.58 + 313.33 + CFS + 2019-11-07 + 15:13:17 +
+ + + + + + + + + + + + + + + +
+ instantaneous + SCIR1 + QINE + forecast + + + + + NaN + SCIR1SNE + 41.75 + -71.59 + -71.59 + 41.75 + 64.01 + CFS + 2019-11-07 + 15:19:28 +
+ + + + + + + + + + + + + + + +
+ instantaneous + HIKN6 + QINE + forecast + + + + + NaN + HIKN6HUD + 43.32 + -75.12 + -75.12 + 43.32 + 385.57 + CFS + 2019-11-07 + 15:57:35 +
+ + + + + + + + + + + + + + + +
+ instantaneous + RKWM1 + RQOT + observed + + + + + NaN + RKWM1ME + 45.58 + -69.72 + -69.72 + 45.58 + 313.33 + CFS + 2019-11-07 + 15:13:17 +
+ + + +
+ instantaneous + BNGM1 + RQOT + observed + + + + + NaN + WYMM1ME + 45.1 + -69.9 + -69.9 + 45.1 + 149.35 + CFS + 2019-11-07 + 15:13:17 +
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + STVC3 + RQOT + observed + + + + + NaN + STVC3SNE + 41.38 + -73.17 + -73.17 + 41.38 + 35.05 + CFS + 2019-11-07 + 15:19:28 +
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HDLN6 + RQOT + observed + + + + + NaN + HDLN6HUD + 43.31 + -73.87 + -73.87 + 43.31 + 177.39 + CFS + 2019-11-07 + 15:57:35 +
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CPNN6 + RQOT + observed + + + + + NaN + CPNN6GRL + 42.92 + -77.23 + -77.23 + 42.92 + 212.45 + CFS + 2019-11-07 + 15:51:13 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GBRN6 + RQOT + observed + + + + + NaN + GBRN6HUD + 42.4 + -74.45 + -74.45 + 42.4 + 0.0 + CFS + 2019-11-07 + 15:57:35 +
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SWRN6 + RQOT + observed + + + + NaN + SWRN6 + 43.9 + -75.05 + -75.05 + 43.9 + 515.1 + CFS +
+ + + + + +
+ instantaneous + CPNN6 + QINE + forecast + + + + + NaN + CPNN6 + 42.92 + -77.23 + -77.23 + 42.92 + 212.4 + CFS +
+ + + + + + + + + + + + + + +
+ instantaneous + GBRN6 + QINE + forecast + + + + + NaN + GBRN6 + 42.4 + -74.45 + -74.45 + 42.4 + 0.0 + CFS +
+ + + + + + + + + + + + + + +
+ instantaneous + HDLN6 + QINE + forecast + + + + + NaN + HDLN6 + 43.31 + -73.87 + -73.87 + 43.31 + 177.4 + CFS +
+ + + + + + + + + + + + + + +
+ instantaneous + STVC3 + QINE + forecast + + + + + NaN + STVC3 + 41.38 + -73.17 + -73.17 + 41.38 + 35.1 + CFS +
+ + + + + + + + + + + + + + +
+ instantaneous + BNGM1 + QINE + forecast + + + + + NaN + BNGM1 + 45.07 + -69.9 + -69.9 + 45.07 + 121.9 + CFS +
+ + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/nerfc/NERFC_11.07.2019.22.xml b/data_assimilation_engine/rfc_ingestion/testdata/nerfc/NERFC_11.07.2019.22.xml new file mode 100644 index 0000000..8250f7b --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/nerfc/NERFC_11.07.2019.22.xml @@ -0,0 +1,684 @@ + + + 0.0 + +
+ accumulative + RKWM1ME + FMAP + + + + + NaN + RKWM1ME + 45.58 + -69.72 + -69.72 + 45.58 + 313.33 + IN +
+ + + + + + + + + + + + + + +
+ instantaneous + SWRN6 + RQOT + forecast + + + + + NaN + SWRN6 + 43.9 + -75.05 + -75.05 + 43.9 + 515.1 + CFS +
+ + + + + + + +
+ instantaneous + STDM1 + RQOT + observed + + + + NaN + STDM1 + 43.78 + -70.51 + -70.51 + 43.78 + 270.0 + CFS +
+ + + + +
+ instantaneous + HIKN6 + RQOT + observed + + + + NaN + HIKN6 + 43.32 + -75.12 + -75.12 + 43.32 + 385.6 + CFS +
+ + + + +
+ instantaneous + SCIR1 + RQOT + observed + + + + NaN + SCIR1 + 41.75 + -71.59 + -71.59 + 41.75 + 64.0 + CFS +
+ + + + + + + + + + + + +
+ instantaneous + STDM1 + QINE + forecast + + + + + NaN + STDM1ME + 43.78 + -70.51 + -70.51 + 43.78 + 0.0 + CFS + 2019-11-07 + 15:13:17 +
+ + + + + + + + + + + + + + + +
+ instantaneous + RKWM1 + QINE + forecast + + + + + NaN + RKWM1ME + 45.58 + -69.72 + -69.72 + 45.58 + 313.33 + CFS + 2019-11-07 + 15:13:17 +
+ + + + + + + + + + + + + + + +
+ instantaneous + SCIR1 + QINE + forecast + + + + + NaN + SCIR1SNE + 41.75 + -71.59 + -71.59 + 41.75 + 64.01 + CFS + 2019-11-07 + 15:19:28 +
+ + + + + + + + + + + + + + + +
+ instantaneous + HIKN6 + QINE + forecast + + + + + NaN + HIKN6HUD + 43.32 + -75.12 + -75.12 + 43.32 + 385.57 + CFS + 2019-11-07 + 15:57:35 +
+ + + + + + + + + + + + + + + +
+ instantaneous + RKWM1 + RQOT + observed + + + + + NaN + RKWM1ME + 45.58 + -69.72 + -69.72 + 45.58 + 313.33 + CFS + 2019-11-07 + 15:13:17 +
+ + +
+ instantaneous + BNGM1 + RQOT + observed + + + + + NaN + WYMM1ME + 45.1 + -69.9 + -69.9 + 45.1 + 149.35 + CFS + 2019-11-07 + 15:13:17 +
+ + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + STVC3 + RQOT + observed + + + + + NaN + STVC3SNE + 41.38 + -73.17 + -73.17 + 41.38 + 35.05 + CFS + 2019-11-07 + 15:19:28 +
+ + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HDLN6 + RQOT + observed + + + + + NaN + HDLN6HUD + 43.31 + -73.87 + -73.87 + 43.31 + 177.39 + CFS + 2019-11-07 + 15:57:35 +
+ + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CPNN6 + RQOT + observed + + + + + NaN + CPNN6GRL + 42.92 + -77.23 + -77.23 + 42.92 + 212.45 + CFS + 2019-11-07 + 18:28:31 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GBRN6 + RQOT + observed + + + + + NaN + GBRN6HUD + 42.4 + -74.45 + -74.45 + 42.4 + 0.0 + CFS + 2019-11-07 + 15:57:35 +
+ + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SWRN6 + RQOT + observed + + + + NaN + SWRN6 + 43.9 + -75.05 + -75.05 + 43.9 + 515.1 + CFS +
+ + + + + +
+ instantaneous + CPNN6 + QINE + forecast + + + + + NaN + CPNN6 + 42.92 + -77.23 + -77.23 + 42.92 + 212.4 + CFS +
+ + + + + + + + + + + + + +
+ instantaneous + GBRN6 + QINE + forecast + + + + + NaN + GBRN6 + 42.4 + -74.45 + -74.45 + 42.4 + 0.0 + CFS +
+ + + + + + + + + + + + + +
+ instantaneous + HDLN6 + QINE + forecast + + + + + NaN + HDLN6 + 43.31 + -73.87 + -73.87 + 43.31 + 177.4 + CFS +
+ + + + + + + + + + + + + +
+ instantaneous + STVC3 + QINE + forecast + + + + + NaN + STVC3 + 41.38 + -73.17 + -73.17 + 41.38 + 35.1 + CFS +
+ + + + + + + + + + + + + +
+ instantaneous + BNGM1 + QINE + forecast + + + + + NaN + BNGM1 + 45.07 + -69.9 + -69.9 + 45.07 + 121.9 + CFS +
+ + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/nerfc/NERFC_Reservoir_Export.xml b/data_assimilation_engine/rfc_ingestion/testdata/nerfc/NERFC_Reservoir_Export.xml new file mode 100644 index 0000000..b17e0c2 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/nerfc/NERFC_Reservoir_Export.xml @@ -0,0 +1,678 @@ + + + 0.0 + +
+ accumulative + RKWM1ME + FMAP + + + + + NaN + RKWM1ME + 45.58 + -69.72 + -69.72 + 45.58 + 313.33 + IN +
+ + + + + + + + + + + + + + +
+ instantaneous + SWRN6 + RQOT + forecast + + + + + NaN + SWRN6 + 43.9 + -75.05 + -75.05 + 43.9 + 515.1 + CFS +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + STDM1 + RQOT + observed + + + + NaN + STDM1 + 43.78 + -70.51 + -70.51 + 43.78 + 270.0 + CFS +
+ + + + + + + + + + + + +
+ instantaneous + HIKN6 + RQOT + observed + + + + NaN + HIKN6 + 43.32 + -75.12 + -75.12 + 43.32 + 385.6 + CFS +
+ + + + + + + + + + + + +
+ instantaneous + SCIR1 + RQOT + observed + + + + NaN + SCIR1 + 41.75 + -71.59 + -71.59 + 41.75 + 64.0 + CFS +
+ + + + + + + + + + + + +
+ instantaneous + STDM1 + QINE + forecast + + + + + NaN + STDM1ME + 43.78 + -70.51 + -70.51 + 43.78 + 0.0 + CFS + 2019-11-25 + 15:13:21 +
+ + + + + + + + + + + + + + +
+ instantaneous + RKWM1 + QINE + forecast + + + + + NaN + RKWM1ME + 45.58 + -69.72 + -69.72 + 45.58 + 313.33 + CFS + 2019-11-25 + 15:13:21 +
+ + + + + + + + + + + + + + +
+ instantaneous + SCIR1 + QINE + forecast + + + + + NaN + SCIR1SNE + 41.75 + -71.59 + -71.59 + 41.75 + 64.01 + CFS + 2019-11-25 + 15:46:47 +
+ + + + + + + + + + + + + + +
+ instantaneous + HIKN6 + QINE + forecast + + + + + NaN + HIKN6HUD + 43.32 + -75.12 + -75.12 + 43.32 + 385.57 + CFS + 2019-11-25 + 15:14:41 +
+ + + + + + + + + + + + + + +
+ instantaneous + RKWM1 + RQOT + observed + + + + + NaN + RKWM1ME + 45.58 + -69.72 + -69.72 + 45.58 + 313.33 + CFS + 2019-11-25 + 15:13:21 +
+ + + + + + + + + + + +
+ instantaneous + BNGM1 + RQOT + observed + + + + NaN + WYMM1SIX + 45.1 + -69.9 + -69.9 + 45.1 + 149.35 + CFS +
+ + + + + + + + + + + +
+ instantaneous + STVC3 + RQOT + observed + + + + NaN + STVC3SIX + 41.38 + -73.17 + -73.17 + 41.38 + 35.05 + CFS +
+ + + + + + + + + + + +
+ instantaneous + HDLN6 + RQOT + observed + + + + NaN + HDLN6SIX + 43.31 + -73.87 + -73.87 + 43.31 + 177.39 + CFS +
+ + + + + + + + + + + +
+ instantaneous + CPNN6 + RQOT + observed + + + + NaN + CPNN6SIX + 42.92 + -77.23 + -77.23 + 42.92 + 212.45 + CFS +
+ + + + + + + + + + + +
+ instantaneous + GBRN6 + RQOT + observed + + + + NaN + GBRN6SIX + 42.4 + -74.45 + -74.45 + 42.4 + 0.0 + CFS +
+ + + + + + + + + + + +
+ instantaneous + SWRN6 + RQOT + observed + + + + NaN + SWRN6 + 43.9 + -75.05 + -75.05 + 43.9 + 515.1 + CFS +
+ + + + + + + + + + + + +
+ instantaneous + CPNN6 + QINE + forecast + + + + + NaN + CPNN6 + 42.92 + -77.23 + -77.23 + 42.92 + 212.4 + CFS +
+ + + + + + + + + + + + + + +
+ instantaneous + GBRN6 + QINE + forecast + + + + + NaN + GBRN6 + 42.4 + -74.45 + -74.45 + 42.4 + 0.0 + CFS +
+ + + + + + + + + + + + + + +
+ instantaneous + HDLN6 + QINE + forecast + + + + + NaN + HDLN6 + 43.31 + -73.87 + -73.87 + 43.31 + 177.4 + CFS +
+ + + + + + + + + + + + + + +
+ instantaneous + STVC3 + QINE + forecast + + + + + NaN + STVC3 + 41.38 + -73.17 + -73.17 + 41.38 + 35.1 + CFS +
+ + + + + + + + + + + + + + +
+ instantaneous + BNGM1 + QINE + forecast + + + + + NaN + BNGM1 + 45.07 + -69.9 + -69.9 + 45.07 + 121.9 + CFS +
+ + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/nwrfc/NWRFC_11.29.2019.2100.xml b/data_assimilation_engine/rfc_ingestion/testdata/nwrfc/NWRFC_11.29.2019.2100.xml new file mode 100644 index 0000000..97eea64 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/nwrfc/NWRFC_11.29.2019.2100.xml @@ -0,0 +1,3774 @@ + + + 0.0 + +
+ instantaneous + OGCN2 + QINE + + + + + -999 + OGCN2 - Wildhorse near Owyhee River + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + AMFI1 + QINE + + + + + -999 + AMFI1 - Snake R at American Falls Res + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LOSO3 + QINE + + + + + -999 + LOSO3 - Lost Ck Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + APPO3 + QINE + + + + + -999 + APPO3 - Applegate R nr Cooper + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DEXO3 + QINE + + + + + -999 + DEXO3 - MF Willamette R nr Dexter + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + COTO3 + QINE + + + + + -999 + COTO3 - Cottage Grove Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DORO3 + QINE + + + + + -999 + DORO3 - Dorena Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FALO3 + QINE + + + + + -999 + FALO3 - Fall Ck Reservoir + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LOPO3 + QINE + + + + + -999 + LOPO3 - Lookout Point Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HCRO3 + QINE + + + + + -999 + HCRO3 - Hills Ck Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WARO3 + QINE + + + + + -999 + WARO3 - Malheur R nr Riverside + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BERO3 + QINE + + + + + -999 + BERO3 - Beulah Reservoir + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BULO3 + QINE + + + + + -999 + BULO3 - Bully Ck Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + OWYO3 + QINE + + + + + -999 + OWYO3 - Owyhee Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ARKI1 + QINE + + + + + -999 + ARKI1 - Arrowrock Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LUCI1 + QINE + + + + + -999 + LUCI1 - Lucky Peak Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ANDI1 + QINE + + + + + -999 + ANDI1 - SF Boise R at Anderson Ranch Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BWMI1 + QINE + + + + + -999 + BWMI1 - Big Wood below Magic Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MAKI1 + QINE + + + + + -999 + MAKI1 - Big Lost R below Mackay Res + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PALI1 + QINE + + + + + -999 + PALI1 - Snake R nr Irwin + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + JLKW4 + QINE + + + + + -999 + JLKW4 - Jackson Lake at Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ISLI1 + QINE + + + + + -999 + ISLI1 - Henrys Fork nr Island Park + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DRBI1 + QINE + + + + + -999 + DRBI1 - Deadwood R bl Deadwood Res + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CSDI1 + QINE + + + + + -999 + CSDI1 - Cascade Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BRNI1 + QINE + + + + + -999 + BRNI1 - Brownlee Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PHLO3 + QINE + + + + + -999 + PHLO3 - Mason Dam at Phillips Lake + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + UNYO3 + QINE + + + + + -999 + UNYO3 - Unity Reservoir + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + OCHO3 + QINE + + + + + -999 + OCHO3 - Ochoco Ck bl Ochoco Res nr Prineville + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PRVO3 + QINE + + + + + -999 + PRVO3 - Crooked R nr Prineville + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CGRO3 + QINE + + + + + -999 + CGRO3 - Cougar Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BLUO3 + QINE + + + + + -999 + BLUO3 - Blue Reservoir + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCLO3 + QINE + + + + + -999 + BCLO3 - N Santiam R at Niagara + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FOSO3 + QINE + + + + + -999 + FOSO3 - Foster Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GPRO3 + QINE + + + + + -999 + GPRO3 - Green Peter Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FRNO3 + QINE + + + + + -999 + FRNO3 - Fern Ridge Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SCOO3 + QINE + + + + + -999 + SCOO3 - Scoggins Ck nr Gaston + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MEWW1 + QINE + + + + + -999 + MEWW1 - Merwin Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MKDO3 + QINE + + + + + -999 + MKDO3 - Mckay Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HCDI1 + QINE + + + + + -999 + HCDI1 - Snake R at Hells Canyon Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DWRI1 + QINE + + + + + -999 + DWRI1 - Dworshak Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MAYW1 + QINE + + + + + -999 + MAYW1 - Cowlitz R bl Mayfield Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MSRW1 + QINE + + + + + -999 + MSRW1 - Mossyrock Reservoir + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WYNW1 + QINE + + + + + -999 + WYNW1 - Wynoochee R nr Grisdale + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HAHW1 + QINE + + + + + -999 + HAHW1 - Green R bl Howard Hanson Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + KEEW1 + QINE + + + + + -999 + KEEW1 - Yakima R nr Martin + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CLEW1 + QINE + + + + + -999 + CLEW1 - Cle Elum R nr Roslyn + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + KACW1 + QINE + + + + + -999 + KACW1 - Kachess R nr Easton + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CHDW1 + QINE + + + + + -999 + CHDW1 - Lake Chelan + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + COEI1 + QINE + + + + + -999 + COEI1 - Coeur D Alene Lake + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CABI1 + QINE + + + + + -999 + CABI1 - Cabinet Gorge Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NOXM8 + QINE + + + + + -999 + NOXM8 - Noxon Rapids Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + KERM8 + QINE + + + + + -999 + KERM8 - Kerr Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HHWM8 + QINE + + + + + -999 + HHWM8 - Hungry Horse Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + KBDM8 + QINE + + + + + -999 + KBDM8 - Kootenai R bl Libby Dam + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PSLI1 + QINE + + + + + -999 + PSLI1 - Priest Lake + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + RODW1 + QINE + + + + + -999 + RODW1 - Ross Res nr Newhalem + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SHAW1 + QINE + + + + + -999 + SHAW1 - Baker R at Concrete + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + UBDW1 + QINE + + + + + -999 + UBDW1 - Upper Baker Lake + KCFS + 2019-11-29 + 21:00:09 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/ohrfc/201911271319_OHRFC_Reservoir_Export_SAGU.xml b/data_assimilation_engine/rfc_ingestion/testdata/ohrfc/201911271319_OHRFC_Reservoir_Export_SAGU.xml new file mode 100644 index 0000000..80113dc --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/ohrfc/201911271319_OHRFC_Reservoir_Export_SAGU.xml @@ -0,0 +1,384 @@ + + + 0.0 + +
+ instantaneous + ADJUSTQ_KNZP1_KNZP1ADJ_Forecast + KNZP1 + QINE + + + + + -999 + Kinzua Dam + 41.8377777778 + -79.0041666667 + -79.0041666667 + 41.8377777778 + 404.7744 + CFS + 2019-11-27 + 13:18:27 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ADJUSTQ_TIOP1_TIOP1ADJ_Forecast + TIOP1 + QINE + + + + + -999 + Tionesta + 41.4730555556 + -79.4394444444 + -79.4394444444 + 41.4730555556 + 335.28 + CFS + 2019-11-27 + 13:18:27 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ADJUSTQ_UCYP1_UCYP1ADJ_Forecast + UCDP1 + QINE + + + + + -999 + Union City + 41.9 + -79.82 + -79.82 + 41.9 + 426.72 + CFS + 2019-11-27 + 13:18:27 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ADJUSTQ_WDRP1_WDRP1ADJ_Forecast + WCDP1 + QINE + + + + + -999 + Woodcock Dam + 41.6958333333 + -80.1083333333 + -80.1083333333 + 41.6958333333 + 344.424 + CFS + 2019-11-27 + 13:18:27 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/ohrfc/201911271327_OHRFC_Reservoir_Export_SAGL.xml b/data_assimilation_engine/rfc_ingestion/testdata/ohrfc/201911271327_OHRFC_Reservoir_Export_SAGL.xml new file mode 100644 index 0000000..886ff31 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/ohrfc/201911271327_OHRFC_Reservoir_Export_SAGL.xml @@ -0,0 +1,479 @@ + + + 0.0 + +
+ instantaneous + ADJUSTQ_GHDP1_GHDP1AJ_Forecast + GHDP1 + QINE + + + + + -999 + Glen Hazel + 41.5530555556 + -78.5963888889 + -78.5963888889 + 41.5530555556 + 466.344 + CFS + 2019-11-27 + 13:25:59 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ADJUSTQ_MHDP1_MHDP1AJ_Forecast + MHDP1 + QINE + + + + + -999 + Mahoning Dam + 40.9275 + -79.2913888889 + -79.2913888889 + 40.9275 + 307.848 + CFS + 2019-11-27 + 13:25:59 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ADJUSTQ_CCDP1_CCDP1AJ_Forecast + CCDP1 + QINE + + + + + -999 + Crooked Creek Dam + 40.7202777778 + -79.5116666667 + -79.5116666667 + 40.7202777778 + 251.46 + CFS + 2019-11-27 + 13:25:59 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ADJUSTQ_CMDP1_CMDP1AJ_Forecast + CMDP1 + QINE + + + + + -999 + Conemaugh Dam + 40.4544444444 + -79.3911111111 + -79.3911111111 + 40.4544444444 + 262.128 + CFS + 2019-11-27 + 13:25:59 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ADJUSTQ_SLTP1_SLTP1AJ_Forecast + LHDP1 + QINE + + + + + -999 + Loyalhanna Dam + 40.4586111111 + -79.4502777778 + -79.4502777778 + 40.4586111111 + 301.752 + CFS + 2019-11-27 + 13:25:59 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/ohrfc/201911271331_OHRFC_Reservoir_Export_SMNU.xml b/data_assimilation_engine/rfc_ingestion/testdata/ohrfc/201911271331_OHRFC_Reservoir_Export_SMNU.xml new file mode 100644 index 0000000..9176f7c --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/ohrfc/201911271331_OHRFC_Reservoir_Export_SMNU.xml @@ -0,0 +1,194 @@ + + + 0.0 + +
+ instantaneous + ADJUSTQ_SWJW2_SSWJW2A_Forecast + SWJW2 + QINE + + + + + -999 + Stonewall Jackson + 39.0033333333 + -80.4741666667 + -80.4741666667 + 39.0033333333 + 310.896 + CFS + 2019-11-27 + 13:30:10 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ADJUSTQ_TYGW2_TYGW2AJ_Forecast + TYGW2 + QINE + + + + + -999 + Tygart Dam + 39.3136111111 + -80.0297222222 + -80.0297222222 + 39.3136111111 + 365.76 + CFS + 2019-11-27 + 13:30:10 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/ohrfc/201911271332_OHRFC_Reservoir_Export_SBVR.xml b/data_assimilation_engine/rfc_ingestion/testdata/ohrfc/201911271332_OHRFC_Reservoir_Export_SBVR.xml new file mode 100644 index 0000000..429fa4b --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/ohrfc/201911271332_OHRFC_Reservoir_Export_SBVR.xml @@ -0,0 +1,479 @@ + + + 0.0 + +
+ instantaneous + ADJUSTQ_BRWO1_BRWO1ADJ_Forecast + BRWO1 + QINE + + + + + -999 + Berlin Dam + 41.0483333333 + -81.0013888889 + -81.0013888889 + 41.0483333333 + 295.656 + CFS + 2019-11-27 + 13:31:20 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ADJUSTQ_BRWO1_MLPO1ADJ_Forecast + MLPO1 + QINE + + + + + -999 + Milton Dam + 41.1313888889 + -80.9713888889 + -80.9713888889 + 41.1313888889 + 275.844 + CFS + 2019-11-27 + 13:31:20 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ADJUSTQ_KITO1_KITO1ADJ_Forecast + KITO1 + QINE + + + + + -999 + Kirwan Dam + 41.1569444444 + -81.0719444444 + -81.0719444444 + 41.1569444444 + 286.512 + CFS + 2019-11-27 + 13:31:20 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ADJUSTQ_MSQO1_MSQO1ADJ_Forecast + MSQO1 + QINE + + + + + -999 + Mosquito Creek Dam + 41.2997222222 + -80.7586111111 + -80.7586111111 + 41.2997222222 + 271.272 + CFS + 2019-11-27 + 13:31:20 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ADJUSTQ_SHDP1_SHDP1ADJ_Forecast + SHDP1 + QINE + + + + + -999 + Shenango Dam + 41.2661111111 + -80.4727777778 + -80.4727777778 + 41.2661111111 + 265.176 + CFS + 2019-11-27 + 13:31:20 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/ohrfc/201911271332_OHRFC_Reservoir_Export_SMNL.xml b/data_assimilation_engine/rfc_ingestion/testdata/ohrfc/201911271332_OHRFC_Reservoir_Export_SMNL.xml new file mode 100644 index 0000000..a542c96 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/ohrfc/201911271332_OHRFC_Reservoir_Export_SMNL.xml @@ -0,0 +1,99 @@ + + + 0.0 + +
+ instantaneous + ADJUSTQ_YGOP1_YGOP1AJ_Forecast + YGOP1 + QINE + + + + + -999 + Youghiogheny Dam + 39.8052777778 + -79.3644444444 + -79.3644444444 + 39.8052777778 + 403.86 + CFS + 2019-11-27 + 13:32:00 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/serfc/201911290600_SERFC_Reservoir_Export.xml b/data_assimilation_engine/rfc_ingestion/testdata/serfc/201911290600_SERFC_Reservoir_Export.xml new file mode 100644 index 0000000..74892f0 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/serfc/201911290600_SERFC_Reservoir_Export.xml @@ -0,0 +1,724 @@ + + + 0.0 + +
+ instantaneous + WDRF1 + QINE + + + + + -9999 + Woodruff + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FOGG1 + QINE + + + + + -9999 + Fort Gaines + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TYLA1 + QINE + + + + + -9999 + Jones Bluff L/D + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MRFA1 + QINE + + + + + -9999 + Millers Ferry + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CLBA1 + QINE + + + + + -9999 + Claiborne + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WETG1 + QINE + + + + + -9999 + West Point + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CHDS1 + QINE + + + + + -9999 + Thurmond Rsvr + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HRTG1 + QINE + + + + + -9999 + Hartwell Dam + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CMMG1 + QINE + + + + + -9999 + Cumming + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CVLG1 + QINE + + + + + -9999 + Allatoona Dam + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CTRG1 + QINE + + + + + -9999 + Carters + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + KERV2 + QINE + + + + + -9999 + Kerr Reservoir + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NUDN7 + QINE + + + + + -9999 + Neuse Falls Dam + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + NHPN7 + QINE + + + + + -9999 + B. Everett Jordan Dam + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WLKN7 + QINE + + + + + -9999 + W Kerr Scott + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PTTV2 + QINE + + + + + -9999 + Philpott Dam + CFS + 2019-11-29 + 06:56:04 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/data_assimilation_engine/rfc_ingestion/testdata/wgrfc/WGRFC_11.27.2019.16.00.xml b/data_assimilation_engine/rfc_ingestion/testdata/wgrfc/WGRFC_11.27.2019.16.00.xml new file mode 100644 index 0000000..9f64298 --- /dev/null +++ b/data_assimilation_engine/rfc_ingestion/testdata/wgrfc/WGRFC_11.27.2019.16.00.xml @@ -0,0 +1,18231 @@ + + 0.0 + +
+ instantaneous + PNTT2 + QINE + + + + + -999.99 + Lake Tawakoni near Wills Point + 32.83 + -95.92 + -95.92 + 32.83 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LFKT2 + QINE + + + + + -999.99 + Lake Fork Reservoir near Quitman + 32.8 + -95.55 + -95.55 + 32.8 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LCRT2 + QINE + + + + + -999.99 + Lake Cherokee + 32.38 + -94.65 + -94.65 + 32.38 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LMVT2 + QINE + + + + + -999.99 + Murvaul Lake near Gary + 32.03 + -94.42 + -94.42 + 32.03 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LMTT2 + QINE + + + + + -999.99 + Martin Lake near Tatum + 32.27 + -94.57 + -94.57 + 32.27 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BKLT2 + QINE + + + + + -999.99 + Toledo Bend Reservoir near Burkeville + 31.175 + -93.5652777778 + -93.5652777778 + 31.175 + 57.912 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LPTT2 + QINE + + + + + -999.99 + Frankston - Lake Palestine + 32.03 + -95.43 + -95.43 + 32.03 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SKCT2 + QINE + + + + + -999.99 + Striker Lake near New Salem + 31.93 + -94.98 + -94.98 + 31.93 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + JSPT2 + QINE + + + + + -999.99 + Sam Rayburn Reservoir + 31.0619444444 + -94.1011111111 + -94.1011111111 + 31.0619444444 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TBLT2 + QINE + + + + + -999.99 + B. A. Steinhagen Lake near Town Bluff + 30.8 + -94.18 + -94.18 + 30.8 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BPRT2 + QINE + + + + + -999.99 + Bridgeport Reservoir above Bridgeport + 33.22 + -97.83 + -97.83 + 33.22 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + EAMT2 + QINE + + + + + -999.99 + Eagle Mountain Reservoir above Fort Worth + 32.88 + -97.47 + -97.47 + 32.88 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FLWT2 + QINE + + + + + -999.99 + Lake Worth above Fort Worth + 32.82 + -97.47 + -97.47 + 32.82 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LWFT2 + QINE + + + + + -999.99 + Lake Weatherford near Weatherford + 32.7725 + -97.6744444444 + -97.6744444444 + 32.7725 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BNBT2 + QINE + + + + + -999.99 + Benbrook - Benbrook Lake + 32.6505555556 + -97.4483333333 + -97.4483333333 + 32.6505555556 + 211.5312 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LART2 + QINE + + + + + -999.99 + Lake Arlington near Arlington + 32.72 + -97.199722 + -97.199722 + 32.72 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GPVT2 + QINE + + + + + -999.99 + Grapevine Lake near Grapevine + 32.97 + -97.05 + -97.05 + 32.97 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + RRLT2 + QINE + + + + + -999.99 + Ray Roberts Lake near Pilot Point + 33.33 + -97.05 + -97.05 + 33.33 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LEWT2 + QINE + + + + + -999.99 + Lewisville - Lewisville Lake + 33.0455555556 + -96.9608333333 + -96.9608333333 + 33.0455555556 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + JPLT2 + QINE + + + + + -999.99 + Joe Pool Lake near Duncanville + 32.6433333333 + -97.0008333333 + -97.0008333333 + 32.6433333333 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GPET2 + QINE + + + + + -999.99 + Venus - Mountain Creek Lake + 32.7319444444 + -96.9430555556 + -96.9430555556 + 32.7319444444 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LVNT2 + QINE + + + + + -999.99 + Lavon Lake near Lavon + 33.03 + -96.48 + -96.48 + 33.03 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FRHT2 + QINE + + + + + -999.99 + Forney - Lake Ray Hubbard + 32.8 + -96.5 + -96.5 + 32.8 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TRNT2 + QINE + + + + + -999.99 + Cedar Creek Reservoir near Trinidad + 32.17 + -96.07 + -96.07 + 32.17 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BDWT2 + QINE + + + + + -999.99 + Ennis - Bardwell Lake + 32.25 + -96.65 + -96.65 + 32.25 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + DAWT2 + QINE + + + + + -999.99 + Dawson - Navarro Mills Lake + 31.95 + -96.6997222222 + -96.6997222222 + 31.95 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FFLT2 + QINE + + + + + -999.99 + Kerens - Richland-Chambers Reservoir + 31.95 + -96.1 + -96.1 + 31.95 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LVDT2 + QINE + + + + + -999.99 + Livingston Reservoir near Goodrich + 30.63 + -95.019722 + -95.019722 + 30.63 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LHPT2 + QINE + + + + + -999.99 + Lake Alan Henry near Justiceburg + 33.07 + -101.049722222 + -101.049722222 + 33.07 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LSWT2 + QINE + + + + + -999.99 + Lake Sweetwater near Sweetwater + 32.43 + -100.26972 + -100.26972 + 32.43 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HAWT2 + QINE + + + + + -999.99 + Nugent - Fort Phantom Hill Reservoir + 32.6 + -99.67 + -99.67 + 32.6 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LSFT2 + QINE + + + + + -999.99 + Paint Creek at Lake Stamford near Haskell + 33.08 + -99.58 + -99.58 + 33.08 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HCRT2 + QINE + + + + + -999.99 + Breckenridge - Hubbard Creek Reservoir + 32.83 + -98.97 + -98.97 + 32.83 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SMKT2 + QINE + + + + + -999.99 + Millers Creek Reservoir near Bomartin + 33.42 + -99.38 + -99.38 + 33.42 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LMGT2 + QINE + + + + + -999.99 + Lake Graham near Graham + 33.13 + -98.62 + -98.62 + 33.13 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PSMT2 + QINE + + + + + -999.99 + Possum Kingdom Lake + 32.87 + -98.43 + -98.43 + 32.87 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LPPT2 + QINE + + + + + -999.99 + Lake Palo Pinto near Santo + 32.65 + -98.269722 + -98.269722 + 32.65 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GBYT2 + QINE + + + + + -999.99 + Granbury - Lake Granbury + 32.3741666667 + -97.6888888889 + -97.6888888889 + 32.3741666667 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LCLT2 + QINE + + + + + -999.99 + Lake Pat Cleburne near Cleburne + 32.28 + -97.42 + -97.42 + 32.28 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + WTYT2 + QINE + + + + + -999.99 + Whitney - Lake Whitney + 31.87 + -97.37 + -97.37 + 31.87 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ALAT2 + QINE + + + + + -999.99 + Aquilla Lake above Aquilla + 31.9 + -97.22 + -97.22 + 31.9 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ACTT2 + QINE + + + + + -999.99 + Waco Lake near Waco + 31.58 + -97.1997222222 + -97.1997222222 + 31.58 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LLET2 + QINE + + + + + -999.99 + Leon Reservoir near Ranger + 32.37 + -98.68 + -98.68 + 32.37 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PCTT2 + QINE + + + + + -999.99 + Proctor Lake near Proctor + 31.97 + -98.5 + -98.5 + 31.97 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BLNT2 + QINE + + + + + -999.99 + Belton Lake near Belton + 31.1 + -97.48 + -97.48 + 31.1 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + STIT2 + QINE + + + + + -999.99 + Stillhouse Hollow Lake near Belton + 31.02 + -97.53 + -97.53 + 31.02 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GGLT2 + QINE + + + + + -999.99 + Georgetown - Lake Georgetown + 30.68 + -97.72 + -97.72 + 30.68 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + GNGT2 + QINE + + + + + -999.99 + Granger - Granger Lake + 30.7 + -97.33 + -97.33 + 30.7 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SOMT2 + QINE + + + + + -999.99 + Somerville Lake near Somerville + 30.3222222222 + -96.5255555556 + -96.5255555556 + 30.3222222222 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LLST2 + QINE + + + + + -999.99 + Lake Limestone near Marquez + 31.35 + -96.32 + -96.32 + 31.35 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CGNT2 + QINE + + + + + -999.99 + Gibbons Creek Reservoir + 30.63 + -96.03 + -96.03 + 30.63 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LTXT2 + QINE + + + + + -999.99 + Lake Texana near Edna + 28.9 + -96.58 + -96.58 + 28.9 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + JBTT2 + QINE + + + + + -999.99 + Vincent - J. B. Thomas Reservoir + 32.58 + -101.13 + -101.13 + 32.58 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CCRT2 + QINE + + + + + -999.99 + Champion Creek Reservoir near Colorado City + 32.27 + -100.87 + -100.87 + 32.27 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CLRT2 + QINE + + + + + -999.99 + Colorado City - Lake Colorado City + 32.33 + -100.92 + -100.92 + 32.33 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + EVST2 + QINE + + + + + -999.99 + Robert Lee - E. V. Spence Reservoir + 31.8794444444 + -100.516944444 + -100.516944444 + 31.8794444444 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BLKT2 + QINE + + + + + -999.99 + Oak Creek Reservoir near Blackwell + 32.05 + -100.3 + -100.3 + 32.05 + 629.412 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SAGT2 + QINE + + + + + -999.99 + SJT - O. C. Fisher Lake + 31.47 + -100.48 + -100.48 + 31.47 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + TBRT2 + QINE + + + + + -999.99 + Twin Buttes Reservoir near San Angelo + 31.3819444444 + -100.538055556 + -100.538055556 + 31.3819444444 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + OHIT2 + QINE + + + + + -999.99 + Voss - O. H. Ivie Reservoir + 31.5 + -99.67 + -99.67 + 31.5 + 472.44 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LKCT2 + QINE + + + + + -999.99 + Lake Coleman near Coleman + 32.05 + -99.5197222222 + -99.5197222222 + 32.05 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LBWT2 + QINE + + + + + -999.99 + Lake Brownwood near Brownwood + 31.83 + -99.0 + -99.0 + 31.83 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BCRT2 + QINE + + + + + -999.99 + Brady Creek Res nr Brady + 31.13 + -99.38 + -99.38 + 31.13 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BUDT2 + QINE + + + + + -999.99 + Lake Buchanan near Burnet + 30.7513888889 + -98.4183333333 + -98.4183333333 + 30.7513888889 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MSDT2 + QINE + + + + + -999.99 + Lake Travis near Austin + 30.3911111111 + -97.9066666667 + -97.9066666667 + 30.3911111111 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SMCT2 + QINE + + + + + -999.99 + Canyon Lake near New Braunfels + 29.8705555556 + -98.1966666667 + -98.1966666667 + 29.8705555556 + 304.8 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CKDT2 + QINE + + + + + -999.99 + Schroeder - Coleto Creek Reservoir + 28.72 + -97.17 + -97.17 + 28.72 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MDLT2 + QINE + + + + + -999.99 + Medina Lake near San Antonio + 29.54 + -98.9336111111 + -98.9336111111 + 29.54 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CTDT2 + QINE + + + + + -999.99 + Choke Canyon Reservoir near Three Rivers + 28.48 + -98.2697222222 + -98.2697222222 + 28.48 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MTHT2 + QINE + + + + + -999.99 + Lake Corpus Christi near Mathis + 28.0380555556 + -97.8797222222 + -97.8797222222 + 28.0380555556 + 8.5344 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + SRLN5 + QINE + + + + + -999.99 + Santa Rosa Lake near Santa Rosa + 35.03 + -104.699722222 + -104.699722222 + 35.03 + 1508.76 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LAKN5 + QINE + + + + + -999.99 + Lake Sumner near Fort Sumner + 34.6169444444 + -104.621944444 + -104.621944444 + 34.6169444444 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + RRRN5 + QINE + + + + + -999.99 + Two Rivers Reservoir + 33.3 + -104.72 + -104.72 + 33.3 + 1127.76 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CBLN5 + QINE + + + + + -999.99 + Brantley Lake near Carlsbad + 32.5466666667 + -104.378611111 + -104.378611111 + 32.5466666667 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + RLAT2 + QINE + + + + + -999.99 + Orla - Red Bluff Reservoir + 31.9061111111 + -103.929444444 + -103.929444444 + 31.9061111111 + 853.44 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + RRRC2 + QINE + + + + + -999.99 + Rio Grande Reservoir near Creede + 37.7166666667 + -107.266666667 + -107.266666667 + 37.7166666667 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ELVN5 + QINE + + + + + -999.99 + El Vado Reservoir near Tierra Amarilla + 36.5941666667 + -106.733333333 + -106.733333333 + 36.5941666667 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ABIN5 + QINE + + + + + -999.99 + Abiquiu Reservoir near Abiquiu + 36.24 + -106.428888889 + -106.428888889 + 36.24 + 1944.624 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + COCN5 + QINE + + + + + -999.99 + Cochiti Lake near Cochiti Pueblo - Rio Grande + 35.63 + -106.32 + -106.32 + 35.63 + 1694.688 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CGRN5 + QINE + + + + + -999.99 + Galisteo Reservior near Cerrillos + 35.47 + -106.22 + -106.22 + 35.47 + 1709.928 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BERN5 + QINE + + + + + -999.99 + Jemez Canyon Reservior near Bernalillo + 35.4 + -106.55 + -106.55 + 35.4 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + EBDN5 + QINE + + + + + -999.99 + Elephant Butte Reservoir near Elephant Butte + 33.1541666667 + -107.191111111 + -107.191111111 + 33.1541666667 + 1343.2536 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PLLC6 + QINE + + + + + -999.99 + Presa Luis Leon, MX + 28.98 + -105.58 + -105.58 + 28.98 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + AMIT2 + QINE + + + + + -999.99 + Amistad Reservoir near Del Rio + 29.4497222222 + -101.055833333 + -101.055833333 + 29.4497222222 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + CRNC7 + QINE + + + + + -999.99 + Presa V Carranza, MX + 27.5 + -100.73 + -100.73 + 27.5 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FALT2 + QINE + + + + + -999.99 + Falcon Reservoir near Falcon Heights + 26.5572222222 + -99.1605555556 + -99.1605555556 + 26.5572222222 + 97.536 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + PMGT4 + QINE + + + + + -999.99 + Presa Marte Gomez, MX + 26.22 + -98.92 + -98.92 + 26.22 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + MADT2 + QINE + + + + + -999.99 + Anzalduas Reservoir near Mission - Rio Grande + 26.13 + -98.33 + -98.33 + 26.13 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + FRGC7 + QINE + + + + + -999.99 + Presa la Fragua, MX + 28.8228 + -100.8333 + -100.8333 + 28.8228 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + LCTT2 + QINE + + + + + -999.99 + Lake Conroe near Conroe + 30.37 + -95.569999 + -95.569999 + 30.37 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + HSJT2 + QINE + + + + + -999.99 + Lake Houston near Sheldon + 29.92 + -95.15 + -95.15 + 29.92 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + ADDT2 + QINE + + + + + -999.99 + Addicks - Addicks Reservoir + 29.8 + -95.62 + -95.62 + 29.8 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ instantaneous + BAKT2 + QINE + + + + + -999.99 + Barker Reservoir near Addicks + 29.77 + -95.65 + -95.65 + 29.77 + 0.0 + CMS + 2019-11-27 + 15:00:03 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/data_assimilation_engine/utils/timeseries.py b/data_assimilation_engine/utils/timeseries.py index b61baed..df7b437 100644 --- a/data_assimilation_engine/utils/timeseries.py +++ b/data_assimilation_engine/utils/timeseries.py @@ -189,7 +189,6 @@ def obs_path(self) -> tuple[str, str]: s3_uri = ( f"s3://{self.obs_prefix}/gages-{self.basin_id}_{self.variable_name}.csv" ) - print(repr(s3_uri)) if fs.exists(s3_uri): return s3_uri else: diff --git a/netcdf_production_sample.py b/netcdf_production_sample.py index 86a7b1c..90a412b 100644 --- a/netcdf_production_sample.py +++ b/netcdf_production_sample.py @@ -1,26 +1,120 @@ import argparse +import os +from datetime import datetime +import time from data_assimilation_engine.output_variables.DataReader import DataReader from data_assimilation_engine.output_variables.DataProcessor import DataProcessor +import data_assimilation_engine.output_variables.utils as utils +import data_assimilation_engine.output_variables.consts as consts def randomize_values(netcdf_file: str, output_file: str) -> None: #Usage: - # python netcdf_wrapper_sample.py randomize sample_data/sample_netcdf/g01123000.nc sample_data/sample_netcdf/catchment_randomvals.nc + # python netcdf_production_sample.py randomize sample_data/sample_netcdf/catchment_output_CNF.nc sample_data/sample_netcdf/catchment_randomvals_cnf.nc reader = DataReader(netcdf_file) reader.assign_random_values(output_file) -def create_template_grid(netcdf_file: str, gpkg_file: str, template_grid_file: str): +def create_template_grid_for_gpkg(netcdf_file: str, gpkg_file: str, template_grid_file: str): #Usage - #python netcdf_wrapper_sample.py create-template-grid sample_data/sample_netcdf/final_test/catchment_randomvals.nc sample_data/sample_gpkg/gages-01123000.gpkg sample_data/sample_netcdf/final_test/grid_template.nc - processor = DataProcessor(netcdf_file, gpkg_file, template_grid_file, True, False) + # python netcdf_production_sample.py create-template-grid sample_data/sample_netcdf/final_test/catchment_randomvals.nc sample_data/sample_gpkg/gages-01123000.gpkg sample_data/sample_netcdf/final_test/new_grid_template.nc sample_data/nwm_output/metadata_config.json + #processor = DataProcessor(netcdf_file, gpkg_file, config_json_file, 'analysis_assim', 'land', 'conus') + processor = DataProcessor(netcdf_file, gpkg_file) + processor.create_template_netcdf_using_config(template_grid_file) -def create_nwm_grid(netcdf_file: str, gpkg_file: str, template_grid_file: str): +def create_nwm_grid(netcdf_file: str, gpkg_file: str, template_grid_file: str, config_json_file: str): #Usage: - #python netcdf_wrapper_sample.py create-nwm-grid sample_data/sample_netcdf/final_test/catchment_randomvals.nc sample_data/sample_gpkg/gages-01123000.gpkg sample_data/sample_netcdf/final_test/grid_template.nc - processor = DataProcessor(netcdf_file, gpkg_file, template_grid_file, False, True) + # python netcdf_production_sample.py create-nwm-grid sample_data/sample_netcdf/final_test/catchment_randomvals.nc sample_data/sample_gpkg/gages-01123000.gpkg sample_data/sample_netcdf/final_test/new_grid_template.nc sample_data/nwm_output/metadata_config.json + processor = DataProcessor(netcdf_file, gpkg_file, config_json_file, 'analysis_assim', 'land', 'conus') + processor.set_template_netcdf(template_grid_file) + output_dir = 'sample_data/sample_netcdf/sample_output/new_test' + processor.produce_nwm_output_grid(output_dir) -def create_nwm_grid_dask(netcdf_file: str, gpkg_file: str, template_grid_file: str): - processor = DataProcessor(netcdf_file, gpkg_file, template_grid_file, False, True) +def download_netcdf_from_nomads(output_folder: str, re_download: bool = False): + #Usage: + # python netcdf_production_sample.py download-nwm-outputs sample_data + utils.download_nwm_data_from_server(output_folder, re_download) + +def extract_netcdf_metadata(netcdf_root_folder: str): + #Usage: + # python netcdf_production_sample.py obtain-netcdf-metadata sample_data/nwm_output metadata + utils.obtain_metadata_information(netcdf_root_folder) + #utils.debug_netcdf_structure_in_folder(netcdf_root_folder) + +def convert_csv_to_netcdf(csv_folder: str): + #Usage: + # python netcdf_production_sample.py convert-csv-to-netcdf sample_data/medium_range_mem2 + utils.convert_csvs_to_netcdf(csv_folder) + +def combine_basin_grids(reference_grid: str, netcdf_folder: str, output_folder: str): + #Usage: + # python netcdf_production_sample.py create-combined-basin-grid sample_data/nwm_output/analysis_assim/nwm.t00z.analysis_assim.land.tm00.conus.nc sample_data/sample_netcdf/sample_output/merge_test sample_data/sample_netcdf/sample_output/merge_test + utils.create_combined_basin_netcdf_products(reference_grid, netcdf_folder, output_folder) + +def overall_netcdf_workflow(ngen_netcdf_output_file: str, ngen_gpkg_file: str, output_folder: str, + troute_output_file: str = '', troute_lakeout_file: str = ''): + #Usage + # python netcdf_production_sample.py test-overall-workflow sample_data/outputs_0629/ngen_outputs/catchment_output_sr_01123000.nc sample_data/sample_gpkg/gauge_01123000.gpkg sample_data/outputs_0629 + download = False + create_config = False + if download: + re_download = False + download_netcdf_from_nomads(output_folder, re_download) # Download unqiue set of NWM data + print('Downloaded NWM data from NOMADS') + + if create_config: + # Create the config file for the nwm output netcdf files + extract_netcdf_metadata(output_folder) + print('Extracted netcdf metadata') + + # Read all metadata from the config file for each NWM output category, class and domain. + json_file = os.path.join(output_folder, consts.NWM_CONFIG_LOCAL_FOLDER, + consts.NWM_CONFIG_FILE_NAME + consts.NWM_CONFIG_FILE_SUFFIX + ".json") + netcdf_metadata_list = [] + if os.path.isfile(json_file): + netcdf_metadata_list = utils.read_output_variables_info_from_config(json_file) + else: + raise ValueError("Specified config file does not exist") + # print('Read output variables info from config') + + # Begin data processing + # reader = DataReader(ngen_netcdf_output_file) + # randomized_nc_name = 'ngen_randomvals_' + datetime.now().strftime("%Y%m%d_%H%M%S") + '.nc' + # ngen_netcdf_output_file = os.path.join(output_folder, randomized_nc_name) + # reader.add_missing_variables(ngen_netcdf_output_file) + # reader.assign_random_values(ngen_netcdf_output_file) + + start_time = time.perf_counter() + + + processor = DataProcessor(ngen_netcdf_output_file, ngen_gpkg_file) + ngen_template_nc_folder = os.path.join(output_folder, consts.NWM_NGEN_TEMPLATE_FOLDER) + nwm_output_folder = os.path.join(output_folder, consts.NWM_OUTPUT_FOLDER) + for mdata in netcdf_metadata_list: + if mdata.output_class == 'analysis_assim' and mdata.category =='channel_rt' and not troute_output_file: + raise ValueError("T-Route output file is not specified") + if mdata.output_class == 'analysis_assim' and mdata.category =='reservoir' and not troute_lakeout_file: + raise ValueError("T-Route lakeout file is not specified") + + # if mdata.output_class == 'medium_range' and mdata.category =='land_1' and mdata.domain == 'conus': + if mdata.output_class == 'analysis_assim' and mdata.category =='land' and mdata.domain == 'conus': + # if mdata.output_class == 'medium_range_blend' and mdata.category =='terrain_rt' and mdata.domain == 'conus': + # if mdata.output_class == 'long_range' and mdata.category =='land_4' and mdata.domain == 'conus': + log_file = os.path.join(output_folder, consts.LOG_FOLDER, 'nwm_' + mdata.output_class + '.' + mdata.category + + '.' + mdata.domain + '.' + datetime.now().strftime("%Y%m%d_%H%M%S") + '.log') + processor.log_file = log_file + processor.create_template_netcdf_using_config(mdata, ngen_template_nc_folder) + if mdata.category == 'channel_rt': + processor.set_troute_netcdf(troute_output_file) + if mdata.category == 'reservoir': + processor.set_troute_lakeout_netcdf(troute_lakeout_file) + # processor.produce_nwm_output_product(mdata, nwm_output_folder) + # print(f"Ready to process: {mdata.output_class}, {mdata.category}, {mdata.domain}") + # if any(cat.lower() in mdata.category.lower() for cat in ['channel_rt', 'reservoir', 'total_water']): + # print(f"----The requested category - {mdata.category} - is not a gridded netcdf. The functionality is not implemented yet") + # continue + end_time = time.perf_counter() + duration_minutes = (end_time - start_time) / 60 + print(f"----Function execution time: {duration_minutes:.2f} minutes") def main() -> None: @@ -34,25 +128,65 @@ def main() -> None: parser_randomize.add_argument("output_file") #create template grid - parser_grid = subparsers.add_parser("create-template-grid") - parser_grid.add_argument("catchment_netcdf_file") - parser_grid.add_argument("catchment_gpkg_file") - parser_grid.add_argument("template_grid_file") + parser_template_grid = subparsers.add_parser("create-template-grid") + parser_template_grid.add_argument("catchment_netcdf_file") + parser_template_grid.add_argument("catchment_gpkg_file") + parser_template_grid.add_argument("template_grid_file") + parser_template_grid.add_argument("config_json_file") #create nwm grid parser_nwm_grid = subparsers.add_parser("create-nwm-grid") parser_nwm_grid.add_argument("catchment_netcdf_file") parser_nwm_grid.add_argument("catchment_gpkg_file") parser_nwm_grid.add_argument("template_grid_file") + parser_nwm_grid.add_argument("config_json_file") + #download current NWM output netcdfs from nomads server + parser_nwm_download = subparsers.add_parser("download-nwm-outputs") + parser_nwm_download.add_argument("output_folder_path") + parser_nwm_download.add_argument("re_download") + + #Extract metadata info from current NWM output netcdfs that are downloaded + parser_nwm_info = subparsers.add_parser("obtain-netcdf-metadata") + parser_nwm_info.add_argument("local_folder_path") + + #convert CSV files from ngen outputs to netcdf + parser_csv_to_nc = subparsers.add_parser("convert-csv-to-netcdf") + parser_csv_to_nc.add_argument("csv_folder_path") + + #create basin level netcdf using timestep netcdfs + parser_basin_grids = subparsers.add_parser("create-combined-basin-grid") + parser_basin_grids.add_argument("reference_grid") + parser_basin_grids.add_argument("timestep_grids_folder") + parser_basin_grids.add_argument("output_basin_grids_folder") + + #overall netcdf workflow + overall_workflow = subparsers.add_parser("test-overall-workflow") + overall_workflow.add_argument("ngen_netcdf_output_file") + overall_workflow.add_argument("ngen_gpkg_file") + overall_workflow.add_argument("output_folder") + overall_workflow.add_argument("troute_out_file") + overall_workflow.add_argument("troute_lakeout_file") + args = parser.parse_args() if args.command == "randomize": randomize_values(args.input_file, args.output_file) elif args.command == "create-template-grid": - create_template_grid(args.catchment_netcdf_file, args.catchment_gpkg_file, args.template_grid_file) + create_template_grid_for_gpkg(args.catchment_netcdf_file, args.catchment_gpkg_file, args.template_grid_file) elif args.command == "create-nwm-grid": - create_nwm_grid(args.catchment_netcdf_file, args.catchment_gpkg_file, args.template_grid_file) + create_nwm_grid(args.catchment_netcdf_file, args.catchment_gpkg_file, args.template_grid_file, args.config_json_file) + elif args.command == "download-nwm-outputs": + download_netcdf_from_nomads(args.download_url, args.output_folder_path) + elif args.command == "obtain-netcdf-metadata": + extract_netcdf_metadata(args.local_folder_path) + elif args.command == "convert-csv-to-netcdf": + convert_csv_to_netcdf(args.csv_folder_path) + elif args.command == "create-combined-basin-grid": + combine_basin_grids(args.reference_grid, args.timestep_grids_folder, args.output_basin_grids_folder) + elif args.command == "test-overall-workflow": + overall_netcdf_workflow(args.ngen_netcdf_output_file, args.ngen_gpkg_file, args.output_folder, + args.troute_out_file, args.troute_lakeout_file) else: parser.print_help() diff --git a/postprocessing_wrapper_sample.py b/postprocessing_wrapper_sample.py new file mode 100644 index 0000000..73c38a7 --- /dev/null +++ b/postprocessing_wrapper_sample.py @@ -0,0 +1,51 @@ +from data_assimilation_engine.output_variables.NetCdfProductionManager import netcdf_production_workflow + +download_inputs = [ + "sample_data/outputs_root", + "download" +] +netcdf_production_workflow(download_inputs) # download and metadata_config.json creation. + +template_inputs = [ + "sample_data/outputs_root", + "sample_data/ngen_netcdfs/catchment_output_3n.nc", + "sample_data/sample_gpkg/vpu_3n.gpkg", + "sample_data/outputs_root/configs/metadata_config.json", + None, + "template", +] +# create templates for geopackage extents. +# assumes download is complete and metadata_config.json is available. +netcdf_production_workflow(template_inputs) + +production_inputs = [ + "sample_data/outputs_root", + "sample_data/ngen_netcdfs/catchment_output_3n.nc", + "sample_data/sample_gpkg/vpu_3n.gpkg", + "sample_data/troute_netcdfs/toute_output_3n.nc", + "sample_data/troute_netcdfs/toute_lakeout_3n.nc", + "sample_data/outputs_root/configs/metadata_config.json", + None, + "4", + "analysis_assim", + "output", +] +# create output for geopackage extents. +# assumes that templating is completed. it also means that the metadata_config.json is available. +netcdf_production_workflow(production_inputs) + +overall_workflow_inputs = [ + "sample_data/outputs_root", + "sample_data/ngen_netcdfs/catchment_output_3n.nc" + "sample_data/sample_gpkg/vpu_3n.gpkg", + "sample_data/troute_netcdfs/toute_output_3n.nc", + "sample_data/troute_netcdfs/toute_lakeout_3n.nc", + "sample_data/outputs_root/configs/metadata_config.json", + None, + "0", + "medium_range_blend", + "all", +] +# overall workflow for creating products for geopackage extents. +# this downloads the nomads data, creates templates and produces outputs. +netcdf_production_workflow(overall_workflow_inputs) diff --git a/sample_data/sample_gpkg/gages-09359500.gpkg b/sample_data/sample_gpkg/gages-09359500.gpkg deleted file mode 100644 index 09d8a29..0000000 Binary files a/sample_data/sample_gpkg/gages-09359500.gpkg and /dev/null differ diff --git a/sample_data/sample_gpkg/gages-13240000.gpkg b/sample_data/sample_gpkg/gages-13240000.gpkg deleted file mode 100644 index 90b72d7..0000000 Binary files a/sample_data/sample_gpkg/gages-13240000.gpkg and /dev/null differ diff --git a/sample_data/sample_gpkg/gages-01123000.gpkg b/sample_data/sample_gpkg/gauge_01123000.gpkg similarity index 56% rename from sample_data/sample_gpkg/gages-01123000.gpkg rename to sample_data/sample_gpkg/gauge_01123000.gpkg index 511295d..93cf5ee 100644 Binary files a/sample_data/sample_gpkg/gages-01123000.gpkg and b/sample_data/sample_gpkg/gauge_01123000.gpkg differ diff --git a/sample_data/sample_gpkg/gauge_09353500.gpkg b/sample_data/sample_gpkg/gauge_09353500.gpkg new file mode 100644 index 0000000..bcfd6f6 Binary files /dev/null and b/sample_data/sample_gpkg/gauge_09353500.gpkg differ diff --git a/sample_data/sample_gpkg/gauge_09359500.gpkg b/sample_data/sample_gpkg/gauge_09359500.gpkg deleted file mode 100644 index 09d8a29..0000000 Binary files a/sample_data/sample_gpkg/gauge_09359500.gpkg and /dev/null differ diff --git a/sample_data/sample_gpkg/gauge_12175500.gpkg b/sample_data/sample_gpkg/gauge_12175500.gpkg new file mode 100644 index 0000000..0e2c2a5 Binary files /dev/null and b/sample_data/sample_gpkg/gauge_12175500.gpkg differ