Catalog (HydrologicEngineeringCenter#16)

* initial catalog function

* initial work for condensed catalog

* housekeeping for testing

* wrapping up first version of catalog

Author: ktarbet

Readme.md

@@ -17,8 +17,9 @@ These are the driving design ideas and goals of the hec-dss-python project (subj
 |Easy transition from Jython|HEC products use an existing [Java/Jython API](https://www.hec.usace.army.mil/confluence/dssdocs/dssvueum/scripting/reading-and-writing-to-hec-dss-files).  We will loosely follow that design | simplify porting from Jython|
 | hec_dss_native.py | native binding layer   | isolate interactions with low level library(if performance is an issue this Ctypes layer can be replaced ) |
 | hec_dss.py | Programmer entry point ; Python API  | Hides interactions with hec_dss_native, seek to be simple user experience|
+|catalog.py|manage list of DSS objects (catalog) |create condensed catalog perspective|
 |Pandas_Series_Utilities.py [future](https://github.com/HydrologicEngineeringCenter/hec-dss-python/issues/8) |NumPy/pandas support |provide features such as dataframes, separate from hec-dss.py; can be developed by different/parallel developers|
-| Easy to get started |nothing to install, just copy python files and shared library   |require minimal privileges to install| 
+|Easy to get started |nothing to install, just copy python files and shared library   |require minimal privileges to install| 
 
 
 ## Features

catalog.py

@@ -0,0 +1,63 @@
+from record_type import RecordType
+from dsspath import DssPath
+from datetime import datetime
+
+class Catalog:
+    """manage list of objects inside a DSS database"""
+    def __init__(self, items, recordTypes):
+        self.rawCatalog = items
+        self.rawRecordTypes = recordTypes
+        self.timeSeriesDictNoDates = {}  # key is path without date, value is a list dates
+        self.recordTypeDict = {} # key is path w/o date, value is recordType
+        self.__create_condensed_catalog()
+ 
+    def __create_condensed_catalog(self):
+        """
+          condensed catalog combines time-series records into a single condensed path
+          other record types are not condensed.
+          time-series records must match all parts except the D (date) part to be combined.
+        """
+        self.items = []
+        for i in range(len(self.rawCatalog)):
+            rawPath = self.rawCatalog[i]
+            recordType = RecordType.RecordTypeFromInt(self.rawRecordTypes[i])
+            path = DssPath(rawPath,recordType)
+            cleanPath = str(path.pathWithoutDate())
+            self.recordTypeDict[cleanPath] = recordType 
+            # if timeseries - accumulate dates within a dataset
+            if path.isTimeSeries():
+                tsRecords = self.timeSeriesDictNoDates.setdefault(cleanPath,[])
+                t = datetime.strptime(path.D,"%d%b%Y")
+                tsRecords.append(t)
+            else: 
+                # add NON time-series to list (nothing else needed)
+                self.items.append(path) 
+
+        # go through each timeSeriesDictNoDates, and sort each list of dates
+        # use first and last to create the condensed path 
+        for key in self.timeSeriesDictNoDates:
+            dateList = sorted(self.timeSeriesDictNoDates[key])
+            condensedDpart = dateList[0].strftime("%d%b%Y")
+            if len(dateList) >1:
+                condensedDpart +="-"+ dateList[-1].strftime("%d%b%Y")
+            # insert condensed D part into path used as key
+            rt = self.recordTypeDict[key]
+            p = DssPath(key,rt)
+            p.D = condensedDpart
+            self.items.append(p)
+    
+    def print(self):
+        for ds in self.items:
+            print(ds)
+
+    def __iter__(self):
+        self.index = 0  # Initialize the index to 0
+        return self
+
+    def __next__(self):
+        if self.index < len(self.items):
+            result = self.items[self.index]
+            self.index += 1
+            return result
+        else:
+            raise StopIteration

dateconverter.py

@@ -3,13 +3,14 @@
 class DateConverter:
 
 
-    #
-    # convert python datetime to DSS 24:00 style string
-    # 2023-08-25 09:32:46.832952 -> 25Aug2023 09:23:47
-    # 2023-08-25 00:00:00.0000000 -> 24Aug2023 24:00
-    # 2023-08-25 00:10:00.0000000 -> 25Aug2023 00:10
     @staticmethod
     def dss_datetime_from_string(dt):
+        """ 
+        convert python datetime to DSS 24:00 style string
+        2023-08-25 09:32:46.832952 -> 25Aug2023 09:23:47
+        2023-08-25 00:00:00.0000000 -> 24Aug2023 24:00
+        2023-08-25 00:10:00.0000000 -> 25Aug2023 00:10
+        """
         if dt.time() == time(0, 0, 0, 0): # midnight
             # subract one day
             dtm1 = dt - timedelta(days=1)
@@ -22,6 +23,9 @@ def dss_datetime_from_string(dt):
 
     @staticmethod
     def date_times_from_julian_array(times_julian, time_granularity_seconds, julian_base_date):
+        """"
+        convert from DSS integer datetime array to python datetime array
+        """
         if times_julian is None:
             raise ValueError("Time Series Times array was None. Something didn't work right in DSS.")
 
@@ -42,10 +46,11 @@ def date_times_from_julian_array(times_julian, time_granularity_seconds, julian_
 
 
 
-dt = datetime.today()
-x = DateConverter.dss_datetime_from_string(dt)
-print(x)
-midnight = datetime(2023,8,3,0,0,0)
-x = DateConverter.dss_datetime_from_string(midnight)
-print(x)
+if __name__ == "__main__":
+    dt = datetime.today()
+    x = DateConverter.dss_datetime_from_string(dt)
+    print(x)
+    midnight = datetime(2023,8,3,0,0,0)
+    x = DateConverter.dss_datetime_from_string(midnight)
+    print(x)
 

dsspath.py

@@ -0,0 +1,45 @@
+from record_type import RecordType
+
+class DssPath:
+    """manage parts of DSS path /A/B/C/D/E/F/ 
+    condenses D part for timeseries records
+    """
+
+    _timeSeriesFamily = [ RecordType.IrregularTimeSeries, RecordType.RegularTimeSeries, RecordType.RegularTimeSeriesProfile ]
+
+
+    def __init__(self,path,recType):
+        """
+        path is raw dss pathname
+        recType is a RecordType , such as RecordType.RegularTimeSeries
+        """
+        if path[0]!='/' or path[-1]!= '/':
+            raise Exception("Invalid DSS Path: '"+path+"'")
+        path = path[1:-1]  # remove beginning and ending '/'
+        self.rawPath= path
+        
+        split_parts = path.split('/')
+        if len(split_parts) >= 6:
+            self.A, self.B, self.C, self.D, self.E, self.F = split_parts[:6]
+        self.recType = recType
+        
+    def __str__(self):
+        return "/"+self.A+"/"+self.B+"/"+self.C+"/"+self.D+"/"+self.E+"/"+self.F+"/"
+    
+    def pathWithoutDate(self):
+        s = "/"+self.A+"/"+self.B+"/"+self.C+"//"+self.E+"/"+self.F+"/"
+        rval = DssPath(s,self.recType)
+        return rval
+    
+    def isTimeSeries(self):
+        return self.recType in DssPath._timeSeriesFamily
+    
+    def print(self):
+        print("a:"+self.path.A)
+        print("b:"+self.path.B)
+        print("c:"+self.path.C)
+        print("d:"+self.path.D)
+        print("e:"+self.path.E)
+        print("f:"+self.path.F)
+
+    

hec_dss.py

@@ -2,6 +2,7 @@
 from datetime import datetime
 from dateconverter import DateConverter
 from timeseries import TimeSeries
+from catalog import Catalog
 import os
 
 class HecDss:
@@ -72,23 +73,22 @@ def put(self,ts):
 
         self._native.hec_dss_tsStoreRegular(ts.pathname,startDate,startTime,ts.values,quality,False,ts.units,ts.dataType)
 
+    def getCatalog(self):
+        paths,recordTypes = self._native.hec_dss_catalog()
+        return Catalog(paths,recordTypes)
 
     def recordCount(self):
         return self._native.hec_dss_record_count()
 
     def setDebugLevel(self,level):
         return self._native.hec_dss_set_debug_level(level)
 
-			
-#import pdb;pdb.set_trace()
-dss = HecDss("sample7.dss")
-print("record count = "+str(dss.recordCount()))
-t1 = datetime(2005, 1, 1)
-t2 = datetime(2005, 1 ,4)
-tsc = dss.get("//SACRAMENTO/PRECIP-INC//1Day/OBS/",t1,t2)
-tsc.print_to_console()
-tsc2 = dss.get("//SACRAMENTO/TEMP-MAX//1Day/OBS/",t1,t2)
-tsc2.print_to_console()
-tsc.pathname = "//SACRAMENTO/PRECIP-INC//1Day/OBS-modified/"
-#dss.setDebugLevel(15)
-dss.put(tsc)
+
+if __name__ == "__main__":
+    #import pdb;pdb.set_trace()
+    dss = HecDss("sample7.dss")
+    catalog = dss.getCatalog()
+    for p in catalog:
+        print(p)
+    #print(catalog[0:5])
+    #dss.setDebugLevel(15)

hec_dss_native.py

@@ -1,14 +1,14 @@
 import ctypes
 from ctypes import c_char, c_double, c_int,byref, create_string_buffer
+import array
 
 
 class HecDssNative:
+    """Wrapper for Native method calls to hecdss.dll or libhecdss.so """
+
     def __init__(self):
         self.dll = ctypes.CDLL("hecdss")
 
-
-    
-
     def hec_dss_open(self,dss_filename):
         self.dll.hec_dss_open.argtypes = [ctypes.c_char_p, ctypes.POINTER(ctypes.c_void_p)]
         self.dll.hec_dss_open.restype = ctypes.c_int
@@ -45,6 +45,8 @@ def __hec_dss_set_value(self,name,value):
     def hec_dss_set_debug_level(self,value):
         self.__hec_dss_set_value("mlvl",value)
 
+    
+
     def hec_dss_export_to_file(self,path,outputFile,startDate,startTime,endDate,endTime):
 
         self.dll.hec_dss_export_to_file.argtypes = [
@@ -60,6 +62,49 @@ def hec_dss_export_to_file(self,path,outputFile,startDate,startTime,endDate,endT
 
         result = self.dll.hec_dss_export_to_file(self.handle,path, outputFile, startDate, startTime, endDate, endTime)
 
+    def hec_dss_CONSTANT_MAX_PATH_SIZE(self): 
+        f = self.dll.hec_dss_CONSTANT_MAX_PATH_SIZE
+        f.restype = ctypes.c_int
+        return f()
+
+    def hec_dss_catalog(self,filter=""):
+        """
+        retrieves a list of objects in a DSS database
+
+        returns a list of paths, and recordTypes
+
+        """
+        count = self.hec_dss_record_count()
+        pathBufferSize = self.hec_dss_CONSTANT_MAX_PATH_SIZE()
+        self.dll.hec_dss_catalog.argtypes = [
+            ctypes.c_void_p,        # dss (assuming it's a pointer)
+            ctypes.c_char_p,        # pathBuffer
+            ctypes.POINTER(ctypes.c_int),  # recordTypes
+            ctypes.c_char_p,        # pathFilter
+            ctypes.c_int,           # count
+            ctypes.c_int            # pathBufferItemSize
+        ]
+        self.dll.hec_dss_catalog.restype = ctypes.c_int
+
+        c_rawCatalog = create_string_buffer(count * pathBufferSize)
+        pathFilter = filter.encode("ascii")
+        recordTypes = (ctypes.c_int32 * count)()
+
+        pathNameList = []
+        
+        numRecords = self.dll.hec_dss_catalog(self.handle,c_rawCatalog, recordTypes, pathFilter, count, pathBufferSize)
+        recordTypeArray =[]
+        recordTypeArray.extend(list(recordTypes[:count]))
+        for i in range(numRecords):
+            start = i * pathBufferSize
+            end = start + pathBufferSize
+            s = c_rawCatalog[start:end].decode('ascii').replace('\x00','')
+            #print(f"str='{s}'")
+            pathNameList.append(s)
+        
+        return pathNameList,recordTypeArray
+
+
     def hec_dss_tsGetSizes(self,pathname,
                             startDate, startTime,
                             endDate, endTime,numberValues,qualityElementSize):

record_type.py

@@ -0,0 +1,36 @@
+from enum import Enum
+
+class RecordType(Enum):
+    Unknown = 0
+    RegularTimeSeriesProfile = 1
+    RegularTimeSeries = 2
+    IrregularTimeSeries = 3
+    PairedData = 4
+    Text = 5
+    Grid = 6
+    Tin = 7
+    LocationInfo = 8
+
+    @staticmethod
+    def RecordTypeFromInt(recType):
+        rval = RecordType.Unknown
+
+        if 100 <= recType < 110:
+            if recType == 102 or recType == 107:
+                rval = RecordType.RegularTimeSeriesProfile
+            else:
+                rval = RecordType.RegularTimeSeries
+        elif 110 <= recType < 200:
+            rval = RecordType.IrregularTimeSeries
+        elif 200 <= recType < 300:
+            rval = RecordType.PairedData
+        elif 300 <= recType < 400:
+            rval = RecordType.Text
+        elif 400 <= recType < 450:
+            rval = RecordType.Grid
+        elif recType == 450:
+            rval = RecordType.Tin
+        elif recType == 20:
+            rval = RecordType.LocationInfo
+
+        return rval

tests.py

@@ -1,6 +1,6 @@
 from hec_dss import HecDss
 from datetime import datetime
-
+from catalog import Catalog
 class Tests:
 
    @staticmethod
@@ -16,9 +16,45 @@ def test_issue9():
         for path in pathnames:
             print(f"reading {path}")
             tsc = dss.get(path,t1,t2)
-            print("len(tsc.values) = {len(tsc.values)}")
+            print(f"len(tsc.values) = {len(tsc.values)}")
             assert(len(tsc.values)>1)
 
+   @staticmethod
+   def basic_tests():
+        dss = HecDss("sample7.dss")
+        print("record count = "+str(dss.recordCount()))
+        catalog = dss.getCatalog()
+        print(catalog[0:5])
+
+        t1 = datetime(2005, 1, 1)
+        t2 = datetime(2005, 1 ,4)
+        tsc = dss.get("//SACRAMENTO/PRECIP-INC//1Day/OBS/",t1,t2)
+        tsc.print_to_console()
+        assert(len(tsc.values)>0)
+        tsc2 = dss.get("//SACRAMENTO/TEMP-MAX//1Day/OBS/",t1,t2)
+        tsc2.print_to_console()
+        assert(len(tsc2.values)>0)
+        # save to a new path
+        tsc.pathname = "//SACRAMENTO/PRECIP-INC//1Day/OBS-modified/"
+        dss.put(tsc)
+        tsc3 = dss.get(tsc.pathname,t1,t2)
+        assert(len(tsc3.values)== len(tsc.values))
+
+   @staticmethod
+   def catalog_test():
+    rawPaths = [
+      "//SACRAMENTO/TEMP-MIN/01Jan1989/1Day/OBS/",
+      "//SACRAMENTO/TEMP-MIN/01Jan1990/1Day/OBS/",
+      "//SACRAMENTO/TEMP-MIN/01Jan1991/1Day/OBS/",
+      "//SACRAMENTO/TEMP-MIN/01Jan1992/1Day/OBS/",
+      "//SACRAMENTO/TEMP-MIN/01Jan1993/1Day/OBS/"
+    ]
+    rt = [100,100,100,100,100]
+    c = Catalog(rawPaths,rt)
+    c.print()
 
-Tests.test_issue9()
 
+if __name__ == "__main__":
+   #Tests.test_issue9()
+   #Tests.basic_tests()
+   Tests.catalog_test()