ArcGIS Server Mapservice probe added

.gitignore

@@ -63,3 +63,4 @@ GeoHealthCheck.conf
 
 # Data
 GeoHealthCheck/data.db
+/.idea

GeoHealthCheck/app.py

@@ -737,8 +737,7 @@ def test(resource_identifier):
         return redirect(request.referrer)
 
     from healthcheck import run_test_resource
-    result = run_test_resource(
-        resource)
+    result = run_test_resource(resource)
 
     if request.method == 'GET':
         if result.message == 'Skipped':

GeoHealthCheck/config_main.py

@@ -115,6 +115,9 @@
     'GeoHealthCheck.plugins.probe.ogcfeat',
     'GeoHealthCheck.plugins.probe.ogc3dtiles',
     'GeoHealthCheck.plugins.probe.esrifs',
+    'GeoHealthCheck.plugins.probe.esrims',
+    'GeoHealthCheck.plugins.probe.oracle',
+    'GeoHealthCheck.plugins.probe.postgres',
     'GeoHealthCheck.plugins.probe.ghcreport',
     'GeoHealthCheck.plugins.probe.mapbox',
 
@@ -166,9 +169,18 @@
     'OGC:3DTiles': {
         'probe_class': 'GeoHealthCheck.plugins.probe.ogc3dtiles.OGC3DTiles'
     },
+    'ORACLE': {
+        'probe_class': 'GeoHealthCheck.plugins.probe.oracle.OracleDrilldown'
+    },
+    'POSTGRES': {
+        'probe_class': 'GeoHealthCheck.plugins.probe.postgres.PostgresDrilldown'
+    },
     'ESRI:FS': {
         'probe_class': 'GeoHealthCheck.plugins.probe.esrifs.ESRIFSDrilldown'
     },
+    'ESRI:MS': {
+        'probe_class': 'GeoHealthCheck.plugins.probe.esrims.ESRIMSDrilldown'
+    },
     'Mapbox:TileJSON': {
         'probe_class': 'GeoHealthCheck.plugins.probe.mapbox.TileJSON'
     },

GeoHealthCheck/enums.py

@@ -78,7 +78,10 @@
         'label': 'OGC 3D Tiles (OGC3D)'
     },
     'ESRI:FS': {
-        'label': 'ESRI ArcGIS FeatureServer (FS)'
+        'label': 'ESRI ArcGIS FeatureServer'
+    },
+    'ESRI:MS': {
+        'label': 'ESRI ArcGIS MapServer'
     },
     'Mapbox:TileJSON': {
         'label': 'Mapbox TileJSON Service (TileJSON)'
@@ -92,6 +95,12 @@
     'FTP': {
         'label': 'File Transfer Protocol (FTP)'
     },
+    'ORACLE': {
+        'label': 'Oracle Database'
+    },
+    'POSTGRES': {
+        'label': 'Postgres Database'
+    },
     'OSGeo:GeoNode': {
         'label': 'GeoNode instance'
     },

GeoHealthCheck/healthcheck.py

@@ -158,6 +158,9 @@ def sniff_test_resource(config, resource_type, url):
                          'OGCFeat': [urlopen],
                          'OGC:3DTiles': [urlopen],
                          'ESRI:FS': [urlopen],
+                         'ESRI:MS': [urlopen],
+                         'ORACLE': [oracle_connect],
+                         'POSTGRES': [postgres_connect],
                          'OGC:STA': [urlopen],
                          'WWW:LINK': [urlopen],
                          'FTP': [urlopen],
@@ -244,7 +247,9 @@ def sniff_test_resource(config, resource_type, url):
             elif resource_type == 'OGCFeat':
                 title = 'OGC API Features (OAFeat)'
             elif resource_type == 'ESRI:FS':
-                title = 'ESRI ArcGIS FS'
+                title = 'ESRI ArcGIS FeatureService'
+            elif resource_type == 'ESRI:MS':
+                title = 'ESRI ArcGIS MapService'
             elif resource_type == 'OGC:3DTiles':
                 title = 'OGC 3D Tiles'
             else:
@@ -309,6 +314,25 @@ def geonode_make_tags(base_url):
     return [tag_name]
 
 
+def oracle_connect(connect_string):
+    d = {}
+    for c in connect_string.split(";"):
+        key = c.split("=")[0]
+        value = c.split("=")[1]
+        d[key] = value
+    base_name = 'Oracle : {}'.format(d["service"])
+    return True
+
+def postgres_connect(connect_string):
+    d = {}
+    for c in connect_string.split(";"):
+        key = c.split("=")[0]
+        value = c.split("=")[1]
+        d[key] = value
+    base_name = 'Postgres : {}'.format(d["database"])
+    return True
+
+
 if __name__ == '__main__':
     print('START - Running health check tests on %s'
           % datetime.utcnow().isoformat())

GeoHealthCheck/models.py

@@ -430,7 +430,7 @@ def run_count(self):
     def get_capabilities_url(self):
         if self.resource_type.startswith('OGC:') \
                 and self.resource_type not in \
-                ['OGC:STA', 'OGCFeat', 'ESRI:FS', 'OGC:3DTiles']:
+                ['OGC:STA', 'OGCFeat', 'ESRI:FS', 'ESRI:MS', 'OGC:3DTiles']:
             url = '%s%s' % (bind_url(self.url),
                             RESOURCE_TYPES[self.resource_type]['capabilities'])
         else:

GeoHealthCheck/plugins/probe/esrims.py

@@ -0,0 +1,200 @@
+from GeoHealthCheck.probe import Probe
+from GeoHealthCheck.result import Result, push_result
+
+
+class ESRIMSDrilldown(Probe):
+    """
+    Probe for ESRI MapServer endpoint "drilldown": starting
+    with top /MapServer endpoint: get Layers and get Features on these.
+    Test e.g. from https://sampleserver6.arcgisonline.com/arcgis/rest/services
+    (at least sampleserver6 is ArcGIS 10.6.1 supporting Paging).
+    """
+
+    NAME = 'ESRIMS Drilldown'
+
+    DESCRIPTION = 'Traverses an ESRI MapServer ' \
+                  '(REST) API endpoint by drilling down'
+
+    RESOURCE_TYPE = 'ESRI:MS'
+
+    REQUEST_METHOD = 'GET'
+
+    PARAM_DEFS = {
+        'drilldown_level': {
+            'type': 'string',
+            'description': 'How heavy the drilldown should be.\
+                            basic: test presence of Capabilities, \
+                            full: go through Layers, get Features',
+            'default': 'basic',
+            'required': True,
+            'range': ['basic', 'full']
+        }
+    }
+    """Param defs"""
+
+    def __init__(self):
+        Probe.__init__(self)
+
+    def get_request_headers(self):
+        headers = Probe.get_request_headers(self)
+
+        # Clear possibly dangling ESRI header
+        # https://github.com/geopython/GeoHealthCheck/issues/293
+        if 'X-Esri-Authorization' in headers:
+            del headers['X-Esri-Authorization']
+
+        if 'Authorization' in headers:
+            # https://enterprise.arcgis.com/en/server/latest/
+            #     administer/linux/about-arcgis-tokens.htm
+            auth_val = headers['Authorization']
+            if 'Bearer' in auth_val:
+                headers['X-Esri-Authorization'] = headers['Authorization']
+        return headers
+
+    def perform_esrims_get_request(self, url):
+        response = self.perform_get_request(url).json()
+        error_msg = 'code=%d message=%s'
+        # May have error like:
+        # {
+        #   "error" :
+        #   {
+        #     "code" : 499,
+        #     "message" : "Token Required",
+        #     "messageCode" : "GWM_0003",
+        #     "details" : [
+        #       "Token Required"
+        #     ]
+        #   }
+        # }
+        if 'error' in response:
+            err = response['error']
+            raise Exception(error_msg % (err['code'], err['message']))
+
+        return response
+
+    def perform_request(self):
+        """
+        Perform the drilldown.
+        """
+
+        # Be sure to use bare root URL http://.../MapServer
+        ms_url = self._resource.url.split('?')[0]
+
+        # Assemble request templates with root MS URL
+        req_tpl = {
+            'ms_caps': ms_url + '?f=json',
+
+            'layer_caps': ms_url + '/%d?f=json',
+
+            'get_features': ms_url +
+            '/%d/query?where=1=1'
+            '&outFields=*&resultOffset=0&'
+            'resultRecordCount=1&f=json',
+
+            'get_feature_by_id': ms_url +
+            '/%d/query?where=%s=%s&outFields=*&f=json'
+        }
+
+        # 1. Test top Service endpoint existence
+        result = Result(True, 'Test Service Endpoint')
+        result.start()
+        layers = []
+        try:
+            ms_caps = self.perform_esrims_get_request(req_tpl['ms_caps'])
+            for attr in ['currentVersion', 'layers']:
+                val = ms_caps.get(attr, None)
+                if val is None:
+                    msg = 'Service: missing attr: %s' % attr
+                    result = push_result(
+                        self, result, False, msg, 'Test Layer:')
+                    continue
+
+            layers = ms_caps.get('layers', [])
+
+        except Exception as err:
+            result.set(False, str(err))
+
+        result.stop()
+        self.result.add_result(result)
+
+        if len(layers) == 0:
+            return
+
+        # 2. Test each Layer Capabilities
+        result = Result(True, 'Test Layer Capabilities')
+        result.start()
+        layer_ids = []
+        layer_caps = []
+        try:
+
+            for layer in layers:
+                if layer['subLayerIds'] is None:
+                    layer_ids.append(layer['id'])
+
+            for layer_id in layer_ids:
+                layer_caps.append(self.perform_esrims_get_request(
+                    req_tpl['layer_caps'] % layer_id))
+
+        except Exception as err:
+            result.set(False, str(err))
+
+        result.stop()
+        self.result.add_result(result)
+
+        if self._parameters['drilldown_level'] == 'basic':
+            return
+
+        # ASSERTION: will do full drilldown from here
+
+        # 3. Test getting Features from Layers
+        result = Result(True, 'Test 1 record for each layer in Layers')
+        result.start()
+        layer_id = -1
+        try:
+            for layer_id in layer_ids:
+                try:
+                    features = self.perform_esrims_get_request(
+                        req_tpl['get_features'] % layer_id)
+                    # Get the name of the OBJECTID FieldName
+                    # In a FeatureService this is direct available from features['objectIdFieldName']
+                    # In a MapService this must be done by looping through the response fields and find the field with type 'esriFieldTypeOID'
+                    obj_id_field_name = None
+                    for f in features['fields']:
+                        if f['type'] == 'esriFieldTypeOID':
+                            obj_id_field_name = f['name']
+                            break
+
+                    features = features['features']
+                    if len(features) == 0:
+                        continue
+
+                    # At least one Feature: use first and try to get by id
+                    object_id = features[0]['attributes'][obj_id_field_name]
+                    feature = self.perform_get_request(
+                        req_tpl['get_feature_by_id'] % (
+                            layer_id, obj_id_field_name,
+                            str(object_id))).json()
+
+                    feature = feature['features']
+                    if len(feature) == 0:
+                        msg = 'layer: %d: missing Feature - id: %s' \
+                              % (layer_id, str(object_id))
+                        result = push_result(
+                            self, result, False, msg,
+                            'Test Layer: %d' % layer_id)
+
+                except Exception as e:
+                    msg = 'GetLayer: id=%d: err=%s ' \
+                          % (layer_id, str(e))
+                    result = push_result(
+                        self, result, False, msg, 'Test Get Features:')
+                    continue
+
+        except Exception as err:
+            result.set(False, 'Layer: id=%d : err=%s'
+                       % (layer_id, str(err)))
+
+        result.stop()
+
+        # Add to overall Probe result
+        self.result.add_result(result)

GeoHealthCheck/plugins/probe/http.py

@@ -58,7 +58,7 @@ class HttpPost(HttpGet):
         """
 
     REQUEST_METHOD = 'POST'
-    REQUEST_HEADERS = {'content-type': '{post_content_type}'}
+    REQUEST_HEADERS = {'Content-Type': '{post_content_type}'}
     REQUEST_TEMPLATE = '{body}'
 
     PARAM_DEFS = {