Skip to content

Commit 172d24b

Browse files
timmarkhuffTim HuffAuto-format BotTim Huff
authored
Inspections (#81)
* adding basic handling for inspection_id * Automatically reformatting code * added inspection_id support * Automatically reformatting code * adding more methods for inspections * Automatically reformatting code * trivial change to comment * adding method to update a detector's confidence threshold * Automatically reformatting code * fixing docstring * Automatically reformatting code * fixing a comment * fixing some linter issues * Automatically reformatting code * fixing another linter issue * Automatically reformatting code * another linter issue * linter issue * Automatically reformatting code * linter issues * Automatically reformatting code * linter issues * changes for linter * more linter changes * trivial changes to comments * Automatically reformatting code * trivial change for linter * adding type hint * Automatically reformatting code * trivial changes to spacing * Automatically reformatting code * changing stop_inspection to return a string of the result * Automatically reformatting code * linter changes * resolving merge conflicts * Automatically reformatting code * responding to PR comments * Automatically reformatting code * linter issues * fixing detector_id conversion issue * removing debugging line * Automatically reformatting code * linter issue * adding some tests * Automatically reformatting code * fixing a test * Automatically reformatting code * adding more tests * adding some more tests and fixing linter issues * Automatically reformatting code * fixing linter issues * Automatically reformatting code * fixing linter issues * fixing linter issues * Automatically reformatting code * fixing linter issues * fixing linter issues * fixing linter issues * fixing linter issues * linter changes * resolving more merge conflicts * refining some tests * bumping version to 0.10.2 * Automatically reformatting code * adding another test for inspections metadata * Automatically reformatting code * responding to PR comments * Automatically reformatting code * fixing params issue * Automatically reformatting code * fixing linting issue * Automatically reformatting code * updating type hint * removing unused testing fixture * Automatically reformatting code * fixing error handling for submitting inspection metadata when inspection is closed * fixing exception for updating metadata --------- Co-authored-by: Tim Huff <[email protected]> Co-authored-by: Auto-format Bot <[email protected]> Co-authored-by: Tim Huff <[email protected]>
1 parent 7c7bde0 commit 172d24b

File tree

4 files changed

+324
-9
lines changed

4 files changed

+324
-9
lines changed

pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ packages = [
99
{include = "**/*.py", from = "src"},
1010
]
1111
readme = "README.md"
12-
version = "0.10.1"
12+
version = "0.11.0"
1313

1414
[tool.poetry.dependencies]
1515
certifi = "^2021.10.8"

src/groundlight/client.py

Lines changed: 45 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -165,12 +165,13 @@ def list_image_queries(self, page: int = 1, page_size: int = 10) -> PaginatedIma
165165
image_queries.results = [self._fixup_image_query(iq) for iq in image_queries.results]
166166
return image_queries
167167

168-
def submit_image_query(
168+
def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments
169169
self,
170170
detector: Union[Detector, str],
171171
image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray],
172172
wait: Optional[float] = None,
173173
human_review: Optional[str] = None,
174+
inspection_id: Optional[str] = None,
174175
) -> ImageQuery:
175176
"""Evaluates an image with Groundlight.
176177
:param detector: the Detector object, or string id of a detector like `det_12345`
@@ -187,9 +188,12 @@ def submit_image_query(
187188
only if the ML prediction is not confident.
188189
If set to `ALWAYS`, always send the image query for human review.
189190
If set to `NEVER`, never send the image query for human review.
191+
:param inspection_id: Most users will omit this. For accounts with Inspection Reports enabled,
192+
this is the ID of the inspection to associate with the image query.
190193
"""
191194
if wait is None:
192195
wait = self.DEFAULT_WAIT
196+
193197
detector_id = detector.id if isinstance(detector, Detector) else detector
194198

195199
image_bytesio: ByteStreamWrapper = parse_supported_image_types(image)
@@ -203,16 +207,25 @@ def submit_image_query(
203207
if human_review is not None:
204208
params["human_review"] = human_review
205209

206-
raw_image_query = self.image_queries_api.submit_image_query(**params)
207-
image_query = ImageQuery.parse_obj(raw_image_query.to_dict())
210+
# If no inspection_id is provided, we submit the image query using image_queries_api (autogenerated via OpenAPI)
211+
# However, our autogenerated code does not currently support inspection_id, so if an inspection_id was
212+
# provided, we use the private API client instead.
213+
if inspection_id is None:
214+
raw_image_query = self.image_queries_api.submit_image_query(**params)
215+
image_query = ImageQuery.parse_obj(raw_image_query.to_dict())
216+
else:
217+
params["inspection_id"] = inspection_id
218+
iq_id = self.api_client.submit_image_query_with_inspection(**params)
219+
image_query = self.get_image_query(iq_id)
220+
208221
if wait:
209222
threshold = self.get_detector(detector).confidence_threshold
210223
image_query = self.wait_for_confident_result(image_query, confidence_threshold=threshold, timeout_sec=wait)
211224
return self._fixup_image_query(image_query)
212225

213226
def wait_for_confident_result(
214227
self,
215-
image_query: ImageQuery,
228+
image_query: Union[ImageQuery, str],
216229
confidence_threshold: float,
217230
timeout_sec: float = 30.0,
218231
) -> ImageQuery:
@@ -222,7 +235,10 @@ def wait_for_confident_result(
222235
:param confidence_threshold: The minimum confidence level required to return before the timeout.
223236
:param timeout_sec: The maximum number of seconds to wait.
224237
"""
225-
# TODO: Add support for ImageQuery id instead of object.
238+
# Convert from image_query_id to ImageQuery if needed.
239+
if isinstance(image_query, str):
240+
image_query = self.get_image_query(image_query)
241+
226242
start_time = time.time()
227243
next_delay = self.POLLING_INITIAL_DELAY
228244
target_delay = 0.0
@@ -263,3 +279,27 @@ def add_label(self, image_query: Union[ImageQuery, str], label: Union[Label, str
263279
api_label = convert_display_label_to_internal(image_query_id, label)
264280

265281
return self.api_client._add_label(image_query_id, api_label) # pylint: disable=protected-access
282+
283+
def start_inspection(self) -> str:
284+
"""For users with Inspection Reports enabled only.
285+
Starts an inspection report and returns the id of the inspection.
286+
"""
287+
return self.api_client.start_inspection()
288+
289+
def update_inspection_metadata(self, inspection_id: str, user_provided_key: str, user_provided_value: str) -> None:
290+
"""For users with Inspection Reports enabled only.
291+
Add/update inspection metadata with the user_provided_key and user_provided_value.
292+
"""
293+
self.api_client.update_inspection_metadata(inspection_id, user_provided_key, user_provided_value)
294+
295+
def stop_inspection(self, inspection_id: str) -> str:
296+
"""For users with Inspection Reports enabled only.
297+
Stops an inspection and raises an exception if the response from the server
298+
indicates that the inspection was not successfully stopped.
299+
Returns a str with result of the inspection (either PASS or FAIL).
300+
"""
301+
return self.api_client.stop_inspection(inspection_id)
302+
303+
def update_detector_confidence_threshold(self, detector_id: str, confidence_threshold: float) -> None:
304+
"""Updates the confidence threshold of a detector given a detector_id."""
305+
self.api_client.update_detector_confidence_threshold(detector_id, confidence_threshold)

src/groundlight/internalapi.py

Lines changed: 182 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
1+
import json
12
import logging
23
import os
34
import random
45
import time
56
import uuid
67
from functools import wraps
7-
from typing import Callable, Optional
8+
from typing import Callable, Dict, Optional, Union
89
from urllib.parse import urlsplit, urlunsplit
910

1011
import requests
1112
from model import Detector, ImageQuery
1213
from openapi_client.api_client import ApiClient, ApiException
1314

15+
from groundlight.images import ByteStreamWrapper
1416
from groundlight.status_codes import is_ok
1517

1618
logger = logging.getLogger("groundlight.sdk")
@@ -225,3 +227,182 @@ def _get_detector_by_name(self, name: str) -> Detector:
225227
f"We found multiple ({parsed['count']}) detectors with the same name. This shouldn't happen.",
226228
)
227229
return Detector.parse_obj(parsed["results"][0])
230+
231+
@RequestsRetryDecorator()
232+
def submit_image_query_with_inspection( # noqa: PLR0913 # pylint: disable=too-many-arguments
233+
self,
234+
detector_id: str,
235+
patience_time: float,
236+
body: ByteStreamWrapper,
237+
inspection_id: str,
238+
human_review: str = "DEFAULT",
239+
) -> str:
240+
"""Submits an image query to the API and returns the ID of the image query.
241+
The image query will be associated to the inspection_id provided.
242+
"""
243+
244+
url = f"{self.configuration.host}/posichecks"
245+
246+
params: Dict[str, Union[str, float, bool]] = {
247+
"inspection_id": inspection_id,
248+
"predictor_id": detector_id,
249+
"patience_time": patience_time,
250+
}
251+
252+
# In the API, 'send_notification' is used to control human_review escalation. This will eventually
253+
# be deprecated, but for now we need to support it in the following manner:
254+
if human_review == "ALWAYS":
255+
params["send_notification"] = True
256+
elif human_review == "NEVER":
257+
params["send_notification"] = False
258+
else:
259+
pass # don't send the send_notifications param, allow "DEFAULT" behavior
260+
261+
headers = self._headers()
262+
headers["Content-Type"] = "image/jpeg"
263+
264+
response = requests.request("POST", url, headers=headers, params=params, data=body.read())
265+
266+
if not is_ok(response.status_code):
267+
logger.info(response)
268+
raise InternalApiError(
269+
status=response.status_code,
270+
reason=f"Error submitting image query with inspection ID {inspection_id} on detector {detector_id}",
271+
http_resp=response,
272+
)
273+
274+
return response.json()["id"]
275+
276+
@RequestsRetryDecorator()
277+
def start_inspection(self) -> str:
278+
"""Starts an inspection, returns the ID."""
279+
url = f"{self.configuration.host}/inspections"
280+
281+
headers = self._headers()
282+
283+
response = requests.request("POST", url, headers=headers, json={})
284+
285+
if not is_ok(response.status_code):
286+
raise InternalApiError(
287+
status=response.status_code,
288+
reason="Error starting inspection.",
289+
http_resp=response,
290+
)
291+
292+
return response.json()["id"]
293+
294+
@RequestsRetryDecorator()
295+
def update_inspection_metadata(self, inspection_id: str, user_provided_key: str, user_provided_value: str) -> None:
296+
"""Add/update inspection metadata with the user_provided_key and user_provided_value.
297+
298+
The API stores inspections metadata in two ways:
299+
1) At the top level of the inspection with user_provided_id_key and user_provided_id_value. This is a
300+
kind of "primary" piece of metadata for the inspection. Only one key/value pair is allowed at this level.
301+
2) In the user_metadata field as a dictionary. Multiple key/value pairs are allowed at this level.
302+
303+
The first piece of metadata presented to an inspection will be assumed to be the user_provided_id_key and
304+
user_provided_id_value. All subsequent pieces metadata will be stored in the user_metadata field.
305+
306+
"""
307+
url = f"{self.configuration.host}/inspections/{inspection_id}"
308+
309+
headers = self._headers()
310+
311+
# Get inspection in order to find out:
312+
# 1) if user_provided_id_key has been set
313+
# 2) if the inspection is closed
314+
response = requests.request("GET", url, headers=headers)
315+
316+
if not is_ok(response.status_code):
317+
raise InternalApiError(
318+
status=response.status_code,
319+
reason=f"Error getting inspection details for inspection {inspection_id}.",
320+
http_resp=response,
321+
)
322+
if response.json()["status"] == "COMPLETE":
323+
raise ValueError(f"Inspection {inspection_id} is closed. Metadata cannot be added.")
324+
325+
payload = {}
326+
327+
# Set the user_provided_id_key and user_provided_id_value if they were not previously set.
328+
response_json = response.json()
329+
if not response_json.get("user_provided_id_key"):
330+
payload["user_provided_id_key"] = user_provided_key
331+
payload["user_provided_id_value"] = user_provided_value
332+
333+
# Get the existing keys and values in user_metadata (if any) so that we don't overwrite them.
334+
metadata = response_json["user_metadata"]
335+
if not metadata:
336+
metadata = {}
337+
338+
# Submit the new metadata
339+
metadata[user_provided_key] = user_provided_value
340+
payload["user_metadata_json"] = json.dumps(metadata)
341+
response = requests.request("PATCH", url, headers=headers, json=payload)
342+
343+
if not is_ok(response.status_code):
344+
raise InternalApiError(
345+
status=response.status_code,
346+
reason=f"Error updating inspection metadata on inspection {inspection_id}.",
347+
http_resp=response,
348+
)
349+
350+
@RequestsRetryDecorator()
351+
def stop_inspection(self, inspection_id: str) -> str:
352+
"""Stops an inspection and raises an exception if the response from the server does not indicate success.
353+
Returns a string that indicates the result (either PASS or FAIL). The URCap requires this.
354+
"""
355+
url = f"{self.configuration.host}/inspections/{inspection_id}"
356+
357+
headers = self._headers()
358+
359+
# Closing an inspection generates a new inspection PDF. Therefore, if the inspection
360+
# is already closed, just return "COMPLETE" to avoid unnecessarily generating a new PDF.
361+
response = requests.request("GET", url, headers=headers)
362+
363+
if not is_ok(response.status_code):
364+
raise InternalApiError(
365+
status=response.status_code,
366+
reason=f"Error checking the status of {inspection_id}.",
367+
http_resp=response,
368+
)
369+
370+
if response.json().get("status") == "COMPLETE":
371+
return "COMPLETE"
372+
373+
payload = {"status": "COMPLETE"}
374+
375+
response = requests.request("PATCH", url, headers=headers, json=payload)
376+
377+
if not is_ok(response.status_code):
378+
raise InternalApiError(
379+
status=response.status_code,
380+
reason=f"Error stopping inspection {inspection_id}.",
381+
http_resp=response,
382+
)
383+
384+
return response.json()["result"]
385+
386+
@RequestsRetryDecorator()
387+
def update_detector_confidence_threshold(self, detector_id: str, confidence_threshold: float) -> None:
388+
"""Updates the confidence threshold of a detector."""
389+
390+
# The API does not validate the confidence threshold,
391+
# so we will validate it here and raise an exception if necessary.
392+
if confidence_threshold < 0 or confidence_threshold > 1:
393+
raise ValueError(f"Confidence threshold must be between 0 and 1. Got {confidence_threshold}.")
394+
395+
url = f"{self.configuration.host}/predictors/{detector_id}"
396+
397+
headers = self._headers()
398+
399+
payload = {"confidence_threshold": confidence_threshold}
400+
401+
response = requests.request("PATCH", url, headers=headers, json=payload)
402+
403+
if not is_ok(response.status_code):
404+
raise InternalApiError(
405+
status=response.status_code,
406+
reason=f"Error updating detector: {detector_id}.",
407+
http_resp=response,
408+
)

0 commit comments

Comments
 (0)