Merge pull request #35 from groundlight/pytest-against-md

Run tests against the python code in our documentation

Author: mjvogelsong

.github/workflows/publish-docs.yaml

@@ -29,8 +29,8 @@ jobs:
       - name: Deploy website
         # Docs: https://github.com/peaceiris/actions-gh-pages#%EF%B8%8F-docusaurus
         uses: peaceiris/actions-gh-pages@v3
-        # Only run this on pushes to the `docs`
-        if: github.ref == 'refs/heads/docs'
+        # Only deploy on pushes to the `main` branch
+        if: github.ref == 'refs/heads/main'
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./docs/build

.github/workflows/publish-pypi.yaml

@@ -11,9 +11,9 @@ jobs:
       POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_PUBLISH_TOKEN }}
     steps:
       - name: install python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
-          python-version: 3.9
+          python-version: 3.10
       - name: install poetry
         run: |
           pip install -U pip

.github/workflows/test-integ.yaml

@@ -28,9 +28,6 @@ jobs:
           ]
         install_numpy: [true, false]
         install_pillow: [true, false]
-    env:
-      # This is associated with the "sdk-integ-test" user, credentials on 1password
-      GROUNDLIGHT_API_TOKEN: ${{ secrets.GROUNDLIGHT_API_TOKEN }}
     steps:
       - name: get code
         uses: actions/checkout@v3
@@ -56,4 +53,12 @@ jobs:
         run: |
           poetry run pip install pillow
       - name: run tests
+        env:
+          # This is associated with the "sdk-integ-test" user, credentials on 1password
+          GROUNDLIGHT_API_TOKEN: ${{ secrets.GROUNDLIGHT_API_TOKEN }}
         run: make test-integ
+      - name: run docs tests
+        run: make test-docs
+        env:
+          # This is associated with the "sdk-test-prod" user, credentials on 1password
+          GROUNDLIGHT_API_TOKEN: ${{ secrets.GROUNDLIGHT_API_TOKEN_PROD }}

Makefile

@@ -2,7 +2,6 @@
 
 install:  ## Install the package from source
 	poetry install
-	npm install
 
 generate: install  ## Generate the SDK from our public openapi spec
 	node_modules/.bin/openapi-generator-cli generate -i spec/public-api.yaml \
@@ -11,10 +10,11 @@ generate: install  ## Generate the SDK from our public openapi spec
 	poetry run datamodel-codegen  --input spec/public-api.yaml --output generated/model.py
 	poetry run black .
 
-
 test-local: install  ## Run integration tests against an API server running at http://localhost:8000/device-api (needs GROUNDLIGHT_API_TOKEN)
 	GROUNDLIGHT_ENDPOINT="http://localhost:8000/" poetry run pytest --cov=src test --log-cli-level INFO
 
-
 test-integ: install  ## Run integration tests against the integ API server (needs GROUNDLIGHT_API_TOKEN)
 	GROUNDLIGHT_ENDPOINT="https://api.integ.groundlight.ai/" poetry run pytest --cov=src test --log-cli-level INFO
+
+test-docs: install  ## Run the example code and tests in our docs against the prod API (needs GROUNDLIGHT_API_TOKEN)
+	poetry run pytest --markdown-docs docs -v

docs/docs/building-applications/working-with-detectors.md

@@ -19,44 +19,80 @@ detector = gl.create_detector(name="your_detector_name", query="is this what we
 
 ### Retrieve an existing detector
 
-```python
+<!-- Don't test because the ID can't be faked -->
+
+```python notest
+from groundlight import Groundlight
+
+gl = Groundlight()
+# highlight-start
 detector = gl.get_detector(id="YOUR_DETECTOR_ID")
+# highlight-end
 ```
 
 ### List your detectors
 
 ```python
+from groundlight import Groundlight
+
+gl = Groundlight()
+# highlight-start
 # Defaults to 10 results per page
 detectors = gl.list_detectors()
 
-# Pagination: 3rd page of 25 results per page
-detectors = gl.list_detectors(page=3, page_size=25)
+# Pagination: 1st page of 5 results per page
+detectors = gl.list_detectors(page=1, page_size=5)
+# highlight-end
 ```
 
 ### Retrieve an image query
 
 In practice, you may want to check for a new result on your query. For example, after a cloud reviewer labels your query. For example, you can use the `image_query.id` after the above `submit_image_query()` call.
 
-```python
-image_query = gl.get_image_query(id="YOUR_IMAGE_QUERY_ID")
+<!-- Don't test because the ID can't be faked -->
+
+```python notest
+from groundlight import Groundlight
+
+gl = Groundlight()
+# highlight-start
+image_query = gl.get_image_query(id="iq_YOUR_IMAGE_QUERY_ID")
+# highlight-end
 ```
 
 ### List your previous image queries
 
 ```python
+from groundlight import Groundlight
+
+gl = Groundlight()
+# highlight-start
 # Defaults to 10 results per page
 image_queries = gl.list_image_queries()
 
-# Pagination: 3rd page of 25 results per page
-image_queries = gl.list_image_queries(page=3, page_size=25)
+# Pagination: 1st page of 5 results per page
+image_queries = gl.list_image_queries(page=1, page_size=5)
+# highlight-end
 ```
 
 ### Adding labels to existing image queries
 
 Groundlight lets you start using models by making queries against your very first image, but there are a few situations where you might either have an existing dataset, or you'd like to handle the escalation response programatically in your own code but still include the label to get better responses in the future. With your `image_query` from either `submit_image_query()` or `get_image_query()` you can add the label directly. Note that if the query is already in the escalation queue due to low ML confidence or audit thresholds, it may also receive labels from another source.
 
 ```python
-add_label(image_query, 'YES').   # or 'NO'
+from groundlight import Groundlight
+from PIL import Image
+import requests
+
+gl = Groundlight()
+d = gl.get_or_create_detector(name="doorway", query="Is the doorway open?")
+image = Image.open(
+    requests.get("https://images.selfstorage.com/large-compress/2174925f24362c479b2.jpg", stream=True).raw
+)
+image_query = gl.submit_image_query(detector=d, image=image)
+# highlight-start
+gl.add_label(image_query, 'YES') # or 'NO'
+# highlight-end
 ```
 
-The only valid labels at this time are `'YES'` and `'NO'`
+The only valid labels at this time are `'YES'` and `'NO'`.

docs/docs/getting-started/getting-started.mdx

@@ -2,14 +2,19 @@
 
 ## Computer Vision powered by Natural Language
 
-Build a working computer vision system in just 5 lines of python:
+Build a working computer vision system in just a few lines of python:
 
 ```python
 from groundlight import Groundlight
+from PIL import Image
+import requests
 
 gl = Groundlight()
-d = gl.get_or_create_detector(name="door", query="Is the door open?")
-image_query = gl.submit_image_query(detector=d, image=jpeg_img)
+d = gl.get_or_create_detector(name="doorway", query="Is the doorway open?")
+image = Image.open(
+    requests.get("https://images.selfstorage.com/large-compress/2174925f24362c479b2.jpg", stream=True).raw
+)
+image_query = gl.submit_image_query(detector=d, image=image)
 print(f"The answer is {image_query.result}")
 ```
 
@@ -27,33 +32,32 @@ _Note: The SDK is currently in "beta" phase. Interfaces are subject to change in
     pip3 install groundlight
     ```
 
-1.  To access the API, you need an API token. You can create one on the
-    [groundlight web app](https://app.groundlight.ai/reef/my-account/api-tokens). The API token
-    should be stored securely. You can use it directly in your code to initialize the SDK like:
-
-    ```python
-    gl = Groundlight(api_token="<YOUR_API_TOKEN>")
-    ```
-
-    This is an easy way to get started, but is NOT a best practice. Please do not commit your API Token to version control! Instead we recommend setting the `GROUNDLIGHT_API_TOKEN` environment variable outside your code so that the SDK can find it automatically.
+1.  Head over to the [groundlight web
+    app](https://app.groundlight.ai/reef/my-account/api-tokens) to create an API token. You will
+    need to set the `GROUNDLIGHT_API_TOKEN` environment variable to access the API.
 
     ```shell
     export GROUNDLIGHT_API_TOKEN=api_2GdXMflhJi6L_example
     ```
 
-1.  Create a python file.
+1.  Create a python script.
 
-    ```python title="hello.py"
+    ```python title="ask.py"
     from groundlight import Groundlight
+    from PIL import Image
+    import requests
 
     gl = Groundlight()
-    d = gl.get_or_create_detector(name="door", query="Is the door open?")
-    image_query = gl.submit_image_query(detector=d, image=jpeg_img)
+    d = gl.get_or_create_detector(name="doorway", query="Is the doorway open?")
+    image = Image.open(
+        requests.get("https://images.selfstorage.com/large-compress/2174925f24362c479b2.jpg", stream=True).raw
+    )
+    image_query = gl.submit_image_query(detector=d, image=image)
     print(f"The answer is {image_query.result}")
     ```
 
 1.  Run it!
 
     ```shell
-    $ python hello.py
+    $ python ask.py
     ```

docs/docs/getting-started/managing-confidence.md

@@ -6,10 +6,25 @@ The desired confidence level is set as the escalation threshold on your detector
 
 For example, say you want to set your desired confidence level to 0.95, but that you're willing to wait up to 60 seconds to get a confident response.
 
-```python
+<!-- TODO: turn on `confidence` parameter, and re-enable tests! -->
+
+```python notest
+from groundlight import Groundlight
+from PIL import Image
+import requests
+
+gl = Groundlight()
+image = Image.open(
+    requests.get("https://www.photos-public-domain.com/wp-content/uploads/2010/11/over_flowing_garbage_can.jpg", stream=True).raw
+)
+
+# highlight-start
 d = gl.get_or_create_detector(name="trash", query="Is the trash can full?", confidence=0.95)
-image_query = gl.submit_image_query(detector=d, image=jpeg_img, wait=60)
+
 # This will wait until either 60 seconds have passed or the confidence reaches 0.95
+image_query = gl.submit_image_query(detector=d, image=jpeg_img, wait=60)
+# highlight-end
+
 print(f"The answer is {image_query.result}")
 ```
 
@@ -22,12 +37,12 @@ Higher confidence will get higher accuracy, but will generally require longer la
 
 Or if you want to run as fast as possible, set `wait=0`. This way you will only get the ML results, without waiting for escalation. Image queries which are below the desired confidence level still be escalated for further analysis, and the results are incorporated as training data to improve your ML model, but your code will not wait for that to happen.
 
-```python
+```python notest continuation
 image_query = gl.submit_image_query(detector=d, image=jpeg_img, wait=0)
 ```
 
 If the returned result was generated from an ML model, you can see the confidence score returned for the image query:
 
-```python
+```python notest continuation
 print(f"The confidence is {image_query.result.confidence}")
 ```

docs/docusaurus.config.js

@@ -74,7 +74,11 @@ const config = {
             position: "left",
             label: "Docs",
           },
-          { to: "/docs/category/building-applications", label: "Applications", position: "left" },
+          {
+            to: "/docs/category/building-applications",
+            label: "Applications",
+            position: "left",
+          },
           {
             href: "https://github.com/groundlight/python-sdk",
             label: "GitHub",

pyproject.toml

@@ -1,40 +1,42 @@
 [tool.poetry]
-name = "groundlight"
-version = "0.7.3"
-license = "MIT"
-readme = "README.md"
-homepage = "https://www.groundlight.ai"
-description = "Build computer vision systems from natural language with Groundlight"
 authors = ["Groundlight AI <[email protected]>"]
+description = "Build computer vision systems from natural language with Groundlight"
+homepage = "https://www.groundlight.ai"
+license = "MIT"
+name = "groundlight"
 packages = [
-    { include = "**/*.py", from = "src" },
-    { include = "**/*.py", from = "generated" },
+    {include = "**/*.py", from = "generated"},
+    {include = "**/*.py", from = "src"},
 ]
+readme = "README.md"
+version = "0.7.4"
 
 [tool.poetry.dependencies]
-python = ">=3.7.0,<4.0"
-python-dateutil = "^2.8.2"
-urllib3 = "^1.26.9"
-frozendict = "^2.3.2"
 certifi = "^2021.10.8"
+frozendict = "^2.3.2"
+pillow = "^9.0.0"
 pydantic = "^1.7.4"
+python = ">=3.7.0,<4.0"
+python-dateutil = "^2.8.2"
 requests = "^2.28.2"
+urllib3 = "^1.26.9"
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.group.dev.dependencies]
+autoflake = "^1.4"
 black = "^22.3.0"
-pytest-cov = "^3.0.0"
-pylint = "^2.13.9"
+datamodel-code-generator = "^0.12.1"
 flake8 = "^4.0.1"
+isort = "^5.10.1"
 mypy = "^0.950"
+pylint = "^2.13.9"
 pytest = "^7.0.1"
-isort = "^5.10.1"
-autoflake = "^1.4"
-datamodel-code-generator = "^0.12.1"
+pytest-cov = "^3.0.0"
+pytest-markdown-docs = "^0.4.3"
 pytest-mock = "^3.10.0"
 
 [build-system]
-requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
+requires = ["poetry-core>=1.2.0"]
 
 [tool.black]
 line-length = 120

src/groundlight/client.py

@@ -82,7 +82,7 @@ def get_detector(self, id: Union[str, Detector]) -> Detector:
 
     def get_detector_by_name(self, name: str) -> Optional[Detector]:
         # TODO: Do this on server.
-        detector_list = self.list_detectors(page_size=100)
+        detector_list = self.list_detectors(page_size=250)
         for d in detector_list.results:
             if d.name == name:
                 return d