Merge branch 'dev' into rm_testtypesfrompages

Author: manuel-sommer

.github/workflows/test-helm-chart.yml

@@ -114,7 +114,12 @@ jobs:
       - name: Update values in HELM chart
         if: startsWith(github.head_ref, 'renovate/') || startsWith(github.head_ref, 'dependabot/')
         run: |
-          yq -i '.annotations."artifacthub.io/changes" += "- kind: changed\n  description: ${{ github.event.pull_request.title }}\n"' helm/defectdojo/Chart.yaml
+          title=${{ github.event.pull_request.title }}
+          chars='{}:[],&*#?|-<>=!%@'
+          for c in $(echo "$chars" | grep -o .); do
+            title="${title//"$c"/_}"
+          done
+          yq -i '.annotations."artifacthub.io/changes" += "- kind: changed\n  description: $title\n"' helm/defectdojo/Chart.yaml
           git add helm/defectdojo/Chart.yaml
           git commit -m "ci: update Chart annotations from PR #${{ github.event.pull_request.number }}" || echo "No changes to commit"
 

components/package.json

@@ -1,6 +1,6 @@
 {
   "name": "defectdojo",
-  "version": "2.53.0-dev",
+  "version": "2.54.0-dev",
   "license" : "BSD-3-Clause",
   "private": true,
   "dependencies": {

docker/entrypoint-uwsgi.sh

@@ -27,9 +27,9 @@ python3 manage.py check
 
 DD_UWSGI_LOGFORMAT_DEFAULT='[pid: %(pid)|app: -|req: -/-] %(addr) (%(dd_user)) {%(vars) vars in %(pktsize) bytes} [%(ctime)] %(method) %(uri) => generated %(rsize) bytes in %(msecs) msecs (%(proto) %(status)) %(headers) headers in %(hsize) bytes (%(switches) switches on core %(core))'
 
-EXTRA_ARGS=""
+DD_UWSGI_EXTRA_ARGS="${DD_UWSGI_EXTRA_ARGS:-}"
 if [ -n "${DD_UWSGI_MAX_FD}" ]; then
-    EXTRA_ARGS="${EXTRA_ARGS} --max-fd ${DD_UWSGI_MAX_FD}"
+    DD_UWSGI_EXTRA_ARGS="${DD_UWSGI_EXTRA_ARGS} --max-fd ${DD_UWSGI_MAX_FD}"
 fi
 
 exec uwsgi \
@@ -42,5 +42,5 @@ exec uwsgi \
   --buffer-size="${DD_UWSGI_BUFFER_SIZE:-8192}" \
   --http 0.0.0.0:8081 --http-to "${DD_UWSGI_ENDPOINT}" \
   --logformat "${DD_UWSGI_LOGFORMAT:-$DD_UWSGI_LOGFORMAT_DEFAULT}" \
-  $EXTRA_ARGS
+  $DD_UWSGI_EXTRA_ARGS
   # HTTP endpoint is enabled for Kubernetes liveness checks. It should not be exposed as a service.

docs/content/en/open_source/upgrading/2.54.md

@@ -8,4 +8,4 @@ description: Dropped support for DD_PARSER_EXCLUDE
 To simplify the management of the DefectDojo application, parser exclusions are no longer controlled via the environment variable DD_PARSER_EXCLUDE or application settings. This variable is now unsupported.
 From now on, you should use the active flag in the Test_Type model to enable or disable parsers. Only parsers associated with active Test_Type entries will be available for use.
 
-There are other instructions for upgrading to 2.54.x. Check the Release Notes for the contents of the release.
+There are other instructions for upgrading to 2.54.x. Check the Release Notes for the contents of the release.

docs/content/en/share_your_findings/troubleshooting_jira.md

@@ -101,6 +101,34 @@ To correct this issue, you can add the 'Epic Name' field to your Project's issue
 
 ![image](images/epic_name_error.png)
 
+## Configuring JIRA Connection Retries and Timeouts
+
+DefectDojo's JIRA integration includes configurable retry and timeout settings to handle rate limiting and connection issues. These settings are important for maintaining system responsiveness, especially when using Celery workers.
+
+### Available Configuration Variables
+
+The following environment variables control JIRA connection behavior:
+
+- **`DD_JIRA_MAX_RETRIES`** (default: `3`): Maximum number of retry attempts for recoverable errors. The integration will automatically retry on HTTP 429 (Too Many Requests), HTTP 503 (Service Unavailable), and connection errors. See the [JIRA rate limiting documentation](https://developer.atlassian.com/cloud/jira/platform/rate-limiting/) for more information.
+
+- **`DD_JIRA_CONNECT_TIMEOUT`** (default: `10` seconds): Connection timeout for establishing a connection to the JIRA server.
+
+- **`DD_JIRA_READ_TIMEOUT`** (default: `30` seconds): Read timeout for waiting for a response from the JIRA server after the connection is established.
+
+**Note on Rate Limiting**: The jira library has a built-in maximum wait time of 60 seconds for rate limiting retries. If JIRA's `Retry-After` header indicates a wait time longer than 60 seconds, the request will fail and not be retried. This is a limitation of the jira library version currently in use.
+
+### Why Conservative Values Matter
+
+**Important**: It is recommended to use conservative (lower) values for these settings. Here's why:
+
+1. **Celery Task Blocking**: JIRA operations in DefectDojo run as asynchronous Celery tasks. When a task is waiting for a retry delay, it blocks that Celery worker from processing other tasks.
+
+2. **Worker Pool Exhaustion**: If multiple JIRA operations are retrying with long delays, you can quickly exhaust your Celery worker pool, causing other tasks (not just JIRA-related) to queue up and wait.
+
+3. **System Responsiveness**: Long retry delays can make the system appear unresponsive, especially during JIRA outages or rate limiting events.
+
+JIRA Rate limiting is new, so please let us know on Slack or GitHub what works best for you.
+
 ## Jira and DefectDojo are out of sync
 
 Sometimes Jira is down, or DefectDojo is down, or there was bug in a webhook. In this case, Jira can become out of sync with DefectDojo. If this is the case for lots of issues, manual reconciliation might not be feasible. For this scenario there is the management command 'jira_status_reconciliation'.

docs/package-lock.json

@@ -26,7 +26,7 @@
     "thulite": "2.6.3"
   },
   "devDependencies": {
-    "prettier": "3.7.2",
+    "prettier": "3.7.3",
     "vite": "7.2.4"
   },
   "engines": {

docs/package.json

@@ -4,6 +4,6 @@
 # Django starts so that shared_task will use this app.
 from .celery import app as celery_app  # noqa: F401
 
-__version__ = "2.53.0-dev"
+__version__ = "2.54.0-dev"
 __url__ = "https://github.com/DefectDojo/django-DefectDojo"
 __docs__ = "https://documentation.defectdojo.com"

dojo/__init__.py

@@ -208,7 +208,6 @@ def can_be_pushed_to_jira(obj, form=None):
             return False, f"Finding below the minimum JIRA severity threshold ({System_Settings.objects.get().jira_minimum_severity}).", "error_below_minimum_threshold"
     elif isinstance(obj, Finding_Group):
         finding_group_status = _safely_get_obj_status_for_jira(obj)
-        logger.error("Finding group status: %s", finding_group_status)
         if "Empty" in finding_group_status:
             return False, f"{to_str_typed(obj)} cannot be pushed to jira as it contains no findings above minimum treshold.", "error_empty"
 
@@ -433,14 +432,19 @@ def has_jira_configured(obj):
 
 
 def connect_to_jira(jira_server, jira_username, jira_password):
+    max_retries = getattr(settings, "JIRA_MAX_RETRIES", 3)
+    timeout = getattr(settings, "JIRA_TIMEOUT", (10, 30))
+
     return JIRA(
         server=jira_server,
         basic_auth=(jira_username, jira_password),
-        max_retries=0,
+        max_retries=max_retries,
+        timeout=timeout,
         options={
             "verify": settings.JIRA_SSL_VERIFY,
             "headers": settings.ADDITIONAL_HEADERS,
-        })
+        },
+    )
 
 
 def get_jira_connect_method():

dojo/jira_link/helper.py

@@ -250,6 +250,16 @@
     # When interacting with jira tickets that attached finding groups, we should no be opening any findings
     # on the DefectDojo side because jira has no way of knowing if a finding really should be reopened or not
     DD_JIRA_WEBHOOK_ALLOW_FINDING_GROUP_REOPEN=(bool, False),
+    # JIRA connection retry and timeout settings: https://developer.atlassian.com/cloud/jira/platform/rate-limiting/
+    # Maximum number of retry attempts for recoverable errors (429, 503, ConnectionError)
+    # See https://jira.readthedocs.io/ for more in the jira library used by DefectDojo
+    # Note: The jira library has a built-in maximum wait time of 60s for rate limiting retries.
+    # If JIRA's Retry-After header indicates a wait time longer than 60s, the request will fail and not be retried.
+    DD_JIRA_MAX_RETRIES=(int, 3),
+    # Connection timeout (seconds) for establishing a connection to the JIRA server
+    DD_JIRA_CONNECT_TIMEOUT=(int, 10),
+    # Read timeout (seconds) for waiting for a response from the JIRA server
+    DD_JIRA_READ_TIMEOUT=(int, 30),
     # You can set extra Jira issue types via a simple env var that supports a csv format, like "Work Item,Vulnerability"
     DD_JIRA_EXTRA_ISSUE_TYPES=(str, ""),
     # if you want to keep logging to the console but in json format, change this here to 'json_console'
@@ -1722,6 +1732,12 @@ def saml2_attrib_map_format(din):
 JIRA_SSL_VERIFY = env("DD_JIRA_SSL_VERIFY")
 JIRA_DESCRIPTION_MAX_LENGTH = env("DD_JIRA_DESCRIPTION_MAX_LENGTH")
 JIRA_WEBHOOK_ALLOW_FINDING_GROUP_REOPEN = env("DD_JIRA_WEBHOOK_ALLOW_FINDING_GROUP_REOPEN")
+# JIRA connection retry and timeout settings
+JIRA_MAX_RETRIES = env("DD_JIRA_MAX_RETRIES")
+JIRA_CONNECT_TIMEOUT = env("DD_JIRA_CONNECT_TIMEOUT")
+JIRA_READ_TIMEOUT = env("DD_JIRA_READ_TIMEOUT")
+# Combine timeouts into a tuple for the JIRA library: (connect_timeout, read_timeout)
+JIRA_TIMEOUT = (JIRA_CONNECT_TIMEOUT, JIRA_READ_TIMEOUT)
 
 # ------------------------------------------------------------------------------
 # LOGGING