Skip to content

Commit 5f730c4

Browse files
committed
adjust code and move script to separate file
1 parent 341746d commit 5f730c4

File tree

2 files changed

+94
-36
lines changed

2 files changed

+94
-36
lines changed

.github/workflows/reusable-build-test-release.yml

Lines changed: 26 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -406,13 +406,7 @@ jobs:
406406
- build
407407
- setup-workflow
408408
- setup
409-
410-
runs-on: ubuntu-latest
411-
strategy:
412-
fail-fast: false
413-
matrix:
414-
python-version:
415-
- "3.11"
409+
runs-on: large-ubuntu-22.04-32core
416410
permissions:
417411
actions: read
418412
deployments: read
@@ -421,16 +415,24 @@ jobs:
421415
statuses: read
422416
checks: write
423417
steps:
424-
- uses: actions/checkout@v4
425-
- uses: actions/setup-python@v5
418+
- name: Checkout TA
419+
uses: actions/checkout@v4
420+
421+
- name: Checkout Security Content
422+
uses: actions/checkout@v4
426423
with:
427-
python-version: ${{ matrix.python-version }}
424+
repository: splunk/security_content
425+
path: security_content
426+
ref: refs/heads/develop
428427

429-
- name: Install Python Dependencies and ContentCTL
428+
- uses: actions/setup-python@v5
429+
with:
430+
python-version: "3.11"
431+
432+
- name: Install dependencies
430433
run: |
431-
pip install contentctl
432-
git clone https://github.com/splunk/security_content.git
433-
434+
python -m pip install --upgrade pip
435+
pip install contentctl pyyaml
434436
435437
- name: Download TA Build Artifact
436438
uses: actions/download-artifact@v4
@@ -444,7 +446,7 @@ jobs:
444446
TA_BUILD_PATH="${{ github.workspace }}/ta_build/$TA_BUILD"
445447
echo "TA_BUILD_PATH=$TA_BUILD_PATH" >> $GITHUB_ENV
446448
447-
- name: Run Python Script
449+
- name: Filter ESCU Detections
448450
id: filter-detection-files
449451
shell: python
450452
run: |
@@ -460,6 +462,8 @@ jobs:
460462
config.read("package/default/app.conf")
461463
APP_ID = config.get("id", "name")
462464
APP_LABEL = config.get("ui", "label")
465+
466+
print(f"APP_ID = {APP_ID}, APP_LABEL = {APP_LABEL}")
463467
464468
# Read the file and remove trailing backslashes
465469
with open("package/default/props.conf", "r") as f:
@@ -480,19 +484,11 @@ jobs:
480484
# Load the YAML content
481485
with open("security_content/contentctl.yml", "r") as file:
482486
data = yaml.safe_load(file)
483-
484-
found = False
485-
486487
for app in data["apps"]:
487-
if app['appid'] == APP_ID or GITHUB_REPOSITORY in app['hardcoded_path'] or app["title"] == APP_LABEL:
488-
app['hardcoded_path'] = "${{ env.TA_BUILD_PATH }}"
489-
found = True
490-
elif app['appid'] == "PALO_ALTO_NETWORKS_ADD_ON_FOR_SPLUNK" and APP_ID == "Splunk_TA_paloalto_networks":
488+
if app['appid'] == APP_ID or APP_ID in app['hardcoded_path'] or GITHUB_REPOSITORY in app['hardcoded_path'] or app["title"] == APP_LABEL or (app['appid'] == "PALO_ALTO_NETWORKS_ADD_ON_FOR_SPLUNK" and APP_ID == "Splunk_TA_paloalto_networks"):
491489
app['hardcoded_path'] = "${{ env.TA_BUILD_PATH }}"
492-
found = True
493-
494-
if not found:
495-
exit(127)
490+
else:
491+
exit(127)
496492
497493
498494
# Write the modified data to the contentctl.yml file
@@ -506,14 +502,11 @@ jobs:
506502
for root, dirs, files in os.walk(base_dir):
507503
for file in files:
508504
file_path = os.path.join(root, file)
509-
510505
try:
511-
with open(file_path, "r") as file:
512-
file_content = yaml.safe_load(file)
506+
with open(file_path, "r") as yaml_file:
507+
file_content = yaml.safe_load(yaml_file)
513508
if "deprecated" not in file_path and (file_content["tests"][0]["attack_data"][0]["sourcetype"] in sourcetypes or file_content["tests"][0]["attack_data"][0]["source"] in sourcetypes):
514509
detection_files += file_path.replace("security_content/", "") + " "
515-
516-
517510
except Exception as e:
518511
continue
519512
@@ -525,19 +518,16 @@ jobs:
525518
526519
- name: Run ESCU Tests
527520
run: |
528-
529521
cd security_content
530-
echo "Content of contentctl.yml file"
522+
echo "Content of contentctl.yml file: "
531523
cat contentctl.yml
532-
533524
contentctl test --container-settings.num-containers 8 --post-test-behavior never_pause --disable-tqdm mode:selected --mode.files ${{ steps.filter-detection-files.outputs.DETECTION_FILES }}
534525
535526
- uses: actions/upload-artifact@v4
536527
if: always()
537528
with:
538529
name: escu_test_summary_results
539-
path: |
540-
security_content/test_results/summary.yml
530+
path: security_content/test_results/summary.yml
541531

542532
run-unit-tests:
543533
name: test-unit-python3-${{ matrix.python-version }}

scripts/filter_escu_detections.py

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import yaml
2+
import os
3+
import configparser
4+
import re
5+
6+
GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY", "")
7+
8+
# Parse app.conf get the appid of the TA.
9+
config = configparser.ConfigParser(strict=False)
10+
config.read("package/default/app.conf")
11+
APP_ID = config.get("id", "name")
12+
APP_LABEL = config.get("ui", "label")
13+
14+
# Read the file and remove trailing backslashes
15+
with open("package/default/props.conf", "r") as f:
16+
content = f.read()
17+
18+
# Remove trailing backslashes followed by a newline
19+
updated_content = re.sub(r"\\\n", "", content)
20+
21+
# Write the cleaned content to a new file
22+
with open("package/default/props.conf", "w") as f:
23+
f.write(updated_content)
24+
25+
# Parse props.conf and collect all the sourcetypes in a list.
26+
config = configparser.ConfigParser(strict=False)
27+
config.read("package/default/props.conf")
28+
sourcetypes = config.sections()
29+
30+
# Load the YAML content
31+
with open("security_content/contentctl.yml", "r") as file:
32+
data = yaml.safe_load(file)
33+
34+
found = False
35+
36+
for app in data["apps"]:
37+
if app['appid'] == APP_ID or GITHUB_REPOSITORY in app['hardcoded_path'] or app["title"] == APP_LABEL or (app['appid'] == "PALO_ALTO_NETWORKS_ADD_ON_FOR_SPLUNK" and APP_ID == "Splunk_TA_paloalto_networks"):
38+
app['hardcoded_path'] = "${{ env.TA_BUILD_PATH }}"
39+
else:
40+
exit(127)
41+
42+
# Write the modified data to the contentctl.yml file
43+
with open("security_content/contentctl.yml", "w") as file:
44+
yaml.dump(data, file, sort_keys=False)
45+
46+
# Filter out the detections based on the collected sourcetypes
47+
base_dir = "security_content/detections"
48+
detection_files = ""
49+
50+
for root, dirs, files in os.walk(base_dir):
51+
for file in files:
52+
file_path = os.path.join(root, file)
53+
54+
try:
55+
with open(file_path, "r") as yaml_file:
56+
file_content = yaml.safe_load(yaml_file)
57+
if "deprecated" not in file_path and (
58+
file_content["tests"][0]["attack_data"][0]["sourcetype"] in sourcetypes or file_content["tests"][0]["attack_data"][0]["source"] in sourcetypes):
59+
detection_files += file_path.replace("security_content/", "") + " "
60+
61+
except Exception as e:
62+
continue
63+
64+
# Save detection_files as an output variable
65+
with open(os.getenv('GITHUB_OUTPUT'), 'w') as output_file:
66+
output_file.write(f"DETECTION_FILES={detection_files}")
67+
68+
print(f"Filtered Detection files = {detection_files}")

0 commit comments

Comments
 (0)