Merge remote-tracking branch 'origin'

Author: ehsandeep

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -36,6 +36,8 @@ body:
       description: |
         Steps to reproduce the behavior, for example, commands to run Nuclei.
 
+        📝 For a more detailed output that could help in troubleshooting, you may want to run Nuclei with the **`-verbose`** or **`-debug`** flags. This will provide additional insights into what's happening under the hood.
+
         :warning: **Please redact any literal target hosts/URLs or other sensitive information.**
       placeholder: |
         1. Run `nuclei -t ...`
@@ -44,7 +46,12 @@ body:
   - type: textarea
     attributes:
       label: Relevant log output
-      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+      description: |
+        Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+
+        📝 For a more detailed output that could help in troubleshooting, you may want to run Nuclei with the **`-verbose`** or **`-debug`** flags. This will provide additional insights into what's happening under the hood.
+
+        :warning: **Please redact any literal target hosts/URLs or other sensitive information.**
       render: shell
   - type: textarea
     attributes:

.github/workflows/generate-pgo.yaml

@@ -0,0 +1,55 @@
+name: 👤 Generate PGO
+
+on:
+  push:
+    branches: ["dev"]
+    paths:
+      - '**.go'
+      - '**.mod'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+# TODO(dwisiswant0): https://go.dev/doc/pgo#merging-profiles
+
+jobs:
+  pgo:
+    strategy:
+      matrix:
+        targets: [150]
+    runs-on: ubuntu-latest-16-cores
+    if: github.repository == 'projectdiscovery/nuclei'
+    permissions:
+      contents: write
+    env:
+      PGO_FILE: "cmd/nuclei/default.pgo"
+      LIST_FILE: "/tmp/targets-${{ matrix.targets }}.txt"
+      PROFILE_MEM: "/tmp/nuclei-profile-${{ matrix.targets }}-targets"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: projectdiscovery/actions/setup/git@v1
+      - uses: projectdiscovery/actions/setup/go@v1
+      - name: Generate list
+        run: for i in {1..${{ matrix.targets }}}; do echo "https://honey.scanme.sh/?_=${i}" >> "${LIST_FILE}"; done
+      # NOTE(dwisiswant0): use `-no-mhe` flag to get better samples.
+      - run: go run . -l "${LIST_FILE}" -profile-mem="${PROFILE_MEM}" -no-mhe
+        working-directory: cmd/nuclei/
+      - run: mv "${PROFILE_MEM}.cpu" ${PGO_FILE}
+      # NOTE(dwisiswant0): shall we prune $PGO_FILE git history?
+      # if we prune it, this won't be linear since it requires a force-push.
+      # if we don't, the git objects will just keep growing bigger.
+      # 
+      # Ref:
+      # - https://go.dev/blog/pgo#:~:text=We%20recommend%20committing%20default.pgo%20files%20to%20your%20repository
+      # - https://gist.github.com/nottrobin/5758221
+      - uses: projectdiscovery/actions/commit@v1
+        with:
+          files: "${PGO_FILE}"
+          message: "build: update PGO profile :robot:"
+      - run: git push origin $GITHUB_REF
+      - uses: actions/upload-artifact@v4
+        with:
+          name: "pgo"
+          path: "${{ env.PGO_FILE }}"

.github/workflows/perf-regression.yaml

@@ -0,0 +1,38 @@
+name: 🔨 Performance Regression
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  perf-regression:
+    runs-on: ubuntu-latest-16-cores
+    if: github.repository == 'projectdiscovery/nuclei'
+    env:
+      BENCH_OUT: "/tmp/bench.out"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: projectdiscovery/actions/setup/go@v1
+      - run: make build-test
+      - run: ./bin/nuclei.test -test.run - -test.bench=. -test.benchmem ./cmd/nuclei/ | tee $BENCH_OUT
+        env:
+          DISABLE_STDOUT: "1"
+      - uses: actions/cache/restore@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      - uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: 'RunEnumeration Benchmark'
+          tool: 'go'
+          output-file-path: ${{ env.BENCH_OUT }}
+          external-data-json-path: ./cache/benchmark-data.json
+          fail-on-alert: false
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-on-alert: true
+          summary-always: true
+      - uses: actions/cache/save@v4
+        if: github.event_name == 'push'
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark

.github/workflows/perf-test.yaml

@@ -20,16 +20,24 @@ jobs:
       - uses: projectdiscovery/actions/setup/go@v1
       - run: make verify
       - name: Generate list
-        run: for i in {1..${{ matrix.count }}}; do echo "https://scanme.sh/?_=${i}" >> "${LIST_FILE}"; done
-      - run: NUCLEI_ARGS=host-error-stats go run . -l "${LIST_FILE}" -profile-mem="${PROFILE_MEM}"
+        run: for i in {1..${{ matrix.count }}}; do echo "https://honey.scanme.sh/?_=${i}" >> "${LIST_FILE}"; done
+      - run: go run . -l "${LIST_FILE}" -profile-mem="${PROFILE_MEM}"
+        env:
+          NUCLEI_ARGS: host-error-stats
         working-directory: cmd/nuclei/
       - uses: projectdiscovery/actions/flamegraph@v1
-        id: flamegraph
+        id: flamegraph-cpu
         with:
-          profile: "${{ env.PROFILE_MEM }}.prof"
-          name: "nuclei-perf-test-${{ matrix.count }}"
+          profile: "${{ env.PROFILE_MEM }}.cpu"
+          name: "${{ env.FLAMEGRAPH_NAME }} CPU profiles"
         continue-on-error: true
-      - if: ${{ steps.flamegraph.outputs.message == '' }}
-        run: echo "::notice::${FLAMEGRAPH_URL}"
-        env:
-          FLAMEGRAPH_URL: ${{ steps.flamegraph.outputs.url }}
+      - uses: projectdiscovery/actions/flamegraph@v1
+        id: flamegraph-mem
+        with:
+          profile: "${{ env.PROFILE_MEM }}.mem"
+          name: "${{ env.FLAMEGRAPH_NAME }} memory profiles"
+        continue-on-error: true
+      - if: ${{ steps.flamegraph-mem.outputs.message == '' }}
+        run: |
+          echo "::notice::CPU flamegraph: ${{ steps.flamegraph-cpu.outputs.url }}"
+          echo "::notice::Memory (heap) flamegraph: ${{ steps.flamegraph-mem.outputs.url }}"

.github/workflows/tests.yaml

@@ -156,13 +156,23 @@ jobs:
           echo "FLAMEGRAPH_NAME=nuclei (PR #${{ github.event.number }})" >> $GITHUB_ENV
       - run: ./bin/nuclei -silent -update-templates
       - run: ./bin/nuclei -silent -u "${TARGET_URL}" -profile-mem="${PROFILE_MEM}"
-      - uses: projectdiscovery/actions/flamegraph@master
-        id: flamegraph
+      - uses: projectdiscovery/actions/flamegraph@v1
+        id: flamegraph-cpu
         with:
-          profile: "${{ env.PROFILE_MEM }}.prof"
-          name: "${{ env.FLAMEGRAPH_NAME }}"
+          profile: "${{ env.PROFILE_MEM }}.cpu"
+          name: "${{ env.FLAMEGRAPH_NAME }} CPU profiles"
         continue-on-error: true
-      - if: ${{ steps.flamegraph.outputs.message == '' }}
-        run: echo "::notice::${FLAMEGRAPH_URL}"
-        env:
-          FLAMEGRAPH_URL: ${{ steps.flamegraph.outputs.url }}
+      - uses: projectdiscovery/actions/flamegraph@v1
+        id: flamegraph-mem
+        with:
+          profile: "${{ env.PROFILE_MEM }}.mem"
+          name: "${{ env.FLAMEGRAPH_NAME }} memory profiles"
+        continue-on-error: true
+      - if: ${{ steps.flamegraph-mem.outputs.message == '' }}
+        run: |
+          echo "::notice::CPU flamegraph: ${{ steps.flamegraph-cpu.outputs.url }}"
+          echo "::notice::Memory (heap) flamegraph: ${{ steps.flamegraph-mem.outputs.url }}"
+
+  perf-regression:
+    needs: ["tests"]
+    uses: ./.github/workflows/perf-regression.yaml

.gitignore

@@ -48,4 +48,6 @@ vendor
 # Profiling & tracing
 *.prof
 *.pprof
-*.trace
+*.trace
+*.mem
+*.cpu

.goreleaser.yml

@@ -1,27 +1,29 @@
 before:
   hooks:
-    - go mod tidy
+    - go mod download
+    - go mod verify
 
 builds:
-- main: cmd/nuclei/main.go
-  binary: nuclei
-  id: nuclei-cli
-
-  env:
-  - CGO_ENABLED=0
-
-  goos: [windows,linux,darwin]
-  goarch: [amd64,386,arm,arm64]
-  ignore:
-    - goos: darwin
-      goarch: 386
-    - goos: windows
-      goarch: arm
-    - goos: windows
-      goarch: arm64
-
-  flags:
-    - -trimpath
+  - main: cmd/nuclei/main.go
+    binary: nuclei
+    id: nuclei-cli
+    env:
+      - CGO_ENABLED=0
+    goos: [windows,linux,darwin]
+    goarch: [amd64,'386',arm,arm64]
+    ignore:
+      - goos: darwin
+        goarch: '386'
+      - goos: windows
+        goarch: arm
+      - goos: windows
+        goarch: arm64
+    flags:
+      - -trimpath
+      - -pgo=auto
+    ldflags:
+      - -s
+      - -w
 
 #- main: cmd/tmc/main.go
 #  binary: tmc
@@ -34,10 +36,10 @@ builds:
 #  goarch: [amd64]
 
 archives:
-- format: zip
-  id: nuclei
-  builds: [nuclei-cli]
-  name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}'
+  - format: zip
+    id: nuclei
+    builds: [nuclei-cli]
+    name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}'
 
 checksum:
   algorithm: sha256

DESIGN.md

@@ -459,35 +459,42 @@ That's it, you've added a new protocol to Nuclei. The next good step would be to
 
 ## Profiling and Tracing
 
-To analyze Nuclei's performance and resource usage, you can generate memory profiles and trace files using the `-profile-mem` flag:
+To analyze Nuclei's performance and resource usage, you can generate CPU & memory profiles and trace files using the `-profile-mem` flag:
 
 ```bash
 nuclei -t nuclei-templates/ -u https://example.com -profile-mem=nuclei-$(git describe --tags)
 ```
 
-This command creates two files:
+This command creates three files:
 
-* `nuclei.prof`: Memory (heap) profile
+* `nuclei.cpu`: CPU profile
+* `nuclei.mem`: Memory (heap) profile
 * `nuclei.trace`: Execution trace
 
-### Analyzing the Memory Profile
+### Analyzing the CPU/Memory Profiles
 
-1. View the profile in the terminal:
+* View the profile in the terminal:
 
 ```bash
-go tool pprof nuclei.prof
+go tool pprof nuclei.{cpu,mem}
 ```
 
-2. Display top memory consumers:
+* Display overall CPU time for processing $$N$$ targets:
+
+```
+go tool pprof -top nuclei.cpu | grep "Total samples"
+```
+
+* Display top memory consumers:
 
 ```bash
-go tool pprof -top nuclei.prof | grep "$(go list -m)" | head -10
+go tool pprof -top nuclei.mem | grep "$(go list -m)" | head -10
 ```
 
-3. Visualize the profile in a web browser:
+* Visualize the profile in a web browser:
 
 ```bash
-go tool pprof -http=:$(shuf -i 1000-99999 -n 1) nuclei.prof
+go tool pprof -http=:$(shuf -i 1000-99999 -n 1) nuclei.{cpu,mem}
 ```
 
 ### Analyzing the Trace File

Makefile

@@ -11,7 +11,7 @@ GOFLAGS := -v
 LDFLAGS := -s -w
 
 ifneq ($(shell go env GOOS),darwin)
-	LDFLAGS = -extldflags "-static"
+	LDFLAGS += -extldflags "-static"
 endif
 
 .PHONY: all build build-stats clean devtools-all devtools-bindgen devtools-scrapefuncs
@@ -26,13 +26,22 @@ clean:
 
 go-build: clean
 go-build:
-	$(GOBUILD) $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \
+	CGO_ENABLED=0 $(GOBUILD) -trimpath $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \
 		 -o '${GOBUILD_OUTPUT}' $(GOBUILD_PACKAGES)
 
+build: GOFLAGS = -v -pgo=auto
 build: GOBUILD_OUTPUT = ./bin/nuclei
 build: GOBUILD_PACKAGES = cmd/nuclei/main.go
 build: go-build
 
+build-test: GOFLAGS = -v -pgo=auto
+build-test: GOBUILD_OUTPUT = ./bin/nuclei.test
+build-test: GOBUILD_PACKAGES = ./cmd/nuclei/
+build-test: clean
+build-test:
+	CGO_ENABLED=0 $(GOCMD) test -c -trimpath $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \
+		 -o '${GOBUILD_OUTPUT}' ${GOBUILD_PACKAGES}
+
 build-stats: GOBUILD_OUTPUT = ./bin/nuclei-stats
 build-stats: GOBUILD_PACKAGES = cmd/nuclei/main.go
 build-stats: GOBUILD_ADDITIONAL_ARGS = -tags=stats