jaeles-project · Dec 25, 2021 · Jul 27, 2022 · Jul 29, 2022 · Mar 31, 2023 · Jul 1, 2023
Showing with 451 additions and 317 deletions.

+1 −3 .gitignore

+6 −7 Dockerfile

+34 −0 Makefile

+14 −13 README.md

+79 −30 core/parser.go

+89 −25 go.mod

+226 −238 go.sum

+1 −0 libs/http.go

+1 −1 libs/version.go
diff --git a/.gitignore b/.gitignore
@@ -2,11 +2,9 @@
 .goreleaser.yml
 .idea
 .vscode
-Makefile
 dist
 out
 passive-*
 old-out
 http-out
-test-scripts
-Makefile
+test-scripts
diff --git a/Dockerfile b/Dockerfile
@@ -1,9 +1,8 @@
-FROM golang:1.14-buster as builder
-RUN GO111MODULE=on GOOS=linux go get -ldflags "-linkmode external -extldflags -static" github.com/jaeles-project/jaeles
-
-FROM alpine:latest
-WORKDIR /
-COPY --from=builder /go/bin/jaeles /bin/jaeles
+FROM golang:1.20-buster as builder
+RUN go install github.com/jaeles-project/jaeles@latest
+RUN apt update -qq \
+    && apt install -y chromium && apt clean
+WORKDIR /root/
 EXPOSE 5000
 RUN jaeles config init -y
-ENTRYPOINT ["/bin/jaeles"]
+ENTRYPOINT ["/go/bin/jaeles"]
diff --git a/Makefile b/Makefile
@@ -0,0 +1,34 @@
+TARGET   ?= jaeles
+GO       ?= go
+GOFLAGS  ?= 
+VERSION  := $(shell cat libs/version.go | grep 'VERSION =' | cut -d '"' -f 2 | sed 's/ /\-/g')
+
+build:
+	go install
+	go build -ldflags="-s -w" -tags netgo -trimpath -buildmode=pie -o dist/$(TARGET)
+
+release:
+	go install
+	@echo "==> Clean up old builds"
+	rm -rf ./dist/*
+	@echo "==> building binaries for for mac intel"
+	GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w" -tags netgo -trimpath -buildmode=pie -o dist/$(TARGET)
+	zip -9 -j dist/$(TARGET)-macos-amd64.zip dist/$(TARGET) && rm -rf ./dist/$(TARGET)
+	@echo "==> building binaries for for mac M1 chip"
+	CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -ldflags="-s -w" -tags netgo -trimpath -buildmode=pie -o dist/$(TARGET)
+	zip -9 -j dist/$(TARGET)-macos-arm64.zip dist/$(TARGET)&& rm -rf ./dist/$(TARGET)
+	@echo "==> building binaries for linux intel build on mac"
+	GOOS=linux GOARCH=amd64 CC="/usr/local/bin/x86_64-linux-musl-gcc" CGO_ENABLED=1 go build -ldflags="-s -w" -tags netgo -trimpath -buildmode=pie -o dist/$(TARGET)
+	zip -9 -j dist/$(TARGET)-linux.zip dist/$(TARGET)&& rm -rf ./dist/$(TARGET)
+	mv dist/$(TARGET)-macos-amd64.zip dist/$(TARGET)-$(VERSION)-macos-amd64.zip
+	mv dist/$(TARGET)-macos-arm64.zip dist/$(TARGET)-$(VERSION)-macos-arm64.zip
+	mv dist/$(TARGET)-linux.zip dist/$(TARGET)-$(VERSION)-linux.zip
+run:
+	$(GO) $(GOFLAGS) run *.go
+
+fmt:
+	$(GO) $(GOFLAGS) fmt ./...; \
+	echo "Done."
+
+test:
+	$(GO) $(GOFLAGS) test ./... -v%
diff --git a/README.md b/README.md
@@ -1,9 +1,9 @@
 <p align="center">
   <img alt="Jaeles" src="https://github.com/jaeles-project/jaeles-plugins/blob/master/assets/jaeles.png?raw=true" height="140" />
   <p align="center">
+    <a href="https://github.com/jaeles-project/jaeles/releases"><img alt="Release" src="https://img.shields.io/github/v/release/jaeles-project/jaeles.svg"></a>
+    <a href="https://jaeles-project.github.io/"><img alt="Documentation" src="https://img.shields.io/badge/Documentation-📖-red.svg?style=flat-square"></a>
     <a href=""><img alt="Software License" src="https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square"></a>
-    <a href="https://github.com/jaeles-project/jaeles"><img alt="Release" src="https://img.shields.io/github/v/release/jaeles-project/jaeles.svg"></a>
-    <a href="https://inventory.rawsec.ml/tools.html#Jaeles"><img src="https://inventory.rawsec.ml/img/badges/Rawsec-inventoried-FF5050_flat.svg" alt="Rawsec&#39;s CyberSecurity Inventory"></a>
   </p>
 </p>
 
@@ -12,23 +12,15 @@ Scanner.
 
 ![Architecture](https://github.com/jaeles-project/jaeles-plugins/blob/master/imgs/jaeles-architecture.png?raw=true)
 
-## Painless integrate Jaeles into your recon workflow?
-
-<p align="center">
-  <img alt="huntersuite" src="https://user-images.githubusercontent.com/23289085/101143253-35ea6b80-3649-11eb-9130-d1fc306c9a76.png" height="200" />
-  <p align="center">
-Enjoying this tool? Support it's development and take your game to the next level by using <a href="https://huntersuite.io">HunterSuite.io</a>
-  </p>
-</p>
 
 ## Installation
 
 Download [precompiled version here](https://github.com/jaeles-project/jaeles/releases).
 
-If you have a Go environment, make sure you have **Go >= 1.13** with Go Modules enable and run the following command.
+If you have a Go environment, make sure you have **Go >= 1.17** with Go Modules enable and run the following command.
 
 ```shell
-GO111MODULE=on go get github.com/jaeles-project/jaeles
+go install github.com/jaeles-project/jaeles@latest
 ```
 
 Please visit the [Official Documention](https://jaeles-project.github.io/) for more details.
@@ -101,10 +93,19 @@ Guide [here](https://youtu.be/1lxsYhfTq3M)
 * Adding more input sources.
 * Adding more APIs to get access to more properties of the request.
 * Adding proxy plugins to directly receive input from browser of http client.
-* ~~Adding passive signature for passive checking each request.~~
 * Adding more action on Web UI.
 * Integrate with many other tools.
 
+## Painless integrate Jaeles into your recon workflow?
+
+<p align="center">
+  <img alt="OsmedeusEngine" src="https://raw.githubusercontent.com/osmedeus/assets/main/part-of-osmedeus-banner.png" />
+  <p align="center">
+    This project was part of Osmedeus Engine. Check out how it was integrated at <a href="https://twitter.com/OsmedeusEngine">@OsmedeusEngine</a>
+  </p>
+</p>
+
+
 ## Contribute
 
 If you have some new idea about this project, issue, feedback or found some valuable tool feel free to open an issue for

diff --git a/core/parser.go b/core/parser.go
@@ -11,6 +11,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"bytes"
 
 	"github.com/jaeles-project/jaeles/libs"
 	"github.com/thoas/go-funk"
@@ -451,45 +452,93 @@ func ParsePayloads(sign libs.Signature) []string {
 func ParseBurpRequest(raw string) (req libs.Request) {
 	var realReq libs.Request
 	realReq.Raw = raw
-	reader := bufio.NewReader(strings.NewReader(raw))
-	parsedReq, err := http.ReadRequest(reader)
-	if err != nil {
-		return realReq
-	}
-	realReq.Method = parsedReq.Method
-	// URL part
-	if parsedReq.URL.Host == "" {
-		realReq.Host = parsedReq.Host
-		parsedReq.URL.Host = parsedReq.Host
-	}
-	if parsedReq.URL.Scheme == "" {
-		if parsedReq.Referer() == "" {
-			realReq.Scheme = "https"
-			parsedReq.URL.Scheme = "https"
-		} else {
-			u, err := url.Parse(parsedReq.Referer())
-			if err == nil {
-				realReq.Scheme = u.Scheme
-				parsedReq.URL.Scheme = u.Scheme
-			}
+
+	// create a scanner to read the request
+	scanner := bufio.NewScanner(strings.NewReader(raw))
+
+	// read the first request line that contains HTTP version, method and path
+	scanner.Scan()
+	requestLine := scanner.Text()
+	parts := strings.Split(requestLine, " ")
+	if len(parts) != 3 {
+		return
+	}
+	httpMethod := parts[0]
+	httpVersion := parts[2]
+	path := parts[1]
+
+	// create a new buffer for the request body
+	bodyBuffer := bytes.NewBuffer([]byte{})
+
+	// create a new map to hold the headers
+	headers := make([]map[string]string, 0)
+
+	// read the headers
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line == "" {
+			break
 		}
+
+		parts := strings.Split(line, ": ")
+		if len(parts) != 2 {
+			return
+		}
+		headerName := parts[0]
+		headerValue := parts[1]
+
+		header := map[string]string{
+			headerName: headerValue,
+		}
+		headers = append(headers, header)
+	}
+
+	// read the request body
+	for scanner.Scan() {
+		bodyBuffer.Write(scanner.Bytes())
+		bodyBuffer.Write([]byte("\r\n"))
 	}
-	realReq.URL = parsedReq.URL.String()
-	realReq.Path = parsedReq.RequestURI
-	realReq.Headers = ParseHeaders(parsedReq.Header)
 
-	body, _ := ioutil.ReadAll(parsedReq.Body)
-	realReq.Body = string(body)
-	// net/http parse something weird here
-	if !strings.HasSuffix(raw, realReq.Body) {
-		if strings.Contains(raw, "\n\n") {
-			realReq.Body = strings.Split(raw, "\n\n")[1]
+	body := bodyBuffer.String()
+
+	var urlScheme string
+	referer := headersGet(headers, "Referer")
+	if referer != "" {
+		refURL, err := url.Parse(referer)
+		if err == nil {
+			urlScheme = refURL.Scheme
 		}
 	}
 
+	if urlScheme == "" {
+		urlScheme = "https"
+	}
+
+	url := urlScheme + "://" + headers[0]["Host"] + path
+
+	realReq.Body = body
+	realReq.Headers = headers
+	realReq.Path = path
+	realReq.URL = url
+	realReq.Method = httpMethod
+	realReq.Scheme = urlScheme
+	realReq.Proto = httpVersion
+
 	return realReq
 }
 
+// helper function to get header value
+func headersGet(headers []map[string]string, headerName string) string {
+	for _, header := range headers {
+		for k, v := range header {
+			if strings.ToLower(k) == strings.ToLower(headerName) {
+				return v
+			}
+		}
+	}
+	return ""
+}
+
 // ParseHeaders parse header for sending method
 func ParseHeaders(rawHeaders map[string][]string) []map[string]string {
 	var headers []map[string]string

diff --git a/go.mod b/go.mod
@@ -1,40 +1,104 @@
 module github.com/jaeles-project/jaeles
 
-go 1.16
+go 1.20
 
 require (
-	github.com/Jeffail/gabs/v2 v2.6.1
-	github.com/Masterminds/sprig/v3 v3.2.2
-	github.com/PuerkitoBio/goquery v1.7.0
-	github.com/appleboy/gin-jwt/v2 v2.6.4
-	github.com/chromedp/cdproto v0.0.0-20210625233425-810000e4a4fc
-	github.com/chromedp/chromedp v0.7.3
+	github.com/Jeffail/gabs/v2 v2.7.0
+	github.com/Masterminds/sprig/v3 v3.2.3
+	github.com/PuerkitoBio/goquery v1.8.1
+	github.com/appleboy/gin-jwt/v2 v2.9.1
+	github.com/chromedp/cdproto v0.0.0-20230816033919-17ee49f3eb4f
+	github.com/chromedp/chromedp v0.9.2
 	github.com/davecgh/go-spew v1.1.1
-	github.com/fatih/color v1.12.0
-	github.com/gin-contrib/cors v1.3.1
+	github.com/fatih/color v1.15.0
+	github.com/gin-contrib/cors v1.4.0
 	github.com/gin-contrib/static v0.0.1
-	github.com/gin-gonic/gin v1.7.2
-	github.com/go-resty/resty/v2 v2.6.0
-	github.com/google/uuid v1.2.0
-	github.com/gorilla/websocket v1.4.2
-	github.com/jinzhu/copier v0.3.2
+	github.com/gin-gonic/gin v1.9.1
+	github.com/go-resty/resty/v2 v2.7.0
+	github.com/google/uuid v1.3.0
+	github.com/gorilla/websocket v1.5.0
+	github.com/jinzhu/copier v0.4.0
 	github.com/jinzhu/gorm v1.9.16
-	github.com/json-iterator/go v1.1.11
+	github.com/json-iterator/go v1.1.12
 	github.com/lixiangzhong/dnsutil v1.4.0
 	github.com/logrusorgru/aurora/v3 v3.0.0
-	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/onsi/ginkgo v1.16.4 // indirect
-	github.com/onsi/gomega v1.13.0 // indirect
 	github.com/panjf2000/ants v1.3.0
-	github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452
-	github.com/sirupsen/logrus v1.8.1
-	github.com/spf13/cast v1.3.1
-	github.com/spf13/cobra v1.2.1
-	github.com/spf13/viper v1.8.1
-	github.com/thoas/go-funk v0.8.0
+	github.com/robertkrimen/otto v0.2.1
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cast v1.5.1
+	github.com/spf13/cobra v1.7.0
+	github.com/spf13/viper v1.16.0
+	github.com/thoas/go-funk v0.9.3
 	github.com/x-cray/logrus-prefixed-formatter v0.5.2
-	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 	gopkg.in/src-d/go-git.v4 v4.13.1
 	gopkg.in/yaml.v2 v2.4.0
 )
+
+require (
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.2.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/bytedance/sonic v1.9.1 // indirect
+	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
+	github.com/chromedp/sysutil v1.0.0 // indirect
+	github.com/emirpasic/gods v1.12.0 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.14.0 // indirect
+	github.com/gobwas/httphead v0.1.0 // indirect
+	github.com/gobwas/pool v0.2.1 // indirect
+	github.com/gobwas/ws v1.2.1 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/huandu/xstrings v1.3.3 // indirect
+	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect
+	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-sqlite3 v1.14.0 // indirect
+	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
+	github.com/miekg/dns v1.1.40 // indirect
+	github.com/mitchellh/copystructure v1.0.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/onsi/ginkgo v1.16.5 // indirect
+	github.com/onsi/gomega v1.27.10 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/sergi/go-diff v1.0.0 // indirect
+	github.com/shopspring/decimal v1.2.0 // indirect
+	github.com/spf13/afero v1.9.5 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/src-d/gcfg v1.4.0 // indirect
+	github.com/subosito/gotenv v1.4.2 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.11 // indirect
+	github.com/xanzy/ssh-agent v0.2.1 // indirect
+	golang.org/x/arch v0.3.0 // indirect
+	golang.org/x/crypto v0.11.0 // indirect
+	golang.org/x/net v0.12.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/term v0.10.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/sourcemap.v1 v1.0.5 // indirect
+	gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)