Skip to content

Commit 12b93fd

Browse files
wrouesnelwrouesnel
committed
Add streaming command support.
Add options - `stream-stdout-in-response` - `stream-stdout-in-response-on-error` - `stream-command-kill-grace-period-seconds` to allow defining webhooks which dynamically stream large content back to the requestor. This allows the creation of download endpoints from scripts, i.e. running a `git archive` command or a database dump from a docker container, without needing to buffer up the original.
1 parent 10396a5 commit 12b93fd

File tree

8 files changed

+564
-114
lines changed

8 files changed

+564
-114
lines changed

‎docs/Hook-Definition.md

+3
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ Hooks are defined as JSON objects. Please note that in order to be considered va
1010
* `response-headers` - specifies the list of headers in format `{"name": "X-Example-Header", "value": "it works"}` that will be returned in HTTP response for the hook
1111
* `include-command-output-in-response` - boolean whether webhook should wait for the command to finish and return the raw output as a response to the hook initiator. If the command fails to execute or encounters any errors while executing the response will result in 500 Internal Server Error HTTP status code, otherwise the 200 OK status code will be returned.
1212
* `include-command-output-in-response-on-error` - boolean whether webhook should include command stdout & stderror as a response in failed executions. It only works if `include-command-output-in-response` is set to `true`.
13+
* `stream-stdout-in-response` - boolean (exclusive with `include-command-output-in-response` and `include-command-output-in-response-on-error`) that will stream the output of a command in the response if the command writes any data to standard output before exiting non-zero.
14+
* `stream-stderr-in-response-on-error` - boolean whether the webhook should send the stream of stderr on error. Only effective if `stream-stdout-in-response` is being used.
15+
* `stream-command-kill-grace-period-seconds` - float number of seconds to wait after trying to kill a stream command with SIGTERM before sending SIGKILL. Default is 0 (do not wait).
1316
* `parse-parameters-as-json` - specifies the list of arguments that contain JSON strings. These parameters will be decoded by webhook and you can access them like regular objects in rules and `pass-arguments-to-command`.
1417
* `pass-arguments-to-command` - specifies the list of arguments that will be passed to the command. Check [Referencing request values page](Referencing-Request-Values) to see how to reference the values from the request. If you want to pass a static string value to your command you can specify it as
1518
`{ "source": "string", "name": "argumentvalue" }`

‎hook/hook.go

+3
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,9 @@ type Hook struct {
385385
ResponseHeaders ResponseHeaders `json:"response-headers,omitempty"`
386386
CaptureCommandOutput bool `json:"include-command-output-in-response,omitempty"`
387387
CaptureCommandOutputOnError bool `json:"include-command-output-in-response-on-error,omitempty"`
388+
StreamCommandStdout bool `json:"stream-stdout-in-response,omitempty"`
389+
StreamCommandStderrOnError bool `json:"stream-stderr-in-response-on-error,omitempty"`
390+
StreamCommandKillGraceSecs float64 `json:"stream-command-kill-grace-period-seconds,omitempty"`
388391
PassEnvironmentToCommand []Argument `json:"pass-environment-to-command,omitempty"`
389392
PassArgumentsToCommand []Argument `json:"pass-arguments-to-command,omitempty"`
390393
PassFileToCommand []Argument `json:"pass-file-to-command,omitempty"`

‎test/hookecho.go

+48-11
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,56 @@ import (
77
"os"
88
"strings"
99
"strconv"
10+
"io"
1011
)
1112

13+
func checkPrefix(prefixMap map[string]struct{}, prefix string, arg string) bool {
14+
if _, found := prefixMap[prefix]; found {
15+
fmt.Printf("prefix specified more then once: %s", arg)
16+
os.Exit(-1)
17+
}
18+
19+
if strings.HasPrefix(arg, prefix) {
20+
prefixMap[prefix] = struct{}{}
21+
return true
22+
}
23+
24+
return false
25+
}
26+
1227
func main() {
28+
var outputStream io.Writer
29+
outputStream = os.Stdout
30+
seenPrefixes := make(map[string]struct{})
31+
exit_code := 0
32+
33+
for _, arg := range os.Args[1:] {
34+
if checkPrefix(seenPrefixes, "stream=", arg) {
35+
switch arg {
36+
case "stream=stdout":
37+
outputStream = os.Stdout
38+
case "stream=stderr":
39+
outputStream = os.Stderr
40+
case "stream=both":
41+
outputStream = io.MultiWriter(os.Stdout, os.Stderr)
42+
default:
43+
fmt.Printf("unrecognized stream specification: %s", arg)
44+
os.Exit(-1)
45+
}
46+
} else if checkPrefix(seenPrefixes, "exit=", arg) {
47+
exit_code_str := arg[5:]
48+
var err error
49+
exit_code_conv, err := strconv.Atoi(exit_code_str)
50+
exit_code = exit_code_conv
51+
if err != nil {
52+
fmt.Printf("Exit code %s not an int!", exit_code_str)
53+
os.Exit(-1)
54+
}
55+
}
56+
}
57+
1358
if len(os.Args) > 1 {
14-
fmt.Printf("arg: %s\n", strings.Join(os.Args[1:], " "))
59+
fmt.Fprintf(outputStream, "arg: %s\n", strings.Join(os.Args[1:], " "))
1560
}
1661

1762
var env []string
@@ -22,16 +67,8 @@ func main() {
2267
}
2368

2469
if len(env) > 0 {
25-
fmt.Printf("env: %s\n", strings.Join(env, " "))
70+
fmt.Fprintf(outputStream, "env: %s\n", strings.Join(env, " "))
2671
}
2772

28-
if (len(os.Args) > 1) && (strings.HasPrefix(os.Args[1], "exit=")) {
29-
exit_code_str := os.Args[1][5:]
30-
exit_code, err := strconv.Atoi(exit_code_str)
31-
if err != nil {
32-
fmt.Printf("Exit code %s not an int!", exit_code_str)
33-
os.Exit(-1)
34-
}
35-
os.Exit(exit_code)
36-
}
73+
os.Exit(exit_code)
3774
}

‎test/hooks.json.tmpl

+31
Original file line numberDiff line numberDiff line change
@@ -180,5 +180,36 @@
180180
"execute-command": "{{ .Hookecho }}",
181181
"include-command-output-in-response": true,
182182
"include-command-output-in-response-on-error": true
183+
},
184+
{
185+
"id": "stream-stdout-in-response",
186+
"pass-arguments-to-command": [
187+
{
188+
"source": "string",
189+
"name": "exit=0"
190+
},
191+
{
192+
"source": "string",
193+
"name": "stream=both"
194+
}
195+
],
196+
"execute-command": "{{ .Hookecho }}",
197+
"stream-stdout-in-response": true
198+
},
199+
{
200+
"id": "stream-stderr-in-response-on-error",
201+
"pass-arguments-to-command": [
202+
{
203+
"source": "string",
204+
"name": "exit=1"
205+
},
206+
{
207+
"source": "string",
208+
"name": "stream=stderr"
209+
}
210+
],
211+
"execute-command": "{{ .Hookecho }}",
212+
"stream-stdout-in-response": true,
213+
"stream-stderr-in-response-on-error": true
183214
}
184215
]

‎test/hooks.yaml.tmpl

+17
Original file line numberDiff line numberDiff line change
@@ -97,3 +97,20 @@
9797
execute-command: '{{ .Hookecho }}'
9898
include-command-output-in-response: true
9999
include-command-output-in-response-on-error: true
100+
- id: stream-stdout-in-response
101+
execute-command: '{{ .Hookecho }}'
102+
stream-stdout-in-response: true
103+
pass-arguments-to-command:
104+
- source: string
105+
name: exit=0
106+
- source: string
107+
name: stream=both
108+
- id: stream-stderr-in-response-on-error
109+
execute-command: '{{ .Hookecho }}'
110+
stream-stdout-in-response: true
111+
stream-stderr-in-response-on-error: true
112+
pass-arguments-to-command:
113+
- source: string
114+
name: exit=1
115+
- source: string
116+
name: stream=stderr

‎test/hookstream/hookstream.go

+107
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// Hook Stream is a simple utility for testing Webhook streaming capability. It spawns a TCP server on execution
2+
// which echos all connections to its stdout until it receives the string EOF.
3+
4+
package main
5+
6+
import (
7+
"fmt"
8+
"os"
9+
"strings"
10+
"strconv"
11+
"io"
12+
"net"
13+
"bufio"
14+
)
15+
16+
func checkPrefix(prefixMap map[string]struct{}, prefix string, arg string) bool {
17+
if _, found := prefixMap[prefix]; found {
18+
fmt.Printf("prefix specified more then once: %s", arg)
19+
os.Exit(-1)
20+
}
21+
22+
if strings.HasPrefix(arg, prefix) {
23+
prefixMap[prefix] = struct{}{}
24+
return true
25+
}
26+
27+
return false
28+
}
29+
30+
func main() {
31+
var outputStream io.Writer
32+
outputStream = os.Stdout
33+
seenPrefixes := make(map[string]struct{})
34+
exit_code := 0
35+
36+
for _, arg := range os.Args[1:] {
37+
if checkPrefix(seenPrefixes, "stream=", arg) {
38+
switch arg {
39+
case "stream=stdout":
40+
outputStream = os.Stdout
41+
case "stream=stderr":
42+
outputStream = os.Stderr
43+
case "stream=both":
44+
outputStream = io.MultiWriter(os.Stdout, os.Stderr)
45+
default:
46+
fmt.Printf("unrecognized stream specification: %s", arg)
47+
os.Exit(-1)
48+
}
49+
} else if checkPrefix(seenPrefixes, "exit=", arg) {
50+
exit_code_str := arg[5:]
51+
var err error
52+
exit_code_conv, err := strconv.Atoi(exit_code_str)
53+
exit_code = exit_code_conv
54+
if err != nil {
55+
fmt.Printf("Exit code %s not an int!", exit_code_str)
56+
os.Exit(-1)
57+
}
58+
}
59+
}
60+
61+
l, err := net.Listen("tcp", "localhost:0")
62+
if err != nil {
63+
fmt.Printf("Error starting tcp server: %v\n", err)
64+
os.Exit(-1)
65+
}
66+
defer l.Close()
67+
68+
// Emit the address of the server
69+
fmt.Printf("%v\n",l.Addr())
70+
71+
manageCh := make(chan struct{})
72+
73+
go func() {
74+
for {
75+
conn, err := l.Accept()
76+
if err != nil {
77+
fmt.Printf("Error accepting connection: %v\n", err)
78+
os.Exit(-1)
79+
}
80+
go handleRequest(manageCh, outputStream, conn)
81+
}
82+
}()
83+
84+
<- manageCh
85+
l.Close()
86+
87+
os.Exit(exit_code)
88+
}
89+
90+
// Handles incoming requests.
91+
func handleRequest(manageCh chan<- struct{}, w io.Writer, conn net.Conn) {
92+
defer conn.Close()
93+
bio := bufio.NewScanner(conn)
94+
for bio.Scan() {
95+
if line := strings.TrimSuffix(bio.Text(), "\n"); line == "EOF" {
96+
// Request program close
97+
select {
98+
case manageCh <- struct{}{}:
99+
// Request sent.
100+
default:
101+
// Already closing
102+
}
103+
break
104+
}
105+
fmt.Fprintf(w, "%s\n", bio.Text())
106+
}
107+
}

0 commit comments

Comments
 (0)