Skip to content

Commit 3c1867d

Browse files
tarokkktarokkk
authored
Document generator for supported plugins (#65)
Document generation and default variables fix
1 parent 547b5dc commit 3c1867d

File tree

11 files changed

+310
-42
lines changed

11 files changed

+310
-42
lines changed

‎cmd/docgen/docgen.go‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package main
2+
3+
import (
4+
"bytes"
5+
"fmt"
6+
"io/ioutil"
7+
"regexp"
8+
"text/template"
9+
"text/template/parse"
10+
11+
"github.com/Masterminds/sprig"
12+
"github.com/banzaicloud/logging-operator/pkg/resources/plugins"
13+
)
14+
15+
//TODO handle parameters
16+
func main() {
17+
pluginMap := plugins.GetAll()
18+
var indexPage bytes.Buffer
19+
indexPage.WriteString("# List of ")
20+
for name, plugin := range pluginMap {
21+
var data bytes.Buffer
22+
data.WriteString(fmt.Sprintf("# Plugin %s\n", name))
23+
t := template.New("PluginTemplate").Funcs(sprig.TxtFuncMap())
24+
t, err := t.Parse(plugin.Template)
25+
if err != nil {
26+
panic(err)
27+
}
28+
data.WriteString("## Variables\n")
29+
data.WriteString("| Variable name | Default | Applied function |\n")
30+
data.WriteString(fmt.Sprintf("|---|---|---|\n"))
31+
for _, item := range listTemplateFields(t) {
32+
regExp, err := regexp.Compile(`{{(?P<Function>\w*)?\s*.(?P<Variable>.*)}}`)
33+
if err != nil {
34+
panic(err)
35+
}
36+
matches := regExp.FindStringSubmatch(item)
37+
vairableName := matches[2]
38+
variableFunc := matches[1]
39+
defaultValue, ok := plugin.DefaultValues[matches[2]]
40+
if !ok {
41+
defaultValue = "-"
42+
}
43+
data.WriteString(fmt.Sprintf("| %s | %s | %s |\n", vairableName, defaultValue, variableFunc))
44+
45+
}
46+
data.WriteString("## Plugin template\n")
47+
data.WriteString("```" + plugin.Template + "\n```")
48+
err = ioutil.WriteFile("docs/plugins/"+name+".md", data.Bytes(), 0644)
49+
if err != nil {
50+
panic(err)
51+
}
52+
}
53+
}
54+
55+
func listTemplateFields(t *template.Template) []string {
56+
return listNodeFields(t.Tree.Root, nil)
57+
}
58+
59+
func listNodeFields(node parse.Node, res []string) []string {
60+
if node.Type() == parse.NodeAction {
61+
res = append(res, node.String())
62+
}
63+
64+
if ln, ok := node.(*parse.ListNode); ok {
65+
for _, n := range ln.Nodes {
66+
res = listNodeFields(n, res)
67+
}
68+
}
69+
return res
70+
}

‎docs/plugins/alibaba.md‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Plugin alibaba
2+
## Variables
3+
| Variable name | Default | Applied function |
4+
|---|---|---|
5+
| pattern | - | |
6+
| aliKeyId | - | |
7+
| aliKeySecret | - | |
8+
| bucket | - | |
9+
| aliBucketEndpoint | - | |
10+
| oss_object_key_format | %{time_slice}/%{host}-%{uuid}.%{file_ext} | |
11+
| buffer_path | /buffers/ali | |
12+
| buffer_chunk_limit | 1m | |
13+
| time_slice_format | %Y%m%d | |
14+
| time_slice_wait | 10m | |
15+
## Plugin template
16+
```
17+
<match {{ .pattern }}.**>
18+
@type oss
19+
oss_key_id {{ .aliKeyId }}
20+
oss_key_secret {{ .aliKeySecret }}
21+
oss_bucket {{ .bucket }}
22+
oss_endpoint {{ .aliBucketEndpoint }}
23+
oss_object_key_format {{ .oss_object_key_format }}
24+
25+
buffer_path {{ .buffer_path }}
26+
buffer_chunk_limit {{ .buffer_chunk_limit }}
27+
time_slice_format {{ .time_slice_format }}
28+
time_slice_wait {{ .time_slice_wait }}
29+
</match>
30+
```

‎docs/plugins/azure.md‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Plugin azure
2+
## Variables
3+
| Variable name | Default | Applied function |
4+
|---|---|---|
5+
| pattern | - | |
6+
| storageAccountName | - | |
7+
| storageAccountKey | - | |
8+
| bucket | - | |
9+
| azure_object_key_format | %{path}%{time_slice}_%{index}.%{file_extension} | |
10+
| path | logs/${tag}/%Y/%m/%d/ | |
11+
| time_slice_format | %Y%m%d-%H | |
12+
| bufferPath | /buffers/azure | |
13+
| timekey | 1h | |
14+
| timekey_wait | 10m | |
15+
| timekey_use_utc | true | |
16+
| format | json | |
17+
## Plugin template
18+
```
19+
<match {{ .pattern }}.**>
20+
@type azurestorage
21+
22+
azure_storage_account {{ .storageAccountName }}
23+
azure_storage_access_key {{ .storageAccountKey }}
24+
azure_container {{ .bucket }}
25+
azure_storage_type blob
26+
store_as gzip
27+
auto_create_container true
28+
azure_object_key_format {{ .azure_object_key_format }}
29+
path {{ .path }}
30+
time_slice_format {{ .time_slice_format }}
31+
# if you want to use ${tag} or %Y/%m/%d/ like syntax in path / object_key_format,
32+
# need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
33+
<buffer tag,time>
34+
@type file
35+
path {{ .bufferPath }}
36+
timekey {{ .timekey }}
37+
timekey_wait {{ .timekey_wait }}
38+
timekey_use_utc {{ .timekey_use_utc }}
39+
</buffer>
40+
41+
<format>
42+
@type {{ .format }}
43+
</format>
44+
</match>
45+
```

‎docs/plugins/gcs.md‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Plugin gcs
2+
## Variables
3+
| Variable name | Default | Applied function |
4+
|---|---|---|
5+
| pattern | - | |
6+
| project | - | |
7+
| private_key | - | toJson |
8+
| client_email | - | |
9+
| bucket | - | |
10+
| object_key_format | %{path}%{time_slice}_%{index}.%{file_extension} | |
11+
| path | logs/${tag}/%Y/%m/%d/ | |
12+
| bufferPath | /buffers/gcs | |
13+
| timekey | 1h | |
14+
| timekey_wait | 10m | |
15+
| timekey_use_utc | true | |
16+
| format | json | |
17+
## Plugin template
18+
```
19+
<match {{ .pattern }}.**>
20+
@type gcs
21+
22+
project {{ .project }}
23+
credentialsJson { "private_key": {{ toJson .private_key }}, "client_email": "{{ .client_email }}" }
24+
bucket {{ .bucket }}
25+
object_key_format {{ .object_key_format }}
26+
path {{ .path }}
27+
28+
# if you want to use ${tag} or %Y/%m/%d/ like syntax in path / object_key_format,
29+
# need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
30+
<buffer tag,time>
31+
@type file
32+
path {{ .bufferPath }}
33+
timekey {{ .timekey }}
34+
timekey_wait {{ .timekey_wait }}
35+
timekey_use_utc {{ .timekey_use_utc }}
36+
</buffer>
37+
38+
<format>
39+
@type {{ .format }}
40+
</format>
41+
</match>
42+
```

‎docs/plugins/parser.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Plugin parser
2+
## Variables
3+
| Variable name | Default | Applied function |
4+
|---|---|---|
5+
| pattern | - | |
6+
| format | - | |
7+
| timeFormat | - | |
8+
| keyName | log | |
9+
## Plugin template
10+
```
11+
<filter {{ .pattern }}.** >
12+
@type parser
13+
format {{ .format }}
14+
time_format {{ .timeFormat }}
15+
key_name {{ .keyName }}
16+
</filter>
17+
18+
```

‎docs/plugins/s3.md‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Plugin s3
2+
## Variables
3+
| Variable name | Default | Applied function |
4+
|---|---|---|
5+
| pattern | - | |
6+
| aws_key_id | - | |
7+
| aws_sec_key | - | |
8+
| s3_bucket | - | |
9+
| s3_region | - | |
10+
| s3_object_key_format | %{path}%{time_slice}_%{index}.%{file_extension} | |
11+
| bufferPath | /buffers/s3 | |
12+
| bufferTimeKey | 3600 | |
13+
| bufferTimeWait | 10m | |
14+
| timekey_use_utc | true | |
15+
| format | json | |
16+
## Plugin template
17+
```
18+
<match {{ .pattern }}.** >
19+
@type s3
20+
21+
aws_key_id {{ .aws_key_id }}
22+
aws_sec_key {{ .aws_sec_key }}
23+
s3_bucket {{ .s3_bucket }}
24+
s3_region {{ .s3_region }}
25+
store_as gzip_command
26+
27+
path logs/${tag}/%Y/%m/%d/
28+
s3_object_key_format {{ .s3_object_key_format }}
29+
30+
# if you want to use ${tag} or %Y/%m/%d/ like syntax in path / s3_object_key_format,
31+
# need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
32+
<buffer tag,time>
33+
@type file
34+
path {{ .bufferPath }}
35+
timekey {{ .bufferTimeKey }}
36+
timekey_wait {{ .bufferTimeWait }}
37+
timekey_use_utc {{ .timekey_use_utc }}
38+
</buffer>
39+
<format>
40+
@type {{ .format }}
41+
</format>
42+
</match>
43+
```

‎pkg/resources/plugins/alibaba.go‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,11 @@ const AlibabaOutput = "alibaba"
2121

2222
// AlibabaDefaultValues for Alibaba OSS output plugin
2323
var AlibabaDefaultValues = map[string]string{
24-
"buffer_chunk_limit": "256m",
25-
"buffer_path": "/buffers/ali",
26-
"time_slice_format": "%Y%m%d",
27-
"time_slice_wait": "10m",
24+
"buffer_chunk_limit": "1m",
25+
"buffer_path": "/buffers/ali",
26+
"time_slice_format": "%Y%m%d",
27+
"time_slice_wait": "10m",
28+
"oss_object_key_format": "%{time_slice}/%{host}-%{uuid}.%{file_ext}",
2829
}
2930

3031
// AlibabaTemplate for Alibaba OSS output plugin
@@ -35,10 +36,10 @@ const AlibabaTemplate = `
3536
oss_key_secret {{ .aliKeySecret }}
3637
oss_bucket {{ .bucket }}
3738
oss_endpoint {{ .aliBucketEndpoint }}
38-
oss_object_key_format "%{time_slice}/%{host}-%{uuid}.%{file_ext}"
39+
oss_object_key_format {{ .oss_object_key_format }}
3940
40-
buffer_path /buffers/ali
41-
buffer_chunk_limit 1m
42-
time_slice_format %Y%m%d
43-
time_slice_wait 10m
41+
buffer_path {{ .buffer_path }}
42+
buffer_chunk_limit {{ .buffer_chunk_limit }}
43+
time_slice_format {{ .time_slice_format }}
44+
time_slice_wait {{ .time_slice_wait }}
4445
</match>`

‎pkg/resources/plugins/azure.go‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,16 @@ const AzureOutput = "azure"
2121

2222
// AzureDefaultValues for Azure ObjectStore output plugin
2323
var AzureDefaultValues = map[string]string{
24-
"bufferTimeKey": "3600",
25-
"bufferTimeWait": "10m",
26-
"bufferPath": "/buffers/azure",
27-
"format": "json",
24+
"bufferTimeKey": "3600",
25+
"bufferTimeWait": "10m",
26+
"bufferPath": "/buffers/azure",
27+
"format": "json",
28+
"timekey": "1h",
29+
"timekey_wait": "10m",
30+
"timekey_use_utc": "true",
31+
"time_slice_format": "%Y%m%d-%H",
32+
"azure_object_key_format": "%{path}%{time_slice}_%{index}.%{file_extension}",
33+
"path": "logs/${tag}/%Y/%m/%d/",
2834
}
2935

3036
// AzureTemplate for Azure ObjectStore output plugin
@@ -38,20 +44,20 @@ const AzureTemplate = `
3844
azure_storage_type blob
3945
store_as gzip
4046
auto_create_container true
41-
azure_object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
42-
path logs/${tag}/%Y/%m/%d/
43-
time_slice_format %Y%m%d-%H
47+
azure_object_key_format {{ .azure_object_key_format }}
48+
path {{ .path }}
49+
time_slice_format {{ .time_slice_format }}
4450
# if you want to use ${tag} or %Y/%m/%d/ like syntax in path / object_key_format,
4551
# need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
4652
<buffer tag,time>
4753
@type file
48-
path /buffers/azure
49-
timekey 1h # 1 hour partition
50-
timekey_wait 10m
51-
timekey_use_utc true # use utc
54+
path {{ .bufferPath }}
55+
timekey {{ .timekey }}
56+
timekey_wait {{ .timekey_wait }}
57+
timekey_use_utc {{ .timekey_use_utc }}
5258
</buffer>
5359
5460
<format>
55-
@type json
61+
@type {{ .format }}
5662
</format>
5763
</match>`

‎pkg/resources/plugins/gcs.go‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,15 @@ const GCSOutput = "gcs"
2121

2222
// GCSDefaultValues for Google Cloud Storage output plugin
2323
var GCSDefaultValues = map[string]string{
24-
"bufferTimeKey": "3600",
25-
"bufferTimeWait": "10m",
26-
"bufferPath": "/buffers/gcs",
27-
"format": "json",
24+
"bufferTimeKey": "3600",
25+
"bufferTimeWait": "10m",
26+
"bufferPath": "/buffers/gcs",
27+
"object_key_format": "%{path}%{time_slice}_%{index}.%{file_extension}",
28+
"path": "logs/${tag}/%Y/%m/%d/",
29+
"timekey": "1h",
30+
"timekey_wait": "10m",
31+
"timekey_use_utc": "true",
32+
"format": "json",
2833
}
2934

3035
// GCSTemplate for Google Cloud Storage output plugin
@@ -35,20 +40,20 @@ const GCSTemplate = `
3540
project {{ .project }}
3641
credentialsJson { "private_key": {{ toJson .private_key }}, "client_email": "{{ .client_email }}" }
3742
bucket {{ .bucket }}
38-
object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
39-
path logs/${tag}/%Y/%m/%d/
43+
object_key_format {{ .object_key_format }}
44+
path {{ .path }}
4045
4146
# if you want to use ${tag} or %Y/%m/%d/ like syntax in path / object_key_format,
4247
# need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
4348
<buffer tag,time>
4449
@type file
45-
path /buffers/gcs
46-
timekey 1h # 1 hour partition
47-
timekey_wait 10m
48-
timekey_use_utc true # use utc
50+
path {{ .bufferPath }}
51+
timekey {{ .timekey }}
52+
timekey_wait {{ .timekey_wait }}
53+
timekey_use_utc {{ .timekey_use_utc }}
4954
</buffer>
5055
5156
<format>
52-
@type json
57+
@type {{ .format }}
5358
</format>
5459
</match>`

0 commit comments

Comments
 (0)