diff --git a/cmd/flatten-approvals/main.go b/cmd/flatten-approvals/main.go new file mode 100644 index 0000000..2b45389 --- /dev/null +++ b/cmd/flatten-approvals/main.go @@ -0,0 +1,113 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "os" + "path/filepath" +) + +var inplace = flag.Bool("i", false, "modify file in place") + +func main() { + flag.Parse() + if err := flatten(flag.Args()); err != nil { + log.Fatal(err) + } +} + +func flatten(args []string) error { + var filepaths []string + for _, arg := range args { + matches, err := filepath.Glob(arg) + if err != nil { + return err + } + filepaths = append(filepaths, matches...) + } + for _, filepath := range filepaths { + if err := transform(filepath); err != nil { + return fmt.Errorf("error transforming %q: %w", filepath, err) + } + } + return nil +} + +// transform []{"events": {"object": {"field": ...}}} to []{"field", "field", ...} +func transform(filepath string) error { + var input struct { + Events []map[string]any `json:"events"` + } + if err := decodeJSONFile(filepath, &input); err != nil { + return fmt.Errorf("could not read existing approved events file: %w", err) + } + out := make([]map[string][]any, 0, len(input.Events)) + for _, event := range input.Events { + fields := make(map[string][]any) + flattenFields("", event, fields) + out = append(out, fields) + } + + var w io.Writer = os.Stdout + if *inplace { + f, err := os.Create(filepath) + if err != nil { + return err + } + defer f.Close() + w = f + } + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(out) +} + +func flattenFields(k string, v any, out map[string][]any) { + switch v := v.(type) { + case map[string]any: + for k2, v := range v { + if k != "" { + k2 = k + "." + k2 + } + flattenFields(k2, v, out) + } + case []any: + for _, v := range v { + flattenFields(k, v, out) + } + default: + out[k] = append(out[k], v) + } +} + +func decodeJSONFile(path string, out interface{}) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + if err := json.NewDecoder(f).Decode(&out); err != nil { + return fmt.Errorf("cannot unmarshal file %q: %w", path, err) + } + return nil +} diff --git a/pkg/approvaltest/approvals.go b/pkg/approvaltest/approvals.go index 453dc1e..c4268d2 100644 --- a/pkg/approvaltest/approvals.go +++ b/pkg/approvaltest/approvals.go @@ -53,20 +53,6 @@ const ( func ApproveEvents(t testing.TB, name string, hits []espoll.SearchHit, dynamic ...string) { t.Helper() - // Fields generated by the server (e.g. observer.*) - // agent which may change between tests. - // - // Ignore their values in comparisons, but compare - // existence: either the field exists in both, or neither. - dynamic = append([]string{ - "ecs.version", - "event.ingested", - "observer.ephemeral_id", - "observer.hostname", - "observer.id", - "observer.version", - }, dynamic...) - // Sort events for repeatable diffs. sort.Slice(hits, func(i, j int) bool { return compareDocumentFields(hits[i].RawFields, hits[j].RawFields) < 0 @@ -79,6 +65,32 @@ func ApproveEvents(t testing.TB, name string, hits []espoll.SearchHit, dynamic . approveEventDocs(t, filepath.Join("approvals", name), sources, dynamic...) } +// ApproveFields compares the fields of the search hits with the +// contents of the file in "approvals/.approved.json". +// +// Dynamic fields (@timestamp, observer.id, etc.) are replaced +// with a static string for comparison. Integration tests elsewhere +// use canned data to test fields that we do not cover here. +// +// TODO(axw) eventually remove ApproveEvents when we have updated +// all calls to use ApproveFields. ApproveFields should be used +// since it includes runtime fields, whereas ApproveEvents only +// looks at _source. +func ApproveFields(t testing.TB, name string, hits []espoll.SearchHit, dynamic ...string) { + t.Helper() + + // Sort events for repeatable diffs. + sort.Slice(hits, func(i, j int) bool { + return compareDocumentFields(hits[i].RawFields, hits[j].RawFields) < 0 + }) + + fields := make([][]byte, len(hits)) + for i, hit := range hits { + fields[i] = hit.RawFields + } + approveFields(t, filepath.Join("approvals", name), fields, dynamic...) +} + // approveEventDocs compares the given event documents with // the contents of the file in ".approved.json". // @@ -89,6 +101,20 @@ func ApproveEvents(t testing.TB, name string, hits []espoll.SearchHit, dynamic . func approveEventDocs(t testing.TB, name string, eventDocs [][]byte, dynamic ...string) { t.Helper() + // Fields generated by the server (e.g. observer.*) + // agent which may change between tests. + // + // Ignore their values in comparisons, but compare + // existence: either the field exists in both, or neither. + dynamic = append([]string{ + "ecs.version", + "event.ingested", + "observer.ephemeral_id", + "observer.hostname", + "observer.id", + "observer.version", + }, dynamic...) + // Rewrite all dynamic fields to have a known value, // so dynamic fields don't affect diffs. events := make([]interface{}, len(eventDocs)) @@ -117,6 +143,42 @@ func approveEventDocs(t testing.TB, name string, eventDocs [][]byte, dynamic ... approve(t, name, received) } +func approveFields(t testing.TB, name string, docs [][]byte, dynamic ...string) { + t.Helper() + + // Fields generated by the server (e.g. observer.*) + // agent which may change between tests. + // + // Ignore their values in comparisons, but compare + // existence: either the field exists in both, or neither. + dynamic = append([]string{ + "ecs.version", + "event.ingested", + "observer.ephemeral_id", + "observer.hostname", + "observer.id", + "observer.version", + }, dynamic...) + + // Rewrite all dynamic fields to have a known value, + // so dynamic fields don't affect diffs. + decodedDocs := make([]any, len(docs)) + for i, doc := range docs { + var fields map[string]any + if err := json.Unmarshal(doc, &fields); err != nil { + t.Fatal(err) + } + for _, field := range dynamic { + if _, ok := fields[field]; ok { + fields[field] = []any{"dynamic"} + } + } + decodedDocs[i] = fields + } + + approve(t, name, decodedDocs) +} + // approve compares the given value with the contents of the file // ".approved.json". //