From 8b9c8b438d9100e92f2db6b9ee916fcbeffeac5f Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Fri, 8 Aug 2025 17:50:04 -0400 Subject: [PATCH 01/11] feat!: replace AssessmentStep with AssessmentMethod Moves from AssessmentStep to AssessmentMethod to allow for metadata to be used to describe the check or test Signed-off-by: Jennifer Power --- layer4/assessment.go | 62 +++++++++------------------- layer4/assessment_method.go | 68 +++++++++++++++++++++++++++++++ layer4/assessment_test.go | 60 +++++++++++++-------------- layer4/control_evaluation.go | 8 ++-- layer4/control_evaluation_test.go | 12 +++--- layer4/test-data.go | 54 ++++++++++++------------ 6 files changed, 155 insertions(+), 109 deletions(-) create mode 100644 layer4/assessment_method.go diff --git a/layer4/assessment.go b/layer4/assessment.go index 346c748..743a296 100644 --- a/layer4/assessment.go +++ b/layer4/assessment.go @@ -1,15 +1,12 @@ package layer4 import ( - "encoding/json" "errors" "fmt" - "reflect" - "runtime" "time" ) -// Assessment is a struct that contains the results of a single step within a ControlEvaluation. +// Assessment is a struct that contains the results of a single method within a ControlEvaluation. type Assessment struct { // RequirementID is the unique identifier for the requirement being tested RequirementId string `yaml:"requirement-id"` @@ -21,10 +18,10 @@ type Assessment struct { Result Result `yaml:"result"` // Message is the human-readable result of the test Message string `yaml:"message"` - // Steps is a slice of steps that were executed during the test - Steps []AssessmentStep `yaml:"steps"` - // StepsExecuted is the number of steps that were executed during the test - StepsExecuted int `yaml:"steps-executed,omitempty"` + // Methods is a slice of assessment methods that were executed during the test + Methods []*AssessmentMethod `yaml:"methods"` + // MethodsExecuted is the number of assessment methods that were executed during the test + MethodsExecuted int `yaml:"methods-executed,omitempty"` // RunDuration is the time it took to run the test RunDuration string `yaml:"run-duration,omitempty"` // Value is the object that was returned during the test @@ -33,54 +30,33 @@ type Assessment struct { Changes map[string]*Change `yaml:"changes,omitempty"` } -// AssessmentStep is a function type that inspects the provided targetData and returns a Result with a message. -// The message may be an error string or other descriptive text. -type AssessmentStep func(payload interface{}, c map[string]*Change) (Result, string) - -func (as AssessmentStep) String() string { - // Get the function pointer correctly - fn := runtime.FuncForPC(reflect.ValueOf(as).Pointer()) - if fn == nil { - return "" - } - return fn.Name() -} - -func (as AssessmentStep) MarshalJSON() ([]byte, error) { - return json.Marshal(as.String()) -} - -func (as AssessmentStep) MarshalYAML() (interface{}, error) { - return as.String(), nil -} - // NewAssessment creates a new Assessment object and returns a pointer to it. -func NewAssessment(requirementId string, description string, applicability []string, steps []AssessmentStep) (*Assessment, error) { +func NewAssessment(requirementId string, description string, applicability []string, methods []*AssessmentMethod) (*Assessment, error) { a := &Assessment{ RequirementId: requirementId, Description: description, Applicability: applicability, Result: NotRun, - Steps: steps, + Methods: methods, } err := a.precheck() return a, err } -// AddStep queues a new step in the Assessment -func (a *Assessment) AddStep(step AssessmentStep) { - a.Steps = append(a.Steps, step) +// AddMethod queues a new method in the Assessment +func (a *Assessment) AddMethod(method AssessmentMethod) { + a.Methods = append(a.Methods, &method) } -func (a *Assessment) runStep(targetData interface{}, step AssessmentStep) Result { - a.StepsExecuted++ - result, message := step(targetData, a.Changes) +func (a *Assessment) runMethod(targetData interface{}, method *AssessmentMethod) Result { + a.MethodsExecuted++ + result, message := method.RunMethod(targetData, a.Changes) a.Result = UpdateAggregateResult(a.Result, result) a.Message = message return result } -// Run will execute all steps, halting if any step does not return layer4.Passed. +// Run will execute all steps, halting if any method does not return layer4.Passed. func (a *Assessment) Run(targetData interface{}, changesAllowed bool) Result { if a.Result != NotRun { return a.Result @@ -97,8 +73,8 @@ func (a *Assessment) Run(targetData interface{}, changesAllowed bool) Result { change.Allow() } } - for _, step := range a.Steps { - if a.runStep(targetData, step) == Failed { + for _, method := range a.Methods { + if a.runMethod(targetData, method) == Failed { return Failed } } @@ -142,10 +118,10 @@ func (a *Assessment) RevertChanges() (corrupted bool) { // precheck verifies that the assessment has all the required fields. // It returns an error if the assessment is not valid. func (a *Assessment) precheck() error { - if a.RequirementId == "" || a.Description == "" || a.Applicability == nil || a.Steps == nil || len(a.Applicability) == 0 || len(a.Steps) == 0 { + if a.RequirementId == "" || a.Description == "" || a.Applicability == nil || a.Methods == nil || len(a.Applicability) == 0 || len(a.Methods) == 0 { message := fmt.Sprintf( - "expected all Assessment fields to have a value, but got: requirementId=len(%v), description=len=(%v), applicability=len(%v), steps=len(%v)", - len(a.RequirementId), len(a.Description), len(a.Applicability), len(a.Steps), + "expected all Assessment fields to have a value, but got: requirementId=len(%v), description=len=(%v), applicability=len(%v), methods=len(%v)", + len(a.RequirementId), len(a.Description), len(a.Applicability), len(a.Methods), ) a.Result = Unknown a.Message = message diff --git a/layer4/assessment_method.go b/layer4/assessment_method.go new file mode 100644 index 0000000..86b9262 --- /dev/null +++ b/layer4/assessment_method.go @@ -0,0 +1,68 @@ +package layer4 + +import ( + "encoding/json" + "reflect" + "runtime" +) + +// AssessmentMethod describes the method used to assess the layer 2 control requirement referenced by requirementID. +type AssessmentMethod struct { + // Id is the unique identifier of the assessment method being executed + Id string `json:"id"` + + // Name is the name of the method used to assess the requirement. + Name string `json:"name"` + + // Description is a detailed explanation of the method. + Description string `json:"description"` + + // Run is a boolean indicating whether the method was run or not. When run is true, result is expected to be present. + Run bool `json:"run"` + + // Remediation guide is a URL to remediation guidance associated with the control's assessment requirement and this specific assessment method. + RemediationGuide URL `json:"remediation-guide,omitempty"` + + // URL to documentation that describes how the assessment method evaluates the control requirement. + Documentation URL `json:"documentation,omitempty"` + + // Result is the status or outcome of an assessed method present when run us true. + Result *Result `json:"result,omitempty"` + // Executor is a function type that inspects the provided targetData and returns a Result with a message. + // The message may be an error string or other descriptive text. + Executor MethodExecutor +} + +// URL describes a specific subset of URLs of interest to the framework +type URL string + +// MethodExecutor is a function type that inspects the provided payload and returns the result of the assessment. +// The payload is the data/evidence that the assessment will be run against. +type MethodExecutor func(payload interface{}, c map[string]*Change) (Result, string) + +// RunMethod executes the assessment method using the provided payload and changes. +// It returns the result of the assessment and any error encountered during execution. +// The payload is the data/evidence that the assessment will be run against. +func (a *AssessmentMethod) RunMethod(payload interface{}, changes map[string]*Change) (Result, string) { + result, message := a.Executor(payload, changes) + a.Result = &result + a.Run = true + return result, message +} + +func (e MethodExecutor) String() string { + // Get the function pointer correctly + fn := runtime.FuncForPC(reflect.ValueOf(e).Pointer()) + if fn == nil { + return "" + } + return fn.Name() +} + +func (e MethodExecutor) MarshalJSON() ([]byte, error) { + return json.Marshal(e.String()) +} + +func (e MethodExecutor) MarshalYAML() (interface{}, error) { + return e.String(), nil +} diff --git a/layer4/assessment_test.go b/layer4/assessment_test.go index 59da860..c3a5f35 100644 --- a/layer4/assessment_test.go +++ b/layer4/assessment_test.go @@ -23,7 +23,7 @@ func getAssessmentsTestData() []struct { assessment: Assessment{}, }, { - testName: "Assessment with one step", + testName: "Assessment with one method", assessment: passingAssessment(), numberOfSteps: 1, numberOfStepsToRun: 1, @@ -53,53 +53,53 @@ func getAssessmentsTestData() []struct { } } -// TestNewStep ensures that NewStep queues a new step in the Assessment +// TestNewStep ensures that NewStep queues a new method in the Assessment func TestAddStep(t *testing.T) { for _, test := range getAssessmentsTestData() { t.Run(test.testName, func(t *testing.T) { - if len(test.assessment.Steps) != test.numberOfSteps { - t.Errorf("Bad test data: expected to start with %d, got %d", test.numberOfSteps, len(test.assessment.Steps)) + if len(test.assessment.Methods) != test.numberOfSteps { + t.Errorf("Bad test data: expected to start with %d, got %d", test.numberOfSteps, len(test.assessment.Methods)) } - test.assessment.AddStep(passingAssessmentStep) - if len(test.assessment.Steps) != test.numberOfSteps+1 { - t.Errorf("expected %d, got %d", test.numberOfSteps, len(test.assessment.Steps)) + test.assessment.AddMethod(passingAssessmentMethod) + if len(test.assessment.Methods) != test.numberOfSteps+1 { + t.Errorf("expected %d, got %d", test.numberOfSteps, len(test.assessment.Methods)) } }) } } -// TestRunStep ensures that runStep runs the step and updates the Assessment +// TestRunStep ensures that runStep runs the method and updates the Assessment func TestRunStep(t *testing.T) { stepsTestData := []struct { testName string - step AssessmentStep + method AssessmentMethod result Result }{ { - testName: "Failing step", - step: failingAssessmentStep, + testName: "Failing method", + method: failingAssessmentMethod, result: Failed, }, { - testName: "Passing step", - step: passingAssessmentStep, + testName: "Passing method", + method: passingAssessmentMethod, result: Passed, }, { - testName: "Needs review step", - step: needsReviewAssessmentStep, + testName: "Needs review method", + method: needsReviewAssessmentMethod, result: NeedsReview, }, { - testName: "Unknown step", - step: unknownAssessmentStep, + testName: "Unknown method", + method: unknownAssessmentMethod, result: Unknown, }, } for _, test := range stepsTestData { t.Run(test.testName, func(t *testing.T) { anyOldAssessment := Assessment{} - result := anyOldAssessment.runStep(nil, test.step) + result := anyOldAssessment.runMethod(nil, &test.method) if result != test.result { t.Errorf("expected %s, got %s", test.result, result) } @@ -110,7 +110,7 @@ func TestRunStep(t *testing.T) { } } -// TestRun ensures that Run executes all steps, halting if any step does not return Passed +// TestRun ensures that Run executes all steps, halting if any method does not return Passed func TestRun(t *testing.T) { for _, data := range getAssessmentsTestData() { t.Run(data.testName, func(t *testing.T) { @@ -119,8 +119,8 @@ func TestRun(t *testing.T) { if result != a.Result { t.Errorf("expected match between Run return value (%s) and assessment Result value (%s)", result, data.expectedResult) } - if a.StepsExecuted != data.numberOfStepsToRun { - t.Errorf("expected to run %d tests, got %d", data.numberOfStepsToRun, a.StepsExecuted) + if a.MethodsExecuted != data.numberOfStepsToRun { + t.Errorf("expected to run %d tests, got %d", data.numberOfStepsToRun, a.MethodsExecuted) } }) } @@ -130,8 +130,8 @@ func TestRunB(t *testing.T) { for _, data := range getAssessmentsTestData() { t.Run(data.testName+"-no-changes", func(t *testing.T) { data.assessment.Run(nil, false) - if data.assessment.StepsExecuted != data.numberOfStepsToRun { - t.Errorf("expected to run %d tests, got %d", data.numberOfStepsToRun, data.assessment.StepsExecuted) + if data.assessment.MethodsExecuted != data.numberOfStepsToRun { + t.Errorf("expected to run %d tests, got %d", data.numberOfStepsToRun, data.assessment.MethodsExecuted) } for _, change := range data.assessment.Changes { if change.Allowed { @@ -234,7 +234,7 @@ func TestNewAssessment(t *testing.T) { requirementId string description string applicability []string - steps []AssessmentStep + methods []*AssessmentMethod expectedError bool }{ { @@ -242,7 +242,7 @@ func TestNewAssessment(t *testing.T) { requirementId: "", description: "test", applicability: []string{"test"}, - steps: []AssessmentStep{passingAssessmentStep}, + methods: []*AssessmentMethod{&passingAssessmentMethod}, expectedError: true, }, { @@ -250,7 +250,7 @@ func TestNewAssessment(t *testing.T) { requirementId: "test", description: "", applicability: []string{"test"}, - steps: []AssessmentStep{passingAssessmentStep}, + methods: []*AssessmentMethod{&passingAssessmentMethod}, expectedError: true, }, { @@ -258,7 +258,7 @@ func TestNewAssessment(t *testing.T) { requirementId: "test", description: "test", applicability: []string{}, - steps: []AssessmentStep{passingAssessmentStep}, + methods: []*AssessmentMethod{&passingAssessmentMethod}, expectedError: true, }, { @@ -266,7 +266,7 @@ func TestNewAssessment(t *testing.T) { requirementId: "test", description: "test", applicability: []string{"test"}, - steps: []AssessmentStep{}, + methods: []*AssessmentMethod{}, expectedError: true, }, { @@ -274,13 +274,13 @@ func TestNewAssessment(t *testing.T) { requirementId: "test", description: "test", applicability: []string{"test"}, - steps: []AssessmentStep{passingAssessmentStep}, + methods: []*AssessmentMethod{&passingAssessmentMethod}, expectedError: false, }, } for _, data := range newAssessmentsTestData { t.Run(data.testName, func(t *testing.T) { - assessment, err := NewAssessment(data.requirementId, data.description, data.applicability, data.steps) + assessment, err := NewAssessment(data.requirementId, data.description, data.applicability, data.methods) if data.expectedError && err == nil { t.Error("expected error, got nil") } diff --git a/layer4/control_evaluation.go b/layer4/control_evaluation.go index fd09d81..abb8c50 100644 --- a/layer4/control_evaluation.go +++ b/layer4/control_evaluation.go @@ -26,8 +26,8 @@ type ControlEvaluation struct { } // AddAssessment creates a new Assessment object and adds it to the ControlEvaluation. -func (c *ControlEvaluation) AddAssessment(requirementId string, description string, applicability []string, steps []AssessmentStep) (assessment *Assessment) { - assessment, err := NewAssessment(requirementId, description, applicability, steps) +func (c *ControlEvaluation) AddAssessment(requirementId string, description string, applicability []string, methods []*AssessmentMethod) (assessment *Assessment) { + assessment, err := NewAssessment(requirementId, description, applicability, methods) if err != nil { c.Result = Failed c.Message = err.Error() @@ -36,8 +36,8 @@ func (c *ControlEvaluation) AddAssessment(requirementId string, description stri return } -// Evaluate runs each step in each assessment, updating the relevant fields on the control evaluation. -// It will halt if a step returns a failed result. The targetData is the data that the assessment will be run against. +// Evaluate runs each method in each assessment, updating the relevant fields on the control evaluation. +// It will halt if a method returns a failed result. The targetData is the data that the assessment will be run against. // The userApplicability is a slice of strings that determine when the assessment is applicable. The changesAllowed // determines whether the assessment is allowed to execute its changes. func (c *ControlEvaluation) Evaluate(targetData interface{}, userApplicability []string, changesAllowed bool) { diff --git a/layer4/control_evaluation_test.go b/layer4/control_evaluation_test.go index 7e64d54..734f6c8 100644 --- a/layer4/control_evaluation_test.go +++ b/layer4/control_evaluation_test.go @@ -1,6 +1,8 @@ package layer4 -import "testing" +import ( + "testing" +) var controlEvaluationTestData = []struct { testName string @@ -137,16 +139,16 @@ func TestEvaluate(t *testing.T) { } } -func TestAddAssesment(t *testing.T) { +func TestAddAssessment(t *testing.T) { - controlEvaluationTestData[0].control.AddAssessment("test", "test", []string{}, []AssessmentStep{}) + controlEvaluationTestData[0].control.AddAssessment("test", "test", []string{}, []*AssessmentMethod{}) if controlEvaluationTestData[0].control.Result != Failed { t.Errorf("Expected Result to be Failed, but it was %v", controlEvaluationTestData[0].control.Result) } - if controlEvaluationTestData[0].control.Message != "expected all Assessment fields to have a value, but got: requirementId=len(4), description=len=(4), applicability=len(0), steps=len(0)" { - t.Errorf("Expected error message to be 'expected all Assessment fields to have a value, but got: requirementId=len(4), description=len=(4), applicability=len(0), steps=len(0)', but instead it was '%v'", controlEvaluationTestData[0].control.Message) + if controlEvaluationTestData[0].control.Message != "expected all Assessment fields to have a value, but got: requirementId=len(4), description=len=(4), applicability=len(0), methods=len(0)" { + t.Errorf("Expected error message to be 'expected all Assessment fields to have a value, but got: requirementId=len(4), description=len=(4), applicability=len(0), methods=len(0)', but instead it was '%v'", controlEvaluationTestData[0].control.Message) } } diff --git a/layer4/test-data.go b/layer4/test-data.go index 70579b0..7b4f874 100644 --- a/layer4/test-data.go +++ b/layer4/test-data.go @@ -23,18 +23,18 @@ var ( } // Assessment Results - passingAssessmentStep = func(interface{}, map[string]*Change) (Result, string) { + passingAssessmentMethod = AssessmentMethod{Executor: func(interface{}, map[string]*Change) (Result, string) { return Passed, "" - } - failingAssessmentStep = func(interface{}, map[string]*Change) (Result, string) { + }} + failingAssessmentMethod = AssessmentMethod{Executor: func(interface{}, map[string]*Change) (Result, string) { return Failed, "" - } - needsReviewAssessmentStep = func(interface{}, map[string]*Change) (Result, string) { + }} + needsReviewAssessmentMethod = AssessmentMethod{Executor: func(interface{}, map[string]*Change) (Result, string) { return NeedsReview, "" - } - unknownAssessmentStep = func(interface{}, map[string]*Change) (Result, string) { + }} + unknownAssessmentMethod = AssessmentMethod{Executor: func(interface{}, map[string]*Change) (Result, string) { return Unknown, "" - } + }} ) func pendingChangePtr() *Change { @@ -155,9 +155,9 @@ func failingAssessment() Assessment { return Assessment{ RequirementId: "failingAssessment()", Description: "failing assessment", - Steps: []AssessmentStep{ - failingAssessmentStep, - passingAssessmentStep, + Methods: []*AssessmentMethod{ + &failingAssessmentMethod, + &passingAssessmentMethod, }, Applicability: testingApplicability, } @@ -171,8 +171,8 @@ func passingAssessment() Assessment { return Assessment{ RequirementId: "passingAssessment()", Description: "passing assessment", - Steps: []AssessmentStep{ - passingAssessmentStep, + Methods: []*AssessmentMethod{ + &passingAssessmentMethod, }, Applicability: testingApplicability, Changes: map[string]*Change{ @@ -189,10 +189,10 @@ func needsReviewAssessment() Assessment { return Assessment{ RequirementId: "needsReviewAssessment()", Description: "needs review assessment", - Steps: []AssessmentStep{ - passingAssessmentStep, - needsReviewAssessmentStep, - passingAssessmentStep, + Methods: []*AssessmentMethod{ + &passingAssessmentMethod, + &needsReviewAssessmentMethod, + &passingAssessmentMethod, }, Applicability: testingApplicability, } @@ -206,10 +206,10 @@ func unknownAssessment() Assessment { return Assessment{ RequirementId: "unknownAssessment()", Description: "unknown assessment", - Steps: []AssessmentStep{ - passingAssessmentStep, - unknownAssessmentStep, - passingAssessmentStep, + Methods: []*AssessmentMethod{ + &passingAssessmentMethod, + &unknownAssessmentMethod, + &passingAssessmentMethod, }, Applicability: testingApplicability, } @@ -218,15 +218,15 @@ func unknownAssessment() Assessment { func badRevertPassingAssessment() Assessment { return Assessment{ RequirementId: "badRevertPassingAssessment()", - Description: "bad revert passing assessment", + Description: "bad revert passing assessment", Changes: map[string]*Change{ "badRevertChange": badRevertChangePtr(), }, - Steps: []AssessmentStep{ - passingAssessmentStep, - passingAssessmentStep, - passingAssessmentStep, - passingAssessmentStep, + Methods: []*AssessmentMethod{ + &passingAssessmentMethod, + &passingAssessmentMethod, + &passingAssessmentMethod, + &passingAssessmentMethod, }, Applicability: testingApplicability, } From 42864d18af769280205e44908c162e503d8e19fe Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Fri, 8 Aug 2025 17:51:06 -0400 Subject: [PATCH 02/11] chore: updates layer4 schema to reflect code changes The layer 4 schema is not up to date with the Go code implemented. This chanes updates the schema and uses evaluation data generated from L4 unit tests to verify againt the schema. Signed-off-by: Jennifer Power --- layer4/testdata/good-evaluation.yml | 28 ++++++++ schemas/layer-4.cue | 106 +++++++++++++++++++++------- 2 files changed, 109 insertions(+), 25 deletions(-) create mode 100644 layer4/testdata/good-evaluation.yml diff --git a/layer4/testdata/good-evaluation.yml b/layer4/testdata/good-evaluation.yml new file mode 100644 index 0000000..10b33f0 --- /dev/null +++ b/layer4/testdata/good-evaluation.yml @@ -0,0 +1,28 @@ +evaluations: + - name: "Test Evaluation" + control-id: "CTRL-1" + result: Passed + message: "The control is an example that passed." + corrupted-state: false + remediation-guide: "https://example-remediation.dev" + assessments: + - requirement-id: passingAssessment() + applicability: + - test-applicability + description: passing assessment + result: Passed + message: "The requirement has passed." + methods: + - id: "method-1" + name: "The first method" + description: "The is the only method executed for this requirement" + run: true + result: Passed + executor: github.com/revanite-io/gemara/layer4.init.func5 + methods-executed: 1 + run-duration: 1.253µs + changes: + pendingChange: + target-name: pendingChange + description: description placeholder + allowed: true diff --git a/schemas/layer-4.cue b/schemas/layer-4.cue index ffea275..fc5617f 100644 --- a/schemas/layer-4.cue +++ b/schemas/layer-4.cue @@ -1,37 +1,93 @@ package schemas #Layer4: { - evaluations: [#ControlEvaluation, ...#ControlEvaluation] + evaluations: [#ControlEvaluation, ...#ControlEvaluation] } -// Types - +// ControlEvaluation is a struct that contains all assessment results, organized by name. #ControlEvaluation: { - name: string - "control-id": string - result: #Result - message: string - "documentation-url"?: =~"^https?://[^\\s]+$" - "corrupted-state"?: bool - "assessment-results"?: [...#AssessmentResult] + // Name is the name of the control being evaluated + name: string + // ControlID is the unique identifier for the control being evaluated + "control-id": string + // Result is the overall result of the control evaluation + result: #Result + // Message is the human-readable result of the final assessment to run in this evaluation + message: string + // CorruptedState is true if the control evaluation was interrupted and changes were not reverted + "corrupted-state": bool + // RemediationGuide is the URL to the documentation for this evaluation + "remediation-guide": string + // Assessments is a map of pointers to Assessment objects to establish idempotency + assessments: [...#Assessment] +} + +// Assessment is a struct that contains the results of a single step within a ControlEvaluation. +#Assessment: { + // RequirementID is the unique identifier for the requirement being tested + "requirement-id": string @go(RequirementId) + // Applicability is a slice of identifier strings to determine when this test is applicable + applicability: [...string] + // Description is a human-readable description of the test + description: string + // Result is true if the test passed + result: #Result + // Message is the human-readable result of the test + message: string + // Methods is a slice of assessment methods that were executed during the test + methods: [...#AssessmentMethod] + // MethodsExecuted is the number of assessment methods that were executed during the test + "methods-executed"?: int @go(MethodExecuted) + // RunDuration is the time it took to run the test + "run-duration"?: string @go(RunDuration) + // Value is the object that was returned during the test + value?: _ + // Changes is a slice of changes that were made during the test + changes?: [string]: #Change } -#AssessmentResult: { - result: #Result - name: string - description: string - message: string - "function-address": string - change?: #Change - value?: _ +// AssessmentMethod describes the method used to assess the layer 2 control requirement referenced by requirementID. +#AssessmentMethod: { + // Id is the unique identifier of the assessment method being executed. + id: string + // Name is the name of the method used to assess the requirement. + name: string + // Description is a detailed explanation of the method. + description: string + // Run is a boolean indicating whether the method was run or not. When run is true, result is expected to be present. + run: bool + // Remediation guide is a URL to remediation guidance associated with the control's assessment requirement and this specific assessment method. + "remediation-guide"?: #URL @go(RemediationGuide) + // URL to documentation that describes how the assessment method evaluates the control requirement. + documentation?: #URL + // Result is the status or outcome of an assessed method present when run us true. + result?: #Result @go(Result,optional=nillable) + // Address or location of the code providing logic for the assessment + executor?: string } -#Result: "Passed" | "Failed" | "Needs Review" +// Result is the outcome of an assessment method when it is executed. +#Result: "Not Run" | "Passed" | "Failed" | "Needs Review" | "Not Applicable" | "Unknown" +// Change is a struct that contains the data and functions associated with a single change to a target resource. #Change: { - "target-name": string - applied: bool - reverted: bool - error?: string - "target-object"?: _ -} \ No newline at end of file + // TargetName is the name or ID of the resource or configuration that is to be changed + "target-name": string @go(TagertName) + // Description is a human-readable description of the change + description: string + // The applyFunc and revertFunc fields are function types and are not directly + // representable in a CUE schema. + // TargetObject is supplemental data describing the object that was changed + "target-object"?: _ @go(TargetName) + // Applied is true if the change was successfully applied at least once + applied?: bool + // Reverted is true if the change was successfully reverted and not applied again + reverted?: bool + // Error is used if any error occurred during the change + error?: _ + // Allowed may be disabled to prevent the change from being applied + allowed?: bool +} + +// URL describes a specific subset of URLs of interest to the framework +#URL: =~"^https?://[^\\s]+$" From 815fef46756028c74f06c0421f58036e8f25f983 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Fri, 8 Aug 2025 18:30:38 -0400 Subject: [PATCH 03/11] chore: add yaml struct tags to AssessmentMethod Signed-off-by: Jennifer Power --- layer4/assessment_method.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/layer4/assessment_method.go b/layer4/assessment_method.go index 86b9262..ee8a0c0 100644 --- a/layer4/assessment_method.go +++ b/layer4/assessment_method.go @@ -9,25 +9,25 @@ import ( // AssessmentMethod describes the method used to assess the layer 2 control requirement referenced by requirementID. type AssessmentMethod struct { // Id is the unique identifier of the assessment method being executed - Id string `json:"id"` + Id string `json:"id" yaml:"id"` // Name is the name of the method used to assess the requirement. - Name string `json:"name"` + Name string `json:"name" yaml:"name"` // Description is a detailed explanation of the method. - Description string `json:"description"` + Description string `json:"description" yaml:"description"` // Run is a boolean indicating whether the method was run or not. When run is true, result is expected to be present. - Run bool `json:"run"` + Run bool `json:"run" yaml:"run"` // Remediation guide is a URL to remediation guidance associated with the control's assessment requirement and this specific assessment method. - RemediationGuide URL `json:"remediation-guide,omitempty"` + RemediationGuide URL `json:"remediation-guide,omitempty" yaml:"remediation-guide,omitempty"` // URL to documentation that describes how the assessment method evaluates the control requirement. - Documentation URL `json:"documentation,omitempty"` + Documentation URL `json:"documentation,omitempty" yaml:"documentation,omitempty"` // Result is the status or outcome of an assessed method present when run us true. - Result *Result `json:"result,omitempty"` + Result *Result `json:"result,omitempty" yaml:"result,omitempty"` // Executor is a function type that inspects the provided targetData and returns a Result with a message. // The message may be an error string or other descriptive text. Executor MethodExecutor From 2b7beee73f4caad4c72b051d31cd569e9011be2f Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Fri, 8 Aug 2025 18:47:15 -0400 Subject: [PATCH 04/11] fix: address inconsistencies in comments and type names Signed-off-by: Jennifer Power --- layer4/assessment.go | 2 +- layer4/control_evaluation.go | 2 +- schemas/layer-4.cue | 12 +++++------- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/layer4/assessment.go b/layer4/assessment.go index 743a296..87f5502 100644 --- a/layer4/assessment.go +++ b/layer4/assessment.go @@ -26,7 +26,7 @@ type Assessment struct { RunDuration string `yaml:"run-duration,omitempty"` // Value is the object that was returned during the test Value interface{} `yaml:"value,omitempty"` - // Changes is a slice of changes that were made during the test + // Changes is a map of changes that were made during the test Changes map[string]*Change `yaml:"changes,omitempty"` } diff --git a/layer4/control_evaluation.go b/layer4/control_evaluation.go index abb8c50..edeffab 100644 --- a/layer4/control_evaluation.go +++ b/layer4/control_evaluation.go @@ -21,7 +21,7 @@ type ControlEvaluation struct { CorruptedState bool `yaml:"corrupted-state"` // RemediationGuide is the URL to the documentation for this evaluation RemediationGuide string `yaml:"remediation-guide"` - // Assessments is a map of pointers to Assessment objects to establish idempotency + // Assessments is a slice of pointers to Assessment objects to establish idempotency Assessments []*Assessment `yaml:"assessments"` } diff --git a/schemas/layer-4.cue b/schemas/layer-4.cue index fc5617f..7c2cd37 100644 --- a/schemas/layer-4.cue +++ b/schemas/layer-4.cue @@ -18,7 +18,7 @@ package schemas "corrupted-state": bool // RemediationGuide is the URL to the documentation for this evaluation "remediation-guide": string - // Assessments is a map of pointers to Assessment objects to establish idempotency + // Assessments is a slice of pointers to Assessment objects to establish idempotency assessments: [...#Assessment] } @@ -37,12 +37,12 @@ package schemas // Methods is a slice of assessment methods that were executed during the test methods: [...#AssessmentMethod] // MethodsExecuted is the number of assessment methods that were executed during the test - "methods-executed"?: int @go(MethodExecuted) + "methods-executed"?: int @go(MethodsExecuted) // RunDuration is the time it took to run the test "run-duration"?: string @go(RunDuration) // Value is the object that was returned during the test value?: _ - // Changes is a slice of changes that were made during the test + // Changes is a map of changes that were made during the test changes?: [string]: #Change } @@ -72,13 +72,11 @@ package schemas // Change is a struct that contains the data and functions associated with a single change to a target resource. #Change: { // TargetName is the name or ID of the resource or configuration that is to be changed - "target-name": string @go(TagertName) + "target-name": string @go(TargetName) // Description is a human-readable description of the change description: string - // The applyFunc and revertFunc fields are function types and are not directly - // representable in a CUE schema. // TargetObject is supplemental data describing the object that was changed - "target-object"?: _ @go(TargetName) + "target-object"?: _ @go(TargetObject) // Applied is true if the change was successfully applied at least once applied?: bool // Reverted is true if the change was successfully reverted and not applied again From affd0760cdc9f6437194fff57f4e59f53350832a Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Fri, 8 Aug 2025 19:06:52 -0400 Subject: [PATCH 05/11] fix: refines comments in layer4 Assessment types Signed-off-by: Jennifer Power --- layer4/assessment.go | 2 +- layer4/assessment_method.go | 2 +- schemas/layer-4.cue | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/layer4/assessment.go b/layer4/assessment.go index 87f5502..6d6982a 100644 --- a/layer4/assessment.go +++ b/layer4/assessment.go @@ -14,7 +14,7 @@ type Assessment struct { Applicability []string `yaml:"applicability"` // Description is a human-readable description of the test Description string `yaml:"description"` - // Result is true if the test passed + // Result is the overall result of the assessment Result Result `yaml:"result"` // Message is the human-readable result of the test Message string `yaml:"message"` diff --git a/layer4/assessment_method.go b/layer4/assessment_method.go index ee8a0c0..a0f1523 100644 --- a/layer4/assessment_method.go +++ b/layer4/assessment_method.go @@ -26,7 +26,7 @@ type AssessmentMethod struct { // URL to documentation that describes how the assessment method evaluates the control requirement. Documentation URL `json:"documentation,omitempty" yaml:"documentation,omitempty"` - // Result is the status or outcome of an assessed method present when run us true. + // Result is the status or outcome of an assessed method present. This field is present when Run is true. Result *Result `json:"result,omitempty" yaml:"result,omitempty"` // Executor is a function type that inspects the provided targetData and returns a Result with a message. // The message may be an error string or other descriptive text. diff --git a/schemas/layer-4.cue b/schemas/layer-4.cue index 7c2cd37..7e9c20d 100644 --- a/schemas/layer-4.cue +++ b/schemas/layer-4.cue @@ -30,7 +30,7 @@ package schemas applicability: [...string] // Description is a human-readable description of the test description: string - // Result is true if the test passed + // Result is the overall result of the assessment result: #Result // Message is the human-readable result of the test message: string @@ -60,7 +60,7 @@ package schemas "remediation-guide"?: #URL @go(RemediationGuide) // URL to documentation that describes how the assessment method evaluates the control requirement. documentation?: #URL - // Result is the status or outcome of an assessed method present when run us true. + // Result is the status or outcome of an assessed method present. This field is present when Run is true. result?: #Result @go(Result,optional=nillable) // Address or location of the code providing logic for the assessment executor?: string From 34334bc8d8394f231cbce7025a985d2279241b06 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Fri, 8 Aug 2025 22:00:46 -0400 Subject: [PATCH 06/11] fix: refines comments and inconsistencies in layer4 schema Some of the comments around AssessementMethod require clarification around intended usage Signed-off-by: Jennifer Power --- layer4/assessment_method.go | 7 ++++--- schemas/layer-4.cue | 10 +++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/layer4/assessment_method.go b/layer4/assessment_method.go index a0f1523..96cb538 100644 --- a/layer4/assessment_method.go +++ b/layer4/assessment_method.go @@ -6,12 +6,12 @@ import ( "runtime" ) -// AssessmentMethod describes the method used to assess the layer 2 control requirement referenced by requirementID. +// AssessmentMethod defines a specific procedure for assessing a Layer 2 control requirement. type AssessmentMethod struct { // Id is the unique identifier of the assessment method being executed Id string `json:"id" yaml:"id"` - // Name is the name of the method used to assess the requirement. + // Name is the human-readable name of the method. Name string `json:"name" yaml:"name"` // Description is a detailed explanation of the method. @@ -26,7 +26,8 @@ type AssessmentMethod struct { // URL to documentation that describes how the assessment method evaluates the control requirement. Documentation URL `json:"documentation,omitempty" yaml:"documentation,omitempty"` - // Result is the status or outcome of an assessed method present. This field is present when Run is true. + // Result is the outcome of the assessment method. + // This field must be present if Run is true. Result *Result `json:"result,omitempty" yaml:"result,omitempty"` // Executor is a function type that inspects the provided targetData and returns a Result with a message. // The message may be an error string or other descriptive text. diff --git a/schemas/layer-4.cue b/schemas/layer-4.cue index 7e9c20d..5e6fff9 100644 --- a/schemas/layer-4.cue +++ b/schemas/layer-4.cue @@ -15,9 +15,9 @@ package schemas // Message is the human-readable result of the final assessment to run in this evaluation message: string // CorruptedState is true if the control evaluation was interrupted and changes were not reverted - "corrupted-state": bool + "corrupted-state"?: bool // RemediationGuide is the URL to the documentation for this evaluation - "remediation-guide": string + "remediation-guide"?: string // Assessments is a slice of pointers to Assessment objects to establish idempotency assessments: [...#Assessment] } @@ -46,11 +46,11 @@ package schemas changes?: [string]: #Change } -// AssessmentMethod describes the method used to assess the layer 2 control requirement referenced by requirementID. +// AssessmentMethod describes a specific procedure for evaluating a Layer 2 control requirement. #AssessmentMethod: { // Id is the unique identifier of the assessment method being executed. id: string - // Name is the name of the method used to assess the requirement. + // Name is the human-readable name of the method. name: string // Description is a detailed explanation of the method. description: string @@ -62,7 +62,7 @@ package schemas documentation?: #URL // Result is the status or outcome of an assessed method present. This field is present when Run is true. result?: #Result @go(Result,optional=nillable) - // Address or location of the code providing logic for the assessment + // Executor is a string identifier for the address or location for the specific assessment function to be used. executor?: string } From f65f6ec1350cafaea8b16413ad68b752d7d71783 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Fri, 8 Aug 2025 22:02:30 -0400 Subject: [PATCH 07/11] chore: updates testdata to reflect a concrete example To better show the impact of the change, the testdata was updated to an example from the pvtr baseline validator Signed-off-by: Jennifer Power --- layer4/testdata/good-evaluation.yml | 28 ---------------------- layer4/testdata/good-osps-evaluation.yml | 30 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 28 deletions(-) delete mode 100644 layer4/testdata/good-evaluation.yml create mode 100644 layer4/testdata/good-osps-evaluation.yml diff --git a/layer4/testdata/good-evaluation.yml b/layer4/testdata/good-evaluation.yml deleted file mode 100644 index 10b33f0..0000000 --- a/layer4/testdata/good-evaluation.yml +++ /dev/null @@ -1,28 +0,0 @@ -evaluations: - - name: "Test Evaluation" - control-id: "CTRL-1" - result: Passed - message: "The control is an example that passed." - corrupted-state: false - remediation-guide: "https://example-remediation.dev" - assessments: - - requirement-id: passingAssessment() - applicability: - - test-applicability - description: passing assessment - result: Passed - message: "The requirement has passed." - methods: - - id: "method-1" - name: "The first method" - description: "The is the only method executed for this requirement" - run: true - result: Passed - executor: github.com/revanite-io/gemara/layer4.init.func5 - methods-executed: 1 - run-duration: 1.253µs - changes: - pendingChange: - target-name: pendingChange - description: description placeholder - allowed: true diff --git a/layer4/testdata/good-osps-evaluation.yml b/layer4/testdata/good-osps-evaluation.yml new file mode 100644 index 0000000..ec4e17f --- /dev/null +++ b/layer4/testdata/good-osps-evaluation.yml @@ -0,0 +1,30 @@ +evaluations: + - control-id: "OSPS-BR-06" + name: "Produce all released software assets with signatures and hashes" + message: "Security insights required for this assessment, but file not found" + result: Needs Review + assessments: + - requirement-id: "OSPS-BR-06.01" + description: "Determine if the project publishes SLSA provenance attestations" + result: Needs Review + message: "Security insights required for this assessment, but file not found" + # This show an example from + # https://github.com/revanite-io/pvtr-github-repo/blob/main/evaluation_plans/osps/build_release/evaluations.go#L174-L178 + # where this support a multistep assessment + methods: + - id: has-made-release + name: "Review the project releases" + description: "Determine if the project has any releases that require review of release artifacts" + run: true + executor: github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasMadeReleases + result: Passed + - id: has-security-insights-file + name: "Check the project for security-insights.yml" + description: "Determine if the project has a security-insights.yml in expected locations" + executor: github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasSecurityInsightsFile + run: true + result: Needs Review + - id: insights-has-attestation + name: "Check the project for SLSA provenance attestation" + description: "Check security-insights.yml attestation for a SLSA provenance attestation" + run: false \ No newline at end of file From b31bb6562cd99a7d47bad382810e033b40f21f50 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Sat, 9 Aug 2025 21:14:53 -0400 Subject: [PATCH 08/11] feat: adds constraints on run logic on Assessment Method This commit refactors the AssessmentMethod schema to use CUE's disjunction feature to formally define conditional logic for the run and result fields. The schema was allowing a state where run was true but a result was missing. Signed-off-by: Jennifer Power --- schemas/layer-4.cue | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/schemas/layer-4.cue b/schemas/layer-4.cue index 5e6fff9..c6bdc71 100644 --- a/schemas/layer-4.cue +++ b/schemas/layer-4.cue @@ -60,14 +60,24 @@ package schemas "remediation-guide"?: #URL @go(RemediationGuide) // URL to documentation that describes how the assessment method evaluates the control requirement. documentation?: #URL - // Result is the status or outcome of an assessed method present. This field is present when Run is true. - result?: #Result @go(Result,optional=nillable) // Executor is a string identifier for the address or location for the specific assessment function to be used. executor?: string } +// Additional constraints on Assessment Method. +#AssessmentMethod: { + run: false + result?: ("Not Run" | *null) @go(Result,optional=nillable) +} | { + run: true + result!: #ResultWhenRun +} + +// Result is valid assessment outcomes before and after execution. +#Result: #ResultWhenRun | "Not Run" + // Result is the outcome of an assessment method when it is executed. -#Result: "Not Run" | "Passed" | "Failed" | "Needs Review" | "Not Applicable" | "Unknown" +#ResultWhenRun: "Passed" | "Failed" | "Needs Review" | "Not Applicable" | "Unknown" // Change is a struct that contains the data and functions associated with a single change to a target resource. #Change: { @@ -82,7 +92,7 @@ package schemas // Reverted is true if the change was successfully reverted and not applied again reverted?: bool // Error is used if any error occurred during the change - error?: _ + error?: string // Allowed may be disabled to prevent the change from being applied allowed?: bool } From f2c2beb4fdc66f6eae92d4f84df24b4cf4a6b197 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Mon, 11 Aug 2025 14:28:04 -0400 Subject: [PATCH 09/11] fix: apply suggestions for layer4 schema comments Co-authored-by: Travis Truman Signed-off-by: Jennifer Power Signed-off-by: Jennifer Power --- schemas/layer-4.cue | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/schemas/layer-4.cue b/schemas/layer-4.cue index c6bdc71..9b002b2 100644 --- a/schemas/layer-4.cue +++ b/schemas/layer-4.cue @@ -4,21 +4,21 @@ package schemas evaluations: [#ControlEvaluation, ...#ControlEvaluation] } -// ControlEvaluation is a struct that contains all assessment results, organized by name. +// ControlEvaluation provides all assessment results for a single control #ControlEvaluation: { // Name is the name of the control being evaluated name: string - // ControlID is the unique identifier for the control being evaluated + // ControlID uniquely identifies the control "control-id": string - // Result is the overall result of the control evaluation + // Result communicates whether the evaluation has been run, and if so, the outcome(s) result: #Result - // Message is the human-readable result of the final assessment to run in this evaluation + // Message describes the result of the evaluation message: string // CorruptedState is true if the control evaluation was interrupted and changes were not reverted "corrupted-state"?: bool - // RemediationGuide is the URL to the documentation for this evaluation + // RemediationGuide provides a URL with guidance on how to remediate systems that fail control evaluation "remediation-guide"?: string - // Assessments is a slice of pointers to Assessment objects to establish idempotency + // Assessments represents the collection of results from evaluation of each control requirement assessments: [...#Assessment] } From 9d8a610c48b335b7f8dfd49bad5e80ceb92bab97 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Mon, 11 Aug 2025 14:51:04 -0400 Subject: [PATCH 10/11] chore: refines layer4 schema comments This change normalizes descriptions between different types defined in cue for layer4 and removes Go type language from the cue schema comments Signed-off-by: Jennifer Power --- schemas/layer-4.cue | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/schemas/layer-4.cue b/schemas/layer-4.cue index 9b002b2..e8f1978 100644 --- a/schemas/layer-4.cue +++ b/schemas/layer-4.cue @@ -22,45 +22,45 @@ package schemas assessments: [...#Assessment] } -// Assessment is a struct that contains the results of a single step within a ControlEvaluation. +// Assessment provides all assessment results from evaluation of a single control requirement #Assessment: { - // RequirementID is the unique identifier for the requirement being tested + // RequirementID uniquely identifies the requirement being tested "requirement-id": string @go(RequirementId) - // Applicability is a slice of identifier strings to determine when this test is applicable + // Applicability provides identifier strings to determine when this assessment is applicable applicability: [...string] - // Description is a human-readable description of the test + // Description provides a detailed explanation of the assessment description: string - // Result is the overall result of the assessment + // Result communicates whether the assessment has been run, and if so, the outcome(s) result: #Result - // Message is the human-readable result of the test + // Message describes the result of the assessment message: string - // Methods is a slice of assessment methods that were executed during the test + // Methods defines the assessment methods associated with the assessment methods: [...#AssessmentMethod] - // MethodsExecuted is the number of assessment methods that were executed during the test + // MethodsExecuted is the number of assessment methods that were executed during the assessment "methods-executed"?: int @go(MethodsExecuted) - // RunDuration is the time it took to run the test + // RunDuration is the time it took to run the assessment "run-duration"?: string @go(RunDuration) - // Value is the object that was returned during the test + // Value is the object that was returned during the assessment value?: _ - // Changes is a map of changes that were made during the test + // Changes describes changes that were made during the assessment changes?: [string]: #Change } // AssessmentMethod describes a specific procedure for evaluating a Layer 2 control requirement. #AssessmentMethod: { - // Id is the unique identifier of the assessment method being executed. + // Id uniquely identifies the assessment method being executed id: string - // Name is the human-readable name of the method. + // Name provides a summary of the method name: string - // Description is a detailed explanation of the method. + // Description provides a detailed explanation of the method description: string - // Run is a boolean indicating whether the method was run or not. When run is true, result is expected to be present. + // Run is a boolean indicating whether the method was run or not. When run is true, result is expected to be present run: bool - // Remediation guide is a URL to remediation guidance associated with the control's assessment requirement and this specific assessment method. + // RemediationGuide provides a URL with remediation guidance associated with the control's assessment requirement and this specific assessment method "remediation-guide"?: #URL @go(RemediationGuide) - // URL to documentation that describes how the assessment method evaluates the control requirement. + // Documentation provides a URL to documentation that describes how the assessment method evaluates the control requirement documentation?: #URL - // Executor is a string identifier for the address or location for the specific assessment function to be used. + // Executor provides the address or location for the specific assessment logic used executor?: string } @@ -73,10 +73,10 @@ package schemas result!: #ResultWhenRun } -// Result is valid assessment outcomes before and after execution. +// Result describes valid assessment outcomes before and after execution. #Result: #ResultWhenRun | "Not Run" -// Result is the outcome of an assessment method when it is executed. +// Result describes the outcome(s) of an assessment method when it is executed. #ResultWhenRun: "Passed" | "Failed" | "Needs Review" | "Not Applicable" | "Unknown" // Change is a struct that contains the data and functions associated with a single change to a target resource. From d431b80c5a8d0469f53a72e820632b9b28cc5e9c Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Mon, 18 Aug 2025 20:33:19 -0400 Subject: [PATCH 11/11] refactor: updates AssessmentMethod to AssessmentProcedure This change refactors the work in the previous commits to allow for top-level metadata to support different types of assessment procedures and move the AssessmentSteps under AssessmentProcedure struct. The concept of "method" is updated to describe the procedure categorically for tools that run automated procedures. Signed-off-by: Jennifer Power --- layer4/assessment.go | 44 +++---- layer4/assessment_method.go | 69 ----------- layer4/assessment_procedure.go | 114 ++++++++++++++++++ layer4/assessment_procedure_test.go | 122 +++++++++++++++++++ layer4/assessment_test.go | 142 +++++------------------ layer4/control_evaluation.go | 8 +- layer4/control_evaluation_test.go | 10 +- layer4/method.go | 39 +++++++ layer4/test-data.go | 93 +++++++++------ layer4/testdata/good-osps-evaluation.yml | 36 +++--- schemas/layer-4.cue | 41 ++++--- 11 files changed, 433 insertions(+), 285 deletions(-) delete mode 100644 layer4/assessment_method.go create mode 100644 layer4/assessment_procedure.go create mode 100644 layer4/assessment_procedure_test.go create mode 100644 layer4/method.go diff --git a/layer4/assessment.go b/layer4/assessment.go index 6d6982a..252c685 100644 --- a/layer4/assessment.go +++ b/layer4/assessment.go @@ -6,7 +6,7 @@ import ( "time" ) -// Assessment is a struct that contains the results of a single method within a ControlEvaluation. +// Assessment is a struct that contains the results of a single step within a ControlEvaluation. type Assessment struct { // RequirementID is the unique identifier for the requirement being tested RequirementId string `yaml:"requirement-id"` @@ -18,10 +18,8 @@ type Assessment struct { Result Result `yaml:"result"` // Message is the human-readable result of the test Message string `yaml:"message"` - // Methods is a slice of assessment methods that were executed during the test - Methods []*AssessmentMethod `yaml:"methods"` - // MethodsExecuted is the number of assessment methods that were executed during the test - MethodsExecuted int `yaml:"methods-executed,omitempty"` + // Procedures is a slice of assessment procedures that were executed during the test + Procedures []*AssessmentProcedure `yaml:"procedures"` // RunDuration is the time it took to run the test RunDuration string `yaml:"run-duration,omitempty"` // Value is the object that was returned during the test @@ -31,32 +29,24 @@ type Assessment struct { } // NewAssessment creates a new Assessment object and returns a pointer to it. -func NewAssessment(requirementId string, description string, applicability []string, methods []*AssessmentMethod) (*Assessment, error) { +func NewAssessment(requirementId string, description string, applicability []string, procedures []*AssessmentProcedure) (*Assessment, error) { a := &Assessment{ RequirementId: requirementId, Description: description, Applicability: applicability, Result: NotRun, - Methods: methods, + Procedures: procedures, } err := a.precheck() return a, err } -// AddMethod queues a new method in the Assessment -func (a *Assessment) AddMethod(method AssessmentMethod) { - a.Methods = append(a.Methods, &method) +// AddProcedure queues a new procedure in the Assessment +func (a *Assessment) AddProcedure(procedure AssessmentProcedure) { + a.Procedures = append(a.Procedures, &procedure) } -func (a *Assessment) runMethod(targetData interface{}, method *AssessmentMethod) Result { - a.MethodsExecuted++ - result, message := method.RunMethod(targetData, a.Changes) - a.Result = UpdateAggregateResult(a.Result, result) - a.Message = message - return result -} - -// Run will execute all steps, halting if any method does not return layer4.Passed. +// Run executes all assessment procedures using the test method. func (a *Assessment) Run(targetData interface{}, changesAllowed bool) Result { if a.Result != NotRun { return a.Result @@ -73,11 +63,15 @@ func (a *Assessment) Run(targetData interface{}, changesAllowed bool) Result { change.Allow() } } - for _, method := range a.Methods { - if a.runMethod(targetData, method) == Failed { - return Failed + + for _, procedure := range a.Procedures { + if procedure.Method == TestMethod { + result := procedure.RunProcedure(targetData, a.Changes) + a.Result = UpdateAggregateResult(a.Result, result) + a.Message = procedure.Message } } + a.RunDuration = time.Since(startTime).String() return a.Result } @@ -118,10 +112,10 @@ func (a *Assessment) RevertChanges() (corrupted bool) { // precheck verifies that the assessment has all the required fields. // It returns an error if the assessment is not valid. func (a *Assessment) precheck() error { - if a.RequirementId == "" || a.Description == "" || a.Applicability == nil || a.Methods == nil || len(a.Applicability) == 0 || len(a.Methods) == 0 { + if a.RequirementId == "" || a.Description == "" || a.Applicability == nil || a.Procedures == nil || len(a.Applicability) == 0 || len(a.Procedures) == 0 { message := fmt.Sprintf( - "expected all Assessment fields to have a value, but got: requirementId=len(%v), description=len=(%v), applicability=len(%v), methods=len(%v)", - len(a.RequirementId), len(a.Description), len(a.Applicability), len(a.Methods), + "expected all Assessment fields to have a value, but got: requirementId=len(%v), description=len=(%v), applicability=len(%v), procedures=len(%v)", + len(a.RequirementId), len(a.Description), len(a.Applicability), len(a.Procedures), ) a.Result = Unknown a.Message = message diff --git a/layer4/assessment_method.go b/layer4/assessment_method.go deleted file mode 100644 index 96cb538..0000000 --- a/layer4/assessment_method.go +++ /dev/null @@ -1,69 +0,0 @@ -package layer4 - -import ( - "encoding/json" - "reflect" - "runtime" -) - -// AssessmentMethod defines a specific procedure for assessing a Layer 2 control requirement. -type AssessmentMethod struct { - // Id is the unique identifier of the assessment method being executed - Id string `json:"id" yaml:"id"` - - // Name is the human-readable name of the method. - Name string `json:"name" yaml:"name"` - - // Description is a detailed explanation of the method. - Description string `json:"description" yaml:"description"` - - // Run is a boolean indicating whether the method was run or not. When run is true, result is expected to be present. - Run bool `json:"run" yaml:"run"` - - // Remediation guide is a URL to remediation guidance associated with the control's assessment requirement and this specific assessment method. - RemediationGuide URL `json:"remediation-guide,omitempty" yaml:"remediation-guide,omitempty"` - - // URL to documentation that describes how the assessment method evaluates the control requirement. - Documentation URL `json:"documentation,omitempty" yaml:"documentation,omitempty"` - - // Result is the outcome of the assessment method. - // This field must be present if Run is true. - Result *Result `json:"result,omitempty" yaml:"result,omitempty"` - // Executor is a function type that inspects the provided targetData and returns a Result with a message. - // The message may be an error string or other descriptive text. - Executor MethodExecutor -} - -// URL describes a specific subset of URLs of interest to the framework -type URL string - -// MethodExecutor is a function type that inspects the provided payload and returns the result of the assessment. -// The payload is the data/evidence that the assessment will be run against. -type MethodExecutor func(payload interface{}, c map[string]*Change) (Result, string) - -// RunMethod executes the assessment method using the provided payload and changes. -// It returns the result of the assessment and any error encountered during execution. -// The payload is the data/evidence that the assessment will be run against. -func (a *AssessmentMethod) RunMethod(payload interface{}, changes map[string]*Change) (Result, string) { - result, message := a.Executor(payload, changes) - a.Result = &result - a.Run = true - return result, message -} - -func (e MethodExecutor) String() string { - // Get the function pointer correctly - fn := runtime.FuncForPC(reflect.ValueOf(e).Pointer()) - if fn == nil { - return "" - } - return fn.Name() -} - -func (e MethodExecutor) MarshalJSON() ([]byte, error) { - return json.Marshal(e.String()) -} - -func (e MethodExecutor) MarshalYAML() (interface{}, error) { - return e.String(), nil -} diff --git a/layer4/assessment_procedure.go b/layer4/assessment_procedure.go new file mode 100644 index 0000000..30d77c4 --- /dev/null +++ b/layer4/assessment_procedure.go @@ -0,0 +1,114 @@ +package layer4 + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "runtime" +) + +// AssessmentProcedure is a specific outline defining how to assess a Layer 2 control requirement. +type AssessmentProcedure struct { + // Id is the unique identifier of the assessment procedure being executed + Id string `json:"id" yaml:"id"` + // Name is the human-readable name of the procedure. + Name string `json:"name" yaml:"name"` + // Description is a detailed explanation of the procedure. + Description string `json:"description" yaml:"description"` + // Method describe the high-level method used to determine the results of the procedure + Method Method `yaml:"method"` + // Remediation guide is a URL to remediation guidance associated with the control's assessment requirement and this specific assessment procedure. + RemediationGuide string `json:"remediation-guide,omitempty" yaml:"remediation-guide,omitempty"` + // URL to documentation that describes how the assessment procedure evaluates the control requirement. + Documentation string `json:"documentation,omitempty" yaml:"documentation,omitempty"` + // Run is a boolean indicating whether the procedure was run or not. When run is true, result is expected to be present. + Run bool `json:"run" yaml:"run"` + // Message is the human-readable result of the procedure + Message string `json:"message,omitempty" yaml:"message,omitempty"` + // Result is the outcome of the assessment procedure. + // This field must be present if Run is true. + Result Result `json:"result,omitempty" yaml:"result,omitempty"` + // StepsExecuted is the number of steps that were executed during the assessment execution + StepsExecuted int `json:"-" yaml:"-"` + // Steps define logical steps to inspect the provided targetData and returns a Result with a message. + // The message may be an error string or other descriptive text. + Steps []AssessmentStep +} + +// NewProcedure creates a new AssessmentProcedure object and returns a pointer to it. +func NewProcedure(id, name, description string, steps []AssessmentStep) (*AssessmentProcedure, error) { + a := &AssessmentProcedure{ + Id: id, + Name: name, + Description: description, + Result: NotRun, + Steps: steps, + } + err := a.precheck() + return a, err +} + +// AssessmentStep is a function type that inspects the provided targetData and returns a Result with a message. +// The message may be an error string or other descriptive text. +type AssessmentStep func(payload interface{}, c map[string]*Change) (Result, string) + +func (as AssessmentStep) String() string { + // Get the function pointer correctly + fn := runtime.FuncForPC(reflect.ValueOf(as).Pointer()) + if fn == nil { + return "" + } + return fn.Name() +} + +func (as AssessmentStep) MarshalJSON() ([]byte, error) { + return json.Marshal(as.String()) +} + +func (as AssessmentStep) MarshalYAML() (interface{}, error) { + return as.String(), nil +} + +// AddStep queues a new step in the Assessment Procedure +func (a *AssessmentProcedure) AddStep(step AssessmentStep) { + a.Steps = append(a.Steps, step) +} + +// RunProcedure executes all assessment steps, halting if any assessment does not return layer4.Passed. +func (a *AssessmentProcedure) RunProcedure(targetData interface{}, changes map[string]*Change) Result { + a.Run = true + + err := a.precheck() + if err != nil { + a.Result = Unknown + return a.Result + } + for _, steps := range a.Steps { + if a.runStep(targetData, changes, steps) == Failed { + return Failed + } + } + return a.Result +} + +func (a *AssessmentProcedure) runStep(targetData interface{}, changes map[string]*Change, step AssessmentStep) Result { + a.StepsExecuted++ + result, message := step(targetData, changes) + a.Result = UpdateAggregateResult(a.Result, result) + a.Message = message + return result +} + +// precheck verifies that the assessment procedure has step fields. +// It returns an error if the assessment is not valid. +func (a *AssessmentProcedure) precheck() error { + if len(a.Steps) == 0 { + message := fmt.Sprintf( + "expected all Assessment Procedure fields steps=len(%v)", + len(a.Steps), + ) + return errors.New(message) + } + return nil +} diff --git a/layer4/assessment_procedure_test.go b/layer4/assessment_procedure_test.go new file mode 100644 index 0000000..d3e3da1 --- /dev/null +++ b/layer4/assessment_procedure_test.go @@ -0,0 +1,122 @@ +package layer4 + +import ( + "testing" +) + +func getProcedures() []struct { + testName string + procedure AssessmentProcedure + numberOfSteps int + numberOfStepsToRun int + expectedResult Result +} { + return []struct { + testName string + procedure AssessmentProcedure + numberOfSteps int + numberOfStepsToRun int + expectedResult Result + }{ + { + testName: "Assessment with no steps", + procedure: AssessmentProcedure{}, + expectedResult: Unknown, + }, + { + testName: "Procedure with one step", + procedure: passingProcedure, + numberOfSteps: 1, + numberOfStepsToRun: 1, + expectedResult: Passed, + }, + { + testName: "Procedure and two steps", + procedure: failingProcedure, + numberOfSteps: 2, + numberOfStepsToRun: 1, + expectedResult: Failed, + }, + { + testName: "Procedure three steps", + procedure: needsReviewProcedure, + numberOfSteps: 3, + numberOfStepsToRun: 3, + expectedResult: NeedsReview, + }, + } +} + +// TestRun ensures that Run executes all steps, halting if any Procedure does not return Passed +func TestRunProcedure(t *testing.T) { + for _, data := range getProcedures() { + t.Run(data.testName, func(t *testing.T) { + a := data.procedure // copy the procedure to prevent duplicate executions in the next test + result := a.RunProcedure(nil, nil) + if result != a.Result { + t.Errorf("expected match between Run return value (%s) and assessment Result value (%s)", result, data.expectedResult) + } + if a.StepsExecuted != data.numberOfStepsToRun { + t.Errorf("expected to run %d tests, got %d", data.numberOfStepsToRun, a.StepsExecuted) + } + }) + } +} + +// TestNewStep ensures that NewStep queues a new step in the Assessment +func TestAddStep(t *testing.T) { + for _, test := range getProcedures() { + t.Run(test.testName, func(t *testing.T) { + if len(test.procedure.Steps) != test.numberOfSteps { + t.Errorf("Bad test data: expected to start with %d, got %d", test.numberOfSteps, len(test.procedure.Steps)) + } + test.procedure.AddStep(passingAssessmentStep) + if len(test.procedure.Steps) != test.numberOfSteps+1 { + t.Errorf("expected %d, got %d", test.numberOfSteps, len(test.procedure.Steps)) + } + }) + } +} + +// TestRunStep ensures that runStep runs the step and updates the Assessment Procedure +func TestRunStep(t *testing.T) { + stepsTestData := []struct { + testName string + step AssessmentStep + result Result + }{ + { + testName: "Failing step", + step: failingAssessmentStep, + result: Failed, + }, + { + testName: "Passing step", + step: passingAssessmentStep, + result: Passed, + }, + { + testName: "Needs review step", + step: needsReviewAssessmentStep, + result: NeedsReview, + }, + { + testName: "Unknown step", + step: unknownAssessmentStep, + result: Unknown, + }, + } + for _, test := range stepsTestData { + t.Run(test.testName, func(t *testing.T) { + anyOldProcedure := AssessmentProcedure{} + + result := anyOldProcedure.runStep(nil, nil, test.step) + if result != test.result { + t.Errorf("expected %s, got %s", test.result, result) + } + if anyOldProcedure.Result != test.result { + t.Errorf("expected %s, got %s", test.result, anyOldProcedure.Result) + } + }) + } +} diff --git a/layer4/assessment_test.go b/layer4/assessment_test.go index c3a5f35..ca82f99 100644 --- a/layer4/assessment_test.go +++ b/layer4/assessment_test.go @@ -4,135 +4,51 @@ import ( "testing" ) -func getAssessmentsTestData() []struct { - testName string - assessment Assessment - numberOfSteps int - numberOfStepsToRun int - expectedResult Result +func getAssessments() []struct { + testName string + assessment Assessment + expectedResult Result } { return []struct { - testName string - assessment Assessment - numberOfSteps int - numberOfStepsToRun int - expectedResult Result + testName string + assessment Assessment + expectedResult Result }{ { testName: "Assessment with no steps", assessment: Assessment{}, }, { - testName: "Assessment with one method", - assessment: passingAssessment(), - numberOfSteps: 1, - numberOfStepsToRun: 1, - expectedResult: Passed, + testName: "Passing assessment", + assessment: passingAssessment(), + expectedResult: Passed, }, { - testName: "Assessment with two steps", - assessment: failingAssessment(), - numberOfSteps: 2, - numberOfStepsToRun: 1, - expectedResult: Failed, + testName: "Failing assessment", + assessment: failingAssessment(), + expectedResult: Failed, }, { - testName: "Assessment with three steps", - assessment: needsReviewAssessment(), - numberOfSteps: 3, - numberOfStepsToRun: 3, - expectedResult: NeedsReview, + testName: "Assessment needs review", + assessment: needsReviewAssessment(), + expectedResult: NeedsReview, }, { - testName: "Assessment with four steps", - assessment: badRevertPassingAssessment(), - numberOfSteps: 4, - numberOfStepsToRun: 4, - expectedResult: Passed, + testName: "Bad change revert", + assessment: badRevertPassingAssessment(), + expectedResult: Passed, }, } } -// TestNewStep ensures that NewStep queues a new method in the Assessment -func TestAddStep(t *testing.T) { - for _, test := range getAssessmentsTestData() { - t.Run(test.testName, func(t *testing.T) { - if len(test.assessment.Methods) != test.numberOfSteps { - t.Errorf("Bad test data: expected to start with %d, got %d", test.numberOfSteps, len(test.assessment.Methods)) - } - test.assessment.AddMethod(passingAssessmentMethod) - if len(test.assessment.Methods) != test.numberOfSteps+1 { - t.Errorf("expected %d, got %d", test.numberOfSteps, len(test.assessment.Methods)) - } - }) - } -} - -// TestRunStep ensures that runStep runs the method and updates the Assessment -func TestRunStep(t *testing.T) { - stepsTestData := []struct { - testName string - method AssessmentMethod - result Result - }{ - { - testName: "Failing method", - method: failingAssessmentMethod, - result: Failed, - }, - { - testName: "Passing method", - method: passingAssessmentMethod, - result: Passed, - }, - { - testName: "Needs review method", - method: needsReviewAssessmentMethod, - result: NeedsReview, - }, - { - testName: "Unknown method", - method: unknownAssessmentMethod, - result: Unknown, - }, - } - for _, test := range stepsTestData { - t.Run(test.testName, func(t *testing.T) { - anyOldAssessment := Assessment{} - result := anyOldAssessment.runMethod(nil, &test.method) - if result != test.result { - t.Errorf("expected %s, got %s", test.result, result) - } - if anyOldAssessment.Result != test.result { - t.Errorf("expected %s, got %s", test.result, anyOldAssessment.Result) - } - }) - } -} - -// TestRun ensures that Run executes all steps, halting if any method does not return Passed func TestRun(t *testing.T) { - for _, data := range getAssessmentsTestData() { - t.Run(data.testName, func(t *testing.T) { + for _, data := range getAssessments() { + t.Run(data.testName+"-no-changes", func(t *testing.T) { a := data.assessment // copy the assessment to prevent duplicate executions in the next test - result := a.Run(nil, true) + result := a.Run(nil, false) if result != a.Result { t.Errorf("expected match between Run return value (%s) and assessment Result value (%s)", result, data.expectedResult) } - if a.MethodsExecuted != data.numberOfStepsToRun { - t.Errorf("expected to run %d tests, got %d", data.numberOfStepsToRun, a.MethodsExecuted) - } - }) - } -} - -func TestRunB(t *testing.T) { - for _, data := range getAssessmentsTestData() { - t.Run(data.testName+"-no-changes", func(t *testing.T) { - data.assessment.Run(nil, false) - if data.assessment.MethodsExecuted != data.numberOfStepsToRun { - t.Errorf("expected to run %d tests, got %d", data.numberOfStepsToRun, data.assessment.MethodsExecuted) - } for _, change := range data.assessment.Changes { if change.Allowed { t.Errorf("expected all changes to be disallowed, but found an allowed change") @@ -234,7 +150,7 @@ func TestNewAssessment(t *testing.T) { requirementId string description string applicability []string - methods []*AssessmentMethod + procedures []*AssessmentProcedure expectedError bool }{ { @@ -242,7 +158,7 @@ func TestNewAssessment(t *testing.T) { requirementId: "", description: "test", applicability: []string{"test"}, - methods: []*AssessmentMethod{&passingAssessmentMethod}, + procedures: []*AssessmentProcedure{&passingProcedure}, expectedError: true, }, { @@ -250,7 +166,7 @@ func TestNewAssessment(t *testing.T) { requirementId: "test", description: "", applicability: []string{"test"}, - methods: []*AssessmentMethod{&passingAssessmentMethod}, + procedures: []*AssessmentProcedure{&passingProcedure}, expectedError: true, }, { @@ -258,7 +174,7 @@ func TestNewAssessment(t *testing.T) { requirementId: "test", description: "test", applicability: []string{}, - methods: []*AssessmentMethod{&passingAssessmentMethod}, + procedures: []*AssessmentProcedure{&passingProcedure}, expectedError: true, }, { @@ -266,7 +182,7 @@ func TestNewAssessment(t *testing.T) { requirementId: "test", description: "test", applicability: []string{"test"}, - methods: []*AssessmentMethod{}, + procedures: []*AssessmentProcedure{}, expectedError: true, }, { @@ -274,13 +190,13 @@ func TestNewAssessment(t *testing.T) { requirementId: "test", description: "test", applicability: []string{"test"}, - methods: []*AssessmentMethod{&passingAssessmentMethod}, + procedures: []*AssessmentProcedure{&passingProcedure}, expectedError: false, }, } for _, data := range newAssessmentsTestData { t.Run(data.testName, func(t *testing.T) { - assessment, err := NewAssessment(data.requirementId, data.description, data.applicability, data.methods) + assessment, err := NewAssessment(data.requirementId, data.description, data.applicability, data.procedures) if data.expectedError && err == nil { t.Error("expected error, got nil") } diff --git a/layer4/control_evaluation.go b/layer4/control_evaluation.go index edeffab..ff8067e 100644 --- a/layer4/control_evaluation.go +++ b/layer4/control_evaluation.go @@ -26,8 +26,8 @@ type ControlEvaluation struct { } // AddAssessment creates a new Assessment object and adds it to the ControlEvaluation. -func (c *ControlEvaluation) AddAssessment(requirementId string, description string, applicability []string, methods []*AssessmentMethod) (assessment *Assessment) { - assessment, err := NewAssessment(requirementId, description, applicability, methods) +func (c *ControlEvaluation) AddAssessment(requirementId string, description string, applicability []string, procedures []*AssessmentProcedure) (assessment *Assessment) { + assessment, err := NewAssessment(requirementId, description, applicability, procedures) if err != nil { c.Result = Failed c.Message = err.Error() @@ -36,8 +36,8 @@ func (c *ControlEvaluation) AddAssessment(requirementId string, description stri return } -// Evaluate runs each method in each assessment, updating the relevant fields on the control evaluation. -// It will halt if a method returns a failed result. The targetData is the data that the assessment will be run against. +// Evaluate runs each test procedure in each assessment, updating the relevant fields on the control evaluation. +// It will halt an assessment if a procedure step returns a failed result. The targetData is the data that the assessment will be run against. // The userApplicability is a slice of strings that determine when the assessment is applicable. The changesAllowed // determines whether the assessment is allowed to execute its changes. func (c *ControlEvaluation) Evaluate(targetData interface{}, userApplicability []string, changesAllowed bool) { diff --git a/layer4/control_evaluation_test.go b/layer4/control_evaluation_test.go index 734f6c8..8ab14a9 100644 --- a/layer4/control_evaluation_test.go +++ b/layer4/control_evaluation_test.go @@ -1,8 +1,6 @@ package layer4 -import ( - "testing" -) +import "testing" var controlEvaluationTestData = []struct { testName string @@ -141,14 +139,14 @@ func TestEvaluate(t *testing.T) { func TestAddAssessment(t *testing.T) { - controlEvaluationTestData[0].control.AddAssessment("test", "test", []string{}, []*AssessmentMethod{}) + controlEvaluationTestData[0].control.AddAssessment("test", "test", []string{}, []*AssessmentProcedure{}) if controlEvaluationTestData[0].control.Result != Failed { t.Errorf("Expected Result to be Failed, but it was %v", controlEvaluationTestData[0].control.Result) } - if controlEvaluationTestData[0].control.Message != "expected all Assessment fields to have a value, but got: requirementId=len(4), description=len=(4), applicability=len(0), methods=len(0)" { - t.Errorf("Expected error message to be 'expected all Assessment fields to have a value, but got: requirementId=len(4), description=len=(4), applicability=len(0), methods=len(0)', but instead it was '%v'", controlEvaluationTestData[0].control.Message) + if controlEvaluationTestData[0].control.Message != "expected all Assessment fields to have a value, but got: requirementId=len(4), description=len=(4), applicability=len(0), procedures=len(0)" { + t.Errorf("Expected error message to be 'expected all Assessment fields to have a value, but got: requirementId=len(4), description=len=(4), applicability=len(0), procedures=len(0)', but instead it was '%v'", controlEvaluationTestData[0].control.Message) } } diff --git a/layer4/method.go b/layer4/method.go new file mode 100644 index 0000000..baac53e --- /dev/null +++ b/layer4/method.go @@ -0,0 +1,39 @@ +package layer4 + +import ( + "encoding/json" +) + +// Method is an enum representing the method used to determine the assessment procedure result. +// This is designed to restrict the possible method values to a set of known types. +type Method int + +const ( + UnknownMethod Method = iota + // TestMethod represents an automated testing assessment method + TestMethod + // ObservationMethod represents an assessment method that requirement + // inspection done by a human + ObservationMethod +) + +// methodToString maps Method values to their string representations. +var methodToString = map[Method]string{ + ObservationMethod: "Observation", + TestMethod: "Test", + UnknownMethod: "Unknown", +} + +func (m Method) String() string { + return methodToString[m] +} + +// MarshalYAML ensures that Method is serialized as a string in YAML +func (m Method) MarshalYAML() (interface{}, error) { + return m.String(), nil +} + +// MarshalJSON ensures that Method is serialized as a string in JSON +func (m Method) MarshalJSON() ([]byte, error) { + return json.Marshal(m.String()) +} diff --git a/layer4/test-data.go b/layer4/test-data.go index 7b4f874..a7bf625 100644 --- a/layer4/test-data.go +++ b/layer4/test-data.go @@ -1,9 +1,9 @@ package layer4 -// This file is for reusable test data to help seed ideas and reduce duplication. - import "errors" +// This file is for reusable test data to help seed ideas and reduce duplication. + var ( // Generic applicability for testing testingApplicability = []string{"test-applicability"} @@ -23,18 +23,53 @@ var ( } // Assessment Results - passingAssessmentMethod = AssessmentMethod{Executor: func(interface{}, map[string]*Change) (Result, string) { + passingAssessmentStep = func(interface{}, map[string]*Change) (Result, string) { return Passed, "" - }} - failingAssessmentMethod = AssessmentMethod{Executor: func(interface{}, map[string]*Change) (Result, string) { + } + failingAssessmentStep = func(interface{}, map[string]*Change) (Result, string) { return Failed, "" - }} - needsReviewAssessmentMethod = AssessmentMethod{Executor: func(interface{}, map[string]*Change) (Result, string) { + } + needsReviewAssessmentStep = func(interface{}, map[string]*Change) (Result, string) { return NeedsReview, "" - }} - unknownAssessmentMethod = AssessmentMethod{Executor: func(interface{}, map[string]*Change) (Result, string) { + } + unknownAssessmentStep = func(interface{}, map[string]*Change) (Result, string) { return Unknown, "" - }} + } + + // AssessmentProcedures + failingProcedure = AssessmentProcedure{ + Steps: []AssessmentStep{ + failingAssessmentStep, + passingAssessmentStep, + }, + Method: TestMethod, + } + + passingProcedure = AssessmentProcedure{ + Steps: []AssessmentStep{ + passingAssessmentStep, + }, + Method: TestMethod, + } + + needsReviewProcedure = AssessmentProcedure{ + Steps: []AssessmentStep{ + passingAssessmentStep, + needsReviewAssessmentStep, + passingAssessmentStep, + }, + Method: TestMethod, + } + + badRevertPassingProcedure = AssessmentProcedure{ + Steps: []AssessmentStep{ + passingAssessmentStep, + passingAssessmentStep, + passingAssessmentStep, + passingAssessmentStep, + }, + Method: TestMethod, + } ) func pendingChangePtr() *Change { @@ -155,10 +190,7 @@ func failingAssessment() Assessment { return Assessment{ RequirementId: "failingAssessment()", Description: "failing assessment", - Methods: []*AssessmentMethod{ - &failingAssessmentMethod, - &passingAssessmentMethod, - }, + Procedures: []*AssessmentProcedure{&failingProcedure}, Applicability: testingApplicability, } } @@ -171,9 +203,7 @@ func passingAssessment() Assessment { return Assessment{ RequirementId: "passingAssessment()", Description: "passing assessment", - Methods: []*AssessmentMethod{ - &passingAssessmentMethod, - }, + Procedures: []*AssessmentProcedure{&passingProcedure}, Applicability: testingApplicability, Changes: map[string]*Change{ "pendingChange": pendingChangePtr(), @@ -189,14 +219,11 @@ func needsReviewAssessment() Assessment { return Assessment{ RequirementId: "needsReviewAssessment()", Description: "needs review assessment", - Methods: []*AssessmentMethod{ - &passingAssessmentMethod, - &needsReviewAssessmentMethod, - &passingAssessmentMethod, - }, + Procedures: []*AssessmentProcedure{&needsReviewProcedure}, Applicability: testingApplicability, } } + func unknownAssessmentPtr() *Assessment { a := unknownAssessment() return &a @@ -206,10 +233,15 @@ func unknownAssessment() Assessment { return Assessment{ RequirementId: "unknownAssessment()", Description: "unknown assessment", - Methods: []*AssessmentMethod{ - &passingAssessmentMethod, - &unknownAssessmentMethod, - &passingAssessmentMethod, + Procedures: []*AssessmentProcedure{ + { + Steps: []AssessmentStep{ + passingAssessmentStep, + unknownAssessmentStep, + passingAssessmentStep, + }, + Method: TestMethod, + }, }, Applicability: testingApplicability, } @@ -218,16 +250,11 @@ func unknownAssessment() Assessment { func badRevertPassingAssessment() Assessment { return Assessment{ RequirementId: "badRevertPassingAssessment()", - Description: "bad revert passing assessment", + Description: "bad revert passing assessment", Changes: map[string]*Change{ "badRevertChange": badRevertChangePtr(), }, - Methods: []*AssessmentMethod{ - &passingAssessmentMethod, - &passingAssessmentMethod, - &passingAssessmentMethod, - &passingAssessmentMethod, - }, + Procedures: []*AssessmentProcedure{&badRevertPassingProcedure}, Applicability: testingApplicability, } } diff --git a/layer4/testdata/good-osps-evaluation.yml b/layer4/testdata/good-osps-evaluation.yml index ec4e17f..d4a15f3 100644 --- a/layer4/testdata/good-osps-evaluation.yml +++ b/layer4/testdata/good-osps-evaluation.yml @@ -8,23 +8,23 @@ evaluations: description: "Determine if the project publishes SLSA provenance attestations" result: Needs Review message: "Security insights required for this assessment, but file not found" - # This show an example from - # https://github.com/revanite-io/pvtr-github-repo/blob/main/evaluation_plans/osps/build_release/evaluations.go#L174-L178 - # where this support a multistep assessment - methods: - - id: has-made-release - name: "Review the project releases" - description: "Determine if the project has any releases that require review of release artifacts" - run: true - executor: github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasMadeReleases - result: Passed - - id: has-security-insights-file - name: "Check the project for security-insights.yml" - description: "Determine if the project has a security-insights.yml in expected locations" - executor: github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasSecurityInsightsFile + procedures: + # This show an example from + # https://github.com/revanite-io/pvtr-github-repo/blob/main/evaluation_plans/osps/build_release/evaluations.go#L174-L178 + # where this support a multistep assessment + - id: "insight-has-attestation" + name: "Automated review of project declared attestations" + description: "Check the security insights file in the repository for SLSA attestations." + method: Test run: true + message: "Security insights required for this assessment, but file not found" result: Needs Review - - id: insights-has-attestation - name: "Check the project for SLSA provenance attestation" - description: "Check security-insights.yml attestation for a SLSA provenance attestation" - run: false \ No newline at end of file + steps: + - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasMadeReleases + - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasSecurityInsightsFile + - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/build_release.insightsHasSlsaAttestation + - id: "release-has-attestation" + name: "Manual review of project release artifacts" + description: "Manually review the project release artifacts to find SLSA attestations." + method: Observation + run: false diff --git a/schemas/layer-4.cue b/schemas/layer-4.cue index e8f1978..8443dcb 100644 --- a/schemas/layer-4.cue +++ b/schemas/layer-4.cue @@ -34,10 +34,8 @@ package schemas result: #Result // Message describes the result of the assessment message: string - // Methods defines the assessment methods associated with the assessment - methods: [...#AssessmentMethod] - // MethodsExecuted is the number of assessment methods that were executed during the assessment - "methods-executed"?: int @go(MethodsExecuted) + // Procedures defines the assessment procedures associated with the assessment + procedures: [...#AssessmentProcedure] // RunDuration is the time it took to run the assessment "run-duration"?: string @go(RunDuration) // Value is the object that was returned during the assessment @@ -46,39 +44,48 @@ package schemas changes?: [string]: #Change } -// AssessmentMethod describes a specific procedure for evaluating a Layer 2 control requirement. -#AssessmentMethod: { - // Id uniquely identifies the assessment method being executed +// AssessmentProcedure describes a testing procedure for evaluating a Layer 2 control requirement. +#AssessmentProcedure: { + // Id uniquely identifies the assessment procedure being executed id: string - // Name provides a summary of the method + // Name provides a summary of the procedure name: string - // Description provides a detailed explanation of the method + // Description provides a detailed explanation of the procedure description: string - // Run is a boolean indicating whether the method was run or not. When run is true, result is expected to be present + // Method describe the high-level method used to determine the results of the procedure + method: #ProcedureMethod + // Run is a boolean indicating whether the procedure was run or not. When run is true, result is expected to be present run: bool - // RemediationGuide provides a URL with remediation guidance associated with the control's assessment requirement and this specific assessment method + // RemediationGuide provides a URL with remediation guidance associated with the control's assessment requirement and this specific assessment procedure "remediation-guide"?: #URL @go(RemediationGuide) - // Documentation provides a URL to documentation that describes how the assessment method evaluates the control requirement + // Documentation provides a URL to documentation that describes how the assessment procedure evaluates the control requirement documentation?: #URL - // Executor provides the address or location for the specific assessment logic used - executor?: string + // Steps provides the address for the assessment steps executed + "steps"?: [...string] } -// Additional constraints on Assessment Method. -#AssessmentMethod: { +// Additional constraints on Assessment Procedure. +#AssessmentProcedure: { run: false + // Message describes the result of the procedure + message?: string + // Result communicates the outcome(s) of the procedure result?: ("Not Run" | *null) @go(Result,optional=nillable) } | { run: true + message!: string result!: #ResultWhenRun } // Result describes valid assessment outcomes before and after execution. #Result: #ResultWhenRun | "Not Run" -// Result describes the outcome(s) of an assessment method when it is executed. +// Result describes the outcome(s) of an assessment procedure when it is executed. #ResultWhenRun: "Passed" | "Failed" | "Needs Review" | "Not Applicable" | "Unknown" +// ProcedureMethod describes method options that can be used to determine the results +#ProcedureMethod: "Test" | "Observation" + // Change is a struct that contains the data and functions associated with a single change to a target resource. #Change: { // TargetName is the name or ID of the resource or configuration that is to be changed