diff --git a/layer4/assessment.go b/layer4/assessment.go index 346c748..252c685 100644 --- a/layer4/assessment.go +++ b/layer4/assessment.go @@ -1,11 +1,8 @@ package layer4 import ( - "encoding/json" "errors" "fmt" - "reflect" - "runtime" "time" ) @@ -17,70 +14,39 @@ type Assessment struct { Applicability []string `yaml:"applicability"` // Description is a human-readable description of the test Description string `yaml:"description"` - // Result is true if the test passed + // Result is the overall result of the assessment Result Result `yaml:"result"` // Message is the human-readable result of the test Message string `yaml:"message"` - // Steps is a slice of steps that were executed during the test - Steps []AssessmentStep `yaml:"steps"` - // StepsExecuted is the number of steps that were executed during the test - StepsExecuted int `yaml:"steps-executed,omitempty"` + // Procedures is a slice of assessment procedures that were executed during the test + Procedures []*AssessmentProcedure `yaml:"procedures"` // RunDuration is the time it took to run the test RunDuration string `yaml:"run-duration,omitempty"` // Value is the object that was returned during the test Value interface{} `yaml:"value,omitempty"` - // Changes is a slice of changes that were made during the test + // Changes is a map of changes that were made during the test Changes map[string]*Change `yaml:"changes,omitempty"` } -// AssessmentStep is a function type that inspects the provided targetData and returns a Result with a message. -// The message may be an error string or other descriptive text. -type AssessmentStep func(payload interface{}, c map[string]*Change) (Result, string) - -func (as AssessmentStep) String() string { - // Get the function pointer correctly - fn := runtime.FuncForPC(reflect.ValueOf(as).Pointer()) - if fn == nil { - return "" - } - return fn.Name() -} - -func (as AssessmentStep) MarshalJSON() ([]byte, error) { - return json.Marshal(as.String()) -} - -func (as AssessmentStep) MarshalYAML() (interface{}, error) { - return as.String(), nil -} - // NewAssessment creates a new Assessment object and returns a pointer to it. -func NewAssessment(requirementId string, description string, applicability []string, steps []AssessmentStep) (*Assessment, error) { +func NewAssessment(requirementId string, description string, applicability []string, procedures []*AssessmentProcedure) (*Assessment, error) { a := &Assessment{ RequirementId: requirementId, Description: description, Applicability: applicability, Result: NotRun, - Steps: steps, + Procedures: procedures, } err := a.precheck() return a, err } -// AddStep queues a new step in the Assessment -func (a *Assessment) AddStep(step AssessmentStep) { - a.Steps = append(a.Steps, step) -} - -func (a *Assessment) runStep(targetData interface{}, step AssessmentStep) Result { - a.StepsExecuted++ - result, message := step(targetData, a.Changes) - a.Result = UpdateAggregateResult(a.Result, result) - a.Message = message - return result +// AddProcedure queues a new procedure in the Assessment +func (a *Assessment) AddProcedure(procedure AssessmentProcedure) { + a.Procedures = append(a.Procedures, &procedure) } -// Run will execute all steps, halting if any step does not return layer4.Passed. +// Run executes all assessment procedures using the test method. func (a *Assessment) Run(targetData interface{}, changesAllowed bool) Result { if a.Result != NotRun { return a.Result @@ -97,11 +63,15 @@ func (a *Assessment) Run(targetData interface{}, changesAllowed bool) Result { change.Allow() } } - for _, step := range a.Steps { - if a.runStep(targetData, step) == Failed { - return Failed + + for _, procedure := range a.Procedures { + if procedure.Method == TestMethod { + result := procedure.RunProcedure(targetData, a.Changes) + a.Result = UpdateAggregateResult(a.Result, result) + a.Message = procedure.Message } } + a.RunDuration = time.Since(startTime).String() return a.Result } @@ -142,10 +112,10 @@ func (a *Assessment) RevertChanges() (corrupted bool) { // precheck verifies that the assessment has all the required fields. // It returns an error if the assessment is not valid. func (a *Assessment) precheck() error { - if a.RequirementId == "" || a.Description == "" || a.Applicability == nil || a.Steps == nil || len(a.Applicability) == 0 || len(a.Steps) == 0 { + if a.RequirementId == "" || a.Description == "" || a.Applicability == nil || a.Procedures == nil || len(a.Applicability) == 0 || len(a.Procedures) == 0 { message := fmt.Sprintf( - "expected all Assessment fields to have a value, but got: requirementId=len(%v), description=len=(%v), applicability=len(%v), steps=len(%v)", - len(a.RequirementId), len(a.Description), len(a.Applicability), len(a.Steps), + "expected all Assessment fields to have a value, but got: requirementId=len(%v), description=len=(%v), applicability=len(%v), procedures=len(%v)", + len(a.RequirementId), len(a.Description), len(a.Applicability), len(a.Procedures), ) a.Result = Unknown a.Message = message diff --git a/layer4/assessment_procedure.go b/layer4/assessment_procedure.go new file mode 100644 index 0000000..30d77c4 --- /dev/null +++ b/layer4/assessment_procedure.go @@ -0,0 +1,114 @@ +package layer4 + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "runtime" +) + +// AssessmentProcedure is a specific outline defining how to assess a Layer 2 control requirement. +type AssessmentProcedure struct { + // Id is the unique identifier of the assessment procedure being executed + Id string `json:"id" yaml:"id"` + // Name is the human-readable name of the procedure. + Name string `json:"name" yaml:"name"` + // Description is a detailed explanation of the procedure. + Description string `json:"description" yaml:"description"` + // Method describe the high-level method used to determine the results of the procedure + Method Method `yaml:"method"` + // Remediation guide is a URL to remediation guidance associated with the control's assessment requirement and this specific assessment procedure. + RemediationGuide string `json:"remediation-guide,omitempty" yaml:"remediation-guide,omitempty"` + // URL to documentation that describes how the assessment procedure evaluates the control requirement. + Documentation string `json:"documentation,omitempty" yaml:"documentation,omitempty"` + // Run is a boolean indicating whether the procedure was run or not. When run is true, result is expected to be present. + Run bool `json:"run" yaml:"run"` + // Message is the human-readable result of the procedure + Message string `json:"message,omitempty" yaml:"message,omitempty"` + // Result is the outcome of the assessment procedure. + // This field must be present if Run is true. + Result Result `json:"result,omitempty" yaml:"result,omitempty"` + // StepsExecuted is the number of steps that were executed during the assessment execution + StepsExecuted int `json:"-" yaml:"-"` + // Steps define logical steps to inspect the provided targetData and returns a Result with a message. + // The message may be an error string or other descriptive text. + Steps []AssessmentStep +} + +// NewProcedure creates a new AssessmentProcedure object and returns a pointer to it. +func NewProcedure(id, name, description string, steps []AssessmentStep) (*AssessmentProcedure, error) { + a := &AssessmentProcedure{ + Id: id, + Name: name, + Description: description, + Result: NotRun, + Steps: steps, + } + err := a.precheck() + return a, err +} + +// AssessmentStep is a function type that inspects the provided targetData and returns a Result with a message. +// The message may be an error string or other descriptive text. +type AssessmentStep func(payload interface{}, c map[string]*Change) (Result, string) + +func (as AssessmentStep) String() string { + // Get the function pointer correctly + fn := runtime.FuncForPC(reflect.ValueOf(as).Pointer()) + if fn == nil { + return "" + } + return fn.Name() +} + +func (as AssessmentStep) MarshalJSON() ([]byte, error) { + return json.Marshal(as.String()) +} + +func (as AssessmentStep) MarshalYAML() (interface{}, error) { + return as.String(), nil +} + +// AddStep queues a new step in the Assessment Procedure +func (a *AssessmentProcedure) AddStep(step AssessmentStep) { + a.Steps = append(a.Steps, step) +} + +// RunProcedure executes all assessment steps, halting if any assessment does not return layer4.Passed. +func (a *AssessmentProcedure) RunProcedure(targetData interface{}, changes map[string]*Change) Result { + a.Run = true + + err := a.precheck() + if err != nil { + a.Result = Unknown + return a.Result + } + for _, steps := range a.Steps { + if a.runStep(targetData, changes, steps) == Failed { + return Failed + } + } + return a.Result +} + +func (a *AssessmentProcedure) runStep(targetData interface{}, changes map[string]*Change, step AssessmentStep) Result { + a.StepsExecuted++ + result, message := step(targetData, changes) + a.Result = UpdateAggregateResult(a.Result, result) + a.Message = message + return result +} + +// precheck verifies that the assessment procedure has step fields. +// It returns an error if the assessment is not valid. +func (a *AssessmentProcedure) precheck() error { + if len(a.Steps) == 0 { + message := fmt.Sprintf( + "expected all Assessment Procedure fields steps=len(%v)", + len(a.Steps), + ) + return errors.New(message) + } + return nil +} diff --git a/layer4/assessment_procedure_test.go b/layer4/assessment_procedure_test.go new file mode 100644 index 0000000..d3e3da1 --- /dev/null +++ b/layer4/assessment_procedure_test.go @@ -0,0 +1,122 @@ +package layer4 + +import ( + "testing" +) + +func getProcedures() []struct { + testName string + procedure AssessmentProcedure + numberOfSteps int + numberOfStepsToRun int + expectedResult Result +} { + return []struct { + testName string + procedure AssessmentProcedure + numberOfSteps int + numberOfStepsToRun int + expectedResult Result + }{ + { + testName: "Assessment with no steps", + procedure: AssessmentProcedure{}, + expectedResult: Unknown, + }, + { + testName: "Procedure with one step", + procedure: passingProcedure, + numberOfSteps: 1, + numberOfStepsToRun: 1, + expectedResult: Passed, + }, + { + testName: "Procedure and two steps", + procedure: failingProcedure, + numberOfSteps: 2, + numberOfStepsToRun: 1, + expectedResult: Failed, + }, + { + testName: "Procedure three steps", + procedure: needsReviewProcedure, + numberOfSteps: 3, + numberOfStepsToRun: 3, + expectedResult: NeedsReview, + }, + } +} + +// TestRun ensures that Run executes all steps, halting if any Procedure does not return Passed +func TestRunProcedure(t *testing.T) { + for _, data := range getProcedures() { + t.Run(data.testName, func(t *testing.T) { + a := data.procedure // copy the procedure to prevent duplicate executions in the next test + result := a.RunProcedure(nil, nil) + if result != a.Result { + t.Errorf("expected match between Run return value (%s) and assessment Result value (%s)", result, data.expectedResult) + } + if a.StepsExecuted != data.numberOfStepsToRun { + t.Errorf("expected to run %d tests, got %d", data.numberOfStepsToRun, a.StepsExecuted) + } + }) + } +} + +// TestNewStep ensures that NewStep queues a new step in the Assessment +func TestAddStep(t *testing.T) { + for _, test := range getProcedures() { + t.Run(test.testName, func(t *testing.T) { + if len(test.procedure.Steps) != test.numberOfSteps { + t.Errorf("Bad test data: expected to start with %d, got %d", test.numberOfSteps, len(test.procedure.Steps)) + } + test.procedure.AddStep(passingAssessmentStep) + if len(test.procedure.Steps) != test.numberOfSteps+1 { + t.Errorf("expected %d, got %d", test.numberOfSteps, len(test.procedure.Steps)) + } + }) + } +} + +// TestRunStep ensures that runStep runs the step and updates the Assessment Procedure +func TestRunStep(t *testing.T) { + stepsTestData := []struct { + testName string + step AssessmentStep + result Result + }{ + { + testName: "Failing step", + step: failingAssessmentStep, + result: Failed, + }, + { + testName: "Passing step", + step: passingAssessmentStep, + result: Passed, + }, + { + testName: "Needs review step", + step: needsReviewAssessmentStep, + result: NeedsReview, + }, + { + testName: "Unknown step", + step: unknownAssessmentStep, + result: Unknown, + }, + } + for _, test := range stepsTestData { + t.Run(test.testName, func(t *testing.T) { + anyOldProcedure := AssessmentProcedure{} + + result := anyOldProcedure.runStep(nil, nil, test.step) + if result != test.result { + t.Errorf("expected %s, got %s", test.result, result) + } + if anyOldProcedure.Result != test.result { + t.Errorf("expected %s, got %s", test.result, anyOldProcedure.Result) + } + }) + } +} diff --git a/layer4/assessment_test.go b/layer4/assessment_test.go index 59da860..ca82f99 100644 --- a/layer4/assessment_test.go +++ b/layer4/assessment_test.go @@ -4,135 +4,51 @@ import ( "testing" ) -func getAssessmentsTestData() []struct { - testName string - assessment Assessment - numberOfSteps int - numberOfStepsToRun int - expectedResult Result +func getAssessments() []struct { + testName string + assessment Assessment + expectedResult Result } { return []struct { - testName string - assessment Assessment - numberOfSteps int - numberOfStepsToRun int - expectedResult Result + testName string + assessment Assessment + expectedResult Result }{ { testName: "Assessment with no steps", assessment: Assessment{}, }, { - testName: "Assessment with one step", - assessment: passingAssessment(), - numberOfSteps: 1, - numberOfStepsToRun: 1, - expectedResult: Passed, + testName: "Passing assessment", + assessment: passingAssessment(), + expectedResult: Passed, }, { - testName: "Assessment with two steps", - assessment: failingAssessment(), - numberOfSteps: 2, - numberOfStepsToRun: 1, - expectedResult: Failed, + testName: "Failing assessment", + assessment: failingAssessment(), + expectedResult: Failed, }, { - testName: "Assessment with three steps", - assessment: needsReviewAssessment(), - numberOfSteps: 3, - numberOfStepsToRun: 3, - expectedResult: NeedsReview, + testName: "Assessment needs review", + assessment: needsReviewAssessment(), + expectedResult: NeedsReview, }, { - testName: "Assessment with four steps", - assessment: badRevertPassingAssessment(), - numberOfSteps: 4, - numberOfStepsToRun: 4, - expectedResult: Passed, + testName: "Bad change revert", + assessment: badRevertPassingAssessment(), + expectedResult: Passed, }, } } -// TestNewStep ensures that NewStep queues a new step in the Assessment -func TestAddStep(t *testing.T) { - for _, test := range getAssessmentsTestData() { - t.Run(test.testName, func(t *testing.T) { - if len(test.assessment.Steps) != test.numberOfSteps { - t.Errorf("Bad test data: expected to start with %d, got %d", test.numberOfSteps, len(test.assessment.Steps)) - } - test.assessment.AddStep(passingAssessmentStep) - if len(test.assessment.Steps) != test.numberOfSteps+1 { - t.Errorf("expected %d, got %d", test.numberOfSteps, len(test.assessment.Steps)) - } - }) - } -} - -// TestRunStep ensures that runStep runs the step and updates the Assessment -func TestRunStep(t *testing.T) { - stepsTestData := []struct { - testName string - step AssessmentStep - result Result - }{ - { - testName: "Failing step", - step: failingAssessmentStep, - result: Failed, - }, - { - testName: "Passing step", - step: passingAssessmentStep, - result: Passed, - }, - { - testName: "Needs review step", - step: needsReviewAssessmentStep, - result: NeedsReview, - }, - { - testName: "Unknown step", - step: unknownAssessmentStep, - result: Unknown, - }, - } - for _, test := range stepsTestData { - t.Run(test.testName, func(t *testing.T) { - anyOldAssessment := Assessment{} - result := anyOldAssessment.runStep(nil, test.step) - if result != test.result { - t.Errorf("expected %s, got %s", test.result, result) - } - if anyOldAssessment.Result != test.result { - t.Errorf("expected %s, got %s", test.result, anyOldAssessment.Result) - } - }) - } -} - -// TestRun ensures that Run executes all steps, halting if any step does not return Passed func TestRun(t *testing.T) { - for _, data := range getAssessmentsTestData() { - t.Run(data.testName, func(t *testing.T) { + for _, data := range getAssessments() { + t.Run(data.testName+"-no-changes", func(t *testing.T) { a := data.assessment // copy the assessment to prevent duplicate executions in the next test - result := a.Run(nil, true) + result := a.Run(nil, false) if result != a.Result { t.Errorf("expected match between Run return value (%s) and assessment Result value (%s)", result, data.expectedResult) } - if a.StepsExecuted != data.numberOfStepsToRun { - t.Errorf("expected to run %d tests, got %d", data.numberOfStepsToRun, a.StepsExecuted) - } - }) - } -} - -func TestRunB(t *testing.T) { - for _, data := range getAssessmentsTestData() { - t.Run(data.testName+"-no-changes", func(t *testing.T) { - data.assessment.Run(nil, false) - if data.assessment.StepsExecuted != data.numberOfStepsToRun { - t.Errorf("expected to run %d tests, got %d", data.numberOfStepsToRun, data.assessment.StepsExecuted) - } for _, change := range data.assessment.Changes { if change.Allowed { t.Errorf("expected all changes to be disallowed, but found an allowed change") @@ -234,7 +150,7 @@ func TestNewAssessment(t *testing.T) { requirementId string description string applicability []string - steps []AssessmentStep + procedures []*AssessmentProcedure expectedError bool }{ { @@ -242,7 +158,7 @@ func TestNewAssessment(t *testing.T) { requirementId: "", description: "test", applicability: []string{"test"}, - steps: []AssessmentStep{passingAssessmentStep}, + procedures: []*AssessmentProcedure{&passingProcedure}, expectedError: true, }, { @@ -250,7 +166,7 @@ func TestNewAssessment(t *testing.T) { requirementId: "test", description: "", applicability: []string{"test"}, - steps: []AssessmentStep{passingAssessmentStep}, + procedures: []*AssessmentProcedure{&passingProcedure}, expectedError: true, }, { @@ -258,7 +174,7 @@ func TestNewAssessment(t *testing.T) { requirementId: "test", description: "test", applicability: []string{}, - steps: []AssessmentStep{passingAssessmentStep}, + procedures: []*AssessmentProcedure{&passingProcedure}, expectedError: true, }, { @@ -266,7 +182,7 @@ func TestNewAssessment(t *testing.T) { requirementId: "test", description: "test", applicability: []string{"test"}, - steps: []AssessmentStep{}, + procedures: []*AssessmentProcedure{}, expectedError: true, }, { @@ -274,13 +190,13 @@ func TestNewAssessment(t *testing.T) { requirementId: "test", description: "test", applicability: []string{"test"}, - steps: []AssessmentStep{passingAssessmentStep}, + procedures: []*AssessmentProcedure{&passingProcedure}, expectedError: false, }, } for _, data := range newAssessmentsTestData { t.Run(data.testName, func(t *testing.T) { - assessment, err := NewAssessment(data.requirementId, data.description, data.applicability, data.steps) + assessment, err := NewAssessment(data.requirementId, data.description, data.applicability, data.procedures) if data.expectedError && err == nil { t.Error("expected error, got nil") } diff --git a/layer4/control_evaluation.go b/layer4/control_evaluation.go index fd09d81..ff8067e 100644 --- a/layer4/control_evaluation.go +++ b/layer4/control_evaluation.go @@ -21,13 +21,13 @@ type ControlEvaluation struct { CorruptedState bool `yaml:"corrupted-state"` // RemediationGuide is the URL to the documentation for this evaluation RemediationGuide string `yaml:"remediation-guide"` - // Assessments is a map of pointers to Assessment objects to establish idempotency + // Assessments is a slice of pointers to Assessment objects to establish idempotency Assessments []*Assessment `yaml:"assessments"` } // AddAssessment creates a new Assessment object and adds it to the ControlEvaluation. -func (c *ControlEvaluation) AddAssessment(requirementId string, description string, applicability []string, steps []AssessmentStep) (assessment *Assessment) { - assessment, err := NewAssessment(requirementId, description, applicability, steps) +func (c *ControlEvaluation) AddAssessment(requirementId string, description string, applicability []string, procedures []*AssessmentProcedure) (assessment *Assessment) { + assessment, err := NewAssessment(requirementId, description, applicability, procedures) if err != nil { c.Result = Failed c.Message = err.Error() @@ -36,8 +36,8 @@ func (c *ControlEvaluation) AddAssessment(requirementId string, description stri return } -// Evaluate runs each step in each assessment, updating the relevant fields on the control evaluation. -// It will halt if a step returns a failed result. The targetData is the data that the assessment will be run against. +// Evaluate runs each test procedure in each assessment, updating the relevant fields on the control evaluation. +// It will halt an assessment if a procedure step returns a failed result. The targetData is the data that the assessment will be run against. // The userApplicability is a slice of strings that determine when the assessment is applicable. The changesAllowed // determines whether the assessment is allowed to execute its changes. func (c *ControlEvaluation) Evaluate(targetData interface{}, userApplicability []string, changesAllowed bool) { diff --git a/layer4/control_evaluation_test.go b/layer4/control_evaluation_test.go index 7e64d54..8ab14a9 100644 --- a/layer4/control_evaluation_test.go +++ b/layer4/control_evaluation_test.go @@ -137,16 +137,16 @@ func TestEvaluate(t *testing.T) { } } -func TestAddAssesment(t *testing.T) { +func TestAddAssessment(t *testing.T) { - controlEvaluationTestData[0].control.AddAssessment("test", "test", []string{}, []AssessmentStep{}) + controlEvaluationTestData[0].control.AddAssessment("test", "test", []string{}, []*AssessmentProcedure{}) if controlEvaluationTestData[0].control.Result != Failed { t.Errorf("Expected Result to be Failed, but it was %v", controlEvaluationTestData[0].control.Result) } - if controlEvaluationTestData[0].control.Message != "expected all Assessment fields to have a value, but got: requirementId=len(4), description=len=(4), applicability=len(0), steps=len(0)" { - t.Errorf("Expected error message to be 'expected all Assessment fields to have a value, but got: requirementId=len(4), description=len=(4), applicability=len(0), steps=len(0)', but instead it was '%v'", controlEvaluationTestData[0].control.Message) + if controlEvaluationTestData[0].control.Message != "expected all Assessment fields to have a value, but got: requirementId=len(4), description=len=(4), applicability=len(0), procedures=len(0)" { + t.Errorf("Expected error message to be 'expected all Assessment fields to have a value, but got: requirementId=len(4), description=len=(4), applicability=len(0), procedures=len(0)', but instead it was '%v'", controlEvaluationTestData[0].control.Message) } } diff --git a/layer4/method.go b/layer4/method.go new file mode 100644 index 0000000..baac53e --- /dev/null +++ b/layer4/method.go @@ -0,0 +1,39 @@ +package layer4 + +import ( + "encoding/json" +) + +// Method is an enum representing the method used to determine the assessment procedure result. +// This is designed to restrict the possible method values to a set of known types. +type Method int + +const ( + UnknownMethod Method = iota + // TestMethod represents an automated testing assessment method + TestMethod + // ObservationMethod represents an assessment method that requirement + // inspection done by a human + ObservationMethod +) + +// methodToString maps Method values to their string representations. +var methodToString = map[Method]string{ + ObservationMethod: "Observation", + TestMethod: "Test", + UnknownMethod: "Unknown", +} + +func (m Method) String() string { + return methodToString[m] +} + +// MarshalYAML ensures that Method is serialized as a string in YAML +func (m Method) MarshalYAML() (interface{}, error) { + return m.String(), nil +} + +// MarshalJSON ensures that Method is serialized as a string in JSON +func (m Method) MarshalJSON() ([]byte, error) { + return json.Marshal(m.String()) +} diff --git a/layer4/test-data.go b/layer4/test-data.go index 70579b0..a7bf625 100644 --- a/layer4/test-data.go +++ b/layer4/test-data.go @@ -1,9 +1,9 @@ package layer4 -// This file is for reusable test data to help seed ideas and reduce duplication. - import "errors" +// This file is for reusable test data to help seed ideas and reduce duplication. + var ( // Generic applicability for testing testingApplicability = []string{"test-applicability"} @@ -35,6 +35,41 @@ var ( unknownAssessmentStep = func(interface{}, map[string]*Change) (Result, string) { return Unknown, "" } + + // AssessmentProcedures + failingProcedure = AssessmentProcedure{ + Steps: []AssessmentStep{ + failingAssessmentStep, + passingAssessmentStep, + }, + Method: TestMethod, + } + + passingProcedure = AssessmentProcedure{ + Steps: []AssessmentStep{ + passingAssessmentStep, + }, + Method: TestMethod, + } + + needsReviewProcedure = AssessmentProcedure{ + Steps: []AssessmentStep{ + passingAssessmentStep, + needsReviewAssessmentStep, + passingAssessmentStep, + }, + Method: TestMethod, + } + + badRevertPassingProcedure = AssessmentProcedure{ + Steps: []AssessmentStep{ + passingAssessmentStep, + passingAssessmentStep, + passingAssessmentStep, + passingAssessmentStep, + }, + Method: TestMethod, + } ) func pendingChangePtr() *Change { @@ -155,10 +190,7 @@ func failingAssessment() Assessment { return Assessment{ RequirementId: "failingAssessment()", Description: "failing assessment", - Steps: []AssessmentStep{ - failingAssessmentStep, - passingAssessmentStep, - }, + Procedures: []*AssessmentProcedure{&failingProcedure}, Applicability: testingApplicability, } } @@ -171,9 +203,7 @@ func passingAssessment() Assessment { return Assessment{ RequirementId: "passingAssessment()", Description: "passing assessment", - Steps: []AssessmentStep{ - passingAssessmentStep, - }, + Procedures: []*AssessmentProcedure{&passingProcedure}, Applicability: testingApplicability, Changes: map[string]*Change{ "pendingChange": pendingChangePtr(), @@ -189,14 +219,11 @@ func needsReviewAssessment() Assessment { return Assessment{ RequirementId: "needsReviewAssessment()", Description: "needs review assessment", - Steps: []AssessmentStep{ - passingAssessmentStep, - needsReviewAssessmentStep, - passingAssessmentStep, - }, + Procedures: []*AssessmentProcedure{&needsReviewProcedure}, Applicability: testingApplicability, } } + func unknownAssessmentPtr() *Assessment { a := unknownAssessment() return &a @@ -206,10 +233,15 @@ func unknownAssessment() Assessment { return Assessment{ RequirementId: "unknownAssessment()", Description: "unknown assessment", - Steps: []AssessmentStep{ - passingAssessmentStep, - unknownAssessmentStep, - passingAssessmentStep, + Procedures: []*AssessmentProcedure{ + { + Steps: []AssessmentStep{ + passingAssessmentStep, + unknownAssessmentStep, + passingAssessmentStep, + }, + Method: TestMethod, + }, }, Applicability: testingApplicability, } @@ -222,12 +254,7 @@ func badRevertPassingAssessment() Assessment { Changes: map[string]*Change{ "badRevertChange": badRevertChangePtr(), }, - Steps: []AssessmentStep{ - passingAssessmentStep, - passingAssessmentStep, - passingAssessmentStep, - passingAssessmentStep, - }, + Procedures: []*AssessmentProcedure{&badRevertPassingProcedure}, Applicability: testingApplicability, } } diff --git a/layer4/testdata/good-osps-evaluation.yml b/layer4/testdata/good-osps-evaluation.yml new file mode 100644 index 0000000..d4a15f3 --- /dev/null +++ b/layer4/testdata/good-osps-evaluation.yml @@ -0,0 +1,30 @@ +evaluations: + - control-id: "OSPS-BR-06" + name: "Produce all released software assets with signatures and hashes" + message: "Security insights required for this assessment, but file not found" + result: Needs Review + assessments: + - requirement-id: "OSPS-BR-06.01" + description: "Determine if the project publishes SLSA provenance attestations" + result: Needs Review + message: "Security insights required for this assessment, but file not found" + procedures: + # This show an example from + # https://github.com/revanite-io/pvtr-github-repo/blob/main/evaluation_plans/osps/build_release/evaluations.go#L174-L178 + # where this support a multistep assessment + - id: "insight-has-attestation" + name: "Automated review of project declared attestations" + description: "Check the security insights file in the repository for SLSA attestations." + method: Test + run: true + message: "Security insights required for this assessment, but file not found" + result: Needs Review + steps: + - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasMadeReleases + - github.com/revanite-io/pvtr-github-repo/evaluation_plans/reusable_steps.HasSecurityInsightsFile + - github.com/revanite-io/pvtr-github-repo/evaluation_plans/osps/build_release.insightsHasSlsaAttestation + - id: "release-has-attestation" + name: "Manual review of project release artifacts" + description: "Manually review the project release artifacts to find SLSA attestations." + method: Observation + run: false diff --git a/schemas/layer-4.cue b/schemas/layer-4.cue index ffea275..8443dcb 100644 --- a/schemas/layer-4.cue +++ b/schemas/layer-4.cue @@ -1,37 +1,108 @@ package schemas #Layer4: { - evaluations: [#ControlEvaluation, ...#ControlEvaluation] + evaluations: [#ControlEvaluation, ...#ControlEvaluation] } -// Types - +// ControlEvaluation provides all assessment results for a single control #ControlEvaluation: { - name: string - "control-id": string - result: #Result - message: string - "documentation-url"?: =~"^https?://[^\\s]+$" - "corrupted-state"?: bool - "assessment-results"?: [...#AssessmentResult] + // Name is the name of the control being evaluated + name: string + // ControlID uniquely identifies the control + "control-id": string + // Result communicates whether the evaluation has been run, and if so, the outcome(s) + result: #Result + // Message describes the result of the evaluation + message: string + // CorruptedState is true if the control evaluation was interrupted and changes were not reverted + "corrupted-state"?: bool + // RemediationGuide provides a URL with guidance on how to remediate systems that fail control evaluation + "remediation-guide"?: string + // Assessments represents the collection of results from evaluation of each control requirement + assessments: [...#Assessment] +} + +// Assessment provides all assessment results from evaluation of a single control requirement +#Assessment: { + // RequirementID uniquely identifies the requirement being tested + "requirement-id": string @go(RequirementId) + // Applicability provides identifier strings to determine when this assessment is applicable + applicability: [...string] + // Description provides a detailed explanation of the assessment + description: string + // Result communicates whether the assessment has been run, and if so, the outcome(s) + result: #Result + // Message describes the result of the assessment + message: string + // Procedures defines the assessment procedures associated with the assessment + procedures: [...#AssessmentProcedure] + // RunDuration is the time it took to run the assessment + "run-duration"?: string @go(RunDuration) + // Value is the object that was returned during the assessment + value?: _ + // Changes describes changes that were made during the assessment + changes?: [string]: #Change +} + +// AssessmentProcedure describes a testing procedure for evaluating a Layer 2 control requirement. +#AssessmentProcedure: { + // Id uniquely identifies the assessment procedure being executed + id: string + // Name provides a summary of the procedure + name: string + // Description provides a detailed explanation of the procedure + description: string + // Method describe the high-level method used to determine the results of the procedure + method: #ProcedureMethod + // Run is a boolean indicating whether the procedure was run or not. When run is true, result is expected to be present + run: bool + // RemediationGuide provides a URL with remediation guidance associated with the control's assessment requirement and this specific assessment procedure + "remediation-guide"?: #URL @go(RemediationGuide) + // Documentation provides a URL to documentation that describes how the assessment procedure evaluates the control requirement + documentation?: #URL + // Steps provides the address for the assessment steps executed + "steps"?: [...string] } -#AssessmentResult: { - result: #Result - name: string - description: string - message: string - "function-address": string - change?: #Change - value?: _ +// Additional constraints on Assessment Procedure. +#AssessmentProcedure: { + run: false + // Message describes the result of the procedure + message?: string + // Result communicates the outcome(s) of the procedure + result?: ("Not Run" | *null) @go(Result,optional=nillable) +} | { + run: true + message!: string + result!: #ResultWhenRun } -#Result: "Passed" | "Failed" | "Needs Review" +// Result describes valid assessment outcomes before and after execution. +#Result: #ResultWhenRun | "Not Run" +// Result describes the outcome(s) of an assessment procedure when it is executed. +#ResultWhenRun: "Passed" | "Failed" | "Needs Review" | "Not Applicable" | "Unknown" + +// ProcedureMethod describes method options that can be used to determine the results +#ProcedureMethod: "Test" | "Observation" + +// Change is a struct that contains the data and functions associated with a single change to a target resource. #Change: { - "target-name": string - applied: bool - reverted: bool - error?: string - "target-object"?: _ -} \ No newline at end of file + // TargetName is the name or ID of the resource or configuration that is to be changed + "target-name": string @go(TargetName) + // Description is a human-readable description of the change + description: string + // TargetObject is supplemental data describing the object that was changed + "target-object"?: _ @go(TargetObject) + // Applied is true if the change was successfully applied at least once + applied?: bool + // Reverted is true if the change was successfully reverted and not applied again + reverted?: bool + // Error is used if any error occurred during the change + error?: string + // Allowed may be disabled to prevent the change from being applied + allowed?: bool +} + +// URL describes a specific subset of URLs of interest to the framework +#URL: =~"^https?://[^\\s]+$"