From bf68f816f868aa28f3889626b462987f90d30cbb Mon Sep 17 00:00:00 2001 From: Farshid Tavakolizadeh Date: Thu, 20 May 2021 13:19:04 +0200 Subject: [PATCH] Validate against multiple schemas (#50) * validate against multiple schemas * configuration, file loading * loading schemas into memory * fix response types * Add dns-sd types to wot package * update tests * Update expiry logic for compliance Give precedence to ttl for calculation of expiry fix ttl test, remove retrieval time * discovery attr validation with json schema * remove storage type from test log * Delete directory-td.jsonld * refactoring to use new validation function * update tests, readme, remove deprecated code * add function to check loaded schemas, refactor tests --- README.md | 7 ++++ catalog/catalog.go | 12 ++---- catalog/catalog_test.go | 38 ------------------- catalog/main_test.go | 5 ++- config.go | 5 +++ main.go | 14 ++++--- sample_conf/thing-directory.json | 3 ++ wot/discovery_schema.json | 32 ++++++++++++++++ wot/discovery_validation.go | 61 ------------------------------- wot/validation.go | 63 ++++++++++++++++++++++++-------- wot/validation_test.go | 63 ++++++++++++++++++++++++++++---- 11 files changed, 167 insertions(+), 136 deletions(-) delete mode 100644 catalog/catalog_test.go create mode 100644 wot/discovery_schema.json delete mode 100644 wot/discovery_validation.go diff --git a/README.md b/README.md index e670c1ff..276edba5 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,13 @@ Copy sample configuration files for server and JSON Schema into `conf` directory mkdir -p conf cp sample_conf/thing-directory.json wot/wot_td_schema.json conf ``` + +Alternatively, download sample config file and schemas: +```bash +curl https://raw.githubusercontent.com/linksmart/thing-directory/master/sample_conf/thing-directory.json --create-dirs -o conf/thing-directory.json +curl https://raw.githubusercontent.com/w3c/wot-thing-description/REC1.0/validation/td-json-schema-validation.json --create-dirs -o conf/wot_td_schema.json +``` + `conf` is the default directory for configuration files. This can be changed with CLI arguments. Get the CLI argument help (linux/macOS): diff --git a/catalog/catalog.go b/catalog/catalog.go index 6efd0e60..71685c82 100644 --- a/catalog/catalog.go +++ b/catalog/catalog.go @@ -20,17 +20,11 @@ const ( ) func validateThingDescription(td map[string]interface{}) ([]wot.ValidationError, error) { - issues, err := wot.ValidateDiscoveryExtensions(&td) + result, err := wot.ValidateTD(&td) if err != nil { - return nil, fmt.Errorf("error validating with JSON schema: %s", err) + return nil, fmt.Errorf("error validating with JSON Schemas: %s", err) } - - tdIssues, err := wot.ValidateMap(&td) - if err != nil { - return nil, fmt.Errorf("error validating with JSON schema: %s", err) - } - - return append(issues, tdIssues...), nil + return result, nil } // Controller interface diff --git a/catalog/catalog_test.go b/catalog/catalog_test.go deleted file mode 100644 index 5587f60e..00000000 --- a/catalog/catalog_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package catalog - -import ( - "testing" -) - -// here are only the tests related to non-standard TD vocabulary -func TestValidateThingDescription(t *testing.T) { - err := loadSchema() - if err != nil { - t.Fatalf("error loading WoT Thing Description schema: %s", err) - } - - t.Run("non-float TTL", func(t *testing.T) { - var td = map[string]any{ - "@context": "https://www.w3.org/2019/wot/td/v1", - "id": "urn:example:test/thing1", - "title": "example thing", - "security": []string{"basic_sc"}, - "securityDefinitions": map[string]any{ - "basic_sc": map[string]string{ - "in": "header", - "scheme": "basic", - }, - }, - "registration": map[string]any{ - "ttl": "60", - }, - } - results, err := validateThingDescription(td) - if err != nil { - t.Fatalf("internal validation error: %s", err) - } - if len(results) == 0 { - t.Fatalf("Didn't return error on string TTL.") - } - }) -} diff --git a/catalog/main_test.go b/catalog/main_test.go index 82b1ef61..86d346c4 100644 --- a/catalog/main_test.go +++ b/catalog/main_test.go @@ -27,11 +27,14 @@ var ( ) func loadSchema() error { + if wot.LoadedJSONSchemas() { + return nil + } path := os.Getenv(envTestSchemaPath) if path == "" { path = defaultSchemaPath } - return wot.LoadSchema(path) + return wot.LoadJSONSchemas([]string{path}) } func serializedEqual(td1 ThingDescription, td2 ThingDescription) bool { diff --git a/config.go b/config.go index 423bba90..f944585a 100644 --- a/config.go +++ b/config.go @@ -17,12 +17,17 @@ import ( type Config struct { ServiceID string `json:"serviceID"` Description string `json:"description"` + Validation Validation `json:"validation"` HTTP HTTPConfig `json:"http"` DNSSD DNSSDConfig `json:"dnssd"` Storage StorageConfig `json:"storage"` ServiceCatalog ServiceCatalog `json:"serviceCatalog"` } +type Validation struct { + JSONSchemas []string `json:"jsonSchemas"` +} + type HTTPConfig struct { PublicEndpoint string `json:"publicEndpoint"` BindAddr string `json:"bindAddr"` diff --git a/main.go b/main.go index 5f30f351..037f82cc 100644 --- a/main.go +++ b/main.go @@ -66,16 +66,20 @@ func main() { if err != nil { panic("Error reading config file:" + err.Error()) } - log.Printf("Loaded config file: " + *confPath) + log.Printf("Loaded config file: %s", *confPath) if config.ServiceID == "" { config.ServiceID = uuid.NewV4().String() log.Printf("Service ID not set. Generated new UUID: %s", config.ServiceID) } - log.Print("Loaded schema file: " + *schemaPath) - err = wot.LoadSchema(*schemaPath) - if err != nil { - panic("error loading WoT Thing Description schema: " + err.Error()) + if len(config.Validation.JSONSchemas) > 0 { + err = wot.LoadJSONSchemas(config.Validation.JSONSchemas) + if err != nil { + panic("error loading validation JSON Schemas: " + err.Error()) + } + log.Printf("Loaded JSON Schemas: %v", config.Validation.JSONSchemas) + } else { + log.Printf("Warning: No configuration for JSON Schemas. TDs will not be validated.") } // Setup API storage diff --git a/sample_conf/thing-directory.json b/sample_conf/thing-directory.json index f6f875e3..ce78e394 100644 --- a/sample_conf/thing-directory.json +++ b/sample_conf/thing-directory.json @@ -1,5 +1,8 @@ { "description": "LinkSmart Thing Directory", + "validation": { + "jsonSchemas": [] + }, "storage": { "type": "leveldb", "dsn": "./data" diff --git a/wot/discovery_schema.json b/wot/discovery_schema.json new file mode 100644 index 00000000..7e0d1c2c --- /dev/null +++ b/wot/discovery_schema.json @@ -0,0 +1,32 @@ +{ + "title": "WoT Discovery TD-extensions Schema - 21 May 2021", + "description": "JSON Schema for validating TD instances with WoT Discovery extensions", + "$schema ": "http://json-schema.org/draft/2019-09/schema#", + "type": "object", + "properties": { + "registration": { + "type": "object", + "properties": { + "created": { + "type": "string", + "format": "date-time" + }, + "expires": { + "type": "string", + "format": "date-time" + }, + "retrieved": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + }, + "ttl": { + "type": "number" + } + } + } + } +} diff --git a/wot/discovery_validation.go b/wot/discovery_validation.go deleted file mode 100644 index aed7e7e7..00000000 --- a/wot/discovery_validation.go +++ /dev/null @@ -1,61 +0,0 @@ -package wot - -import ( - "fmt" - - "github.com/xeipuuv/gojsonschema" -) - -const DiscoverySchema = ` -{ - "type":"object", - "properties":{ - "registration":{ - "type":"object", - "properties":{ - "created":{ - "type":"string", - "format":"date-time" - }, - "expires":{ - "type":"string", - "format":"date-time" - }, - "retrieved":{ - "type":"string", - "format":"date-time" - }, - "modified":{ - "type":"string", - "format":"date-time" - }, - "ttl":{ - "type":"number" - } - } - } - } -} -` - -func ValidateDiscoveryExtensions(td *map[string]interface{}) ([]ValidationError, error) { - schema, err := gojsonschema.NewSchema(gojsonschema.NewStringLoader(DiscoverySchema)) - if err != nil { - return nil, fmt.Errorf("error loading schema: %s", err) - } - - result, err := schema.Validate(gojsonschema.NewGoLoader(td)) - if err != nil { - return nil, err - } - - if !result.Valid() { - var issues []ValidationError - for _, re := range result.Errors() { - issues = append(issues, ValidationError{Field: re.Field(), Descr: re.Description()}) - } - return issues, nil - } - - return nil, nil -} diff --git a/wot/validation.go b/wot/validation.go index 3c02cece..c053ca35 100644 --- a/wot/validation.go +++ b/wot/validation.go @@ -7,33 +7,47 @@ import ( "github.com/xeipuuv/gojsonschema" ) -var schema *gojsonschema.Schema +type jsonSchema = *gojsonschema.Schema -// LoadSchema loads the schema into the package -func LoadSchema(path string) error { - if schema != nil { - // already loaded - return nil - } +var loadedJSONSchemas []jsonSchema +// ReadJSONSchema reads the a JSONSchema from a file +func readJSONSchema(path string) (jsonSchema, error) { file, err := ioutil.ReadFile(path) if err != nil { - return fmt.Errorf("error reading file: %s", err) + return nil, fmt.Errorf("error reading file: %s", err) } - schema, err = gojsonschema.NewSchema(gojsonschema.NewBytesLoader(file)) + schema, err := gojsonschema.NewSchema(gojsonschema.NewBytesLoader(file)) if err != nil { - return fmt.Errorf("error loading schema: %s", err) + return nil, fmt.Errorf("error loading schema: %s", err) } - return nil + return schema, nil } -// ValidateMap validates the input against the loaded WoT Thing Description schema -func ValidateMap(td *map[string]interface{}) ([]ValidationError, error) { - if schema == nil { - return nil, fmt.Errorf("WoT Thing Description schema is not loaded") +// LoadJSONSchemas loads one or more JSON Schemas into memory +func LoadJSONSchemas(paths []string) error { + if len(loadedJSONSchemas) != 0 { + panic("Unexpected re-loading of JSON Schemas.") + } + var schemas []jsonSchema + for _, path := range paths { + schema, err := readJSONSchema(path) + if err != nil { + return err + } + schemas = append(schemas, schema) } + loadedJSONSchemas = schemas + return nil +} + +// LoadedJSONSchemas checks whether any JSON Schema has been loaded into memory +func LoadedJSONSchemas() bool { + return len(loadedJSONSchemas) > 0 +} +func validateAgainstSchema(td *map[string]interface{}, schema jsonSchema) ([]ValidationError, error) { result, err := schema.Validate(gojsonschema.NewGoLoader(td)) if err != nil { return nil, err @@ -49,3 +63,22 @@ func ValidateMap(td *map[string]interface{}) ([]ValidationError, error) { return nil, nil } + +func validateAgainstSchemas(td *map[string]interface{}, schemas ...jsonSchema) ([]ValidationError, error) { + var validationErrors []ValidationError + for _, schema := range schemas { + result, err := validateAgainstSchema(td, schema) + if err != nil { + return nil, err + } + validationErrors = append(validationErrors, result...) + } + + return validationErrors, nil +} + +// ValidateTD performs input validation using one or more pre-loaded JSON Schemas +// If no schema has been pre-loaded, the function returns as if there are no validation errors +func ValidateTD(td *map[string]interface{}) ([]ValidationError, error) { + return validateAgainstSchemas(td, loadedJSONSchemas...) +} diff --git a/wot/validation_test.go b/wot/validation_test.go index 371b5fcc..73d12ee1 100644 --- a/wot/validation_test.go +++ b/wot/validation_test.go @@ -1,8 +1,11 @@ package wot import ( + "io/ioutil" "os" "testing" + + "github.com/xeipuuv/gojsonschema" ) const ( @@ -10,18 +13,38 @@ const ( defaultSchemaPath = "../wot/wot_td_schema.json" ) -func TestLoadSchema(t *testing.T) { +func TestLoadSchemas(t *testing.T) { + if !LoadedJSONSchemas() { + path := os.Getenv(envTestSchemaPath) + if path == "" { + path = defaultSchemaPath + } + err := LoadJSONSchemas([]string{path}) + if err != nil { + t.Fatalf("error loading WoT Thing Description schema: %s", err) + } + } + if len(loadedJSONSchemas) == 0 { + t.Fatalf("JSON Schema was not loaded into memory") + } +} + +func TestValidateAgainstSchema(t *testing.T) { path := os.Getenv(envTestSchemaPath) if path == "" { path = defaultSchemaPath } - err := LoadSchema(path) + + // load the schema + file, err := ioutil.ReadFile(path) if err != nil { - t.Fatalf("error loading WoT Thing Description schema: %s", err) + t.Fatalf("error reading file: %s", err) + } + schema, err := gojsonschema.NewSchema(gojsonschema.NewBytesLoader(file)) + if err != nil { + t.Fatalf("error loading schema: %s", err) } -} -func TestValidateMap(t *testing.T) { t.Run("non-URI ID", func(t *testing.T) { var td = map[string]any{ "@context": "https://www.w3.org/2019/wot/td/v1", @@ -35,7 +58,7 @@ func TestValidateMap(t *testing.T) { }, }, } - results, err := ValidateMap(&td) + results, err := validateAgainstSchema(&td, schema) if err != nil { t.Fatalf("internal validation error: %s", err) } @@ -57,7 +80,7 @@ func TestValidateMap(t *testing.T) { }, }, } - results, err := ValidateMap(&td) + results, err := validateAgainstSchema(&td, schema) if err != nil { t.Fatalf("internal validation error: %s", err) } @@ -65,4 +88,30 @@ func TestValidateMap(t *testing.T) { t.Fatalf("Didn't return error on missing mandatory title.") } }) + + // TODO test discovery validations + //t.Run("non-float TTL", func(t *testing.T) { + // var td = map[string]any{ + // "@context": "https://www.w3.org/2019/wot/td/v1", + // "id": "urn:example:test/thing1", + // "title": "example thing", + // "security": []string{"basic_sc"}, + // "securityDefinitions": map[string]any{ + // "basic_sc": map[string]string{ + // "in": "header", + // "scheme": "basic", + // }, + // }, + // "registration": map[string]any{ + // "ttl": "60", + // }, + // } + // results, err := validateAgainstSchema(&td, schema) + // if err != nil { + // t.Fatalf("internal validation error: %s", err) + // } + // if len(results) == 0 { + // t.Fatalf("Didn't return error on string TTL.") + // } + //}) }