Validate against multiple schemas (#50)

* validate against multiple schemas

* configuration, file loading

* loading schemas into memory

* fix response types

* Add dns-sd types to wot package

* update tests

* Update expiry logic for compliance

Give precedence to ttl for calculation of expiry

fix ttl test, remove retrieval time

* discovery attr validation with json schema

* remove storage type from test log

* Delete directory-td.jsonld

* refactoring to use new validation function

* update tests, readme, remove deprecated code

* add function to check loaded schemas, refactor tests

README.md

@@ -51,6 +51,13 @@ Copy sample configuration files for server and JSON Schema into `conf` directory
 mkdir -p conf
 cp sample_conf/thing-directory.json wot/wot_td_schema.json conf
 ```
+
+Alternatively, download sample config file and schemas:
+```bash
+curl https://raw.githubusercontent.com/linksmart/thing-directory/master/sample_conf/thing-directory.json --create-dirs -o conf/thing-directory.json
+curl https://raw.githubusercontent.com/w3c/wot-thing-description/REC1.0/validation/td-json-schema-validation.json --create-dirs -o conf/wot_td_schema.json
+```
+
 `conf` is the default directory for configuration files. This can be changed with CLI arguments.
 
 Get the CLI argument help (linux/macOS):

catalog/catalog.go

@@ -20,17 +20,11 @@ const (
 )
 
 func validateThingDescription(td map[string]interface{}) ([]wot.ValidationError, error) {
-	issues, err := wot.ValidateDiscoveryExtensions(&td)
+	result, err := wot.ValidateTD(&td)
 	if err != nil {
-		return nil, fmt.Errorf("error validating with JSON schema: %s", err)
+		return nil, fmt.Errorf("error validating with JSON Schemas: %s", err)
 	}
-
-	tdIssues, err := wot.ValidateMap(&td)
-	if err != nil {
-		return nil, fmt.Errorf("error validating with JSON schema: %s", err)
-	}
-
-	return append(issues, tdIssues...), nil
+	return result, nil
 }
 
 // Controller interface

catalog/main_test.go

@@ -27,11 +27,14 @@ var (
 )
 
 func loadSchema() error {
+	if wot.LoadedJSONSchemas() {
+		return nil
+	}
 	path := os.Getenv(envTestSchemaPath)
 	if path == "" {
 		path = defaultSchemaPath
 	}
-	return wot.LoadSchema(path)
+	return wot.LoadJSONSchemas([]string{path})
 }
 
 func serializedEqual(td1 ThingDescription, td2 ThingDescription) bool {

config.go

@@ -17,12 +17,17 @@ import (
 type Config struct {
 	ServiceID      string         `json:"serviceID"`
 	Description    string         `json:"description"`
+	Validation     Validation     `json:"validation"`
 	HTTP           HTTPConfig     `json:"http"`
 	DNSSD          DNSSDConfig    `json:"dnssd"`
 	Storage        StorageConfig  `json:"storage"`
 	ServiceCatalog ServiceCatalog `json:"serviceCatalog"`
 }
 
+type Validation struct {
+	JSONSchemas []string `json:"jsonSchemas"`
+}
+
 type HTTPConfig struct {
 	PublicEndpoint string         `json:"publicEndpoint"`
 	BindAddr       string         `json:"bindAddr"`

main.go

@@ -66,16 +66,20 @@ func main() {
 	if err != nil {
 		panic("Error reading config file:" + err.Error())
 	}
-	log.Printf("Loaded config file: " + *confPath)
+	log.Printf("Loaded config file: %s", *confPath)
 	if config.ServiceID == "" {
 		config.ServiceID = uuid.NewV4().String()
 		log.Printf("Service ID not set. Generated new UUID: %s", config.ServiceID)
 	}
-	log.Print("Loaded schema file: " + *schemaPath)
 
-	err = wot.LoadSchema(*schemaPath)
-	if err != nil {
-		panic("error loading WoT Thing Description schema: " + err.Error())
+	if len(config.Validation.JSONSchemas) > 0 {
+		err = wot.LoadJSONSchemas(config.Validation.JSONSchemas)
+		if err != nil {
+			panic("error loading validation JSON Schemas: " + err.Error())
+		}
+		log.Printf("Loaded JSON Schemas: %v", config.Validation.JSONSchemas)
+	} else {
+		log.Printf("Warning: No configuration for JSON Schemas. TDs will not be validated.")
 	}
 
 	// Setup API storage

sample_conf/thing-directory.json

@@ -1,5 +1,8 @@
 {
   "description": "LinkSmart Thing Directory",
+  "validation": {
+    "jsonSchemas": []
+  },
   "storage": {
     "type": "leveldb",
     "dsn": "./data"

wot/discovery_schema.json

@@ -0,0 +1,32 @@
+{
+  "title": "WoT Discovery TD-extensions Schema - 21 May 2021",
+  "description": "JSON Schema for validating TD instances with WoT Discovery extensions",
+  "$schema ": "http://json-schema.org/draft/2019-09/schema#",
+  "type": "object",
+  "properties": {
+    "registration": {
+      "type": "object",
+      "properties": {
+        "created": {
+          "type": "string",
+          "format": "date-time"
+        },
+        "expires": {
+          "type": "string",
+          "format": "date-time"
+        },
+        "retrieved": {
+          "type": "string",
+          "format": "date-time"
+        },
+        "modified": {
+          "type": "string",
+          "format": "date-time"
+        },
+        "ttl": {
+          "type": "number"
+        }
+      }
+    }
+  }
+}

wot/validation.go

@@ -7,33 +7,47 @@ import (
 	"github.com/xeipuuv/gojsonschema"
 )
 
-var schema *gojsonschema.Schema
+type jsonSchema = *gojsonschema.Schema
 
-// LoadSchema loads the schema into the package
-func LoadSchema(path string) error {
-	if schema != nil {
-		// already loaded
-		return nil
-	}
+var loadedJSONSchemas []jsonSchema
 
+// ReadJSONSchema reads the a JSONSchema from a file
+func readJSONSchema(path string) (jsonSchema, error) {
 	file, err := ioutil.ReadFile(path)
 	if err != nil {
-		return fmt.Errorf("error reading file: %s", err)
+		return nil, fmt.Errorf("error reading file: %s", err)
 	}
 
-	schema, err = gojsonschema.NewSchema(gojsonschema.NewBytesLoader(file))
+	schema, err := gojsonschema.NewSchema(gojsonschema.NewBytesLoader(file))
 	if err != nil {
-		return fmt.Errorf("error loading schema: %s", err)
+		return nil, fmt.Errorf("error loading schema: %s", err)
 	}
-	return nil
+	return schema, nil
 }
 
-// ValidateMap validates the input against the loaded WoT Thing Description schema
-func ValidateMap(td *map[string]interface{}) ([]ValidationError, error) {
-	if schema == nil {
-		return nil, fmt.Errorf("WoT Thing Description schema is not loaded")
+// LoadJSONSchemas loads one or more JSON Schemas into memory
+func LoadJSONSchemas(paths []string) error {
+	if len(loadedJSONSchemas) != 0 {
+		panic("Unexpected re-loading of JSON Schemas.")
+	}
+	var schemas []jsonSchema
+	for _, path := range paths {
+		schema, err := readJSONSchema(path)
+		if err != nil {
+			return err
+		}
+		schemas = append(schemas, schema)
 	}
+	loadedJSONSchemas = schemas
+	return nil
+}
+
+// LoadedJSONSchemas checks whether any JSON Schema has been loaded into memory
+func LoadedJSONSchemas() bool {
+	return len(loadedJSONSchemas) > 0
+}
 
+func validateAgainstSchema(td *map[string]interface{}, schema jsonSchema) ([]ValidationError, error) {
 	result, err := schema.Validate(gojsonschema.NewGoLoader(td))
 	if err != nil {
 		return nil, err
@@ -49,3 +63,22 @@ func ValidateMap(td *map[string]interface{}) ([]ValidationError, error) {
 
 	return nil, nil
 }
+
+func validateAgainstSchemas(td *map[string]interface{}, schemas ...jsonSchema) ([]ValidationError, error) {
+	var validationErrors []ValidationError
+	for _, schema := range schemas {
+		result, err := validateAgainstSchema(td, schema)
+		if err != nil {
+			return nil, err
+		}
+		validationErrors = append(validationErrors, result...)
+	}
+
+	return validationErrors, nil
+}
+
+// ValidateTD performs input validation using one or more pre-loaded JSON Schemas
+// If no schema has been pre-loaded, the function returns as if there are no validation errors
+func ValidateTD(td *map[string]interface{}) ([]ValidationError, error) {
+	return validateAgainstSchemas(td, loadedJSONSchemas...)
+}