Merge pull request #9 from seralf/master

added pre-built docker image for [DAF-126] POC

Author: seralf

.gitignore

@@ -1,2 +1,9 @@
 *.class
 *.log
+target/
+bin/
+.project
+.classpath
+.settings/
+.cache-main
+.cache-tests

README.md

@@ -1,2 +1,28 @@
-# daf-semantics
-Daf Semantics repository
+
+daf-semantics
+====================
+
+The Daf Semantics repository collects some different components, designed for integrating ontologies, RDF data and to provide some different "semantic" functionalities to the [DAF](https://github.com/italia/daf) platform.
+
+The [semantic_manager]() component exposes the central access point for a subset of the microservices' functionalities:
+<img src="./docs/semantic_manager-v4.png" alt="semantic_manager" width="60%" height="auto">
+
+The planned components are:
+
++ [***semantic_frontend***](https://github.com/seralf/daf-semantics/tree/master/semantic_frontend): 
+the front end for the OntoPA catalog [TODO]
++ [***semantic_manager***](https://github.com/seralf/daf-semantics/tree/master/semantic_manager): 
+the main interface between DAF and the daf-semantics microservices [WIP]
++ [***ontonethub***](https://github.com/seralf/teamdigitale/ontonethub): 
+a component providing indexing/search capabilities for the catalog [WIP]
++ [***semantic_repository***](https://github.com/seralf/daf-semantics/tree/master/semantic_repository): 
+an abstraction over different triplestores [WIP]
++ [***semantic_validator***](https://github.com/seralf/daf-semantics/tree/master/semantic_validator): 
+a component for validating an ontology over DCAT-AP_IT standard [WIP]
++ [***semantic_standardization***](https://github.com/seralf/daf-semantics/tree/master/semantic_standardization): 
+a component exposing vocabulary data and hierarchies, useful for simple standardization [POC]
++ [***semantic_spreadsheet***](https://github.com/seralf/daf-semantics/tree/master/semantic_spreadsheet): 
+a repository collecting recipes for creating RDF data from spreadsheets, using google refine [WIP]
++ [***semantic_mapping***](#): 
+a component for mapping of incoming data (typically in CSV) to RDF, using W3C standards [TODO]
+

docs/semantic_manager-v4.png

@@ -0,0 +1 @@
+<mxfile userAgent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36" version="7.5.0" editor="www.draw.io" type="github"><diagram id="277c755e-be6f-885f-7c0a-642f4f8962d8" name="Page-1">7Vtbd6I6FP41PtolRNA+9ubMOauzZk4765zpY4SImQJxhVjt/PqzgYRbIrUWtTf7UNiEEPa3v30DeugiWn/heDH/xnwS9uyBv+6hy55tW7Y1hH+p5DGXOKdWLgg49eWgUnBL/xApHEjpkvokqQ0UjIWCLupCj8Ux8URNhjlnq/qwGQvrV13ggGiCWw+HuvQ/6ot5Lh3bo1L+ldBgrq5suaf5kSn27gPOlrG8Xs9Gs+yXH46wmkveaDLHPltVROiqhy44YyLfitYXJEx1q9SWnzfZcLRYNyex2OYEWy5DPKpbJz5oQu4yLuYsYDEOr0rpeXZ7JJ1gAHtzEYWwacEmXJM//pLybOcu3TlxYDcRmIuzFBoQxSwmSjahYVic4jdGgKRy/DcR4lEaC14KBqJyhdeMLeQ6EsHZPblgIePZPaFh9iuOKEjTsbrCpA4TtuSe1Im0ZlhuQNQoCWCqrsp5UstfCIsI3D8M4CTEgj7UDQtL+wyKcSVGsCFhMkOGPiHbDbLx0RBzOoVsTcWvAqR0764JpqWD6eNkns1mvQSV8+zPiMqMxcI0cle0kH0suORiHnC4lJPekgjHgnog/YZjcP5cA3Q1p4LcLnB2RyuIinXQcLLI49SMrlMYGvAWYWBg0rqb/TSt2+m8IQ1i2PZAu7AqgAG4Vzl15qR/bTA8EC7IulXD8qiNpM+TQd1y5f6qEiKlaF6Jjkr2DEzY9HeqLnsQ4ikJVSjNJsH5HgXtrWkcgAgMD244yOcrUHQq9yIBfnUucxoy7/7nnMY1v/kihk4mCH5GW6m5AOVF1HZ+DyPHYEJjj3jetkweG0Kl1TWT5ak/GI1FaZ+OCo3KPlHD8PJ1yrNK2wM08GNl2CIdkGy+josa1xk3cqwnxg+HTsP08xWURCh0YuIG7Ep6mN2Xq7mvjVypEAJ8gKjbOScJ/YOn2YDUbqVaYLRz3nMuTc4n9SUUsugzeSCivp+RKaPxeZEbV60rz45N1uVscFBF2i8XV0udTY6rD4ZtIWmZLzSyUQ3K/mn9fDabJUT0mo6tFU8NwNE+soU6z22nxZHZmier+qmqO7P27s5emgaa3BE6VmIx1pj5HegYE/F1OT1oQtGiVY3THSQOyG04THtviYOm89NWMskQ/Uz2tJFHy7XfdBWlqtwaf45WR6nVmDPzG7JgCRUsndMN02A2hTTdDURxsx+DXYWaDsAuldpVEWGhXh2Bwhbppo8FTgCi1MSfgOAVa3zUSDSHBo2PDBofdaHx9o7dtg6tpXlQ8W93hcLfsg9DBh/WeUmyNX5IZ4zA8ZSF79lHjVTQOEIGoJ5FfITq/8VccXSuDI9GFVUAllS5ZlBd6uH9e17VUpLokUdTRAVMFZaidZA+xzqZhWzlzQGwk2gZCtr3mbeMMn0Zo1UL4TQSbYaqA3qNm40PR2fXWGeX2wW73CPEo/edYNudF6gb+mVO3WxQ0WOVU+SL0vplO7Q0rG57Gm+i9mrpxG5nGgfpXezWAR01Sg7niQ5oY7x6crBjB1Q3L72zMiM+4aACFh+5GmwxAy1I1Iy3g7gwbOJ0yNLwVE90f5zd/HOtaX+n1jMwMclJWc1ZQzJLp2r2oUXK0vMEAKZx8DOjbL+j0Os0VYx0FbsGFdsdqLh4iaQTr/r2s9fiVaHXkL3aemvkhkRMkM/09Yn01R4cLn1VKU8FpQfKxZIlTMfi9TSxug8U9focGerzfXW0bL0jcnM5Gf6t84TG/QgoxPWnl68Img7QQO4R0WjvlnzWc89P2pWP2Xs9N2jUc4Pt3n/QJhqOG/bX9LTdFYao2+acsr6K7TUe1725V+NsQ0vueO/G2XpPrvII7l/whuCCGf/YT+BMz4P2VWYZXir4jJ41NJwDPp1D7RXZZ/R8vq/r/rXtDUHPakRPa7ug9+REGs03hOFdome33wI8+2WX7E2xQ0bM7eqeqkkhQ0/geB8C6D2BCeORKVrmEp8+KNFfcUCSaj8TLlY5/J7jqyp/irz2YNFVJVrmdGfCwUzBdX4g3e8xtYHd8ru33BmWHxeiq/8B</diagram></mxfile>

docs/semantic_manager-v4.xml

@@ -17,6 +17,15 @@ import scala.util._
 
 import javax.inject._
 
+import play.api.mvc.{Action,Controller}
+import play.api.data.validation.Constraint
+import play.api.i18n.MessagesApi
+import play.api.inject.{ApplicationLifecycle,ConfigurationProvider}
+import de.zalando.play.controllers._
+import PlayBodyParsing._
+import PlayValidations._
+import scala.util._
+import javax.inject._
 import play.api.libs.ws.WSClient
 import utilities.JSONHelper
 import scala.concurrent.ExecutionContext.Implicits._
@@ -40,6 +49,9 @@ import OntonetHubClient.models._
 import OntonetHubClient.models._
 import OntonetHubClient.models._
 import OntonetHubClient.models._
+import OntonetHubClient.models._
+import OntonetHubClient.models._
+import OntonetHubClient.models._
 
 /**
  * This controller is re-generated after each change in the specification.
@@ -48,7 +60,7 @@ import OntonetHubClient.models._
 
 package semantic_manager.yaml {
     // ----- Start of unmanaged code area for package Semantic_managerYaml
-                    
+                                
     // ----- End of unmanaged code area for package Semantic_managerYaml
     class Semantic_managerYaml @Inject() (
         // ----- Start of unmanaged code area for injections Semantic_managerYaml

semantic_manager/app/generated_controllers/semantic_manager.yaml.scala

@@ -317,7 +317,7 @@ import java.net.URI
 
 package semantic_repository.yaml {
     // ----- Start of unmanaged code area for package Semantic_repositoryYaml
-                        
+                            
     // ----- End of unmanaged code area for package Semantic_repositoryYaml
     class Semantic_repositoryYaml @Inject() (
         // ----- Start of unmanaged code area for injections Semantic_repositoryYaml

semantic_repository/app/generated_controllers/semantic_repository.yaml.scala

@@ -12,10 +12,111 @@ Two endpoints are provided:
 
 The idea is that each endpoint (and its configured queries) acts for a very specific domain, so the next versions could introduce new vocabularies and ontologies, but needs to create ad-hoc SPARQL queries for retrieving the informations needed.
 
+## semantic annotation in DAF ingestion
 
-## example: retrieving a vocabulary dataset
+The [DAF](https://github.com/italia/daf) `semantic_annotation` has currently the following structure: `{ontology}.{concept}.{property}`.
+During the ingestion phase of datasets in DAF platform a `semantic_annotation` is used, in order to relate some column of a dataset to the most appropriate property of a given existing concept, from the controlled vocabularies.
+
+**Note** that while the annotation is used to relate cells with vocabularies, it does not save explicitly a reference to the vocabularies used. A reference to concept from an ontology is used instead.
+
+
+## examples
+
+
+### example: sequence of calls
+
+1. retrieves (vocabulary,ontology) reference from semantic_annotation tag
+```
+curl -X GET http://localhost:9000/kb/v1/daf/annotation/lookup?semantic_annotation=POI-AP_IT.PointOfInterestCategory.POIcategoryIdentifier -H  "accept: application/json" -H  "content-type: application/json"
+```
+
+2. retrieves the hierarchies for a given property
+```
+curl -X GET http://localhost:9000/kb/v1/hierarchies/properties?vocabulary_name=POICategoryClassification&ontology_name=poiapit&lang=it -H  "accept: application/json" -H  "content-type: application/json"
+```
+
+3. retrieves the dataset values for a certain vocaulary
+```
+curl -X GET http://localhost:9000/kb/v1/vocabularies/POICategoryClassification?lang=it -H  "accept: application/json" -H  "content-type: application/json"
+```
+
+----
+
+### example: retrieves informations from the semantic_annotation tag
+With this endpoint we can retrieve informations about the vocabulary/ontology pair related to a given `semantic_annotation` tag:
+
+```
+curl -X GET http://localhost:9000/kb/v1/daf/annotation/lookup?semantic_annotation={semantic_annotation} \ 
+-H  "accept: application/json" -H  "content-type: application/json"
+```
+
+for example, for the Point Of Interest vocabulary:
+
+```
+curl -X GET 'http://localhost:9000/kb/v1/daf/annotation/lookup?semantic_annotation=POI-AP_IT.PointOfInterestCategory.POIcategoryIdentifier' \
+-H  "accept: application/json" -H  "content-type: application/json"
+```
+
+This will return a datastructure similar to the following one for each tag:
+
+```
+[
+  {
+    "vocabulary_id": "POICategoryClassification",
+    "vocabulary": "http://dati.gov.it/onto/controlledvocabulary/POICategoryClassification",
+    "ontology": "http://dati.gov.it/onto/poiapit",
+    "semantic_annotation": "POI-AP_IT.PointOfInterestCategory.POIcategoryIdentifier",
+    "property_id": "POIcategoryIdentifier",
+    "concept_id": "PointOfInterestCategory",
+    "ontology_prefix": "poiapit",
+    "ontology_id": "POI-AP_IT",
+    "concept": "http://dati.gov.it/onto/poiapit#PointOfInterestCategory",
+    "property": "http://dati.gov.it/onto/poiapit#POIcategoryIdentifier"
+  }
+]
+```
+
+the idea is to be able to have as much informations as possible to eventually relate the annotation to ontologies and vocabularies.
+
+
+### example: retrieving a vocabulary dataset
 
 We can obtain a de-normalized, tabular version of the vocabulary `Istat-Classificazione-08-Territorio` using the curl call:
+
+```
+curl -X GET http://localhost:9000/kb/v1/hierarchies/properties?vocabulary_name={vocabulary_name}&ontology_name={ontology_prefix}&lang={lang} \ 
+-H  "accept: application/json" -H  "content-type: application/json"
+```
+
+A `SPARQL` query is used to create a proper tabular representation of the data.
+
+#### example: PontOfInterest / POI_AP-IT
+
+```
+curl -X GET http://localhost:9000/kb/v1/hierarchies/properties?vocabulary_name=POICategoryClassification&ontology_name=poiapit&lang=it -H  "accept: application/json" -H  "content-type: application/json"
+```
+
+this will return a data structure:
+
+```
+[
+  {
+    "vocabulary": "POI-AP_IT",
+    "path": "POI-AP_IT.PointOfInterestCategory.definition",
+    "hierarchy_flat": "PointOfInterestCategory",
+    "hierarchy": [
+      {
+        "class": "PointOfInterestCategory",
+        "level": 0
+      }
+    ]
+  },
+  ...
+]
+```
+
+
+#### example: Luoghi Istat / CLV_AP-IT
 ```
 $ curl -X GET "http://localhost:9000/kb/v1/vocabularies/Istat-Classificazione-08-Territorio?lang=it" -H  "accept: application/json" -H  "content-type: application/json"
 ```
@@ -39,12 +140,52 @@ this will return a result structure similar to the following one:
 ]
 ```
 
-## example: retrieve th hierarchies for the properties used 
+For technical reason, currently a value of `CLV-AP_IT_Region_name` is used in place of `CLV-AP_IT.Region.name`.
+
+### example: retrieve the hierarchies for the properties used 
 
 If we have the example vocabulary `Istat-Classificazione-08-Territorio`, which uses terms from the ontology `clvapit`, we can retrieve the local hierarchy associated to each property with the curl command:
 
 ```
-$ curl -X GET http://localhost:9000/kb/v1/hierarchies/properties?vocabulary_name=Istat-Classificazione-08-Territorio&ontology_name=clvapit&lang=it -H  "accept: application/json" -H  "content-type: application/json"
+$ curl -X GET http://localhost:9000/kb/v1/hierarchies/properties?vocabulary_name={vocabulary_name}&ontology_name={ontology_prefix}&lang={lang} \ 
+-H  "accept: application/json" -H  "content-type: application/json"
+```
+
+#### example: POI / POI_AP-IT
+
+```
+curl -X GET http://localhost:9000/kb/v1/vocabularies/POICategoryClassification?lang=it \ 
+-H  "accept: application/json" -H  "content-type: application/json"
+```
+
+which will return results:
+
+```
+[
+  [
+    {
+      "key": "POI-AP_IT_PointOfInterestCategory_definition",
+      "value": "Rientrano in questa categoria tutti i punti di interesse connessi all'intrattenimento come zoo, discoteche, pub, teatri, acquari, stadi, casino, parchi divertimenti, ecc."
+    },
+    {
+      "key": "POI-AP_IT_PointOfInterestCategory_POICategoryName",
+      "value": "Settore intrattenimento"
+    },
+    {
+      "key": "POI-AP_IT_PointOfInterestCategory_POICategoryIdentifier",
+      "value": "cat_1"
+    }
+ ],
+ ...
+]
+```
+
+
+#### example: Luoghi Istat / CLV_AP-IT
+
+```
+$ curl -X GET http://localhost:9000/kb/v1/hierarchies/properties?vocabulary_name=Istat-Classificazione-08-Territorio&ontology_name=clvapit&lang=it \ 
+-H  "accept: application/json" -H  "content-type: application/json"
 ```
 
 which will return the results:
@@ -68,7 +209,7 @@ which will return the results:
 ```
 
 
-## example configurations
+### example configurations
 
 An example configuration for working with a vocabulary (VocabularyAPI):
 
@@ -122,8 +263,13 @@ Eventually the idea of pre-loading ontologies and vocabularies from disk can be
 
 ----
 
-TODO:
+## TODO
 
 + more documentation / comments
 + more proper tests
-+ remove redundant classes for RDFRepository, importing external kb-core dependency, instead
++ remove redundant classes for RDFRepository, importing external kb-core dependency, instead
+
+
+## known ISSUES
+
+...