Repair addressing #105 (failure to honor 'choice' semantics), still w…

…anting testing; with a small testing application
usnistgov · Jun 5, 2024 · 837a42b · 837a42b
1 parent 048b85a
commit 837a42b
Show file tree

Hide file tree

Showing 28 changed files with 5,445 additions and 21 deletions.
diff --git a/src/common/metaschema-ws-strip.xsl b/src/common/metaschema-ws-strip.xsl
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+   xmlns:xs="http://www.w3.org/2001/XMLSchema"
+
+   xmlns:mx="http://csrc.nist.gov/ns/csd/metaschema-xslt"
+   xmlns:math="http://www.w3.org/2005/xpath-functions/math"
+   exclude-result-prefixes="xs math mx"
+   xmlns="http://csrc.nist.gov/ns/oscal/metaschema/1.0"
+   xpath-default-namespace="http://csrc.nist.gov/ns/oscal/metaschema/1.0"
+   version="3.0">
+
+   <xsl:strip-space elements="*"/>
+
+   <xsl:preserve-space elements="p h1 h2 h3 h4 h5 h6 pre li td
+      a insert code em i b strong sub sup"/>
+
+   <xsl:mode on-no-match="shallow-copy"/>
+
+   <xsl:function name="mx:scrub-up" as="element()">
+      <xsl:param name="in" as="node()"/>
+      <xsl:apply-templates select="$in"/>
+   </xsl:function>
+
+</xsl:stylesheet>
diff --git a/src/schema-gen/JSON-schema/Makefile b/src/schema-gen/JSON-schema/Makefile
@@ -22,7 +22,7 @@ test: unit-test spec-test ## Run all tests
 
 .PHONY: spec-test
 spec-test: ## Run all specification tests
-	LOGFILE="$(output_folder)/inspector-functional-tests.log" $(xspec_ci_script) \
+	LOGFILE="$(output_folder)/jsonschemagen-mapping.log" $(xspec_ci_script) \
 		"folder=$(module_path)/testing/tests/jsonschemagen-mapping-xspec/" \
 		"report-to=$(output_folder)/jsonschemagen-mapping_report.html" \
 		"junit-to=$(output_folder)/jsonschemagen-mapping_junit-report.xml" \
@@ -31,7 +31,7 @@ spec-test: ## Run all specification tests
 
 .PHONY: unit-test
 unit-test: ## Run all unit tests
-	LOGFILE="$(output_folder)/inspector-generation-tests.log" $(xspec_ci_script) \
+	LOGFILE="$(output_folder)/jsonschemagen-runtime.log" $(xspec_ci_script) \
 		"folder=$(module_path)/testing/tests/jsonschemagen-runtime-xspec" \
 		"report-to=$(output_folder)/jsonschemagen-runtime_report.html" \
 		"junit-to=$(output_folder)/jsonschemagen-runtime_junit-report.xml" \

diff --git a/src/schema-gen/JSON-schema/choice-split.xsl b/src/schema-gen/JSON-schema/choice-split.xsl
@@ -10,7 +10,7 @@
     <!-- Purpose: Provide splitting logic for choices, rendering a model sequence containing choices as a choice of model sequences. -->
 
     <!-- Note: this XSLT will only be used on its own for development and debugging.
-        It is however imported by `produce-json-converter.xsl` and possibly other stylesheets. -->
+        It is however imported by `make-json-schema-metamap.xsl` and possibly other stylesheets. -->
 
    <xsl:strip-space elements="*"/>
 

diff --git a/src/schema-gen/JSON-schema/make-json-schema-metamap.xsl b/src/schema-gen/JSON-schema/make-json-schema-metamap.xsl
@@ -146,7 +146,7 @@
         </map>
     </xsl:template>
 
-    <xsl:template match="define-assembly">
+    <xsl:template match="define-assembly" mode="old">
         <xsl:apply-templates select="formal-name, description"/>
         <xsl:call-template name="give-id"/>
         <string key="type">object</string>
@@ -156,7 +156,68 @@
             </map>
         </xsl:where-populated>
         <xsl:call-template name="require-or-allow"/>
-    </xsl:template>
+
+       <!--split out model into newmodels (sans choice)
+       if count(newmodels) gt 1 then anyOf / for-each / map
+       if count(newmodels eq 0 then map with properties-->
+
+
+   </xsl:template>
+
+   <xsl:import href="choice-split.xsl"/>
+
+   <xsl:template match="define-assembly">
+      <xsl:variable name="flags-here" select="flag | define-flag"/>
+      <xsl:variable name="my-model" select="model"/>
+
+      <xsl:apply-templates select="formal-name, description"/>
+      <xsl:call-template name="give-id"/>
+      <string key="type">object</string>
+
+      <xsl:variable name="split-models" as="element(model)*">
+         <xsl:apply-templates select="model" mode="splitting"/>
+      </xsl:variable>
+      <xsl:choose>
+         <xsl:when test="count($split-models) gt 1">
+            <array key="anyOf">
+               <xsl:for-each select="$split-models">
+                  <map>
+                     <map key="properties">
+                        <xsl:apply-templates mode="define" select="$flags-here"/>
+                        <xsl:apply-templates mode="define" select="."/>
+                     </map>
+                     <xsl:call-template name="require-or-allow">
+                        <xsl:with-param name="with-model" select="."/>
+                     </xsl:call-template>
+                  </map>
+               </xsl:for-each>
+            </array>
+         </xsl:when>
+         <xsl:when test="count($split-models) eq 1">
+            <map key="properties">
+               <xsl:apply-templates mode="define" select="$flags-here"/>
+               <xsl:apply-templates mode="define" select="$my-model"/>
+            </map>
+            <xsl:call-template name="require-or-allow"/>
+
+         </xsl:when>
+         <xsl:otherwise>
+            <xsl:call-template name="require-or-allow"/>
+         </xsl:otherwise>
+      </xsl:choose>
+
+
+
+      <!--<xsl:apply-templates mode="define" select="$flags-here"/>
+       <xsl:apply-templates mode="define" select="."/>
+       <xsl:apply-templates mode="define" select="$flags-here"/>
+       
+       model -> flatmodel+
+       for each flatmodel
+       do xsl:apply-templates mode="define" select="flag | define-flag | model"/>-->
+   </xsl:template>
+
+
 
     <xsl:template match="define-field">
             <xsl:apply-templates select="formal-name, description"/>
@@ -167,6 +228,7 @@
                     <xsl:apply-templates select="." mode="properties"/>
                 </map>
             </xsl:where-populated>
+
             <xsl:call-template name="require-or-allow"/>
     </xsl:template>
 
@@ -281,6 +343,8 @@
     </xsl:template>
 
     <xsl:template name="require-or-allow">
+<!-- $with-model allows a proxy 'model' when choices are split out      -->
+       <xsl:param name="with-model" select="child::model"/>
         <xsl:variable name="requirements" as="element()*">
             <!-- A value string is always required except on empty fields -->
             <xsl:variable name="value-property">
@@ -296,7 +360,7 @@
             <xsl:apply-templates mode="property-name"
                 select="flag[@required = 'yes'][not(@ref = $implicit-flags)] |
                         define-flag[@required = 'yes'][not(@name = $implicit-flags)] |
-                        model/(field|define-field|assembly|define-assembly)[@min-occurs &gt; 0]"/>
+                        $with-model/(field|define-field|assembly|define-assembly)[@min-occurs &gt; 0]"/>
         </xsl:variable> 
         <xsl:if test="exists( $requirements )">
             <array key="required">
@@ -305,7 +369,7 @@
         </xsl:if>
         <boolean key="additionalProperties">
             <xsl:choose>
-                <xsl:when test="exists(json-value-key-flag | model/(.|choice)/any)">true</xsl:when>
+                <xsl:when test="exists(json-value-key-flag | $with-model/(.|choice)/any)">true</xsl:when>
                 <xsl:otherwise>false</xsl:otherwise>
             </xsl:choose>
         </boolean>
@@ -425,8 +489,8 @@
     <xsl:template mode="define" priority="5"
         match="define-assembly[group-as/@in-json='BY_KEY'][exists(json-key)] |
         define-field[group-as/@in-json='BY_KEY'][exists(json-key)] |
-        assembly[group-as/@in-json='BY_KEY'][exists(key('assembly-definition-by-name',@_key-ref)/json-key)] |
-        field[group-as/@in-json='BY_KEY'][exists(key('field-definition-by-name',@_key-ref)/json-key)]">
+        assembly[group-as/@in-json='BY_KEY'][exists(key('assembly-definition-by-name',@_key-ref, $composed-metaschema)/json-key)] |
+        field[group-as/@in-json='BY_KEY'][exists(key('field-definition-by-name', @_key-ref, $composed-metaschema)/json-key)]">
         <xsl:variable name="group-name" select="group-as/@name"/>
         <map key="{ $group-name }">
             <string key="type">object</string>
@@ -444,7 +508,7 @@
     <!-- Always a map when max-occurs is 1 or implicit -->
     <xsl:template mode="define" priority="4"
         match="assembly[empty(@max-occurs) or number(@max-occurs) = 1] | define-assembly[empty(@max-occurs) or number(@max-occurs) = 1]">
-        <xsl:variable name="decl" select="key('assembly-definition-by-name', @_key-ref) | self::define-assembly"/>
+        <xsl:variable name="decl" select="key('assembly-definition-by-name', @_key-ref, $composed-metaschema) | self::define-assembly"/>
         <map key="{ (use-name,$decl/use-name,$decl/@name)[1] }">
             <xsl:apply-templates select="." mode="definition-or-reference"/>
         </map>
@@ -453,7 +517,7 @@
     <!-- Always a map when max-occurs is 1 or implicit -->
     <xsl:template mode="define" priority="4"
         match="field[empty(@max-occurs) or number(@max-occurs) = 1] | define-field[empty(@max-occurs) or number(@max-occurs) = 1]">
-        <xsl:variable name="decl" select="key('field-definition-by-name', @_key-ref) | self::define-field"/>
+       <xsl:variable name="decl" select="key('field-definition-by-name', @_key-ref, $composed-metaschema) | self::define-field"/>
         <map key="{ (use-name,$decl/use-name,$decl/@name)[1] }">
             <xsl:apply-templates select="." mode="definition-or-reference"/>
         </map>
@@ -509,17 +573,17 @@
     </xsl:template>
 
     <xsl:template match="flag" mode="definition-or-reference">
-        <xsl:variable name="definition" select="key('flag-definition-by-name',@_key-ref)"/>
+        <xsl:variable name="definition" select="key('flag-definition-by-name', @_key-ref, $composed-metaschema)"/>
         <xsl:apply-templates select="$definition" mode="make-ref"/>
     </xsl:template>
 
     <xsl:template match="field" mode="definition-or-reference">
-        <xsl:variable name="definition" select="key('field-definition-by-name',@_key-ref)"/>
+       <xsl:variable name="definition" select="key('field-definition-by-name', @_key-ref, $composed-metaschema)"/>
         <xsl:apply-templates select="$definition" mode="make-ref"/>
     </xsl:template>
 
     <xsl:template match="assembly" mode="definition-or-reference">
-        <xsl:variable name="definition" select="key('assembly-definition-by-name',@_key-ref)"/>
+       <xsl:variable name="definition" select="key('assembly-definition-by-name', @_key-ref, $composed-metaschema)"/>
         <xsl:apply-templates select="$definition" mode="make-ref"/>
     </xsl:template>
 
@@ -550,8 +614,8 @@
             <xsl:when test="exists(@as-type)">
                 <xsl:next-match/>
             </xsl:when>
-            <xsl:when test="exists(key('field-definition-by-name',@_key-ref)/@as-type)">
-                <xsl:apply-templates mode="#current" select="key('field-definition-by-name',@_key-ref)"/>
+            <xsl:when test="exists(key('field-definition-by-name',@_key-ref, $composed-metaschema)/@as-type)">
+                <xsl:apply-templates mode="#current" select="key('field-definition-by-name',@_key-ref, $composed-metaschema)"/>
             </xsl:when>
             <xsl:otherwise>
                 <string key="type">string</string> 

diff --git a/src/schema-gen/JSON-schema/testing/tests/jsonschemagen-mapping-xspec/metamap-test.xspec b/src/schema-gen/JSON-schema/testing/tests/jsonschemagen-mapping-xspec/metamap-test.xspec
@@ -2,10 +2,17 @@
 <?xml-model href="../../../../../testing/xspec-assurance.sch"?>
 <?xml-model href="../../../../../util/xspec-fixup.sch"?>
 <x:description xmlns:mx="http://csrc.nist.gov/ns/csd/metaschema-xslt"
-   xmlns="http://example.com/ns/computer"
-   xmlns:x="http://www.jenitennison.com/xslt/xspec" xmlns:xs="http://www.w3.org/2001/XMLSchema"
-   stylesheet="../../../make-json-schema-metamap.xsl" xslt-version="3.0">
+   xmlns:m="http://csrc.nist.gov/ns/oscal/metaschema/1.0"
+   xmlns="http://csrc.nist.gov/ns/metaschema-xslt/tinydata"
+   xmlns:t="http://csrc.nist.gov/ns/metaschema-xslt/tinydata"
+   xmlns:x="http://www.jenitennison.com/xslt/xspec"
+   xmlns:xs="http://www.w3.org/2001/XMLSchema"
+   xmlns:fn="http://www.w3.org/2005/xpath-functions"
+   stylesheet="../../../make-json-schema-metamap.xsl"
+   xslt-version="3.0" run-as="external">
 
+   <x:helper stylesheet="../../../../../common/metaschema-ws-strip.xsl"/>
+
    <!--<x:variable name="mx:makes-a-date" as="function(*)"
       select="function($v as item()) as xs:boolean { $v castable as xs:date }"/>-->
 
@@ -21,4 +28,24 @@
       </x:scenario>
    </x:scenario>
 
+<!-- Tests following use as input the testing metaschema ../../../../../testing/tinydata/tiny_metaschema.xml
+   with static artifacts produced for the directory `current` in that folder
+   
+   refresh these artifacts with the script ../../../../../testing/tinydata/refresh-tiny-schemas.sh
+   -->
+
+   <!--
+   <map xmlns="http://www.w3.org/2005/xpath-functions"
+      xmlns:nm="http://csrc.nist.gov/ns/metaschema"
+      key="tiny-tiny:sheet">
+      -->
+
+   <x:scenario label="[B] modeling choice - issue 105">
+      <x:variable name="mx:tiny-composed"    href="../../../../../testing/tinydata/current/tiny_COMPOSED.xml"/>
+      <x:variable name="mx:corrected-schema" href="../../../../../testing/tinydata/tiny_json-schema-corrected.xml"/>
+      <x:scenario label="An assembly from the JSON Schema produced from the Tiny model, containing 'choice'">
+         <x:context select="$mx:tiny-composed/*/m:define-assembly[@name='sheet']"/>
+         <x:expect label="Looks like it does in the handmade target model" test="mx:scrub-up($x:result)" select="$mx:corrected-schema/descendant::fn:map[@key='tiny-tiny:sheet'] => mx:scrub-up()"/>
+      </x:scenario>
+   </x:scenario>
 </x:description>
diff --git a/src/schema-gen/METASCHEMA-ALL-VALIDATORS.xpl b/src/schema-gen/METASCHEMA-ALL-VALIDATORS.xpl
@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<p:declare-step xmlns:p="http://www.w3.org/ns/xproc"
+  xmlns:c="http://www.w3.org/ns/xproc-step" version="1.0"
+  xmlns:metaschema="http://csrc.nist.gov/ns/metaschema/1.0"
+  type="metaschema:METASCHEMA-ALL-VALIDATORS" name="METASCHEMA-ALL-VALIDATORS">
+
+  <!-- &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& -->
+  <!-- Ports -->
+
+  <p:input port="METASCHEMA" primary="true"/>
+  <p:input port="parameters" kind="parameter"/>
+
+  <p:serialization port="INT_0_echo-input" indent="true"  method="xml"/>
+  <p:output port="INT_0_echo-input" primary="false">
+    <p:pipe port="result" step="echo-input"/>
+  </p:output>
+
+  <p:serialization port="INT_1_composed-metaschema" indent="true" method="xml"/>
+  <p:output port="INT_1_composed-metaschema" primary="false">
+    <p:pipe port="result" step="composed"/>
+  </p:output>
+
+  <p:serialization port="OUT_json-schema-xml" indent="true"  method="xml"/>
+  <p:output port="OUT_json-schema-xml" primary="false">
+    <p:pipe port="result" step="make-json-schema-xml"/>
+  </p:output>
+
+  <p:serialization port="OUT_json-schema"     indent="true"  method="text" omit-xml-declaration="false"/>
+  <p:output port="OUT_json-schema" primary="false">
+    <p:pipe port="result" step="serialize-json"/>
+  </p:output>
+
+   <p:serialization port="OUT_xml-schema" indent="true"  method="xml"/>
+   <p:output port="OUT_xml-schema" primary="false">
+      <p:pipe port="result" step="make-xsd"/>
+   </p:output>
+
+   <p:serialization port="OUT_inspector-xsl" indent="true"  method="xml"/>
+   <p:output port="OUT_inspector-xsl" primary="true">
+      <p:pipe port="result" step="make-inspector-xslt"/>
+   </p:output>
+
+   <!-- &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& -->
+  <!-- Import (subpipeline) -->
+
+  <p:import href="../compose/metaschema-compose.xpl"/>
+
+  <!-- &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& &&& -->
+  <!-- Pipeline -->
+
+  <p:identity name="echo-input"/>
+
+  <metaschema:metaschema-compose name="compose"/>
+
+  <p:identity name="composed"/>
+
+  <p:xslt name="make-xsd">
+    <p:input port="stylesheet">
+      <p:document href="XSD/make-metaschema-xsd.xsl"/>
+    </p:input>
+  </p:xslt>
+
+  <p:sink/>
+
+  <p:xslt name="make-json-schema-xml">
+    <p:input port="source">
+      <p:pipe port="result" step="composed"/>
+    </p:input>
+    <p:input port="stylesheet">
+      <p:document href="JSON-schema/make-json-schema-metamap.xsl"/>
+    </p:input>
+  </p:xslt>
+
+  <!--<p:identity name="serialize-json"/>-->
+  <p:xslt name="serialize-json">
+    <p:input port="stylesheet">
+      <p:document href="../common/xpath-json-to-json.xsl"/>
+    </p:input>
+  </p:xslt>
+
+   <p:sink/>
+
+   <p:xslt name="make-inspector-xslt">
+      <p:input port="source">
+         <p:pipe port="result" step="composed"/>
+      </p:input>
+      <p:input port="stylesheet">
+         <p:document href="InspectorXSLT/generator/generate-inspector-xslt.xsl"/>
+      </p:input>
+   </p:xslt>
+
+</p:declare-step>
diff --git a/src/testing/readme.md b/src/testing/readme.md
@@ -21,3 +21,28 @@ Dynamic testing of Metaschema-based data conversions, XML to JSON and back.
 - test-xml-conversions.xpl
 - test-xml-json-roundtrip.xpl
 
+Don't change any of these files without keeping the schema generation testing in place as it depends on them.
+
+## Tinydata model
+
+See the [tinydata](./tinydata) folder for a fully worked example, made to a different set of design requirements than models_metaschema.xml.
+
+It has its own tests and build scripts.
+
+The 'models' metaschema aims to present all assembly, field and flag model configurations and definitions in a 'lab-only' tag set.
+
+Tinydata, in contrast, is designed to be actually useful for some range of use cases, for example glossaries, data sheets or spreadsheet dumps.
+
+## Schema generation testing overhaul planning Summer 2024
+
+- [ ] **Phase I** plan out, test and code solution to Issue 105 - JSON schema `choice` fail
+  - [x] use tinydata example to unit test this in schema-gen/JSON-schema
+  - [ ] demonstrate with some go/no-go testing in tinydata that this works
+  - [ ] make a new OSCAL catalog JSON Schema available for field testing in oscal-xproc3
+  - [ ] clean up and push to address #105
+- [ ] **Phase II** build out XSpecs for schema generation - at least minimally (foothold)
+  - [ ] XSD and JSON Schema in detail
+  - [ ] both models_metaschema and tinydata
+- [ ] **Phase III** Inspector XSLT generation
+  - [ ] free all schema generators from `metaschema` module runtime dependency
+  - [ ] make all schema generation available as runtimes in XSLT, XProc 1/3