New packages galaxy-tool-util-models, galaxy-tool-shed-schema.

The galaxy-tool-shed-schema package is basically pydantic models for the whole tool shed API. It is used by both the backend code for the new tool shed and the testing code that runs on both the old and new tool shed.

The creation of galaxy-tool-util-models package is something I've wanted to do for a while - it should solve some TODOs I've left in the base. I think beyond enabling galaxy-tool-shed-schema package - the other nice thing is that galaxy-schema code can now rely on various tool stuff without requiring dependency on any of the parser or runtime code.

This should also fix things for the new record types in galaxyproject#19377. I've introduced a type that can be used tools and is included schema for collection creation. The galaxy-tool-util-models packages can now be dependended on by galaxy-schema and galaxy-tool-util and these two packages do not need to depend on each other - either direction of that dependency would make me uncomfortable. xref
https://github.com/galaxyproject/galaxy/actions/runs/13704893929/job/38327692654?pr=19377

lib/galaxy/managers/citations.py

@@ -11,7 +11,7 @@
 from beaker.util import parse_cache_config_options
 
 from galaxy.structured_app import BasicSharedApp
-from galaxy.tool_util.parser.interface import Citation
+from galaxy.tool_util_models.tool_source import Citation
 from galaxy.util import (
     DEFAULT_SOCKET_TIMEOUT,
     requests,

lib/galaxy/tool_util/model_factory.py

@@ -0,0 +1,51 @@
+
+from typing import (
+    Type,
+    TypeVar,
+)
+
+from galaxy.tool_util_models import ParsedTool
+from .parameters import input_models_for_tool_source
+from .parser.interface import (
+    ToolSource,
+)
+from .parser.output_objects import from_tool_source
+
+
+def parse_tool(tool_source: ToolSource) -> ParsedTool:
+    return parse_tool_custom(tool_source, ParsedTool)
+
+
+P = TypeVar("P", bound=ParsedTool)
+
+
+def parse_tool_custom(tool_source: ToolSource, model_type: Type[P]) -> P:
+    id = tool_source.parse_id()
+    version = tool_source.parse_version()
+    name = tool_source.parse_name()
+    description = tool_source.parse_description()
+    inputs = input_models_for_tool_source(tool_source).parameters
+    outputs = from_tool_source(tool_source)
+    citations = tool_source.parse_citations()
+    license = tool_source.parse_license()
+    profile = tool_source.parse_profile()
+    edam_operations = tool_source.parse_edam_operations()
+    edam_topics = tool_source.parse_edam_topics()
+    xrefs = tool_source.parse_xrefs()
+    help = tool_source.parse_help()
+
+    return model_type(
+        id=id,
+        version=version,
+        name=name,
+        description=description,
+        profile=profile,
+        inputs=inputs,
+        outputs=outputs,
+        license=license,
+        citations=citations,
+        edam_operations=edam_operations,
+        edam_topics=edam_topics,
+        xrefs=xrefs,
+        help=help,
+    )

lib/galaxy/tool_util/ontologies/ontology_data.py

@@ -9,10 +9,8 @@
 )
 
 from galaxy.tool_util.biotools import BiotoolsMetadataSource
-from galaxy.tool_util.parser import (
-    ToolSource,
-    XrefDict,
-)
+from galaxy.tool_util.parser import ToolSource
+from galaxy.tool_util_models.tool_source import XrefDict
 from galaxy.util.resources import resource_string
 
 

lib/galaxy/tool_util/parameters/__init__.py

@@ -1,26 +1,4 @@
-from .case import (
-    test_case_state,
-    validate_test_cases_for_tool_source,
-)
-from .convert import (
-    decode,
-    dereference,
-    encode,
-    encode_test,
-    fill_static_defaults,
-    landing_decode,
-    landing_encode,
-)
-from .factory import (
-    from_input_source,
-    input_models_for_pages,
-    input_models_for_tool_source,
-    input_models_from_json,
-    ParameterDefinitionError,
-    tool_parameter_bundle_from_json,
-)
-from .json import to_json_schema_string
-from .models import (
+from galaxy.tool_util_models.parameters import (
     BooleanParameterModel,
     ColorParameterModel,
     ConditionalParameterModel,
@@ -53,6 +31,30 @@
     ToolParameterBundleModel,
     ToolParameterModel,
     ToolParameterT,
+)
+from .case import (
+    test_case_state,
+    validate_test_cases_for_tool_source,
+)
+from .convert import (
+    decode,
+    dereference,
+    encode,
+    encode_test,
+    fill_static_defaults,
+    landing_decode,
+    landing_encode,
+)
+from .factory import (
+    from_input_source,
+    input_models_for_pages,
+    input_models_for_tool_source,
+    input_models_from_json,
+    ParameterDefinitionError,
+    tool_parameter_bundle_from_json,
+)
+from .json import to_json_schema_string
+from .model_validation import (
     validate_against_model,
     validate_internal_job,
     validate_internal_landing_request,

lib/galaxy/tool_util/parameters/case.py

@@ -22,7 +22,7 @@
 )
 from galaxy.util import asbool
 from .factory import input_models_for_tool_source
-from .models import (
+from galaxy.tool_util_models.parameters import (
     BooleanParameterModel,
     ConditionalParameterModel,
     ConditionalWhen,

lib/galaxy/tool_util/parameters/convert.py

@@ -11,11 +11,7 @@
     Union,
 )
 
-from galaxy.tool_util.parser.interface import (
-    JsonTestCollectionDefDict,
-    JsonTestDatasetDefDict,
-)
-from .models import (
+from galaxy.tool_util_models.parameters import (
     BooleanParameterModel,
     ConditionalParameterModel,
     ConditionalWhen,
@@ -37,6 +33,10 @@
     ToolParameterBundle,
     ToolParameterT,
 )
+from galaxy.tool_util_models.tool_source import (
+    JsonTestCollectionDefDict,
+    JsonTestDatasetDefDict,
+)
 from .state import (
     JobInternalToolState,
     LandingRequestInternalToolState,

lib/galaxy/tool_util/parameters/factory.py

@@ -14,21 +14,22 @@
     PagesSource,
     ToolSource,
 )
-from galaxy.tool_util.parser.parameter_validators import (
+from galaxy.tool_util.parser.parameter_validators import static_validators
+from galaxy.tool_util_models.parameter_validators import (
     EmptyFieldParameterValidatorModel,
     ExpressionParameterValidatorModel,
     InRangeParameterValidatorModel,
     LengthParameterValidatorModel,
     NoOptionsParameterValidatorModel,
     RegexParameterValidatorModel,
-    static_validators,
+
 )
 from galaxy.tool_util.parser.util import (
     parse_profile_version,
     text_input_is_optional,
 )
 from galaxy.util import string_as_bool
-from .models import (
+from galaxy.tool_util_models.parameters import (
     BaseUrlParameterModel,
     BooleanParameterModel,
     ColorParameterModel,

lib/galaxy/tool_util/parameters/model_validation.py

@@ -0,0 +1,57 @@
+from typing import (
+    Any,
+    Dict,
+    Optional,
+    Type,
+)
+
+from pydantic import (
+    BaseModel,
+    ValidationError,
+)
+from typing_extensions import (
+    Protocol,
+)
+
+from galaxy.exceptions import RequestParameterInvalidException
+from galaxy.tool_util_models.parameters import (
+    DEFAULT_MODEL_NAME,
+    create_field_model,
+    ToolParameterBundle,
+    RawStateDict,
+    StateRepresentationT,
+)
+
+def validate_against_model(pydantic_model: Type[BaseModel], parameter_state: Dict[str, Any]) -> None:
+    try:
+        pydantic_model(**parameter_state)
+    except ValidationError as e:
+        # TODO: Improve this or maybe add a handler for this in the FastAPI exception
+        # handler.
+        raise RequestParameterInvalidException(str(e))
+
+
+class ValidationFunctionT(Protocol):
+
+    def __call__(self, tool: ToolParameterBundle, request: RawStateDict, name: Optional[str] = None) -> None: ...
+
+
+def validate_model_type_factory(state_representation: StateRepresentationT) -> ValidationFunctionT:
+
+    def validate_request(tool: ToolParameterBundle, request: Dict[str, Any], name: Optional[str] = None) -> None:
+        name = name or DEFAULT_MODEL_NAME
+        pydantic_model = create_field_model(tool.parameters, name=name, state_representation=state_representation)
+        validate_against_model(pydantic_model, request)
+
+    return validate_request
+
+
+validate_request = validate_model_type_factory("request")
+validate_internal_request = validate_model_type_factory("request_internal")
+validate_internal_request_dereferenced = validate_model_type_factory("request_internal_dereferenced")
+validate_landing_request = validate_model_type_factory("landing_request")
+validate_internal_landing_request = validate_model_type_factory("landing_request_internal")
+validate_internal_job = validate_model_type_factory("job_internal")
+validate_test_case = validate_model_type_factory("test_case_xml")
+validate_workflow_step = validate_model_type_factory("workflow_step")
+validate_workflow_step_linked = validate_model_type_factory("workflow_step_linked")

lib/galaxy/tool_util/parameters/state.py

@@ -14,7 +14,7 @@
 from pydantic import BaseModel
 from typing_extensions import Literal
 
-from .models import (
+from galaxy.tool_util_models.parameters import (
     create_job_internal_model,
     create_landing_request_internal_model,
     create_landing_request_model,
@@ -28,6 +28,8 @@
     ToolParameterBundle,
     ToolParameterBundleModel,
     ToolParameterT,
+)
+from .model_validation import (
     validate_against_model,
 )
 

lib/galaxy/tool_util/parameters/visitor.py

@@ -11,7 +11,7 @@
 
 from typing_extensions import Protocol
 
-from .models import (
+from galaxy.tool_util_models.parameters import (
     ConditionalParameterModel,
     ConditionalWhen,
     RepeatParameterModel,

lib/galaxy/tool_util/parser/__init__.py

@@ -8,7 +8,6 @@
 from .interface import (
     RequiredFiles,
     ToolSource,
-    XrefDict,
 )
 from .output_objects import ToolOutputCollectionPart
 
@@ -19,5 +18,4 @@
     "RequiredFiles",
     "ToolOutputCollectionPart",
     "ToolSource",
-    "XrefDict",
 )

lib/galaxy/tool_util/parser/cwl.py

@@ -10,8 +10,8 @@
 
 from galaxy.tool_util.cwl.parser import tool_proxy
 from galaxy.tool_util.deps import requirements
+from galaxy.tool_util_models.tool_source import HelpContent
 from .interface import (
-    HelpContent,
     PageSource,
     PagesSource,
     ToolSource,