Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add auto-generated Configuration schema from Pydantic model #16518

Draft
wants to merge 28 commits into
base: dev
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
5404142
Add basic script to generate GalaxyConfigModel
davelopez Aug 4, 2023
1522897
Add makefile commands
davelopez Aug 4, 2023
66130e4
WIP: Add basic generated GalaxyConfigModel
davelopez Aug 4, 2023
ad8f491
Enhance GalaxyConfigModel generation
davelopez Aug 6, 2023
6e1f37a
Revert "Add makefile commands"
davelopez Aug 7, 2023
dd8a3db
WIP: Split config options into models
davelopez Aug 7, 2023
273ab80
Fix some type annotations
davelopez Aug 8, 2023
a6f97c3
Fix single_user field type duality
davelopez Aug 8, 2023
5eafa77
Fix metadata_strategy type annotation
davelopez Aug 9, 2023
7fd44f9
Make optional options, well... optional
davelopez Aug 9, 2023
250aad8
Annotate job_config_file option as optional
davelopez Aug 9, 2023
8646b29
Add bool as possible WatchToolOptions type
davelopez Aug 9, 2023
2d2974a
Fix typo in tests for `extended_celery` option
davelopez Aug 9, 2023
f9c1406
Validate configuration on startup
davelopez Aug 9, 2023
c9613b4
Handle `panel_views` type duality
davelopez Aug 10, 2023
ecbed37
Refactor model names + update client schema
davelopez Aug 10, 2023
6bcef0c
Add missing valid values for simplified_workflow_run_ui_target_history
davelopez Aug 10, 2023
e7386a8
Annotate more non-required options as optional
davelopez Aug 13, 2023
cca9481
Add tool_shed_urls to admin computed configuration
davelopez Aug 22, 2023
e4a8cbb
Remove unused delay_tool_initialization option
davelopez Aug 23, 2023
8910fe6
Annotate more non-required config params as optional
davelopez Aug 23, 2023
6e5a104
Annotate deprecated options
davelopez Aug 23, 2023
667cc45
Update client API schema
davelopez Aug 23, 2023
9c4f948
Annotate per_host options
davelopez Aug 24, 2023
cf5b90b
Annotate the rest of custom properties
davelopez Aug 24, 2023
c46e31d
Fix bool assignment for use_remote_user
davelopez Aug 25, 2023
669453c
Add basic schema generation (for testing)
davelopez Aug 25, 2023
b27ee9a
Avoid unwanted spacing in descriptions
davelopez Sep 11, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,9 @@ reports-config-convert: ## convert old style reports ini to yaml
reports-config-lint: ## lint reports YAML configuration file
$(CONFIG_MANAGE) lint reports

config-generate-schema: ## build galaxy YAML configuration schema
$(CONFIG_MANAGE) generate_schema galaxy

config-validate: ## validate galaxy YAML configuration file
$(CONFIG_MANAGE) validate galaxy

Expand Down
1,407 changes: 1,406 additions & 1 deletion client/src/schema/schema.ts

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions lib/galaxy/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
from galaxy.managers.api_keys import ApiKeyManager
from galaxy.managers.citations import CitationsManager
from galaxy.managers.collections import DatasetCollectionManager
from galaxy.managers.configuration import ConfigurationManager
from galaxy.managers.dbkeys import GenomeBuilds
from galaxy.managers.folders import FolderManager
from galaxy.managers.hdas import HDAManager
Expand Down Expand Up @@ -244,6 +245,7 @@ def __init__(self, fsmon=False, **kwargs) -> None:
self.is_webapp = False
# Read config file and check for errors
self.config = self._register_singleton(config.GalaxyAppConfiguration, config.GalaxyAppConfiguration(**kwargs))
ConfigurationManager.validate_config(self.config._raw_config)
self.config.check()
self._configure_object_store(fsmon=True)
self._register_singleton(BaseObjectStore, self.object_store)
Expand Down
2 changes: 1 addition & 1 deletion lib/galaxy/config/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -914,7 +914,7 @@ def _process_config(self, kwargs: Dict[str, Any]) -> None:
self.galaxy_data_manager_data_path = self.galaxy_data_manager_data_path or self.tool_data_path
self.tool_secret = kwargs.get("tool_secret", "")
self.metadata_strategy = kwargs.get("metadata_strategy", "directory")
self.use_remote_user = self.use_remote_user or self.single_user
self.use_remote_user = self.use_remote_user or bool(self.single_user)
self.fetch_url_allowlist_ips = [
ipaddress.ip_network(unicodify(ip.strip())) # If it has a slash, assume 127.0.0.1/24 notation
if "/" in ip
Expand Down
12 changes: 12 additions & 0 deletions lib/galaxy/config/config_manage.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
TOOL_SHED_CONFIG_SCHEMA_PATH,
)
from galaxy.config.schema import AppSchema
from galaxy.schema.configuration import FullGalaxyConfig
from galaxy.util import safe_makedirs
from galaxy.util.properties import (
nice_config_parser,
Expand Down Expand Up @@ -524,12 +525,23 @@ def _get_option_desc(option: Dict[str, Any]) -> str:
return desc


def _generate_schema(args: Namespace, app_desc: App) -> None:
"""Generates the configuration YAML schema from the Pydantic model."""
if app_desc.app_name != "galaxy":
raise Exception("Only Galaxy is supported for configuration schema generation")
galaxy_config_schema = FullGalaxyConfig.schema()
target_path = app_desc.schema_path + ".test.yml"
with open(target_path, "w") as f:
yaml.dump(galaxy_config_schema, f)


ACTIONS: Dict[str, Callable] = {
"convert": _run_conversion,
"build_sample_yaml": _build_sample_yaml,
"validate": _validate,
"lint": _lint,
"build_rst": _to_rst,
"generate_schema": _generate_schema,
}


Expand Down
9 changes: 9 additions & 0 deletions lib/galaxy/managers/configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from galaxy.managers.context import ProvidesUserContext
from galaxy.managers.markdown_util import weasyprint_available
from galaxy.schema import SerializationParams
from galaxy.schema.configuration import FullGalaxyConfig
from galaxy.structured_app import StructuredApp
from galaxy.web.framework.base import server_starttime

Expand Down Expand Up @@ -84,6 +85,14 @@ def tool_conf_to_dict(conf):
def reload_toolbox(self):
self._app.queue_worker.send_control_task("reload_toolbox")

@staticmethod
def validate_config(config_dict: Dict[str, Any]):
"""Validate the configuration against the schema model.

It will raise an exception if the configuration is invalid.
"""
return FullGalaxyConfig(**config_dict)


# TODO: this is a bit of an odd duck. It uses the serializer structure from managers
# but doesn't have a model like them. It might be better in config.py or a
Expand Down
Loading