diff --git a/README.md b/README.md index c304feb..f74dadd 100644 --- a/README.md +++ b/README.md @@ -1,95 +1,8 @@ -from sio3pack.graph import GraphOperation - # SIO3Pack -## Prerequisites -``` -- Python 3.9 or higher -- pip -- Linux or macOS operating system -- Django 4.2.x (for Django support) -``` - -## Instalation -``` -pip install sio3pack -``` -## Example usage (in python) - -### In OIOIOI - -```python -# Package unpacking -import sio3pack, sio3workers -from django.conf import settings - -package = sio3pack.from_file(path_to_package, django_settings=settings) -graph_op: GraphOperation = package.get_unpack_operation() -results = sioworkers.run(graph_op) -graph_op.return_results(results) -package.save_to_db(problem_id=1) -``` - -### Locally (for example `sinol-make`) - -```python -import sio3pack, sio3workers.local - -package = sio3pack.from_file(path_to_package) -graph_op: GraphOperation = package.get_unpack_operation() -results = sio3workers.local.run(graph_op) -graph_op.return_results(results) -``` - ---- - -## Development - -### Test without django support - -Install the package in editable mode and make sure that `django` and -`pytest-django` are not installed. - -```bash -pip install -e ".[tests]" -pip uninstall django pytest-django -``` - -Then follow the instructions in -[General testing information](#general-testing-information). - - -### Test with django support - -Install the package in editable mode along with Django dependencies: - -```bash -pip install -e ".[django,tests,django_tests]" -``` - -Then follow the instructions in -[General testing information](#general-testing-information). - - -### General testing information - -Run the tests with `pytest` in the root directory of -the repository. - -```bash -pytest -v -``` - -To run tests in parallel, use the following command. - -```bash -pytest -v -n auto -``` - -To run coverage tests, use the following command. - -```bash -pytest -v --cov=sio3pack --cov-report=html -``` +SIO3Pack is a Python package designed to facilitate the creation and manipulation +of packages supported by the SIO2 system. It provides a set of classes and functions +to handle various operations related to package interaction, including workflow +management and creation. -The coverage report will be generated in the file `htmlcov/index.html`. +Full documentation is available at [SIO3Pack Documentation](https://sio2project.github.io/SIO3Pack/). diff --git a/docs/conf.py b/docs/conf.py index f11ff4d..4be36eb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -7,6 +7,15 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information import sio3pack +import django +import sys +import os + + +sys.path.append(os.path.abspath('../tests/test_django')) +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'test_django.settings') +django.setup() + project = 'SIO3Pack' copyright = '2025, Tomasz Kwiatkowski, Mateusz Masiarz, Jakub Rożek, Stanisław Struzik' @@ -21,6 +30,7 @@ 'sphinx.ext.autodoc', # Also required by AutoAPI. 'sphinx.ext.viewcode', 'sphinx.ext.intersphinx', + 'sphinx.ext.coverage', ] templates_path = ['_templates'] @@ -38,6 +48,7 @@ autoapi_include = [ "sio3pack.django.common.handler.DjangoHandler", "sio3pack.django.sinolpack.handler.SinolpackDjangoHandler", + "sio3pack.files.remote_file.RemoteFile", ] autodoc_typehints = 'description' diff --git a/docs/development.rst b/docs/development.rst new file mode 100644 index 0000000..12d2522 --- /dev/null +++ b/docs/development.rst @@ -0,0 +1,64 @@ +Development +=========== + +This section is intended for developers who want to contribute to the project or understand its inner workings. +It provides guidelines on how to set up the development environment, run tests, and contribute code. + +Setting Up the Development Environment +-------------------------------------- + +If you want to develop or run tests without Django support, you can use the following command: + +.. code-block:: bash + + pip install -e .[tests] + pip uninstall django pytest-django + +This will install the package in editable mode, without Django support. Uninstall Django and pytest-django to make sure +that the tests run without Django support. + +If you want to run tests with Django support, you can use the following command: + +.. code-block:: bash + + pip install -e .[django,tests,django_tests] + +This will install the package in editable mode with Django support, along with the necessary testing dependencies. + +Running Tests +------------- + +To run the tests, you can use the following command: + +.. code-block:: bash + + pytest -v + +The tests can also be run in parallel using the `pytest-xdist` plugin. To do this, you can use the following command: + +.. code-block:: bash + + pytest -v -n auto + +You can also run coverage reports to see how much of the code is covered by tests. To do this, you can use the following +command: + +.. code-block:: bash + + pytest --cov=src --cov-report=html --cov-report=html + +The coverage report will be generated in the `htmlcov` directory, and you can open the `index.html` file in your web browser +to view the report. + +Contributing Code +----------------- + +If you want to contribute code to the project, please follow these guidelines: +1. Fork the repository and create a new branch for your feature or bug fix. +2. Write tests for your code and make sure they pass. +3. Make sure your code follows the project's coding style and conventions. +4. Submit a pull request with a clear description of your changes and why they are needed. +5. Be open to feedback and willing to make changes based on code reviews. +6. Ensure that your code is well-documented, including docstrings for functions and classes. +7. Update the documentation if your changes affect the usage or functionality of the package. +8. Keep your pull request focused on a single feature or bug fix to make it easier to review. diff --git a/docs/index.rst b/docs/index.rst index 3b86ce7..c1e8057 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,4 +8,7 @@ SIO3Pack documentation :maxdepth: 2 :caption: Contents: + usage + workflows sinolpack + development diff --git a/docs/usage.rst b/docs/usage.rst new file mode 100644 index 0000000..d565388 --- /dev/null +++ b/docs/usage.rst @@ -0,0 +1,265 @@ +Usage guide +=========== + +This library allows to interact with packages supported by SIO2 system. It's used in two important components of SIO2: + +- `OIOIOI `_, which is the web interface for SIO2 system, +- `sinol-make `_, which is the local tool for testing packages. + +The purpose of this library is to keep the code for package management in one place, so that it can be reused in both components. +This will also allow developers to write their own tools that can interact with SIO2 packages. + +Supported package types +----------------------- + +Currently, this library supports the following package types: + +- `sinolpack` -- a package format originally used by SIO2 system, which is a simple archive format with metadata, + and is used for most packages in SIO2 system. + +Next planned package types to be supported are: + +- `sinol3pack` -- a new package format that is being developed for SIO2/SIO3 system, which will be a more advanced format + with better support for metadata and dependencies, +- Codeforces packages -- a package format used by `Codeforces system `_, which is a popular + competitive programming platform. + +Installation +------------ + +To install the library, you can use `pip`. There are two ways to install it, depending on the way you want to use it: + +- if you want to use the library locally, without the need for Django, you can install it with the following command: + + .. code-block:: shell + + pip install sio3pack + +- if you want to use the library with Django support, for example when developing OIOIOI, you can install it with the following command: + + .. code-block:: shell + + pip install sio3pack[django] + +Initializing without Django support +----------------------------------- +When using SIO3Pack without Django support, there is only one way to initialize a package, which is by using the +:py:func:`sio3pack.from_file` function. This function takes a file path to a directory with the package or a archived +package file, and returns an instance of :py:class:`sio3pack.packages.package.Package` class, which can be used to +interact with the package. + +This class is an abstract base class, so for different package types, you will receive different subclasses of this class. +For example for `sinolpack` packages, you will get an instance of :py:class:`sio3pack.packages.sinolpack.Sinolpack` class. + +Example when importing a `sinolpack` package: + +.. code-block:: python + + import sio3pack + from sio3pack.packages.sinolpack import Sinolpack + + + package: Sinolpack = sio3pack.from_file("path/to/sinolpack/package") + + print(package.full_name) # Prints the full title of the package + +The :py:func:`sio3pack.from_file` function can also take an optional argument `configuration`, which is an instance +of :py:class:`sio3pack.packages.package.SIO3PackConfig` class. For local usage, this class mainly provides available +compilers. It is possible to automatically detect the configuration by using the :py:meth:`sio3pack.packages.package.SIO3PackConfig.detect` +method, which will try to detect the configuration based on the current environment. Example usage: + +.. code-block:: python + + import sio3pack + from sio3pack.packages.sinolpack import Sinolpack + from sio3pack.packages.package import SIO3PackConfig + + configuration = SIO3PackConfig.detect() + package: Sinolpack = sio3pack.from_file("path/to/sinolpack/package", configuration=configuration) + + print(package.full_name) # Prints the full title of the package + +Initializing with Django support +-------------------------------- + +When using SIO3Pack with Django support, you can initialize a package using the same :py:func:`sio3pack.from_file` function, +but you can also use the :py:func:`sio3pack.from_db` function, which allows you to initialize a package from the database. + +After initializing the package from file, you can use the :py:meth:`sio3pack.packages.package.Package.save_to_db` method +to save the package to the database. This method will create a new package in the database, or update an existing one if +it already exists. + +The :py:class:`sio3pack.packages.package.SIO3PackConfig` class can also take the Django settings, which is useful when you want to +use the library with Django support. + +The important distinction between a package from file and a package from the database is that the package from database +is lazy-loaded, meaning that the metadata and files are not loaded until they are accessed. This is useful for +saving memory and improving performance, especially for OIOIOI. + +Example usage: + +.. code-block:: python + + import sio3pack + from sio3pack.packages.sinolpack import Sinolpack + from sio3pack.packages.package import SIO3PackConfig + + from django.conf import settings + + + package: Sinolpack = sio3pack.from_file("path/to/sinolpack/package") + problem_id = 1 # The `save_to_db` functions requires a problem ID to save the package to the database + package.save_to_db(problem_id) + + + configuration = SIO3PackConfig.detect() + configuration.django_settings = settings # Set the Django settings for the configuration + from_db: Sinolpack = sio3pack.from_db(problem_id, configuration=configuration) + print(from_db.full_name) # Prints the full title of the package from the database + +Interacting with workflows +-------------------------- + +The most important part of the new packages specification are the workflows, which are a set of steps that can be +executed by the sio3workers. There are default workflows for all types of packages, but you can also create your own +workflows by creating a `workflows.json` file in the package directory. For more information, read the +:doc:`workflows section ` of the documentation. + +There are a couple of important workflows that can be used: + +- unpack workflow -- for the first time unpacking the package, which will generate tests and verify them +- run workflow -- for running a program on the tests from the package +- generate user out workflow -- for generating an output of user's program on a test from the package +- test run workflow -- for generating an output for user's program on a user-provided test + +Each method for generating workflows returns an instance of :py:class:`sio3pack.workflow.WorkflowOperation` class, +which can be used to get the workflows. This class has a method :py:meth:`sio3pack.workflow.WorkflowOperation.get_workflow`, +which yields instances of :py:class:`sio3pack.workflow.Workflow` class until there are workflows to be run. +The :py:class:`sio3pack.workflow.Workflow` class can be passed to a sio3worker executor, which will then run the workflow +and return the results. After each workflow run, the results should be passed to :py:meth:`sio3pack.workflow.WorkflowOperation.return_results` +method. Example usage of a workflow operation: + +.. code-block:: python + + import sio3pack, sio3worker + from sio3pack.packages.sinolpack import Sinolpack + from sio3pack.workflow import WorkflowOperation + + package: Sinolpack = sio3pack.from_file("path/to/sinolpack/package") + wf_op: WorkflowOperation = package.get_some_workflow_operation() + + for workflow in wf_op.get_workflow(): + results = sio3worker.run_workflow(workflow) # This will run the workflow using the sio3worker executor + wf_op.return_results(results) # This will return the results of the workflow to the operation + + +Unpack workflow +~~~~~~~~~~~~~~~ + +This workflow should be used after initializing a package from file. It will generate tests, verify them and save the +information about the tests on the package. This workflow uses these sub-workflows: + +- `ingen` -- for generating input tests +- `outgen` -- for generating output tests +- `inwer` -- for verifying the input tests + +These workflows can be created by calling the :py:meth:`sio3pack.packages.package.Package.get_unpack_operation` method on +the package instance. This method will return the workflow operation. Example usage: + +.. code-block:: python + + import sio3pack, sio3worker + from sio3pack.packages.sinolpack import Sinolpack + from sio3pack.workflow import WorkflowOperation + + package: Sinolpack = sio3pack.from_file("path/to/sinolpack/package") + unpack_op: WorkflowOperation = package.get_unpack_operation() + + for workflow in unpack_op.get_workflow(): + results = sio3worker.run_workflow(workflow) # This will run the workflow using the sio3worker executor + unpack_op.return_results(results) # This will return the results of the workflow to the operation + + # After unpacking, you can save the package to the database + package.save_to_db(problem_id) + +Run workflow +~~~~~~~~~~~~ + +This workflow is used for running a program on the tests from the package. It can be created by calling the +:py:meth:`sio3pack.packages.package.Package.get_run_operation` method on the package instance. This method requires +a `program` argument, which is an instance of :py:class:`sio3pack.files.File` class, which represents the program to be run. +It also has an optional argument `tests`, which allows to specify a list of tests to be run. If not specified, +all tests from the package will be used. Example usage: + +.. code-block:: python + + import sio3pack, sio3worker + from sio3pack.packages.sinolpack import Sinolpack + from sio3pack.files import File + from sio3pack.workflow import WorkflowOperation + + package: Sinolpack = sio3pack.from_file("path/to/sinolpack/package") + program: File = package.get_program("main") # Get the main program file from the package + run_op: WorkflowOperation = package.get_run_operation(program) + + for workflow in run_op.get_workflow(): + results = sio3worker.run_workflow(workflow) # This will run the workflow using the sio3worker executor + run_op.return_results(results) # This will return the results of the workflow to the operation + +User out workflow +~~~~~~~~~~~~~~~~~ + +This workflow is used for generating an output of user's program on a test from the package. It can be created by +calling the :py:meth:`sio3pack.packages.package.Package.get_user_out_operation` method on the package instance. +This method requires two arguments: `program` and `test`. The `program` argument is an instance of +:py:class:`sio3pack.files.File` class, which represents the user's program to be run, and the `test` argument is +an instance of :py:class:`sio3pack.test.Test` class, which represents the test to be run. Example usage: + +.. code-block:: python + + import sio3pack, sio3worker + from sio3pack.packages.sinolpack import Sinolpack + from sio3pack.files import File + from sio3pack.test import Test + from sio3pack.workflow import WorkflowOperation + + package: Sinolpack = sio3pack.from_file("path/to/sinolpack/package") + program: File = package.get_program("main") # Get the main program file from the package + test: Test = package.tests[0] # Get the first test from the package + user_out_op: WorkflowOperation = package.get_user_out_operation(program, test) + + for workflow in user_out_op.get_workflow(): + results = sio3worker.run_workflow(workflow) # This will run the workflow using the sio3worker executor + user_out_op.return_results(results) # This will return the results of the workflow to the operation + +Test run workflow +~~~~~~~~~~~~~~~~~ + +This workflow is used for generating an output for user's program on a user-provided test. It can be created by +calling the :py:meth:`sio3pack.packages.package.Package.get_test_run_operation` method on the package instance. +This method requires two arguments: `program` and `test`. Both arguments are instances of +:py:class:`sio3pack.files.File` class, where `program` represents the user's program to be run, and `test` +represents the user-provided test to be run. Example usage: + +.. code-block:: python + + import sio3pack, sio3worker + from sio3pack.packages.sinolpack import Sinolpack + from sio3pack.files import LocalFile + from sio3pack.workflow import WorkflowOperation + + package: Sinolpack = sio3pack.from_file("path/to/sinolpack/package") + program: File = package.get_program("main") # Get the main program file from the package + user_test: LocalFile = LocalFile("path/to/user/test") # User-provided test file + test_run_op: WorkflowOperation = package.get_test_run_operation(program, user_test) + + for workflow in test_run_op.get_workflow(): + results = sio3worker.run_workflow(workflow) # This will run the workflow using the sio3worker executor + test_run_op.return_results(results) # This will return the results of the workflow to the operation + +More information +---------------- + +For more information on how to use the library, you can check the documentation for the specific package types, such as +:py:class:`sio3pack.packages.sinolpack.Sinolpack`. You can also check the :doc:`workflows section ` for examples +of how to implement your own workflows using the library. diff --git a/docs/workflows.rst b/docs/workflows.rst new file mode 100644 index 0000000..3df6d93 --- /dev/null +++ b/docs/workflows.rst @@ -0,0 +1,302 @@ +SIO3Worker workflows documentation +================================== + +This document provides an overview of the workflow's specification +and how to use it effectively. + + +Workflow Specification +---------------------- + +A workflow is a collection of tasks that can run programs, execute Lua scripts, write objects, +read and write from registers, and perform various other operations. Workflows are defined by a +JSON file that describes the tasks, their dependencies, and the data they use. + +Objects +~~~~~~~ + +Objects are files that can be provided from file storage, can be read of written by workflows and +can be saved to file storage. An object can be a compiled executable, source code or a test. +Objects are identified by their handle and can be referenced in the workflow. + +There are three types of objects: + +- **external object** -- an object that is provided from file storage, such as a compiled executable or source code. + These objects can be used by workflows. These objects have to be specified in the `external_objects` key in + the workflow JSON file. This is an array of handles of external objects that can be used by the workflow. +- **observable object** -- an object that is created by the workflow and saved to the file storage. + This type of object is for example a compiled executable or generated tests. These objects have to be + specified in the `observable_objects` key in the workflow JSON file. This is an array of handles of observable + objects that can be used by the workflow. +- **normal object** -- any other object. This type of object can be created by workflow and after finishing + they are deleted. An example of this object is a user's generated output which is only used by a checker. + These objects aren't specified in the workflow definition. They can be specified in tasks. + +Registers +~~~~~~~~~ + +Registers are used to store any data that can be used by workflow. Only one task can write to a register, but +multiple tasks can read from it. Registers are identified by their number, starting from 0. + +There are two types of registers: + +- **observable register** -- after the workflow finishes, the value of this register is returned to the user. + This type of register is used to store the final result of the workflow, such as the output of a program or + the result of a test. These registers have numbers starting from 0. The number of observable registers has + to be specified in the `observable_registers` key in the workflow definition. +- **normal register** -- this type of register is used to store intermediate results of the workflow. These registers + can be used by any task in the workflow, but their values are not returned to the user. These registers have + numbers starting from the number of observable registers. The total number of registers (including observable) has + to be specified in the `registers` key in the workflow definition. + +Tasks +~~~~~ + +Tasks are the building blocks of a workflow. Each task can perform a specific operation, such as running a program +or executing a Lua script. Tasks can depend on other tasks by using registers or objects. Tasks are defined in the +`tasks` key in the workflow definition, as an array of task definitions. + +There are two types of tasks: script tasks and execution tasks. The type of task is specified by the `type` key in the +task definition (either `script` or `execution`). + +Script Tasks +~~~~~~~~~~~~ + +Script tasks are tasks that execute Lua scripts. They can read and write to registers and read objects. + +These keys are used to define script tasks: + +- `type` -- the type of the task, which is `script` for script tasks. +- `name` -- the name of the task, used for debugging and logging. +- `input_registers` -- an array of register numbers that the task reads from. +- `output_registers` -- an array of register numbers that the task writes to. +- `objects` -- an array of object handles that the task reads from. +- `reactive` -- a boolean value that indicates whether the task is reactive. If true, the task will be executed + whenever any of its input registers or objects change. +- `script` -- the Lua script to be executed by the task. + +Execution Tasks +~~~~~~~~~~~~~~~ + +Execution tasks are tasks that can run processes, such as compiled executables or scripts. They can mount objects +inside filesystems, limit resources, mount in specific images, attach specific files to file descriptors, and more. +Security is mostly ensured by Linux namespaces, which isolate and limit the execution environment of the task. + +These keys are used to define execution tasks: + +- `type` -- the type of the task, which is `execution` for execution tasks. +- `name` -- the name of the task, used for debugging and logging. +- `channels` -- an array of configurations for pipes that the task uses. Each configuration is an object with the following keys: + + - `buffer_size` -- The maximum amount of data stored in the channel that has been written by + the writer, but not yet read by the reader. This value must be positive. + - `source_pipe` -- The pipe this channel will be reading from. + - `target_pipe` -- The pipe this channel will be writing to. + - `file_buffer_size` (optional) -- Controls whether this channel is backed by a file on the disk. + A larger buffer may then be allocated on the disk. + - `limit` (optional) -- Limits the maximum amount of data sent through the channel. +- `exclusive` -- a boolean value that indicates whether the task is exclusive. If true, the task will not run concurrently + with other tasks. +- `hard_time_limit` -- the maximum amount of time the task can run, in seconds. If the task exceeds this limit, it will be terminated. +- `output_register` -- the register number that the execution results will be written to. This register will contain various + information about the execution, such as the exit code, output, and error messages. +- `pid_namespaces` -- number of PID namespaces that the task will use. This is used to isolate the process tree of the task. +- `pipes` -- number of pipes that the task will use. Pipes are used for inter-process communication. +- `filesystems` -- an array of configuration for filesystems. Multiple filesystems can be mounted for a process. + There are multiple types of filesystems: + + - Image filesystem -- a filesystem that is mounted from an image file. The configuration is an object with the following keys: + + - `type` -- the type of the filesystem, which is `image` for image filesystems. + - `path` -- the path to the image file. + - Empty filesystem -- a filesystem that is mounted as an empty directory. The configuration is an object with the following keys: + + - `type` -- the type of the filesystem, which is `empty` for empty filesystems. + - Object filesystem -- a filesystem that is an object. The configuration is an object with the following keys: + + - `type` -- the type of the filesystem, which is `object` for object filesystems. + - `object` -- the handle of the object that is used as a filesystem. +- `mount_namespaces` -- an array of mount namespace configurations. Each configuration is an object with the following keys: + + - `root` -- ? + - `mountpoints` -- an array of mountpoint configurations. Each configuration can mount a filesystem at a given path, + specyfing whether this file is writable. These keys are used to define mountpoints: + + - `source` -- index of the filesystem that is mounted at this mountpoint. + - `target` -- the path where the filesystem is mounted. + - `writable` -- a boolean value that indicates whether the mountpoint is writable. +- `resource_groups` -- an array of resource group configurations. Each configuration is an object with the following keys: + + - `cpu_usage_limit` -- the maximum percentage of CPU that the task can use. This value must be between 0 and 100 and is + a floating point number. + - `instruction_limit` -- the maximum number of cpu instructions that the task can execute. This value must be a positive integer. + - `memory_limit` -- the maximum amount of memory that the task can use, in bytes. This value must be a positive integer. + - `oom_terminate_all_tasks` -- a boolean value that indicates whether the task should terminate all tasks in the workflow + if it runs out of memory. If true, all tasks will be terminated if the task runs out of memory. + - `pid_limit` -- the maximum number of processes that the task can create. This value must be a positive integer. + - `swap_limit` -- the maximum amount of swap memory that the task can use, in bytes. This value must be a non-negative integer. + - `time_limit` -- the maximum amount of time the task can run, in microseconds. This value must be a positive integer. +- `processes` -- an array of process configurations. Each configuration is an object with the following keys: + + - `arguments` -- an array of strings that are passed as arguments to the process. + - `environment` -- array of environment variables that are passed to the process. Each variable is a string in the format `KEY=VALUE`. + - `image` -- the name of the image that is used to run the process. + - `mount_namespace` -- the index of the mount namespace that the process will use. + - `resource_group` -- the index of the resource group that the process will use. + - `pid_namespace` -- the index of the PID namespace that the process will use. + - `working_directorfy` -- the working directory of the process. This is the directory where the process will be executed. + - `descriptors` -- a dictionary of file descriptors that are attached to the process. Each key is a file descriptor number + (as a string) and each value is a stream. There are several types of streams (specified by `type` key): + + - file stream -- a stream which attaches a file to the file descriptor. It uses following keys: + + - `type` -- the type of the stream, which is `file` for file streams. + - `filesystem` -- the index of the filesystem that contains the file. + - `path` -- the path to the file in the filesystem. + - `mode` -- the mode of the file, which can be `read`, `read_write`, `read_write_append`, `read_write_truncate`, + `write`, `write_append`, or `write_truncate`. + - null stream -- a stream that is a null device. It uses the following keys: + + - `type` -- the type of the stream, which is `null` for null streams. + - object read stream -- a stream that allows reading from an object. It uses the following keys: + + - `type` -- the type of the stream, which is `object_read` for object read streams. + - `handle` -- the handle of the object that is used as a stream. + - object write stream -- a stream that allows writing to an object. It uses the following keys: + + - `type` -- the type of the stream, which is `object_write` for object write streams. + - `handle` -- the handle of the object that is used as a stream. + - pipe read stream -- a stream that allows reading from a pipe. It uses the following keys: + + - `type` -- the type of the stream, which is `pipe_read` for pipe read streams. + - `pipe` -- the index of the pipe to read from. + - pipe write stream -- a stream that allows writing to a pipe. It uses the following keys: + + - `type` -- the type of the stream, which is `pipe_write` for pipe write streams. + - `pipe` -- the index of the pipe to write to. + - `start_after` -- an array of process indices that this process will start after. This is used to define dependencies between processes. + + +Example workflows +------------------ + +Workflow examples can be found in the `example_workflows` directory in the SIO3Pack repository (`here `_). +Every workflow in this directory was generated by SIO3Pack. Files ending with `_workflows.json` are examples of +user defined workflows, that can be used in packages. This is explained later in this document. + +How workflow creation works in SIO3Pack +--------------------------------------- + +SIO3Pack allows a more object-oriented and user friendly way of creating workflows. It provides a set of classes that can +represent workflows, tasks, objects and all the other components of a workflow. These classes can be used to create workflows in a more intuitive way, without +having to write JSON files manually or worry about indexes or register numbers. + +SIO3Pack allows joining multiple workflows together, allowing for writing small workflows that can be reused in larger workflows. +For example, a workflow for generating output tests is created by joining multiple workflows, which are responsible for +generating a single test. In SIO3Pack, registers can be named by strings, which makes it easier to understand the workflow +and allows joining workflows together without worrying about register numbers. All registers starting with `obsreg:` +are considered observable registers, and all other registers are considered normal registers. When a workflow is +converted to JSON, all registers are converted to numbers, and the observable registers are placed at the beginning of the register list. +SIO3Pack also has a simple templating system, for replacing strings in the workflow with values from the context. +Examples of such templates are ``, `` or special `` and ``, +which are replaced with the path to the extra file or executable in the workflow context. + +Detailed documentation of SIO3Pack's workflow classes can be found in the :py:mod:`sio3pack.workflow` module documentation. +Below is a description on how to create own workflows for use in packages. + +User Defined Workflows +---------------------- + +In a package, you can define your own workflows that will be used by SIO3Pack to generate workflows for the package. +These workflows are stored in `workflows.json` file in the package root directory. This files contains a dictionary +of workflows, where keys are names of workflows that you want to override and values are the workflow definitions. + +Here are all currently used workflows and their descriptions: + +- `compile_cpp` -- workflow for compiling C++ source code into an executable. It's used whenever a C++ file needs + to be compiled into an executable. It uses the `g++` compiler and supports various options for compilation. + It uses two templates: `` for the path to the source file and `` for the path to the output executable. +- `compile_python` -- workflow for compiling Python source code into an executable. It's used whenever a Python file needs + to be compiled into an executable. It typically add she-bang to the file and makes it executable. Uses the same + templates as `compile_cpp`. +- `compile_extra` -- a workflow that can be defined to compile any extra files. It is used during unpacking of package, + before compiling any other common files, like checker. It doesn't have any extra templates. +- `ingen` -- workflow for generating input tests. It doesn't have any extra templates. +- `outgen_test` -- workflow for generating a single output test. This workflow is generated for each input test + and then they are combined into a single workflow. Used templates: + + - `` -- the path to the input test file. + - `` -- the path to the output test file. + - `` -- the ID of the test, which should be used to give unique names for registers. + - `` -- the path to the compiled output generator executable. + + The execution output register should be named `r:ougen_res_`. +- `verify_outgen` -- a workflow which verifies if output generation was successful. Typically, this is a script + task that checks if exit status of execution tasks are 0. It uses the following templates: + + - `` -- a Lua template, which generates a map of test IDs to registers. + - `` -- a template for use in `input_registers` key in script tasks. It is replaced with an array + of registers which are output registers of output generation tasks (the `r:ougen_res_` registers). + + The output register of this workflow should be named `obsreg:result`, as it is a final task of outgen. + +- `inwer` -- a workflow which runs inwer (input verification program) for one test. This workflow is generated + for each input test and then they are combined into a single workflow. Used templates: + + - `` -- the path to the input test file. + - `` -- the ID of the test, which should be used to give unique names for registers. + - `` -- path to the compiled inwer executable. + + The execution output register should be named `r:inwer_res_`. + +- `verify_inwer` -- a workflow which verifies that input verification was successful. Typically, this is a script + task that checks if exit status of execution tasks are 0. It uses the following templates: + + - `` -- a Lua template, which generates a map of test IDs to registers. + - `` -- a template for use in `input_registers` key in script tasks. It is replaced with an array + of registers which are output registers of input verification tasks (the `r:inwer_res_` registers). + + The output register of this workflow should be named `obsreg:result`, as it is a final task of inwer. +- `run_test` -- a workflow which runs a program on a single test and grades the solution on this test. This workflow is + generated for each input test, then they are grouped by test groups and finally results of workflows for grading groups + are connected into grading the whole solution. Used templates: + + - `` -- the ID of the test, which should be used to give unique names for registers. + - `` -- the path to the input test file. + - `` -- the path to the output test file. + - `` -- the path to the solution executable. + + The grading results for a test should be stored in `r:grade_res_` register. +- `grade_group` -- a workflow which grades a group of tests. Typically, this is a script task, which takes grading results + as input and produces a grading for a given group. Used templates: + + - `` -- a Lua template, which generates a map of test IDs to registers. + - `` -- a template for use in `input_registers` key in script tasks. It is replaced with an array + of registers which are output registers of grading tasks (the `r:grade_res_` registers). + - `` -- the ID of the group being graded, which should be used to give unique names for registers. + + The group grading results should be stored in `r:grup_grade_res_` register. +- `grade_run` -- a workflow which grades the whole solution, based on grading results of groups. Typically, this is a + script task that takes group grading results as input and produces a final grading for the solution. Used templates: + + - `` -- a Lua template, which generates a map of group IDs to registers. + - `` -- a template for use in `input_registers` key in script tasks. It is replaced with an array + of registers which are output registers of group grading tasks (the `r:group_grade_res_` registers). + + The final grading result should be stored in `obsreg:result` register, as it is a final task of grading the solution. +- `user_out` -- a workflow for generating program's output on a test. Used templates: + + - `` -- the ID of the test. + - `` -- the path to the input test file. + - `` -- path to the program. + + This workflow should generate an observable object `user_out_`, as well as store execution results in + `obsreg:result` register. +- `test_run` -- a workflow for generating program's output on a user-provided test. Used templates: + + - `` -- the path to the input file. + - `` -- path to the program. + - `` -- a path to the user output file. This should be the final observable object. + + This workflow should generate an observable object `` as well as store execution results in + `obsreg:result` register. diff --git a/src/sio3pack/django/common/handler.py b/src/sio3pack/django/common/handler.py index 8bfbc73..27b91dd 100644 --- a/src/sio3pack/django/common/handler.py +++ b/src/sio3pack/django/common/handler.py @@ -26,14 +26,15 @@ class DjangoHandler: Base class for handling Django models. Allows to save the package to the database and retrieve its data. - :param sio3pack.Package package: The package to handle. + :param Package package: The package to handle. :param int problem_id: The problem ID. """ - def __init__(self, package: "sio3pack.Package", problem_id: int): + def __init__(self, package: "Package", problem_id: int): """ Initialize the handler with the package and problem ID. - :param sio3pack.Package package: The package to handle. + + :param Package package: The package to handle. :param int problem_id: The problem ID. """ self.package = package @@ -47,6 +48,8 @@ def __init__(self, package: "sio3pack.Package", problem_id: int): def save_to_db(self): """ Save the package to the database. + + :raises PackageAlreadyExists: If a package with the same problem ID already exists. """ if SIO3Package.objects.filter(problem_id=self.problem_id).exists(): raise PackageAlreadyExists(self.problem_id) @@ -136,7 +139,8 @@ def _save_workflows(self): def get_executable_path(self, program: File | str) -> str | None: """ Get the executable path for the given program. - :param program: The program to get the path for. + + :param File | str program: The program to get the path for. :return: The executable path or None if not found. """ if isinstance(program, File): @@ -170,14 +174,14 @@ def lang_titles(self) -> dict[str, str]: def model_solutions(self) -> list[dict[str, Any]]: """ A list of model solutions, where each element is a dictionary containing - a :class:`sio3pack.RemoteFile` object. + a :class:`RemoteFile` object. """ return [{"file": RemoteFile(s.source_file)} for s in self.db_package.model_solutions.all()] @property def main_model_solution(self) -> RemoteFile: """ - The main model solution as a :class:`sio3pack.RemoteFile`. + The main model solution as a :class:`RemoteFile`. """ return RemoteFile(self.db_package.main_model_solution.source_file) @@ -207,6 +211,6 @@ def tests(self) -> list[Test]: @property def workflows(self) -> dict[str, Workflow]: """ - A dictionary of workflows, where keys are workflow names and values are :class:`sio3pack.Workflow` objects. + A dictionary of workflows, where keys are workflow names and values are :class:`Workflow` objects. """ return {w.name: w.workflow for w in self.db_package.workflows.all()} diff --git a/src/sio3pack/django/sinolpack/handler.py b/src/sio3pack/django/sinolpack/handler.py index e69bae1..d41313e 100644 --- a/src/sio3pack/django/sinolpack/handler.py +++ b/src/sio3pack/django/sinolpack/handler.py @@ -20,9 +20,12 @@ class SinolpackDjangoHandler(DjangoHandler): """ Handler for Sinolpack packages in Django. Has additional properties like config, model_solutions, additional_files and attachments. + + :param Sinolpack package: The Sinolpack package to handle. + :param int problem_id: The problem ID. """ - def __init__(self, package: "sio3pack.Sinolpack", problem_id: int): + def __init__(self, package: "Sinolpack", problem_id: int): super().__init__(package, problem_id) @transaction.atomic @@ -108,7 +111,7 @@ def config(self) -> dict[str, Any]: @property def model_solutions(self) -> list[dict[str, Any]]: """ - A list of model solutions, where each element is a dictionary containing a :class:`sio3pack.RemoteFile` object + A list of model solutions, where each element is a dictionary containing a :class:`RemoteFile` object and the :class:`sio3pack.packages.sinolpack.enums.ModelSolutionKind` kind. """ solutions = SinolpackModelSolution.objects.filter(package=self.db_package) @@ -117,14 +120,14 @@ def model_solutions(self) -> list[dict[str, Any]]: @property def additional_files(self) -> list[RemoteFile]: """ - A list of additional files (as :class:`sio3pack.RemoteFile`) for the problem. + A list of additional files (as :class:`RemoteFile`) for the problem. """ return [RemoteFile(f.file) for f in self.db_package.additional_files.all()] @property def special_files(self) -> dict[str, RemoteFile]: """ - A dictionary of special files (as :class:`sio3pack.RemoteFile`) for the problem. + A dictionary of special files (as :class:`RemoteFile`) for the problem. The keys are the types of the special files. """ res = {} @@ -139,7 +142,7 @@ def special_files(self) -> dict[str, RemoteFile]: @property def extra_execution_files(self) -> list[RemoteFile]: """ - A list of extra execution files (as :class:`sio3pack.RemoteFile`) specified in the config file. + A list of extra execution files (as :class:`RemoteFile`) specified in the config file. """ files = self.config.get("extra_execution_files", []) return [RemoteFile(f.file) for f in self.db_package.additional_files.filter(name__in=files)] @@ -147,7 +150,7 @@ def extra_execution_files(self) -> list[RemoteFile]: @property def extra_compilation_files(self) -> list[RemoteFile]: """ - A list of extra compilation files (as :class:`sio3pack.RemoteFile`) specified in the config file. + A list of extra compilation files (as :class:`RemoteFile`) specified in the config file. """ files = self.config.get("extra_compilation_files", []) return [RemoteFile(f.file) for f in self.db_package.additional_files.filter(name__in=files)] @@ -155,14 +158,14 @@ def extra_compilation_files(self) -> list[RemoteFile]: @property def attachments(self) -> list[RemoteFile]: """ - A list of attachments (as :class:`sio3pack.RemoteFile`) related to the problem. + A list of attachments (as :class:`RemoteFile`) related to the problem. """ return [RemoteFile(f.content) for f in self.db_package.attachments.all()] @property def extra_files(self) -> dict[str, RemoteFile]: """ - A dictionary of extra files (as :class:`sio3pack.RemoteFile`) for the problem, as + A dictionary of extra files (as :class:`RemoteFile`) for the problem, as specified in the config file. The keys are the paths of the files in the package. """ files = self.db_package.extra_files.all() @@ -170,10 +173,10 @@ def extra_files(self) -> dict[str, RemoteFile]: def get_extra_file(self, package_path: str) -> RemoteFile | None: """ - Get an extra file (as :class:`sio3pack.RemoteFile`) for the problem. + Get an extra file (as :class:`RemoteFile`) for the problem. :param package_path: The path of the file in the package. - :return: The extra file (as :class:`sio3pack.RemoteFile`) or None if it does not exist. + :return: The extra file (as :class:`RemoteFile`) or None if it does not exist. """ try: extra_file = self.db_package.extra_files.get(package_path=package_path) diff --git a/src/sio3pack/files/local_file.py b/src/sio3pack/files/local_file.py index 572476d..f17115b 100644 --- a/src/sio3pack/files/local_file.py +++ b/src/sio3pack/files/local_file.py @@ -5,7 +5,10 @@ class LocalFile(File): """ - Base class for a file in a package that is stored locally. + Class for a file in a package that is stored locally. + + :param str path: The path to the file. + :param str filename: The name of the file. """ @classmethod diff --git a/src/sio3pack/files/remote_file.py b/src/sio3pack/files/remote_file.py index fce848d..f8961f4 100644 --- a/src/sio3pack/files/remote_file.py +++ b/src/sio3pack/files/remote_file.py @@ -5,7 +5,10 @@ class RemoteFile(File): """ - Base class for a file that is tracked by filetracker. + Class for a file that is tracked by filetracker. + + :param oioioi.filetracker.fields.FileField file: The file field from the filetracker. + :param str filename: The name of the file. """ try: diff --git a/src/sio3pack/lua/__init__.py b/src/sio3pack/lua/__init__.py index c32c663..be46369 100644 --- a/src/sio3pack/lua/__init__.py +++ b/src/sio3pack/lua/__init__.py @@ -5,8 +5,8 @@ def get_script(name: str, templates: dict[str, str] = None) -> str: """ Get the script for the given name and replace templates with the given replacements. - :param name: The name of the script. - :param templates: The templates to replace. + :param str name: The name of the script. + :param dict[str, str] templates: The templates to replace. """ script = os.path.join(os.path.dirname(__file__), "scripts", f"{name}.lua") if not os.path.exists(script): @@ -21,7 +21,7 @@ def get_script(name: str, templates: dict[str, str] = None) -> str: def to_lua_map(data: dict[str, str]) -> str: """ - Convert a dictionary to a Lua map. + Convert a Python dictionary to a Lua map. :param data: The dictionary to convert. """ diff --git a/src/sio3pack/packages/__init__.py b/src/sio3pack/packages/__init__.py index 22919aa..e69de29 100644 --- a/src/sio3pack/packages/__init__.py +++ b/src/sio3pack/packages/__init__.py @@ -1 +0,0 @@ -from sio3pack.packages.sinolpack import Sinolpack diff --git a/src/sio3pack/packages/package/__init__.py b/src/sio3pack/packages/package/__init__.py index f3d8b76..245779f 100644 --- a/src/sio3pack/packages/package/__init__.py +++ b/src/sio3pack/packages/package/__init__.py @@ -1 +1,2 @@ +from sio3pack.packages.package.configuration import CompilerConfig, SIO3PackConfig from sio3pack.packages.package.model import Package diff --git a/src/sio3pack/packages/package/configuration.py b/src/sio3pack/packages/package/configuration.py index 49ce603..52a2b09 100644 --- a/src/sio3pack/packages/package/configuration.py +++ b/src/sio3pack/packages/package/configuration.py @@ -1,4 +1,13 @@ class CompilerConfig: + """ + Configuration class for a compiler. It holds the name, full name, path, and flags of the compiler. + + :param name: The name of the compiler. + :param full_name: The full name of the compiler, including version. + :param path: The path to the compiler executable. + :param flags: The flags to use when compiling with this compiler. + """ + def __init__(self, name: str, full_name: str, path: str, flags: list[str]): self.name = name self.full_name = full_name @@ -19,7 +28,14 @@ def detect(cls) -> dict[str, "CompilerConfig"]: class SIO3PackConfig: """ - Configuration class for SIO3Pack. + Configuration class for SIO3Pack. It holds the configuration for compilers and file extensions. + It can be initialized with Django settings or detected automatically. + + :param django_settings: Django settings object. + :param compilers_config: Dictionary of compiler configurations. The keys are the compiler names, + and the values are CompilerConfig objects. + :param extensions_config: Dictionary of language configurations. The keys are the file extensions, + and the values are the corresponding languages. """ def __init__( diff --git a/src/sio3pack/packages/sinolpack/__init__.py b/src/sio3pack/packages/sinolpack/__init__.py index e2c0d63..26bd795 100644 --- a/src/sio3pack/packages/sinolpack/__init__.py +++ b/src/sio3pack/packages/sinolpack/__init__.py @@ -1 +1,2 @@ from sio3pack.packages.sinolpack.model import Sinolpack +from sio3pack.packages.sinolpack.workflows import SinolpackWorkflowManager diff --git a/src/sio3pack/packages/sinolpack/workflows.py b/src/sio3pack/packages/sinolpack/workflows.py index f5bcf37..9fa769a 100644 --- a/src/sio3pack/packages/sinolpack/workflows.py +++ b/src/sio3pack/packages/sinolpack/workflows.py @@ -22,6 +22,16 @@ class UnpackStage(Enum): class SinolpackWorkflowManager(WorkflowManager): + """ + A workflow manager for Sinolpack packages. It extends the base WorkflowManager + and provides additional workflows for ingen, outgen, inwer, and other Sinolpack-specific tasks. + It also overrides the `get_compile_file_workflow` method to add extra compilation files + to the workflow. + + :param Sinolpack package: The Sinolpack package to manage workflows for. + :param dict[str, Workflow] workflows: A dictionary of workflows to manage. + """ + def __init__(self, package: "Sinolpack", workflows: dict[str, Any]): super().__init__(package, workflows) self._has_ingen = False @@ -257,7 +267,6 @@ def _get_compile_files_workflows(self, data: dict) -> tuple[Workflow, bool]: # Get the workflow for compiling any extra files from package's workflow's config extra_wf = self.get("compile_extra") - print("xddd", extra_wf) if extra_wf is not None: to_replace = self._add_extra_files_to_replace(extra_wf, {}) extra_wf.replace_templates(to_replace) diff --git a/src/sio3pack/test/test.py b/src/sio3pack/test/test.py index 6341f44..e6e3c64 100644 --- a/src/sio3pack/test/test.py +++ b/src/sio3pack/test/test.py @@ -4,6 +4,12 @@ class Test: """ Represents an input and output test. + + :param str test_name: Name of the test. + :param str test_id: Unique identifier for the test. + :param File in_file: Input file for the test. + :param File out_file: Output file for the test. + :param str group: Group identifier for the test. """ def __init__(self, test_name: str, test_id: str, in_file: File, out_file: File, group: str): diff --git a/src/sio3pack/visualizer/__init__.py b/src/sio3pack/visualizer/__init__.py index 41754a5..22b4551 100644 --- a/src/sio3pack/visualizer/__init__.py +++ b/src/sio3pack/visualizer/__init__.py @@ -15,6 +15,10 @@ def main(): + """ + Main entry point for the SIO3Worker visualizer. + """ + app = dash.Dash(__name__) app.layout = html.Div( [ diff --git a/src/sio3pack/workflow/__init__.py b/src/sio3pack/workflow/__init__.py index e72525b..e0078c3 100644 --- a/src/sio3pack/workflow/__init__.py +++ b/src/sio3pack/workflow/__init__.py @@ -1,4 +1,4 @@ -from sio3pack.workflow.object import Object +from sio3pack.workflow.object import Object, ObjectList, ObjectsManager from sio3pack.workflow.tasks import ExecutionTask, ScriptTask, Task from sio3pack.workflow.workflow import Workflow from sio3pack.workflow.workflow_manager import WorkflowManager diff --git a/src/sio3pack/workflow/execution/__init__.py b/src/sio3pack/workflow/execution/__init__.py index dc2a7a3..54924e0 100644 --- a/src/sio3pack/workflow/execution/__init__.py +++ b/src/sio3pack/workflow/execution/__init__.py @@ -1,7 +1,25 @@ from sio3pack.workflow.execution.channels import Channel from sio3pack.workflow.execution.descriptors import DescriptorManager -from sio3pack.workflow.execution.filesystems import Filesystem -from sio3pack.workflow.execution.mount_namespace import MountNamespace +from sio3pack.workflow.execution.filesystems import ( + EmptyFilesystem, + Filesystem, + FilesystemManager, + ImageFilesystem, + ObjectFilesystem, +) +from sio3pack.workflow.execution.mount_namespace import MountNamespace, MountNamespaceManager, Mountpoint from sio3pack.workflow.execution.process import Process -from sio3pack.workflow.execution.resource_group import ResourceGroup -from sio3pack.workflow.execution.stream import * +from sio3pack.workflow.execution.resource_group import ResourceGroup, ResourceGroupManager +from sio3pack.workflow.execution.stream import ( + FileMode, + FileStream, + NullStream, + ObjectReadStream, + ObjectStream, + ObjectWriteStream, + PipeReadStream, + PipeStream, + PipeWriteStream, + Stream, + StreamType, +) diff --git a/src/sio3pack/workflow/execution/descriptors.py b/src/sio3pack/workflow/execution/descriptors.py index 9fac4a4..116c4a6 100644 --- a/src/sio3pack/workflow/execution/descriptors.py +++ b/src/sio3pack/workflow/execution/descriptors.py @@ -48,7 +48,7 @@ def to_json(self) -> dict: # Convert the fd numbers to strings, since in JSON keys cant be ints. return {str(fd): stream.to_json() for fd, stream in self.descriptors.items()} - def items(self) -> ItemsView[int, Stream]: + def items(self) -> ItemsView[int, "Stream"]: """ Get the items in the descriptor manager. @@ -56,7 +56,7 @@ def items(self) -> ItemsView[int, Stream]: """ return self.descriptors.items() - def all(self) -> dict[int, Stream]: + def all(self) -> dict[int, "Stream"]: """ Get all the streams in the descriptor manager. @@ -64,7 +64,7 @@ def all(self) -> dict[int, Stream]: """ return self.descriptors - def get(self, fd: int) -> Stream: + def get(self, fd: int) -> "Stream": """ Get a stream by its file descriptor. diff --git a/src/sio3pack/workflow/execution/mount_namespace.py b/src/sio3pack/workflow/execution/mount_namespace.py index 23a8f9e..03895ea 100644 --- a/src/sio3pack/workflow/execution/mount_namespace.py +++ b/src/sio3pack/workflow/execution/mount_namespace.py @@ -5,13 +5,13 @@ class Mountpoint: """ A class to represent a mountpoint. - :param Filesystem source: The source filesystem. + :param "Filesystem" source: The source filesystem. :param str target: The target path in the filesystem. :param bool writable: Whether the mountpoint is writable or not. :param int capacity: The capacity of the mountpoint. If None, the capacity is unlimited. """ - def __init__(self, source: Filesystem, target: str, writable: bool = False, capacity: int | None = None): + def __init__(self, source: "Filesystem", target: str, writable: bool = False, capacity: int | None = None): """ Represent a mountpoint. @@ -50,7 +50,7 @@ def to_json(self) -> dict: class MountNamespace: """ A class to represent a mount namespace. - It can mount an in the target filesystem. + It can mount :class:`Mountpoint` instances in the target filesystem. :param int id: The id of the mount namespace. :param list[Mountpoint] mountpoints: The mountpoints in the mount namespace. @@ -99,10 +99,12 @@ def add_mountpoint(self, mountpoint: Mountpoint): class MountNamespaceManager: - def __init__(self, task: "Task", filesystem_manager: FilesystemManager): + def __init__(self, task: "Task", filesystem_manager: "FilesystemManager"): """ Create a new mount namespace manager. - :param task: The task the mount namespace manager belongs to. + + :param Task task: The task the mount namespace manager belongs to. + :param FilesystemManager filesystem_manager: Workflow's filesystem manager. """ self.mount_namespaces: list[MountNamespace] = [] self.id = 0 @@ -112,8 +114,8 @@ def __init__(self, task: "Task", filesystem_manager: FilesystemManager): def from_json(self, data: list[dict]): """ Create a new mount namespace manager from a list of dictionaries. + :param data: The list of dictionaries to create the mount namespace manager from. - :param FilesystemManager filesystem_manager: The filesystem manager to use. """ for mount_namespace in data: self.add(MountNamespace.from_json(mount_namespace, self.id, self.filesystem_manager)) @@ -122,7 +124,8 @@ def from_json(self, data: list[dict]): def add(self, mount_namespace: MountNamespace): """ Add a mount namespace to the manager. - :param mount_namespace: The mount namespace to add. + + :param MountNamespace mount_namespace: The mount namespace to add. """ mount_namespace._set_id(self.id) self.mount_namespaces.append(mount_namespace) @@ -130,6 +133,7 @@ def add(self, mount_namespace: MountNamespace): def get_by_id(self, id: int) -> MountNamespace: """ Get a mount namespace by its id. + :param id: The id of the mount namespace. """ return self.mount_namespaces[id] diff --git a/src/sio3pack/workflow/execution/process.py b/src/sio3pack/workflow/execution/process.py index 2dd833c..e9c8ed3 100644 --- a/src/sio3pack/workflow/execution/process.py +++ b/src/sio3pack/workflow/execution/process.py @@ -4,6 +4,21 @@ class Process: + """ + A class to represent a process in a workflow. + + :param Workflow workflow: The workflow the process belongs to. + :param Executiontask task: The task which the process belongs to. + :param list[str] arguments: Executable arguments for the process. + :param dict[str, str] environment: Environment variables for the process. + :param str image: The image of the process, which can be a Docker image or similar. + :param MountNamespace mount_namespace: The mount namespace to use for the process. + :param ResourceGroup resource_group: The resource group of the process. + :param str working_directory: The working directory of the process. + :param int pid_namespace: The PID namespace of the process. + :param list[int] start_after: The processes that must be finished before this process starts. + """ + def __init__( self, workflow: "Workflow", @@ -19,6 +34,7 @@ def __init__( ): """ Represent a process. + :param arguments: The arguments of the process. :param environment: The environment of the process. :param image: The image of the process. @@ -43,6 +59,10 @@ def __init__( self.start_after = start_after or [] def to_json(self) -> dict: + """ + Convert the process to a JSON-serializable dictionary. + """ + return { "arguments": self.arguments, "environment": [f"{key}={value}" for key, value in self.environment.items()], @@ -57,6 +77,14 @@ def to_json(self) -> dict: @classmethod def from_json(cls, data: dict, workflow: "Workflow", task: "Task"): + """ + Create a new process from a dictionary. + + :param data: The dictionary to create the process from. + :param workflow: The workflow the process belongs to. + :param task: The task the process belongs to. + """ + env = {} for var in data["environment"]: key, value = var.split("=", 1) @@ -79,7 +107,8 @@ def from_json(cls, data: dict, workflow: "Workflow", task: "Task"): def replace_templates(self, replacements: dict[str, str]): """ Replace strings in the process with the given replacements. - :param replacements: The replacements to make. + + :param dict[str, str] replacements: The replacements to make. """ for key, value in replacements.items(): if key in self.image: diff --git a/src/sio3pack/workflow/execution/resource_group.py b/src/sio3pack/workflow/execution/resource_group.py index 43a5750..c000580 100644 --- a/src/sio3pack/workflow/execution/resource_group.py +++ b/src/sio3pack/workflow/execution/resource_group.py @@ -1,4 +1,18 @@ class ResourceGroup: + """ + A resource group is a set of limits that can be applied to a task. + It can limit CPU usage, instruction usage, memory usage, and more. + + :param id: The id of the resource group. + :param cpu_usage_limit: The CPU usage limit. + :param instruction_limit: The instruction usage limit. + :param memory_limit: The memory limit. + :param oom_terminate_all_tasks: Whether to terminate all tasks on OOM. + :param pid_limit: The PID limit. + :param swap_limit: The swap limit. + :param time_limit: The time limit. + """ + def __init__( self, cpu_usage_limit: int = 100.0, @@ -12,6 +26,7 @@ def __init__( ): """ Create a new resource group. + :param id: The id of the resource group. :param cpu_usage_limit: The CPU usage limit. :param instruction_limit: The instruction usage limit. @@ -41,6 +56,7 @@ def _set_id(self, id: int): def set_limits(self, cpu_usage_limit: int, instruction_limit: int, memory_limit: int, time_limit: int): """ Set the limits of the resource group. + :param cpu_usage_limit: The CPU usage limit. :param instruction_limit: The instruction usage limit. :param memory_limit: The memory limit. @@ -55,6 +71,7 @@ def set_limits(self, cpu_usage_limit: int, instruction_limit: int, memory_limit: def from_json(cls, data: dict, id: int): """ Create a new resource group from a dictionary. + :param data: The dictionary to create the resource group from. :param id: The id of the resource group. """ @@ -85,9 +102,14 @@ def to_json(self) -> dict: class ResourceGroupManager: + """ + A class to manage resource groups in a workflow. Allows creation, retrieval, and management of resource groups. + """ + def __init__(self, task: "Task"): """ Create a new resource group manager. + :param task: The task the resource group manager belongs to. """ self.resource_groups: list[ResourceGroup] = [] @@ -96,7 +118,8 @@ def __init__(self, task: "Task"): def add(self, resource_group: ResourceGroup): """ Add a resource group to the resource group manager. - :param resource_group: The resource group to add. + + :param ResourceGroup resource_group: The resource group to add. """ resource_group._set_id(self.id) self.resource_groups.append(resource_group) @@ -105,6 +128,7 @@ def add(self, resource_group: ResourceGroup): def get_by_id(self, id: int) -> ResourceGroup: """ Get a resource group by its id. + :param id: The id of the resource group to get. """ return self.resource_groups[id] @@ -118,6 +142,7 @@ def to_json(self) -> list[dict]: def from_json(self, data: list[dict]): """ Create a new resource group manager from a list of dictionaries. + :param data: The list of dictionaries to create the resource group manager from. """ for resource_group in data: diff --git a/src/sio3pack/workflow/execution/stream.py b/src/sio3pack/workflow/execution/stream.py index b5a8793..1cd92d6 100644 --- a/src/sio3pack/workflow/execution/stream.py +++ b/src/sio3pack/workflow/execution/stream.py @@ -1,8 +1,7 @@ from enum import Enum -from sio3pack.workflow import Object from sio3pack.workflow.execution.filesystems import Filesystem, FilesystemManager -from sio3pack.workflow.object import ObjectsManager +from sio3pack.workflow.object import Object, ObjectsManager class StreamType(Enum): @@ -70,6 +69,9 @@ def from_json(cls, data: dict, objects_manager: ObjectsManager, filesystem_manag raise ValueError(f"Unknown stream type: {type}") def to_json(self) -> dict: + """ + Convert the stream to a JSON-serializable dictionary. + """ raise NotImplementedError("Subclasses must implement to_json method") def replace_templates(self, replacements: dict[str, str]): diff --git a/src/sio3pack/workflow/object.py b/src/sio3pack/workflow/object.py index f6b5b1e..dd08771 100644 --- a/src/sio3pack/workflow/object.py +++ b/src/sio3pack/workflow/object.py @@ -13,7 +13,7 @@ def __init__(self, handle: str): """ Create a new object. - :param handle: The handle of the object. + :param str handle: The handle of the object. """ self.handle = handle @@ -27,7 +27,7 @@ def replace_templates(self, replacements: dict[str, str]): """ Replace strings in the object with the given replacements. - :param replacements: The replacements to make. + :param dict[str, str] replacements: The replacements to make. """ for key, value in replacements.items(): if key in self.handle: @@ -35,13 +35,18 @@ def replace_templates(self, replacements: dict[str, str]): class ObjectsManager: + """ + A class to manage objects in a workflow. Allows creation, retrieval, and management of objects. + """ + def __init__(self): self.objects = {} def create_object(self, handle: str) -> Object: """ Create and return a new object. - :param handle: The handle of the object. + + :param str handle: The handle of the object. :return: The created object. """ obj = Object(handle) @@ -51,21 +56,24 @@ def create_object(self, handle: str) -> Object: def add_object(self, obj: Object): """ Add an object to the manager. - :param obj: The object to add. + + :param Object obj: The object to add. """ self.objects[obj.handle] = obj def get_object(self, handle: str) -> Object: """ Get an object by its handle. - :param handle: The handle of the object. + + :param str handle: The handle of the object. """ return self.objects[handle] def get_or_create_object(self, handle: str) -> Object: """ Get an object by its handle, creating it if it does not exist. - :param handle: The handle of the object. + + :param str handle: The handle of the object. """ if handle not in self.objects: return self.create_object(handle) diff --git a/src/sio3pack/workflow/tasks.py b/src/sio3pack/workflow/tasks.py index 5a6d1ab..9827839 100644 --- a/src/sio3pack/workflow/tasks.py +++ b/src/sio3pack/workflow/tasks.py @@ -1,11 +1,11 @@ import re -from sio3pack.workflow import Object from sio3pack.workflow.execution.channels import Channel from sio3pack.workflow.execution.filesystems import Filesystem, FilesystemManager from sio3pack.workflow.execution.mount_namespace import MountNamespace, MountNamespaceManager from sio3pack.workflow.execution.process import Process from sio3pack.workflow.execution.resource_group import ResourceGroup, ResourceGroupManager +from sio3pack.workflow.object import Object class Task: diff --git a/src/sio3pack/workflow/workflow.py b/src/sio3pack/workflow/workflow.py index 2c35642..ac3a249 100644 --- a/src/sio3pack/workflow/workflow.py +++ b/src/sio3pack/workflow/workflow.py @@ -1,11 +1,10 @@ -from sio3pack.workflow import ExecutionTask, Object, ScriptTask -from sio3pack.workflow.object import ObjectList, ObjectsManager -from sio3pack.workflow.tasks import Task +from sio3pack.workflow.object import Object, ObjectList, ObjectsManager +from sio3pack.workflow.tasks import ExecutionTask, ScriptTask, Task class Workflow: """ - A class to represent a job workflow. Number of registers is not required, + A class to represent a workflow. Number of registers is not required, as it is calculated automatically. :param str name: The name of the workflow. diff --git a/src/sio3pack/workflow/workflow_manager.py b/src/sio3pack/workflow/workflow_manager.py index 9b2a964..71c60e8 100644 --- a/src/sio3pack/workflow/workflow_manager.py +++ b/src/sio3pack/workflow/workflow_manager.py @@ -4,10 +4,11 @@ from sio3pack.files import File from sio3pack.test import Test -from sio3pack.workflow import ExecutionTask, constants +from sio3pack.workflow import constants from sio3pack.workflow.execution import MountNamespace, ObjectWriteStream, Process, ResourceGroup from sio3pack.workflow.execution.filesystems import ObjectFilesystem from sio3pack.workflow.execution.mount_namespace import Mountpoint +from sio3pack.workflow.tasks import ExecutionTask from sio3pack.workflow.workflow import Workflow from sio3pack.workflow.workflow_op import WorkflowOperation @@ -25,6 +26,18 @@ class UnpackStage(Enum): class WorkflowManager: + """ + A class to manage workflows for a package. Allows to get workflows by name, + manages default and user created workflows, and provides methods + to get workflows for compiling files, generating tests, and verifying results. + This class can be overridden to provide custom workflows + in other package types. + + :param Package package: The package for which the workflows are managed. + :param dict[str, Workflow] workflows: A dictionary of user defined workflows, + where the key is the workflow name and the value is the Workflow object. + """ + def __init__(self, package: "Package", workflows: dict[str, Any]): for name, wf in workflows.items(): if isinstance(wf, dict): @@ -41,9 +54,9 @@ def get(self, name: str) -> Workflow: """ Get the workflow with the given name. If the workflow does not exist, return default - workflow for this name from self.get_default. + workflow for this name from :meth:`WorkflowManager.get_default`. - :param name: The name of the workflow. + :param str name: The name of the workflow. :return: The workflow with the given name. """ if name not in self.workflows: @@ -53,7 +66,7 @@ def get(self, name: str) -> Workflow: def all(self) -> dict[str, Workflow]: """ - Get all workflows. + Get all user defined workflows. :return: A dictionary of all workflows. """ @@ -89,7 +102,7 @@ def get_compile_file_workflow(self, file: File | str) -> tuple[Workflow, str]: The files are not added as external or observable objects, since they don't have to be. - :param file: The file (or the path to the file) to compile. + :param File | str file: The file (or the path to the file) to compile. :return: A tuple of the workflow and the path to the compiled file. """ if isinstance(file, File): @@ -270,6 +283,13 @@ def _get_unpack_workflows(self, data: dict) -> tuple[Workflow, bool]: def get_unpack_operation( self, has_test_gen: bool, has_verify: bool, return_func: callable = None ) -> WorkflowOperation: + """ + Get the operation for unpacking the package. + + :param bool has_test_gen: Whether the package has test generation. + :param bool has_verify: Whether the package has verification. + :param callable return_func: A function to call with the results of the workflow execution. + """ self._has_test_gen = has_test_gen self._has_verify = has_verify # At first, compile all required files @@ -281,10 +301,31 @@ def get_unpack_operation( def get_run_operation( self, program: File, tests: list[Test] | None = None, return_func: callable = None ) -> WorkflowOperation: + """ + Get the operation for running the program with the given tests. + + :param File program: The program file to run. + :param list[Test] | None tests: The list of tests to run the program with or None if all tests should be run. + :param callable return_func: A function to call with the results of the workflow execution. + """ raise NotImplementedError def get_user_out_operation(self, program: File, test: Test, return_func: callable = None) -> WorkflowOperation: + """ + Get the operation for running the user output with the given program and test. + + :param File program: The program file to run. + :param Test test: The test to run the program with. + :param callable return_func: A function to call with the results of the workflow execution. + """ raise NotImplementedError def get_test_run_operation(self, program: File, test: File, return_func: callable = None) -> WorkflowOperation: + """ + Get the operation for running the test with the given program. + + :param File program: The program file to run. + :param File test: The test file to run the program with. + :param callable return_func: A function to call with the results of the workflow execution. + """ raise NotImplementedError diff --git a/src/sio3pack/workflow/workflow_op.py b/src/sio3pack/workflow/workflow_op.py index fd1277e..4c97abb 100644 --- a/src/sio3pack/workflow/workflow_op.py +++ b/src/sio3pack/workflow/workflow_op.py @@ -1,12 +1,33 @@ class WorkflowOperation: + """ + A class to handle workflow operations, allowing for the retrieval of workflows + and the return of results from those workflows. + + :param callable get_workflow_func: Function to retrieve workflows. + :param bool return_results: Whether to return results from the workflow. + :param callable return_results_func: Function to handle returning results. + :param list[Any] wf_args: Additional positional arguments for the workflow function. + :param dict[str, Any] wf_kwargs: Additional keyword arguments for the workflow function. + """ + def __init__( self, get_workflow_func: callable, - return_results=False, + return_results: bool = False, return_results_func: callable = None, *wf_args, **wf_kwargs ): + """ + Initialize the WorkflowOperation with a function to get workflows and + optionally a function to return results. + + :param callable get_workflow_func: Function to retrieve workflows. + :param bool return_results: Whether to return results from the workflow. + :param callable return_results_func: Function to handle returning results. + :param wf_args: Additional positional arguments for the workflow function. + :param wf_kwargs: Additional keyword arguments for the workflow function. + """ self.get_workflow_func = get_workflow_func self.return_results = return_results self.return_results_func = return_results_func @@ -17,6 +38,11 @@ def __init__( self._workflow_kwargs = wf_kwargs def get_workflow(self): + """ + A function to retrieve workflows. It yields workflows until the last one is reached. + This function uses the provided `get_workflow_func` to get workflows and + yields them one by one. + """ while not self._last: self._workflow, self._last = self.get_workflow_func( self._data, *self._workflow_args, **self._workflow_kwargs @@ -24,6 +50,11 @@ def get_workflow(self): yield self._workflow def return_results(self, data: dict): + """ + A function to return results from the workflow. If `return_results_func` is provided, + it will be called with the workflow and data. Also stores the data internally for next + workflow retrieval. + """ if self.return_results_func: return self.return_results_func(self._workflow, data) self._data = data diff --git a/tests/packages/sinolpack/test_utils.py b/tests/packages/sinolpack/test_utils.py index 34e503b..4366d50 100644 --- a/tests/packages/sinolpack/test_utils.py +++ b/tests/packages/sinolpack/test_utils.py @@ -1,8 +1,7 @@ import pytest import sio3pack -from sio3pack.packages import Sinolpack -from sio3pack.packages.sinolpack import constants +from sio3pack.packages.sinolpack import Sinolpack, constants from sio3pack.test import Test from tests.fixtures import PackageInfo, get_package diff --git a/tests/packages/sinolpack/test_workflows.py b/tests/packages/sinolpack/test_workflows.py index 0594002..dc81bc6 100644 --- a/tests/packages/sinolpack/test_workflows.py +++ b/tests/packages/sinolpack/test_workflows.py @@ -6,8 +6,8 @@ import sio3pack from sio3pack.exceptions import WorkflowCreationError -from sio3pack.packages import Sinolpack from sio3pack.packages.package.configuration import SIO3PackConfig +from sio3pack.packages.sinolpack import Sinolpack from sio3pack.workflow import ExecutionTask, ScriptTask, Workflow from sio3pack.workflow.execution import ObjectReadStream, ObjectWriteStream from sio3pack.workflow.execution.filesystems import ObjectFilesystem diff --git a/tests/packages/sinolpack/utils.py b/tests/packages/sinolpack/utils.py index 2e689d2..e9c4a54 100644 --- a/tests/packages/sinolpack/utils.py +++ b/tests/packages/sinolpack/utils.py @@ -1,5 +1,5 @@ from sio3pack import Package -from sio3pack.packages import Sinolpack +from sio3pack.packages.sinolpack import Sinolpack from tests.fixtures import PackageInfo diff --git a/tests/test_django/test_sio3pack/test_sinolpack.py b/tests/test_django/test_sio3pack/test_sinolpack.py index 413a04b..cf6ab1d 100644 --- a/tests/test_django/test_sio3pack/test_sinolpack.py +++ b/tests/test_django/test_sio3pack/test_sinolpack.py @@ -9,7 +9,7 @@ SinolpackModelSolution, SinolpackSpecialFile, ) -from sio3pack.packages import Sinolpack +from sio3pack.packages.sinolpack import Sinolpack from tests.fixtures import Compression, PackageInfo, get_archived_package, get_package from tests.utils import assert_contents_equal