|
| 1 | +# Copyright (C) 2021 Intel Corporation |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, |
| 10 | +# software distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions |
| 13 | +# and limitations under the License. |
| 14 | + |
| 15 | +import glob |
| 16 | +import logging |
| 17 | +import os |
| 18 | +import os.path as osp |
| 19 | +from collections import namedtuple |
| 20 | +from copy import deepcopy |
| 21 | +from pprint import pformat |
| 22 | +from typing import Any, Callable, Dict, List, Optional |
| 23 | + |
| 24 | +import pytest |
| 25 | +import yaml |
| 26 | +from ote_sdk.entities.datasets import DatasetEntity |
| 27 | +from ote_sdk.entities.label_schema import LabelSchemaEntity |
| 28 | +from ote_sdk.entities.subset import Subset |
| 29 | + |
| 30 | +from mmseg.apis.ote.extension.datasets.mmdataset import load_dataset_items |
| 31 | + |
| 32 | +from ote_sdk.test_suite.e2e_test_system import DataCollector, e2e_pytest_performance |
| 33 | +from ote_sdk.test_suite.training_tests_common import (make_path_be_abs, |
| 34 | + make_paths_be_abs, |
| 35 | + KEEP_CONFIG_FIELD_VALUE, |
| 36 | + REALLIFE_USECASE_CONSTANT, |
| 37 | + ROOT_PATH_KEY) |
| 38 | +from ote_sdk.test_suite.training_tests_helper import (OTETestHelper, |
| 39 | + DefaultOTETestCreationParametersInterface, |
| 40 | + OTETrainingTestInterface) |
| 41 | + |
| 42 | + |
| 43 | +logger = logging.getLogger(__name__) |
| 44 | + |
| 45 | +def DATASET_PARAMETERS_FIELDS() -> List[str]: |
| 46 | + return deepcopy(['annotations_train', |
| 47 | + 'images_train_dir', |
| 48 | + 'annotations_val', |
| 49 | + 'images_val_dir', |
| 50 | + 'annotations_test', |
| 51 | + 'images_test_dir', |
| 52 | + ]) |
| 53 | + |
| 54 | +DatasetParameters = namedtuple('DatasetParameters', DATASET_PARAMETERS_FIELDS()) |
| 55 | + |
| 56 | + |
| 57 | +def _get_dataset_params_from_dataset_definitions(dataset_definitions, dataset_name): |
| 58 | + if dataset_name not in dataset_definitions: |
| 59 | + raise ValueError(f'dataset {dataset_name} is absent in dataset_definitions, ' |
| 60 | + f'dataset_definitions.keys={list(dataset_definitions.keys())}') |
| 61 | + cur_dataset_definition = dataset_definitions[dataset_name] |
| 62 | + training_parameters_fields = {k: v for k, v in cur_dataset_definition.items() |
| 63 | + if k in DATASET_PARAMETERS_FIELDS()} |
| 64 | + make_paths_be_abs(training_parameters_fields, dataset_definitions[ROOT_PATH_KEY]) |
| 65 | + |
| 66 | + assert set(DATASET_PARAMETERS_FIELDS()) == set(training_parameters_fields.keys()), \ |
| 67 | + f'ERROR: dataset definitions for name={dataset_name} does not contain all required fields' |
| 68 | + assert all(training_parameters_fields.values()), \ |
| 69 | + f'ERROR: dataset definitions for name={dataset_name} contains empty values for some required fields' |
| 70 | + |
| 71 | + params = DatasetParameters(**training_parameters_fields) |
| 72 | + return params |
| 73 | + |
| 74 | + |
| 75 | +def _create_segmentation_dataset_and_labels_schema(dataset_params): |
| 76 | + logger.debug(f'Using for train annotation file {dataset_params.annotations_train}') |
| 77 | + logger.debug(f'Using for val annotation file {dataset_params.annotations_val}') |
| 78 | + labels_list = [] |
| 79 | + items = load_dataset_items( |
| 80 | + ann_file_path=dataset_params.annotations_train, |
| 81 | + data_root_dir=dataset_params.images_train_dir, |
| 82 | + subset=Subset.TRAINING, |
| 83 | + labels_list=labels_list) |
| 84 | + items.extend(load_dataset_items( |
| 85 | + ann_file_path=dataset_params.annotations_val, |
| 86 | + data_root_dir=dataset_params.images_val_dir, |
| 87 | + subset=Subset.VALIDATION, |
| 88 | + labels_list=labels_list)) |
| 89 | + items.extend(load_dataset_items( |
| 90 | + ann_file_path=dataset_params.annotations_test, |
| 91 | + data_root_dir=dataset_params.images_test_dir, |
| 92 | + subset=Subset.TESTING, |
| 93 | + labels_list=labels_list)) |
| 94 | + dataset = DatasetEntity(items=items) |
| 95 | + labels_schema = LabelSchemaEntity.from_labels(labels_list) |
| 96 | + return dataset, labels_schema |
| 97 | + |
| 98 | + |
| 99 | +class SegmentationTrainingTestParameters(DefaultOTETestCreationParametersInterface): |
| 100 | + def test_bunches(self) -> List[Dict[str, Any]]: |
| 101 | + test_bunches = [ |
| 102 | + dict( |
| 103 | + model_name=[ |
| 104 | + 'Custom_Semantic_Segmentation_Lite-HRNet-18_OCR', |
| 105 | + ], |
| 106 | + dataset_name='kvasir_seg_shortened', |
| 107 | + usecase='precommit', |
| 108 | + ), |
| 109 | + dict( |
| 110 | + model_name=[ |
| 111 | + 'Custom_Semantic_Segmentation_Lite-HRNet-18_OCR', |
| 112 | + ], |
| 113 | + dataset_name='kvasir_seg', |
| 114 | + num_training_iters=KEEP_CONFIG_FIELD_VALUE, |
| 115 | + batch_size=KEEP_CONFIG_FIELD_VALUE, |
| 116 | + usecase=REALLIFE_USECASE_CONSTANT, |
| 117 | + ), |
| 118 | + ] |
| 119 | + return deepcopy(test_bunches) |
| 120 | + |
| 121 | +class TestOTEReallifeSegmentation(OTETrainingTestInterface): |
| 122 | + """ |
| 123 | + The main class of running test in this file. |
| 124 | + """ |
| 125 | + PERFORMANCE_RESULTS = None # it is required for e2e system |
| 126 | + helper = OTETestHelper(SegmentationTrainingTestParameters()) |
| 127 | + |
| 128 | + @classmethod |
| 129 | + def get_list_of_tests(cls, usecase: Optional[str] = None): |
| 130 | + """ |
| 131 | + This method should be a classmethod. It is called before fixture initialization, during |
| 132 | + tests discovering. |
| 133 | + """ |
| 134 | + return cls.helper.get_list_of_tests(usecase) |
| 135 | + |
| 136 | + @pytest.fixture |
| 137 | + def params_factories_for_test_actions_fx(self, current_test_parameters_fx, |
| 138 | + dataset_definitions_fx, template_paths_fx) -> Dict[str,Callable[[], Dict]]: |
| 139 | + logger.debug('params_factories_for_test_actions_fx: begin') |
| 140 | + |
| 141 | + test_parameters = deepcopy(current_test_parameters_fx) |
| 142 | + dataset_definitions = deepcopy(dataset_definitions_fx) |
| 143 | + template_paths = deepcopy(template_paths_fx) |
| 144 | + def _training_params_factory() -> Dict: |
| 145 | + if dataset_definitions is None: |
| 146 | + pytest.skip('The parameter "--dataset-definitions" is not set') |
| 147 | + |
| 148 | + model_name = test_parameters['model_name'] |
| 149 | + dataset_name = test_parameters['dataset_name'] |
| 150 | + num_training_iters = test_parameters['num_training_iters'] |
| 151 | + batch_size = test_parameters['batch_size'] |
| 152 | + |
| 153 | + dataset_params = _get_dataset_params_from_dataset_definitions(dataset_definitions, dataset_name) |
| 154 | + |
| 155 | + if model_name not in template_paths: |
| 156 | + raise ValueError(f'Model {model_name} is absent in template_paths, ' |
| 157 | + f'template_paths.keys={list(template_paths.keys())}') |
| 158 | + template_path = make_path_be_abs(template_paths[model_name], template_paths[ROOT_PATH_KEY]) |
| 159 | + |
| 160 | + logger.debug('training params factory: Before creating dataset and labels_schema') |
| 161 | + dataset, labels_schema = _create_segmentation_dataset_and_labels_schema(dataset_params) |
| 162 | + logger.debug('training params factory: After creating dataset and labels_schema') |
| 163 | + |
| 164 | + return { |
| 165 | + 'dataset': dataset, |
| 166 | + 'labels_schema': labels_schema, |
| 167 | + 'template_path': template_path, |
| 168 | + 'num_training_iters': num_training_iters, |
| 169 | + 'batch_size': batch_size, |
| 170 | + } |
| 171 | + |
| 172 | + params_factories_for_test_actions = { |
| 173 | + 'training': _training_params_factory |
| 174 | + } |
| 175 | + logger.debug('params_factories_for_test_actions_fx: end') |
| 176 | + return params_factories_for_test_actions |
| 177 | + |
| 178 | + @pytest.fixture |
| 179 | + def test_case_fx(self, current_test_parameters_fx, params_factories_for_test_actions_fx): |
| 180 | + """ |
| 181 | + This fixture returns the test case class OTEIntegrationTestCase that should be used for the current test. |
| 182 | + Note that the cache from the test helper allows to store the instance of the class |
| 183 | + between the tests. |
| 184 | + If the main parameters used for this test are the same as the main parameters used for the previous test, |
| 185 | + the instance of the test case class will be kept and re-used. It is helpful for tests that can |
| 186 | + re-use the result of operations (model training, model optimization, etc) made for the previous tests, |
| 187 | + if these operations are time-consuming. |
| 188 | + If the main parameters used for this test differs w.r.t. the previous test, a new instance of |
| 189 | + test case class will be created. |
| 190 | + """ |
| 191 | + test_case = type(self).helper.get_test_case(current_test_parameters_fx, |
| 192 | + params_factories_for_test_actions_fx) |
| 193 | + return test_case |
| 194 | + |
| 195 | + # TODO(lbeynens): move to common fixtures |
| 196 | + @pytest.fixture |
| 197 | + def data_collector_fx(self, request) -> DataCollector: |
| 198 | + setup = deepcopy(request.node.callspec.params) |
| 199 | + setup['environment_name'] = os.environ.get('TT_ENVIRONMENT_NAME', 'no-env') |
| 200 | + setup['test_type'] = os.environ.get('TT_TEST_TYPE', 'no-test-type') # TODO: get from e2e test type |
| 201 | + setup['scenario'] = 'api' # TODO(lbeynens): get from a fixture! |
| 202 | + setup['test'] = request.node.name |
| 203 | + setup['subject'] = 'custom-segmentation' |
| 204 | + setup['project'] = 'ote' |
| 205 | + if 'test_parameters' in setup: |
| 206 | + assert isinstance(setup['test_parameters'], dict) |
| 207 | + if 'dataset_name' not in setup: |
| 208 | + setup['dataset_name'] = setup['test_parameters'].get('dataset_name') |
| 209 | + if 'model_name' not in setup: |
| 210 | + setup['model_name'] = setup['test_parameters'].get('model_name') |
| 211 | + if 'test_stage' not in setup: |
| 212 | + setup['test_stage'] = setup['test_parameters'].get('test_stage') |
| 213 | + if 'usecase' not in setup: |
| 214 | + setup['usecase'] = setup['test_parameters'].get('usecase') |
| 215 | + logger.info(f'creating DataCollector: setup=\n{pformat(setup, width=140)}') |
| 216 | + data_collector = DataCollector(name='TestOTEIntegration', |
| 217 | + setup=setup) |
| 218 | + with data_collector: |
| 219 | + logger.info('data_collector is created') |
| 220 | + yield data_collector |
| 221 | + logger.info('data_collector is released') |
| 222 | + |
| 223 | + @e2e_pytest_performance |
| 224 | + def test(self, |
| 225 | + test_parameters, |
| 226 | + test_case_fx, data_collector_fx, |
| 227 | + cur_test_expected_metrics_callback_fx): |
| 228 | + test_case_fx.run_stage(test_parameters['test_stage'], data_collector_fx, |
| 229 | + cur_test_expected_metrics_callback_fx) |
0 commit comments