diff --git a/dbbot/reader/database_writer.py b/dbbot/reader/database_writer.py index 545b5ba..c139377 100644 --- a/dbbot/reader/database_writer.py +++ b/dbbot/reader/database_writer.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from sqlalchemy import create_engine, Column, DateTime, ForeignKey, Integer, MetaData, Sequence, String, Table, Text, \ +from sqlalchemy import create_engine, Column, DateTime, ForeignKey, Interval, Integer, MetaData, Sequence, String, Table, Text, \ UniqueConstraint from sqlalchemy.sql import and_, select from sqlalchemy.exc import IntegrityError @@ -57,7 +57,7 @@ def _create_table_test_run_status(self): return self._create_table('test_run_status', ( Column('test_run_id', Integer, ForeignKey('test_runs.id'), nullable=False), Column('name', String(256), nullable=False), - Column('elapsed', Integer), + Column('elapsed', Interval), Column('failed', Integer, nullable=False), Column('passed', Integer, nullable=False) ), ('test_run_id', 'name')) @@ -76,7 +76,7 @@ def _create_table_tag_status(self): Column('test_run_id', Integer, ForeignKey('test_runs.id'), nullable=False), Column('name', String(256), nullable=False), Column('critical', Integer, nullable=False), - Column('elapsed', Integer), + Column('elapsed', Interval), Column('failed', Integer, nullable=False), Column('passed', Integer, nullable=False) ), ('test_run_id', 'name')) @@ -94,7 +94,7 @@ def _create_table_suite_status(self): return self._create_table('suite_status', ( Column('test_run_id', Integer, ForeignKey('test_runs.id'), nullable=False), Column('suite_id', Integer, ForeignKey('suites.id'), nullable=False), - Column('elapsed', Integer, nullable=False), + Column('elapsed', Interval, nullable=False), Column('failed', Integer, nullable=False), Column('passed', Integer, nullable=False), Column('status', String(4), nullable=False) @@ -114,7 +114,7 @@ def _create_table_test_status(self): Column('test_run_id', Integer, ForeignKey('test_runs.id'), nullable=False), Column('test_id', Integer, ForeignKey('tests.id'), nullable=False), Column('status', String(4), nullable=False), - Column('elapsed', Integer, nullable=False) + Column('elapsed', Interval, nullable=False), ), ('test_run_id', 'test_id')) def _create_table_keywords(self): @@ -126,14 +126,14 @@ def _create_table_keywords(self): Column('type', String(64), nullable=False), Column('timeout', String(4)), Column('doc', Text) - ), ('name', 'type')) + )) def _create_table_keyword_status(self): return self._create_table('keyword_status', ( Column('test_run_id', Integer, ForeignKey('test_runs.id'), nullable=False), Column('keyword_id', Integer, ForeignKey('keywords.id'), nullable=False), Column('status', String(4), nullable=False), - Column('elapsed', Integer, nullable=False) + Column('elapsed', Interval, nullable=False), )) def _create_table_messages(self): @@ -167,7 +167,7 @@ def _create_table(self, table_name, columns, unique_columns=()): def fetch_id(self, table_name, criteria): table = getattr(self, table_name) - sql_statement = select([table.c.id]).where( + sql_statement = select(table.c.id).where( and_(*(getattr(table.c, key) == value for key, value in criteria.items())) ) result = self._connection.execute(sql_statement).first() @@ -178,7 +178,7 @@ def fetch_id(self, table_name, criteria): def insert(self, table_name, criteria): sql_statement = getattr(self, table_name).insert() - result = self._connection.execute(sql_statement, **criteria) + result = self._connection.execute(sql_statement, criteria) return result.inserted_primary_key[0] def insert_or_ignore(self, table_name, criteria): @@ -190,4 +190,5 @@ def insert_or_ignore(self, table_name, criteria): def close(self): self._verbose('- Closing database connection') + self._connection.commit() self._connection.close() diff --git a/dbbot/reader/robot_results_parser.py b/dbbot/reader/robot_results_parser.py index b2fbdec..5645023 100644 --- a/dbbot/reader/robot_results_parser.py +++ b/dbbot/reader/robot_results_parser.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. from __future__ import with_statement -from datetime import datetime +import datetime + from hashlib import sha1 from robot.api import ExecutionResult from sqlalchemy.exc import IntegrityError @@ -35,14 +36,14 @@ def xml_to_db(self, xml_file): try: test_run_id = self._db.insert('test_runs', { 'hash': hash_string, - 'imported_at': datetime.utcnow(), - 'source_file': test_run.source, + 'source_file': str(test_run.source), + 'imported_at': datetime.datetime.now(datetime.UTC), 'started_at': self._format_robot_timestamp(test_run.suite.starttime), 'finished_at': self._format_robot_timestamp(test_run.suite.endtime) }) except IntegrityError: test_run_id = self._db.fetch_id('test_runs', { - 'source_file': test_run.source, + 'source_file': str(test_run.source), 'started_at': self._format_robot_timestamp(test_run.suite.starttime), 'finished_at': self._format_robot_timestamp(test_run.suite.endtime) }) @@ -65,7 +66,7 @@ def _parse_errors(self, errors, test_run_id): for error in errors: self._db.insert_or_ignore('test_run_errors', { 'test_run_id': test_run_id, 'level': error.level, - 'timestamp': self._format_robot_timestamp(error.timestamp), + 'timestamp': error.timestamp, 'content': error.message, 'content_hash': self._string_hash(error.message) }) @@ -86,7 +87,7 @@ def _parse_tag_stats(self, stat, test_run_id): self._db.insert_or_ignore('tag_status', { 'test_run_id': test_run_id, 'name': stat.name, - 'critical': int(stat.critical), + 'skipped': int(stat.skipped), 'elapsed': getattr(stat, 'elapsed', None), 'failed': stat.failed, 'passed': stat.passed @@ -108,26 +109,25 @@ def _parse_suite(self, suite, test_run_id, parent_suite_id=None): 'suite_id': parent_suite_id, 'xml_id': suite.id, 'name': suite.name, - 'source': suite.source, + 'source': str(suite.source), 'doc': suite.doc }) except IntegrityError: suite_id = self._db.fetch_id('suites', { 'name': suite.name, - 'source': suite.source + 'source': str(suite.source) }) self._parse_suite_status(test_run_id, suite_id, suite) self._parse_suites(suite, test_run_id, suite_id) self._parse_tests(suite.tests, test_run_id, suite_id) - self._parse_keywords(suite.keywords, test_run_id, suite_id, None) def _parse_suite_status(self, test_run_id, suite_id, suite): self._db.insert_or_ignore('suite_status', { 'test_run_id': test_run_id, 'suite_id': suite_id, - 'passed': suite.statistics.all.passed, - 'failed': suite.statistics.all.failed, - 'elapsed': suite.elapsedtime, + 'passed': suite.statistics.passed, + 'failed': suite.statistics.failed, + 'elapsed': suite.elapsed_time, 'status': suite.status }) @@ -154,14 +154,14 @@ def _parse_test(self, test, test_run_id, suite_id): }) self._parse_test_status(test_run_id, test_id, test) self._parse_tags(test.tags, test_id) - self._parse_keywords(test.keywords, test_run_id, None, test_id) + self._parse_keywords(test.body, test_run_id, None, test_id) def _parse_test_status(self, test_run_id, test_id, test): self._db.insert_or_ignore('test_status', { 'test_run_id': test_run_id, 'test_id': test_id, 'status': test.status, - 'elapsed': test.elapsedtime + 'elapsed': test.elapsed_time }) def _parse_tags(self, tags, test_id): @@ -173,6 +173,8 @@ def _parse_keywords(self, keywords, test_run_id, suite_id, test_id, keyword_id=N [self._parse_keyword(keyword, test_run_id, suite_id, test_id, keyword_id) for keyword in keywords] def _parse_keyword(self, keyword, test_run_id, suite_id, test_id, keyword_id): + if keyword.type != "KEYWORD": + return try: keyword_id = self._db.insert('keywords', { 'suite_id': suite_id, @@ -191,21 +193,21 @@ def _parse_keyword(self, keyword, test_run_id, suite_id, test_id, keyword_id): self._parse_keyword_status(test_run_id, keyword_id, keyword) self._parse_messages(keyword.messages, keyword_id) self._parse_arguments(keyword.args, keyword_id) - self._parse_keywords(keyword.keywords, test_run_id, None, None, keyword_id) + self._parse_keywords(keyword.body, test_run_id, None, None, keyword_id) def _parse_keyword_status(self, test_run_id, keyword_id, keyword): self._db.insert_or_ignore('keyword_status', { 'test_run_id': test_run_id, 'keyword_id': keyword_id, 'status': keyword.status, - 'elapsed': keyword.elapsedtime + 'elapsed': keyword.elapsed_time }) def _parse_messages(self, messages, keyword_id): for message in messages: self._db.insert_or_ignore('messages', { 'keyword_id': keyword_id, 'level': message.level, - 'timestamp': self._format_robot_timestamp(message.timestamp), + 'timestamp': message.timestamp, 'content': message.message, 'content_hash': self._string_hash(message.message) }) @@ -220,7 +222,7 @@ def _parse_arguments(self, args, keyword_id): @staticmethod def _format_robot_timestamp(timestamp): - return datetime.strptime(timestamp, '%Y%m%d %H:%M:%S.%f') if timestamp else None + return datetime.datetime.strptime(timestamp, '%Y%m%d %H:%M:%S.%f') if timestamp else None @staticmethod def _string_hash(string):