From 72a1cf6476742b878c39ececdf92f0265840d8d9 Mon Sep 17 00:00:00 2001 From: Vladimir Kamarzin Date: Tue, 30 Jul 2019 15:38:57 +0500 Subject: [PATCH] Extend "orders" table Add new columns "virtual" and "custom" to Orders table. "Virtual" is for indicating virtual order, "Custom" is for adding any custom data in string format, may be used by any strategy to add additional info about order. This commit also introduces database migrations mechanism via alembic tool. Seems like it's the best solution for sqlalchemy to handle schema updates. --- dexbot/migrations/README | 13 ++++ dexbot/migrations/env.py | 72 +++++++++++++++++++ dexbot/migrations/script.py.mako | 24 +++++++ .../d1e6672520b2_extend_orders_table.py | 26 +++++++ dexbot/storage.py | 36 ++++++++-- requirements.txt | 1 + tests/migrations/conftest.py | 62 ++++++++++++++++ tests/migrations/test_migrations.py | 5 ++ 8 files changed, 235 insertions(+), 4 deletions(-) create mode 100644 dexbot/migrations/README create mode 100644 dexbot/migrations/env.py create mode 100644 dexbot/migrations/script.py.mako create mode 100644 dexbot/migrations/versions/d1e6672520b2_extend_orders_table.py create mode 100644 tests/migrations/conftest.py create mode 100644 tests/migrations/test_migrations.py diff --git a/dexbot/migrations/README b/dexbot/migrations/README new file mode 100644 index 000000000..d3f637d2c --- /dev/null +++ b/dexbot/migrations/README @@ -0,0 +1,13 @@ +Dexbot database migrations are handled by alembic. See https://alembic.sqlalchemy.org/ + +## Create new migration + +``` +alembic revision -m "Short summary of changes" +``` + +Next, modify the migration script in dexbot/migrations/versions/ + +Migration will be applied automatically on next run of dexbot, see `run_migrations()` in dexbot/storage.py + +Don't forget to change table definitions in dexbot/storage.py. diff --git a/dexbot/migrations/env.py b/dexbot/migrations/env.py new file mode 100644 index 000000000..be82b9ecc --- /dev/null +++ b/dexbot/migrations/env.py @@ -0,0 +1,72 @@ +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +# fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/dexbot/migrations/script.py.mako b/dexbot/migrations/script.py.mako new file mode 100644 index 000000000..2c0156303 --- /dev/null +++ b/dexbot/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/dexbot/migrations/versions/d1e6672520b2_extend_orders_table.py b/dexbot/migrations/versions/d1e6672520b2_extend_orders_table.py new file mode 100644 index 000000000..732d5bfb1 --- /dev/null +++ b/dexbot/migrations/versions/d1e6672520b2_extend_orders_table.py @@ -0,0 +1,26 @@ +"""extend orders table + +Revision ID: d1e6672520b2 +Revises: +Create Date: 2019-07-29 17:38:09.136485 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd1e6672520b2' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('orders', sa.Column('virtual', sa.Boolean(create_constraint=False))) + op.add_column('orders', sa.Column('custom', sa.String)) + + +def downgrade(): + op.drop_column('orders', 'virtual') + op.drop_column('orders', 'custom') diff --git a/dexbot/storage.py b/dexbot/storage.py index 115419051..558022d49 100644 --- a/dexbot/storage.py +++ b/dexbot/storage.py @@ -1,17 +1,23 @@ import os +import os.path +import inspect import json import threading import queue import uuid +import alembic +import alembic.config + from appdirs import user_data_dir from . import helper from dexbot import APP_NAME, AUTHOR -from sqlalchemy import create_engine, Column, String, Integer, Float +from sqlalchemy import create_engine, Column, String, Integer, Float, Boolean from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker + Base = declarative_base() # For dexbot.sqlite file @@ -39,11 +45,15 @@ class Orders(Base): worker = Column(String) order_id = Column(String) order = Column(String) + virtual = Column(Boolean) + custom = Column(String) - def __init__(self, worker, order_id, order): + def __init__(self, worker, order_id, order, virtual, custom): self.worker = worker self.order_id = order_id self.order = order + self.virtual = virtual + self.custom = custom class Balances(Base): @@ -152,12 +162,18 @@ def __init__(self): super().__init__() # Obtain engine and session - engine = create_engine('sqlite:///%s' % sqlDataBaseFile, echo=False) + dsn = 'sqlite:///{}'.format(sqlDataBaseFile) + engine = create_engine(dsn, echo=False) Session = sessionmaker(bind=engine) self.session = Session() Base.metadata.create_all(engine) self.session.commit() + # Run migrations + import dexbot + migrations_dir = '{}/migrations'.format(os.path.dirname(inspect.getfile(dexbot))) + self.run_migrations(migrations_dir, dsn) + self.task_queue = queue.Queue() self.results = {} @@ -166,6 +182,18 @@ def __init__(self): self.daemon = True self.start() + @staticmethod + def run_migrations(script_location, dsn): + """ Apply database migrations using alembic + + :param str script_location: path to migration scripts + :param str dsn: database URL + """ + alembic_cfg = alembic.config.Config() + alembic_cfg.set_main_option('script_location', script_location) + alembic_cfg.set_main_option('sqlalchemy.url', dsn) + alembic.command.upgrade(alembic_cfg, 'head') + def run(self): for func, args, token in iter(self.task_queue.get, None): if token is not None: @@ -279,7 +307,7 @@ def _save_order(self, worker, order_id, order): if e: e.value = value else: - e = Orders(worker, order_id, value) + e = Orders(worker, order_id, value, None, None) self.session.add(e) self.session.commit() diff --git a/requirements.txt b/requirements.txt index b38cfb0ab..9711a1be8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ websocket-client==0.56.0 sdnotify==0.3.2 sqlalchemy==1.3.0 click==7.0 +alembic==1.0.11 diff --git a/tests/migrations/conftest.py b/tests/migrations/conftest.py new file mode 100644 index 000000000..acb64d33c --- /dev/null +++ b/tests/migrations/conftest.py @@ -0,0 +1,62 @@ +import os +import pytest +import tempfile +import logging + +from sqlalchemy import create_engine, Column, String, Integer, Float +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +log = logging.getLogger("dexbot") +log.setLevel(logging.DEBUG) + +Base = declarative_base() + +# Classes are represent initial table structure + + +class Config(Base): + __tablename__ = 'config' + + id = Column(Integer, primary_key=True) + category = Column(String) + key = Column(String) + value = Column(String) + + +class Orders(Base): + __tablename__ = 'orders' + + id = Column(Integer, primary_key=True) + worker = Column(String) + order_id = Column(String) + order = Column(String) + + +class Balances(Base): + __tablename__ = 'balances' + + id = Column(Integer, primary_key=True) + account = Column(String) + worker = Column(String) + base_total = Column(Float) + base_symbol = Column(String) + quote_total = Column(Float) + quote_symbol = Column(String) + center_price = Column(Float) + timestamp = Column(Integer) + + +@pytest.fixture +def initial_db(): + + _, db_file = tempfile.mkstemp() # noqa: F811 + engine = create_engine('sqlite:///{}'.format(db_file), echo=False) + Session = sessionmaker(bind=engine) + session = Session() + Base.metadata.create_all(engine) + session.commit() + log.debug('Prepared db on {}'.format(db_file)) + + yield db_file + os.unlink(db_file) diff --git a/tests/migrations/test_migrations.py b/tests/migrations/test_migrations.py new file mode 100644 index 000000000..06eab6fcf --- /dev/null +++ b/tests/migrations/test_migrations.py @@ -0,0 +1,5 @@ +from dexbot.storage import DatabaseWorker + + +def test_apply_migrations(initial_db): + DatabaseWorker.run_migrations('dexbot/migrations', 'sqlite:///{}'.format(initial_db))