From abc2e66c293f608f0fdb9e58ad2e673806e27c39 Mon Sep 17 00:00:00 2001 From: mcinj <98779161+mcinj@users.noreply.github.com> Date: Thu, 19 May 2022 18:03:35 -0400 Subject: [PATCH] version control db --- .gitignore | 3 +- alembic/README | 1 + alembic/env.py | 79 ++++++++++++++++ alembic/script.py.mako | 24 +++++ .../2022_05_19-15_50_19-1da33402b659_init.py | 70 ++++++++++++++ requirements.txt | 3 +- src/{tables.py => database.py} | 94 ++++++------------- src/enter_giveaways.py | 10 +- src/models.py | 52 ++++++++++ src/notification.py | 4 +- src/run.py | 14 ++- 11 files changed, 278 insertions(+), 76 deletions(-) create mode 100644 alembic/README create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/2022_05_19-15_50_19-1da33402b659_init.py rename src/{tables.py => database.py} (57%) create mode 100644 src/models.py diff --git a/.gitignore b/.gitignore index 42e2346..e292dc2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ __pycache__/ .idea config/config.ini config/*.log* -config/sqlite.db \ No newline at end of file +config/sqlite.db +.DS_STORE \ No newline at end of file diff --git a/alembic/README b/alembic/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..d622584 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,79 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from src.models import TableNotification, TableSteamItem, TableGiveaway, Base +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/2022_05_19-15_50_19-1da33402b659_init.py b/alembic/versions/2022_05_19-15_50_19-1da33402b659_init.py new file mode 100644 index 0000000..cbb80be --- /dev/null +++ b/alembic/versions/2022_05_19-15_50_19-1da33402b659_init.py @@ -0,0 +1,70 @@ +"""init + +Revision ID: 1da33402b659 +Revises: +Create Date: 2022-05-19 15:50:19.539401 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = '1da33402b659' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('notification', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('type', sa.String(length=50), nullable=False), + sa.Column('message', sa.String(length=300), nullable=False), + sa.Column('medium', sa.String(length=50), nullable=False), + sa.Column('success', sa.Boolean(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), + nullable=True), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), + nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('steam_item', + sa.Column('steam_id', sa.String(length=15), nullable=False), + sa.Column('game_name', sa.String(length=200), nullable=False), + sa.Column('steam_url', sa.String(length=100), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), + nullable=True), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), + nullable=True), + sa.PrimaryKeyConstraint('steam_id') + ) + op.create_table('giveaway', + sa.Column('giveaway_id', sa.String(length=10), nullable=False), + sa.Column('steam_id', sa.Integer(), nullable=False), + sa.Column('giveaway_uri', sa.String(length=200), nullable=False), + sa.Column('user', sa.String(length=40), nullable=False), + sa.Column('giveaway_created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('giveaway_ended_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('cost', sa.Integer(), nullable=False), + sa.Column('copies', sa.Integer(), nullable=False), + sa.Column('contributor_level', sa.Integer(), nullable=False), + sa.Column('entered', sa.Boolean(), nullable=False), + sa.Column('won', sa.Boolean(), nullable=False), + sa.Column('game_entries', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), + nullable=True), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), + nullable=True), + sa.ForeignKeyConstraint(['steam_id'], ['steam_item.steam_id'], ), + sa.PrimaryKeyConstraint('giveaway_id', 'steam_id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('giveaway') + op.drop_table('steam_item') + op.drop_table('notification') + # ### end Alembic commands ### diff --git a/requirements.txt b/requirements.txt index ec416c3..47e10c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ sqlalchemy_utils==0.38.2 python-dateutil==2.8.2 Flask==2.1.2 Flask-BasicAuth==0.2.0 -pyopenssl==22.0.0 \ No newline at end of file +pyopenssl==22.0.0 +alembic==1.7.7 \ No newline at end of file diff --git a/src/tables.py b/src/database.py similarity index 57% rename from src/tables.py rename to src/database.py index 366339f..d699dc2 100644 --- a/src/tables.py +++ b/src/database.py @@ -1,27 +1,35 @@ from datetime import datetime, timedelta +import sqlalchemy +from alembic import command +from alembic.config import Config from dateutil import tz -from sqlalchemy import create_engine, Integer, String, Column, DateTime, Boolean, func, ForeignKey -from sqlalchemy.orm import registry, relationship, Session -from sqlalchemy_utils import database_exists, create_database +from sqlalchemy import func +from sqlalchemy.orm import Session -mapper_registry = registry() -mapper_registry.metadata -Base = mapper_registry.generate_base() -engine = create_engine('sqlite:///../config/sqlite.db', echo=False) +import log +from models import TableNotification, TableGiveaway, TableSteamItem + +logger = log.get_logger(__name__) +engine = None -class TableNotification(Base): - __tablename__ = 'notification' - id = Column(Integer, primary_key=True, nullable=False) - type = Column(String(50), nullable=False) - message = Column(String(300), nullable=False) - medium = Column(String(50), nullable=False) - success = Column(Boolean, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) +def create_engine(db_url: str): + global engine + if not engine: + engine = sqlalchemy.create_engine(db_url, echo=False) + engine.connect() - __mapper_args__ = {"eager_defaults": True} + +def run_migrations(script_location: str, dsn: str) -> None: + logger.info('Running DB migrations in %r on %r', script_location, dsn) + alembic_cfg = Config() + alembic_cfg.set_main_option('script_location', script_location) + alembic_cfg.set_main_option('sqlalchemy.url', dsn) + command.upgrade(alembic_cfg, 'head') + + +class NotificationHelper: @classmethod def insert(cls, type_of_error, message, medium, success): @@ -61,37 +69,7 @@ class TableNotification(Base): .all() -class TableSteamItem(Base): - __tablename__ = 'steam_item' - steam_id = Column(String(15), primary_key=True, nullable=False) - game_name = Column(String(200), nullable=False) - steam_url = Column(String(100), nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - giveaways = relationship("TableGiveaway", back_populates="steam_item") - - -class TableGiveaway(Base): - __tablename__ = 'giveaway' - giveaway_id = Column(String(10), primary_key=True, nullable=False) - steam_id = Column(Integer, ForeignKey('steam_item.steam_id'), primary_key=True) - giveaway_uri = Column(String(200), nullable=False) - user = Column(String(40), nullable=False) - giveaway_created_at = Column(DateTime(timezone=True), nullable=False) - giveaway_ended_at = Column(DateTime(timezone=True), nullable=False) - cost = Column(Integer(), nullable=False) - copies = Column(Integer(), nullable=False) - contributor_level = Column(Integer(), nullable=False) - entered = Column(Boolean(), nullable=False) - won = Column(Boolean(), nullable=False) - game_entries = Column(Integer(), nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - steam_item = relationship("TableSteamItem", back_populates="giveaways") - - __mapper_args__ = {"eager_defaults": True} +class GiveawayHelper: @classmethod def unix_timestamp_to_utc_datetime(cls, timestamp): @@ -122,8 +100,8 @@ class TableGiveaway(Base): steam_id=steam_id, giveaway_uri=giveaway.giveaway_uri, user=giveaway.user, - giveaway_created_at=TableGiveaway.unix_timestamp_to_utc_datetime(giveaway.time_created_timestamp), - giveaway_ended_at=TableGiveaway.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp), + giveaway_created_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_created_timestamp), + giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp), cost=giveaway.cost, copies=giveaway.copies, contributor_level=giveaway.contributor_level, @@ -135,9 +113,9 @@ class TableGiveaway(Base): @classmethod def upsert_giveaway(cls, giveaway, entered): - result = TableGiveaway.get_by_ids(giveaway) + result = GiveawayHelper.get_by_ids(giveaway) if not result: - TableGiveaway.insert(giveaway, entered) + GiveawayHelper.insert(giveaway, entered) else: with Session(engine) as session: g = TableGiveaway( @@ -154,14 +132,4 @@ class TableGiveaway(Base): won=False, game_entries=giveaway.game_entries) session.merge(g) - session.commit() - - -if not database_exists(engine.url): - create_database(engine.url) - # emitting DDL - mapper_registry.metadata.create_all(engine) - Base.metadata.create_all(engine) -else: - # Connect the database if exists. - engine.connect() + session.commit() \ No newline at end of file diff --git a/src/enter_giveaways.py b/src/enter_giveaways.py index 42e1004..81a3f99 100644 --- a/src/enter_giveaways.py +++ b/src/enter_giveaways.py @@ -8,8 +8,8 @@ from requests.adapters import HTTPAdapter from urllib3.util import Retry import log +from database import NotificationHelper, GiveawayHelper from giveaway import Giveaway -from tables import TableNotification, TableGiveaway logger = log.get_logger(__name__) @@ -90,7 +90,7 @@ class EnterGiveaways: won = soup.select("a[title='Giveaways Won'] div") if won: number_won = soup.select_one("a[title='Giveaways Won'] div").text - won_notifications = TableNotification.get_won_notifications_today() + won_notifications = NotificationHelper.get_won_notifications_today() if won_notifications and len(won_notifications) >= 1: logger.info("🆒️ Win(s) detected, but we have already notified that there are won games waiting " "to be received. Doing nothing.") @@ -203,15 +203,15 @@ class EnterGiveaways: if if_enter_giveaway: res = self.enter_giveaway(giveaway) if res: - TableGiveaway.upsert_giveaway(giveaway, True) + GiveawayHelper.upsert_giveaway(giveaway, True) self.points -= int(giveaway.cost) txt = f"✅ Entered giveaway '{giveaway.game_name}'" logger.info(txt) sleep(randint(4, 15)) else: - TableGiveaway.upsert_giveaway(giveaway, False) + GiveawayHelper.upsert_giveaway(giveaway, False) else: - TableGiveaway.upsert_giveaway(giveaway, False) + GiveawayHelper.upsert_giveaway(giveaway, False) # if we are on any filter type except New and we get to a giveaway that exceeds our # max time left amount, then we don't need to continue to look at giveaways as any # after this point will also exceed the max time left diff --git a/src/models.py b/src/models.py new file mode 100644 index 0000000..f9174f8 --- /dev/null +++ b/src/models.py @@ -0,0 +1,52 @@ +from sqlalchemy import Integer, String, Column, DateTime, Boolean, func, ForeignKey +from sqlalchemy.orm import registry, relationship + +mapper_registry = registry() +metadata = mapper_registry.metadata +Base = mapper_registry.generate_base() + + +class TableNotification(Base): + __tablename__ = 'notification' + id = Column(Integer, primary_key=True, nullable=False) + type = Column(String(50), nullable=False) + message = Column(String(300), nullable=False) + medium = Column(String(50), nullable=False) + success = Column(Boolean, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + __mapper_args__ = {"eager_defaults": True} + + +class TableSteamItem(Base): + __tablename__ = 'steam_item' + steam_id = Column(String(15), primary_key=True, nullable=False) + game_name = Column(String(200), nullable=False) + steam_url = Column(String(100), nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + giveaways = relationship("TableGiveaway", back_populates="steam_item") + + +class TableGiveaway(Base): + __tablename__ = 'giveaway' + giveaway_id = Column(String(10), primary_key=True, nullable=False) + steam_id = Column(Integer, ForeignKey('steam_item.steam_id'), primary_key=True) + giveaway_uri = Column(String(200), nullable=False) + user = Column(String(40), nullable=False) + giveaway_created_at = Column(DateTime(timezone=True), nullable=False) + giveaway_ended_at = Column(DateTime(timezone=True), nullable=False) + cost = Column(Integer(), nullable=False) + copies = Column(Integer(), nullable=False) + contributor_level = Column(Integer(), nullable=False) + entered = Column(Boolean(), nullable=False) + won = Column(Boolean(), nullable=False) + game_entries = Column(Integer(), nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + steam_item = relationship("TableSteamItem", back_populates="giveaways") + + __mapper_args__ = {"eager_defaults": True} diff --git a/src/notification.py b/src/notification.py index 16b1f86..57fb3b0 100644 --- a/src/notification.py +++ b/src/notification.py @@ -2,7 +2,7 @@ import http.client import urllib import log -from tables import TableNotification +from database import NotificationHelper logger = log.get_logger(__name__) @@ -48,4 +48,4 @@ class Notification: else: logger.error(f"Pushover notification failed. Code {response.getcode()}: {response.read().decode()}") success = False - TableNotification.insert(type_of_error, f"{message}", 'pushover', success) + NotificationHelper.insert(type_of_error, f"{message}", 'pushover', success) diff --git a/src/run.py b/src/run.py index 320ca17..51a68ef 100644 --- a/src/run.py +++ b/src/run.py @@ -5,26 +5,30 @@ from config_reader import ConfigReader, ConfigException from enter_giveaways import SteamGiftsException from giveaway_thread import GiveawayThread from notification import Notification +from database import run_migrations, create_engine from webserver_thread import WebServerThread logger = log.get_logger(__name__) +config_file_name = '../config/config.ini' +db_url = 'sqlite:///../config/sqlite.db' +alembic_migration_files = '../alembic' def run(): logger.info("Starting Steamgifts bot.") - file_name = '../config/config.ini' + config = None try: - config = ConfigReader(file_name) + config = ConfigReader(config_file_name) except IOError: - txt = f"{file_name} doesn't exist. Rename {file_name}.example to {file_name} and fill out." + txt = f"{config_file_name} doesn't exist. Rename {config_file_name}.example to {config_file_name} and fill out." logger.warning(txt) exit(-1) except ConfigException as e: logger.error(e) exit(-1) - config.read(file_name) + config.read(config_file_name) notification = Notification(config['NOTIFICATIONS'].get('notification.prefix')) pushover_enabled = config['NOTIFICATIONS'].getboolean('pushover.enabled') @@ -68,4 +72,6 @@ if __name__ == '__main__': |___/ ------------------------------------------------------------------------------------- """) + run_migrations(alembic_migration_files, db_url) + create_engine(db_url) run()