version control db

This commit is contained in:
mcinj 2022-05-19 18:03:35 -04:00
parent 948281dbfb
commit abc2e66c29
11 changed files with 278 additions and 76 deletions

1
.gitignore vendored
View file

@ -6,3 +6,4 @@ __pycache__/
config/config.ini
config/*.log*
config/sqlite.db
.DS_STORE

1
alembic/README Normal file
View file

@ -0,0 +1 @@
Generic single-database configuration.

79
alembic/env.py Normal file
View file

@ -0,0 +1,79 @@
from logging.config import fileConfig
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from alembic import context
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
if config.config_file_name is not None:
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
from src.models import TableNotification, TableSteamItem, TableGiveaway, Base
target_metadata = Base.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(
connection=connection, target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

24
alembic/script.py.mako Normal file
View file

@ -0,0 +1,24 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

View file

@ -0,0 +1,70 @@
"""init
Revision ID: 1da33402b659
Revises:
Create Date: 2022-05-19 15:50:19.539401
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = '1da33402b659'
down_revision = None
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('notification',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('type', sa.String(length=50), nullable=False),
sa.Column('message', sa.String(length=300), nullable=False),
sa.Column('medium', sa.String(length=50), nullable=False),
sa.Column('success', sa.Boolean(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
nullable=True),
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('steam_item',
sa.Column('steam_id', sa.String(length=15), nullable=False),
sa.Column('game_name', sa.String(length=200), nullable=False),
sa.Column('steam_url', sa.String(length=100), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
nullable=True),
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
nullable=True),
sa.PrimaryKeyConstraint('steam_id')
)
op.create_table('giveaway',
sa.Column('giveaway_id', sa.String(length=10), nullable=False),
sa.Column('steam_id', sa.Integer(), nullable=False),
sa.Column('giveaway_uri', sa.String(length=200), nullable=False),
sa.Column('user', sa.String(length=40), nullable=False),
sa.Column('giveaway_created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('giveaway_ended_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('cost', sa.Integer(), nullable=False),
sa.Column('copies', sa.Integer(), nullable=False),
sa.Column('contributor_level', sa.Integer(), nullable=False),
sa.Column('entered', sa.Boolean(), nullable=False),
sa.Column('won', sa.Boolean(), nullable=False),
sa.Column('game_entries', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
nullable=True),
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'),
nullable=True),
sa.ForeignKeyConstraint(['steam_id'], ['steam_item.steam_id'], ),
sa.PrimaryKeyConstraint('giveaway_id', 'steam_id')
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('giveaway')
op.drop_table('steam_item')
op.drop_table('notification')
# ### end Alembic commands ###

View file

@ -7,3 +7,4 @@ python-dateutil==2.8.2
Flask==2.1.2
Flask-BasicAuth==0.2.0
pyopenssl==22.0.0
alembic==1.7.7

View file

@ -1,27 +1,35 @@
from datetime import datetime, timedelta
import sqlalchemy
from alembic import command
from alembic.config import Config
from dateutil import tz
from sqlalchemy import create_engine, Integer, String, Column, DateTime, Boolean, func, ForeignKey
from sqlalchemy.orm import registry, relationship, Session
from sqlalchemy_utils import database_exists, create_database
from sqlalchemy import func
from sqlalchemy.orm import Session
mapper_registry = registry()
mapper_registry.metadata
Base = mapper_registry.generate_base()
engine = create_engine('sqlite:///../config/sqlite.db', echo=False)
import log
from models import TableNotification, TableGiveaway, TableSteamItem
logger = log.get_logger(__name__)
engine = None
class TableNotification(Base):
__tablename__ = 'notification'
id = Column(Integer, primary_key=True, nullable=False)
type = Column(String(50), nullable=False)
message = Column(String(300), nullable=False)
medium = Column(String(50), nullable=False)
success = Column(Boolean, nullable=False)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
def create_engine(db_url: str):
global engine
if not engine:
engine = sqlalchemy.create_engine(db_url, echo=False)
engine.connect()
__mapper_args__ = {"eager_defaults": True}
def run_migrations(script_location: str, dsn: str) -> None:
logger.info('Running DB migrations in %r on %r', script_location, dsn)
alembic_cfg = Config()
alembic_cfg.set_main_option('script_location', script_location)
alembic_cfg.set_main_option('sqlalchemy.url', dsn)
command.upgrade(alembic_cfg, 'head')
class NotificationHelper:
@classmethod
def insert(cls, type_of_error, message, medium, success):
@ -61,37 +69,7 @@ class TableNotification(Base):
.all()
class TableSteamItem(Base):
__tablename__ = 'steam_item'
steam_id = Column(String(15), primary_key=True, nullable=False)
game_name = Column(String(200), nullable=False)
steam_url = Column(String(100), nullable=False)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
giveaways = relationship("TableGiveaway", back_populates="steam_item")
class TableGiveaway(Base):
__tablename__ = 'giveaway'
giveaway_id = Column(String(10), primary_key=True, nullable=False)
steam_id = Column(Integer, ForeignKey('steam_item.steam_id'), primary_key=True)
giveaway_uri = Column(String(200), nullable=False)
user = Column(String(40), nullable=False)
giveaway_created_at = Column(DateTime(timezone=True), nullable=False)
giveaway_ended_at = Column(DateTime(timezone=True), nullable=False)
cost = Column(Integer(), nullable=False)
copies = Column(Integer(), nullable=False)
contributor_level = Column(Integer(), nullable=False)
entered = Column(Boolean(), nullable=False)
won = Column(Boolean(), nullable=False)
game_entries = Column(Integer(), nullable=False)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
steam_item = relationship("TableSteamItem", back_populates="giveaways")
__mapper_args__ = {"eager_defaults": True}
class GiveawayHelper:
@classmethod
def unix_timestamp_to_utc_datetime(cls, timestamp):
@ -122,8 +100,8 @@ class TableGiveaway(Base):
steam_id=steam_id,
giveaway_uri=giveaway.giveaway_uri,
user=giveaway.user,
giveaway_created_at=TableGiveaway.unix_timestamp_to_utc_datetime(giveaway.time_created_timestamp),
giveaway_ended_at=TableGiveaway.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp),
giveaway_created_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_created_timestamp),
giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp),
cost=giveaway.cost,
copies=giveaway.copies,
contributor_level=giveaway.contributor_level,
@ -135,9 +113,9 @@ class TableGiveaway(Base):
@classmethod
def upsert_giveaway(cls, giveaway, entered):
result = TableGiveaway.get_by_ids(giveaway)
result = GiveawayHelper.get_by_ids(giveaway)
if not result:
TableGiveaway.insert(giveaway, entered)
GiveawayHelper.insert(giveaway, entered)
else:
with Session(engine) as session:
g = TableGiveaway(
@ -155,13 +133,3 @@ class TableGiveaway(Base):
game_entries=giveaway.game_entries)
session.merge(g)
session.commit()
if not database_exists(engine.url):
create_database(engine.url)
# emitting DDL
mapper_registry.metadata.create_all(engine)
Base.metadata.create_all(engine)
else:
# Connect the database if exists.
engine.connect()

View file

@ -8,8 +8,8 @@ from requests.adapters import HTTPAdapter
from urllib3.util import Retry
import log
from database import NotificationHelper, GiveawayHelper
from giveaway import Giveaway
from tables import TableNotification, TableGiveaway
logger = log.get_logger(__name__)
@ -90,7 +90,7 @@ class EnterGiveaways:
won = soup.select("a[title='Giveaways Won'] div")
if won:
number_won = soup.select_one("a[title='Giveaways Won'] div").text
won_notifications = TableNotification.get_won_notifications_today()
won_notifications = NotificationHelper.get_won_notifications_today()
if won_notifications and len(won_notifications) >= 1:
logger.info("🆒️ Win(s) detected, but we have already notified that there are won games waiting "
"to be received. Doing nothing.")
@ -203,15 +203,15 @@ class EnterGiveaways:
if if_enter_giveaway:
res = self.enter_giveaway(giveaway)
if res:
TableGiveaway.upsert_giveaway(giveaway, True)
GiveawayHelper.upsert_giveaway(giveaway, True)
self.points -= int(giveaway.cost)
txt = f"✅ Entered giveaway '{giveaway.game_name}'"
logger.info(txt)
sleep(randint(4, 15))
else:
TableGiveaway.upsert_giveaway(giveaway, False)
GiveawayHelper.upsert_giveaway(giveaway, False)
else:
TableGiveaway.upsert_giveaway(giveaway, False)
GiveawayHelper.upsert_giveaway(giveaway, False)
# if we are on any filter type except New and we get to a giveaway that exceeds our
# max time left amount, then we don't need to continue to look at giveaways as any
# after this point will also exceed the max time left

52
src/models.py Normal file
View file

@ -0,0 +1,52 @@
from sqlalchemy import Integer, String, Column, DateTime, Boolean, func, ForeignKey
from sqlalchemy.orm import registry, relationship
mapper_registry = registry()
metadata = mapper_registry.metadata
Base = mapper_registry.generate_base()
class TableNotification(Base):
__tablename__ = 'notification'
id = Column(Integer, primary_key=True, nullable=False)
type = Column(String(50), nullable=False)
message = Column(String(300), nullable=False)
medium = Column(String(50), nullable=False)
success = Column(Boolean, nullable=False)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
__mapper_args__ = {"eager_defaults": True}
class TableSteamItem(Base):
__tablename__ = 'steam_item'
steam_id = Column(String(15), primary_key=True, nullable=False)
game_name = Column(String(200), nullable=False)
steam_url = Column(String(100), nullable=False)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
giveaways = relationship("TableGiveaway", back_populates="steam_item")
class TableGiveaway(Base):
__tablename__ = 'giveaway'
giveaway_id = Column(String(10), primary_key=True, nullable=False)
steam_id = Column(Integer, ForeignKey('steam_item.steam_id'), primary_key=True)
giveaway_uri = Column(String(200), nullable=False)
user = Column(String(40), nullable=False)
giveaway_created_at = Column(DateTime(timezone=True), nullable=False)
giveaway_ended_at = Column(DateTime(timezone=True), nullable=False)
cost = Column(Integer(), nullable=False)
copies = Column(Integer(), nullable=False)
contributor_level = Column(Integer(), nullable=False)
entered = Column(Boolean(), nullable=False)
won = Column(Boolean(), nullable=False)
game_entries = Column(Integer(), nullable=False)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
steam_item = relationship("TableSteamItem", back_populates="giveaways")
__mapper_args__ = {"eager_defaults": True}

View file

@ -2,7 +2,7 @@ import http.client
import urllib
import log
from tables import TableNotification
from database import NotificationHelper
logger = log.get_logger(__name__)
@ -48,4 +48,4 @@ class Notification:
else:
logger.error(f"Pushover notification failed. Code {response.getcode()}: {response.read().decode()}")
success = False
TableNotification.insert(type_of_error, f"{message}", 'pushover', success)
NotificationHelper.insert(type_of_error, f"{message}", 'pushover', success)

View file

@ -5,26 +5,30 @@ from config_reader import ConfigReader, ConfigException
from enter_giveaways import SteamGiftsException
from giveaway_thread import GiveawayThread
from notification import Notification
from database import run_migrations, create_engine
from webserver_thread import WebServerThread
logger = log.get_logger(__name__)
config_file_name = '../config/config.ini'
db_url = 'sqlite:///../config/sqlite.db'
alembic_migration_files = '../alembic'
def run():
logger.info("Starting Steamgifts bot.")
file_name = '../config/config.ini'
config = None
try:
config = ConfigReader(file_name)
config = ConfigReader(config_file_name)
except IOError:
txt = f"{file_name} doesn't exist. Rename {file_name}.example to {file_name} and fill out."
txt = f"{config_file_name} doesn't exist. Rename {config_file_name}.example to {config_file_name} and fill out."
logger.warning(txt)
exit(-1)
except ConfigException as e:
logger.error(e)
exit(-1)
config.read(file_name)
config.read(config_file_name)
notification = Notification(config['NOTIFICATIONS'].get('notification.prefix'))
pushover_enabled = config['NOTIFICATIONS'].getboolean('pushover.enabled')
@ -68,4 +72,6 @@ if __name__ == '__main__':
|___/
-------------------------------------------------------------------------------------
""")
run_migrations(alembic_migration_files, db_url)
create_engine(db_url)
run()