diff --git a/requirements.txt b/requirements.txt index 517508f..cbc0638 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,10 +2,6 @@ requests==2.27.1 beautifulsoup4==4.11.1 urllib3==1.26.9 sqlalchemy==1.4.36 -sqlalchemy_utils==0.38.2 -paginate-sqlalchemy==0.3.1 alembic==1.7.7 python-dateutil==2.8.2 -Flask==2.1.2 -Flask-BasicAuth==0.2.0 -pyopenssl==22.0.0 \ No newline at end of file +apscheduler==3.9.1 \ No newline at end of file diff --git a/src/bot/database.py b/src/bot/database.py index c261518..99cbfb0 100644 --- a/src/bot/database.py +++ b/src/bot/database.py @@ -11,7 +11,7 @@ from sqlalchemy.orm import Session, joinedload from sqlalchemy_utils import database_exists from .log import get_logger -from .models import TableNotification, TableGiveaway, TableSteamItem, TableTask +from .models import TableNotification, TableGiveaway, TableSteamItem logger = get_logger(__name__) engine = sqlalchemy.create_engine(f"{os.getenv('BOT_DB_URL', 'sqlite:///./config/sqlite.db')}", echo=False) @@ -116,6 +116,16 @@ class GiveawayHelper: with Session(engine) as session: return session.query(TableGiveaway).filter_by(entered=True).count() + @classmethod + def total_won(cls): + with Session(engine) as session: + return session.query(TableGiveaway).filter_by(won=True).count() + + @classmethod + def get_by_giveaway_id(cls, game_id): + with Session(engine) as session: + return session.query(TableGiveaway).filter_by(giveaway_id=game_id).one_or_none() + @classmethod def unix_timestamp_to_utc_datetime(cls, timestamp): return datetime.utcfromtimestamp(timestamp) @@ -126,6 +136,17 @@ class GiveawayHelper: return session.query(TableGiveaway).filter_by(giveaway_id=giveaway.giveaway_game_id, steam_id=giveaway.steam_app_id).all() + @classmethod + def mark_game_as_won(cls, game_id): + with Session(engine) as session: + result = session.query(TableGiveaway).filter_by(giveaway_id=game_id).all() + if result: + won_giveaway = result[0] + if not won_giveaway.won: + won_giveaway.won = True + session.add(won_giveaway) + session.commit() + @classmethod def insert(cls, giveaway, entered, won): with Session(engine) as session: @@ -149,7 +170,7 @@ class GiveawayHelper: giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp), cost=giveaway.cost, copies=giveaway.copies, - contributor_level=giveaway._contributor_level, + contributor_level=giveaway.contributor_level, entered=entered, won=won, game_entries=giveaway.game_entries) @@ -157,7 +178,7 @@ class GiveawayHelper: session.commit() @classmethod - def upsert_giveaway(cls, giveaway, entered, won): + def upsert_giveaway_with_details(cls, giveaway, entered, won): result = GiveawayHelper.get_by_ids(giveaway) if not result: GiveawayHelper.insert(giveaway, entered, won) @@ -172,7 +193,7 @@ class GiveawayHelper: giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp), cost=giveaway.cost, copies=giveaway.copies, - contributor_level=giveaway._contributor_level, + contributor_level=giveaway.contributor_level, entered=entered, won=won, game_entries=giveaway.game_entries) @@ -195,7 +216,7 @@ class GiveawayHelper: giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp), cost=giveaway.cost, copies=giveaway.copies, - contributor_level=giveaway._contributor_level, + contributor_level=giveaway.contributor_level, game_entries=giveaway.game_entries) session.merge(g) session.commit() diff --git a/src/bot/enter_giveaways.py b/src/bot/enter_giveaways.py index 7eeb922..bff1345 100644 --- a/src/bot/enter_giveaways.py +++ b/src/bot/enter_giveaways.py @@ -223,13 +223,13 @@ class EnterGiveaways: if if_enter_giveaway: res = self._enter_giveaway(giveaway) if res: - GiveawayHelper.upsert_giveaway(giveaway, True, False) + GiveawayHelper.upsert_giveaway_with_details(giveaway, True, False) self._points -= int(giveaway.cost) txt = f"✅ Entered giveaway '{giveaway.game_name}'" logger.info(txt) sleep(randint(4, 15)) else: - GiveawayHelper.upsert_giveaway(giveaway, False, False) + GiveawayHelper.upsert_giveaway_with_details(giveaway, False, False) else: GiveawayHelper.upsert_giveaway(giveaway) # if we are on any filter type except New and we get to a giveaway that exceeds our diff --git a/src/bot/evaluate_won_giveaways.py b/src/bot/evaluate_won_giveaways.py new file mode 100644 index 0000000..d0ad64c --- /dev/null +++ b/src/bot/evaluate_won_giveaways.py @@ -0,0 +1,90 @@ +import json +from random import randint +from time import sleep + +import requests +from bs4 import BeautifulSoup +from requests.adapters import HTTPAdapter +from urllib3.util import Retry + +from .log import get_logger +from .database import NotificationHelper, GiveawayHelper +from .won_entry import WonEntry + +logger = get_logger(__name__) + + +class SteamGiftsException(Exception): + pass + + +class EvaluateWonGiveaways: + + def __init__(self, cookie, user_agent, notification): + self._contributor_level = None + self._xsrf_token = None + self._cookie = { + 'PHPSESSID': cookie + } + self._user_agent = user_agent + self._notification = notification + + self._base = "https://www.steamgifts.com/giveaways/won" + self._session = requests.Session() + + def start(self): + self._evaluate_won_giveaways() + + def _requests_retry_session( + self, + retries=5, + backoff_factor=0.3 + ): + session = self._session or requests.Session() + retry = Retry( + total=retries, + read=retries, + connect=retries, + backoff_factor=backoff_factor, + status_forcelist=(500, 502, 504), + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + return session + + def _get_soup_from_page(self, url): + headers = { + 'User-Agent': self._user_agent + } + self._requests_retry_session().get(url, headers=headers) + r = requests.get(url, cookies=self._cookie) + soup = BeautifulSoup(r.text, 'html.parser') + return soup + + def _evaluate_won_giveaways(self, page=1): + soup = self._get_soup_from_page(self._base) + + try: + self._xsrf_token = soup.find('input', {'name': 'xsrf_token'})['value'] + self._points = int(soup.find('span', {'class': 'nav__points'}).text) # storage points + self._contributor_level = int(float(soup.select_one('nav a>span[title]')['title'])) + except TypeError: + logger.error("⛔⛔⛔ Cookie is not valid. A new one must be added.⛔⛔⛔") + raise SteamGiftsException("Cookie is not valid. A new one must be added.") + + won_game_list = soup.select('div[class=table__row-inner-wrap]') + + if not len(won_game_list): + txt = f"🟡 No won games to evaluate" + logger.info(txt) + + logger.info("Evaluating won giveaways") + for item in won_game_list: + won_giveaway = WonEntry(item) + w = GiveawayHelper.get_by_giveaway_id(won_giveaway.giveaway_game_id) + logger.debug(f"Giveaway in db: {w}") + if w and not w.won and w.entered: + logger.info(f"Found new won giveaway not already marked as won. Marking. {w}") + GiveawayHelper.mark_game_as_won(won_giveaway.giveaway_game_id) + diff --git a/src/bot/giveaway_thread.py b/src/bot/giveaway_thread.py index be15ae9..4cd7b1e 100644 --- a/src/bot/giveaway_thread.py +++ b/src/bot/giveaway_thread.py @@ -7,8 +7,10 @@ from time import sleep from dateutil import tz +from .evaluate_won_giveaways import EvaluateWonGiveaways from .log import get_logger from .enter_giveaways import EnterGiveaways +from .scheduler import Scheduler logger = get_logger(__name__) @@ -47,6 +49,17 @@ class GiveawayThread(threading.Thread): sleep(10) exit(-1) + logger.debug("Creating scheduler") + scheduler = Scheduler() + won_giveaway_job = scheduler.get_job(job_id='eval_giveaways') + if won_giveaway_job: + logger.debug("Previous won giveaway evaluator job exists. Removing.") + won_giveaway_job.remove() + scheduler.add_job(EvaluateWonGiveaways(cookie, user_agent, notification).start, + id='eval_giveaways', trigger='interval', max_instances=1, replace_existing=True, days=1, + next_run_time=datetime.now() + timedelta(minutes=1)) + scheduler.start() + while True: logger.info("🟢 Evaluating giveaways.") if wishlist_page_enabled: diff --git a/src/bot/models.py b/src/bot/models.py index 23272be..419f2c5 100644 --- a/src/bot/models.py +++ b/src/bot/models.py @@ -19,6 +19,9 @@ class TableNotification(Base): __mapper_args__ = {"eager_defaults": True} + def __str__(self): + return str(self.__class__) + ": " + str(self.__dict__) + class TableSteamItem(Base): __tablename__ = 'steam_item' @@ -30,6 +33,9 @@ class TableSteamItem(Base): giveaways = relationship("TableGiveaway", back_populates="steam_item") + def __str__(self): + return str(self.__class__) + ": " + str(self.__dict__) + class TableGiveaway(Base): __tablename__ = 'giveaway' @@ -43,6 +49,7 @@ class TableGiveaway(Base): copies = Column(Integer(), nullable=False) contributor_level = Column(Integer(), nullable=False) entered = Column(Boolean(), nullable=False) + won = Column(Boolean(), nullable=False) game_entries = Column(Integer(), nullable=False) created_at = Column(DateTime(timezone=True), server_default=func.now()) updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) @@ -50,3 +57,6 @@ class TableGiveaway(Base): steam_item = relationship("TableSteamItem", back_populates="giveaways") __mapper_args__ = {"eager_defaults": True} + + def __str__(self): + return str(self.__class__) + ": " + str(self.__dict__) diff --git a/src/bot/scheduler.py b/src/bot/scheduler.py new file mode 100644 index 0000000..c8dad93 --- /dev/null +++ b/src/bot/scheduler.py @@ -0,0 +1,33 @@ +import os + +from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore +from apscheduler.schedulers.background import BackgroundScheduler + +from .log import get_logger + +logger = get_logger(__name__) + + +class Scheduler: + class __Scheduler: + def __init__(self): + jobstores = { + 'default': SQLAlchemyJobStore(url=f"{os.getenv('BOT_DB_URL', 'sqlite:///./config/sqlite.db')}") + } + job_defaults = { + 'coalesce': True, + 'max_instances': 3 + } + self.scheduler = BackgroundScheduler(jobstores=jobstores, job_defaults=job_defaults) + + def __getattr__(self, name): + return getattr(self.scheduler, name) + + instance = None + + def __init__(self): + if not Scheduler.instance: + Scheduler.instance = Scheduler.__Scheduler() + + def __getattr__(self, name): + return getattr(self.instance, name) diff --git a/src/bot/won_entry.py b/src/bot/won_entry.py new file mode 100644 index 0000000..04a8c15 --- /dev/null +++ b/src/bot/won_entry.py @@ -0,0 +1,22 @@ +import re +from .log import get_logger +import time + +logger = get_logger(__name__) + + +class WonEntry: + + def __init__(self, soup_item): + self.game_name = None + self.giveaway_game_id = None + self.giveaway_uri = None + + logger.debug(f"Won Giveaway html: {soup_item}") + self.game_name = soup_item.find('a', {'class': 'table__column__heading'}).text + self.giveaway_game_id = soup_item.find('a', {'class': 'table__column__heading'})['href'].split('/')[2] + self.giveaway_uri = soup_item.select_one('a.table__column__heading')['href'] + logger.debug(f"Scraped Won Giveaway: {self}") + + def __str__(self): + return str(self.__class__) + ": " + str(self.__dict__) diff --git a/src/web/templates/stats.html b/src/web/templates/stats.html index 9e1a8d2..f525a15 100644 --- a/src/web/templates/stats.html +++ b/src/web/templates/stats.html @@ -5,5 +5,6 @@ {% endblock %} \ No newline at end of file diff --git a/src/web/webserver_thread.py b/src/web/webserver_thread.py index 3f4b9ed..3f01a5b 100644 --- a/src/web/webserver_thread.py +++ b/src/web/webserver_thread.py @@ -74,7 +74,8 @@ class WebServerThread(threading.Thread): def stats(): totals = GiveawayHelper.total_giveaways() entered = GiveawayHelper.total_entered() - return render_template('stats.html', name=self.prefix, totals=totals, entered=entered) + won = GiveawayHelper.total_won() + return render_template('stats.html', name=self.prefix, totals=totals, entered=entered, won=won) if self.enabled: logger.info("Webserver Enabled. Running")