win scraping

- will scape won games at start to update db for giveaways the bot has entered
- schedules a once a day run of scraping won giveaways

requirements.txt

@@ -2,10 +2,6 @@ requests==2.27.1
 beautifulsoup4==4.11.1
 urllib3==1.26.9
 sqlalchemy==1.4.36
-sqlalchemy_utils==0.38.2
-paginate-sqlalchemy==0.3.1
 alembic==1.7.7
 python-dateutil==2.8.2
-Flask==2.1.2
+apscheduler==3.9.1
-Flask-BasicAuth==0.2.0
-pyopenssl==22.0.0

src/bot/database.py

@@ -11,7 +11,7 @@ from sqlalchemy.orm import Session, joinedload
 from sqlalchemy_utils import database_exists
 
 from .log import get_logger
-from .models import TableNotification, TableGiveaway, TableSteamItem, TableTask
+from .models import TableNotification, TableGiveaway, TableSteamItem
 
 logger = get_logger(__name__)
 engine = sqlalchemy.create_engine(f"{os.getenv('BOT_DB_URL', 'sqlite:///./config/sqlite.db')}", echo=False)
@@ -116,6 +116,16 @@ class GiveawayHelper:
         with Session(engine) as session:
             return session.query(TableGiveaway).filter_by(entered=True).count()
 
+    @classmethod
+    def total_won(cls):
+        with Session(engine) as session:
+            return session.query(TableGiveaway).filter_by(won=True).count()
+
+    @classmethod
+    def get_by_giveaway_id(cls, game_id):
+        with Session(engine) as session:
+            return session.query(TableGiveaway).filter_by(giveaway_id=game_id).one_or_none()
+
     @classmethod
     def unix_timestamp_to_utc_datetime(cls, timestamp):
         return datetime.utcfromtimestamp(timestamp)
@@ -126,6 +136,17 @@ class GiveawayHelper:
             return session.query(TableGiveaway).filter_by(giveaway_id=giveaway.giveaway_game_id,
                                                           steam_id=giveaway.steam_app_id).all()
 
+    @classmethod
+    def mark_game_as_won(cls, game_id):
+        with Session(engine) as session:
+            result = session.query(TableGiveaway).filter_by(giveaway_id=game_id).all()
+            if result:
+                won_giveaway = result[0]
+                if not won_giveaway.won:
+                    won_giveaway.won = True
+                    session.add(won_giveaway)
+                    session.commit()
+
     @classmethod
     def insert(cls, giveaway, entered, won):
         with Session(engine) as session:
@@ -149,7 +170,7 @@ class GiveawayHelper:
                 giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp),
                 cost=giveaway.cost,
                 copies=giveaway.copies,
-                contributor_level=giveaway._contributor_level,
+                contributor_level=giveaway.contributor_level,
                 entered=entered,
                 won=won,
                 game_entries=giveaway.game_entries)
@@ -157,7 +178,7 @@ class GiveawayHelper:
             session.commit()
 
     @classmethod
-    def upsert_giveaway(cls, giveaway, entered, won):
+    def upsert_giveaway_with_details(cls, giveaway, entered, won):
         result = GiveawayHelper.get_by_ids(giveaway)
         if not result:
             GiveawayHelper.insert(giveaway, entered, won)
@@ -172,7 +193,7 @@ class GiveawayHelper:
                     giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp),
                     cost=giveaway.cost,
                     copies=giveaway.copies,
-                    contributor_level=giveaway._contributor_level,
+                    contributor_level=giveaway.contributor_level,
                     entered=entered,
                     won=won,
                     game_entries=giveaway.game_entries)
@@ -195,7 +216,7 @@ class GiveawayHelper:
                     giveaway_ended_at=GiveawayHelper.unix_timestamp_to_utc_datetime(giveaway.time_remaining_timestamp),
                     cost=giveaway.cost,
                     copies=giveaway.copies,
-                    contributor_level=giveaway._contributor_level,
+                    contributor_level=giveaway.contributor_level,
                     game_entries=giveaway.game_entries)
                 session.merge(g)
                 session.commit()

src/bot/enter_giveaways.py

@@ -223,13 +223,13 @@ class EnterGiveaways:
                 if if_enter_giveaway:
                     res = self._enter_giveaway(giveaway)
                     if res:
-                        GiveawayHelper.upsert_giveaway(giveaway, True, False)
+                        GiveawayHelper.upsert_giveaway_with_details(giveaway, True, False)
                         self._points -= int(giveaway.cost)
                         txt = f"✅ Entered giveaway '{giveaway.game_name}'"
                         logger.info(txt)
                         sleep(randint(4, 15))
                     else:
-                        GiveawayHelper.upsert_giveaway(giveaway, False, False)
+                        GiveawayHelper.upsert_giveaway_with_details(giveaway, False, False)
                 else:
                     GiveawayHelper.upsert_giveaway(giveaway)
                 # if we are on any filter type except New and we get to a giveaway that exceeds our

src/bot/evaluate_won_giveaways.py

@@ -0,0 +1,90 @@
+import json
+from random import randint
+from time import sleep
+
+import requests
+from bs4 import BeautifulSoup
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
+
+from .log import get_logger
+from .database import NotificationHelper, GiveawayHelper
+from .won_entry import WonEntry
+
+logger = get_logger(__name__)
+
+
+class SteamGiftsException(Exception):
+    pass
+
+
+class EvaluateWonGiveaways:
+
+    def __init__(self, cookie, user_agent, notification):
+        self._contributor_level = None
+        self._xsrf_token = None
+        self._cookie = {
+            'PHPSESSID': cookie
+        }
+        self._user_agent = user_agent
+        self._notification = notification
+
+        self._base = "https://www.steamgifts.com/giveaways/won"
+        self._session = requests.Session()
+
+    def start(self):
+        self._evaluate_won_giveaways()
+
+    def _requests_retry_session(
+            self,
+            retries=5,
+            backoff_factor=0.3
+    ):
+        session = self._session or requests.Session()
+        retry = Retry(
+            total=retries,
+            read=retries,
+            connect=retries,
+            backoff_factor=backoff_factor,
+            status_forcelist=(500, 502, 504),
+        )
+        adapter = HTTPAdapter(max_retries=retry)
+        session.mount('http://', adapter)
+        session.mount('https://', adapter)
+        return session
+
+    def _get_soup_from_page(self, url):
+        headers = {
+            'User-Agent': self._user_agent
+        }
+        self._requests_retry_session().get(url, headers=headers)
+        r = requests.get(url, cookies=self._cookie)
+        soup = BeautifulSoup(r.text, 'html.parser')
+        return soup
+
+    def _evaluate_won_giveaways(self, page=1):
+        soup = self._get_soup_from_page(self._base)
+
+        try:
+            self._xsrf_token = soup.find('input', {'name': 'xsrf_token'})['value']
+            self._points = int(soup.find('span', {'class': 'nav__points'}).text)  # storage points
+            self._contributor_level = int(float(soup.select_one('nav a>span[title]')['title']))
+        except TypeError:
+            logger.error("⛔⛔⛔  Cookie is not valid. A new one must be added.⛔⛔⛔")
+            raise SteamGiftsException("Cookie is not valid. A new one must be added.")
+
+        won_game_list = soup.select('div[class=table__row-inner-wrap]')
+
+        if not len(won_game_list):
+            txt = f"🟡 No won games to evaluate"
+            logger.info(txt)
+
+        logger.info("Evaluating won giveaways")
+        for item in won_game_list:
+            won_giveaway = WonEntry(item)
+            w = GiveawayHelper.get_by_giveaway_id(won_giveaway.giveaway_game_id)
+            logger.debug(f"Giveaway in db: {w}")
+            if w and not w.won and w.entered:
+                logger.info(f"Found new won giveaway not already marked as won. Marking. {w}")
+                GiveawayHelper.mark_game_as_won(won_giveaway.giveaway_game_id)
+

src/bot/giveaway_thread.py

@@ -7,8 +7,10 @@ from time import sleep
 
 from dateutil import tz
 
+from .evaluate_won_giveaways import EvaluateWonGiveaways
 from .log import get_logger
 from .enter_giveaways import EnterGiveaways
+from .scheduler import Scheduler
 
 logger = get_logger(__name__)
 
@@ -47,6 +49,17 @@ class GiveawayThread(threading.Thread):
             sleep(10)
             exit(-1)
 
+        logger.debug("Creating scheduler")
+        scheduler = Scheduler()
+        won_giveaway_job = scheduler.get_job(job_id='eval_giveaways')
+        if won_giveaway_job:
+            logger.debug("Previous won giveaway evaluator job exists. Removing.")
+            won_giveaway_job.remove()
+        scheduler.add_job(EvaluateWonGiveaways(cookie, user_agent, notification).start,
+                          id='eval_giveaways', trigger='interval', max_instances=1, replace_existing=True, days=1,
+                          next_run_time=datetime.now() + timedelta(minutes=1))
+        scheduler.start()
+
         while True:
             logger.info("🟢 Evaluating giveaways.")
             if wishlist_page_enabled:

src/bot/models.py

@@ -19,6 +19,9 @@ class TableNotification(Base):
 
     __mapper_args__ = {"eager_defaults": True}
 
+    def __str__(self):
+        return str(self.__class__) + ": " + str(self.__dict__)
+
 
 class TableSteamItem(Base):
     __tablename__ = 'steam_item'
@@ -30,6 +33,9 @@ class TableSteamItem(Base):
 
     giveaways = relationship("TableGiveaway", back_populates="steam_item")
 
+    def __str__(self):
+        return str(self.__class__) + ": " + str(self.__dict__)
+
 
 class TableGiveaway(Base):
     __tablename__ = 'giveaway'
@@ -43,6 +49,7 @@ class TableGiveaway(Base):
     copies = Column(Integer(), nullable=False)
     contributor_level = Column(Integer(), nullable=False)
     entered = Column(Boolean(), nullable=False)
+    won = Column(Boolean(), nullable=False)
     game_entries = Column(Integer(), nullable=False)
     created_at = Column(DateTime(timezone=True), server_default=func.now())
     updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
@@ -50,3 +57,6 @@ class TableGiveaway(Base):
     steam_item = relationship("TableSteamItem", back_populates="giveaways")
 
     __mapper_args__ = {"eager_defaults": True}
+
+    def __str__(self):
+        return str(self.__class__) + ": " + str(self.__dict__)

src/bot/scheduler.py

@@ -0,0 +1,33 @@
+import os
+
+from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
+from apscheduler.schedulers.background import BackgroundScheduler
+
+from .log import get_logger
+
+logger = get_logger(__name__)
+
+
+class Scheduler:
+    class __Scheduler:
+        def __init__(self):
+            jobstores = {
+                'default': SQLAlchemyJobStore(url=f"{os.getenv('BOT_DB_URL', 'sqlite:///./config/sqlite.db')}")
+            }
+            job_defaults = {
+                'coalesce': True,
+                'max_instances': 3
+            }
+            self.scheduler = BackgroundScheduler(jobstores=jobstores, job_defaults=job_defaults)
+
+        def __getattr__(self, name):
+            return getattr(self.scheduler, name)
+
+    instance = None
+
+    def __init__(self):
+        if not Scheduler.instance:
+            Scheduler.instance = Scheduler.__Scheduler()
+
+    def __getattr__(self, name):
+        return getattr(self.instance, name)

src/bot/won_entry.py

@@ -0,0 +1,22 @@
+import re
+from .log import get_logger
+import time
+
+logger = get_logger(__name__)
+
+
+class WonEntry:
+
+    def __init__(self, soup_item):
+        self.game_name = None
+        self.giveaway_game_id = None
+        self.giveaway_uri = None
+
+        logger.debug(f"Won Giveaway html: {soup_item}")
+        self.game_name = soup_item.find('a', {'class': 'table__column__heading'}).text
+        self.giveaway_game_id = soup_item.find('a', {'class': 'table__column__heading'})['href'].split('/')[2]
+        self.giveaway_uri = soup_item.select_one('a.table__column__heading')['href']
+        logger.debug(f"Scraped Won Giveaway: {self}")
+
+    def __str__(self):
+        return str(self.__class__) + ": " + str(self.__dict__)

src/web/templates/stats.html

@@ -5,5 +5,6 @@
     <ul>
         <li>Total Giveaways Considered: {{totals}}</li>
         <li>Giveaways Entered: {{entered}}</li>
+        <li>Giveaways Won (with the bot): {{won}}</li>
     </ul>
 {% endblock %}

src/web/webserver_thread.py

@@ -74,7 +74,8 @@ class WebServerThread(threading.Thread):
         def stats():
             totals = GiveawayHelper.total_giveaways()
             entered = GiveawayHelper.total_entered()
-            return render_template('stats.html', name=self.prefix, totals=totals, entered=entered)
+            won = GiveawayHelper.total_won()
+            return render_template('stats.html', name=self.prefix, totals=totals, entered=entered, won=won)
 
         if self.enabled:
             logger.info("Webserver Enabled. Running")