Change return type, code function

util/scrape.py

@@ -3,25 +3,63 @@
 
     Utility functions for scaping web data relative to selected games
 """
+import os
+from dotenv import load_dotenv
 
 import httpx
-import pandas as pd
+from dateutil import parser, utils
 from bs4 import BeautifulSoup
 
+load_dotenv()
 
-def scrape_url(game: str, year: int | None) -> pd.DataFrame:
+def scrape_url(game: str, year: int | None) -> dict:
     """
         Scrape game data from the url provided.  This function requires specific environment variables be present
         in order to function properly.
         :param:game The target game to scrape.
         :param:year The year to scrape.  If no year is provided, scrapes games from the current year.
 
-        :return: Pandas DataFrame
+        :return: dict()
 
         Examples:
             >>>scrape_url(game='mega-millions')
-            pd.DataFrame()
+            draw_date {'white_balls': ...}
 
             >>>scrape_url(game='mega-millions', year=2020)
-            pd.DataFrame()
+            draw_date {'white_balls': ...}
     """
+    drawings = {}
+
+    if year is None:
+        year = utils.now().year
+
+    url = f"{os.getenv('BASE_SCRAPE_URL')}{game}/numbers/{year}"
+    page = httpx.get(url)
+    soup = BeautifulSoup(page.text, 'html.parser')
+    results = soup.find_all(os.getenv('RESULT_CONTAINER'), class_=os.getenv('RESULT_CLASS'))
+
+    for result in results:
+        pick5 = []
+        draw_date = parser.parse(result.find(os.getenv('DRAW_DATE_CONTAINER'),
+                 class_=os.getenv('DRAW_DATE_CLASS')).text).date().strftime('%Y-%m-%d')
+        white_balls = result.find_all(os.getenv('BALL_CONTAINER'), class_=os.getenv('BALL_CLASS'))
+        game_ball = result.find(os.getenv('BALL_CONTAINER'),
+                class_=[os.getenv('POWERBALL_CLASS'), os.getenv('MEGABALL_CLASS')])
+        multiplier = result.find(os.getenv('BALL_CONTAINER'),
+                 class_=[os.getenv('POWERBALL_MULTIPLIER_CLASS'), os.getenv('MEGA_MILLIONS_MULTIPLIER_CLASS')])
+
+        for ball in white_balls:
+            pick5.append(int(ball.text.strip()))
+
+        game_ball = int(game_ball.find('span').text)
+        if multiplier:
+            multiplier = int(multiplier.find('span').text)
+        else:
+            multiplier = 1
+
+        if game == 'mega-millions':
+            drawings[draw_date] = {'white_balls': pick5, 'mega-ball': game_ball, 'multiplier': multiplier}
+        elif game == 'powerball':
+            drawings[draw_date] = {'white_balls': pick5, 'powerball': game_ball, 'multiplier': multiplier}
+
+    return drawings