diff --git a/util/scrape.py b/util/scrape.py index 8949394..299a411 100644 --- a/util/scrape.py +++ b/util/scrape.py @@ -3,25 +3,63 @@ Utility functions for scaping web data relative to selected games """ +import os +from dotenv import load_dotenv import httpx -import pandas as pd +from dateutil import parser, utils from bs4 import BeautifulSoup +load_dotenv() -def scrape_url(game: str, year: int | None) -> pd.DataFrame: +def scrape_url(game: str, year: int | None) -> dict: """ Scrape game data from the url provided. This function requires specific environment variables be present in order to function properly. :param:game The target game to scrape. :param:year The year to scrape. If no year is provided, scrapes games from the current year. - :return: Pandas DataFrame + :return: dict() Examples: >>>scrape_url(game='mega-millions') - pd.DataFrame() + draw_date {'white_balls': ...} >>>scrape_url(game='mega-millions', year=2020) - pd.DataFrame() + draw_date {'white_balls': ...} """ + drawings = {} + + if year is None: + year = utils.now().year + + url = f"{os.getenv('BASE_SCRAPE_URL')}{game}/numbers/{year}" + page = httpx.get(url) + soup = BeautifulSoup(page.text, 'html.parser') + results = soup.find_all(os.getenv('RESULT_CONTAINER'), class_=os.getenv('RESULT_CLASS')) + + for result in results: + pick5 = [] + draw_date = parser.parse(result.find(os.getenv('DRAW_DATE_CONTAINER'), + class_=os.getenv('DRAW_DATE_CLASS')).text).date().strftime('%Y-%m-%d') + white_balls = result.find_all(os.getenv('BALL_CONTAINER'), class_=os.getenv('BALL_CLASS')) + game_ball = result.find(os.getenv('BALL_CONTAINER'), + class_=[os.getenv('POWERBALL_CLASS'), os.getenv('MEGABALL_CLASS')]) + multiplier = result.find(os.getenv('BALL_CONTAINER'), + class_=[os.getenv('POWERBALL_MULTIPLIER_CLASS'), os.getenv('MEGA_MILLIONS_MULTIPLIER_CLASS')]) + + for ball in white_balls: + pick5.append(int(ball.text.strip())) + + game_ball = int(game_ball.find('span').text) + if multiplier: + multiplier = int(multiplier.find('span').text) + else: + multiplier = 1 + + if game == 'mega-millions': + drawings[draw_date] = {'white_balls': pick5, 'mega-ball': game_ball, 'multiplier': multiplier} + elif game == 'powerball': + drawings[draw_date] = {'white_balls': pick5, 'powerball': game_ball, 'multiplier': multiplier} + + return drawings