""" util/scrape.py Utility functions for scaping web data relative to selected games """ import os from dotenv import load_dotenv import httpx from dateutil import parser, utils from bs4 import BeautifulSoup load_dotenv() def scrape_game_history(game: str, year: int = utils.today().year) -> dict: """ Scrape game data from the url provided. This function requires specific environment variables be present in order to function properly. :param:game The target game to scrape. :param:year The year to scrape. If no year is provided, scrapes games from the current year. :return: dict() Examples: >>>scrape_game_history(game='mega-millions') draw_date {'main_balls': ...} >>>scrape_game_history(game='mega-millions', year=2020) draw_date {'main_balls': ...} """ drawings = {} url = f"{os.getenv('BASE_SCRAPE_URL')}{game}/numbers/{year}" page = httpx.get(url) soup = BeautifulSoup(page.text, 'html.parser') results = soup.find_all(os.getenv('RESULT_CONTAINER'), class_=os.getenv('RESULT_CLASS')) for result in results: pick5 = [] draw_date = parser.parse(result.find(os.getenv('DRAW_DATE_CONTAINER'), class_=os.getenv('DRAW_DATE_CLASS')).text).date().strftime('%Y-%m-%d') main_balls = result.find_all(os.getenv('BALL_CONTAINER'), class_=os.getenv('BALL_CLASS')) game_ball = result.find(os.getenv('BALL_CONTAINER'), class_=[os.getenv('POWERBALL_CLASS'), os.getenv('MEGABALL_CLASS')]) multiplier = result.find(os.getenv('BALL_CONTAINER'), class_=[os.getenv('POWERBALL_MULTIPLIER_CLASS'), os.getenv('MEGA_MILLIONS_MULTIPLIER_CLASS')]) for ball in main_balls: pick5.append(int(ball.text.strip())) game_ball = int(game_ball.find('span').text) if multiplier: multiplier = int(multiplier.find('span').text) else: multiplier = 1 if game == 'mega-millions': drawings[draw_date] = {'main_balls': pick5, 'mega-ball': game_ball, 'multiplier': multiplier} elif game == 'powerball': drawings[draw_date] = {'main_balls': pick5, 'powerball': game_ball, 'multiplier': multiplier} return drawings