X-Git-Url: https://git.armaanb.net/?p=phrases.git;a=blobdiff_plain;f=extract.py;h=4a0a6b00cb3b44a01e3b02e54506dcd6197379b7;hp=a454569a2eebff59cb883c512b389bd7bb98434b;hb=HEAD;hpb=ce91c175943f73684423627c098ae3d7f4fa94ab diff --git a/extract.py b/extract.py index a454569..4a0a6b0 100755 --- a/extract.py +++ b/extract.py @@ -1,23 +1,75 @@ #!/usr/bin/env python3 -# Tool to extract famous phrases from wikipedia -from bs4 import BeautifulSoup +# Extract Latin famous phrases from Wikipedia +# Armaan Bhojwani 2021 + +import argparse +import sqlite3 import requests +from bs4 import BeautifulSoup + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Generate database of Latin famous phrases from Wikipedia") + parser.add_argument("-o", "--output", + default="phrases.db", + help="set custom output file location") + parser.add_argument("-v", "--version", + action="version", + version="phrases-extract 1.0.3") + return parser.parse_args() + + +def get_html(url): + print("downloading webpage") + return BeautifulSoup(requests.get(url).content, "html.parser") + + +def prep_database(): + print("prepping database") + c.execute("DROP TABLE IF EXISTS phrases") + c.execute("""CREATE TABLE phrases( + id INTEGER, + latin TEXT, + english TEXT, + notes TEXT, + length INTEGER)""") + + +def fill_database(list_table): + i = 0 # phrase id + print("iterating through tables") + for table in list_table: + for row in table.tbody.find_all("tr", recursive=False): + cell = row.find_all("td", recursive=False) + if len(cell) > 2: + print(i, end="\r") + + latin = (cell[0].get_text(" ", strip=True)).rstrip() + english = (cell[1].get_text(" ", strip=True)).rstrip() + notes = (cell[2].get_text(" ", strip=True)).rstrip() + + c.execute("""INSERT INTO phrases + (id, latin, english, notes, length) + VALUES(?, ?, ?, ?, ?)""", + (i, latin, english, notes, len(latin))) + conn.commit() + i = i + 1 -url = 'https://en.wikipedia.org/wiki/List_of_Latin_phrases_(full)' -response = requests.get(url) -html = response.content -soup = BeautifulSoup(html, "html.parser") -list_table = soup.find_all("table", attrs={"class":"wikitable"}) +def get_tables(): + url = ("""https://en.wikipedia.org/w/index.php?title=List_of_Latin_phrases_( + full)&oldid=986793908""") + return get_html(url).find_all("table", attrs={"class": "wikitable"}) -output = [] -for table in list_table: - for row in table.find_all("tr")[1:]: - cell = row.find_all("td") - for content in cell: - text = content.get_text() - output.append(text) +def main(): + prep_database() + fill_database(get_tables()) -print(output) +if __name__ == "__main__": + args = parse_args() + conn = sqlite3.connect(args.output) + c = conn.cursor() + main()