# Extract Latin famous phrases from wikipedia
# Armaan Bhojwani 2020
-from bs4 import BeautifulSoup
+import argparse
+import sqlite3
import requests
-import csv
-
-def main():
- url = 'https://en.wikipedia.org/wiki/List_of_Latin_phrases_(full)'
- response = requests.get(url)
- html = response.content
+from bs4 import BeautifulSoup
- soup = BeautifulSoup(html, "html.parser")
- list_table = soup.find_all("table", attrs={"class":"wikitable"})
- with open('phrases.csv', 'w') as f:
- writer = csv.writer(f, lineterminator="\n")
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description="Generate SQLite db of Latin famous phrases from Wikipedia.")
+ parser.add_argument("-o", "--output",
+ default="phrases.db",
+ help="set custom output file location")
+ parser.add_argument("-v", "--version",
+ action="store_true",
+ help="print script version")
+ return parser.parse_args()
- i = 0 # For the phrase id
+def get_html(url):
+ print("downloading webpage")
+ return BeautifulSoup(requests.get(url).content, "html.parser")
- # write header
- headers = ['id', 'Latin', 'English', 'Notes', 'Length']
- writer.writerow(headers)
+def prep_database():
+ print("prepping database")
+ c.execute("DROP TABLE IF EXISTS phrases")
+ c.execute("""CREATE TABLE phrases(
+ id INTEGER,
+ latin TEXT,
+ english TEXT,
+ notes TEXT,
+ length INTEGER)""")
- # iterate through the tables in the page
- for table in list_table:
- for row in table.tbody.find_all("tr", recursive=False):
- cell = row.find_all("td", recursive=False)
- rowc = []
+def fill_database(list_table):
+ i = 0 # phrase id
+ print("iterating through tables")
+ for table in list_table:
+ for row in table.tbody.find_all("tr", recursive=False):
+ cell = row.find_all("td", recursive=False)
+ if len(cell) > 2:
+ print(i, end="\r")
- rowc.append(i) # append phrase id
+ latin = (cell[0].get_text(" ", strip=True)).rstrip()
+ english = (cell[1].get_text(" ", strip=True)).rstrip()
+ notes = (cell[2].get_text(" ", strip=True)).rstrip()
+
+ c.execute("""INSERT INTO phrases
+ (id, latin, english, notes, length)
+ VALUES(?, ?, ?, ?, ?)""",
+ (i, latin, english, notes, len(latin)))
+ conn.commit()
+ i = i + 1
- # add cell content
- for content in cell:
- text = (content.get_text(" ", strip=True)).rstrip()
- rowc.append(text)
+def get_tables():
+ url = ("""https://en.wikipedia.org/w/index.php?title=List_of_Latin_phrases_(
+ full)&oldid=986793908""")
+ return get_html(url).find_all("table", attrs={"class":"wikitable"})
- if len(rowc) > 1:
- rowc.append(len(rowc[1]))
- writer.writerow(rowc)
- i = i + 1
- f.close()
+def main():
+ if args.version:
+ print(version)
+ else:
+ prep_database()
+ fill_database(get_tables())
if __name__ == "__main__":
+ version = "phrases extract.py 1.0.1"
+ args = parse_args()
+ conn = sqlite3.connect(args.output)
+ c = conn.cursor()
main()