X-Git-Url: https://git.armaanb.net/?p=phrases.git;a=blobdiff_plain;f=extract.py;h=4a0a6b00cb3b44a01e3b02e54506dcd6197379b7;hp=833594da483748f0608acaa5c143e88249d007e9;hb=HEAD;hpb=58ad575e75427a13fdca64a17ced4ae2f3f986ad

diff --git a/extract.py b/extract.py
index 833594d..4a0a6b0 100755
--- a/extract.py
+++ b/extract.py
@@ -1,25 +1,31 @@
 #!/usr/bin/env python3
-# Extract Latin famous phrases from wikipedia
-# Armaan Bhojwani 2020
+# Extract Latin famous phrases from Wikipedia
+# Armaan Bhojwani 2021
 
 import argparse
 import sqlite3
 import requests
 from bs4 import BeautifulSoup
 
+
 def parse_args():
     parser = argparse.ArgumentParser(
-        description="Generate SQLite db of Latin famous phrases from Wikipedia.")
+        description="Generate database of Latin famous phrases from Wikipedia")
     parser.add_argument("-o", "--output",
-                       default="phrases.db",
-                       help="set custom output file location")
+                        default="phrases.db",
+                        help="set custom output file location")
+    parser.add_argument("-v", "--version",
+                        action="version",
+                        version="phrases-extract 1.0.3")
     return parser.parse_args()
 
+
 def get_html(url):
     print("downloading webpage")
     return BeautifulSoup(requests.get(url).content, "html.parser")
 
-def prep_database(c):
+
+def prep_database():
     print("prepping database")
     c.execute("DROP TABLE IF EXISTS phrases")
     c.execute("""CREATE TABLE phrases(
@@ -29,8 +35,9 @@ def prep_database(c):
               notes TEXT,
               length INTEGER)""")
 
-def fill_database(list_table, c, conn):
-    i = 0 # phrase id
+
+def fill_database(list_table):
+    i = 0  # phrase id
     print("iterating through tables")
     for table in list_table:
         for row in table.tbody.find_all("tr", recursive=False):
@@ -41,24 +48,28 @@ def fill_database(list_table, c, conn):
                 latin = (cell[0].get_text(" ", strip=True)).rstrip()
                 english = (cell[1].get_text(" ", strip=True)).rstrip()
                 notes = (cell[2].get_text(" ", strip=True)).rstrip()
-    
+
                 c.execute("""INSERT INTO phrases
                          (id, latin, english, notes, length)
                          VALUES(?, ?, ?, ?, ?)""",
-                         (i, latin, english, notes, len(latin)))
+                          (i, latin, english, notes, len(latin)))
                 conn.commit()
             i = i + 1
 
+
 def get_tables():
     url = ("""https://en.wikipedia.org/w/index.php?title=List_of_Latin_phrases_(
           full)&oldid=986793908""")
-    return get_html(url).find_all("table", attrs={"class":"wikitable"})
+    return get_html(url).find_all("table", attrs={"class": "wikitable"})
+
+
+def main():
+    prep_database()
+    fill_database(get_tables())
 
-def main(args):
-    conn = sqlite3.connect(args.output)
-    c = conn.cursor()
-    prep_database(c)
-    fill_database(get_tables(), c, conn)
 
 if __name__ == "__main__":
-    main(parse_args())
+    args = parse_args()
+    conn = sqlite3.connect(args.output)
+    c = conn.cursor()
+    main()