X-Git-Url: https://git.armaanb.net/?a=blobdiff_plain;f=extract.py;h=c83755fd1caf2f655299112cb37bb86e05caf9dd;hb=832a355da65ad719cac5712d5fabcee41b64c602;hp=8f8877a8fd51968751022a80bc2272edb120de49;hpb=9ffdc6ab29db2f90e880f31c9cb7c5febfda7818;p=phrases.git

diff --git a/extract.py b/extract.py
index 8f8877a..c83755f 100755
--- a/extract.py
+++ b/extract.py
@@ -4,27 +4,26 @@
 
 import argparse
 import sqlite3
-import sys
 import requests
 from bs4 import BeautifulSoup
 
-def main(args=sys.argv[1:]):
-    # Argument parsing
+def parse_args():
     parser = argparse.ArgumentParser(
         description="Generate SQLite db of Latin famous phrases from Wikipedia.")
     parser.add_argument("-o", "--output",
                        default="phrases.db",
                        help="set custom output file location")
-    args = parser.parse_args()
+    parser.add_argument("-v", "--version",
+                        action="store_true",
+                        help="print script version")
+    return parser.parse_args()
 
-    url = ("""https://en.wikipedia.org/w/index.php?title=List_of_Latin_phrases_(
-          full)&oldid=986793908""")
+def get_html(url):
     print("downloading webpage")
-    soup = BeautifulSoup(requests.get(url).content, "html.parser")
+    return BeautifulSoup(requests.get(url).content, "html.parser")
 
+def prep_database():
     print("prepping database")
-    conn = sqlite3.connect(args.output)
-    c = conn.cursor()
     c.execute("DROP TABLE IF EXISTS phrases")
     c.execute("""CREATE TABLE phrases(
               id INTEGER,
@@ -33,29 +32,41 @@ def main(args=sys.argv[1:]):
               notes TEXT,
               length INTEGER)""")
 
-    i = 0 # For the phrase id
-
-    # iterate through the tables in the page
-    list_table = soup.find_all("table", attrs={"class":"wikitable"})
+def fill_database(list_table):
+    i = 0 # phrase id
     print("iterating through tables")
     for table in list_table:
         for row in table.tbody.find_all("tr", recursive=False):
             cell = row.find_all("td", recursive=False)
             if len(cell) > 2:
                 print(i, end="\r")
+
                 latin = (cell[0].get_text(" ", strip=True)).rstrip()
                 english = (cell[1].get_text(" ", strip=True)).rstrip()
                 notes = (cell[2].get_text(" ", strip=True)).rstrip()
     
-                c.execute("""INSERT INTO phrases (id, latin, english, notes, length)
-                          VALUES(?, ?, ?, ?, ?)""", (i, latin, english, notes, len(latin)))
+                c.execute("""INSERT INTO phrases
+                         (id, latin, english, notes, length)
+                         VALUES(?, ?, ?, ?, ?)""",
+                         (i, latin, english, notes, len(latin)))
                 conn.commit()
-
             i = i + 1
 
-    print("closing database")
-    c.close()
-    conn.close()
+def get_tables():
+    url = ("""https://en.wikipedia.org/w/index.php?title=List_of_Latin_phrases_(
+          full)&oldid=986793908""")
+    return get_html(url).find_all("table", attrs={"class":"wikitable"})
+
+def main():
+    if args.version:
+        print(version)
+    else:
+        prep_database()
+        fill_database(get_tables())
 
 if __name__ == "__main__":
+    version = "phrases extract.py 1.0.1"
+    args = parse_args()
+    conn = sqlite3.connect(args.output)
+    c = conn.cursor()
     main()