extract.py

   1 #!/usr/bin/env python3
   2 # Extract Latin famous phrases from wikipedia
   3 # Armaan Bhojwani 2020
   4
   5 from bs4 import BeautifulSoup
   6 import requests
   7 import csv
   8
   9 def main():
  10     url = 'https://en.wikipedia.org/wiki/List_of_Latin_phrases_(full)'
  11     response = requests.get(url)
  12     html = response.content
  13
  14     soup = BeautifulSoup(html, "html.parser")
  15     list_table = soup.find_all("table", attrs={"class":"wikitable"})
  16     with open('phrases.csv', 'w') as f:
  17         writer = csv.writer(f)
  18
  19         i = 0 # For the phrase id
  20         # iterate through the tables in the page
  21         for table in list_table:
  22             for row in table.find_all("tr")[1:]:
  23                 cell = row.find_all("td")
  24                 rowc = []
  25
  26                 # append phrase id
  27                 rowc.append(i)
  28
  29                 # avoid out of bounds errors
  30                 if len(cell) == 2:
  31                     lan = 2
  32                 else:
  33                     lan = 3
  34
  35                 # add cell content
  36                 for j in range (0, lan):
  37                     content = cell[j]
  38                     text=(content.get_text()).rstrip()
  39                     rowc.append(text)
  40
  41                 # append length of phrase
  42                 rowc.append(len(rowc[1]))
  43                 writer.writerow(rowc)
  44                 i = i + 1
  45     f.close()
  46
  47     # Strip empty lines
  48     with open('phrases.csv', 'r+') as f:
  49         lines = f.readlines()
  50         f.seek(0)
  51         f.writelines(line for line in lines if line.strip())
  52         f.truncate()
  53     f.close()
  54
  55 if __name__ == "__main__":
  56     main()