extract.py

   1 #!/usr/bin/env python3
   2 # Extract Latin famous phrases from wikipedia
   3 # Armaan Bhojwani 2020
   4
   5 from bs4 import BeautifulSoup
   6 import requests
   7 import csv
   8
   9 def main():
  10     url = 'https://en.wikipedia.org/wiki/List_of_Latin_phrases_(full)'
  11     response = requests.get(url)
  12     html = response.content
  13
  14     soup = BeautifulSoup(html, "html.parser")
  15     list_table = soup.find_all("table", attrs={"class":"wikitable"})
  16     with open('phrases.csv', 'w') as f:
  17         writer = csv.writer(f, lineterminator="\n")
  18
  19         i = 0 # For the phrase id
  20
  21         # write header
  22         headers = ['id', 'Latin', 'English', 'Notes', 'Length']
  23         writer.writerow(headers)
  24
  25         # iterate through the tables in the page
  26         for table in list_table:
  27             for row in table.tbody.find_all("tr", recursive=False):
  28                 cell = row.find_all("td", recursive=False)
  29                 rowc = []
  30
  31                 rowc.append(i) # append phrase id
  32
  33                 # add cell content
  34                 for content in cell:
  35                     text = (content.get_text(" ", strip=True)).rstrip()
  36                     rowc.append(text)
  37
  38                 if len(rowc) > 1:
  39                     rowc.append(len(rowc[1]))
  40                     writer.writerow(rowc)
  41                 i = i + 1
  42     f.close()
  43
  44 if __name__ == "__main__":
  45     main()