Minor changes to scrape.py

Author: carlson9

Day4/scrape.py

@@ -9,6 +9,11 @@
 import random
 import time
 import os
+import re
+
+#from nltk.util import clean_html
+#import urllib2 
+
 
 # Open a web page
 web_address='https://polisci.wustl.edu/faculty/specialization'
@@ -25,6 +30,7 @@
 
 # Get the attributes
 my_a_tag=soup.find_all('a')[2]
+my_a_tag = re.sub(r'<[^>]+>', '', str(my_a_tag)) #remove tags
 my_a_tag.attrs #Gives a dictionary with the attributes
 my_a_tag.attrs.keys()
 my_a_tag['alt']
@@ -36,6 +42,8 @@
 mysection=soup.find_all('div')[0]
 mysection.a #Gives the 'a' tag within the 'div' tag
 mysection.find_all('a') #Gives the list of all 'a' tags within the 'div' tag
+mysection.get_text()
+
 
 # Creating a tree of objects
 

Day4/scrape.py~

@@ -9,6 +9,11 @@ import urllib2
 import random
 import time
 import os
+import re
+
+#from nltk.util import clean_html
+#import urllib2 
+
 
 # Open a web page
 web_address='https://polisci.wustl.edu/faculty/specialization'
@@ -22,12 +27,10 @@ soup.prettify()
 soup.find_all('h3')
 soup.find_all('a')
 
-# Get the script of a certain tag
-mytitle=soup.find_all('h3')[0]
-mytitle.script
 
 # Get the attributes
 my_a_tag=soup.find_all('a')[2]
+my_a_tag = re.sub(r'<[^>]+>', '', str(my_a_tag)) #remove tags
 my_a_tag.attrs #Gives a dictionary with the attributes
 my_a_tag.attrs.keys()
 my_a_tag['alt']