Notes for KR lab 1

A very small example program in Python 3 to introduce scraping and Google search api: it does a google search for Andrus Ansip with 8 results per page and then goes through the result pages (each containing 8 links).

The m function replaces some chars with whitespace, splits the page to words and prints out the words with word count, starting with the ones having the highest word count.

No file caching used here. Also, we do not check for words near the name, etc. As it is, the program is not a suitable answer for the first lab: you can, as said. use it as a starting point.

import urllib
import urllib.request
import json

def m(url):
  #print ("reading html")
  try:  
    u=urllib.request.urlopen(url)
    html = u.read().decode("utf-8")
  except:
    print("failed to read url")
    return
  #print (str(html))
  r=html.replace("."," . ").replace(","," ").replace("\""," ")
  r=r.replace("<"," ").replace(">", " ")
  s=r.split()
  d={}
  for w in s:
    if w in d:
      d[w]=d[w]+1
    else:
      d[w]=1
  l=[]
  for k in d:
    l.append((k,d[k]))
  t=sorted(l, key=lambda e: e[1])   
  print (str(t))

def g():
  print ("googling")
  start=0
  while start<100:
      print("start "+str(start))
      url="https://ajax.googleapis.com/ajax/services/search/web?v=1.0&rsz=8"
      url+="&start="+str(start)
      url+="&q="
      url+="Andrus+Ansip"
      try:
          u=urllib.request.urlopen(url)
          html = u.read().decode("iso-8859-1")
          j=json.loads(html)
          if not j or not "responseData" in j:
            print ("failed to read/parse")
            return
          p=j["responseData"]
          if not p or not "results" in p:
            print("no more results")
            return
          p=p["results"]
          for r in p:
            url=r["url"]
            print ("******* url *****")
            print (str(url))
            m(url)
      except:
        print("failed to read google search url")      
      start+=8

g()