python-oxweb/ox/google.py

# -*- Mode: Python; -*-
# -*- coding: utf-8 -*-
# vi:si:et:sw=2:sts=2:ts=2
import re
import time
import urllib
import urllib2
import weakref
import threading
import Queue

import oxutils 
from oxutils import stripTags


'''
usage:
import google
google.find(query)
<generator object at 0x833aeac>

for result in google.find(query): result

result is title, url, description

google.find(query, max_results)

FIXME: how search depper than first page?
'''
DEFAULT_MAX_RESULTS = 10

def getUrl(url, data=None, headers=oxutils.net.DEFAULT_HEADERS):
  google_timeout=24*60*60
  return oxutils.cache.getUrl(url, data, headers, google_timeout)

def quote_plus(s):
  return urllib.quote_plus(s.encode('utf-8'))

def find(query, max_results=DEFAULT_MAX_RESULTS):
  url = "http://www.google.com/search?q=%s" % quote_plus(query)
  data = getUrl(url)
  link_re = r'<a href="(?P<url>[^"]*?)" class=l.*?>(?P<name>.*?)</a>' +  \
            r'.*?(?:<br>|<table.*?>)' +  \
            r'(?P<desc>.*?)' + '(?:<font color=#008000>|<a)'
  results = []
  for match in re.compile(link_re, re.DOTALL).finditer(data):
    (name, url, desc) = match.group('name', 'url', 'desc')
    results.append([(stripTags(name), url, stripTags(desc))])
  if len(results) > max_results:
    results = results[:max_results]
  return results