python-oxweb/oxweb/google.py

# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import re
import time
import urllib
import urllib2
import weakref
import threading
import Queue

import oxlib 
from oxlib import stripTags


'''
usage:
import google
google.find(query)

for result in google.find(query): result

result is title, url, description

google.find(query, max_results)

FIXME: how search depper than first page?
'''
DEFAULT_MAX_RESULTS = 10

def getUrl(url, data=None, headers=oxlib.net.DEFAULT_HEADERS):
    google_timeout=24*60*60
    return oxlib.cache.getUrl(url, data, headers, google_timeout)

def quote_plus(s):
    return urllib.quote_plus(s.encode('utf-8'))

def find(query, max_results=DEFAULT_MAX_RESULTS):
    url = "http://www.google.com/search?q=%s" % quote_plus(query)
    data = getUrl(url)
    link_re = r'<a href="(?P<url>[^"]*?)" class=l.*?>(?P<name>.*?)</a>' +  \
              r'.*?(?:<br>|<table.*?>)' +  \
              r'(?P<desc>.*?)' + '(?:<font color=#008000>|<a)'
    results = []
    for match in re.compile(link_re, re.DOTALL).finditer(data):
        (name, url, desc) = match.group('name', 'url', 'desc')
        results.append((stripTags(name), url, stripTags(desc)))
    if len(results) > max_results:
        results = results[:max_results]
    return results