python-oxweb/ox/google.py

51 lines
1.3 KiB
Python
Raw Normal View History

2008-04-28 09:52:21 +00:00
# -*- coding: utf-8 -*-
2008-06-19 09:47:02 +00:00
# vi:si:et:sw=4:sts=4:ts=4
2008-04-28 09:52:21 +00:00
import re
import time
import urllib
import urllib2
import weakref
import threading
import Queue
import oxutils
from oxutils import stripTags
'''
usage:
import google
google.find(query)
for result in google.find(query): result
result is title, url, description
google.find(query, max_results)
FIXME: how search depper than first page?
2008-04-28 09:52:21 +00:00
'''
DEFAULT_MAX_RESULTS = 10
def getUrl(url, data=None, headers=oxutils.net.DEFAULT_HEADERS):
2008-06-19 09:47:02 +00:00
google_timeout=24*60*60
return oxutils.cache.getUrl(url, data, headers, google_timeout)
2008-04-28 09:52:21 +00:00
def quote_plus(s):
2008-06-19 09:47:02 +00:00
return urllib.quote_plus(s.encode('utf-8'))
2008-04-28 09:52:21 +00:00
def find(query, max_results=DEFAULT_MAX_RESULTS):
2008-06-19 09:47:02 +00:00
url = "http://www.google.com/search?q=%s" % quote_plus(query)
data = getUrl(url)
link_re = r'<a href="(?P<url>[^"]*?)" class=l.*?>(?P<name>.*?)</a>' + \
r'.*?(?:<br>|<table.*?>)' + \
r'(?P<desc>.*?)' + '(?:<font color=#008000>|<a)'
results = []
for match in re.compile(link_re, re.DOTALL).finditer(data):
(name, url, desc) = match.group('name', 'url', 'desc')
results.append((stripTags(name), url, stripTags(desc)))
if len(results) > max_results:
results = results[:max_results]
return results
2008-04-28 09:52:21 +00:00