python-ox/ox/web/google.py

72 lines
1.8 KiB
Python
Raw Normal View History

2010-07-07 23:25:57 +00:00
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import re
import time
import urllib
import urllib2
import weakref
import threading
import Queue
import ox
from ox import stripTags
2010-07-28 13:04:43 +00:00
from ox.utils import json
2010-07-07 23:25:57 +00:00
'''
usage:
import google
google.find(query)
for result in google.find(query): result
result is title, url, description
google.find(query, max_results)
FIXME: how search depper than first page?
'''
DEFAULT_MAX_RESULTS = 10
DEFAULT_TIMEOUT = 24*60*60
def readUrl(url, data=None, headers=ox.net.DEFAULT_HEADERS, timeout=DEFAULT_TIMEOUT):
return ox.cache.readUrl(url, data, headers, timeout)
def quote_plus(s):
2010-07-28 13:04:43 +00:00
if not isinstance(s, str):
s = s.encode('utf-8')
return urllib.quote_plus(s)
2010-07-07 23:25:57 +00:00
def find(query, max_results=DEFAULT_MAX_RESULTS, timeout=DEFAULT_TIMEOUT):
2010-07-28 13:04:43 +00:00
"""
Return max_results tuples with title, url, description
>>> find("The Matrix site:imdb.com", 1)[0][0]
u'The Matrix (1999)'
>>> find("The Matrix site:imdb.com", 1)[0][1]
u'http://www.imdb.com/title/tt0133093/'
"""
2010-12-27 13:26:11 +00:00
_results = _find(query, timeout=timeout)
2010-07-07 23:25:57 +00:00
results = []
2010-07-28 13:04:43 +00:00
for r in _results:
results.append((r['titleNoFormatting'], r['unescapedUrl'], stripTags(r['content'])))
if len(results) >= max_results:
break
2010-07-07 23:25:57 +00:00
return results
2010-07-28 13:04:43 +00:00
def _find(query, timeout=DEFAULT_TIMEOUT):
"""
Return parsed json results from google ajax api
>>> _find("The Matrix site:imdb.com")[0]['titleNoFormatting']
u'The Matrix (1999)'
>>> _find("The Matrix site:imdb.com")[0]['url']
u'http://www.imdb.com/title/tt0133093/'
"""
2010-07-07 23:25:57 +00:00
url = 'http://ajax.googleapis.com/ajax/services/search/web?v=1.0&q=%s' % quote_plus(query)
2010-12-31 07:23:24 +00:00
results = json.loads(ox.cache.readUrl(url, timeout=timeout))['responseData']['results']
2010-07-07 23:25:57 +00:00
return results