0x2620/python-ox
2
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

support returning more than 10 results

Browse source
This commit is contained in:
j 2013-09-08 15:56:57 +02:00
parent cb45a25a7c
commit 98ab0e29db
1 changed files with 12 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
ox/web/google.py
View file

@ -27,13 +27,18 @@ def find(query, max_results=DEFAULT_MAX_RESULTS, timeout=DEFAULT_TIMEOUT):
>>> find("The Matrix site:imdb.com", 1)[0][1]
u'http://www.imdb.com/title/tt0133093/'
"""
url = 'http://google.com/search?q=%s' % quote_plus(query)
data = read_url(url, timeout=timeout)
results = []
data = re.sub('<span class="f">(.*?)</span>', '\\1', data)
for a in re.compile('<a href="(htt\S+?)".*?>(.*?)</a>.*?<span class="st">(.*?)<\/span>').findall(data):
results.append((strip_tags(decode_html(a[1])), a[0], strip_tags(decode_html(a[2]))))
if len(results) >= max_results:
break
offset = 0
while len(results) < max_results:
url = 'http://google.com/search?q=%s' % quote_plus(query)
if offset:
url += '&start=%d' % offset
data = read_url(url, timeout=timeout)
data = re.sub('<span class="f">(.*?)</span>', '\\1', data)
for a in re.compile('<a href="(htt\S+?)".*?>(.*?)</a>.*?<span class="st">(.*?)<\/span>').findall(data):
results.append((strip_tags(decode_html(a[1])), a[0], strip_tags(decode_html(a[2]))))
if len(results) >= max_results:
break
offset += 10
return results